Bug 1725660 - Include timestamps in packaged dumps r=robwu

Since the dumps don't contain tombstones, this prevents situations where the dump would not be reloaded (with `loadDumpIfNewer`) if the last operation on the collection is a delete. Differential Revision: https://phabricator.services.mozilla.com/D144428
2022-04-29 08:07:25 +00:00 · 2022-04-29 08:07:25 +00:00 · 04b6b45a7f
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@ -10,7 +10,7 @@ support-files =
 [test_kinto.js]
 tags = blocklist
 [test_storage_adapter.js]
-tags = remote-settingsblocklist
+tags = remote-settings blocklist
 [test_storage_adapter_shutdown.js]
 tags = remote-settings blocklist

--- a/services/settings/RemoteSettingsWorker.js
+++ b/services/settings/RemoteSettingsWorker.js
@ -72,7 +72,7 @@ const Agent = {
   * @returns {int} Number of records loaded from dump or -1 if no dump found.
   */
  async importJSONDump(bucket, collection) {
-    const { data: records } = await SharedUtils.loadJSONDump(
+    const { data: records, timestamp } = await SharedUtils.loadJSONDump(
      bucket,
      collection
    );
@ -83,7 +83,7 @@ const Agent = {
    if (gShutdown) {
      throw new Error("Can't import when we've started shutting down.");
    }
-    await importDumpIDB(bucket, collection, records);
+    await importDumpIDB(bucket, collection, records, timestamp);
    return records.length;
  },

@ -121,8 +121,8 @@ const Agent = {
    }
  },

-  _test_only_import(bucket, collection, records) {
-    return importDumpIDB(bucket, collection, records);
+  _test_only_import(bucket, collection, records, timestamp) {
+    return importDumpIDB(bucket, collection, records, timestamp);
  },
 };

@ -153,8 +153,9 @@ let gPendingTransactions = new Set();
 * @param {String} bucket
 * @param {String} collection
 * @param {Array<Object>} records
+ * @param {Number} timestamp
 */
-async function importDumpIDB(bucket, collection, records) {
+async function importDumpIDB(bucket, collection, records, timestamp) {
  // Open the DB. It will exist since if we are running this, it means
  // we already tried to read the timestamp in `remote-settings.js`
  const db = await IDBHelpers.openIDB(false /* do not allow upgrades */);
@ -172,11 +173,7 @@ async function importDumpIDB(bucket, collection, records) {
    records.forEach(item => {
      item._cid = cid;
    });
-    // Store the highest timestamp as the collection timestamp (or zero if dump is empty).
-    const timestamp =
-      records.length === 0
-        ? 0
-        : Math.max(...records.map(record => record.last_modified));
+    // Store the collection timestamp.
    let { transaction, promise } = IDBHelpers.executeIDB(
      db,
      [IDB_RECORDS_STORE, IDB_TIMESTAMPS_STORE],
--- a/services/settings/SharedUtils.jsm
+++ b/services/settings/SharedUtils.jsm
@ -50,7 +50,7 @@ var SharedUtils = {
      response = await fetch(fileURI);
    } catch (e) {
      // Return null if file is missing.
-      return { data: null };
+      return { data: null, timestamp: null };
    }
    // Will throw if JSON is invalid.
    return response.json();
--- a/services/settings/Utils.jsm
+++ b/services/settings/Utils.jsm
@ -11,11 +11,11 @@ const { XPCOMUtils } = ChromeUtils.import(
 const { ServiceRequest } = ChromeUtils.import(
  "resource://gre/modules/ServiceRequest.jsm"
 );
-ChromeUtils.defineModuleGetter(
-  this,
-  "AppConstants",
-  "resource://gre/modules/AppConstants.jsm"
-);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  SharedUtils: "resource://services-settings/SharedUtils.jsm",
+  AppConstants: "resource://gre/modules/AppConstants.jsm",
+});

 XPCOMUtils.defineLazyServiceGetter(
  this,
@ -255,17 +255,12 @@ var Utils = {
    const identifier = `${bucket}/${collection}`;
    let lastModified = this._dumpStats[identifier];
    if (lastModified === undefined) {
-      try {
-        let res = await fetch(
-          `resource://app/defaults/settings/${bucket}/${collection}.json`
-        );
-        let records = (await res.json()).data;
-        // Records in dumps are sorted by last_modified, newest first.
-        // https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
-        lastModified = records[0]?.last_modified || 0;
-      } catch (e) {
-        lastModified = -1;
-      }
+      const { timestamp: dumpTimestamp } = await SharedUtils.loadJSONDump(
+        bucket,
+        collection
+      );
+      // Client recognize -1 as missing dump.
+      lastModified = dumpTimestamp ?? -1;
      this._dumpStats[identifier] = lastModified;
    }
    return lastModified;
--- a/services/settings/dumps/blocklists/addons-bloomfilters.json
+++ b/services/settings/dumps/blocklists/addons-bloomfilters.json
@ -1,5 +1,20 @@
 {
  "data": [
+    {
+      "stash": {
+        "blocked": [
+          "{06c64973-abc7-4bfe-a569-3ce2ca77d3be}:5.1.0",
+          "{86a2b09b-f6b0-4eec-a0c3-2ea5d0414c46}:1.0.2",
+          "{567f2fd5-3d33-4ee6-b1d4-b6c4dfaec43a}:1.0"
+        ],
+        "unblocked": []
+      },
+      "schema": 1651084687010,
+      "key_format": "{guid}:{version}",
+      "stash_time": 1651170906940,
+      "id": "131b9bf7-9abf-4aa3-9a8a-08ec8c21e489",
+      "last_modified": 1651171060377
+    },
    {
      "stash": {
        "blocked": [
@ -700,5 +715,6 @@
      "id": "7120faf8-4b30-4f60-8f63-4d7ddfc6daef",
      "last_modified": 1639658383523
    }
-  ]
+  ],
+  "timestamp": 1651171060377
 }
--- a/services/settings/dumps/blocklists/addons.json
+++ b/services/settings/dumps/blocklists/addons.json
@ -21956,5 +21956,6 @@
      "id": "aad4545f-8f9d-dd53-2aa8-e8945cad6185",
      "last_modified": 1480349192987
    }
-  ]
+  ],
+  "timestamp": 1604940558744
 }
--- a/services/settings/dumps/blocklists/gfx.json
+++ b/services/settings/dumps/blocklists/gfx.json
@ -1139,5 +1139,6 @@
      "id": "0afc5fe9-1f81-7fde-2424-0cb4d933c173",
      "last_modified": 1480349134090
    }
-  ]
+  ],
+  "timestamp": 1643818378440
 }
--- a/services/settings/dumps/blocklists/plugins.json
+++ b/services/settings/dumps/blocklists/plugins.json
@ -3510,5 +3510,6 @@
      "id": "38797744-cb92-1a49-4c81-2a900691fdba",
      "last_modified": 1480349146047
    }
-  ]
+  ],
+  "timestamp": 1603126502200
 }
--- a/services/settings/dumps/gen_last_modified.py
+++ b/services/settings/dumps/gen_last_modified.py
@ -13,23 +13,21 @@ def get_last_modified(full_path_to_remote_settings_dump_file):
    """
    Get the last_modified for the given file name.
    - File must exist
-    - Must be a JSON dictionary with a data list, e.g. `{"data": []}`
+    - Must be a JSON dictionary with a data list and a timestamp,
+      e.g. `{"data": [], "timestamp": 42}`
    - Every element in `data` should contain a "last_modified" key.
    - The first element must have the highest "last_modified" value.
    """
    with open(full_path_to_remote_settings_dump_file, "r") as f:
-        records = json.load(f)["data"]
-        assert isinstance(records, list)
+        changeset = json.load(f)

-    # Various RemoteSettings client code default to 0 when the set of
-    # records is empty (-1 is reserved for failures / non-existing files).
-    last_modified = 0
-    if records:
-        # Records in dumps are sorted by last_modified, newest first:
-        # https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304 # NOQA: E501
-        last_modified = records[0]["last_modified"]
+    records = changeset["data"]
+    assert isinstance(records, list)
+    last_modified = changeset.get("timestamp")
+    assert isinstance(
+        last_modified, int
+    ), f"{full_path_to_remote_settings_dump_file} is missing the timestamp. See Bug 1725660"

-    assert isinstance(last_modified, int)
    return last_modified


--- a/services/settings/dumps/main/anti-tracking-url-decoration.json
+++ b/services/settings/dumps/main/anti-tracking-url-decoration.json
@ -6,5 +6,6 @@
      "id": "60e82333-914d-4cfa-95b1-5f034b5a704b",
      "last_modified": 1564511755134
    }
-  ]
+  ],
+  "timestamp": 1564511755134
 }
--- a/services/settings/dumps/main/devtools-compatibility-browsers.json
+++ b/services/settings/dumps/main/devtools-compatibility-browsers.json
@ -2,93 +2,300 @@
  "data": [
    {
      "name": "Chrome",
-      "schema": 1648568861836,
+      "schema": 1651151565765,
      "status": "nightly",
-      "version": "101",
+      "version": "102",
      "browserid": "chrome",
-      "id": "eef07db5-7d8f-4d31-aa2a-71b5d0bb0554",
-      "last_modified": 1648570256624
+      "id": "aa24fd15-49da-4fe8-aae4-25236d574612",
+      "last_modified": 1651179345959
+    },
+    {
+      "name": "Chrome",
+      "schema": 1651151566603,
+      "status": "planned",
+      "version": "103",
+      "browserid": "chrome",
+      "id": "98226134-72f5-4e8b-9fe1-91c6f0a73db8",
+      "last_modified": 1651179345956
    },
    {
      "name": "Chrome Android",
-      "schema": 1648568865018,
+      "schema": 1651151569951,
      "status": "nightly",
-      "version": "101",
+      "version": "102",
      "browserid": "chrome_android",
-      "id": "c4f461a1-e339-4b5d-a365-f9d68e60f57f",
-      "last_modified": 1648570256619
+      "id": "681ca4e6-2d0f-439d-9b06-e039eb9f2481",
+      "last_modified": 1651179345953
+    },
+    {
+      "name": "Chrome Android",
+      "schema": 1651151570787,
+      "status": "planned",
+      "version": "103",
+      "browserid": "chrome_android",
+      "id": "38e7422e-9f31-44d4-81f8-8f273d675d1b",
+      "last_modified": 1651179345950
+    },
+    {
+      "name": "Deno",
+      "schema": 1651151575830,
+      "status": "nightly",
+      "version": "1.21",
+      "browserid": "deno",
+      "id": "24332717-5d03-4758-bb21-e0289482ca57",
+      "last_modified": 1651179345947
    },
    {
      "name": "Edge",
-      "schema": 1648568868092,
+      "schema": 1651151579223,
      "status": "nightly",
-      "version": "101",
+      "version": "102",
      "browserid": "edge",
-      "id": "4df63f83-28b4-429d-a395-f448cc22a35f",
-      "last_modified": 1648570256615
+      "id": "a0b9fe57-1667-4824-9f33-59b28047dbcd",
+      "last_modified": 1651179345944
+    },
+    {
+      "name": "Edge",
+      "schema": 1651151580081,
+      "status": "planned",
+      "version": "103",
+      "browserid": "edge",
+      "id": "4afba52a-82ff-43e3-974b-3a00dd9def94",
+      "last_modified": 1651179345942
    },
    {
      "name": "Firefox",
-      "schema": 1648568871139,
+      "schema": 1651151583440,
      "status": "nightly",
-      "version": "100",
+      "version": "101",
      "browserid": "firefox",
-      "id": "34614896-26ad-4518-bde5-6b01ced257b2",
-      "last_modified": 1648570256610
+      "id": "9e175d7d-1109-497b-b384-f14976d7ebe3",
+      "last_modified": 1651179345939
    },
    {
      "name": "Firefox for Android",
-      "schema": 1648568874646,
+      "schema": 1651151586791,
      "status": "nightly",
-      "version": "100",
+      "version": "101",
      "browserid": "firefox_android",
-      "id": "cc6e84d0-fc13-4054-a136-4fe1fd63bb77",
-      "last_modified": 1648570256605
+      "id": "935febef-4d7f-4c06-be1c-d957cbe79b71",
+      "last_modified": 1651179345936
+    },
+    {
+      "name": "Node.js",
+      "schema": 1651151590287,
+      "status": "current",
+      "version": "17.3.0",
+      "browserid": "nodejs",
+      "id": "790f107f-8b26-49d0-9caf-6f90cb5b127b",
+      "last_modified": 1651179345933
+    },
+    {
+      "name": "Opera",
+      "schema": 1651151593683,
+      "status": "beta",
+      "version": "87",
+      "browserid": "opera",
+      "id": "aeb7f795-0d90-49ec-8484-d1fef2bef61c",
+      "last_modified": 1651179345930
+    },
+    {
+      "name": "Opera",
+      "schema": 1651151594520,
+      "status": "nightly",
+      "version": "88",
+      "browserid": "opera",
+      "id": "d93c936e-7b34-4f00-8715-d78b0851613c",
+      "last_modified": 1651179345927
+    },
+    {
+      "name": "Opera Android",
+      "schema": 1651151596201,
+      "status": "current",
+      "version": "68",
+      "browserid": "opera_android",
+      "id": "5daf9d70-05c3-4c13-b6a2-00a9f5034ea8",
+      "last_modified": 1651179345924
+    },
+    {
+      "name": "Safari",
+      "schema": 1651151597052,
+      "status": "beta",
+      "version": "15.5",
+      "browserid": "safari",
+      "id": "d18a11fc-76c1-4a25-b40c-16b6834ad3e3",
+      "last_modified": 1651179345921
+    },
+    {
+      "name": "Safari on iOS",
+      "schema": 1651151598745,
+      "status": "beta",
+      "version": "15.5",
+      "browserid": "safari_ios",
+      "id": "880d9032-1db7-4989-936a-d935fd344b36",
+      "last_modified": 1651179345919
+    },
+    {
+      "name": "Samsung Internet",
+      "schema": 1651151599588,
+      "status": "beta",
+      "version": "17.0",
+      "browserid": "samsunginternet_android",
+      "id": "33853107-4475-46de-a22b-84972be87dfd",
+      "last_modified": 1651179345916
    },
    {
      "name": "WebView Android",
-      "schema": 1648568880910,
+      "schema": 1651151602949,
      "status": "nightly",
-      "version": "101",
+      "version": "102",
      "browserid": "webview_android",
-      "id": "ce15215f-213e-48f2-97f1-829e6a7c9b92",
-      "last_modified": 1648570256601
+      "id": "4b9c29d7-4560-4aa0-b5da-6f741e68465b",
+      "last_modified": 1651179345913
    },
    {
-      "name": "Firefox for Android",
-      "schema": 1648568873804,
-      "status": "beta",
-      "version": "99",
-      "browserid": "firefox_android",
-      "id": "f31e3a1b-2af3-4a7f-b77d-e7734c6be47e",
-      "last_modified": 1648570256596
+      "name": "WebView Android",
+      "schema": 1651151603791,
+      "status": "planned",
+      "version": "103",
+      "browserid": "webview_android",
+      "id": "a07ea66a-9e79-4dae-937f-0b5c42b961bb",
+      "last_modified": 1651179345910
    },
    {
-      "name": "Firefox for Android",
-      "schema": 1648568872901,
+      "name": "Deno",
+      "schema": 1651151574995,
      "status": "current",
-      "version": "98",
-      "browserid": "firefox_android",
-      "id": "1f7048ea-af49-4f72-8b86-c1156a3439b7",
-      "last_modified": 1648570256592
+      "version": "1.20",
+      "browserid": "deno",
+      "id": "12eda7ab-d1f2-4d6d-8fbb-4d3d55778543",
+      "last_modified": 1651179345905
+    },
+    {
+      "name": "Node.js",
+      "schema": 1651151588509,
+      "status": "esr",
+      "version": "16.11.0",
+      "browserid": "nodejs",
+      "id": "4d3d7ec6-7211-437e-aa6b-8439d719f1e1",
+      "last_modified": 1651179345896
+    },
+    {
+      "name": "Opera",
+      "schema": 1651151592837,
+      "status": "current",
+      "version": "86",
+      "browserid": "opera",
+      "id": "5a4e3909-1ac1-4a78-a10d-70a59240f673",
+      "last_modified": 1651179345881
+    },
+    {
+      "name": "Chrome",
+      "schema": 1651151564042,
+      "status": "current",
+      "version": "100",
+      "browserid": "chrome",
+      "id": "2a76cc10-7765-4a3e-b558-d73ff18b5db5",
+      "last_modified": 1651179345877
+    },
+    {
+      "name": "Chrome Android",
+      "schema": 1651151568284,
+      "status": "current",
+      "version": "100",
+      "browserid": "chrome_android",
+      "id": "b282f95f-b994-4b35-9a62-e47d3ea6ec6f",
+      "last_modified": 1651179345873
+    },
+    {
+      "name": "Edge",
+      "schema": 1651151577537,
+      "status": "current",
+      "version": "100",
+      "browserid": "edge",
+      "id": "874c95fb-afd2-48a1-81c4-83fd54c5b237",
+      "last_modified": 1651179345868
+    },
+    {
+      "name": "WebView Android",
+      "schema": 1651151601269,
+      "status": "current",
+      "version": "100",
+      "browserid": "webview_android",
+      "id": "be6227cb-2414-4ac2-8134-6d80c37cb1a4",
+      "last_modified": 1651179345864
    },
    {
      "name": "Firefox",
-      "schema": 1648568870430,
-      "status": "beta",
+      "schema": 1651151581743,
+      "status": "current",
      "version": "99",
      "browserid": "firefox",
      "id": "8be3e7b3-f3fd-4bd0-9e88-c1c2ea93bd62",
-      "last_modified": 1648570256585
+      "last_modified": 1651179345859
+    },
+    {
+      "name": "Firefox for Android",
+      "schema": 1651151585101,
+      "status": "current",
+      "version": "99",
+      "browserid": "firefox_android",
+      "id": "f31e3a1b-2af3-4a7f-b77d-e7734c6be47e",
+      "last_modified": 1651179345855
+    },
+    {
+      "name": "WebView Android",
+      "schema": 1651151602115,
+      "status": "beta",
+      "version": "101",
+      "browserid": "webview_android",
+      "id": "ce15215f-213e-48f2-97f1-829e6a7c9b92",
+      "last_modified": 1651179345853
+    },
+    {
+      "name": "Firefox for Android",
+      "schema": 1651151585954,
+      "status": "beta",
+      "version": "100",
+      "browserid": "firefox_android",
+      "id": "cc6e84d0-fc13-4054-a136-4fe1fd63bb77",
+      "last_modified": 1651179345850
    },
    {
      "name": "Firefox",
-      "schema": 1648568869718,
-      "status": "current",
-      "version": "98",
+      "schema": 1651151582589,
+      "status": "beta",
+      "version": "100",
      "browserid": "firefox",
-      "id": "c112510f-d9cc-41f3-a14a-c6660b719cc3",
-      "last_modified": 1648570256582
+      "id": "34614896-26ad-4518-bde5-6b01ced257b2",
+      "last_modified": 1651179345848
+    },
+    {
+      "name": "Edge",
+      "schema": 1651151578377,
+      "status": "beta",
+      "version": "101",
+      "browserid": "edge",
+      "id": "4df63f83-28b4-429d-a395-f448cc22a35f",
+      "last_modified": 1651179345846
+    },
+    {
+      "name": "Chrome Android",
+      "schema": 1651151569117,
+      "status": "beta",
+      "version": "101",
+      "browserid": "chrome_android",
+      "id": "c4f461a1-e339-4b5d-a365-f9d68e60f57f",
+      "last_modified": 1651179345843
+    },
+    {
+      "name": "Chrome",
+      "schema": 1651151564906,
+      "status": "beta",
+      "version": "101",
+      "browserid": "chrome",
+      "id": "eef07db5-7d8f-4d31-aa2a-71b5d0bb0554",
+      "last_modified": 1651179345841
    },
    {
      "name": "Safari on iOS",
@ -99,24 +306,6 @@
      "id": "6d71d3c6-a98d-4acf-846c-11a10b8d8af6",
      "last_modified": 1648570256575
    },
-    {
-      "name": "WebView Android",
-      "schema": 1648568880169,
-      "status": "beta",
-      "version": "100",
-      "browserid": "webview_android",
-      "id": "be6227cb-2414-4ac2-8134-6d80c37cb1a4",
-      "last_modified": 1648570256568
-    },
-    {
-      "name": "WebView Android",
-      "schema": 1648568879457,
-      "status": "current",
-      "version": "99",
-      "browserid": "webview_android",
-      "id": "21bcaf3c-ccc6-4d3a-a330-a67ed37b650a",
-      "last_modified": 1648570256565
-    },
    {
      "name": "Safari",
      "schema": 1648568876276,
@ -126,114 +315,6 @@
      "id": "94f97615-86ab-4e62-b229-1f0256175e9a",
      "last_modified": 1648570256555
    },
-    {
-      "name": "Edge",
-      "schema": 1648568867371,
-      "status": "beta",
-      "version": "100",
-      "browserid": "edge",
-      "id": "874c95fb-afd2-48a1-81c4-83fd54c5b237",
-      "last_modified": 1648570256552
-    },
-    {
-      "name": "Edge",
-      "schema": 1648568866503,
-      "status": "current",
-      "version": "99",
-      "browserid": "edge",
-      "id": "0c0a8c3a-f4c0-4983-a584-9d909e7813ec",
-      "last_modified": 1648570256548
-    },
-    {
-      "name": "Chrome Android",
-      "schema": 1648568864091,
-      "status": "beta",
-      "version": "100",
-      "browserid": "chrome_android",
-      "id": "b282f95f-b994-4b35-9a62-e47d3ea6ec6f",
-      "last_modified": 1648570256541
-    },
-    {
-      "name": "Chrome Android",
-      "schema": 1648568863279,
-      "status": "current",
-      "version": "99",
-      "browserid": "chrome_android",
-      "id": "26c276ee-05c6-4430-a98d-18fd3559260a",
-      "last_modified": 1648570256538
-    },
-    {
-      "name": "Chrome",
-      "schema": 1648568861054,
-      "status": "beta",
-      "version": "100",
-      "browserid": "chrome",
-      "id": "2a76cc10-7765-4a3e-b558-d73ff18b5db5",
-      "last_modified": 1648570256531
-    },
-    {
-      "name": "Chrome",
-      "schema": 1648568860308,
-      "status": "current",
-      "version": "99",
-      "browserid": "chrome",
-      "id": "371d2018-13b0-4523-8b0d-cdc5b02981af",
-      "last_modified": 1648570256528
-    },
-    {
-      "name": "Opera",
-      "schema": 1646381478384,
-      "status": "nightly",
-      "version": "86",
-      "browserid": "opera",
-      "id": "5a4e3909-1ac1-4a78-a10d-70a59240f673",
-      "last_modified": 1646392425340
-    },
-    {
-      "name": "Opera",
-      "schema": 1646381477291,
-      "status": "beta",
-      "version": "85",
-      "browserid": "opera",
-      "id": "1997a792-cfa4-44ac-bd48-3865034530c7",
-      "last_modified": 1646392425335
-    },
-    {
-      "name": "Opera",
-      "schema": 1646381476206,
-      "status": "current",
-      "version": "84",
-      "browserid": "opera",
-      "id": "f7b88440-7a99-434c-a265-de4fde85e787",
-      "last_modified": 1646392425332
-    },
-    {
-      "name": "Deno",
-      "schema": 1645203727939,
-      "status": "current",
-      "version": "1.16",
-      "browserid": "deno",
-      "id": "a7d0618e-3d5e-4568-b5da-6aa7686e246c",
-      "last_modified": 1645448267610
-    },
-    {
-      "name": "Deno",
-      "schema": 1645203728980,
-      "status": "current",
-      "version": "1.17",
-      "browserid": "deno",
-      "id": "5e981c57-284f-4a45-878f-34345a077416",
-      "last_modified": 1645448267605
-    },
-    {
-      "name": "Deno",
-      "schema": 1645203729978,
-      "status": "current",
-      "version": "1.18",
-      "browserid": "deno",
-      "id": "40d2eb08-3ef6-49ce-9f6f-61e76e6109bc",
-      "last_modified": 1645448267600
-    },
    {
      "name": "Firefox",
      "schema": 1645203736177,
@ -261,42 +342,6 @@
      "id": "25f4d331-04a6-4fc6-842a-5960bf855fd1",
      "last_modified": 1645448267570
    },
-    {
-      "name": "Node.js",
-      "schema": 1645203747348,
-      "status": "current",
-      "version": "16.9.0",
-      "browserid": "nodejs",
-      "id": "509820e6-0a62-423a-b159-2bb16db842d3",
-      "last_modified": 1645448267565
-    },
-    {
-      "name": "Node.js",
-      "schema": 1645203748401,
-      "status": "current",
-      "version": "16.11.0",
-      "browserid": "nodejs",
-      "id": "4d3d7ec6-7211-437e-aa6b-8439d719f1e1",
-      "last_modified": 1645448267560
-    },
-    {
-      "name": "Node.js",
-      "schema": 1645203749482,
-      "status": "current",
-      "version": "17.0.0",
-      "browserid": "nodejs",
-      "id": "59d45343-21f4-4fb9-9103-4d8fbc04aed3",
-      "last_modified": 1645448267555
-    },
-    {
-      "name": "Safari on iOS",
-      "schema": 1645203756942,
-      "status": "current",
-      "version": "15.2",
-      "browserid": "safari_ios",
-      "id": "3bd6dc9d-0d84-4881-a14a-f3319885911c",
-      "last_modified": 1645448267530
-    },
    {
      "name": "Samsung Internet",
      "schema": 1645203760081,
@ -314,33 +359,7 @@
      "browserid": "nodejs",
      "id": "4436cf1f-e30d-401a-893f-306dfd0b1e4e",
      "last_modified": 1645448267500
-    },
-    {
-      "name": "Deno",
-      "schema": 1645426860619,
-      "status": "current",
-      "version": "1.19",
-      "browserid": "deno",
-      "id": "1e64b9e3-bf42-48cc-90eb-1fe792ba5662",
-      "last_modified": 1645448267485
-    },
-    {
-      "name": "Deno",
-      "schema": 1645426897808,
-      "status": "nightly",
-      "version": "1.20",
-      "browserid": "deno",
-      "id": "12eda7ab-d1f2-4d6d-8fbb-4d3d55778543",
-      "last_modified": 1645448267481
-    },
-    {
-      "name": "Opera Android",
-      "schema": 1645426908452,
-      "status": "current",
-      "version": "67",
-      "browserid": "opera_android",
-      "id": "0299f77d-0498-481a-9896-498095a7e301",
-      "last_modified": 1645448267445
    }
-  ]
+  ],
+  "timestamp": 1651179345959
 }
--- a/services/settings/dumps/main/hijack-blocklists.json
+++ b/services/settings/dumps/main/hijack-blocklists.json
@ -54,5 +54,6 @@
      "id": "homepage-urls",
      "last_modified": 1605801189250
    }
-  ]
+  ],
+  "timestamp": 1605801189258
 }
--- a/services/settings/dumps/main/language-dictionaries.json
+++ b/services/settings/dumps/main/language-dictionaries.json
@ -642,5 +642,6 @@
      "id": "af",
      "last_modified": 1539698843264
    }
-  ]
+  ],
+  "timestamp": 1569410800356
 }
--- a/services/settings/dumps/main/password-recipes.json
+++ b/services/settings/dumps/main/password-recipes.json
@ -129,5 +129,6 @@
      "id": "00814138-7ede-4f56-8953-b6d1c99d5f26",
      "last_modified": 1600889016416
    }
-  ]
+  ],
+  "timestamp": 1642005109349
 }
--- a/services/settings/dumps/main/password-rules.json
+++ b/services/settings/dumps/main/password-rules.json
@ -1560,5 +1560,6 @@
      "id": "5fe248fb-3e63-4300-8f67-3e6006af0e87",
      "last_modified": 1624479576629
    }
-  ]
+  ],
+  "timestamp": 1650898337595
 }
--- a/services/settings/dumps/main/search-config.json
+++ b/services/settings/dumps/main/search-config.json
@ -2907,5 +2907,6 @@
      "id": "43559b11-05dc-4750-b131-afdbd9d25905",
      "last_modified": 1583937832210
    }
-  ]
+  ],
+  "timestamp": 1648132005528
 }
--- a/services/settings/dumps/main/search-default-override-allowlist.json
+++ b/services/settings/dumps/main/search-default-override-allowlist.json
@ -12,5 +12,6 @@
      "id": "c5de2b3f-da15-4bc5-b36f-9555043b5ef3",
      "last_modified": 1595254618540
    }
-  ]
+  ],
+  "timestamp": 1595254618540
 }
--- a/services/settings/dumps/main/search-telemetry-v2.json
+++ b/services/settings/dumps/main/search-telemetry-v2.json
@ -149,5 +149,6 @@
      "id": "19c434a3-d173-4871-9743-290ac92a3f6a",
      "last_modified": 1643136933989
    }
-  ]
+  ],
+  "timestamp": 1647619735693
 }
--- a/services/settings/dumps/main/sites-classification.json
+++ b/services/settings/dumps/main/sites-classification.json
@ -458,5 +458,6 @@
      "id": "48ec9db9-0de1-49aa-9b82-1a2372dce31e",
      "last_modified": 1539907283705
    }
-  ]
+  ],
+  "timestamp": 1544035467383
 }
--- a/services/settings/dumps/main/top-sites.json
+++ b/services/settings/dumps/main/top-sites.json
@ -396,5 +396,6 @@
      "id": "89e3da9e-1d84-4c24-beab-d0692fc7cc45",
      "last_modified": 1599146777345
    }
-  ]
+  ],
+  "timestamp": 1647020600359
 }
--- a/services/settings/dumps/main/url-classifier-skip-urls.json
+++ b/services/settings/dumps/main/url-classifier-skip-urls.json
@ -1,3 +1,4 @@
 {
-  "data": []
+  "data": [],
+  "timestamp": 1606870304609
 }
--- a/services/settings/dumps/main/websites-with-shared-credential-backends.json
+++ b/services/settings/dumps/main/websites-with-shared-credential-backends.json
@ -714,5 +714,6 @@
      "id": "8c3d4151-8e68-4bb3-a3fd-babf4aba2cdc",
      "last_modified": 1650898092205
    }
-  ]
+  ],
+  "timestamp": 1650898092205
 }
--- a/services/settings/dumps/security-state/intermediates.json
+++ b/services/settings/dumps/security-state/intermediates.json
@ -1,5 +1,77 @@
 {
  "data": [
+    {
+      "schema": 1651200582692,
+      "derHash": "Fed8WUBAFWWe4m+SVh8Aeq9TmGonkm8OcHWFwqfzAbg=",
+      "subject": "CN=DigiCert Global G3 TLS EE\\+ ECC384 SHA384 2022 CA1,O=DigiCert\\, Inc.,C=US",
+      "subjectDN": "MGIxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwgSW5jLjE6MDgGA1UEAxMxRGlnaUNlcnQgR2xvYmFsIEczIFRMUyBFRSsgRUNDMzg0IFNIQTM4NCAyMDIyIENBMQ==",
+      "whitelist": false,
+      "attachment": {
+        "hash": "d698eb7c871e953a2d52f5a3e37a7c94099ce9cd6e5e351f175434f9b4647831",
+        "size": 1276,
+        "filename": "qX0VkaXNOTvb_iR2T3sjz2frmsxWXSBKxeCVlcqFpuM=.pem",
+        "location": "security-state-staging/intermediates/b22276db-4c8e-4141-a014-1fe8448280d6.pem",
+        "mimetype": "application/x-pem-file"
+      },
+      "pubKeyHash": "qX0VkaXNOTvb/iR2T3sjz2frmsxWXSBKxeCVlcqFpuM=",
+      "crlite_enrolled": false,
+      "id": "3504d097-0f0d-4eb1-bcab-f8c9cff23a8e",
+      "last_modified": 1651201056425
+    },
+    {
+      "schema": 1651200581750,
+      "derHash": "uRW4Bqi3vi44CfLpMplufBDMgmkYZKiWBr1QCA3C6rM=",
+      "subject": "CN=DigiCert Global G2 TLS EE\\+ RSA4096 SHA256 2022 CA1,O=DigiCert\\, Inc.,C=US",
+      "subjectDN": "MGMxCzAJBgNVBAYTAlVTMRcwFQYDVQQKEw5EaWdpQ2VydCwgSW5jLjE7MDkGA1UEAxMyRGlnaUNlcnQgR2xvYmFsIEcyIFRMUyBFRSsgUlNBNDA5NiBTSEEyNTYgMjAyMiBDQTE=",
+      "whitelist": false,
+      "attachment": {
+        "hash": "2770a30970c5f0e28e817c1788eb2c06bff044a066ecca40a7a2b96211daf4fc",
+        "size": 2077,
+        "filename": "nfWaEttE0IGAYQsFsn8KzN13pSJzWY96Gu0Dlx4QYig=.pem",
+        "location": "security-state-staging/intermediates/4916e83a-41de-416d-910e-e4d6d8f4f953.pem",
+        "mimetype": "application/x-pem-file"
+      },
+      "pubKeyHash": "nfWaEttE0IGAYQsFsn8KzN13pSJzWY96Gu0Dlx4QYig=",
+      "crlite_enrolled": false,
+      "id": "168daeb9-d058-4d6b-946a-87f8af07577f",
+      "last_modified": 1651201056418
+    },
+    {
+      "schema": 1651200580812,
+      "derHash": "9dTK1Ts/LvOYENptPNX/dmP0q7h3OIJ8zsr/b8w+r8Q=",
+      "subject": "CN=MSFT RS256 CA-1,O=Microsoft Corporation,C=US",
+      "subjectDN": "MEcxCzAJBgNVBAYTAlVTMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xGDAWBgNVBAMTD01TRlQgUlMyNTYgQ0EtMQ==",
+      "whitelist": false,
+      "attachment": {
+        "hash": "a023fc76ccc16c88e584716119317138d7bdceaf8896157e4bb9aa525e227e9b",
+        "size": 1695,
+        "filename": "4dcjodQDXb57s8lG7TLWD3Xe9ytnPSATT61TJbQtAwI=.pem",
+        "location": "security-state-staging/intermediates/dbe4ae7c-598c-423b-a4f2-aa6674a8ec18.pem",
+        "mimetype": "application/x-pem-file"
+      },
+      "pubKeyHash": "4dcjodQDXb57s8lG7TLWD3Xe9ytnPSATT61TJbQtAwI=",
+      "crlite_enrolled": false,
+      "id": "50064930-2d6a-442e-8edb-9151ed42396b",
+      "last_modified": 1651201056411
+    },
+    {
+      "schema": 1651200583442,
+      "derHash": "lzpBJ2/9AeAnoqrUnjTDeEbT6Xb/amILZxLjODIEGqY=",
+      "subject": "CN=Go Daddy Secure Certificate Authority - G2,OU=http://certs.godaddy.com/repository/,O=GoDaddy.com\\, Inc.,L=Scottsdale,ST=Arizona,C=US",
+      "subjectDN": "MIG0MQswCQYDVQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5jb20sIEluYy4xLTArBgNVBAsTJGh0dHA6Ly9jZXJ0cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzEzMDEGA1UEAxMqR28gRGFkZHkgU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcy",
+      "whitelist": false,
+      "attachment": {
+        "hash": "a111efc20329e87b541cfc3e7f526fc54bb24624fddf210f4347b9fadfb1ab6b",
+        "size": 1727,
+        "filename": "8Rw90Ej3Ttt8RRkrg-WYDS9n7IS03bk5bjP_UXPtaY8=.pem",
+        "location": "security-state-staging/intermediates/a2a04dfe-6201-4d0a-b8cf-80d2975bfb51.pem",
+        "mimetype": "application/x-pem-file"
+      },
+      "pubKeyHash": "8Rw90Ej3Ttt8RRkrg+WYDS9n7IS03bk5bjP/UXPtaY8=",
+      "crlite_enrolled": false,
+      "id": "549c92e2-3738-4c8c-ae37-407e4e6158ce",
+      "last_modified": 1651201056399
+    },
    {
      "schema": 1650962920282,
      "derHash": "Ocdjqc8Z2SP5d9I2JquJBEmkRKuLeVyBXvHvgf68Hjg=",
@ -2106,24 +2178,6 @@
      "id": "3ef3ce19-bcfb-484b-9e2e-253adcc956b5",
      "last_modified": 1647701842304
    },
-    {
-      "schema": 1647701328071,
-      "derHash": "lzpBJ2/9AeAnoqrUnjTDeEbT6Xb/amILZxLjODIEGqY=",
-      "subject": "CN=Go Daddy Secure Certificate Authority - G2,OU=http://certs.godaddy.com/repository/,O=GoDaddy.com\\, Inc.,L=Scottsdale,ST=Arizona,C=US",
-      "subjectDN": "MIG0MQswCQYDVQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5jb20sIEluYy4xLTArBgNVBAsTJGh0dHA6Ly9jZXJ0cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzEzMDEGA1UEAxMqR28gRGFkZHkgU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcy",
-      "whitelist": false,
-      "attachment": {
-        "hash": "a111efc20329e87b541cfc3e7f526fc54bb24624fddf210f4347b9fadfb1ab6b",
-        "size": 1727,
-        "filename": "8Rw90Ej3Ttt8RRkrg-WYDS9n7IS03bk5bjP_UXPtaY8=.pem",
-        "location": "security-state-staging/intermediates/a2a04dfe-6201-4d0a-b8cf-80d2975bfb51.pem",
-        "mimetype": "application/x-pem-file"
-      },
-      "pubKeyHash": "8Rw90Ej3Ttt8RRkrg+WYDS9n7IS03bk5bjP/UXPtaY8=",
-      "crlite_enrolled": true,
-      "id": "549c92e2-3738-4c8c-ae37-407e4e6158ce",
-      "last_modified": 1647701842291
-    },
    {
      "schema": 1647680259122,
      "derHash": "4i5rJZCOEQemB68GDgsk5QxtlWL/BPRVvg+N9BpQMsA=",
@ -24552,5 +24606,6 @@
      "id": "586b6f01-c7f7-4a5e-ba71-e7dca3e77add",
      "last_modified": 1559865863642
    }
-  ]
+  ],
+  "timestamp": 1651201056425
 }
--- a/services/settings/dumps/security-state/onecrl.json
+++ b/services/settings/dumps/security-state/onecrl.json
@ -21780,5 +21780,6 @@
      "id": "ae8bec3c-3b92-822e-53f1-68394cbb1758",
      "last_modified": 1480349158647
    }
-  ]
+  ],
+  "timestamp": 1648145180742
 }
--- a/services/settings/static-dumps/main/doh-config.json
+++ b/services/settings/static-dumps/main/doh-config.json
@ -10,5 +10,6 @@
      "id": "global",
      "last_modified": 1621943462970
    }
-  ]
+  ],
+  "timestamp": 1621943462970
 }
--- a/services/settings/static-dumps/main/doh-providers.json
+++ b/services/settings/static-dumps/main/doh-providers.json
@ -18,5 +18,6 @@
      "id": "cloudflare-global",
      "last_modified": 1621943542615
    }
-  ]
+  ],
+  "timestamp": 1621943542621
 }
--- a/services/settings/test/unit/test_remote_settings_dump_lastmodified.js
+++ b/services/settings/test/unit/test_remote_settings_dump_lastmodified.js
@ -11,17 +11,9 @@ async function getLocalDumpLastModified(bucket, collection) {
  let res = await fetch(
    `resource://app/defaults/settings/${bucket}/${collection}.json`
  );
-  let records = (await res.json()).data;
-
-  if (records.some(r => r.last_modified > records[0].last_modified)) {
-    // The dump importer should ensure that the newest record is at the front:
-    // https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
-    ok(false, `${bucket}/${collection} - newest record should be in the front`);
-  }
-  return records.reduce(
-    (max, { last_modified }) => Math.max(last_modified, max),
-    0
-  );
+  const { timestamp } = await res.json();
+  ok(timestamp >= 0, `${bucket}/${collection} dump has timestamp`);
+  return timestamp;
 }

 add_task(async function lastModified_of_non_existing_dump() {
--- a/services/settings/test/unit/test_remote_settings_offline.js
+++ b/services/settings/test/unit/test_remote_settings_offline.js
@ -26,12 +26,10 @@ add_task(async function setup() {
    bucketName: TEST_BUCKET,
  });

-  DUMP_RECORDS = (await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION))
-    .data;
-  DUMP_LAST_MODIFIED = DUMP_RECORDS.reduce(
-    (max, { last_modified }) => Math.max(last_modified, max),
-    -Infinity
-  );
+  const dump = await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION);
+  DUMP_RECORDS = dump.data;
+  DUMP_LAST_MODIFIED = dump.timestamp;
+
  // Dumps are fetched via the following, which sorts the records, newest first.
  // https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
  equal(
@ -46,6 +44,7 @@ async function importData(records) {
    TEST_BUCKET,
    TEST_COLLECTION,
    records,
+    records[0]?.last_modified || 0,
  ]);
 }

--- a/services/settings/test/unit/test_shutdown_handling.js
+++ b/services/settings/test/unit/test_shutdown_handling.js
@ -108,7 +108,7 @@ add_task(async function test_shutdown_worker() {
  let records = [{}];
  let importPromise = RemoteSettingsWorker._execute(
    "_test_only_import",
-    ["main", "language-dictionaries", records],
+    ["main", "language-dictionaries", records, 0],
    { mustComplete: true }
  );
  let stringifyPromise = RemoteSettingsWorker.canonicalStringify(
--- a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
+++ b/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
@ -290,9 +290,9 @@ function compare_remote_settings_files {
  # 1. List remote settings collections from server.
  echo "INFO: fetch remote settings list from server"
  ${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
-    ${JQ} -r '.data[] | .bucket+"/"+.collection' |\
-    # 2. For each entry ${bucket, collection}
-  while IFS="/" read -r bucket collection; do
+    ${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\
+    # 2. For each entry ${bucket, collection, last_modified}
+  while IFS="/" read -r bucket collection last_modified; do

    # 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
    hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
@ -307,10 +307,10 @@ function compare_remote_settings_files {
    fi

    # 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
-    remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
+    remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}"
    local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
    mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
-    ${WGET} -qO- "$remote_records_url" | ${JQ} . > "${local_location_output}"
+    ${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}"

    # 5. Download attachments if needed.
    if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then
--- a/toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js
+++ b/toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js
@ -35,6 +35,7 @@ add_task(async function test_selector_db_out_of_date() {
        last_modified: 1606227264000,
      },
    ],
+    1606227264000,
  ]);

  // Now load the configuration and check we get what we expect.
--- a/toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.ini
+++ b/toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.ini
@ -4,7 +4,7 @@ head = head_searchconfig.js
 dupe-manifest =
 support-files =
  ../../../../../../browser/locales/all-locales
-tags=searchconfig
+tags=searchconfig remote-settings
 # These are extensive tests, we don't need to run them on asan/tsan.
 # They are also skipped for mobile and Thunderbird as these are specifically
 # testing the Firefox config at the moment.