feat: merge password rules updater (#3)

* docs: update description and links

* feat: merge updater from password-rules

* docs: update function docs

* refactor: remove unused code

* refactor: change apiEndpoint to not use all caps

* refactor: clarify code by using intermediate variable

* refactor: clarify and simplify createAndUpdateRulesRecords

* refactor: extract related realms updating into separate function

* fix: fix wrong variable being used

* fix: fix missing domain in update logic of the password rules updater

* docs: fix docs of getSourceRecords

* refactor: remove debugger statement

* refactor: remove testing bucket comment

README.md

@@ -1,6 +1,6 @@
-# passwordmgr-related-realms-updater
+# passwordmgr-remote-settings-updater
 
-Script that adds new related websites to the "websites-with-shared-credential-backends" Remote Setting collection via [Apple's open sourced password manager resources](https://github.com/apple/password-manager-resources/blob/e0d5ba899c57482b06776a18c56b1ad714efd928/quirks/websites-with-shared-credential-backends.json).
+Script that adds new related websites to the "websites-with-shared-credential-backends" Remote Setting collection via [Apple's open sourced password manager resources](https://github.com/apple/password-manager-resources/blob/e0d5ba899c57482b06776a18c56b1ad714efd928/quirks/websites-with-shared-credential-backends.json) and adds new password rules to the "password-rules" Remote Setting collection via [Apple's open sourced password manager rules](https://github.com/apple/password-manager-resources/blob/main/quirks/password-rules.json).
 
 ## Usage
 

package.json

@@ -1,7 +1,7 @@
 {
-  "name": "update-websites-with-shared-credential-backends",
+  "name": "update-passwordmgr-remote-settings-collections",
-  "version": "0.0.1",
+  "version": "0.0.2",
-  "description": "Updates the 'websites-with-shared-credential-backends' collection on Remote Settings",
+  "description": "Updates the 'websites-with-shared-credential-backends' and 'password-rules' collections on Remote Settings",
   "author": "Tim Giles",
   "main": "index.js",
   "engines": {
@@ -14,13 +14,13 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/mozilla/passwordmgr-related-realms-updater.git"
+    "url": "git+https://github.com/mozilla/passwordmgr-remote-settings-updater.git"
   },
   "license": "MPL-2.0",
   "bugs": {
-    "url": "https://github.com/mozilla/passwordmgr-related-realms-updater/issues"
+    "url": "https://github.com/mozilla/passwordmgr-remote-settings-updater/issues"
   },
-  "homepage": "https://github.com/mozilla/passwordmgr-related-realms-updater#readme",
+  "homepage": "https://github.com/mozilla/passwordmgr-remote-settings-updater#readme",
   "dependencies": {
     "atob": "^2.1.2",
     "btoa": "^1.2.1",

update-script.js

@@ -3,25 +3,28 @@ const btoa = require("btoa");
 const fetch = require("node-fetch");
 const AppConstants = require("./app-constants");
 
-const COLLECTION_ID = "websites-with-shared-credential-backends";
+const RELATED_REALMS_COLLECTION_ID = "websites-with-shared-credential-backends";
+const PASSWORD_RULES_COLLECTION_ID = "password-rules";
 /** @type {String} */
 const FX_RS_WRITER_USER = AppConstants.FX_REMOTE_SETTINGS_WRITER_USER;
 /** @type {String} */
 const FX_RS_WRITER_PASS = AppConstants.FX_REMOTE_SETTINGS_WRITER_PASS;
 /** @type {String} */
 const SERVER_ADDRESS = AppConstants.FX_REMOTE_SETTINGS_WRITER_SERVER;
-const BUCKET = "main-workspace";
+const BUCKET = "main";
-const APPLE_API_ENDPOINT = "https://api.github.com/repos/apple/password-manager-resources/contents/quirks/websites-with-shared-credential-backends.json";
+const RELATED_REALMS_API_ENDPOINT = "https://api.github.com/repos/apple/password-manager-resources/contents/quirks/websites-with-shared-credential-backends.json";
+const PASSWORD_RULES_API_ENDPOINT = "https://api.github.com/repos/apple/password-manager-resources/contents/quirks/password-rules.json";
 
 /**
- * Fetches the source records from the APPLE_API_ENDPOINT.
+ * Fetches the source records from the apiEndpoint param
  *
  * Since this script should run once every two weeks, we don't need a GitHub token.
  * See also: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
- * @return {String[][]} The related realms
+ * @param {string} apiEndpoint either `RELATED_REALMS_API_ENDPOINT` or `PASSWORD_RULES_API_ENDPOINT`
+ * @return {String[][]} The source records
  */
-const getSourceRecords = async () => {
+const getSourceRecords = async (apiEndpoint) => {
-  const response = await fetch(APPLE_API_ENDPOINT, {
+  const response = await fetch(apiEndpoint, {
     headers: {
       "Accept": "application/vnd.github.v3.raw"
     }
@@ -38,7 +41,7 @@ const arrayEquals = (a, b) => {
 };
 
 /**
- * Updates the existing record in Remote Settings with the updated data from Apple's GitHub repository
+ * Updates the existing record in the "websites-with-shared-credential-backends" Remote Settings collection with the updated data from Apple's GitHub repository
  *
  * @param {KintoClient} client KintoClient instance
  * @param {string} bucket Name of the Remote Settings bucket
@@ -46,15 +49,16 @@ const arrayEquals = (a, b) => {
  * @param {string} newRecord.id ID from the current related realms object from the Remote Settings server
  * @param {string[][]} newRecord.relatedRealms Updated related realms array from GitHub
  */
-const updateRecord = async (client, bucket, newRecord) => {
+const updateRelatedRealmsRecord = async (client, bucket, newRecord) => {
-  await client.bucket(bucket).collection(COLLECTION_ID).updateRecord(newRecord);
+  const cid = RELATED_REALMS_COLLECTION_ID;
-  const postServerData = await client.bucket(bucket).collection(COLLECTION_ID).getData();
+  await client.bucket(bucket).collection(cid).updateRecord(newRecord);
+  const postServerData = await client.bucket(bucket).collection(cid).getData();
   const setDataObject = {
     status: "to-review",
     last_modified: postServerData.last_modified
   };
-  await client.bucket(bucket).collection(COLLECTION_ID).setData(setDataObject, { patch: true });
+  await client.bucket(bucket).collection(cid).setData(setDataObject, { patch: true });
-  console.log(`Found new records, committed changes to ${COLLECTION_ID} collection.`);
+  console.log(`Found new records, committed changes to ${cid} collection.`);
 };
 
 /**
@@ -63,13 +67,14 @@ const updateRecord = async (client, bucket, newRecord) => {
  * @param {KintoClient} client
  * @param {string} bucket
  */
-const createRecord = async (client, bucket, sourceRecords) => {
+const createRelatedRealmsRecord = async (client, bucket, sourceRecords) => {
-  const result = await client.bucket(bucket).collection(COLLECTION_ID).createRecord({
+  const cid = RELATED_REALMS_COLLECTION_ID;
+  const result = await client.bucket(bucket).collection(cid).createRecord({
     relatedRealms: sourceRecords
   });
-  const postServerData = await client.bucket(bucket).collection(COLLECTION_ID).getData();
+  const postServerData = await client.bucket(bucket).collection(cid).getData();
-  await client.bucket(bucket).collection(COLLECTION_ID).setData({ status: "to-review", last_modified: postServerData.last_modified }, { patch: true });
+  await client.bucket(bucket).collection(cid).setData({ status: "to-review", last_modified: postServerData.last_modified }, { patch: true });
-  console.log(`Added new record to ${COLLECTION_ID}`, result);
+  console.log(`Added new record to ${cid}`, result);
 };
 
 const printSuccessMessage = () => {
@@ -77,13 +82,13 @@ const printSuccessMessage = () => {
 }
 
 /**
- * Determines if there are new records from the GitHub source
+ * Determines if there are new records from the GitHub source for the "websites-with-shared-credential-backends" collection
  *
  * @param {String[][]} sourceRecords Related realms from Apple's GitHub
  * @param {String[][]} destinationRecords Related realms from Remote Settings
  * @return {Boolean} `true` if there are new records, `false` if there are no new records 
  */
-const checkIfNewRecords = (sourceRecords, destinationRecords) => {
+const checkIfNewRelatedRealmsRecords = (sourceRecords, destinationRecords) => {
   let areNewRecords = false;
   if (sourceRecords.length !== destinationRecords.length) {
     areNewRecords = true;
@@ -97,6 +102,106 @@ const checkIfNewRecords = (sourceRecords, destinationRecords) => {
   return areNewRecords;
 }
 
+/**
+ * Converts the records from the "password-rules" Remote Settings collection into a Map 
+ * for easier comparison against the GitHub source of truth records.
+ *
+ * @param {Object[]} records
+ * @param {string} records.Domain
+ * @param {string} records[password-rules]
+ * @return {Map} 
+ */
+const passwordRulesRecordsToMap = (records) => {
+  let map = new Map();
+  for (let record of records) {
+    let { id, Domain: domain, "password-rules": rules } = record;
+    map.set(domain, { id: id, "password-rules": rules });
+  }
+  return map;
+}
+
+/**
+ * Creates and/or updates the existing records in the "password-rules" Remote Settings collection with the updated data from Apple's GitHub repository
+ *
+ * @param {KintoClient} client KintoClient instance
+ * @param {string} bucket Name of the Remote Settings bucket
+ */
+const createAndUpdateRulesRecords = async (client, bucket) => {
+  let collection = client.bucket(bucket).collection(PASSWORD_RULES_COLLECTION_ID);
+  let sourceRulesByDomain = await getSourceRecords(PASSWORD_RULES_API_ENDPOINT);
+  let { data: remoteSettingsRecords } = await collection.listRecords();
+  let remoteSettingsRulesByDomain = passwordRulesRecordsToMap(remoteSettingsRecords);
+  let batchRecords = [];
+
+  for (let domain in sourceRulesByDomain) {
+    let passwordRules = sourceRulesByDomain[domain]["password-rules"];
+    let { id, "password-rules": oldRules } = remoteSettingsRulesByDomain.get(domain);
+    if (!id) {
+      let newRecord = { "Domain": domain, "password-rules": passwordRules };
+      batchRecords.push(newRecord);
+      console.log("Added new record to batch!", newRecord);
+    }
+    if (id && oldRules !== passwordRules) {
+      let updatedRecord = { id, "Domain": domain, "password-rules": passwordRules };
+      batchRecords.push(updatedRecord);
+      console.log("Added updated record to batch!", updatedRecord);
+    }
+
+  }
+  await collection.batch(batch => {
+    for (let record of batchRecords) {
+      if (record.id) {
+        batch.updateRecord(record);
+      } else {
+        batch.createRecord(record);
+      }
+    }
+  });
+
+  const postServerData = await collection.getData();
+  const setDataObject = {
+    status: "to-review",
+    last_modified: postServerData.last_modified
+  };
+  await collection.setData(setDataObject, { patch: true });
+  if (batchRecords.length) {
+    console.log(`Found new and/or updated records, committed changes to ${PASSWORD_RULES_COLLECTION_ID} collection.`);
+  } else {
+    console.log(`Found no new or updated records for the ${PASSWORD_RULES_COLLECTION_ID} collection.`);
+  }
+};
+
+/**
+ * Creates and/or updates the existing records in the "websites-with-shared-credential-backends" Remote Settings collection 
+ * with the updated data from Apple's GitHub repository.
+ *
+ * @param {KintoClient} client
+ * @param {string} bucket
+ */
+const createAndUpdateRelatedRealmsRecords = async (client, bucket) => {
+  let { data: relatedRealmsData } = await client.bucket(bucket).collection(RELATED_REALMS_COLLECTION_ID).listRecords();
+  let realmsGithubRecords = await getSourceRecords(RELATED_REALMS_API_ENDPOINT);
+  let id = relatedRealmsData[0]?.id;
+  // If there is no ID from Remote Settings, we need to create a new record in the related realms collection
+  if (!id) {
+    await createRelatedRealmsRecord(client, bucket, realmsGithubRecords);
+  } else {
+    // If there is an ID, we can compare the source and destination records
+    let currentRecords = relatedRealmsData[0].relatedRealms;
+    let areNewRecords = checkIfNewRelatedRealmsRecords(realmsGithubRecords, currentRecords);
+    // If there are new records, we need to update the data of the record using the current ID
+    if (areNewRecords) {
+      let newRecord = {
+        id: id,
+        relatedRealms: realmsGithubRecords
+      };
+      await updateRelatedRealmsRecord(client, bucket, newRecord)
+    } else {
+      console.log(`No new records! Not committing any changes to ${RELATED_REALMS_COLLECTION_ID} collection.`);
+    }
+  }
+};
+
 /**
  * The runner for the script.
  * 
@@ -115,28 +220,8 @@ const main = async () => {
       }
     });
 
-    let records = await client.bucket(BUCKET).collection(COLLECTION_ID).listRecords();
+    await createAndUpdateRelatedRealmsRecords(client, BUCKET);
-    let data = records.data;
+    await createAndUpdateRulesRecords(client, BUCKET);
-    let githubRecords = await getSourceRecords();
-    let id = data[0]?.id;
-    // If there is no ID from Remote Settings, we need to create a new record
-    if (!id) {
-      await createRecord(client, BUCKET, githubRecords);
-    } else {
-      // If there is an ID, we can compare the source and destination records
-      let currentRecords = data[0].relatedRealms;
-      let areNewRecords = checkIfNewRecords(githubRecords, currentRecords);
-      // If there are new records, we need to update the data of the record using the current ID
-      if (areNewRecords) {
-        let newRecord = {
-          id: id,
-          relatedRealms: githubRecords
-        };
-        await updateRecord(client, BUCKET, newRecord)
-      } else {
-        console.log("No new records! Not committing any changes to Remote Settings collection.");
-      }
-    }
   } catch (e) {
     console.error(e);
     return 1;