feat: add update script to automatically update appropriate Remote Settings collection (#1)
* chore: add gitignore * build: add package.json * feat: add update script * build: update dependencies * fix: add error handling for empty username/password * fix: fix missing async keywords * refactor: add success logging * refactor: make create record logging consistent with update record * docs: update types for consts and getSourceRecords * docs: add README * fix: remove debug collection name * fix: add NODE_ENV variable to .env * refactor: remove unused path import * docs: update README * refactor: rename env to env sample * docs: clarify that VPN is required for Remote Settings server access * refactor: rename collection variable * refactor: update tense in updateRecord logging message * refactor: extract bucket to global const * refactor: simplify main function * build: update Node version for optional chaining * fix: fix main loop logic when there is no data in RS
This commit is contained in:
Tim Giles Jr
GitHub
Родитель
f98215694b
Коммит
c8a4777e4f
|
|
@ -0,0 +1,4 @@
|
|||
FX_REMOTE_SETTINGS_WRITER_SERVER="https://settings-writer.prod.mozaws.net/v1"
|
||||
FX_REMOTE_SETTINGS_WRITER_USER=""
|
||||
FX_REMOTE_SETTINGS_WRITER_PASS=""
|
||||
NODE_ENV="production"
|
||||
|
|
@ -0,0 +1 @@
|
|||
node_modules
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# passwordmgr-related-realms-updater
|
||||
|
||||
Script that adds new related websites to the "websites-with-shared-credential-backends" Remote Setting collection via [Apple's open sourced password manager resources](https://github.com/apple/password-manager-resources/blob/e0d5ba899c57482b06776a18c56b1ad714efd928/quirks/websites-with-shared-credential-backends.json).
|
||||
|
||||
## Usage
|
||||
|
||||
The script will _not run_ without the following environment variables set in `.env`:
|
||||
- `FX_REMOTE_SETTINGS_WRITER_USER`
|
||||
- `FX_REMOTE_SETTINGS_WRITER_PASS`
|
||||
- `FX_REMOTE_SETTINGS_WRITER_SERVER`
|
||||
- `NODE_ENV`
|
||||
|
||||
To run this script:
|
||||
|
||||
`$ node update-script.js`
|
||||
|
||||
**Note**: Corporate VPN access is required to access the Remote Settings servers
|
||||
|
||||
The script will exit with a `0` for success and a `1` if there were any errors.
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
"use strict";
|
||||
|
||||
require("dotenv").config();
|
||||
|
||||
const environmentVariables = [
|
||||
"NODE_ENV",
|
||||
"FX_REMOTE_SETTINGS_WRITER_SERVER",
|
||||
"FX_REMOTE_SETTINGS_WRITER_USER",
|
||||
"FX_REMOTE_SETTINGS_WRITER_PASS"
|
||||
]
|
||||
|
||||
const AppConstants = {};
|
||||
|
||||
for (const v of environmentVariables) {
|
||||
if (process.env[v] === undefined) {
|
||||
throw new Error(`Required environment variable was not set: ${v}`);
|
||||
}
|
||||
AppConstants[v] = process.env[v];
|
||||
}
|
||||
|
||||
module.exports = Object.freeze(AppConstants);
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
{
|
||||
"name": "update-websites-with-shared-credential-backends",
|
||||
"version": "0.0.1",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"asynckit": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
|
||||
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
|
||||
},
|
||||
"atob": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
|
||||
"integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
|
||||
},
|
||||
"btoa": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/btoa/-/btoa-1.2.1.tgz",
|
||||
"integrity": "sha512-SB4/MIGlsiVkMcHmT+pSmIPoNDoHg+7cMzmt3Uxt628MTz2487DKSqK/fuhFBrkuqrYv5UCEnACpF4dTFNKc/g=="
|
||||
},
|
||||
"combined-stream": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
|
||||
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
|
||||
"requires": {
|
||||
"delayed-stream": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"delayed-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
|
||||
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
|
||||
},
|
||||
"dotenv": {
|
||||
"version": "8.2.0",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
|
||||
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
|
||||
},
|
||||
"form-data": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
|
||||
"integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"kinto-http": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/kinto-http/-/kinto-http-5.1.1.tgz",
|
||||
"integrity": "sha512-Hg1Yqc/HgmMHv2VFAxHoXp91hVbOZeymQG7/qoniRndUHJ4p7bUkzYEZ/1lDnWxmkziI1zoU0w6d0dHEVaJ5Tg==",
|
||||
"requires": {
|
||||
"uuid": "^7.0.0"
|
||||
}
|
||||
},
|
||||
"mime-db": {
|
||||
"version": "1.45.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz",
|
||||
"integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w=="
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.28",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz",
|
||||
"integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==",
|
||||
"requires": {
|
||||
"mime-db": "1.45.0"
|
||||
}
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"uuid": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
|
||||
"integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"name": "update-websites-with-shared-credential-backends",
|
||||
"version": "0.0.1",
|
||||
"description": "Updates the 'websites-with-shared-credential-backends' collection on Remote Settings",
|
||||
"author": "Tim Giles",
|
||||
"main": "index.js",
|
||||
"engines": {
|
||||
"node": "14"
|
||||
},
|
||||
"private": "true",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/TGiles/bug-1120684.git"
|
||||
},
|
||||
"license": "MPL-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/TGiles/bug-1120684/issues"
|
||||
},
|
||||
"homepage": "https://github.com/TGiles/bug-1120684#readme",
|
||||
"dependencies": {
|
||||
"atob": "^2.1.2",
|
||||
"btoa": "^1.2.1",
|
||||
"dotenv": "^8.2.0",
|
||||
"form-data": "^3.0.0",
|
||||
"kinto-http": "^5.1.1",
|
||||
"node-fetch": "^2.6.1"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
const KintoClient = require("kinto-http").default;
|
||||
const btoa = require("btoa");
|
||||
const fetch = require("node-fetch");
|
||||
const AppConstants = require("./app-constants");
|
||||
|
||||
const COLLECTION_ID = "websites-with-shared-credential-backends";
|
||||
/** @type {String} */
|
||||
const FX_RS_WRITER_USER = AppConstants.FX_REMOTE_SETTINGS_WRITER_USER;
|
||||
/** @type {String} */
|
||||
const FX_RS_WRITER_PASS = AppConstants.FX_REMOTE_SETTINGS_WRITER_PASS;
|
||||
/** @type {String} */
|
||||
const SERVER_ADDRESS = AppConstants.FX_REMOTE_SETTINGS_WRITER_SERVER;
|
||||
const BUCKET = "main-workspace";
|
||||
const APPLE_API_ENDPOINT = "https://api.github.com/repos/apple/password-manager-resources/contents/quirks/websites-with-shared-credential-backends.json";
|
||||
|
||||
/**
|
||||
* Fetches the source records from the APPLE_API_ENDPOINT.
|
||||
*
|
||||
* Since this script should run once every two weeks, we don't need a GitHub token.
|
||||
* See also: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting
|
||||
* @return {String[][]} The related realms
|
||||
*/
|
||||
const getSourceRecords = async () => {
|
||||
const response = await fetch(APPLE_API_ENDPOINT, {
|
||||
headers: {
|
||||
"Accept": "application/vnd.github.v3.raw"
|
||||
}
|
||||
});
|
||||
const data = await response.json();
|
||||
return data;
|
||||
}
|
||||
|
||||
const arrayEquals = (a, b) => {
|
||||
return Array.isArray(a) &&
|
||||
Array.isArray(b) &&
|
||||
a.length === b.length &&
|
||||
a.every((val, index) => val === b[index]);
|
||||
};
|
||||
|
||||
/**
|
||||
* Updates the existing record in Remote Settings with the updated data from Apple's GitHub repository
|
||||
*
|
||||
* @param {KintoClient} client KintoClient instance
|
||||
* @param {string} bucket Name of the Remote Settings bucket
|
||||
* @param {Object} newRecord Object containing the updated related realms object
|
||||
* @param {string} newRecord.id ID from the current related realms object from the Remote Settings server
|
||||
* @param {string[][]} newRecord.relatedRealms Updated related realms array from GitHub
|
||||
*/
|
||||
const updateRecord = async (client, bucket, newRecord) => {
|
||||
// ? Why do we ignore the result of the `updateRecord` call?
|
||||
await client.bucket(bucket).collection(COLLECTION_ID).updateRecord(newRecord);
|
||||
const postServerData = await client.bucket(bucket).collection(COLLECTION_ID).getData();
|
||||
const setDataObject = {
|
||||
status: "to-review",
|
||||
last_modified: postServerData.last_modified
|
||||
};
|
||||
await client.bucket(bucket).collection(COLLECTION_ID).setData(setDataObject, { patch: true });
|
||||
console.log(`Found new records, committed changes to ${COLLECTION_ID} collection.`);
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a new record in Remote Settings if there are no records in the WEBSITES_WITH_SHARED_CREDENTIAL_COLLECTION
|
||||
*
|
||||
* @param {KintoClient} client
|
||||
* @param {string} bucket
|
||||
*/
|
||||
const createRecord = async (client, bucket, sourceRecords) => {
|
||||
const result = await client.bucket(bucket).collection(COLLECTION_ID).createRecord({
|
||||
relatedRealms: sourceRecords
|
||||
});
|
||||
const postServerData = await client.bucket(bucket).collection(COLLECTION_ID).getData();
|
||||
await client.bucket(bucket).collection(COLLECTION_ID).setData({ status: "to-review", last_modified: postServerData.last_modified }, { patch: true });
|
||||
console.log(`Added new record to ${COLLECTION_ID}`, result);
|
||||
};
|
||||
|
||||
const printSuccessMessage = () => {
|
||||
console.log("Script finished successfully!");
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if there are new records from the GitHub source
|
||||
*
|
||||
* @param {String[][]} sourceRecords Related realms from Apple's GitHub
|
||||
* @param {String[][]} destinationRecords Related realms from Remote Settings
|
||||
* @return {Boolean} `true` if there are new records, `false` if there are no new records
|
||||
*/
|
||||
const checkIfNewRecords = (sourceRecords, destinationRecords) => {
|
||||
let areNewRecords = false;
|
||||
if (sourceRecords.length !== destinationRecords.length) {
|
||||
areNewRecords = true;
|
||||
}
|
||||
for (let i = 0; i < sourceRecords.length; i++) {
|
||||
if (areNewRecords) {
|
||||
break;
|
||||
}
|
||||
areNewRecords = !arrayEquals(sourceRecords[i], destinationRecords[i]);
|
||||
}
|
||||
return areNewRecords;
|
||||
}
|
||||
|
||||
/**
|
||||
* The runner for the script.
|
||||
*
|
||||
* @return {Number} 0 for success, 1 for failure.
|
||||
*/
|
||||
const main = async () => {
|
||||
if (FX_RS_WRITER_USER === "" || FX_RS_WRITER_PASS === "") {
|
||||
console.error("No username or password set, quitting!");
|
||||
return 1;
|
||||
}
|
||||
const secretString = `${FX_RS_WRITER_USER}:${FX_RS_WRITER_PASS}`;
|
||||
try {
|
||||
const client = new KintoClient(SERVER_ADDRESS, {
|
||||
headers: {
|
||||
Authorization: "Basic " + btoa(secretString)
|
||||
}
|
||||
});
|
||||
|
||||
let records = await client.bucket(BUCKET).collection(COLLECTION_ID).listRecords();
|
||||
let data = records.data;
|
||||
let githubRecords = await getSourceRecords();
|
||||
let id = data[0]?.id;
|
||||
// If there is no ID from Remote Settings, we need to create a new record
|
||||
if (!id) {
|
||||
await createRecord(client, BUCKET, githubRecords);
|
||||
} else {
|
||||
// If there is an ID, we can compare the source and destination records
|
||||
let currentRecords = data[0].relatedRealms;
|
||||
let areNewRecords = checkIfNewRecords(githubRecords, currentRecords);
|
||||
// If there are new records, we need to update the data of the record using the current ID
|
||||
if (areNewRecords) {
|
||||
let newRecord = {
|
||||
id: id,
|
||||
relatedRealms: githubRecords
|
||||
};
|
||||
await updateRecord(client, BUCKET, newRecord)
|
||||
} else {
|
||||
console.log("No new records! Not committing any changes to Remote Settings collection.");
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
return 1;
|
||||
}
|
||||
printSuccessMessage();
|
||||
return 0;
|
||||
};
|
||||
|
||||
main();
|
||||
Загрузка…
Ссылка в новой задаче