From c6ad40270a63cca38e8ef7c2c5d949e02cfb52ee Mon Sep 17 00:00:00 2001 From: Phil Booth Date: Thu, 9 Aug 2018 18:43:29 +0100 Subject: [PATCH] feat(email): write live email-sending config to redis --- .travis.yml | 1 + README.md | 59 +++++++++ package.json | 1 + scripts/email-config.js | 141 ++++++++++++++++++++++ test/local/senders/email.js | 67 +++++++++++ test/scripts/email-config.js | 224 +++++++++++++++++++++++++++++++++++ 6 files changed, 493 insertions(+) create mode 100644 scripts/email-config.js create mode 100644 test/scripts/email-config.js diff --git a/.travis.yml b/.travis.yml index 6398d624..d56c6081 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,6 +40,7 @@ script: - if [ $DB == "mysql" ]; then ./scripts/start-travis-auth-db-mysql.sh; fi - npm run test-ci - npm run test-e2e + - npm run test-scripts # Test fxa-auth-mailer - grunt templates && git status -s | (! grep 'M lib/senders/templates/') - grunt l10n-extract diff --git a/README.md b/README.md index 9b2af07a..2daf6508 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,65 @@ can be overridden in two ways: export CONFIG_FILES="~/fxa-content-server.json,~/fxa-db.json" ``` +### Email config + +There is also some live config +loaded from Redis for the email service. +This config is stored as a JSON string +that looks like this +(every property is optional): + +```json +{ + "sendgrid": { + "percentage": 100, + "regex": "^.+@example\\.com$" + }, + "socketlabs": { + "percentage": 100, + "regex": "^.+@example\\.org$" + }, + "ses": { + "percentage": 10, + "regex": ".*" + } +} +``` + +`scripts/email-config.js` +has been written to help +manage this config. + +* To print the current live config to stdout: + + ``` + node scripts/email-config read + ``` + +* To set the live config from a JSON file on disk: + + ``` + cat foo.json | node scripts/email-config write + ``` + +* To set the live config from a string: + + ``` + echo '{"sendgrid":{"percentage":10}}' | node scripts/email-config write + ``` + +* To undo the last change: + + ``` + node scripts/email-config revert + ``` + +* To check the resolved config for a specific email address: + + ``` + node scripts/email-config check foo@example.com + ``` + ## Troubleshooting Firefox Accounts authorization is a complicated flow. You can get verbose logging by adjusting the log level in the `config.json` on your deployed instance. Add a stanza like: diff --git a/package.json b/package.json index ccd10eba..cc7908ae 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test": "VERIFIER_VERSION=0 MEMCACHE_METRICS_CONTEXT_ADDRESS=none NO_COVERAGE=1 scripts/test-local.sh", "test-ci": "scripts/test-local.sh", "test-e2e": "NODE_ENV=dev mocha test/e2e", + "test-scripts": "mocha test/scripts", "test-remote": "MAILER_HOST=restmail.net MAILER_PORT=80 CORS_ORIGIN=http://baz mocha --timeout=300000 test/remote" }, "repository": { diff --git a/scripts/email-config.js b/scripts/email-config.js new file mode 100644 index 00000000..060e5ab2 --- /dev/null +++ b/scripts/email-config.js @@ -0,0 +1,141 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict' + +const ROOT_DIR = '..' +const LIB_DIR = `${ROOT_DIR}/lib` + +const config = require(`${ROOT_DIR}/config`).getProperties() +const log = require(`${LIB_DIR}/log`)(config.log.level, 'email-config') +const Promise = require(`${LIB_DIR}/promise`) +const redis = require(`${LIB_DIR}/redis`)({ ...config.redis, ...config.redis.email }, log) + +if (! redis) { + console.error('redis is disabled in config, aborting') + process.exit(1) +} + +const COMMANDS = { + read, + write, + revert, + check +} +const KEYS = { + current: 'config', + previous: 'config.previous' +} + +const { argv } = process + +main() + .then(() => redis.close()) + +async function main () { + try { + const command = argv[2] + switch (command) { + case 'read': + case 'write': + case 'revert': + assertArgs(0) + break + case 'check': + assertArgs(1) + break + default: + usageError() + } + const result = await COMMANDS[command](...argv.slice(3)) + if (result) { + console.log(result) + } + } catch (error) { + console.error(error.message) + process.exit(1) + } +} + +function assertArgs (count) { + if (argv.length !== count + 3) { + usageError() + } +} + +function usageError () { + const scriptName = argv[1].substr(argv[1].indexOf('/scripts/') + 1) + throw new Error([ + 'Usage:', + `${scriptName} read - Read the current config to stdout`, + `${scriptName} write - Write the current config from stdin`, + `${scriptName} revert - Undo the last write or revert`, + `${scriptName} check - Check whether matches config` + ].join('\n')) +} + +async function read () { + const current = await redis.get(KEYS.current) + if (current) { + // Parse then stringify for pretty printing + return JSON.stringify(JSON.parse(current), null, ' ') + } +} + +async function write () { + // Parse then stringify for validation + const current = JSON.stringify(JSON.parse(await stdin())) + const previous = await redis.get(KEYS.current) + await redis.set(KEYS.current, current) + if (previous) { + await redis.set(KEYS.previous, previous) + } +} + +function stdin () { + return new Promise((resolve, reject) => { + const chunks = [] + process.stdin.on('readable', () => { + const chunk = process.stdin.read() + if (chunk !== null) { + chunks.push(chunk) + } + }) + process.stdin.on('error', reject) + process.stdin.on('end', () => resolve(chunks.join(''))) + }) +} + +async function revert () { + const previous = await redis.get(KEYS.previous) + const current = await redis.get(KEYS.current) + if (previous) { + await redis.set(KEYS.current, previous) + } else { + await redis.del(KEYS.current) + } + if (current) { + await redis.set(KEYS.previous, current) + } +} + +async function check (emailAddress) { + const current = await redis.get(KEYS.current) + if (current) { + const config = JSON.parse(await redis.get(KEYS.current)) + const result = Object.entries(config) + .filter(([ sender, senderConfig ]) => { + if (senderConfig.regex) { + return new RegExp(senderConfig.regex).test(emailAddress) + } + + return true + }) + .reduce((matches, [ sender, senderConfig ]) => { + matches[sender] = senderConfig + return matches + }, {}) + return JSON.stringify(result, null, ' ') + } +} diff --git a/test/local/senders/email.js b/test/local/senders/email.js index 25b55a14..db155ccf 100644 --- a/test/local/senders/email.js +++ b/test/local/senders/email.js @@ -7,11 +7,15 @@ const ROOT_DIR = '../../..' const assert = require('insist') +const cp = require('child_process') const mocks = require('../../mocks') const P = require('bluebird') +const path = require('path') const proxyquire = require('proxyquire').noPreserveCache() const sinon = require('sinon') +cp.execAsync = P.promisify(cp.exec) + const config = require(`${ROOT_DIR}/config`) const TEMPLATE_VERSIONS = require(`${ROOT_DIR}/lib/senders/templates/_versions.json`) @@ -1873,3 +1877,66 @@ describe('call selectEmailServices with mocked safe-regex, regex-only match and }) }) }) + +if (config.get('redis.email.enabled')) { + const emailAddress = 'foo@example.com'; + + [ 'sendgrid', 'ses', 'socketlabs' ].reduce((promise, service) => { + return promise.then(() => { + return new P((resolve, reject) => { + describe(`call selectEmailServices with real redis containing ${service} config:`, () => { + let mailer, result + + before(() => { + return P.all([ + require(`${ROOT_DIR}/lib/senders/translator`)(['en'], 'en'), + require(`${ROOT_DIR}/lib/senders/templates`).init() + ]).spread((translator, templates) => { + const mockLog = mocks.mockLog() + const Mailer = require(`${ROOT_DIR}/lib/senders/email`)(mockLog, config.getProperties()) + mailer = new Mailer(translator, templates, config.get('smtp')) + return redisWrite({ + [service]: { + regex: '^foo@example\.com$', + percentage: 100 + } + }) + }) + .then(() => mailer.selectEmailServices({ email: emailAddress })) + .then(r => result = r) + }) + + after(() => { + return redisRevert() + .then(() => mailer.stop()) + .then(resolve) + .catch(reject) + }) + + it('returned the correct result', () => { + assert.deepEqual(result, [ + { + emailAddresses: [ emailAddress ], + emailService: 'fxa-email-service', + emailSender: service, + mailer: mailer.emailService + } + ]) + }) + }) + }) + }) + }, P.resolve()) +} + +function redisWrite (config) { + return cp.execAsync(`echo '${JSON.stringify(config)}' | node scripts/email-config write`, { + cwd: path.resolve(__dirname, '../../..') + }) +} + +function redisRevert () { + return cp.execAsync('node scripts/email-config revert', { + cwd: path.resolve(__dirname, '../../..') + }) +} diff --git a/test/scripts/email-config.js b/test/scripts/email-config.js new file mode 100644 index 00000000..56ebf3fa --- /dev/null +++ b/test/scripts/email-config.js @@ -0,0 +1,224 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict' + +const ROOT_DIR = '../..' +const LIB_DIR = `${ROOT_DIR}/lib` + +const assert = require('insist') +const cp = require('child_process') +const mocks = require(`${ROOT_DIR}/test/mocks`) +const P = require('bluebird') +const path = require('path') + +cp.execAsync = P.promisify(cp.exec) + +const config = require(`${ROOT_DIR}/config`).getProperties() +const redis = require(`${LIB_DIR}/redis`)({ + ...config.redis, + ...config.redis.email +}, mocks.mockLog()) + +const cwd = path.resolve(__dirname, ROOT_DIR) + +const KEYS = { + current: 'config', + previous: 'config.previous' +} + +describe('scripts/email-config:', () => { + let current, previous + + beforeEach(() => { + return redis.get(KEYS.current) + .then(result => { + current = result + return redis.get(KEYS.previous) + }) + .then(result => { + previous = result + return redis.del(KEYS.current) + }) + .then(() => redis.del(KEYS.previous)) + }) + + afterEach(() => { + return P.resolve() + .then(() => { + if (current) { + return redis.set(KEYS.current, current) + } + + return redis.del(KEYS.current) + }) + .then(() => { + if (previous) { + return redis.set(KEYS.previous, previous) + } + + return redis.del(KEYS.previous) + }) + }) + + it('read does not fail', () => { + return cp.execAsync('node scripts/email-config read', { cwd }) + }) + + it('write does not fail', () => { + return cp.execAsync('echo "{}" | node scripts/email-config write', { cwd }) + }) + + it('write fails if stdin is not valid JSON', () => { + return cp.execAsync('echo "wibble" | node scripts/email-config write', { cwd }) + .then(() => assert(false, 'script should have failed')) + .catch(() => {}) + }) + + it('revert does not fail', () => { + return cp.execAsync('node scripts/email-config revert', { cwd }) + }) + + it('check does not fail', () => { + return cp.execAsync('node scripts/email-config check foo@example.com', { cwd }) + }) + + it('check fails without argument', () => { + return cp.execAsync('node scripts/email-config check', { cwd }) + .then(() => assert(false, 'script should have failed')) + .catch(() => {}) + }) + + describe('write config with regex:', () => { + let config + + beforeEach(() => { + config = { + sendgrid: { + regex: 'foo', + percentage: 42 + } + } + return cp.execAsync(`echo '${JSON.stringify(config)}' | node scripts/email-config write`, { cwd }) + }) + + it('read prints the config to stdout', () => { + return cp.execAsync('node scripts/email-config read', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, `${JSON.stringify(config, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check matching email prints the config to stdout', () => { + return cp.execAsync('node scripts/email-config check foo@example.com', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, `${JSON.stringify(config, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check non-matching email does not print', () => { + return cp.execAsync('node scripts/email-config check bar@example.com', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, '{}\n') + assert.equal(stderr, undefined) + }) + }) + + describe('write config without regex:', () => { + beforeEach(() => { + config.socketlabs = { + percentage: 10 + } + return cp.execAsync(`echo '${JSON.stringify(config)}' | node scripts/email-config write`, { cwd }) + }) + + it('read prints the config to stdout', () => { + return cp.execAsync('node scripts/email-config read', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, `${JSON.stringify(config, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check matching email prints the both configs to stdout', () => { + return cp.execAsync('node scripts/email-config check foo@example.com', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, `${JSON.stringify(config, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check non-matching email prints one config to stdout', () => { + return cp.execAsync('node scripts/email-config check bar@example.com', { cwd }) + .then((stdout, stderr) => { + const expected = { + socketlabs: config.socketlabs + } + assert.equal(stdout, `${JSON.stringify(expected, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + describe('revert:', () => { + beforeEach(() => { + return cp.execAsync('node scripts/email-config revert', { cwd }) + }) + + it('read prints the previous config to stdout', () => { + return cp.execAsync('node scripts/email-config read', { cwd }) + .then((stdout, stderr) => { + const expected = { + sendgrid: config.sendgrid + } + assert.equal(stdout, `${JSON.stringify(expected, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check matching email prints the previous config to stdout', () => { + return cp.execAsync('node scripts/email-config check foo@example.com', { cwd }) + .then((stdout, stderr) => { + const expected = { + sendgrid: config.sendgrid + } + assert.equal(stdout, `${JSON.stringify(expected, null, ' ')}\n`) + assert.equal(stderr, undefined) + }) + }) + + it('check non-matching email does not print', () => { + return cp.execAsync('node scripts/email-config check bar@example.com', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, '{}\n') + assert.equal(stderr, undefined) + }) + }) + }) + }) + + describe('revert:', () => { + beforeEach(() => { + return cp.execAsync('node scripts/email-config revert', { cwd }) + }) + + it('read does not print', () => { + return cp.execAsync('node scripts/email-config read', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, '') + assert.equal(stderr, undefined) + }) + }) + + it('check matching email does not print', () => { + return cp.execAsync('node scripts/email-config check foo@example.com', { cwd }) + .then((stdout, stderr) => { + assert.equal(stdout, '') + assert.equal(stderr, undefined) + }) + }) + }) + }) +})