Merge pull request #123 from mozilla/mailer-verification-reminders r=vbudhram

feat(reminders): add verification reminders

bin/server.js

@@ -10,6 +10,7 @@ var mailConfig = config.get('mail')
 
 var packageJson = require('../package.json')
 var P = require('bluebird')
+var DB = require('../lib/db')()
 
 // NOTE: Mailer is also used by fxa-auth-server directly with an old logging interface
 // the legacy log module provides an interface to convert old logs to new mozlog logging.
@@ -29,6 +30,19 @@ P.all(
     log.info('config', mailConfig)
     log.info('templates', Object.keys(templates))
 
+    // fetch and process verification reminders
+
+    DB.connect(config.get(config.get('db').backend))
+      .then(
+        function (db) {
+          var verificationReminders = require('../lib/verification-reminders')(mailer, db)
+          verificationReminders.poll()
+        },
+        function (err) {
+          log.error('db', { err: err })
+        }
+      )
+
     var api = restify.createServer()
     api.use(restify.bodyParser())
     /*/

config.js

@@ -66,6 +66,38 @@ var conf = convict({
     format: String,
     default: 'en'
   },
+  verificationReminders: {
+    reminderTimeFirst: {
+      doc: 'Milliseconds since account creation after which the first reminder is sent',
+      default: '48 hours',
+      format: 'duration'
+    },
+    reminderTimeSecond: {
+      doc: 'Milliseconds since account creation after which the second reminder is sent',
+      default: '168 hours',
+      format: 'duration'
+    },
+    reminderTimeFirstOutdated: {
+      doc: 'Milliseconds since account creation after which the reminder should not be sent',
+      default: '167 hours',
+      format: 'duration'
+    },
+    reminderTimeSecondOutdated: {
+      doc: 'Milliseconds since account creation after which the reminder should not be sent',
+      default: '300 hours',
+      format: 'duration'
+    },
+    pollFetch: {
+      doc: 'Number of reminder record to fetch when polling.',
+      format: Number,
+      default: 20
+    },
+    pollTime: {
+      doc: 'Poll duration in milliseconds.',
+      format: 'duration',
+      default: '30 seconds'
+    }
+  },
   mail: {
     host: {
       doc: 'The ip address the server should bind',

config/dev.json

@@ -2,5 +2,10 @@
   "logging": {
     "level": "ALL",
     "fmt": "pretty"
+  },
+  "verificationReminders": {
+    "reminderTimeFirst": 1000,
+    "reminderTimeSecond": 5000,
+    "pollTime": 5000
   }
 }

lib/db.js

@@ -6,6 +6,7 @@ var butil = require('./crypto/butil')
 var log = require('../log')('db')
 var P = require('./promise')
 var Pool = require('./pool')
+var qs = require('querystring')
 
 var bufferize = butil.bufferize
 
@@ -48,5 +49,48 @@ module.exports = function () {
       )
   }
 
+  DB.prototype.account = function (uid) {
+    log.trace({ op: 'DB.account', uid: uid })
+    return this.pool.get('/account/' + uid.toString('hex'))
+      .then(
+        function (body) {
+          var data = bufferize(body)
+          data.emailVerified = !!data.emailVerified
+          return data
+        },
+        function (err) {
+          throw err
+        }
+      )
+  }
+
+  /**
+   * Create a new reminder
+   * @param reminderData
+   * @param {string} reminderData.uid - The uid to remind.
+   * @param {string} reminderData.type - The type of a reminder.
+   */
+  DB.prototype.createVerificationReminder = function (reminderData) {
+    log.debug('createVerificationReminder', reminderData)
+
+    return this.pool.post('/verificationReminders', reminderData)
+  }
+
+  /**
+   * Fetch reminders given reminder age
+   *
+   * @param options
+   * @param {string} options.type - Type of reminder. Can be 'first' or 'second'.
+   * @param {string} options.reminderTime - Reminder age in MS.
+   * @param {string} options.reminderTimeOutdated - Reminder outdated age in MS.
+   * @param {string} options.limit - Number of records to fetch.
+   */
+  DB.prototype.fetchReminders = function (options) {
+    log.debug('fetchReminders', options)
+
+    var query = '?' + qs.stringify(options)
+    return this.pool.get('/verificationReminders' + query, options)
+  }
+
   return DB
 }

lib/pool.js

@@ -83,7 +83,6 @@ Pool.prototype.get = function (path) {
   return this.request('GET', path)
 }
 
-
 Pool.prototype.close = function () {
   /*/
    This is a hack to coax the server to close its existing connections

lib/verification-reminders.js

@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var logger = require('../log')('verification-reminders')
+var P = require('./promise')
+var config = require('../config')
+var reminderConfig = config.get('verificationReminders')
+
+module.exports = function (mailer, db, options) {
+  options = options || {}
+  var log = options.log || logger
+
+  return {
+    /**
+     * Process a fetch reminder, sends the email if account is not verified.
+     *
+     * @param reminderData
+     * @param {buffer} reminderData.uid - The uid to remind.
+     * @param {string} reminderData.type - The type of a reminder.
+     * @returns {Promise}
+     * @private
+     */
+    _processReminder: function (reminderData) {
+        log.debug('_processReminder', reminderData)
+
+        return db.account(reminderData.uid)
+          .then(function (account) {
+            if (! account.emailVerified) {
+              // if account is not verified then send the reminder
+              mailer.verificationReminderEmail({
+                email: account.email,
+                uid: account.uid.toString('hex'),
+                code: account.emailCode.toString('hex'),
+                type: reminderData.type,
+                acceptLanguage: account.locale
+              })
+            } else {
+              log.debug('_processReminder', { msg: 'Already Verified' })
+            }
+          }, function (err) {
+            log.debug('_processReminder', { err: err })
+          })
+    },
+    _continuousPoll: function () {
+      var self = this
+      // fetch reminders for both types, separately
+      var firstReminder = db.fetchReminders({
+        reminderTime: reminderConfig.reminderTimeFirst,
+        reminderTimeOutdated: reminderConfig.reminderTimeFirstOutdated,
+        type: 'first',
+        limit: reminderConfig.pollFetch
+      })
+
+      var secondReminder = db.fetchReminders({
+        reminderTime: reminderConfig.reminderTimeSecond,
+        reminderTimeOutdated: reminderConfig.reminderTimeSecondOutdated,
+        type: 'second',
+        limit: reminderConfig.pollFetch
+      })
+
+      return P.all([firstReminder, secondReminder])
+        .then(
+          function (reminderResultsByType) {
+            reminderResultsByType.forEach(function (reminders) {
+              reminders.forEach(function (reminder) {
+                self._processReminder(reminder)
+              })
+            })
+          }
+        ).catch(function (err) {
+          log.error('_continuousPoll', { err: err })
+        })
+    },
+    poll: function poll () {
+      var self = this
+
+      setTimeout(function () {
+        self.poll()
+      }, reminderConfig.pollTime)
+
+      self._continuousPoll()
+    }
+  }
+}

mailer.js

@@ -447,7 +447,15 @@ module.exports = function (log) {
   }
 
   Mailer.prototype.verificationReminderEmail = function (message) {
-    log.trace({ op: 'mailer.verificationReminderEmail', email: message.email, uid: message.uid })
+    log.trace({ op: 'mailer.verificationReminderEmail', email: message.email, type: message.type })
+
+    if (! message || ! message.code || ! message.email) {
+      log.error({
+        op: 'mailer.verificationReminderEmail',
+        err: 'Missing code or email'
+      })
+      return
+    }
 
     var subject = gettext('Hello again.')
     var template = 'verificationReminderFirstEmail'
@@ -461,20 +469,15 @@ module.exports = function (log) {
       code: message.code
     }
 
-    if (message.service) { query.service = message.service }
-    if (message.redirectTo) { query.redirectTo = message.redirectTo }
-    if (message.resume) { query.resume = message.resume }
-
     var link = this.verificationUrl + '?' + qs.stringify(query)
     query.one_click = true
     var oneClickLink = this.verificationUrl + '?' + qs.stringify(query)
 
     return this.send({
-      acceptLanguage: message.acceptLanguage,
+      acceptLanguage: message.acceptLanguage || 'en',
       email: message.email,
       headers: {
         'X-Link': link,
-        'X-Service-ID': message.service,
         'X-Uid': message.uid,
         'X-Verify-Code': message.code
       },

test/helpers.js

@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*eslint no-console: 0*/
+
+var uuid = require('uuid')
+
+var zeroBuffer16 = Buffer('00000000000000000000000000000000', 'hex')
+var zeroBuffer32 = Buffer('0000000000000000000000000000000000000000000000000000000000000000', 'hex')
+
+function createTestAccount() {
+  var account = {
+    uid: uuid.v4('binary'),
+    email: 'foo' + Math.random() + '@bar.com',
+    emailCode: zeroBuffer16,
+    emailVerified: false,
+    verifierVersion: 1,
+    verifyHash: zeroBuffer32,
+    authSalt: zeroBuffer32,
+    kA: zeroBuffer32,
+    wrapWrapKb: zeroBuffer32,
+    createdAt: Date.now(),
+    verifierSetAt: Date.now(),
+    locale: 'da, en-gb;q=0.8, en;q=0.7'
+  }
+
+  account.normalizedEmail = account.email.toLowerCase()
+
+  return account
+}
+
+module.exports = {
+  createTestAccount: createTestAccount
+}

test/local/db_tests.js

@@ -4,39 +4,16 @@
 
 var tap = require('tap')
 var test = tap.test
-var uuid = require('uuid')
 
 var butil = require('../../lib/crypto/butil')
 var unbuffer = butil.unbuffer
 var config = require('../../config').getProperties()
 var TestServer = require('../test_server')
 
-var zeroBuffer16 = Buffer('00000000000000000000000000000000', 'hex')
-var zeroBuffer32 = Buffer('0000000000000000000000000000000000000000000000000000000000000000', 'hex')
-
+var testHelpers = require('../helpers')
 
 var DB = require('../../lib/db')()
 
-function createTestAccount() {
-  var account = {
-    uid: uuid.v4('binary'),
-    email: 'foo' + Math.random() + '@bar.com',
-    emailCode: zeroBuffer16,
-    emailVerified: false,
-    verifierVersion: 1,
-    verifyHash: zeroBuffer32,
-    authSalt: zeroBuffer32,
-    kA: zeroBuffer32,
-    wrapWrapKb: zeroBuffer32,
-    createdAt: Date.now(),
-    verifierSetAt: Date.now()
-  }
-
-  account.normalizedEmail = account.email.toLowerCase()
-
-  return account
-}
-
 var dbServer
 var dbConn = TestServer.start(config)
   .then(
@@ -62,7 +39,6 @@ test(
   }
 )
 
-
 test(
   'get email record',
   function (t) {
@@ -74,7 +50,7 @@ test(
     return dbConn
       .then(function (dbObj) {
         db = dbObj
-        accountData = createTestAccount()
+        accountData = testHelpers.createTestAccount()
 
         return db.pool.put(
           '/account/' + accountData.uid.toString('hex'),

test/local/reminder_db_tests.js

@@ -0,0 +1,99 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var tap = require('tap')
+var test = tap.test
+
+var P = require('../../lib/promise')
+var butil = require('../../lib/crypto/butil')
+var unbuffer = butil.unbuffer
+var config = require('../../config').getProperties()
+var TestServer = require('../test_server')
+var testHelpers = require('../helpers')
+
+var DB = require('../../lib/db')()
+
+var dbServer
+var dbConn = TestServer.start(config)
+  .then(
+    function (server) {
+      dbServer = server
+      return DB.connect(config[config.db.backend])
+    }
+  )
+
+test(
+  'fetchReminders',
+  function (t) {
+    var accountData
+    var db
+
+    return dbConn
+      .then(function (dbObj) {
+        db = dbObj
+        accountData = testHelpers.createTestAccount()
+
+        return db.pool.put(
+          '/account/' + accountData.uid.toString('hex'),
+          unbuffer(accountData)
+        )
+      })
+      .then(function () {
+        var rem1 = db.createVerificationReminder({
+          type: 'first',
+          uid: accountData.uid.toString('hex')
+        })
+
+        var rem2 = db.createVerificationReminder({
+          type: 'second',
+          uid: accountData.uid.toString('hex')
+        })
+
+        return P.all([rem1, rem2]).catch(function (err) {
+          throw err
+        })
+      })
+      .then(
+        function () {
+          return db.fetchReminders({
+            // fetch reminders older than 'reminderTime'
+            reminderTime: 1,
+            reminderTimeOutdated: 5000,
+            type: 'first',
+            limit: 200
+          })
+        }
+      )
+      .then(
+        function (reminders) {
+          var reminderFound = false
+          reminders.some(function (reminder) {
+            if (reminder.uid === accountData.uid.toString('hex')) {
+              reminderFound = true
+              return true
+            }
+          })
+
+          t.ok(reminderFound, 'fetched the created reminder')
+          t.end()
+        },
+        function (err) {
+          throw err
+        }
+      )
+  }
+)
+
+test(
+  'teardown',
+  function (t) {
+    return dbConn.then(function(db) {
+      return db.close()
+    }).then(function() {
+      return dbServer.stop()
+    }).then(function () {
+      t.end()
+    })
+  }
+)

test/local/reminder_tests.js

@@ -0,0 +1,156 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var sinon = require('sinon')
+var extend = require('util')._extend
+var tap = require('tap')
+var test = tap.test
+var P = require('../../lib/promise')
+
+var TEST_EMAIL = 'test@restmail.net'
+var TEST_ACCOUNT_RECORD = {
+  emailVerified: false,
+  email: TEST_EMAIL,
+  emailCode: new Buffer('foo'),
+  uid: new Buffer('bar'),
+  locale: 'da, en-gb;q=0.8, en;q=0.7'
+}
+var REMINDER_TYPE = 'first'
+var SAMPLE_REMINDER = {uid: TEST_ACCOUNT_RECORD.uid, type: REMINDER_TYPE}
+
+var sandbox
+var mockMailer = {
+  verificationReminderEmail: function () {}
+}
+
+var mockDb = {
+  fetchReminders: function () {},
+  account: function () { return P.resolve(TEST_ACCOUNT_RECORD) }
+}
+
+var LOG_METHOD_NAMES = ['trace', 'increment', 'info', 'error', 'begin', 'debug']
+
+var mockLog = function(methods) {
+  var log = extend({}, methods)
+  LOG_METHOD_NAMES.forEach(function(name) {
+    if (!log[name]) {
+      log[name] = function() {}
+    }
+  })
+  return log
+}
+
+test('_processReminder sends first reminder for unverified emails', function (t) {
+  setup()
+
+  var legacyMailerLog = require('../../legacy_log')(mockLog())
+  var Mailer = require('../../mailer')(legacyMailerLog)
+  var mailer = new Mailer({}, {}, {})
+  sandbox.stub(mailer, 'send', function (vals) {
+    t.equal(vals.acceptLanguage, TEST_ACCOUNT_RECORD.locale, 'correct locale')
+    t.equal(vals.uid, TEST_ACCOUNT_RECORD.uid.toString('hex'), 'correct uid')
+    t.equal(vals.email, TEST_ACCOUNT_RECORD.email, 'correct email')
+    t.equal(vals.template, 'verificationReminderFirstEmail', 'correct template')
+    t.equal(vals.subject, 'Hello again.', 'correct subject')
+    t.equal(vals.headers['X-Verify-Code'], TEST_ACCOUNT_RECORD.emailCode.toString('hex'), 'correct code')
+    t.ok(vals.templateValues.link.indexOf(TEST_ACCOUNT_RECORD.emailCode.toString('hex')) >= 0, 'correct link')
+    done(t)
+  })
+
+  require('../../lib/verification-reminders')(mailer, mockDb)
+    ._processReminder(SAMPLE_REMINDER)
+})
+
+test('_processReminder sends second reminder for unverified emails', function (t) {
+  setup()
+  var legacyMailerLog = require('../../legacy_log')(mockLog())
+  var Mailer = require('../../mailer')(legacyMailerLog)
+  var mailer = new Mailer({}, {}, {})
+  sandbox.stub(mailer, 'send', function (vals) {
+    t.equal(vals.template, 'verificationReminderSecondEmail', 'correct template')
+    t.equal(vals.headers['X-Verify-Code'], TEST_ACCOUNT_RECORD.emailCode.toString('hex'), 'correct code')
+    t.ok(vals.templateValues.link.indexOf(TEST_ACCOUNT_RECORD.emailCode.toString('hex')) >= 0, 'correct link')
+    done(t)
+  })
+
+  require('../../lib/verification-reminders')(mailer, mockDb)
+    ._processReminder({uid: TEST_ACCOUNT_RECORD.uid, type: 'second'})
+})
+
+test('_processReminder - does not send email for verified emails', function (t) {
+  setup()
+  sandbox.stub(mockDb, 'account', function () {
+    return P.resolve({
+      emailVerified: true
+    })
+  })
+
+  var log = mockLog({
+    debug: function (op, data) {
+      if (data.msg === 'Already Verified') {
+        done(t)
+      }
+    }
+  })
+
+  require('../../lib/verification-reminders')(mockMailer, mockDb, { log: log })
+    ._processReminder(SAMPLE_REMINDER)
+})
+
+test('_processReminder - catches errors', function (t) {
+  setup()
+  var errorMsg = 'Something is wrong.'
+
+  sandbox.stub(mockDb, 'account', function () {
+    return P.reject(new Error(errorMsg))
+  })
+
+  var log = mockLog({
+    debug: function (op, data) {
+      if (data.err && data.err.message === errorMsg) {
+        done(t)
+      }
+    }
+  })
+
+  require('../../lib/verification-reminders')(mockMailer, mockDb, { log: log })
+    ._processReminder(SAMPLE_REMINDER)
+})
+
+
+test('_continuousPoll - calls _continuousPoll', function (t) {
+  setup()
+
+  sandbox.stub(mockDb, 'fetchReminders', function (options) {
+    t.ok(options.type)
+    t.ok(options.reminderTime)
+    t.ok(options.limit)
+
+    if (options.type === 'first') {
+      return P.resolve([SAMPLE_REMINDER])
+    } else {
+      return P.resolve([])
+    }
+  })
+
+  sandbox.stub(mockDb, 'account', function () {
+    done(t)
+    return P.resolve({
+      emailVerified: true
+    })
+  })
+
+  require('../../lib/verification-reminders')(mockMailer, mockDb)
+    ._continuousPoll()
+})
+
+
+function setup() {
+  sandbox = sinon.sandbox.create()
+}
+
+function done(t) {
+  sandbox.restore()
+  t.done()
+}