diff --git a/bin/server.js b/bin/server.js index 5c85cc02..386f7720 100644 --- a/bin/server.js +++ b/bin/server.js @@ -10,7 +10,6 @@ var mailConfig = config.get('mail') var packageJson = require('../package.json') var P = require('bluebird') -var DB = require('../lib/db')() // NOTE: Mailer is also used by fxa-auth-server directly with an old logging interface // the legacy log module provides an interface to convert old logs to new mozlog logging. @@ -18,6 +17,8 @@ var mailerLog = require('../log')('mailer') var legacyMailerLog = require('../legacy_log')(mailerLog) var Mailer = require('../mailer')(legacyMailerLog) +var dbConnect = require('../lib/db_connect')() + P.all( [ require('../translator')(config.get('locales')), @@ -30,18 +31,15 @@ P.all( log.info('config', mailConfig) log.info('templates', Object.keys(templates)) - // fetch and process verification reminders - - DB.connect(config.get(config.get('db').backend)) - .then( - function (db) { - var verificationReminders = require('../lib/verification-reminders')(mailer, db) - verificationReminders.poll() - }, - function (err) { - log.error('db', { err: err }) - } - ) + dbConnect() + .then(function (db) { + // fetch and process verification reminders + var verificationReminders = require('../lib/verification-reminders')(mailer, db) + verificationReminders.poll() + }) + .catch(function (err) { + log.error('server', {err: err}) + }) var api = restify.createServer() api.use(restify.bodyParser()) diff --git a/config.js b/config.js index c4608c31..f32447ab 100644 --- a/config.js +++ b/config.js @@ -28,6 +28,18 @@ var conf = convict({ backend: { default: 'httpdb', env: 'DB_BACKEND' + }, + connectionRetry: { + default: '10 seconds', + env: 'DB_CONNECTION_RETRY', + doc: 'Time in milliseconds to retry a database connection attempt', + format: 'duration' + }, + connectionTimeout: { + default: '5 minutes', + env: 'DB_CONNECTION_TIMEOUT', + doc: 'Timeout in milliseconds after which the mailer will stop trying to connect to the database', + format: 'duration' } }, httpdb: { diff --git a/lib/db_connect.js b/lib/db_connect.js new file mode 100644 index 00000000..8f88bf93 --- /dev/null +++ b/lib/db_connect.js @@ -0,0 +1,51 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var P = require('bluebird') +var config = require('../config') +var log = require('../log')('db') + +var DB = require('./db')() + +var dbConnectionTimeout = config.get('db').connectionTimeout +var dbConnectionRetry = config.get('db').connectionRetry +var dbBackend = config.get('db').backend + +/** + * This modules exports a function that polls and waits for the database connection to become available. + * + * @returns {dbConnect} + */ +module.exports = function () { + function dbConnect() { + function dbConnectPoll() { + return DB.connect(config.get(dbBackend)) + .then( + function (db) { + return db + }, + function (err) { + if (err && err.message && err.message.indexOf('ECONNREFUSED') > -1) { + log.warn('db', {message: 'Failed to connect to database, retrying...'}) + return P.delay(dbConnectionRetry).then(dbConnectPoll) + } else { + log.error('db', {err: err}) + } + } + ) + } + + return dbConnectPoll() + // 'cancellable' means the Promise chain will stop polling after the timeout below. + .cancellable() + .timeout(dbConnectionTimeout) + .catch(function (err) { + // report any errors to the db log and rethrow it to the consumer. + log.error('db', {err: err}) + throw err + }); + } + + return dbConnect +} diff --git a/test/local/db_connect_tests.js b/test/local/db_connect_tests.js new file mode 100644 index 00000000..42f15019 --- /dev/null +++ b/test/local/db_connect_tests.js @@ -0,0 +1,85 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +var P = require('bluebird') +var tap = require('tap') +var test = tap.test +var proxyquire = require('proxyquire') +var dbConnect + +var mockLog = function () { + return { + error: function () {}, + info: function () {}, + warn: function () {} + } +} + +var mockConfig = { + get: function (prop) { + if (prop === 'db') { + return { + connectionRetry: 10, + connectionTimeout: 100 + } + } + } +} + +test( + 'keeps polling until time limit', + function (t) { + dbConnect = proxyquire('../../lib/db_connect', { + '../log': mockLog, + './db': function () { + return { + connect: function() { + return P.reject(new Error('ECONNREFUSED')) + } + } + }, + '../config': mockConfig + })() + + dbConnect() + .catch(function (err) { + t.equal(err.name, 'TimeoutError') + t.equal(err.message, 'operation timed out') + t.end() + }) + } +) + +test( + 'connects to db after polling', + function (t) { + var dbConnectedMs = 30 + var testStart = Date.now() + + dbConnect = proxyquire('../../lib/db_connect', { + '../log': mockLog, + './db': function () { + return { + connect: function() { + if ((Date.now() - testStart) > dbConnectedMs) { + return P.resolve({ + name: 'TestDB' + }) + } else { + return P.reject(new Error('ECONNREFUSED')) + } + + } + } + }, + '../config': mockConfig + })() + + dbConnect() + .then(function (db) { + t.equal(db.name, 'TestDB') + t.end() + }) + } +)