feat(redis): Testing customs server using redis

packages/fxa-customs-server/lib/cache.js

@@ -24,6 +24,9 @@ class Cache {
 
   async setAsync(key, value, lifetime) {
     if (this.useRedis) {
+      if (lifetime === 0) {
+        return this.client.redis.set(key, JSON.stringify(value));
+      }
       // Set the value in redis. We use 'EX' to set the expiration time in seconds.
       return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime);
     }
@@ -33,7 +36,11 @@ class Cache {
   async getAsync(key) {
     if (this.useRedis) {
       const value = await this.client.redis.get(key);
-      return JSON.parse(value);
+      try {
+        return JSON.parse(value);
+      } catch (err) {
+        return {};
+      }
     } else {
       return this.client.getAsync(key);
     }

packages/fxa-customs-server/scripts/test-local.sh

@@ -4,4 +4,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+echo 'Testing with Memcache database'
 tap test/local test/remote test/scripts --no-coverage --jobs=1
+
+echo 'Testing with Redis database'
+CUSTOMS_REDIS_ENABLED=true tap test/local test/remote test/scripts --no-coverage --jobs=1

packages/fxa-customs-server/test/memcache-helper.js

@@ -5,13 +5,21 @@
 'use strict';
 
 var P = require('bluebird');
-var Memcached = require('memcached');
-P.promisifyAll(Memcached.prototype);
+const Cache = require('../lib/cache');
 
 var config = {
   memcache: {
     address: process.env.MEMCACHE_ADDRESS || 'localhost:11211',
   },
+  redis: {
+    customs: {
+      enabled: process.env.CUSTOMS_REDIS_ENABLED === 'true',
+      host: 'localhost',
+      password: '',
+      port: 6379,
+      prefix: 'customs:',
+    },
+  },
   limits: {
     blockIntervalSeconds: 1,
     rateLimitIntervalSeconds: 1,
@@ -43,14 +51,7 @@ var config = {
   },
 };
 
-var mc = new Memcached(config.memcache.address, {
-  timeout: 500,
-  retries: 1,
-  retry: 1000,
-  reconnect: 1000,
-  idle: 30000,
-  namespace: 'fxa~',
-});
+var mc = new Cache(config);
 
 module.exports.mc = mc;
 
@@ -85,22 +86,35 @@ var IpRecord = require('../lib/ip_record')(limits);
 
 module.exports.limits = limits;
 
-function blockedEmailCheck(cb) {
-  setTimeout(
-    // give memcache time to flush the writes
-    function () {
-      mc.get(TEST_EMAIL, function (err, data) {
-        var er = EmailRecord.parse(data);
-        mc.end();
-        cb(er.shouldBlock());
-      });
-    }
-  );
+async function blockedEmailCheck(email, cb) {
+  if (config.redis.customs.enabled) {
+    return P.resolve(true).then(async () => {
+      const result = await mc.getAsync(email);
+      var er = EmailRecord.parse(result);
+      cb(er.shouldBlock());
+    });
+  }
+  // give memcache time to flush the writes
+  setTimeout(function () {
+    mc.client.get(email, function (err, data) {
+      var er = EmailRecord.parse(data);
+      mc.client.end();
+      cb(er.shouldBlock());
+    });
+  });
 }
 
 module.exports.blockedEmailCheck = blockedEmailCheck;
 
 function blockedIpCheck(cb) {
+  if (config.redis.customs.enabled) {
+    return Promise.resolve(true).then(async () => {
+      const result = await mc.getAsync(TEST_IP);
+      var er = IpRecord.parse(result);
+      cb(er.shouldBlock());
+    });
+  }
+
   setTimeout(
     // give memcache time to flush the writes
     function () {
@@ -124,7 +138,7 @@ function badLoginCheck() {
     var ipEmailRecord = IpEmailRecord.parse(d1);
     var ipRecord = IpRecord.parse(d2);
     var emailRecord = EmailRecord.parse(d3);
-    mc.end();
+    mc.client.end();
     return {
       ipEmailRecord: ipEmailRecord,
       ipRecord: ipRecord,
@@ -136,7 +150,17 @@ function badLoginCheck() {
 module.exports.badLoginCheck = badLoginCheck;
 
 function clearEverything(cb) {
-  mc.itemsAsync()
+  if (config.redis.customs.enabled) {
+    return mc.client.redis.flushall(function (err) {
+      if (err) {
+        return cb(err);
+      }
+      cb();
+    });
+  }
+
+  mc.client
+    .itemsAsync()
     .then(function (result) {
       var firstServer = result[0];
 
@@ -148,7 +172,7 @@ function clearEverything(cb) {
       // get a cachedump for each slabid and slab.number
       var cachedumps = keys
         .map(function (stats) {
-          return mc.cachedumpAsync(
+          return mc.client.cachedumpAsync(
             firstServer.server,
             stats,
             firstServer[stats].number
@@ -167,7 +191,7 @@ function clearEverything(cb) {
               dump
                 .filter((item) => /^fxa~/.test(item.key))
                 .map(function (item) {
-                  return mc.delAsync(item.key.replace(/^fxa~/, ''));
+                  return mc.client.delAsync(item.key.replace(/^fxa~/, ''));
                 })
             );
           });
@@ -176,7 +200,7 @@ function clearEverything(cb) {
       return P.all(cachedumps);
     })
     .then(function () {
-      mc.end();
+      mc.client.end();
       cb();
     })
     .catch(cb);
@@ -191,7 +215,7 @@ function setLimits(settings) {
     limits[k] = settings[k];
   }
   return limits.push().then(function (s) {
-    mc.end();
+    mc.client.end();
     return s;
   });
 }
@@ -201,7 +225,7 @@ module.exports.setLimits = setLimits;
 function setAllowedIPs(ips) {
   allowedIPs.setAll(ips);
   return allowedIPs.push().then(function (ips) {
-    mc.end();
+    mc.client.end();
     return ips;
   });
 }
@@ -211,7 +235,7 @@ module.exports.setAllowedIPs = setAllowedIPs;
 function setAllowedEmailDomains(domains) {
   allowedEmailDomains.setAll(domains);
   return allowedEmailDomains.push().then(function (domains) {
-    mc.end();
+    mc.client.end();
     return domains;
   });
 }
@@ -221,7 +245,7 @@ module.exports.setAllowedEmailDomains = setAllowedEmailDomains;
 function setAllowedPhoneNumbers(phoneNumbers) {
   allowedPhoneNumbers.setAll(phoneNumbers);
   return allowedPhoneNumbers.push().then((phoneNumbers) => {
-    mc.end();
+    mc.client.end();
     return phoneNumbers;
   });
 }
@@ -237,9 +261,13 @@ function setRequestChecks(settings) {
     requestChecks[k] = settings[k];
   }
   return requestChecks.push().then(function (s) {
-    mc.end();
+    mc.client.end();
     return s;
   });
 }
 
 module.exports.setAllowedEmailDomains = setAllowedEmailDomains;
+
+if (config.redis.customs.enabled) {
+  mc.client.end = function () {};
+}

packages/fxa-customs-server/test/remote/memcache_error_tests.js

@@ -11,6 +11,7 @@ var TEST_IP = '192.0.2.1';
 
 const config = require('../../lib/config').getProperties();
 config.memcache.address = '128.0.0.1:12131';
+config.redis.customs.port = '6380';
 
 var testServer = new TestServer(config);
 
@@ -31,18 +32,22 @@ var client = restifyClients.createJsonClient({
   url: 'http://localhost:' + config.listen.port,
 });
 
-test('request with disconnected memcache', function (t) {
-  client.post(
-    '/check',
-    { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' },
-    function (err, req, res, obj) {
-      t.equal(res.statusCode, 200, 'check worked');
-      t.equal(obj.block, true, 'request was blocked');
-      t.equal(obj.retryAfter, 900, 'retry after');
-      t.end();
-    }
-  );
-});
+test(
+  'request with disconnected memcache',
+  { skip: config.redis.customs.enabled },
+  function (t) {
+    client.post(
+      '/check',
+      { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' },
+      function (err, req, res, obj) {
+        t.equal(res.statusCode, 200, 'check worked');
+        t.equal(obj.block, true, 'request was blocked');
+        t.equal(obj.retryAfter, 900, 'retry after');
+        t.end();
+      }
+    );
+  }
+);
 
 test('teardown', async function (t) {
   await testServer.stop();

packages/fxa-customs-server/test/remote/never_blocked.js

@@ -65,7 +65,7 @@ test('maximum number of emails', function (t) {
       t.equal(obj.block, false, 'resending the code');
 
       return new Promise(function (resolve, reject) {
-        mcHelper.blockedEmailCheck(function (isBlocked) {
+        mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) {
           t.equal(isBlocked, false, 'account is still not blocked');
           resolve();
         });

packages/fxa-customs-server/test/remote/too_many_emails.js

@@ -6,9 +6,10 @@ var restifyClients = require('restify-clients');
 var TestServer = require('../test_server');
 var Promise = require('bluebird');
 var mcHelper = require('../memcache-helper');
+const { randomEmail, randomIp } = require('../utils');
 
-var TEST_EMAIL = 'test@example.com';
-var TEST_IP = '192.0.2.1';
+var TEST_EMAIL = randomEmail();
+var TEST_IP = randomIp();
 
 const config = require('../../lib/config').getProperties();
 config.limits.rateLimitIntervalSeconds = 1;
@@ -76,7 +77,7 @@ test('too many sent emails', function (t) {
       t.equal(obj.block, true, 'operation blocked');
 
       return new Promise(function (resolve, reject) {
-        mcHelper.blockedEmailCheck(function (isBlocked) {
+        mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) {
           t.equal(isBlocked, true, 'account is blocked');
           resolve();
         });
@@ -90,7 +91,7 @@ test('too many sent emails', function (t) {
 
 test('failed logins expire', function (t) {
   setTimeout(function () {
-    mcHelper.blockedEmailCheck(function (isBlocked) {
+    mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) {
       t.equal(isBlocked, false, 'account no longer blocked');
       t.end();
     });