From 62436c0b48a5f55f51166f00757c8c3d47b102d9 Mon Sep 17 00:00:00 2001 From: Vijay Budhram Date: Mon, 26 Feb 2024 14:25:07 -0700 Subject: [PATCH] feat(redis): Testing customs server using redis --- packages/fxa-customs-server/lib/cache.js | 9 +- .../fxa-customs-server/scripts/test-local.sh | 4 + .../test/memcache-helper.js | 90 ++++++++++++------- .../test/remote/memcache_error_tests.js | 29 +++--- .../test/remote/never_blocked.js | 2 +- .../test/remote/too_many_emails.js | 9 +- 6 files changed, 94 insertions(+), 49 deletions(-) diff --git a/packages/fxa-customs-server/lib/cache.js b/packages/fxa-customs-server/lib/cache.js index 99fb40bb7e..1754bc8f42 100644 --- a/packages/fxa-customs-server/lib/cache.js +++ b/packages/fxa-customs-server/lib/cache.js @@ -24,6 +24,9 @@ class Cache { async setAsync(key, value, lifetime) { if (this.useRedis) { + if (lifetime === 0) { + return this.client.redis.set(key, JSON.stringify(value)); + } // Set the value in redis. We use 'EX' to set the expiration time in seconds. return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime); } @@ -33,7 +36,11 @@ class Cache { async getAsync(key) { if (this.useRedis) { const value = await this.client.redis.get(key); - return JSON.parse(value); + try { + return JSON.parse(value); + } catch (err) { + return {}; + } } else { return this.client.getAsync(key); } diff --git a/packages/fxa-customs-server/scripts/test-local.sh b/packages/fxa-customs-server/scripts/test-local.sh index f288297b9e..69450d8ae0 100755 --- a/packages/fxa-customs-server/scripts/test-local.sh +++ b/packages/fxa-customs-server/scripts/test-local.sh @@ -4,4 +4,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +echo 'Testing with Memcache database' tap test/local test/remote test/scripts --no-coverage --jobs=1 + +echo 'Testing with Redis database' +CUSTOMS_REDIS_ENABLED=true tap test/local test/remote test/scripts --no-coverage --jobs=1 diff --git a/packages/fxa-customs-server/test/memcache-helper.js b/packages/fxa-customs-server/test/memcache-helper.js index d9083e4696..859549c267 100644 --- a/packages/fxa-customs-server/test/memcache-helper.js +++ b/packages/fxa-customs-server/test/memcache-helper.js @@ -5,13 +5,21 @@ 'use strict'; var P = require('bluebird'); -var Memcached = require('memcached'); -P.promisifyAll(Memcached.prototype); +const Cache = require('../lib/cache'); var config = { memcache: { address: process.env.MEMCACHE_ADDRESS || 'localhost:11211', }, + redis: { + customs: { + enabled: process.env.CUSTOMS_REDIS_ENABLED === 'true', + host: 'localhost', + password: '', + port: 6379, + prefix: 'customs:', + }, + }, limits: { blockIntervalSeconds: 1, rateLimitIntervalSeconds: 1, @@ -43,14 +51,7 @@ var config = { }, }; -var mc = new Memcached(config.memcache.address, { - timeout: 500, - retries: 1, - retry: 1000, - reconnect: 1000, - idle: 30000, - namespace: 'fxa~', -}); +var mc = new Cache(config); module.exports.mc = mc; @@ -85,22 +86,35 @@ var IpRecord = require('../lib/ip_record')(limits); module.exports.limits = limits; -function blockedEmailCheck(cb) { - setTimeout( - // give memcache time to flush the writes - function () { - mc.get(TEST_EMAIL, function (err, data) { - var er = EmailRecord.parse(data); - mc.end(); - cb(er.shouldBlock()); - }); - } - ); +async function blockedEmailCheck(email, cb) { + if (config.redis.customs.enabled) { + return P.resolve(true).then(async () => { + const result = await mc.getAsync(email); + var er = EmailRecord.parse(result); + cb(er.shouldBlock()); + }); + } + // give memcache time to flush the writes + setTimeout(function () { + mc.client.get(email, function (err, data) { + var er = EmailRecord.parse(data); + mc.client.end(); + cb(er.shouldBlock()); + }); + }); } module.exports.blockedEmailCheck = blockedEmailCheck; function blockedIpCheck(cb) { + if (config.redis.customs.enabled) { + return Promise.resolve(true).then(async () => { + const result = await mc.getAsync(TEST_IP); + var er = IpRecord.parse(result); + cb(er.shouldBlock()); + }); + } + setTimeout( // give memcache time to flush the writes function () { @@ -124,7 +138,7 @@ function badLoginCheck() { var ipEmailRecord = IpEmailRecord.parse(d1); var ipRecord = IpRecord.parse(d2); var emailRecord = EmailRecord.parse(d3); - mc.end(); + mc.client.end(); return { ipEmailRecord: ipEmailRecord, ipRecord: ipRecord, @@ -136,7 +150,17 @@ function badLoginCheck() { module.exports.badLoginCheck = badLoginCheck; function clearEverything(cb) { - mc.itemsAsync() + if (config.redis.customs.enabled) { + return mc.client.redis.flushall(function (err) { + if (err) { + return cb(err); + } + cb(); + }); + } + + mc.client + .itemsAsync() .then(function (result) { var firstServer = result[0]; @@ -148,7 +172,7 @@ function clearEverything(cb) { // get a cachedump for each slabid and slab.number var cachedumps = keys .map(function (stats) { - return mc.cachedumpAsync( + return mc.client.cachedumpAsync( firstServer.server, stats, firstServer[stats].number @@ -167,7 +191,7 @@ function clearEverything(cb) { dump .filter((item) => /^fxa~/.test(item.key)) .map(function (item) { - return mc.delAsync(item.key.replace(/^fxa~/, '')); + return mc.client.delAsync(item.key.replace(/^fxa~/, '')); }) ); }); @@ -176,7 +200,7 @@ function clearEverything(cb) { return P.all(cachedumps); }) .then(function () { - mc.end(); + mc.client.end(); cb(); }) .catch(cb); @@ -191,7 +215,7 @@ function setLimits(settings) { limits[k] = settings[k]; } return limits.push().then(function (s) { - mc.end(); + mc.client.end(); return s; }); } @@ -201,7 +225,7 @@ module.exports.setLimits = setLimits; function setAllowedIPs(ips) { allowedIPs.setAll(ips); return allowedIPs.push().then(function (ips) { - mc.end(); + mc.client.end(); return ips; }); } @@ -211,7 +235,7 @@ module.exports.setAllowedIPs = setAllowedIPs; function setAllowedEmailDomains(domains) { allowedEmailDomains.setAll(domains); return allowedEmailDomains.push().then(function (domains) { - mc.end(); + mc.client.end(); return domains; }); } @@ -221,7 +245,7 @@ module.exports.setAllowedEmailDomains = setAllowedEmailDomains; function setAllowedPhoneNumbers(phoneNumbers) { allowedPhoneNumbers.setAll(phoneNumbers); return allowedPhoneNumbers.push().then((phoneNumbers) => { - mc.end(); + mc.client.end(); return phoneNumbers; }); } @@ -237,9 +261,13 @@ function setRequestChecks(settings) { requestChecks[k] = settings[k]; } return requestChecks.push().then(function (s) { - mc.end(); + mc.client.end(); return s; }); } module.exports.setAllowedEmailDomains = setAllowedEmailDomains; + +if (config.redis.customs.enabled) { + mc.client.end = function () {}; +} diff --git a/packages/fxa-customs-server/test/remote/memcache_error_tests.js b/packages/fxa-customs-server/test/remote/memcache_error_tests.js index 872875f6ce..a5d45dc76e 100644 --- a/packages/fxa-customs-server/test/remote/memcache_error_tests.js +++ b/packages/fxa-customs-server/test/remote/memcache_error_tests.js @@ -11,6 +11,7 @@ var TEST_IP = '192.0.2.1'; const config = require('../../lib/config').getProperties(); config.memcache.address = '128.0.0.1:12131'; +config.redis.customs.port = '6380'; var testServer = new TestServer(config); @@ -31,18 +32,22 @@ var client = restifyClients.createJsonClient({ url: 'http://localhost:' + config.listen.port, }); -test('request with disconnected memcache', function (t) { - client.post( - '/check', - { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' }, - function (err, req, res, obj) { - t.equal(res.statusCode, 200, 'check worked'); - t.equal(obj.block, true, 'request was blocked'); - t.equal(obj.retryAfter, 900, 'retry after'); - t.end(); - } - ); -}); +test( + 'request with disconnected memcache', + { skip: config.redis.customs.enabled }, + function (t) { + client.post( + '/check', + { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' }, + function (err, req, res, obj) { + t.equal(res.statusCode, 200, 'check worked'); + t.equal(obj.block, true, 'request was blocked'); + t.equal(obj.retryAfter, 900, 'retry after'); + t.end(); + } + ); + } +); test('teardown', async function (t) { await testServer.stop(); diff --git a/packages/fxa-customs-server/test/remote/never_blocked.js b/packages/fxa-customs-server/test/remote/never_blocked.js index 88ce92c713..985557f4ef 100644 --- a/packages/fxa-customs-server/test/remote/never_blocked.js +++ b/packages/fxa-customs-server/test/remote/never_blocked.js @@ -65,7 +65,7 @@ test('maximum number of emails', function (t) { t.equal(obj.block, false, 'resending the code'); return new Promise(function (resolve, reject) { - mcHelper.blockedEmailCheck(function (isBlocked) { + mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) { t.equal(isBlocked, false, 'account is still not blocked'); resolve(); }); diff --git a/packages/fxa-customs-server/test/remote/too_many_emails.js b/packages/fxa-customs-server/test/remote/too_many_emails.js index 5b2d083c3f..f8a205dfd9 100644 --- a/packages/fxa-customs-server/test/remote/too_many_emails.js +++ b/packages/fxa-customs-server/test/remote/too_many_emails.js @@ -6,9 +6,10 @@ var restifyClients = require('restify-clients'); var TestServer = require('../test_server'); var Promise = require('bluebird'); var mcHelper = require('../memcache-helper'); +const { randomEmail, randomIp } = require('../utils'); -var TEST_EMAIL = 'test@example.com'; -var TEST_IP = '192.0.2.1'; +var TEST_EMAIL = randomEmail(); +var TEST_IP = randomIp(); const config = require('../../lib/config').getProperties(); config.limits.rateLimitIntervalSeconds = 1; @@ -76,7 +77,7 @@ test('too many sent emails', function (t) { t.equal(obj.block, true, 'operation blocked'); return new Promise(function (resolve, reject) { - mcHelper.blockedEmailCheck(function (isBlocked) { + mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) { t.equal(isBlocked, true, 'account is blocked'); resolve(); }); @@ -90,7 +91,7 @@ test('too many sent emails', function (t) { test('failed logins expire', function (t) { setTimeout(function () { - mcHelper.blockedEmailCheck(function (isBlocked) { + mcHelper.blockedEmailCheck(TEST_EMAIL, function (isBlocked) { t.equal(isBlocked, false, 'account no longer blocked'); t.end(); });