fix(cache): Remove memcache service and references to it

feat(sync): Return estimated sync devices in recovery key exist route
This commit is contained in:
Vijay Budhram 2024-04-15 12:26:25 -04:00
Родитель 4b1d485fc6
Коммит 0e00017b0d
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 9778545895B2532B
64 изменённых файлов: 116 добавлений и 759 удалений

Просмотреть файл

@ -119,7 +119,6 @@ executors:
- image: cimg/mysql:8.0
command: --default-authentication-plugin=mysql_native_password
- image: jdlk7/firestore-emulator
- image: memcached
- image: redis
- image: ghcr.io/aertje/cloud-tasks-emulator:1.2.0
command: -queue "projects/test/locations/test/queues/delete-accounts-queue"
@ -141,7 +140,6 @@ executors:
docker:
- image: mozilla/fxa-circleci:ci-functional-test-runner-v3
- image: redis
- image: memcached
- image: pafortin/goaws
- image: cimg/mysql:8.0
command: --default-authentication-plugin=mysql_native_password

Просмотреть файл

@ -31,7 +31,6 @@ services:
- DB=mysql
- CONFIG_FILES=config/secrets.json
- MYSQL_HOST=mysql
- MEMCACHE_METRICS_CONTEXT_ADDRESS=memcached:11211
- HTTPDB_URL=http://authdb:8000
- EMAIL_SERVICE_HOST=email
- REDIS_HOST=redis
@ -80,7 +79,6 @@ services:
- "1111:1111"
depends_on:
- mysql
- memcached
profile-worker:
image: fxa-profile-server:build
entrypoint: /bin/bash -c
@ -99,7 +97,6 @@ services:
- "1113:1113"
depends_on:
- mysql
- memcached
profile-static:
image: fxa-profile-server:build
entrypoint: /bin/bash -c
@ -116,7 +113,6 @@ services:
- "1112:1112"
depends_on:
- mysql
- memcached
redis:
image: redis
mysql:
@ -125,8 +121,6 @@ services:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_ROOT_HOST=%
- MYSQL_DATABASE=fxa
memcached:
image: memcached
maildev:
image: djfarrelly/maildev
ports:

Просмотреть файл

@ -17,13 +17,6 @@ module.exports = {
min_uptime: '2m',
kill_timeout: 20000,
},
{
name: 'memcache',
script: '_scripts/memcached.sh',
max_restarts: '1',
min_uptime: '2m',
kill_timeout: 20000,
},
{
name: 'sns',
script: '_scripts/goaws.sh',

Просмотреть файл

@ -3,7 +3,6 @@
PORTS=(
3306 # MySQL server
6379 # redis
11211 # memcached
4100 # Fake SQS/SNS
8085 # google-pubsub-emulator
9090 # google-firestore-emulator

Просмотреть файл

@ -1,3 +0,0 @@
#!/bin/bash -ex
docker run --rm --net fxa -p 11211:11211 --name memcache memcached

Просмотреть файл

@ -1,12 +0,0 @@
---
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: fxa-customs-cache
description: Caches rate limits and IP reuptation scores.
tags:
- memcached
spec:
type: cache
owner: fxa-devs
system: mozilla-accounts

Просмотреть файл

@ -22,3 +22,15 @@ spec:
type: cache
owner: fxa-devs
system: mozilla-accounts
---
apiVersion: backstage.io/v1alpha1
kind: Resource
metadata:
name: fxa-customs-cache
description: Caches rate limits and IP reputation scores.
tags:
- redis
spec:
type: cache
owner: fxa-devs
system: mozilla-accounts

Просмотреть файл

@ -10,7 +10,6 @@ spec:
- ./backstage/mozilla-accounts.system.yaml
- ./backstage/mysql.resources.yaml
- ./backstage/redis.resources.yaml
- ./backstage/memcached.resources.yaml
- ./packages/123done/backstage.yaml
- ./packages/browserid-verifier/backstage.yaml
- ./packages/fxa-admin-panel/backstage.yaml

Просмотреть файл

@ -70,7 +70,7 @@ We also use [Chai](https://www.chaijs.com/) for making assertions in tests. As o
### Testing with non-local databases
Executing tests using remote databases (MySQL, Redis, Memcached) is possible by specifying (and exporting) the following environment variables:
Executing tests using remote databases (MySQL, Redis) is possible by specifying (and exporting) the following environment variables:
- MySQL:
- MYSQL_HOST
@ -80,8 +80,7 @@ Executing tests using remote databases (MySQL, Redis, Memcached) is possible by
- REDIS_HOST
- ACCESS_TOKEN_REDIS_HOST
- REFRESH_TOKEN_REDIS_HOST
- Memcached:
- MEMCACHE_METRICS_CONTEXT_ADDRESS
- METRICS_REDIS_HOST
This also allows to use temporary throw-away Docker containers to provide these.

Просмотреть файл

@ -160,7 +160,6 @@
"@types/jws": "^3.2.3",
"@types/lodash": "^4",
"@types/luxon": "^3",
"@types/memcached": "^2.2.6",
"@types/mocha": "^10.0.6",
"@types/nock": "^11.1.0",
"@types/node": "^20.11.1",

Просмотреть файл

@ -658,7 +658,7 @@ describe('metricsContext', () => {
.catch((err) => assert.fail(err));
});
it('metricsContext.clear with memcached error', () => {
it('metricsContext.clear with cache error', () => {
const token = {
uid: Array(64).fill('7').join(''),
id: 'wibble',
@ -677,7 +677,7 @@ describe('metricsContext', () => {
assert.equal(
err.message,
'blee',
'metricsContext.clear should have rejected with memcached error'
'metricsContext.clear should have rejected with cache error'
);
assert.equal(cache.del.callCount, 1, 'cache.del was called once');
});

Просмотреть файл

@ -54,11 +54,6 @@ const makeRoutes = function (options = {}, requireMocks = {}) {
config.oauth = config.oauth || {};
config.verifierVersion = config.verifierVersion || 0;
config.smtp = config.smtp || {};
config.memcached = config.memcached || {
address: 'none',
idle: 500,
lifetime: 30,
};
config.lastAccessTimeUpdates = {};
config.signinConfirmation = config.signinConfirmation || {};
config.signinUnblock = config.signinUnblock || {};

Просмотреть файл

@ -23,11 +23,6 @@ const mockAuthorizedClients = {
function makeRoutes(options = {}) {
const config = options.config || {};
config.smtp = config.smtp || {};
config.memcached = config.memcached || {
address: 'localhost:1121',
idle: 500,
lifetime: 30,
};
config.i18n = {
supportedLanguages: ['en', 'fr'],
defaultLanguage: 'en',
@ -378,9 +373,9 @@ describe('/account/attached_clients', () => {
it('filters out idle devices', async () => {
const now = Date.now();
const FIVE_DAYS_AGO = now - (1000 * 60 * 60 * 24 * 5);
const ONE_DAY_AGO = now - (1000 * 60 * 60 * 24);
const FIVE_DAYS_AGO = now - 1000 * 60 * 60 * 24 * 5;
const ONE_DAY_AGO = now - 1000 * 60 * 60 * 24;
request.query.filterIdleDevicesTimestamp = ONE_DAY_AGO; // Filter for devices active in the last day
const DEVICES = [
{
@ -402,7 +397,7 @@ describe('/account/attached_clients', () => {
createdAt: now,
lastAccessTime: now,
location: { country: 'Germany' },
}
},
];
const OAUTH_CLIENTS = [];

Просмотреть файл

@ -21,11 +21,6 @@ function makeRoutes(options = {}, requireMocks) {
const config = options.config || {};
config.oauth = config.oauth || {};
config.smtp = config.smtp || {};
config.memcached = config.memcached || {
address: 'localhost:1121',
idle: 500,
lifetime: 30,
};
config.i18n = {
supportedLanguages: ['en', 'fr'],
defaultLanguage: 'en',

Просмотреть файл

@ -153,11 +153,6 @@ const makeRoutes = function (options = {}, requireMocks) {
const config = options.config || {};
config.verifierVersion = config.verifierVersion || 0;
config.smtp = config.smtp || {};
config.memcached = config.memcached || {
address: 'none',
idle: 500,
lifetime: 30,
};
config.i18n = {
supportedLanguages: ['en'],
defaultLanguage: 'en',

Просмотреть файл

@ -1,9 +0,0 @@
{
"processes": [
"bin/customs_server.js"
],
"packages": [
"memcached"
],
"ssl": "disable"
}

Просмотреть файл

@ -63,7 +63,7 @@ We currently have the following policies in place:
- manual blocking of an account (see `/blockEmail` API call)
- manual blocking of an IP address (see `/blockIp` API call)
The data that these policies are based on is stored in a memcache instance (keyed by `email`, `ip` or `ip + email` depending on the policy) and the code that implements them is split across these three files:
The data that these policies are based on is stored in a redis cache instance (keyed by `email`, `ip` or `ip + email` depending on the policy) and the code that implements them is split across these three files:
- `email_record.js` handles blocking and rate-limiting based only on the email address
- `ip_email_record.js` handles rate-limiting based on both the email and IP address of the request

Просмотреть файл

@ -1,48 +1,28 @@
const Memcached = require('memcached');
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
const { RedisShared } = require('fxa-shared/db/redis');
const P = require('bluebird');
P.promisifyAll(Memcached.prototype);
class Cache {
constructor(config) {
const customsRedisConfig = config.redis.customs;
this.useRedis = customsRedisConfig.enabled;
if (this.useRedis) {
this.client = new RedisShared(config.redis.customs);
} else {
this.client = new Memcached(config.memcache.address, {
timeout: 500,
retries: 1,
retry: 1000,
reconnect: 1000,
idle: 30000,
namespace: 'fxa~',
});
}
this.client = new RedisShared(config.redis.customs);
}
async setAsync(key, value, lifetime) {
if (this.useRedis) {
if (lifetime === 0) {
return this.client.redis.set(key, JSON.stringify(value));
}
// Set the value in redis. We use 'EX' to set the expiration time in seconds.
return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime);
if (lifetime === 0) {
return this.client.redis.set(key, JSON.stringify(value));
}
return this.client.setAsync(key, value, lifetime);
// Set the value in redis. We use 'EX' to set the expiration time in seconds.
return this.client.redis.set(key, JSON.stringify(value), 'EX', lifetime);
}
async getAsync(key) {
if (this.useRedis) {
const value = await this.client.redis.get(key);
try {
return JSON.parse(value);
} catch (err) {
return {};
}
} else {
return this.client.getAsync(key);
const value = await this.client.redis.get(key);
try {
return JSON.parse(value);
} catch (err) {
return {};
}
}
}

Просмотреть файл

@ -174,14 +174,9 @@ module.exports = function (fs, path, url, convict) {
env: 'MAX_ACCOUNT_ACCESS',
},
},
memcache: {
address: {
doc: 'Hostname/IP:Port of the memcache server',
default: 'localhost:11211',
env: 'MEMCACHE_ADDRESS',
},
cache: {
recordLifetimeSeconds: {
doc: 'Memcache record expiry',
doc: 'Record expiry',
default: 900,
format: 'nat',
env: 'RECORD_LIFETIME_SECONDS',

Просмотреть файл

@ -1,7 +1,7 @@
{
"redis": {
"customs": {
"enabled": false
"enabled": true
}
}
}

Просмотреть файл

@ -9,14 +9,14 @@ class Record {
this.hits = object.hits || []; // timestamps when last hit occurred
Object.defineProperty(this, 'limits', {
// limits is not saved to memcached
// limits is not saved to cache
enumerable: false,
get() {
return config.limits;
},
});
Object.defineProperty(this, 'actions', {
// actions is not saved to memcached
// actions is not saved to cache
enumerable: false,
get() {
return config.actions;

Просмотреть файл

@ -74,7 +74,7 @@ module.exports = async function createServer(config, log) {
mc,
reputationService,
limits,
config.memcache.recordLifetimeSeconds,
config.cache.recordLifetimeSeconds,
statsd
);
@ -123,7 +123,7 @@ module.exports = async function createServer(config, log) {
await configureSentry(api, config, log);
function logError(err) {
log.error({ op: 'memcachedError', err: err });
log.error({ op: 'cacheError', err: err });
throw err;
}
function normalizePath(path) {

Просмотреть файл

@ -30,7 +30,7 @@ module.exports = (config, Settings, log) => {
op: 'allowedEmailDomains.validate.invalid',
data: domains,
});
throw new Settings.Missing('invalid allowedEmailDomains from memcache');
throw new Settings.Missing('invalid allowedEmailDomains from cache');
}
return domains;
}

Просмотреть файл

@ -28,7 +28,7 @@ module.exports = (config, Settings, log) => {
validate(ips) {
if (!Array.isArray(ips)) {
log.error({ op: 'allowedIPs.validate.invalid', data: ips });
throw new Settings.Missing('invalid allowedIPs from memcache');
throw new Settings.Missing('invalid allowedIPs from cache');
}
return ips.filter(function (ip) {
var is = net.isIPv4(ip);

Просмотреть файл

@ -26,7 +26,7 @@ module.exports = (config, Settings, log) => {
op: 'allowedPhoneNumbers.validate.invalid',
data: phoneNumbers,
});
throw new Settings.Missing('invalid allowedPhoneNumbers from memcache');
throw new Settings.Missing('invalid allowedPhoneNumbers from cache');
}
return phoneNumbers;
}

Просмотреть файл

@ -53,7 +53,7 @@ module.exports = (config, Settings, log) => {
validate(settings) {
if (typeof settings !== 'object') {
log.error({ op: 'limits.validate.invalid', data: settings });
throw new Settings.Missing('invalid limits from memcache');
throw new Settings.Missing('invalid limits from cache');
}
var keys = Object.keys(config.limits);
for (var i = 0; i < keys.length; i++) {

Просмотреть файл

@ -21,8 +21,8 @@ module.exports = (config, Settings, log) => {
if (!this.flowIdExemptUserAgentREs) {
this.flowIdExemptUserAgentCompiledREs = [];
} else {
this.flowIdExemptUserAgentCompiledREs = this.flowIdExemptUserAgentREs.map(
function (re) {
this.flowIdExemptUserAgentCompiledREs =
this.flowIdExemptUserAgentREs.map(function (re) {
// Log the regex at startup to check that we have correct values.
const regex = new RegExp(re);
log.info({
@ -30,8 +30,7 @@ module.exports = (config, Settings, log) => {
regex: String(regex),
});
return regex;
}
);
});
}
return this;
}
@ -41,7 +40,7 @@ module.exports = (config, Settings, log) => {
validate(settings) {
if (typeof settings !== 'object') {
log.error({ op: 'requestChecks.validate.invalid', data: settings });
throw new Settings.Missing('invalid requestChecks from memcache');
throw new Settings.Missing('invalid requestChecks from cache');
}
const keys = Object.keys(config.requestChecks);
for (let i = 0; i < keys.length; i++) {

Просмотреть файл

@ -11,12 +11,12 @@ const POLL_INTERVAL = Symbol();
module.exports = (config, mc, log) => {
// A sentinel error for signaling that the result was invalid/missing,
// and we should try to push our own representation to memcached.
// and we should try to push our own representation to cache.
class Missing extends Error {}
// An abstract class that stores options in memcached.
// An abstract class that stores options in cache.
//
// Others extend this class to hotload options from memcached.
// Others extend this class to hotload options from cache.
class Settings {
constructor(key) {
assert(typeof key === 'string');
@ -61,7 +61,7 @@ module.exports = (config, mc, log) => {
return result.then(
(value) => this.setAll(value),
(err) => {
log.error({ op: this[KEY] + '.refresh', err: err.message});
log.error({ op: this[KEY] + '.refresh', err: err.message });
throw err;
}
);

Просмотреть файл

@ -22,8 +22,7 @@
"lint": "eslint .",
"test": "scripts/test-local.sh",
"test-unit": "yarn make-artifacts-dir && tap test/local --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-unit.xml",
"test-integration": "yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration.xml",
"test-integration-redis": "CUSTOMS_REDIS_ENABLED=true yarn make-artifacts-dir && tap test/remote test/scripts --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration-redis.xml",
"test-integration": "yarn make-artifacts-dir && tap test/remote --jobs=1 | tap-xunit > ../../artifacts/tests/$npm_package_name/tap-integration.xml",
"format": "prettier --write --config ../../_dev/.prettierrc '**'",
"make-artifacts-dir": "mkdir -p ../../artifacts/tests/$npm_package_name"
},
@ -44,8 +43,7 @@
"ip": "^2.0.1",
"ip-reputation-js-client": "^6.0.4",
"lodash.isequal": "4.5.0",
"lodash.merge": "4.6.2",
"memcached": "^2.2.2"
"lodash.merge": "4.6.2"
},
"devDependencies": {
"@types/dedent": "^0",

Просмотреть файл

@ -1,78 +0,0 @@
#!/usr/bin/env node
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*/
Block an ip address for a number of seconds.
Example:
$ ./block-ip.js localhost:11211 127.0.0.1 600
/*/
/* eslint-disable no-console */
var net = require('net');
var Memcached = require('memcached');
const P = require('bluebird');
P.promisifyAll(Memcached.prototype);
if (process.argv.length < 5) {
var usage = 'Usage: block-ip.js <memcacheHost:port> <ip> <seconds>';
console.error(usage);
process.exit(1);
}
if (!/.+:\d+/.test(process.argv[2])) {
var x = "use a host:port like 'localhost:11211'";
console.error(x);
process.exit(1);
}
if (!net.isIPv4(process.argv[3])) {
console.error('second argument must be an IPv4');
process.exit(1);
}
var lifetime = parseInt(process.argv[4], 10);
if (!lifetime || lifetime < 0) {
console.error('third argument must be a positive integer of seconds');
process.exit(1);
}
var mc = new Memcached(process.argv[2], { namespace: 'fxa~' });
var IpRecord = require('../lib/ip_record')({
blockIntervalMs: lifetime * 1000,
ipRateLimitIntervalMs: lifetime * 1000,
ipRateLimitBanDurationMs: lifetime * 1000,
});
const { fetchRecord, setRecord } = require('../lib/records')(
mc,
{},
{},
lifetime
);
async function main() {
const ip = process.argv[3];
try {
const ipRecord = await fetchRecord(ip, IpRecord.parse);
ipRecord.block();
await setRecord(ipRecord);
console.log('successfully blocked', ipRecord, lifetime);
} catch (err) {
console.error(err);
return process.exit(1);
}
mc.end();
}
main();

Просмотреть файл

@ -1,91 +0,0 @@
#!/usr/bin/env node
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*/
Use in combination with config-set.js to update customs server settings
in memcache. config-list.js reads the settings from the memcache server
specified by the first argument and writes json to stdout.
Example:
$ config-list.js localhost:11211 > settings.json
$ vi settings.json
$ cat settings.json | config-set.js localhost:11211
/*/
/* eslint-disable no-console */
var Memcached = require('memcached');
var P = require('bluebird');
P.promisifyAll(Memcached.prototype);
if (process.argv.length < 3) {
var usage = 'Usage: config-list.js <memcacheHost:port>';
console.error(usage);
process.exit(1);
}
if (!/.+:\d+/.test(process.argv[2])) {
var x = "use a host:port like 'localhost:11211'";
console.error(x);
process.exit(1);
}
var mc = new Memcached(process.argv[2], { namespace: 'fxa~' });
var output = {
limits: {},
allowedIPs: [],
allowedEmailDomains: [],
allowedPhoneNumbers: [],
requestChecks: {},
};
mc.getAsync('limits')
.then(function (data) {
if (!data) {
console.error('no limits set');
} else {
output.limits = data;
}
return mc.getAsync('allowedIPs');
})
.then(function (data) {
if (!data) {
console.error('no allowedIPs set');
} else {
output.allowedIPs = data;
}
return mc.getAsync('allowedEmailDomains');
})
.then(function (data) {
if (!data) {
console.error('no allowedEmailDomains set');
} else {
output.allowedEmailDomains = data;
}
return mc.getAsync('allowedPhoneNumbers');
})
.then(function (data) {
if (!data) {
console.error('no allowedPhoneNumbers set');
} else {
output.allowedPhoneNumbers = data;
}
return mc.getAsync('requestChecks');
})
.then(function (data) {
if (!data) {
console.error('no requestChecks set');
} else {
output.requestChecks = data;
}
})
.then(function () {
mc.end();
console.log(JSON.stringify(output, null, 2));
});

Просмотреть файл

@ -1,113 +0,0 @@
#!/usr/bin/env node
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*/
Use in combination with config-list.js to update customs server settings
in memcache. config-set.js reads json from stdin and writes it to the
memcache server specified by the first argument.
Example:
$ config-list.js localhost:11211 > settings.json
$ vi settings.json
$ cat settings.json | config-set.js localhost:11211
/*/
/* eslint-disable no-console */
var Memcached = require('memcached');
var P = require('bluebird');
var BL = require('bl');
var merge = require('lodash.merge');
var isEqual = require('lodash.isequal');
P.promisifyAll(Memcached.prototype);
if (process.argv.length < 3) {
var usage = 'Usage: cat settings.json | config-set.js <memcacheHost:port>';
console.error(usage);
process.exit(1);
}
if (!/.+:\d+/.test(process.argv[2])) {
var x = "use a host:port like 'localhost:11211'";
console.error(x);
process.exit(1);
}
function writeMergedSettings(mc, key, newSettings) {
return mc.getAsync(key).then(function (settings) {
if (typeof newSettings !== 'object' || Array.isArray(newSettings)) {
throw new Error('tried to merge non-Object-typed settings value');
}
settings = settings || {};
merge(settings, newSettings);
return mc.setAsync(key, settings, 0);
});
}
function clobberSettings(mc, key, newSettings) {
return mc.getAsync(key).then(function (settings) {
if (!Array.isArray(newSettings)) {
throw new Error('tried to clobber non-Array-typed settings value');
}
if (settings && !isEqual(settings, newSettings)) {
console.warn('Clobbering existing settings for "' + key + '":');
console.warn('Old:', settings);
console.warn('New:', newSettings);
}
return mc.setAsync(key, newSettings, 0);
});
}
var b = process.stdin.pipe(new BL());
process.stdin.on('end', function () {
var input = null;
try {
input = JSON.parse(b.toString());
} catch (e) {
return console.error('Input is not valid JSON');
}
var mc = new Memcached(process.argv[2], { namespace: 'fxa~' });
var actions = [];
if (input.limits) {
actions.push(writeMergedSettings(mc, 'limits', input.limits));
}
if (input.allowedIPs) {
// It's an array, we can't sensibly merge it.
actions.push(clobberSettings(mc, 'allowedIPs', input.allowedIPs));
}
if (input.allowedEmailDomains) {
// It's an array, we can't sensibly merge it.
actions.push(
clobberSettings(mc, 'allowedEmailDomains', input.allowedEmailDomains)
);
}
if (input.allowedPhoneNumbers) {
// It's an array, we can't sensibly merge it.
actions.push(
clobberSettings(mc, 'allowedPhoneNumbers', input.allowedPhoneNumbers)
);
}
if (input.requestChecks) {
actions.push(writeMergedSettings(mc, 'requestChecks', input.requestChecks));
}
P.all(actions)
.catch(function (err) {
console.error('update failed');
console.error(err);
})
.then(function () {
mc.end();
});
});

Просмотреть файл

@ -1,66 +0,0 @@
#!/usr/bin/env node
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/*/
Look up customs info for an ip, email, or ip+email.
Examples:
$ ./customs-info.js localhost:11211 127.0.0.1
$ ./customs-info.js localhost:11211 test@example.com
$ ./customs-info.js localhost:11211 127.0.0.1test@example.com
/*/
/* eslint-disable no-console */
var Memcached = require('memcached');
if (process.argv.length < 4) {
var usage = 'Usage: customs-info.js <memcacheHost:port> <key>';
console.error(usage);
process.exit(1);
}
if (!/.+:\d+/.test(process.argv[2])) {
var x = "use a host:port like 'localhost:11211'";
console.error(x);
process.exit(1);
}
var mc = new Memcached(process.argv[2], { namespace: 'fxa~' });
mc.get(process.argv[3], function (err, data) {
if (err) {
console.error(err);
process.exit(1);
}
if (!data) {
mc.end();
console.error('no record');
process.exit(1);
}
if (data.bk) {
console.error('blocked at %s', new Date(data.bk));
}
if (data.lf && data.lf.length > 0) {
console.error('login failures: %d', data.lf.length);
}
if (data.as && data.as.length > 0) {
console.error('account status: %d', data.as.length);
}
if (data.rl) {
console.error('rate limited at %s', new Date(data.rl));
}
if (data.xs && data.xs.length > 0) {
console.error('emails sent %d', data.xs.length);
}
if (data.pr) {
console.error('password reset at %s', new Date(data.pr));
}
console.error('raw data:');
console.log(JSON.stringify(data, null, 2));
mc.end();
});

Просмотреть файл

@ -4,8 +4,4 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
echo 'Testing with Memcache database'
tap test/local test/remote test/scripts --no-coverage --jobs=1
echo 'Testing with Redis database'
CUSTOMS_REDIS_ENABLED=true tap test/local test/remote test/scripts --no-coverage --jobs=1
tap test/local test/remote --no-coverage --jobs=1

Просмотреть файл

@ -8,12 +8,9 @@ var P = require('bluebird');
const Cache = require('../lib/cache');
var config = {
memcache: {
address: process.env.MEMCACHE_ADDRESS || 'localhost:11211',
},
redis: {
customs: {
enabled: process.env.CUSTOMS_REDIS_ENABLED === 'true',
enabled: 'true',
host: 'localhost',
password: '',
port: 6379,
@ -87,44 +84,21 @@ var IpRecord = require('../lib/ip_record')(limits);
module.exports.limits = limits;
async function blockedEmailCheck(email, cb) {
if (config.redis.customs.enabled) {
return P.resolve(true).then(async () => {
const result = await mc.getAsync(email);
var er = EmailRecord.parse(result);
cb(er.shouldBlock());
});
}
// give memcache time to flush the writes
setTimeout(function () {
mc.client.get(email, function (err, data) {
var er = EmailRecord.parse(data);
mc.client.end();
cb(er.shouldBlock());
});
return P.resolve(true).then(async () => {
const result = await mc.getAsync(email);
var er = EmailRecord.parse(result);
cb(er.shouldBlock());
});
}
module.exports.blockedEmailCheck = blockedEmailCheck;
function blockedIpCheck(cb) {
if (config.redis.customs.enabled) {
return Promise.resolve(true).then(async () => {
const result = await mc.getAsync(TEST_IP);
var er = IpRecord.parse(result);
cb(er.shouldBlock());
});
}
setTimeout(
// give memcache time to flush the writes
function () {
mc.get(TEST_IP, function (err, data) {
var ir = IpRecord.parse(data);
mc.end();
cb(ir.shouldBlock());
});
}
);
return Promise.resolve(true).then(async () => {
const result = await mc.getAsync(TEST_IP);
var er = IpRecord.parse(result);
cb(er.shouldBlock());
});
}
module.exports.blockedIpCheck = blockedIpCheck;
@ -150,60 +124,12 @@ function badLoginCheck() {
module.exports.badLoginCheck = badLoginCheck;
function clearEverything(cb) {
if (config.redis.customs.enabled) {
return mc.client.redis.flushall(function (err) {
if (err) {
return cb(err);
}
cb();
});
}
mc.client
.itemsAsync()
.then(function (result) {
var firstServer = result[0];
// we don't need the "server" key, but the other indicate the slab id's
var keys = Object.keys(firstServer)
.filter((k) => /[0-9]+/i.test(k))
.map(Number);
// get a cachedump for each slabid and slab.number
var cachedumps = keys
.map(function (stats) {
return mc.client.cachedumpAsync(
firstServer.server,
stats,
firstServer[stats].number
);
})
.map(function (dumpPromise) {
return dumpPromise.then(function (dump) {
if (!dump) {
return;
}
// when one key is return as an object pretend it's an array
if (dump.key && !dump.length) {
dump = [dump];
}
return P.all(
dump
.filter((item) => /^fxa~/.test(item.key))
.map(function (item) {
return mc.client.delAsync(item.key.replace(/^fxa~/, ''));
})
);
});
});
return P.all(cachedumps);
})
.then(function () {
mc.client.end();
cb();
})
.catch(cb);
return mc.client.redis.flushall(function (err) {
if (err) {
return cb(err);
}
cb();
});
}
module.exports.clearEverything = clearEverything;
@ -268,6 +194,4 @@ function setRequestChecks(settings) {
module.exports.setAllowedEmailDomains = setAllowedEmailDomains;
if (config.redis.customs.enabled) {
mc.client.end = function () {};
}
mc.client.end = function () {};

Просмотреть файл

@ -19,26 +19,16 @@ services:
volumes:
- ${PWD}/iprepd.yaml:/app/iprepd.yaml
memcached:
container_name: customs_memcached
image: memcached:1.4
ports:
- '11211:11211'
customs:
container_name: customs_app
image: fxa-customs-server:test
build:
context: ../../..
dockerfile: Dockerfile
environment:
MEMCACHE_ADDRESS: memcached:11211
links:
- iprepd:iprepd.host
- memcached
depends_on:
- iprepd
- memcached
ports:
- '7000:7000'
command: node_modules/.bin/tap test/integration/reputation/iprepd_tests.js

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var TestServer = require('../../test_server');
var Promise = require('bluebird');
var restifyClients = require('restify-clients');
var mcHelper = require('../../memcache-helper');
var mcHelper = require('../../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -74,7 +74,7 @@ test('refresh pushOnMissing works on Missing error', (t) => {
});
test('refresh pushOnMissing returns other Errors', (t) => {
const mcError = new Error('memcached error');
const mcError = new Error('cache error');
mc.getAsync = () => P.reject(mcError);
mc.setAsync = (key, val) => {
return P.reject(new Error('setAsync should not have been called'));

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var TestServer = require('../test_server');
var Promise = require('bluebird');
var restifyClients = require('restify-clients');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const testUtils = require('../utils');
var TEST_EMAIL = 'test@example.com';

Просмотреть файл

@ -6,7 +6,7 @@ var TestServer = require('../test_server');
var ReputationServer = require('../test_reputation_server');
var Promise = require('bluebird');
var restifyClients = require('restify-clients');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -3,7 +3,7 @@
'use strict';
const mcHelper = require('../memcache-helper');
const mcHelper = require('../cache-helper');
const Promise = require('bluebird');
const restifyClients = require('restify-clients');
const test = require('tap').test;
@ -60,17 +60,16 @@ Promise.promisifyAll(client, { multiArgs: true });
});
test('missing action', function (t) {
client.post('/check', { email: TEST_EMAIL, ip: TEST_IP }, function (
err,
req,
res,
obj
) {
t.equal(res.statusCode, 400, 'bad request returns a 400');
t.type(obj.code, 'string', 'bad request returns an error code');
t.type(obj.message, 'string', 'bad request returns an error message');
t.end();
});
client.post(
'/check',
{ email: TEST_EMAIL, ip: TEST_IP },
function (err, req, res, obj) {
t.equal(res.statusCode, 400, 'bad request returns a 400');
t.type(obj.code, 'string', 'bad request returns an error code');
t.type(obj.message, 'string', 'bad request returns an error message');
t.end();
}
);
});
test('missing email, ip and action', function (t) {

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var TestServer = require('../test_server');
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const {
randomIp,
randomEmail,
@ -53,7 +53,7 @@ test('change limits', function (t) {
})
.then(function (settings) {
t.equal(x + 1, settings.blockIntervalSeconds, 'helper sees the change');
// Wait for background polling to detect the new value in memcache
// Wait for background polling to detect the new value in cache
return Promise.delay(1010);
})
.then(function () {
@ -83,7 +83,7 @@ test('change nested limits', function (t) {
})
.then(function (settings) {
t.equal(x + 1, settings.maxChecksPerUid, 'helper sees the change');
// Wait for background polling to detect the new value in memcache
// Wait for background polling to detect the new value in cache
return Promise.delay(1010);
})
.then(function () {
@ -110,7 +110,7 @@ test('change allowedIPs', function (t) {
})
.then(function (ips) {
t.deepEqual(x, ips, 'helper sees the change');
// Wait for background polling to detect the new value in memcache
// Wait for background polling to detect the new value in cache
return Promise.delay(1010);
})
.then(function () {
@ -137,7 +137,7 @@ test('change allowedEmailDomains', function (t) {
})
.then(function (ips) {
t.deepEqual(x, ips, 'helper sees the change');
// Wait for background polling to detect the new value in memcache
// Wait for background polling to detect the new value in cache
return Promise.delay(1010);
})
.then(function () {
@ -168,7 +168,7 @@ test('change allowedPhoneNumbers', function (t) {
})
.then(function (phoneNumbers) {
t.deepEqual(allowedPhoneNumbers, phoneNumbers, 'helper sees the change');
// Wait for background polling to detect the new value in memcache
// Wait for background polling to detect the new value in cache
return Promise.delay(1010);
})
.then(function () {

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var Promise = require('bluebird');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -4,7 +4,7 @@
var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail, randomIp } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var clients = require('restify-clients');
var Promise = require('bluebird');
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -1,55 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* http://creativecommons.org/publicdomain/zero/1.0/ */
var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';
const config = require('../../lib/config').getProperties();
config.memcache.address = '128.0.0.1:12131';
config.redis.customs.port = '6380';
var testServer = new TestServer(config);
test('startup', async function (t) {
await testServer.start();
t.type(testServer.server, 'object', 'test server was started');
t.end();
});
test('clear everything', function (t) {
mcHelper.clearEverything(function (err) {
t.notOk(err, 'no errors were returned');
t.end();
});
});
var client = restifyClients.createJsonClient({
url: 'http://localhost:' + config.listen.port,
});
test(
'request with disconnected memcache',
{ skip: config.redis.customs.enabled },
function (t) {
client.post(
'/check',
{ email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' },
function (err, req, res, obj) {
t.equal(res.statusCode, 200, 'check worked');
t.equal(obj.block, true, 'request was blocked');
t.equal(obj.retryAfter, 900, 'retry after');
t.end();
}
);
}
);
test('teardown', async function (t) {
await testServer.stop();
t.end();
});

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var Promise = require('bluebird');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var Promise = require('bluebird');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail, randomIp } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -4,7 +4,7 @@
var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -7,7 +7,7 @@ var TestServer = require('../test_server');
var ReputationServer = require('../test_reputation_server');
var Promise = require('bluebird');
var restifyClients = require('restify-clients');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const testUtils = require('../utils');
var TEST_EMAIL = 'test@example.com';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var TestServer = require('../test_server');
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_IP = '192.0.2.1';
var ACCOUNT_STATUS_CHECK = 'accountStatusCheck';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var TestServer = require('../test_server');
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_IP = '192.0.2.1';
var TEST_UID = 'abc123';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var Promise = require('bluebird');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_EMAIL = 'test@example.com';
var TEST_IP = '192.0.2.1';

Просмотреть файл

@ -5,7 +5,7 @@ var test = require('tap').test;
var restifyClients = require('restify-clients');
var TestServer = require('../test_server');
var Promise = require('bluebird');
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
const { randomEmail, randomIp } = require('../utils');
var TEST_EMAIL = randomEmail();

Просмотреть файл

@ -10,7 +10,7 @@ var test = require('tap').test;
var TestServer = require('../test_server');
var Promise = require('bluebird');
var restifyClients = Promise.promisifyAll(require('restify-clients'));
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var TEST_IP = '192.0.2.1';
var ACCOUNT_LOGIN = 'accountLogin';

Просмотреть файл

@ -3,7 +3,7 @@
'use strict';
const memcached = require('../memcache-helper');
const cached = require('../cache-helper');
const Promise = require('bluebird');
const restifyClients = Promise.promisifyAll(require('restify-clients'));
const test = require('tap').test;
@ -32,8 +32,8 @@ test('startup', async function (t) {
});
test('clear everything', (t) => {
memcached.clearEverything((err) => {
t.notOk(err, 'memcached.clearEverything should not return an error');
cached.clearEverything((err) => {
t.notOk(err, 'cache.clearEverything should not return an error');
t.end();
});
});

Просмотреть файл

@ -25,7 +25,7 @@ config.limits.ipRateLimitBanDurationSeconds = 1;
config.limits.rateLimitIntervalSeconds = 1;
config.allowedPhoneNumbers = [ALLOWED_PHONE_NUMBER];
var mcHelper = require('../memcache-helper');
var mcHelper = require('../cache-helper');
var testServer = new TestServer(config);

Просмотреть файл

@ -5,7 +5,7 @@ const test = require('tap').test;
const TestServer = require('../test_server');
const Promise = require('bluebird');
const restifyClients = Promise.promisifyAll(require('restify-clients'));
const mcHelper = require('../memcache-helper');
const mcHelper = require('../cache-helper');
const testUtils = require('../utils');
const TEST_IP = '192.0.2.1';

Просмотреть файл

@ -7,7 +7,7 @@ const test = require('tap').test;
const restifyClients = require('restify-clients');
const TestServer = require('../test_server');
const Promise = require('bluebird');
const mcHelper = require('../memcache-helper');
const mcHelper = require('../cache-helper');
const TEST_EMAIL = 'test@example.com';
const TEST_IP = '192.0.2.1';

Просмотреть файл

@ -5,7 +5,7 @@ const test = require('tap').test;
const TestServer = require('../test_server');
const Promise = require('bluebird');
const restifyClients = Promise.promisifyAll(require('restify-clients'));
const mcHelper = require('../memcache-helper');
const mcHelper = require('../cache-helper');
function randomEmail() {
return Math.floor(Math.random() * 10000) + '@email.com';

Просмотреть файл

@ -1,18 +0,0 @@
const ROOT_DIR = '../..';
const test = require('tap').test;
const cp = require('child_process');
const Promise = require('bluebird');
const path = require('path');
cp.execAsync = Promise.promisify(cp.exec);
const cwd = path.resolve(__dirname, ROOT_DIR);
test('can run script', async function (t) {
cp.execAsync('node scripts/block-ip.js localhost:11211 127.0.0.1 600', {
cwd,
});
t.end();
});

Просмотреть файл

@ -30246,13 +30246,6 @@ __metadata:
languageName: node
linkType: hard
"connection-parse@npm:0.0.x":
version: 0.0.7
resolution: "connection-parse@npm:0.0.7"
checksum: 82c242157205babf80bd6467a65b4f05c21eed91cc46f76ca26398ef10b031a7a0bb63efbda3386e6d24dbfaf74a9d786b0e2fb6425b3a159c762b18052a5070
languageName: node
linkType: hard
"consola@npm:^2.15.0":
version: 2.15.0
resolution: "consola@npm:2.15.0"
@ -37524,7 +37517,6 @@ fsevents@~2.1.1:
"@types/jws": ^3.2.3
"@types/lodash": ^4
"@types/luxon": ^3
"@types/memcached": ^2.2.6
"@types/mjml": ^4.7.4
"@types/mocha": ^10.0.6
"@types/nock": ^11.1.0
@ -37855,7 +37847,6 @@ fsevents@~2.1.1:
load-grunt-tasks: ^5.1.0
lodash.isequal: 4.5.0
lodash.merge: 4.6.2
memcached: ^2.2.2
pm2: ^5.3.0
prettier: ^2.3.1
proxyquire: ^2.1.3
@ -40644,16 +40635,6 @@ fsevents@~2.1.1:
languageName: node
linkType: hard
"hashring@npm:3.2.x":
version: 3.2.0
resolution: "hashring@npm:3.2.0"
dependencies:
connection-parse: 0.0.x
simple-lru-cache: 0.0.x
checksum: d46733d341227a48df48c9c4d31f36a174e8c773df2f9013baed42357f93e07cd99e60dcb40c986b813654d505e5938f0a677bf33935e7be8ee20a0a1fbeb1db
languageName: node
linkType: hard
"hasown@npm:^2.0.0":
version: 2.0.0
resolution: "hasown@npm:2.0.0"
@ -43557,15 +43538,6 @@ fsevents@~2.1.1:
languageName: node
linkType: hard
"jackpot@npm:>=0.0.6":
version: 0.0.6
resolution: "jackpot@npm:0.0.6"
dependencies:
retry: 0.6.0
checksum: 57af0f39f8ffbf12825aaabe60295d2bb9c0e56ef6123168a542f73ce56935498ef35350b53171111214f3bcfe1ac4ff75cf2b64662e714adb1218b5e92ad6b9
languageName: node
linkType: hard
"jackspeak@npm:^1.4.1":
version: 1.4.1
resolution: "jackspeak@npm:1.4.1"
@ -48254,16 +48226,6 @@ fsevents@~2.1.1:
languageName: node
linkType: hard
"memcached@npm:^2.2.2":
version: 2.2.2
resolution: "memcached@npm:2.2.2"
dependencies:
hashring: 3.2.x
jackpot: ">=0.0.6"
checksum: bb87f0c9422e5a4dc1ca2235115b88ac1b4f3cf0d10435bb1296f54e3c14971ad764b1c26a071c0e24baa6e3095eef8e18ccfe82d70ca8272334ccd4340db1a5
languageName: node
linkType: hard
"memfs@npm:^3.1.2":
version: 3.2.2
resolution: "memfs@npm:3.2.2"
@ -58055,13 +58017,6 @@ resolve@1.1.7:
languageName: node
linkType: hard
"retry@npm:0.6.0":
version: 0.6.0
resolution: "retry@npm:0.6.0"
checksum: 44c7b1179ef9627d3ff11b9948e49094226c28f8838c43e39e3cd6a15689daacd59e82db53c5177793717768a6a15e16db6606b5b51cce7622751349c6a1ccfa
languageName: node
linkType: hard
"reusify@npm:^1.0.4":
version: 1.0.4
resolution: "reusify@npm:1.0.4"
@ -59523,13 +59478,6 @@ resolve@1.1.7:
languageName: node
linkType: hard
"simple-lru-cache@npm:0.0.x":
version: 0.0.2
resolution: "simple-lru-cache@npm:0.0.2"
checksum: 75bc9b788be4dad48b96ff2225604fe9c62d2e69327491ecb4d034b482cb88e5cd810e7b84f6ded3c4a87f1ecd407f1f2fafd5da09cf07dec3b9ce100cc13a44
languageName: node
linkType: hard
"simple-swizzle@npm:^0.2.2":
version: 0.2.2
resolution: "simple-swizzle@npm:0.2.2"