feat(server): Add uid_record and checkAuthenticated endpoint (#121) r=vladikoff,rfk

docs/api.md

@@ -26,6 +26,7 @@ content-type of "application/json".
 * [POST /blockEmail](#post-blockemail)
 * [POST /blockIp](#post-blockip)
 * [POST /check](#post-check)
+* [POST /checkAuthenticated](#post-checkauthenticated)
 * [POST /failedLoginAttempt](#post-failedloginattempt)
 * [POST /passwordReset](#post-passwordreset)
 
@@ -142,6 +143,52 @@ Failing requests may be due to the following errors:
 
 * status code 400, code MissingParameters: email, ip and action are all required
 
+
+## POST /checkAuthenticated
+
+Called by the auth server before performing an authenticated action to
+check whether or not the action should be blocked.
+
+___Parameters___
+
+* action - the name of the action under consideration
+* ip - the IP address where the request originates
+* uid - account identifier
+
+### Request
+
+```sh
+curl -v \
+-H "Content-Type: application/json" \
+"http://127.0.0.1:7000/checkAuthenticated" \
+-d '{
+  "action": "devicesNotify"
+  "ip": "192.0.2.1",
+  "uid": "0b65dd742b5a415487f2108cca597044",
+}'
+```
+
+### Response
+
+Successful requests will produce a "200 OK" response with the blocking
+advice in the JSON body:
+
+```json
+{
+  "block": true,
+  "retryAfter": 86396
+}
+```
+
+`block` indicates whether or not the action should be blocked and
+`retryAfter` tells the client how long it should wait (in seconds)
+before attempting this action again.
+
+Failing requests may be due to the following errors:
+
+* status code 400, code MissingParameters: action, ip and uid are all required
+
+
 ## POST /failedLoginAttempt
 
 Called by the auth server to signal to the customs server that a

lib/config/config.js

@@ -94,6 +94,26 @@ module.exports = function (fs, path, url, convict) {
         format: 'nat',
         env: 'IP_RATE_LIMIT_BAN_DURATION_SECONDS'
       },
+      uidRateLimit: {
+        limitIntervalSeconds: {
+          doc: 'Duration of automatic throttling for uids',
+          default: 60 * 15,
+          format: 'nat',
+          env: 'UID_RATE_LIMIT_INTERVAL_SECONDS'
+        },
+        banDurationSeconds: {
+          doc: 'Duration of automatic ban',
+          default: 60 * 15,
+          format: 'nat',
+          env: 'UID_RATE_LIMIT_BAN_DURATION_SECONDS'
+        },
+        maxChecks: {
+          doc: 'Number of checks within uidRateLimitBanDurationSeconds before blocking',
+          default: 100,
+          format: 'nat',
+          env: 'UID_RATE_LIMIT'
+        }
+      },
       maxAccountStatusCheck: {
         doc: 'Number of account status checks within rateLimitIntervalSeconds before throttling',
         default: 5,

lib/limits.js

@@ -27,6 +27,9 @@ module.exports = function (config, mc, log) {
     this.ipRateLimitBanDurationMs = settings.ipRateLimitBanDurationSeconds * 1000
     this.maxAccountStatusCheck = settings.maxAccountStatusCheck
     this.badLoginErrnoWeights = settings.badLoginErrnoWeights || {}
+    this.maxChecksPerUid = settings.uidRateLimit.maxChecks
+    this.uidRateLimitBanDurationMs = settings.uidRateLimit.banDurationSeconds * 1000
+    this.uidRateLimitIntervalMs = settings.uidRateLimit.limitIntervalSeconds * 1000
     return this
   }
 

lib/server.js

@@ -44,6 +44,7 @@ module.exports = function createServer(config, log) {
   var IpEmailRecord = require('./ip_email_record')(limits)
   var EmailRecord = require('./email_record')(limits)
   var IpRecord = require('./ip_record')(limits)
+  var UidRecord = require('./uid_record')(limits)
 
   var handleBan = P.promisify(require('./bans/handler')(config.memcache.recordLifetimeSeconds, mc, EmailRecord, IpRecord, log))
 
@@ -163,6 +164,55 @@ module.exports = function createServer(config, log) {
     }
   )
 
+  api.post(
+    '/checkAuthenticated',
+    function (req, res, next) {
+      var action = req.body.action
+      var ip = req.body.ip
+      var uid = req.body.uid
+
+      if(!action || !ip || !uid){
+        var err = {code: 'MissingParameters', message: 'action, ip and uid are all required'}
+        log.error({op:'request.checkAuthenticated', action: action, ip: ip, uid: uid, err: err})
+        res.send(400, err)
+        return next()
+      }
+
+      mc.getAsync(uid)
+        .then(UidRecord.parse, UidRecord.parse)
+        .then(
+          function (uidRecord) {
+            var retryAfter = uidRecord.addCount(action, uid)
+
+            return setRecord(uid, uidRecord)
+              .then(
+                function () {
+                  return {
+                    block: retryAfter > 0,
+                    retryAfter: retryAfter
+                  }
+                }
+              )
+          }
+        )
+        .then(
+          function (result) {
+            log.info({ op: 'request.checkAuthenticated', block: result.block })
+            res.send(result)
+          },
+          function (err) {
+            log.error({ op: 'request.checkAuthenticated', err: err })
+            // Default is to block request on any server based error
+            res.send({
+              block: true,
+              retryAfter: limits.blockIntervalMs
+            })
+          }
+        )
+        .done(next, next)
+    }
+  )
+
   api.post(
     '/failedLoginAttempt',
     function (req, res, next) {

lib/uid_record.js

@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This module keeps track of events related to a specific uid
+ */
+module.exports = function (limits, now) {
+
+  now = now || Date.now
+  function UidRecord() {
+    this.timestampsByAction = {}
+    this.rateLimitedTimestamp = undefined
+  }
+
+  UidRecord.parse = function (object) {
+    var rec = new UidRecord()
+    object = object || {}
+    rec.rateLimitedTimestamp = object.rateLimitedTimestamp // timestamp when the account was rateLimited
+    rec.timestampsByAction = object.timestampsByAction || {}  // timestamps on each kind of action
+    return rec
+  }
+
+  UidRecord.prototype.getMinLifetimeMS = function () {
+    return Math.max(
+      limits.uidRateLimitIntervalMs,
+      limits.uidRateLimitBanDurationMs
+    )
+  }
+
+  /*
+   * stores check timestamps if account is not blocked.
+   * returns remaining time until uid unblick
+   */
+  UidRecord.prototype.addCount = function (action) {
+    var timestamp = now()
+    this.trimCounts(action, timestamp)
+
+    if(!this.isRateLimited()){
+      this.timestampsByAction[action].push(timestamp)
+    }
+
+    return this.getPossibleRetryDelay(action)
+  }
+
+  UidRecord.prototype.trimCounts = function (action, now) {
+    this.timestampsByAction[action] = this._trim(now, this.timestampsByAction[action], limits.maxChecksPerUid)
+  }
+
+  UidRecord.prototype._trim = function (now, items, max) {
+    if (!items || items.length === 0) { return [] }
+    // the list is naturally ordered from oldest to newest,
+    // and we only need to keep data for up to max+ 1.
+    var i = items.length - 1
+    var n = 0
+    var item = items[i]
+    while (item > (now - limits.uidRateLimitIntervalMs) && n < max) {
+      item = items[--i]
+      n++
+      if (i === -1) {
+        break
+      }
+    }
+    return items.slice(i + 1)
+  }
+
+  UidRecord.prototype.rateLimit = function () {
+    this.rateLimitedTimestamp = now()
+  }
+
+  UidRecord.prototype.getRetryAfter = function() {
+    return this.rateLimitedTimestamp + limits.uidRateLimitBanDurationMs - now()
+  }
+
+  UidRecord.prototype.isRateLimited = function() {
+    //uid is rateLimited if there's a valid rateLimited timestamp and
+    //waiting time interval wasn't reached
+    return (this.rateLimitedTimestamp && (this.rateLimitedTimestamp + limits.uidRateLimitBanDurationMs > now()))
+  }
+
+  UidRecord.prototype.getPossibleRetryDelay = function(action) {
+    if(this.timestampsByAction[action].length < limits.maxChecksPerUid){
+      this.rateLimitedTimestamp = undefined
+      return 0
+    } else {
+      if(!this.isRateLimited()){
+        // we have more attempts than allowed. rateLimit it
+        this.rateLimit()
+      }
+      return this.getRetryAfter()
+    }
+  }
+
+  return UidRecord
+}

test/memcache-helper.js

@@ -19,7 +19,12 @@ var config = {
     maxBadLogins: 2,
     maxBadLoginsPerIp: Number(process.env.MAX_BAD_LOGINS_PER_IP) || 3,
     ipRateLimitIntervalSeconds: Number(process.env.IP_RATE_LIMIT_INTERVAL_SECONDS) || 60 * 15,
-    ipRateLimitBanDurationSeconds: Number(process.env.IP_RATE_LIMIT_BAN_DURATION_SECONDS) || 60 * 15
+    ipRateLimitBanDurationSeconds: Number(process.env.IP_RATE_LIMIT_BAN_DURATION_SECONDS) || 60 * 15,
+    uidRateLimit: {
+      limitIntervalSeconds: Number(process.env.UID_RATE_LIMIT_INTERVAL_SECONDS) || 60 * 15,
+      banDurationSeconds: Number(process.env.UID_RATE_LIMIT_BAN_DURATION_SECONDS) || 60 * 15,
+      maxChecks: Number(process.env.UID_RATE_LIMIT) || 3
+    }
   }
 }
 

test/remote/config_update_tests.js

@@ -56,7 +56,7 @@ test(
       })
       .then(function (bis) {
         x = bis
-        return mcHelper.setLimits({ blockIntervalSeconds: bis + 1 })
+        return mcHelper.setLimits({ blockIntervalSeconds: bis + 1, uidRateLimit:{} })
       })
       .then(function (settings) {
         t.equal(x + 1, settings.blockIntervalSeconds, 'helper sees the change')

test/remote/too_many_authenticated_checks.js

@@ -0,0 +1,159 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Override limit values for testing
+process.env.UID_RATE_LIMIT = 3
+process.env.UID_RATE_LIMIT_INTERVAL_SECONDS = 2
+process.env.UID_RATE_LIMIT_BAN_DURATION_SECONDS = 2
+
+var test = require('tap').test
+var TestServer = require('../test_server')
+var Promise = require('bluebird')
+var restify = Promise.promisifyAll(require('restify'))
+var mcHelper = require('../memcache-helper')
+
+var TEST_IP = '192.0.2.1'
+var TEST_UID = 'abc123'
+var ACTION_ONE = 'action1'
+var ACTION_A = 'actionA'
+var ACTION_B = 'actionB'
+
+var config = {
+  listen: {
+    port: 7000
+  }
+}
+
+var testServer = new TestServer(config)
+
+var client = restify.createJsonClient({
+  url: 'http://127.0.0.1:' + config.listen.port
+})
+
+Promise.promisifyAll(client, { multiArgs: true })
+
+test(
+  'startup',
+  function (t) {
+    testServer.start(function (err) {
+      t.type(testServer.server, 'object', 'test server was started')
+      t.notOk(err, 'no errors were returned')
+      t.end()
+    })
+  }
+)
+
+test(
+  'clear everything',
+  function (t) {
+    mcHelper.clearEverything(
+      function (err) {
+        t.notOk(err, 'no errors were returned')
+        t.end()
+      }
+    )
+  }
+)
+
+test(
+  '/checkAuthenticated with same action',
+  function (t) {
+    // Send requests until it blocks
+    return client.postAsync('/checkAuthenticated', { action: ACTION_ONE, ip: TEST_IP, uid: TEST_UID})
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200 check1')
+        t.equal(obj.block, false, 'not rate limited')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_ONE, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200 check2')
+        t.equal(obj.block, false, 'not rate limited')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_ONE, ip: TEST_IP, uid: TEST_UID})
+      })
+      // uid should be now blocked
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200 check3')
+        t.equal(obj.block, true, 'uid is rate limited')
+
+        // Delay ~2s for rate limit to go away
+        return Promise.delay(2010)
+      })
+      // uid should be now unblocked
+      .then(function(){
+        return client.postAsync('/checkAuthenticated', { action: ACTION_ONE, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'is not rate limited after UID_RATE_LIMIT_BAN_DURATION_SECONDS')
+        t.end()
+      })
+      .catch(function(err){
+        t.fail(err)
+        t.end()
+      })
+  }
+)
+
+test(
+  '/checkAuthenticated with different actions',
+  function (t) {
+    // Send requests until one gets rate limited
+    return client.postAsync('/checkAuthenticated', { action: ACTION_A, ip: TEST_IP, uid: TEST_UID})
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'not rate limited on check1, actionA')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_B, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'not rate limited on check1, actionB')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_A, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'not rate limited on check2, actionA')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_A, ip: TEST_IP, uid: TEST_UID})
+      })
+      // uid should be now blocked to action1
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, true, 'uid is actionA rate limited after check3')
+
+        return client.postAsync('/checkAuthenticated', { action: ACTION_B, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'not rate limited for actionB after check2')
+
+        // Delay ~2s for rate limit to go away
+        return Promise.delay(2010)
+      })
+      // uid should be now unblocked
+      .then(function(){
+        return client.postAsync('/checkAuthenticated', { action: ACTION_A, ip: TEST_IP, uid: TEST_UID})
+      })
+      .spread(function(req, res, obj){
+        t.equal(res.statusCode, 200, 'returns a 200')
+        t.equal(obj.block, false, 'is not rate limited after UID_RATE_LIMIT_BAN_DURATION_SECONDS')
+        t.end()
+      })
+      .catch(function(err){
+        t.fail(err)
+        t.end()
+      })
+  }
+)
+
+test(
+  'teardown',
+  function (t) {
+    testServer.stop()
+    t.equal(testServer.server.killed, true, 'test server has been killed')
+    t.end()
+  }
+)

test/remote/too_many_login_checks.js

@@ -91,7 +91,7 @@ test(
         t.equal(res.statusCode, 200, 'returns a 200')
         t.equal(obj.block, true, 'ip is still rate limited')
 
-        // Delay ~3s for rate limit to go away
+        // Delay ~5s for rate limit to go away
         return Promise.delay(5010)
       })
       // IP should be now unblocked
@@ -158,7 +158,6 @@ test(
   }
 )
 
-
 test(
   'teardown',
   function (t) {