feat(scripts): use redlock to prevent >1 instance of paypal-processor

Because:
 - we could easily end up running two instances of the paypal-processor
   during a deploy

This commit:
 - use a redis based distributed lock to ensure only one
   paypal-processor can run per env
 - add script options to control the lock name and duration, as well as
   completely bypassing the lock

package.json

@@ -38,6 +38,7 @@
     "nps": "^5.10.0",
     "pm2": "^5.1.2",
     "prettier": "^2.3.1",
+    "redlock": "^5.0.0-beta.2",
     "replace-in-file": "^6.1.0",
     "semver": "^7.3.5"
   },

packages/fxa-auth-server/lib/payments/paypal/processor.ts

@@ -350,7 +350,7 @@ export class PaypalProcessor {
     return;
   }
 
-  public async processInvoices() {
+  public async *processInvoices() {
     // Generate a time `invoiceAge` hours prior.
     const invoiceAgeInSeconds = hoursBeforeInSeconds(this.invoiceAge);
 
@@ -369,6 +369,8 @@ export class PaypalProcessor {
         });
         reportSentryError(err);
       }
+
+      yield;
     }
   }
 }

packages/fxa-auth-server/scripts/paypal-processor.ts

@@ -3,6 +3,8 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 import program from 'commander';
 import { StatsD } from 'hot-shots';
+import Redis from 'ioredis';
+import Redlock, { Lock } from 'redlock';
 import Container from 'typedi';
 import { promisify } from 'util';
 
@@ -13,6 +15,21 @@ import { setupProcessingTaskObjects } from '../lib/payments/processing-tasks-set
 
 const pckg = require('../package.json');
 const config = require('../config').getProperties();
+const PAYPAL_PROCESSOR_LOCK = 'fxa-paypal-processor-lock';
+const DEFAULT_LOCK_DURATION_MS = 300000;
+let lock: Lock;
+
+const initTimer = () => {
+  let start = Date.now();
+
+  const reset = () => (start = Date.now());
+  const elapsed = () => Date.now() - start;
+
+  return {
+    reset,
+    elapsed,
+  };
+};
 
 export async function init() {
   // Load program options
@@ -29,8 +46,28 @@ export async function init() {
       'How old in hours the invoice must be to get processed. Defaults to 6.',
       '6'
     )
+    .option(
+      '-l, --use-lock [bool]',
+      'Whether to require a distributed lock to run.  Use "false" to disable.  Defaults to true.',
+      true
+    )
+    .option(
+      '-n, --lock-name [name]',
+      `The name of the resource for which to acquire a distributed lock. Defaults to ${PAYPAL_PROCESSOR_LOCK}.`,
+      PAYPAL_PROCESSOR_LOCK
+    )
+    .option(
+      '-d, --lock-duration [milliseconds]',
+      `The max duration in milliseconds to hold the lock.  The lock will be extended as needed.  Defaults to ${DEFAULT_LOCK_DURATION_MS}.`,
+      DEFAULT_LOCK_DURATION_MS
+    )
     .parse(process.argv);
 
+  // every arg is a string
+  const useLock = program.useLock !== 'false';
+  const lockDuration =
+    parseInt(`${program.lockDuration}`) || DEFAULT_LOCK_DURATION_MS;
+
   const { log, database, senders } = await setupProcessingTaskObjects(
     'paypal-processor'
   );
@@ -53,7 +90,26 @@ export async function init() {
   );
   const statsd = Container.get(StatsD);
   statsd.increment('paypal-processor.startup');
-  await processor.processInvoices();
+
+  const timer = initTimer();
+
+  if (useLock) {
+    try {
+      const redis = new Redis(config.redis);
+      const redlock = new Redlock([redis], { retryCount: 1 });
+      lock = await redlock.acquire([program.lockName], lockDuration);
+    } catch (err) {
+      throw new Error(`Cannot acquire lock to run: ${err.message}`);
+    }
+  }
+
+  for await (const _ of processor.processInvoices()) {
+    if (useLock && timer.elapsed() > Math.floor(lockDuration / 2)) {
+      await lock.extend(timer.elapsed());
+      timer.reset();
+    }
+  }
+
   statsd.increment('paypal-processor.shutdown');
   await promisify(statsd.close).bind(statsd)();
   return 0;
@@ -65,5 +121,6 @@ if (require.main === module) {
       console.error(err);
       process.exit(1);
     })
-    .then((result) => process.exit(result));
+    .then((result) => process.exit(result))
+    .finally(() => lock?.release());
 }

packages/fxa-auth-server/test/local/payments/paypal-processor.js

@@ -605,7 +605,11 @@ describe('PaypalProcessor', () => {
           yield invoice;
         },
       });
-      await processor.processInvoices();
+      // eslint-disable-next-line
+      for await (const _ of processor.processInvoices()) {
+        // No value yield'd; yielding control for potential distributed lock
+        // extension in actual use case
+      }
       sinon.assert.calledOnceWithExactly(
         mockLog.info,
         'processInvoice.processing',
@@ -628,7 +632,11 @@ describe('PaypalProcessor', () => {
         },
       });
       try {
-        await processor.processInvoices();
+        // eslint-disable-next-line
+        for await (const _ of processor.processInvoices()) {
+          // No value yield'd; yielding control for potential distributed lock
+          // extension in actual use case
+        }
         assert.fail('Process invoicce should fail');
       } catch (err) {
         sinon.assert.calledOnceWithExactly(

yarn.lock

@@ -23122,6 +23122,7 @@ fsevents@~2.1.1:
     nps: ^5.10.0
     pm2: ^5.1.2
     prettier: ^2.3.1
+    redlock: ^5.0.0-beta.2
     replace-in-file: ^6.1.0
     semver: ^7.3.5
     stylelint: ^13.13.1
@@ -32166,6 +32167,13 @@ fsevents@~2.1.1:
   languageName: node
   linkType: hard
 
+"node-abort-controller@npm:^3.0.1":
+  version: 3.0.1
+  resolution: "node-abort-controller@npm:3.0.1"
+  checksum: 2b3d75c65249fea99e8ba22da3a8bc553f034f44dd12f5f4b38b520f718b01c88718c978f0c24c2a460fc01de9a80b567028f547b94440cb47adeacfeb82c2ee
+  languageName: node
+  linkType: hard
+
 "node-addon-api@npm:^4.3.0":
   version: 4.3.0
   resolution: "node-addon-api@npm:4.3.0"
@@ -37346,6 +37354,15 @@ fsevents@~2.1.1:
   languageName: node
   linkType: hard
 
+"redlock@npm:^5.0.0-beta.2":
+  version: 5.0.0-beta2
+  resolution: "redlock@npm:5.0.0-beta2"
+  dependencies:
+    node-abort-controller: ^3.0.1
+  checksum: d8a0d6d472922d146077e3c12946b942108e3041439fdadced79c3dc285ec3d509a48cee33f7da125e71b3868c65ba248b612fb65825aacfa5e67c118e1ba543
+  languageName: node
+  linkType: hard
+
 "reduce-css-calc@npm:^2.1.6, reduce-css-calc@npm:^2.1.8":
   version: 2.1.8
   resolution: "reduce-css-calc@npm:2.1.8"