Merge pull request #16250 from mozilla/FXA-8843-delete-unverified-script

feat(scripts): add script to delete unverified accounts thru cloud tasks

packages/fxa-auth-server/lib/account-delete.ts

@@ -15,7 +15,7 @@ import { StripeHelper } from './payments/stripe';
 import push from './push';
 import pushboxApi from './pushbox';
 import { accountDeleteCloudTaskPath } from './routes/cloud-tasks';
-import { AppConfig, AuthLogger } from './types';
+import { AccountDeleteReasons, AppConfig, AuthLogger } from './types';
 /*
 import {
   uid as uidValidator,
@@ -23,11 +23,6 @@ import {
 } from './routes/validators';
 //*/
 
-export const AccountDeleteReasons = [
-  'fxa_unverified_account_delete',
-  'fxa_user_requested_account_delete',
-] as const;
-
 type FxaDbDeleteAccount = Pick<
   Awaited<ReturnType<ReturnType<typeof DB>['connect']>>,
   'deleteAccount' | 'accountRecord'

packages/fxa-auth-server/lib/db/index.js

@@ -291,6 +291,11 @@ module.exports = (config, log, Token, UnblockCode = null) => {
     return await Account.listAllUnverified({ include: ['emails'] });
   };
 
+  DB.prototype.getEmailUnverifiedAccounts = async function (options) {
+    log.trace('DB.getEmailUnverifiedAccounts');
+    return await Account.getEmailUnverifiedAccounts(options);
+  };
+
   DB.prototype.devices = async function (uid) {
     log.trace('DB.devices', { uid });
 

packages/fxa-auth-server/lib/routes/validators.js

@@ -8,7 +8,7 @@
 const { URL } = require('url');
 const punycode = require('punycode.js');
 const isA = require('joi');
-const { AccountDeleteReasons } = require('../account-delete');
+const { AccountDeleteReasons } = require('../types');
 const { MozillaSubscriptionTypes } = require('fxa-shared/subscriptions/types');
 const {
   minimalConfigSchema,

packages/fxa-auth-server/lib/types.ts

@@ -84,6 +84,13 @@ export interface AuthLogger extends Logger {
   ): Promise<void>;
 }
 
+// Exporting this here to avoid a circular dependency.  Can be moved to
+// lib/account-delete if we are ever at full ESM.
+export const AccountDeleteReasons = [
+  'fxa_unverified_account_delete',
+  'fxa_user_requested_account_delete',
+] as const;
+
 // Container token types
 // eslint-disable-next-line @typescript-eslint/no-redeclare
 export const AuthLogger = new Token<AuthLogger>('AUTH_LOGGER');

packages/fxa-auth-server/scripts/delete-unverified-accounts.ts

@@ -0,0 +1,240 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Command } from 'commander';
+import { parseDryRun } from './lib/args';
+import { AccountDeleteManager } from '../lib/account-delete';
+import { Container } from 'typedi';
+import { StripeHelper, createStripeHelper } from '../lib/payments/stripe';
+import { StatsD } from 'hot-shots';
+import { setupFirestore } from '../lib/firestore-db';
+import { AuthFirestore } from '../lib/types';
+import { CurrencyHelper } from '../lib/payments/currencies';
+import appConfig from '../config';
+import initLog from '../lib/log';
+import { AuthLogger, AppConfig } from '../lib/types';
+import DB from '../lib/db';
+import Token from '../lib/tokens';
+import * as random from '../lib/crypto/random';
+import initRedis from '../lib/redis';
+import oauthDb from '../lib/oauth/db';
+import initPush from '../lib/push';
+import { pushboxApi } from '../lib/pushbox';
+
+const collect = () => (val: string, xs: string[]) => {
+  xs.push(val);
+  return xs;
+};
+const uid = collect();
+const email = collect();
+
+const limitSpecifiedAccounts = (
+  program: Command,
+  limit: number
+): { uids: string[]; emails: string[] } => {
+  if (limit === Infinity) {
+    return { uids: program.uid, emails: program.email };
+  }
+  if (program.uid.size >= limit) {
+    return { uids: program.uid.slice(0, limit), emails: [] };
+  }
+  return {
+    uids: program.uid,
+    emails: program.email.slice(0, limit - program.uid.length),
+  };
+};
+
+const dryRun = (
+  program: Command,
+  useSpecifiedAccounts: boolean,
+  useDateRange: boolean,
+  limit: number
+) => {
+  if (useSpecifiedAccounts) {
+    const { uids, emails } = limitSpecifiedAccounts(program, limit);
+    console.log(
+      `When not in dry-run mode this call will enqueue ${
+        uids.length + emails.length
+      } account deletions with the following account info`
+    );
+    uids.forEach((x) => console.log(`uid: ${x}`));
+    emails.forEach((x) => console.log(`uid: ${x}`));
+  }
+  if (useDateRange) {
+    const start = new Date(program.startDate);
+    const end = new Date(program.endDate);
+    console.log(
+      `When not in dry-run mode this call will enqueue up to ${limit} account deletions in the date range ${start.toLocaleDateString()} ${start.toLocaleTimeString()} - ${end.toLocaleDateString()} ${end.toLocaleTimeString()} inclusive.`
+    );
+    console.log('Scanning a large database table for a range will be slow!');
+    console.log('Consider running this on a db replica.');
+  }
+  return 0;
+};
+
+const init = async () => {
+  const program = new Command();
+  program
+    .description(
+      'enqueue account deletes to Cloud Task with uid/email or a date range.'
+    )
+    .option('-u, --uid [uid]', 'An account uid, repeatable.', uid, [])
+    .option(
+      '--email [email]',
+      'An account primary email address, repeatable.',
+      email,
+      []
+    )
+    .option(
+      '--start-date [date]',
+      'Start of date range of account creation date, inclusive.',
+      Date.parse
+    )
+    .option(
+      '--end-date [date]',
+      'End of date range of account creation date, inclusive.',
+      Date.parse
+    )
+    .option('--limit', 'The number of delete tasks to enqueue.')
+    .option(
+      '--dry-run [true|false]',
+      'Print what the script would do instead of performing the action.  Defaults to true.',
+      true
+    )
+    .option(
+      '--table-scan [true|false]',
+      'Acknowledge that you are fine with a table scan on the accounts table.  Defaults to false.',
+      false
+    );
+
+  program.parse(process.argv);
+  const isDryRun = parseDryRun(program.dryRun);
+  const limit = program.limit ? parseInt(program.limit) : Infinity;
+  const hasUid = program.uid.length > 0;
+  const hasEmail = program.email.length > 0;
+  const hasDateRange =
+    program.startDate && program.endDate && program.endDate > program.startDate;
+  const reason = 'fxa_unverified_account_delete';
+
+  if (!hasUid && !hasEmail && !hasDateRange) {
+    throw new Error(
+      'The program needs at least a uid, an email, or valid date range.'
+    );
+  }
+  if ((hasUid || hasEmail) && hasDateRange) {
+    throw new Error(
+      'Sorry, but the script does not support uid/email arguments and a date range in the same invocation.'
+    );
+  }
+  if (limit <= 0) {
+    throw new Error('The limit should be a positive integer.');
+  }
+
+  const useSpecifiedAccounts = hasUid || hasEmail;
+  const useDateRange = !useSpecifiedAccounts && hasDateRange;
+
+  if (isDryRun) {
+    console.log(
+      'Dry run mode is on.  It is the default; use --dry-run=false when you are ready.'
+    );
+
+    return dryRun(program, useSpecifiedAccounts, useDateRange, limit);
+  }
+
+  const config = appConfig.getProperties();
+  const log = initLog({
+    ...config.log,
+  });
+  const statsd = new StatsD({ ...config.statsd });
+  const redis = initRedis(
+    { ...config.redis, ...config.redis.sessionTokens },
+    log
+  );
+  const db = DB(
+    config,
+    log,
+    Token(log, config),
+    random.base32(config.signinUnblock.codeLength) as any // TS type inference is failing pretty hard with this
+  );
+  const fxaDb = await db.connect(config, redis);
+  const push = initPush(log, fxaDb, config, statsd);
+  const pushbox = pushboxApi(log, config, statsd);
+
+  Container.set(AppConfig, config);
+  Container.set(AuthLogger, log);
+
+  const authFirestore = setupFirestore(config);
+  Container.set(AuthFirestore, authFirestore);
+  const currencyHelper = new CurrencyHelper(config);
+  Container.set(CurrencyHelper, currencyHelper);
+  const stripeHelper = createStripeHelper(log, config, statsd);
+  Container.set(StripeHelper, stripeHelper);
+
+  const accountDeleteManager = new AccountDeleteManager({
+    fxaDb,
+    oauthDb,
+    push,
+    pushbox,
+    statsd,
+  });
+
+  if (useSpecifiedAccounts) {
+    const { uids, emails } = limitSpecifiedAccounts(program, limit);
+
+    for (const x of uids) {
+      const acct = await fxaDb.account(x);
+      if (acct && acct.emailVerified) {
+        console.error(`Account with uid ${x} is verified.  Skipping.`);
+        continue;
+      }
+      const result = await accountDeleteManager.enqueue({ uid: x, reason });
+      console.log(`Created cloud task ${result} for uid ${x}`);
+    }
+
+    for (const x of emails) {
+      const acct = await fxaDb.accountRecord(x);
+      if (acct && acct.emailVerified) {
+        console.error(`Account with email ${x} is verified.  Skipping.`);
+        continue;
+      }
+      const result = await accountDeleteManager.enqueue({ email: x, reason });
+      console.log(`Created cloud task ${result} for uid ${x}`);
+    }
+  }
+
+  if (useDateRange) {
+    if (program.tableScan !== 'true') {
+      console.log('Please call with --table-scan if you are sure.');
+      return 0;
+    }
+
+    const accounts = await fxaDb.getEmailUnverifiedAccounts({
+      startCreatedAtDate: program.startDate,
+      endCreatedAtDate: program.endDate,
+      limit: limit === Infinity ? undefined : limit,
+      fields: ['uid'],
+    });
+
+    if (accounts.length === 0) {
+      console.log('No unverified accounts found with the given date range.');
+      return 0;
+    }
+
+    for (const x of accounts) {
+      const result = await accountDeleteManager.enqueue({ uid: x.uid, reason });
+      console.log(`Created cloud task ${result} for uid ${x.uid}`);
+    }
+  }
+
+  return 0;
+};
+
+if (require.main === module) {
+  init()
+    .catch((err: Error) => {
+      console.error(err);
+      process.exit(1);
+    })
+    .then((exitCode: number) => process.exit(exitCode));
+}

packages/fxa-auth-server/scripts/lib/args.ts

@@ -0,0 +1,7 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export const parseDryRun = (dryRun: boolean | string) => {
+  return `${dryRun}`.toLowerCase() !== 'false';
+};

packages/fxa-auth-server/scripts/refund-unverified-accounts.ts

@@ -9,6 +9,7 @@ import Container from 'typedi';
 import { setupProcessingTaskObjects } from '../lib/payments/processing-tasks-setup';
 import { AppConfig } from '../lib/types';
 import { promises as fs } from 'fs';
+import { parseDryRun } from './lib/args';
 
 const pckg = require('../package.json');
 
@@ -81,10 +82,6 @@ const isCustomer = (
   return false;
 };
 
-const parseDryRun = (dryRun: boolean | string) => {
-  return `${dryRun}`.toLowerCase() !== 'false';
-};
-
 const parseDateForFirestore = (date: number) => Math.floor(date / 1000);
 
 const parseStartDate = (date: string): number =>

packages/fxa-auth-server/scripts/stripe-products-and-plans-to-firestore-documents.ts

@@ -7,17 +7,14 @@ import Container from 'typedi';
 
 import { setupProcessingTaskObjects } from '../lib/payments/processing-tasks-setup';
 import { AppConfig } from '../lib/types';
+import { parseDryRun } from './lib/args';
 import {
-  StripeProductsAndPlansConverter,
   OutputTarget,
+  StripeProductsAndPlansConverter,
 } from './stripe-products-and-plans-to-firestore-documents/stripe-products-and-plans-converter';
 
 const pckg = require('../package.json');
 
-const parseDryRun = (dryRun: boolean | string) => {
-  return `${dryRun}`.toLowerCase() !== 'false';
-};
-
 const parseTarget = (target: any): OutputTarget => {
   if (Object.values(OutputTarget).includes(target)) {
     return target;

packages/fxa-auth-server/scripts/test-ci.sh

@@ -6,7 +6,7 @@ cd "$DIR/.."
 export NODE_ENV=dev
 export CORS_ORIGIN="http://foo,http://bar"
 
-DEFAULT_ARGS="--require esbuild-register --require tsconfig-paths/register --recursive --timeout 5000 --exit "
+DEFAULT_ARGS="--require esbuild-register --require tsconfig-paths/register --recursive --timeout 10000 --exit "
 if [ "$TEST_TYPE" == 'unit' ]; then GREP_TESTS="--grep #integration --invert "; fi;
 if [ "$TEST_TYPE" == 'integration' ]; then GREP_TESTS="--grep #integration "; fi;
 

packages/fxa-auth-server/test/scripts/delete-unverified-accounts.ts

@@ -0,0 +1,91 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import childProcess from 'child_process';
+import util from 'util';
+import path from 'path';
+import { assert } from 'chai';
+
+const exec = util.promisify(childProcess.exec);
+const ROOT_DIR = '../..';
+const cwd = path.resolve(__dirname, ROOT_DIR);
+const execOptions = {
+  cwd,
+  env: {
+    ...process.env,
+    NODE_ENV: 'dev',
+  },
+};
+
+const command = [
+  'node',
+  '-r esbuild-register',
+  'scripts/delete-unverified-accounts.ts',
+];
+
+describe('enqueue delete unverified account tasks script', () => {
+  it('needs uid, email, or date range', async () => {
+    try {
+      await exec(command.join(' '), execOptions);
+    } catch (err) {
+      assert.include(
+        err.stderr,
+        'The program needs at least a uid, an email, or valid date range.'
+      );
+    }
+  });
+
+  it('allows uid/email or date range but not both', async () => {
+    try {
+      const cmd = [
+        ...command,
+        '--uid 0f0f0f',
+        '--email testo@example.gg',
+        '--start-date 2022-11-22',
+        '--end-date 2022-11-30',
+      ];
+      await exec(cmd.join(' '), execOptions);
+    } catch (err) {
+      assert.include(
+        err.stderr,
+        'the script does not support uid/email arguments and a date range'
+      );
+    }
+  });
+
+  it('needs a positive integer for the limit', async () => {
+    try {
+      const cmd = [
+        ...command,
+        '--start-date 2022-11-22',
+        '--end-date 2022-11-30',
+        '--limit null',
+      ];
+      await exec(cmd.join(' '), execOptions);
+    } catch (err) {
+      assert.include(err.stderr, 'The limit should be a positive integer.');
+    }
+  });
+
+  it('executes in dry-run mode by default', async () => {
+    const cmd = [
+      ...command,
+      '--start-date 2022-11-22',
+      '--end-date 2022-11-30',
+    ];
+    const { stdout } = await exec(cmd.join(' '), execOptions);
+    assert.include(stdout, 'Dry run mode is on.');
+  });
+
+  it('warns about table scan', async () => {
+    const cmd = [
+      ...command,
+      '--start-date 2022-11-22',
+      '--end-date 2022-11-30',
+      '--dry-run=false',
+    ];
+    const { stdout } = await exec(cmd.join(' '), execOptions);
+    assert.include(stdout, 'Please call with --table-scan if you are sure.');
+  });
+});

packages/fxa-shared/db/models/auth/account.ts

@@ -499,6 +499,26 @@ export class Account extends BaseAuthModel {
     return account;
   }
 
+  static async getEmailUnverifiedAccounts(options: {
+    startCreatedAtDate: number;
+    endCreatedAtDate: number;
+    limit?: number;
+    fields?: string[];
+  }) {
+    const accounts = Account.query()
+      .select(...(options.fields || selectFields))
+      .whereBetween('createdAt', [
+        options.startCreatedAtDate,
+        options.endCreatedAtDate,
+      ])
+      .andWhere('emailVerified', 0);
+    if (options.limit) {
+      accounts.limit(options.limit);
+    }
+
+    return await accounts;
+  }
+
   static async setMetricsOpt(
     uid: string,
     state: 'in' | 'out',

packages/fxa-shared/test/db/models/auth/index.spec.ts

@@ -540,4 +540,23 @@ describe('#integration - auth', () => {
       });
     });
   });
+
+  describe('unverifiedAccounts', () => {
+    it('returns a list of unverified accounts in a date range', async () => {
+      for (let x = 0; x < 3; x++) {
+        const acct = randomAccount();
+        const email = randomEmail(acct);
+        await Account.query().insertGraph({
+          ...acct,
+          emailVerified: false,
+          emails: [email],
+        });
+      }
+      const unverifiedAccounts = await Account.getEmailUnverifiedAccounts({
+        startCreatedAtDate: 0,
+        endCreatedAtDate: Date.now(),
+      });
+      assert.equal(unverifiedAccounts.length, 3);
+    });
+  });
 });