task(auth): Have auth-server use cloud-tasks nx lib

Because:
- We want to use the `cloud-tasks` nx lib that introduced last sprint

This commit
- Updates account end points to use the nx lib
- Updates CI to include cloud-task emulator
This commit is contained in:
dschom 2024-03-26 15:43:57 -07:00
Родитель 08b405470c
Коммит 7c0c70d2a4
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: F26AEE99174EE68B
18 изменённых файлов: 1158 добавлений и 1332 удалений

Просмотреть файл

@ -121,11 +121,14 @@ executors:
- image: jdlk7/firestore-emulator
- image: memcached
- image: redis
- image: ghcr.io/aertje/cloud-tasks-emulator:1.2.0
command: -queue "projects/test/locations/test/queues/delete-accounts-queue"
environment:
NODE_ENV: development
FIRESTORE_EMULATOR_HOST: localhost:9090
CUSTOMS_SERVER_URL: none
HUSKY_SKIP_INSTALL: 1
AUTH_CLOUDTASKS_USE_LOCAL_EMULATOR: true
# For anything that needs a full stack to run and needs browsers available for
# ui test automation. This image requires a restored workspace state.
@ -143,6 +146,8 @@ executors:
- image: cimg/mysql:8.0
command: --default-authentication-plugin=mysql_native_password
- image: jdlk7/firestore-emulator
- image: ghcr.io/aertje/cloud-tasks-emulator:1.2.0
command: -queue "projects/test/locations/test/queues/delete-accounts-queue"
environment:
NODE_ENV: development
FXA_EMAIL_ENV: development
@ -164,6 +169,7 @@ executors:
REACT_CONVERSION_POST_VERIFY_CAD_VIA_QR_ROUTES: true
CUSTOMS_SERVER_URL: none
HUSKY_SKIP_INSTALL: 1
AUTH_CLOUDTASKS_USE_LOCAL_EMULATOR: true
# Contains a pre-installed fxa stack and browsers for doing ui test
# automation. Perfect for running smoke tests against remote targets.

Просмотреть файл

@ -20,7 +20,11 @@ export class AccountTasks extends CloudTasks {
super(config, cloudTaskClient);
}
/** Add an account to the task queue. */
/**
* Adds an account to the delete account task queue.
* @param deleteTask The info necessary to queue an account deletion.
* @returns A taskName
*/
public async deleteAccount(deleteTask: DeleteAccountTask) {
try {
const result = await this.enqueueTask({

Просмотреть файл

@ -12,8 +12,6 @@ export type DeleteAccountCloudTaskConfig = CloudTasksConfig & {
queueName: string;
};
};
publicUrl: string;
apiVersion: string;
};
/** Reasons an account can be deleted. */
@ -28,7 +26,7 @@ export type DeleteAccountTask = {
/** The account id */
uid: string;
/** The customer id, i.e. a stripe customer id if applicable */
customerId?: string;
customerId: string | undefined;
/** Reason for deletion */
reason: ReasonForDeletion;
};

Просмотреть файл

@ -6,6 +6,24 @@ import { CloudTasksConfig } from './cloud-tasks.types';
/** Base class for encapsulating common cloud task operations */
export class CloudTasks {
/**
* Indicates if a queue has been configured and is enabled.
*/
public get queueEnabled() {
// If a keyFilename was supplied, the cloud task queue can be considered enabled.
if (this.config.cloudTasks.credentials.keyFilename) {
return true;
}
// If we specify a local emulator is being used, then no keyFilename is required,
// and the task queue can be considered enabled.
if (this.config.cloudTasks.useLocalEmulator) {
return true;
}
return false;
}
protected constructor(
protected readonly config: CloudTasksConfig,
protected readonly client: Pick<CloudTasksClient, 'createTask' | 'getTask'>

Просмотреть файл

@ -37,6 +37,10 @@ const { AccountDeleteManager } = require('../lib/account-delete');
const { gleanMetrics } = require('../lib/metrics/glean');
const Customs = require('../lib/customs');
const Profile = require('../lib/profile/client');
const {
AccountTasks,
AccountTasksFactory,
} = require('@fxa/shared/cloud-tasks');
async function run(config) {
Container.set(AppConfig, config);
@ -176,12 +180,15 @@ async function run(config) {
// The AccountDeleteManager is dependent on some of the object set into
// Container above.
const accountTasks = AccountTasksFactory(config, statsd);
Container.set(AccountTasks, accountTasks);
const accountDeleteManager = new AccountDeleteManager({
fxaDb: database,
oauthDb,
config,
push,
pushbox,
statsd,
});
Container.set(AccountDeleteManager, accountDeleteManager);

Просмотреть файл

@ -4,13 +4,10 @@
import {
deleteAllPayPalBAs,
getAccountCustomerByUid,
getAllPayPalBAByUid,
} from 'fxa-shared/db/models/auth';
import { StatsD } from 'hot-shots';
import { Container } from 'typedi';
import { CloudTasksClient } from '@google-cloud/tasks';
import * as Sentry from '@sentry/node';
import { ConfigType } from '../config';
@ -21,10 +18,10 @@ import { AppleIAP } from './payments/iap/apple-app-store/apple-iap';
import { PlayBilling } from './payments/iap/google-play/play-billing';
import { PayPalHelper } from './payments/paypal/helper';
import { StripeHelper } from './payments/stripe';
import { ReasonForDeletion } from '@fxa/shared/cloud-tasks';
import { StripeFirestoreMultiError } from './payments/stripe-firestore';
import pushBuilder from './push';
import pushboxApi from './pushbox';
import { accountDeleteCloudTaskPath } from './routes/cloud-tasks';
import { AppConfig, AuthLogger, AuthRequest } from './types';
type FxaDbDeleteAccount = Pick<
@ -42,54 +39,17 @@ type PushForDeleteAccount = Pick<
>;
type Log = AuthLogger & { activityEvent: (data: Record<string, any>) => void };
export const ReasonForDeletionOptions = {
UserRequested: 'fxa_user_requested_account_delete',
Unverified: 'fxa_unverified_account_delete',
Cleanup: 'fxa_cleanup_account_delete',
} as const;
export type ReasonForDeletion =
(typeof ReasonForDeletionOptions)[keyof typeof ReasonForDeletionOptions];
export const ReasonForDeletionValues = Object.values(ReasonForDeletionOptions);
type DeleteTask = {
uid: string;
customerId?: string;
reason: ReasonForDeletion;
};
type EnqueueByUidParam = {
uid: string;
reason: ReasonForDeletion;
};
type EnqueueByEmailParam = {
email: string;
reason: ReasonForDeletion;
};
const isEnqueueByUidParam = (
x: EnqueueByUidParam | EnqueueByEmailParam
): x is EnqueueByUidParam => (x as EnqueueByUidParam).uid !== undefined;
const isEnqueueByEmailParam = (
x: EnqueueByUidParam | EnqueueByEmailParam
): x is EnqueueByEmailParam => (x as EnqueueByEmailParam).email !== undefined;
export class AccountDeleteManager {
private fxaDb: FxaDbDeleteAccount;
private oauthDb: OAuthDbDeleteAccount;
private push: PushForDeleteAccount;
private pushbox: PushboxDeleteAccount;
private statsd: StatsD;
private stripeHelper?: StripeHelper;
private paypalHelper?: PayPalHelper;
private appleIap?: AppleIAP;
private playBilling?: PlayBilling;
private log: Log;
private config: ConfigType;
private tasksEnabled = false;
private tasksRequired = false;
private cloudTasksClient: CloudTasksClient;
private queueName;
private taskUrl;
constructor({
fxaDb,
@ -97,21 +57,18 @@ export class AccountDeleteManager {
config,
push,
pushbox,
statsd,
}: {
fxaDb: FxaDbDeleteAccount;
oauthDb: OAuthDbDeleteAccount;
config: ConfigType;
push: PushForDeleteAccount;
pushbox: PushboxDeleteAccount;
statsd: StatsD;
}) {
this.fxaDb = fxaDb;
this.oauthDb = oauthDb;
this.config = config;
this.push = push;
this.pushbox = pushbox;
this.statsd = statsd;
if (Container.has(StripeHelper)) {
this.stripeHelper = Container.get(StripeHelper);
@ -125,81 +82,11 @@ export class AccountDeleteManager {
if (Container.has(PlayBilling)) {
this.playBilling = Container.get(PlayBilling);
}
this.log = Container.get(AuthLogger) as Log;
// Is this intentional? Config is passed in the constructor
this.config = Container.get(AppConfig);
const tasksConfig = this.config.cloudTasks;
this.cloudTasksClient = new CloudTasksClient({
projectId: tasksConfig.projectId,
keyFilename: tasksConfig.credentials.keyFilename ?? undefined,
});
this.queueName = `projects/${tasksConfig.projectId}/locations/${tasksConfig.locationId}/queues/${tasksConfig.deleteAccounts.queueName}`;
this.taskUrl = `${this.config.publicUrl}/v${this.config.apiVersion}${accountDeleteCloudTaskPath}`;
this.tasksEnabled = !!tasksConfig.credentials.keyFilename;
this.tasksRequired = ['stage', 'prod'].includes(this.config.env);
}
public async enqueue(options: EnqueueByUidParam | EnqueueByEmailParam) {
if (isEnqueueByUidParam(options)) {
return this.enqueueByUid(options);
}
if (isEnqueueByEmailParam(options)) {
return this.enqueueByEmail(options);
}
throw new Error(
`Failed to enqueue account delete cloud task with ${options}.`
);
}
private async enqueueByUid(options: EnqueueByUidParam) {
const { stripeCustomerId } =
(await getAccountCustomerByUid(options.uid)) || {};
const task: DeleteTask = {
uid: options.uid,
customerId: stripeCustomerId,
reason: options.reason,
};
return this.enqueueTask(task);
}
private async enqueueByEmail(options: EnqueueByEmailParam) {
const account = await this.fxaDb.accountRecord(options.email);
const { stripeCustomerId } =
(await getAccountCustomerByUid(account.uid)) || {};
const task: DeleteTask = {
uid: account.uid,
customerId: stripeCustomerId,
reason: options.reason,
};
return this.enqueueTask(task);
}
private async enqueueTask(task: DeleteTask) {
try {
const taskResult = await this.cloudTasksClient.createTask({
parent: this.queueName,
task: {
httpRequest: {
url: this.taskUrl,
httpMethod: 'POST',
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(JSON.stringify(task)).toString('base64'),
oidcToken: {
audience: this.config.cloudTasks.oidc.aud,
serviceAccountEmail:
this.config.cloudTasks.oidc.serviceAccountEmail,
},
},
},
});
this.statsd.increment('cloud-tasks.account-delete.enqueue.success');
return taskResult[0].name;
} catch (err) {
this.statsd.increment('cloud-tasks.account-delete.enqueue.failure');
throw err;
}
}
/**
@ -240,7 +127,7 @@ export class AccountDeleteManager {
* deletion.
*/
public async quickDelete(uid: string, reason: ReasonForDeletion) {
if (reason !== ReasonForDeletionOptions.UserRequested) {
if (reason !== ReasonForDeletion.UserRequested) {
throw new Error('quickDelete only supports user requested deletions');
}
@ -252,10 +139,6 @@ export class AccountDeleteManager {
// still queue the account for cleanup.
this.log.error('quickDelete', { uid, error });
}
// Only queue if enabled or is in stage/prod.
if (this.tasksEnabled || this.tasksRequired) {
await this.enqueueByUid({ uid, reason });
}
}
/**
@ -329,7 +212,7 @@ export class AccountDeleteManager {
});
// Currently only support auto refund of invoices for unverified accounts
if (deleteReason !== ReasonForDeletionOptions.Unverified || !customerId) {
if (deleteReason !== ReasonForDeletion.Unverified || !customerId) {
return;
}

Просмотреть файл

@ -1,7 +1,7 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { Account } from 'fxa-shared/db/models/auth';
import { Account, getAccountCustomerByUid } from 'fxa-shared/db/models/auth';
import {
AppStoreSubscription,
PlayStoreSubscription,
@ -42,11 +42,9 @@ import requestHelper from './utils/request_helper';
import validators from './validators';
import { AccountEventsManager } from '../account-events';
import { gleanMetrics } from '../metrics/glean';
import {
AccountDeleteManager,
ReasonForDeletionOptions,
} from '../account-delete';
import { AccountDeleteManager } from '../account-delete';
import { uuidTransformer } from 'fxa-shared/db/transformers';
import { AccountTasks, ReasonForDeletion } from '@fxa/shared/cloud-tasks';
const METRICS_CONTEXT_SCHEMA = require('../metrics/context').schema;
@ -67,6 +65,7 @@ export class AccountHandler {
private capabilityService: CapabilityService;
private accountEventsManager: AccountEventsManager;
private accountDeleteManager: AccountDeleteManager;
private accountTasks: AccountTasks;
constructor(
private log: AuthLogger,
@ -82,7 +81,6 @@ export class AccountHandler {
private subscriptionAccountReminders: any,
private oauth: any,
private stripeHelper: StripeHelper,
private pushbox: any,
private glean: ReturnType<typeof gleanMetrics>
) {
this.otpUtils = require('./utils/otp')(log, config, db);
@ -104,6 +102,7 @@ export class AccountHandler {
this.capabilityService = Container.get(CapabilityService);
this.accountEventsManager = Container.get(AccountEventsManager);
this.accountDeleteManager = Container.get(AccountDeleteManager);
this.accountTasks = Container.get(AccountTasks);
}
private async generateRandomValues() {
@ -1846,11 +1845,21 @@ export class AccountHandler {
await this.accountDeleteManager.quickDelete(
accountRecord.uid,
ReasonForDeletionOptions.UserRequested
ReasonForDeletion.UserRequested
);
await request.emitMetricsEvent('account.deleted', {
uid: accountRecord.uid,
});
if (this.accountTasks.queueEnabled) {
const result = await getAccountCustomerByUid(accountRecord.uid);
await this.accountTasks.deleteAccount({
uid: accountRecord.uid,
customerId: result?.stripeCustomerId,
reason: ReasonForDeletion.UserRequested,
});
}
return {};
}
@ -1935,7 +1944,6 @@ export const accountRoutes = (
subscriptionAccountReminders,
oauth,
stripeHelper,
pushbox,
glean
);
const routes = [

Просмотреть файл

@ -7,19 +7,11 @@ import { Container } from 'typedi';
import { ConfigType } from '../../config';
import DESCRIPTION from '../../docs/swagger/shared/descriptions';
import {
AccountDeleteManager,
ReasonForDeletion,
ReasonForDeletionValues,
} from '../account-delete';
import { AccountDeleteManager } from '../account-delete';
import { AuthLogger, AuthRequest } from '../types';
import validators from './validators';
export type DeleteAccountTaskPayload = {
uid: string;
customerId?: string;
reason: ReasonForDeletion;
};
import { DeleteAccountTask } from '@fxa/shared/cloud-tasks';
export class CloudTaskHandler {
private accountDeleteManager: AccountDeleteManager;
@ -28,7 +20,7 @@ export class CloudTaskHandler {
this.accountDeleteManager = Container.get(AccountDeleteManager);
}
async deleteAccount(taskPayload: DeleteAccountTaskPayload) {
async deleteAccount(taskPayload: DeleteAccountTask) {
this.log.debug('Received delete account task', taskPayload);
await this.accountDeleteManager.deleteAccount(
taskPayload.uid,
@ -64,14 +56,12 @@ export const cloudTaskRoutes = (log: AuthLogger, config: ConfigType) => {
.string()
.optional()
.description(DESCRIPTION.customerId),
reason: isA.string().valid(...ReasonForDeletionValues),
reason: validators.reasonForAccountDeletion,
}),
},
},
handler: (request: AuthRequest) =>
cloudTaskHandler.deleteAccount(
request.payload as DeleteAccountTaskPayload
),
cloudTaskHandler.deleteAccount(request.payload as DeleteAccountTask),
},
];

Просмотреть файл

@ -35,6 +35,7 @@ const {
const {
VX_REGEX: CLIENT_SALT_STRING,
} = require('../../lib/routes/utils/client-key-stretch');
const { ReasonForDeletion } = require('@fxa/shared/cloud-tasks');
// Match any non-empty hex-encoded string.
const HEX_STRING = /^(?:[a-fA-F0-9]{2})+$/;
@ -121,6 +122,9 @@ module.exports.uid = module.exports.hexString.length(32);
module.exports.clientId = module.exports.hexString.length(16);
module.exports.clientSecret = module.exports.hexString;
module.exports.idToken = module.exports.jwt;
module.exports.reasonForAccountDeletion = isA
.string()
.valid(...Object.values(ReasonForDeletion));
module.exports.refreshToken = module.exports.hexString.length(64);
module.exports.sessionToken = module.exports.hexString.length(64);
module.exports.sessionTokenId = module.exports.hexString.length(64);

Просмотреть файл

@ -3,27 +3,28 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { Command } from 'commander';
import { AccountCustomers } from 'fxa-shared/db/models/auth';
import {
AccountCustomers,
getAccountCustomerByUid,
} from 'fxa-shared/db/models/auth';
import { StatsD } from 'hot-shots';
import { Container } from 'typedi';
import appConfig from '../config';
import {
AccountDeleteManager,
ReasonForDeletionOptions,
} from '../lib/account-delete';
import * as random from '../lib/crypto/random';
import DB from '../lib/db';
import { setupFirestore } from '../lib/firestore-db';
import initLog from '../lib/log';
import oauthDb from '../lib/oauth/db';
import { CurrencyHelper } from '../lib/payments/currencies';
import { createStripeHelper, StripeHelper } from '../lib/payments/stripe';
import { pushboxApi } from '../lib/pushbox';
import initRedis from '../lib/redis';
import Token from '../lib/tokens';
import { AppConfig, AuthFirestore, AuthLogger } from '../lib/types';
import { parseDryRun } from './lib/args';
import {
AccountTasksFactory,
ReasonForDeletion,
} from '@fxa/shared/cloud-tasks';
const dryRun = async (program: Command, limit: number) => {
const countQuery = AccountCustomers.query()
@ -66,7 +67,7 @@ const init = async () => {
program.parse(process.argv);
const isDryRun = parseDryRun(program.dryRun);
const limit = program.limit ? parseInt(program.limit) : Infinity;
const reason = ReasonForDeletionOptions.Cleanup;
const reason = ReasonForDeletion.Cleanup;
if (limit <= 0) {
throw new Error('The limit should be a positive integer.');
@ -88,7 +89,7 @@ const init = async () => {
random.base32(config.signinUnblock.codeLength) as any // TS type inference is failing pretty hard with this
);
// connect to db here so the dry run could get a row count
const fxaDb = await db.connect(config, redis);
await db.connect(config, redis);
if (isDryRun) {
console.log(
@ -98,7 +99,6 @@ const init = async () => {
return dryRun(program, limit);
}
const pushbox = pushboxApi(log, config, statsd);
Container.set(AppConfig, config);
Container.set(AuthLogger, log);
const authFirestore = setupFirestore(config);
@ -108,14 +108,7 @@ const init = async () => {
const stripeHelper = createStripeHelper(log, config, statsd);
Container.set(StripeHelper, stripeHelper);
const accountDeleteManager = new AccountDeleteManager({
fxaDb,
oauthDb,
config,
push: {} as any, // Push isn't needed for enqueuing
pushbox,
statsd,
});
const accountTasks = AccountTasksFactory(config, statsd);
const query = AccountCustomers.query()
.select({ uid: 'accountCustomers.uid' })
@ -131,7 +124,11 @@ const init = async () => {
const rows = await query;
for (const x of rows) {
const result = await accountDeleteManager.enqueue({ uid: x.uid, reason });
const result = await accountTasks.deleteAccount({
uid: x.uid,
customerId: (await getAccountCustomerByUid(x.uid))?.stripeCustomerId,
reason,
});
console.log(`Created cloud task ${result} for uid ${x.uid}`);
}

Просмотреть файл

@ -8,31 +8,31 @@ import { parseDryRun } from './lib/args';
import { FieldPath, Firestore } from '@google-cloud/firestore';
import { AppConfig, AuthFirestore, AuthLogger } from '../lib/types';
import Container from 'typedi';
import {
AccountDeleteManager,
ReasonForDeletionOptions,
} from '../lib/account-delete';
import { ConfigType } from '../config';
import oauthDb from '../lib/oauth/db';
import { StatsD } from 'hot-shots';
import pushboxApi from '../lib/pushbox';
import { setupAccountDatabase } from '@fxa/shared/db/mysql/account';
import { AccountManager } from '@fxa/shared/account/account';
import { uuidTransformer } from 'packages/fxa-shared/db/transformers';
import * as pckg from '../package.json';
import {
AccountTasks,
ReasonForDeletion,
AccountTasksFactory,
} from '@fxa/shared/cloud-tasks';
import { getAccountCustomerByUid } from 'fxa-shared/db/models/auth';
class CleanupFirestoreHelper {
private firestore: Firestore;
private log: AuthLogger;
private config: ConfigType;
private accountDeleteManager: AccountDeleteManager;
private accountTasks: AccountTasks;
private accountManager: AccountManager;
constructor(private batchSize: number, private dryRun: boolean) {
this.firestore = Container.get<Firestore>(AuthFirestore);
this.log = Container.get(AuthLogger);
this.config = Container.get(AppConfig);
this.accountDeleteManager = Container.get(AccountDeleteManager);
this.accountTasks = Container.get(AccountTasks);
this.accountManager = Container.get(AccountManager);
}
@ -104,10 +104,11 @@ class CleanupFirestoreHelper {
}
await Promise.all(
uids.map((uid) =>
this.accountDeleteManager.enqueue({
uids.map(async (uid) =>
this.accountTasks.deleteAccount({
uid,
reason: ReasonForDeletionOptions.Cleanup,
customerId: (await getAccountCustomerByUid(uid))?.stripeCustomerId,
reason: ReasonForDeletion.Cleanup,
})
)
);
@ -153,25 +154,14 @@ export async function init() {
const batchSize = parseInt(options.batchSize);
const isDryRun = parseDryRun(options.dryRun);
const { database: fxaDb } = await setupProcessingTaskObjects(
'cleanup-delete-partial-firestore'
);
// TBD, do we still need this? fxaDb is no longer referenced...
await setupProcessingTaskObjects('cleanup-delete-partial-firestore');
const config = Container.get(AppConfig);
const statsd = Container.get(StatsD);
const log = Container.get(AuthLogger);
const pushbox = pushboxApi(log, config, statsd);
const accountDeleteManager = new AccountDeleteManager({
fxaDb,
oauthDb,
config,
push: {} as any,
pushbox,
statsd,
});
Container.set(AccountDeleteManager, accountDeleteManager);
const accountTasks = AccountTasksFactory(config, statsd);
Container.set(AccountTasks, accountTasks);
const accountDb = await setupAccountDatabase(config.database.mysql.auth);
const accountManager = new AccountManager(accountDb);

Просмотреть файл

@ -37,6 +37,10 @@ import {
AuthLogger,
ProfileClient,
} from '../lib/types';
import {
AccountTasks,
AccountTasksFactory,
} from '../../../libs/shared/cloud-tasks/src';
const config = configProperties.getProperties();
const mailer = null;
@ -121,13 +125,16 @@ DB.connect(config).then(async (db: any) => {
Container.set(PayPalHelper, paypalHelper);
}
const accountTasks = AccountTasksFactory(config, statsd);
Container.set(AccountTasks, accountTasks);
const accountDeleteManager = new AccountDeleteManager({
fxaDb: db,
oauthDb: oauthDB,
config,
push,
pushbox,
statsd,
} as any);
});
Container.set(AccountDeleteManager, accountDeleteManager);
// Load the account-deletion route, so we can use its logic directly.

Просмотреть файл

@ -8,25 +8,23 @@ import PQueue from 'p-queue';
import { Container } from 'typedi';
import { setupAccountDatabase } from '@fxa/shared/db/mysql/account';
import { CloudTasksClient } from '@google-cloud/tasks';
import appConfig from '../config';
import {
AccountDeleteManager,
ReasonForDeletionOptions,
} from '../lib/account-delete';
import * as random from '../lib/crypto/random';
import DB from '../lib/db';
import { setupFirestore } from '../lib/firestore-db';
import initLog from '../lib/log';
import oauthDb from '../lib/oauth/db';
import { CurrencyHelper } from '../lib/payments/currencies';
import { createStripeHelper, StripeHelper } from '../lib/payments/stripe';
import { pushboxApi } from '../lib/pushbox';
import initRedis from '../lib/redis';
import Token from '../lib/tokens';
import { AppConfig, AuthFirestore, AuthLogger } from '../lib/types';
import { parseDryRun } from './lib/args';
import {
AccountTasksFactory,
ReasonForDeletion,
} from '@fxa/shared/cloud-tasks';
import { getAccountCustomerByUid } from 'fxa-shared/db/models/auth';
const collect = () => (val: string, xs: string[]) => {
xs.push(val);
@ -129,7 +127,7 @@ const init = async () => {
const hasEmail = program.email.length > 0;
const hasDateRange =
program.startDate && program.endDate && program.endDate > program.startDate;
const reason = ReasonForDeletionOptions.Unverified;
const reason = ReasonForDeletion.Unverified;
const taskLimit = program.taskEnqueueLimit
? parseInt(program.taskEnqueueLimit)
: 200;
@ -175,7 +173,6 @@ const init = async () => {
random.base32(config.signinUnblock.codeLength) as any // TS type inference is failing pretty hard with this
);
const fxaDb = await db.connect(config, redis);
const pushbox = pushboxApi(log, config, statsd);
Container.set(AppConfig, config);
Container.set(AuthLogger, log);
@ -187,22 +184,7 @@ const init = async () => {
const stripeHelper = createStripeHelper(log, config, statsd);
Container.set(StripeHelper, stripeHelper);
const accountDeleteManager = new AccountDeleteManager({
fxaDb,
oauthDb,
config,
push: {} as any, // Not needed when enqueuing
pushbox,
statsd,
});
// Replace the client with one that uses the fallback to HTTP for higher concurrency
const tasksConfig = config.cloudTasks;
accountDeleteManager['cloudTasksClient'] = new CloudTasksClient({
projectId: tasksConfig.projectId,
keyFilename: tasksConfig.credentials.keyFilename ?? undefined,
fallback: true,
});
const accountTasks = AccountTasksFactory(config, statsd);
if (useSpecifiedAccounts) {
const { uids, emails } = limitSpecifiedAccounts(program, limit);
@ -213,7 +195,11 @@ const init = async () => {
console.error(`Account with uid ${x} is verified. Skipping.`);
continue;
}
const result = await accountDeleteManager.enqueue({ uid: x, reason });
const result = await accountTasks.deleteAccount({
uid: x,
customerId: (await getAccountCustomerByUid(x))?.stripeCustomerId,
reason,
});
console.log(`Created cloud task ${result} for uid ${x}`);
}
@ -223,7 +209,11 @@ const init = async () => {
console.error(`Account with email ${x} is verified. Skipping.`);
continue;
}
const result = await accountDeleteManager.enqueue({ email: x, reason });
const result = await accountTasks.deleteAccount({
uid: acct.uid,
customerId: (await getAccountCustomerByUid(acct.uid))?.stripeCustomerId,
reason,
});
console.log(`Created cloud task ${result} for uid ${x}`);
}
}
@ -272,7 +262,7 @@ const init = async () => {
queue.add(async () => {
try {
const result = await accountDeleteManager['enqueueTask']({
const result = await accountTasks.deleteAccount({
uid: row.uid.toString('hex'),
customerId: row.stripeCustomerId || undefined,
reason,

Просмотреть файл

@ -44,14 +44,12 @@ describe('AccountDeleteManager', function () {
let mockConfig;
let accountDeleteManager;
let mockAuthModels;
let createTaskStub;
beforeEach(() => {
const { PayPalHelper } = require('../../lib/payments/paypal/helper');
const { StripeHelper } = require('../../lib/payments/stripe');
sandbox.reset();
createTaskStub = sandbox.stub();
mockFxaDb = {
...mocks.mockDB({ email: email, uid: uid }),
fetchAccountSubscriptions: sinon.spy(
@ -107,7 +105,7 @@ describe('AccountDeleteManager', function () {
return [{ status: 'Active', billingAgreementId: 'B-test' }];
});
mockAuthModels.deleteAllPayPalBAs = sinon.spy(async () => {});
mockAuthModels.getAccountCustomerByUid = sinon.spy(async () => {
mockAuthModels.getAccountCustomerByUid = sinon.spy(async (...args) => {
return { stripeCustomerId: 'cus_993' };
});
@ -124,14 +122,6 @@ describe('AccountDeleteManager', function () {
Container.set(PlayBilling, mockPlayBilling);
const { AccountDeleteManager } = proxyquire('../../lib/account-delete', {
'@google-cloud/tasks': {
CloudTasksClient: class CloudTasksClient {
constructor() {}
createTask(...args) {
return createTaskStub.apply(null, args);
}
},
},
'fxa-shared/db/models/auth': mockAuthModels,
});
@ -153,104 +143,6 @@ describe('AccountDeleteManager', function () {
assert.ok(accountDeleteManager);
});
describe('create tasks', function () {
it('creates a delete account task by uid', async () => {
const taskId = 'proj/testo/loc/us-n/q/del0/tasks/123';
createTaskStub = sandbox.stub().resolves([{ name: taskId }]);
const result = await accountDeleteManager.enqueue({
uid,
reason: 'fxa_unverified_account_delete',
});
sinon.assert.calledOnceWithExactly(
mockAuthModels.getAccountCustomerByUid,
uid
);
sinon.assert.calledOnceWithExactly(
mockStatsd.increment,
'cloud-tasks.account-delete.enqueue.success'
);
sinon.assert.calledOnceWithExactly(createTaskStub, {
parent: `projects/${mockConfig.cloudTasks.projectId}/locations/${mockConfig.cloudTasks.locationId}/queues/${mockConfig.cloudTasks.deleteAccounts.queueName}`,
task: {
httpRequest: {
url: `${mockConfig.publicUrl}/v${mockConfig.apiVersion}/cloud-tasks/accounts/delete`,
httpMethod: 'POST',
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(
JSON.stringify({
uid,
customerId: 'cus_993',
reason: 'fxa_unverified_account_delete',
})
).toString('base64'),
oidcToken: {
audience: mockConfig.cloudTasks.oidc.aud,
serviceAccountEmail:
mockConfig.cloudTasks.oidc.serviceAccountEmail,
},
},
},
});
assert.equal(result, taskId);
});
it('creates a delete account task by email', async () => {
const taskId = 'proj/testo/loc/us-n/q/del0/tasks/134';
createTaskStub = sandbox.stub().resolves([{ name: taskId }]);
const result = await accountDeleteManager.enqueue({
email,
reason: 'fxa_unverified_account_delete',
});
sinon.assert.calledOnceWithExactly(
mockAuthModels.getAccountCustomerByUid,
uid
);
sinon.assert.calledOnceWithExactly(createTaskStub, {
parent: `projects/${mockConfig.cloudTasks.projectId}/locations/${mockConfig.cloudTasks.locationId}/queues/${mockConfig.cloudTasks.deleteAccounts.queueName}`,
task: {
httpRequest: {
url: `${mockConfig.publicUrl}/v${mockConfig.apiVersion}/cloud-tasks/accounts/delete`,
httpMethod: 'POST',
headers: { 'Content-Type': 'application/json' },
body: Buffer.from(
JSON.stringify({
uid,
customerId: 'cus_993',
reason: 'fxa_unverified_account_delete',
})
).toString('base64'),
oidcToken: {
audience: mockConfig.cloudTasks.oidc.aud,
serviceAccountEmail:
mockConfig.cloudTasks.oidc.serviceAccountEmail,
},
},
},
});
assert.equal(result, taskId);
});
it('throws when task creation fails', async () => {
const fetchCustomerStub = sandbox.stub().resolves({ id: 'cus_997' });
mockStripeHelper['fetchCustomer'] = fetchCustomerStub;
createTaskStub = sandbox.stub().throws();
try {
await accountDeleteManager.enqueue({
uid,
reason: 'fxa_unverified_account_delete',
});
assert.fail('An error should have been thrown.');
} catch (err) {
sinon.assert.calledOnceWithExactly(
mockStatsd.increment,
'cloud-tasks.account-delete.enqueue.failure'
);
assert.instanceOf(err, Error);
}
});
});
describe('delete account', function () {
it('should delete the account', async () => {
mockPush.notifyAccountDestroyed = sinon.fake.resolves();
@ -346,17 +238,12 @@ describe('AccountDeleteManager', function () {
});
describe('quickDelete', () => {
it('should delete the account and queue', async () => {
createTaskStub = sandbox.stub().resolves([{ name: 'test' }]);
it('should delete the account', async () => {
await accountDeleteManager.quickDelete(uid, deleteReason);
sinon.assert.calledWithMatch(mockFxaDb.deleteAccount, {
uid,
});
sinon.assert.calledOnceWithExactly(
mockAuthModels.getAccountCustomerByUid,
uid
);
sinon.assert.calledOnceWithExactly(mockOAuthDb.removeTokensAndCodes, uid);
});
@ -368,25 +255,6 @@ describe('AccountDeleteManager', function () {
assert.match(err.message, /^quickDelete only supports user/);
}
});
it('should enqueue if an error happens during delete', async () => {
createTaskStub = sandbox.stub().resolves([{ name: 'test' }]);
mockFxaDb.deleteAccount = sandbox.stub().throws();
await accountDeleteManager.quickDelete(uid, deleteReason);
sinon.assert.calledWithMatch(mockFxaDb.deleteAccount, {
uid,
});
sinon.assert.calledOnceWithExactly(
mockAuthModels.getAccountCustomerByUid,
uid
);
sinon.assert.callCount(mockOAuthDb.removeTokensAndCodes, 0);
sinon.assert.calledOnceWithExactly(
mockStatsd.increment,
'cloud-tasks.account-delete.enqueue.success'
);
});
});
describe('refundSubscriptions', () => {

Просмотреть файл

@ -10,6 +10,7 @@ const assert = { ...sinon.assert, ...require('chai').assert };
const mocks = require('../../mocks');
const getRoute = require('../../routes_helpers').getRoute;
const proxyquire = require('proxyquire');
const { AccountTasks, ReasonForDeletion } = require('@fxa/shared/cloud-tasks');
const uuid = require('uuid');
const crypto = require('crypto');
@ -19,10 +20,7 @@ const otplib = require('otplib');
const { Container } = require('typedi');
const { CapabilityService } = require('../../../lib/payments/capability');
const { AccountEventsManager } = require('../../../lib/account-events');
const {
AccountDeleteManager,
ReasonForDeletionOptions,
} = require('../../../lib/account-delete');
const { AccountDeleteManager } = require('../../../lib/account-delete');
const { normalizeEmail } = require('fxa-shared').email.helpers;
const { MozillaSubscriptionTypes } = require('fxa-shared/subscriptions/types');
const {
@ -44,7 +42,11 @@ function hexString(bytes) {
return crypto.randomBytes(bytes).toString('hex');
}
const mockAccountQuickDelete = sinon.fake.resolves();
let mockAccountQuickDelete = sinon.fake.resolves();
let mockAccountTasksDeleteAccount = sinon.fake(async (...args) => {});
const mockGetAccountCustomerByUid = sinon.fake.resolves({
stripeCustomerId: 'customer123',
});
const makeRoutes = function (options = {}, requireMocks = {}) {
Container.set(CapabilityService, options.capabilityService || sinon.fake);
@ -95,10 +97,12 @@ const makeRoutes = function (options = {}, requireMocks = {}) {
options.verificationReminders || mocks.mockVerificationReminders();
const subscriptionAccountReminders =
options.subscriptionAccountReminders || mocks.mockVerificationReminders();
const { accountRoutes } = proxyquire(
'../../../lib/routes/account',
requireMocks || {}
);
const { accountRoutes } = proxyquire('../../../lib/routes/account', {
...(requireMocks || {}),
'fxa-shared/db/models/auth': {
getAccountCustomerByUid: mockGetAccountCustomerByUid,
},
});
const signupUtils =
options.signupUtils ||
require('../../../lib/routes/utils/signup')(
@ -116,11 +120,20 @@ const makeRoutes = function (options = {}, requireMocks = {}) {
...(options.oauth || {}),
};
mockAccountTasksDeleteAccount = sinon.fake.resolves();
const accountTasks = {
deleteAccount: mockAccountTasksDeleteAccount,
queueEnabled: true,
};
Container.set(AccountTasks, accountTasks);
// We have to do some redirection with proxyquire because dependency
// injection changes the class
const AccountDeleteManagerMock = proxyquire('../../../lib/account-delete', {
'fxa-shared/db/models/auth':
requireMocks['fxa-shared/db/models/auth'] || {},
'fxa-shared/db/models/auth': {
...(requireMocks['fxa-shared/db/models/auth'] || {}),
getAccountCustomerByUid: mockGetAccountCustomerByUid,
},
});
const accountManagerMock = new AccountDeleteManagerMock.AccountDeleteManager({
fxaDb: db,
@ -128,10 +141,11 @@ const makeRoutes = function (options = {}, requireMocks = {}) {
config,
push,
pushbox,
statsd: { increment: sinon.fake.returns({}) },
});
mockAccountQuickDelete = sinon.fake(async (...args) => {
return {};
});
accountManagerMock.quickDelete = mockAccountQuickDelete;
mockAccountQuickDelete.resetHistory();
Container.set(AccountDeleteManager, accountManagerMock);
return accountRoutes(
@ -3810,7 +3824,7 @@ describe('/account/destroy', () => {
return getRoute(accountRoutes, '/account/destroy');
}
it('should delete the account', () => {
it('should delete the account and enqueue account task', () => {
const route = buildRoute();
return runTest(route, mockRequest, () => {
@ -3823,11 +3837,43 @@ describe('/account/destroy', () => {
userAgent: 'test user-agent',
uid: uid,
});
sinon.assert.calledOnceWithExactly(
mockAccountQuickDelete,
uid,
ReasonForDeletionOptions.UserRequested
ReasonForDeletion.UserRequested
);
sinon.assert.calledOnceWithExactly(mockGetAccountCustomerByUid, uid);
sinon.assert.calledOnceWithExactly(mockAccountTasksDeleteAccount, {
uid,
customerId: 'customer123',
reason: ReasonForDeletion.UserRequested,
});
});
});
it('should delete the account and enqueue account task on error', () => {
const route = buildRoute();
// Here we act like there's an error when calling accountDeleteManager.quickDelete(...)
mockAccountQuickDelete = sinon.fake.rejects();
return runTest(route, mockRequest, () => {
sinon.assert.calledOnceWithExactly(mockDB.accountRecord, email);
sinon.assert.calledOnceWithExactly(mockLog.activityEvent, {
country: 'United States',
event: 'account.deleted',
region: 'California',
service: undefined,
userAgent: 'test user-agent',
uid: uid,
});
sinon.assert.calledOnceWithExactly(mockAccountTasksDeleteAccount, {
uid,
customerId: 'customer123',
reason: ReasonForDeletion.UserRequested,
});
});
});
@ -3846,7 +3892,7 @@ describe('/account/destroy', () => {
sinon.assert.calledOnceWithExactly(
mockAccountQuickDelete,
uid,
ReasonForDeletionOptions.UserRequested
ReasonForDeletion.UserRequested
);
});
});

Просмотреть файл

@ -8,11 +8,8 @@ const sinon = require('sinon');
const mocks = require('../../mocks');
const getRoute = require('../../routes_helpers').getRoute;
const { cloudTaskRoutes } = require('../../../lib/routes/cloud-tasks');
const {
AccountDeleteManager,
ReasonForDeletionOptions,
} = require('../../../lib/account-delete');
const { AccountDeleteManager } = require('../../../lib/account-delete');
const { ReasonForDeletion } = require('@fxa/shared/cloud-tasks');
const mockConfig = {
cloudTasks: {
deleteAccounts: { queueName: 'del-accts' },
@ -45,7 +42,7 @@ describe('/cloud-tasks/accounts/delete', () => {
it('should delete the account', async () => {
try {
const req = {
payload: { uid, reason: ReasonForDeletionOptions.Unverified },
payload: { uid, reason: ReasonForDeletion.Unverified },
};
await route.handler(req);

Просмотреть файл

@ -11,6 +11,7 @@ const plan1 = require('../payments/fixtures/stripe/plan1.json');
const validProductMetadata = plan1.product.metadata;
const { MozillaSubscriptionTypes } = require('fxa-shared/subscriptions/types');
const { deepCopy } = require('../payments/util');
const { ReasonForDeletion } = require('@fxa/shared/cloud-tasks');
describe('lib/routes/validators:', () => {
it('isValidEmailAddress returns true for valid email addresses', () => {
@ -1138,4 +1139,27 @@ describe('lib/routes/validators:', () => {
assert.exists(validators.recoveryCode(11).validate('123456').error);
});
});
describe('reason for account deletion', () => {
it('validates valid reason', () => {
assert.notExists(
validators.reasonForAccountDeletion.validate(ReasonForDeletion.Cleanup)
.error
);
assert.notExists(
validators.reasonForAccountDeletion.validate(
ReasonForDeletion.UserRequested
).error
);
assert.notExists(
validators.reasonForAccountDeletion.validate(
ReasonForDeletion.Unverified
).error
);
});
it('requires valid reason', () => {
assert.exists(validators.reasonForAccountDeletion.validate('blah').error);
});
});
});

Просмотреть файл

@ -19,10 +19,8 @@ const {
} = require('../../lib/payments/iap/apple-app-store/subscriptions');
// Note, intentionally not indenting for code review.
[{version:""},{version:"V2"}].forEach((testOptions) => {
describe(`#integration${testOptions.version} - remote account create`, function () {
[{ version: '' }, { version: 'V2' }].forEach((testOptions) => {
describe(`#integration${testOptions.version} - remote account create`, function () {
this.timeout(50000);
let server;
before(async () => {
@ -205,7 +203,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
const client = new Client(config.publicUrl, testOptions);
await client.setupCredentials(email, password);
if (testOptions.version === "V2") {
if (testOptions.version === 'V2') {
await client.setupCredentialsV2(email, password);
}
@ -214,16 +212,23 @@ describe(`#integration${testOptions.version} - remote account create`, function
assert.exists(stubResponse.setup_token);
// Finish the setup.
const response = await client.finishAccountSetup(stubResponse.setup_token)
const response = await client.finishAccountSetup(
stubResponse.setup_token
);
assert.exists(response.uid);
assert.exists(response.sessionToken);
assert.exists(response.verified);
assert.isFalse(response.verified);
// Now a client should be able login
const client2 = await Client.login(config.publicUrl, email, password, testOptions);
assert.exists(client2.sessionToken)
})
const client2 = await Client.login(
config.publicUrl,
email,
password,
testOptions
);
assert.exists(client2.sessionToken);
});
it('cannot stub the same account twice', async () => {
const email = server.uniqueEmail();
@ -232,7 +237,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
const client = new Client(config.publicUrl, testOptions);
await client.setupCredentials(email, password);
if (testOptions.version === "V2") {
if (testOptions.version === 'V2') {
await client.setupCredentialsV2(email, password);
}
@ -244,7 +249,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
// stubbed
await stub();
assert.isRejected(stub());
})
});
it('fails to create account with a corrupt setup token', async () => {
const email = server.uniqueEmail();
@ -252,7 +257,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
const client = new Client(config.publicUrl, testOptions);
await client.setupCredentials(email, password);
if (testOptions.version === "V2") {
if (testOptions.version === 'V2') {
await client.setupCredentialsV2(email, password);
}
@ -260,11 +265,13 @@ describe(`#integration${testOptions.version} - remote account create`, function
assert.exists(stubResponse.setup_token);
// modify the setup token and make sure it fails
stubResponse.setup_token = stubResponse.setup_token.toString().substring(2);
stubResponse.setup_token = stubResponse.setup_token
.toString()
.substring(2);
// Finish the setup. Should fail because the setup token is bad
assert.isRejected(client.finishAccountSetup(stubResponse.setup_token))
})
assert.isRejected(client.finishAccountSetup(stubResponse.setup_token));
});
it('fails to call finish setup again', async () => {
const email = server.uniqueEmail();
@ -272,7 +279,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
const client = new Client(config.publicUrl, testOptions);
await client.setupCredentials(email, password);
if (testOptions.version === "V2") {
if (testOptions.version === 'V2') {
await client.setupCredentialsV2(email, password);
}
@ -281,7 +288,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
//Should fail because finish account setup was already called
assert.isRejected(client.finishAccountSetup(stubResponse.setup_token));
})
});
it('/account/create works with proper data', () => {
const email = server.uniqueEmail();
@ -819,10 +826,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
return server.mailbox.waitForCode(email);
})
.then((code) => {
assert.ok(
code,
'the next email was reset-password, not post-verify'
);
assert.ok(code, 'the next email was reset-password, not post-verify');
});
});
@ -861,10 +865,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
return server.mailbox.waitForCode(email);
})
.then((code) => {
assert.ok(
code,
'the next email was reset-password, not post-verify'
);
assert.ok(code, 'the next email was reset-password, not post-verify');
});
});
@ -906,10 +907,8 @@ describe(`#integration${testOptions.version} - remote account create`, function
});
});
it('maintains single kB value for account create with V1 & V2 credentials', async function () {
if (testOptions.version !== "V2") {
if (testOptions.version !== 'V2') {
return this.skip();
}
@ -938,23 +937,20 @@ describe(`#integration${testOptions.version} - remote account create`, function
await clientV1.getKeysV1();
const kB1 = clientV1.kB;
const clientV2 = await login(email, password, "V2");
const clientV2 = await login(email, password, 'V2');
await clientV2.getKeysV2();
const kB2 = clientV2.kB;
assert.exists(originalKb);
assert.isTrue(
clientSalt.startsWith(
'identity.mozilla.com/picl/v1/quickStretchV2:'
)
clientSalt.startsWith('identity.mozilla.com/picl/v1/quickStretchV2:')
);
assert.equal(kB1, originalKb);
assert.equal(kB2, originalKb);
});
it('maintains single kB value after account password upgrade from V1 to V2', async function () {
if (testOptions.version !== "V2") {
if (testOptions.version !== 'V2') {
return this.skip();
}
@ -984,7 +980,7 @@ describe(`#integration${testOptions.version} - remote account create`, function
const kB1 = clientV1.kB;
const clientV2 = await login(email, password, "V2");
const clientV2 = await login(email, password, 'V2');
await clientV2.getKeysV2();
const kB2 = clientV2.kB;
@ -993,24 +989,17 @@ describe(`#integration${testOptions.version} - remote account create`, function
assert.equal(kB2, originalKb);
});
after(async () => {
await TestServer.stop(server);
});
async function login(email, password, version="") {
return await Client.login(
config.publicUrl,
email,
password,
{
async function login(email, password, version = '') {
return await Client.login(config.publicUrl, email, password, {
...testOptions,
version,
keys: true,
service: 'sync',
});
}
);
}
});
});
});