Port breach alerts cronjob to TS
This job uses appConstants, and ESBuild bundles everything into a single module. That means that resolving the .env files relative to the dirname would fail for the cronjob. Thus, I reverted that back to have dotenv-flow autodetect the .env file, and migrated the DB script (for which the dirname-relative resolving was added) to use process.env directly instead.
This commit is contained in:
Vincent
Vincent
Родитель
85841725e0
Коммит
309ed8f39b
|
|
@ -145,6 +145,8 @@ Monitor uses GCP PubSub for processing incoming breach data, this can be tested
|
|||
gcloud beta emulators pubsub start --project=your-project-name
|
||||
```
|
||||
|
||||
(Set `your-project-name` as the value for `GCP_PUBSUB_PROJECT_ID` in your `.env.local`.)
|
||||
|
||||
### In a different shell, set the environment to point at the emulator and run Monitor in dev mode:
|
||||
|
||||
```sh
|
||||
|
|
@ -160,10 +162,13 @@ curl -d '{ "breachName": "000webhost", "hashPrefix": "test", "hashSuffixes": ["t
|
|||
http://localhost:6060/api/v1/hibp/notify
|
||||
```
|
||||
|
||||
This emulates HIBP notifying our API that a new breach was found. Our API will
|
||||
then add it to the (emulated) pubsub queue.
|
||||
|
||||
### This pubsub queue will be consumed by this cron job, which is responsible for looking up and emailing impacted users:
|
||||
|
||||
```sh
|
||||
node src/scripts/emailBreachAlerts.js
|
||||
npm run dev:cron:breach-alerts
|
||||
```
|
||||
|
||||
### Emails
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
"dev": "npm run build-nimbus && next dev --port=6060",
|
||||
"dev:cron:first-data-broker-removal-fixed": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/firstDataBrokerRemovalFixed.tsx",
|
||||
"dev:cron:monthly-activity": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/monthlyActivity.tsx",
|
||||
"dev:cron:breach-alerts": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/emailBreachAlerts.ts",
|
||||
"dev:cron:db-delete-unverified-subscribers": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/deleteUnverifiedSubscribers.ts",
|
||||
"dev:cron:db-pull-breaches": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/syncBreaches.ts",
|
||||
"dev:cron:remote-settings-pull-breaches": "tsx --tsconfig tsconfig.cronjobs.json src/scripts/cronjobs/updateBreachesInRemoteSettings.ts",
|
||||
|
|
@ -27,7 +28,7 @@
|
|||
"e2e:smoke": "playwright test src/e2e/ --grep @smoke",
|
||||
"cron:first-data-broker-removal-fixed": "node dist/scripts/cronjobs/firstDataBrokerRemovalFixed.js",
|
||||
"cron:monthly-activity": "node dist/scripts/cronjobs/monthlyActivity.js",
|
||||
"cron:breach-alerts": "node src/scripts/emailBreachAlerts.js",
|
||||
"cron:breach-alerts": "node dist/scripts/cronjobs/emailBreachAlerts.js",
|
||||
"cron:db-delete-unverified-subscribers": "node dist/scripts/cronjobs/deleteUnverifiedSubscribers.js",
|
||||
"cron:db-pull-breaches": "node dist/scripts/cronjobs/syncBreaches.js",
|
||||
"cron:remote-settings-pull-breaches": "node dist/scripts/cronjobs/updateBreachesInRemoteSettings.js",
|
||||
|
|
|
|||
|
|
@ -2,21 +2,14 @@
|
|||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
// TODO: these vars were copy/pasted from the old app-constants.js and should be cleaned up
|
||||
import path from "path";
|
||||
import url from "url";
|
||||
|
||||
if (typeof process.env.NEXT_RUNTIME === "undefined" && typeof process.env.STORYBOOK === "undefined") {
|
||||
// Next.js already loads env vars by itself, and dotenv-flow will throw an
|
||||
// error if loaded in that context (about `fs` not existing), so only load
|
||||
// it if we're not running in a Next.js-context (e.g. cron jobs):
|
||||
const __filename = url.fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const dotenvFlow = await import("dotenv-flow");
|
||||
dotenvFlow.config({ path: path.resolve(__dirname, "../") });
|
||||
await import("dotenv-flow/config");
|
||||
}
|
||||
|
||||
// TODO: these vars were copy/pasted from the old app-constants.js and should be cleaned up
|
||||
const requiredEnvVars = [
|
||||
'ADMINS',
|
||||
'APP_ENV',
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
// `pg-connection-string` works, triggering a false positive for this lint rule:
|
||||
/* eslint-disable import/no-named-as-default-member */
|
||||
import pgConnectionStr from "pg-connection-string";
|
||||
import AppConstants from "../appConstants.js";
|
||||
import "dotenv-flow/config";
|
||||
|
||||
/**
|
||||
* @typedef {object} KnexConfig
|
||||
|
|
@ -14,9 +14,12 @@ import AppConstants from "../appConstants.js";
|
|||
* @property {import("pg-connection-string").ConnectionOptions} connection
|
||||
*/
|
||||
|
||||
const { DATABASE_URL, APP_ENV, NODE_ENV, PG_HOST } = AppConstants;
|
||||
const DATABASE_URL = process.env.DATABASE_URL ?? "";
|
||||
const APP_ENV = process.env.APP_ENV ?? "production";
|
||||
/** @type {string} */
|
||||
const NODE_ENV = process.env.NODE_ENV ?? "production";
|
||||
const connectionObj = pgConnectionStr.parse(DATABASE_URL);
|
||||
if (APP_ENV === "heroku") {
|
||||
if (typeof process.env.APP_ENV === "string" && process.env.APP_ENV === "heroku") {
|
||||
// @ts-ignore TODO: Check if this typing error is correct, or if the types are wrong?
|
||||
connectionObj.ssl = { rejectUnauthorized: false };
|
||||
}
|
||||
|
|
@ -40,7 +43,7 @@ let exportConfig = NODE_ENV === "tests" ? TESTS_CONFIG : RUNTIME_CONFIG
|
|||
if (APP_ENV === "cloudrun") {
|
||||
// @ts-ignore TODO: Check if this typing error is correct, or if the types are wrong?
|
||||
connectionObj.ssl = false;
|
||||
connectionObj.host = PG_HOST
|
||||
connectionObj.host = /** @type {string} */ (process.env.PG_HOST)
|
||||
exportConfig = {
|
||||
client: "pg",
|
||||
connection: connectionObj
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@
|
|||
|
||||
import { test, expect, jest } from "@jest/globals";
|
||||
|
||||
process.env.GCP_PUBSUB_PROJECT_ID = "arbitrary-id";
|
||||
process.env.GCP_PUBSUB_SUBSCRIPTION_NAME = "arbitrary-name";
|
||||
|
||||
jest.mock("@sentry/nextjs", () => {
|
||||
return {
|
||||
init: jest.fn(),
|
||||
|
|
@ -11,7 +14,7 @@ jest.mock("@sentry/nextjs", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../utils/email.js", () => {
|
||||
jest.mock("../../utils/email.js", () => {
|
||||
return {
|
||||
initEmail: jest.fn(),
|
||||
EmailTemplateType: jest.fn(),
|
||||
|
|
@ -20,7 +23,7 @@ jest.mock("../utils/email.js", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../utils/hibp.js", () => {
|
||||
jest.mock("../../utils/hibp.js", () => {
|
||||
return {
|
||||
getAddressesAndLanguageForEmail: jest.fn(() => {
|
||||
return {
|
||||
|
|
@ -34,19 +37,19 @@ jest.mock("../utils/hibp.js", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/subscribers.js", () => {
|
||||
jest.mock("../../db/tables/subscribers.js", () => {
|
||||
return {
|
||||
getSubscribersByHashes: jest.fn(() => [""]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/emailAddresses.js", () => {
|
||||
jest.mock("../../db/tables/emailAddresses.js", () => {
|
||||
return {
|
||||
getEmailAddressesByHashes: jest.fn(() => [""]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/email_notifications.js", () => {
|
||||
jest.mock("../../db/tables/email_notifications.js", () => {
|
||||
return {
|
||||
getNotifiedSubscribersForBreach: jest.fn(() => [""]),
|
||||
addEmailNotification: jest.fn(),
|
||||
|
|
@ -54,7 +57,7 @@ jest.mock("../db/tables/email_notifications.js", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../utils/fluent.js", () => {
|
||||
jest.mock("../../utils/fluent.js", () => {
|
||||
return {
|
||||
initFluentBundles: jest.fn(),
|
||||
getMessage: jest.fn(),
|
||||
|
|
@ -62,24 +65,24 @@ jest.mock("../utils/fluent.js", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../emails/email2022.js", () => {
|
||||
jest.mock("../../emails/email2022.js", () => {
|
||||
return {
|
||||
getTemplate: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../emails/emailBreachAlert.js", () => {
|
||||
jest.mock("../../emails/emailBreachAlert.js", () => {
|
||||
return {
|
||||
breachAlertEmailPartial: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const subClient = {
|
||||
const subClient: any = {
|
||||
subscriptionPath: jest.fn(),
|
||||
acknowledge: jest.fn(),
|
||||
};
|
||||
|
||||
function buildReceivedMessages(testBreachAlert) {
|
||||
function buildReceivedMessages(testBreachAlert: any) {
|
||||
return [
|
||||
{
|
||||
ackId: "testAckId",
|
||||
|
|
@ -101,12 +104,14 @@ beforeEach(() => {
|
|||
});
|
||||
|
||||
test("rejects invalid messages", async () => {
|
||||
const { poll } = await import("./emailBreachAlerts.js");
|
||||
const { poll } = await import("./emailBreachAlerts");
|
||||
|
||||
const consoleError = jest
|
||||
.spyOn(console, "error")
|
||||
.mockImplementation(() => {});
|
||||
const consoleLog = jest.spyOn(console, "log").mockImplementation();
|
||||
const consoleLog = jest
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
await poll(
|
||||
subClient,
|
||||
|
|
@ -174,14 +179,19 @@ test("rejects invalid messages", async () => {
|
|||
});
|
||||
|
||||
test("processes valid messages", async () => {
|
||||
const consoleLog = jest.spyOn(console, "log").mockImplementation();
|
||||
const consoleLog = jest
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation(() => undefined);
|
||||
// It's not clear if the calls to console.info are important enough to remain,
|
||||
// but since they were already there when adding the "no logs" rule in tests,
|
||||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation();
|
||||
const { sendEmail } = await import("../utils/email.js");
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const emailMod = await import("../../utils/email.js");
|
||||
const sendEmail = emailMod.sendEmail as jest.Mock<
|
||||
(typeof emailMod)["sendEmail"]
|
||||
>;
|
||||
|
||||
const mockedUtilsHibp = jest.requireMock("../utils/hibp.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -195,7 +205,7 @@ test("processes valid messages", async () => {
|
|||
hashSuffixes: ["test-suffix1"],
|
||||
});
|
||||
|
||||
const { poll } = await import("./emailBreachAlerts.js");
|
||||
const { poll } = await import("./emailBreachAlerts");
|
||||
|
||||
await poll(subClient, receivedMessages);
|
||||
// Fabricated but valid breach is acknowledged.
|
||||
|
|
@ -265,13 +275,15 @@ test("processes valid messages", async () => {
|
|||
});
|
||||
|
||||
test("skipping email when subscriber id exists in email_notifications table", async () => {
|
||||
const consoleLog = jest.spyOn(console, "log").mockImplementation();
|
||||
const consoleLog = jest
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation(() => undefined);
|
||||
// It's not clear if the calls to console.info are important enough to remain,
|
||||
// but since they were already there when adding the "no logs" rule in tests,
|
||||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation();
|
||||
const { sendEmail } = await import("../utils/email.js");
|
||||
const mockedUtilsHibp = jest.requireMock("../utils/hibp.js");
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -280,19 +292,19 @@ test("skipping email when subscriber id exists in email_notifications table", as
|
|||
Id: 1,
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/subscribers.js", () => {
|
||||
jest.mock("../../db/tables/subscribers.js", () => {
|
||||
return {
|
||||
getSubscribersByHashes: jest.fn(() => [{ id: 1 }]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/emailAddresses.js", () => {
|
||||
jest.mock("../../db/tables/emailAddresses.js", () => {
|
||||
return {
|
||||
getEmailAddressesByHashes: jest.fn(() => []),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/email_notifications.js", () => {
|
||||
jest.mock("../../db/tables/email_notifications.js", () => {
|
||||
return {
|
||||
getNotifiedSubscribersForBreach: jest.fn(() => [1]),
|
||||
addEmailNotification: jest.fn(),
|
||||
|
|
@ -305,7 +317,7 @@ test("skipping email when subscriber id exists in email_notifications table", as
|
|||
hashSuffixes: ["test-suffix1"],
|
||||
});
|
||||
|
||||
const { poll } = await import("./emailBreachAlerts.js");
|
||||
const { poll } = await import("./emailBreachAlerts");
|
||||
|
||||
await poll(subClient, receivedMessages);
|
||||
// Verified, not fabricated, not spam list breaches are acknowledged.
|
||||
|
|
@ -318,13 +330,15 @@ test("skipping email when subscriber id exists in email_notifications table", as
|
|||
});
|
||||
|
||||
test("throws an error when addEmailNotification fails", async () => {
|
||||
const consoleLog = jest.spyOn(console, "log").mockImplementation();
|
||||
const consoleLog = jest
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation(() => undefined);
|
||||
// It's not clear if the calls to console.info are important enough to remain,
|
||||
// but since they were already there when adding the "no logs" rule in tests,
|
||||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation();
|
||||
const { sendEmail } = await import("../utils/email.js");
|
||||
const mockedUtilsHibp = jest.requireMock("../utils/hibp.js");
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -333,19 +347,19 @@ test("throws an error when addEmailNotification fails", async () => {
|
|||
Id: 1,
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/subscribers.js", () => {
|
||||
jest.mock("../../db/tables/subscribers.js", () => {
|
||||
return {
|
||||
getSubscribersByHashes: jest.fn(() => [{ id: 1 }]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/emailAddresses.js", () => {
|
||||
jest.mock("../../db/tables/emailAddresses.js", () => {
|
||||
return {
|
||||
getEmailAddressesByHashes: jest.fn(() => [""]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/email_notifications.js", () => {
|
||||
jest.mock("../../db/tables/email_notifications.js", () => {
|
||||
return {
|
||||
getNotifiedSubscribersForBreach: jest.fn(() => [2]),
|
||||
addEmailNotification: jest.fn().mockImplementationOnce(() => {
|
||||
|
|
@ -359,13 +373,13 @@ test("throws an error when addEmailNotification fails", async () => {
|
|||
hashSuffixes: ["test-suffix1"],
|
||||
});
|
||||
|
||||
const { poll } = await import("./emailBreachAlerts.js");
|
||||
const { poll } = await import("./emailBreachAlerts");
|
||||
|
||||
try {
|
||||
await poll(subClient, receivedMessages);
|
||||
} catch (e) {
|
||||
} catch (e: unknown) {
|
||||
expect(console.error).toBeCalled();
|
||||
expect(e.message).toBe("add failed");
|
||||
expect((e as Error).message).toBe("add failed");
|
||||
}
|
||||
|
||||
expect(consoleLog).toHaveBeenCalledWith(
|
||||
|
|
@ -375,13 +389,15 @@ test("throws an error when addEmailNotification fails", async () => {
|
|||
});
|
||||
|
||||
test("throws an error when markEmailAsNotified fails", async () => {
|
||||
const consoleLog = jest.spyOn(console, "log").mockImplementation();
|
||||
const consoleLog = jest
|
||||
.spyOn(console, "log")
|
||||
.mockImplementation(() => undefined);
|
||||
// It's not clear if the calls to console.info are important enough to remain,
|
||||
// but since they were already there when adding the "no logs" rule in tests,
|
||||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation();
|
||||
const { sendEmail } = await import("../utils/email.js");
|
||||
const mockedUtilsHibp = jest.requireMock("../utils/hibp.js");
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -390,19 +406,19 @@ test("throws an error when markEmailAsNotified fails", async () => {
|
|||
Id: 1,
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/subscribers.js", () => {
|
||||
jest.mock("../../db/tables/subscribers.js", () => {
|
||||
return {
|
||||
getSubscribersByHashes: jest.fn(() => [{ id: 1 }]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/emailAddresses.js", () => {
|
||||
jest.mock("../../db/tables/emailAddresses.js", () => {
|
||||
return {
|
||||
getEmailAddressesByHashes: jest.fn(() => [""]),
|
||||
};
|
||||
});
|
||||
|
||||
jest.mock("../db/tables/email_notifications.js", () => {
|
||||
jest.mock("../../db/tables/email_notifications.js", () => {
|
||||
return {
|
||||
getNotifiedSubscribersForBreach: jest.fn(() => [2]),
|
||||
addEmailNotification: jest.fn(),
|
||||
|
|
@ -417,13 +433,13 @@ test("throws an error when markEmailAsNotified fails", async () => {
|
|||
hashSuffixes: ["test-suffix1"],
|
||||
});
|
||||
|
||||
const { poll } = await import("./emailBreachAlerts.js");
|
||||
const { poll } = await import("./emailBreachAlerts");
|
||||
|
||||
try {
|
||||
await poll(subClient, receivedMessages);
|
||||
} catch (e) {
|
||||
} catch (e: unknown) {
|
||||
expect(console.error).toBeCalled();
|
||||
expect(e.message).toBe("mark failed");
|
||||
expect((e as Error).message).toBe("mark failed");
|
||||
}
|
||||
expect(consoleLog).toHaveBeenCalledWith(
|
||||
'Received message: {"breachName":"test1","hashPrefix":"test-prefix1","hashSuffixes":["test-suffix1"]}',
|
||||
|
|
@ -4,40 +4,42 @@
|
|||
|
||||
import Sentry from "@sentry/nextjs";
|
||||
import { acceptedLanguages, negotiateLanguages } from "@fluent/langneg";
|
||||
import { localStorage } from "../utils/localStorage.js";
|
||||
import { localStorage } from "../../utils/localStorage.js";
|
||||
|
||||
import * as pubsub from "@google-cloud/pubsub";
|
||||
import * as grpc from "@grpc/grpc-js";
|
||||
import type { SubscriberClient } from "@google-cloud/pubsub/build/src/v1/subscriber_client.js";
|
||||
import type { EmailAddressRow, SubscriberRow } from "knex/types/tables";
|
||||
|
||||
import {
|
||||
getSubscribersByHashes,
|
||||
knexSubscribers,
|
||||
} from "../db/tables/subscribers.js";
|
||||
} from "../../db/tables/subscribers.js";
|
||||
import {
|
||||
getEmailAddressesByHashes,
|
||||
knexEmailAddresses,
|
||||
} from "../db/tables/emailAddresses.js";
|
||||
} from "../../db/tables/emailAddresses.js";
|
||||
import {
|
||||
getNotifiedSubscribersForBreach,
|
||||
addEmailNotification,
|
||||
markEmailAsNotified,
|
||||
} from "../db/tables/email_notifications.js";
|
||||
import { getTemplate } from "../emails/email2022.js";
|
||||
import { breachAlertEmailPartial } from "../emails/emailBreachAlert.js";
|
||||
} from "../../db/tables/email_notifications.js";
|
||||
import { getTemplate } from "../../emails/email2022.js";
|
||||
import { breachAlertEmailPartial } from "../../emails/emailBreachAlert.js";
|
||||
import {
|
||||
initEmail,
|
||||
EmailTemplateType,
|
||||
getEmailCtaDashboardHref,
|
||||
sendEmail,
|
||||
} from "../utils/email.js";
|
||||
} from "../../utils/email.js";
|
||||
|
||||
import { initFluentBundles, getMessage } from "../utils/fluent.js";
|
||||
import { initFluentBundles, getMessage } from "../../utils/fluent.js";
|
||||
import {
|
||||
getAddressesAndLanguageForEmail,
|
||||
getBreachByName,
|
||||
getAllBreachesFromDb,
|
||||
knexHibp,
|
||||
} from "../utils/hibp.js";
|
||||
} from "../../utils/hibp.js";
|
||||
|
||||
const SENTRY_SLUG = "cron-breach-alerts";
|
||||
|
||||
|
|
@ -55,7 +57,8 @@ const checkInId = Sentry.captureCheckIn({
|
|||
// Only process this many messages before exiting.
|
||||
/* c8 ignore start */
|
||||
const maxMessages = parseInt(
|
||||
process.env.EMAIL_BREACH_ALERT_MAX_MESSAGES || 10000,
|
||||
process.env.EMAIL_BREACH_ALERT_MAX_MESSAGES ?? "10000",
|
||||
10,
|
||||
);
|
||||
/* c8 ignore stop */
|
||||
const projectId = process.env.GCP_PUBSUB_PROJECT_ID;
|
||||
|
|
@ -71,7 +74,28 @@ const subscriptionName = process.env.GCP_PUBSUB_SUBSCRIPTION_NAME;
|
|||
*
|
||||
* More about how account identities are anonymized: https://blog.mozilla.org/security/2018/06/25/scanning-breached-accounts-k-anonymity/
|
||||
*/
|
||||
export async function poll(subClient, receivedMessages) {
|
||||
export async function poll(
|
||||
subClient: SubscriberClient,
|
||||
receivedMessages: Array<{
|
||||
ackId?: string | null;
|
||||
deliveryAttempt?: number | null;
|
||||
message?: {
|
||||
messageId?: string | null;
|
||||
orderingKey?: string | undefined | null;
|
||||
data?: string | Uint8Array | null;
|
||||
} | null;
|
||||
}>,
|
||||
) {
|
||||
// These env vars are always set in tests:
|
||||
/* c8 ignore next 8 */
|
||||
if (!projectId) {
|
||||
throw new Error("Environment variable [$GCP_PUBSUB_PROJECT_ID] not set.");
|
||||
}
|
||||
if (!subscriptionName) {
|
||||
throw new Error(
|
||||
"Environment variable [$GCP_PUBSUB_SUBSCRIPTION_NAME] not set.",
|
||||
);
|
||||
}
|
||||
const formattedSubscription = subClient.subscriptionPath(
|
||||
projectId,
|
||||
subscriptionName,
|
||||
|
|
@ -81,8 +105,12 @@ export async function poll(subClient, receivedMessages) {
|
|||
|
||||
// Process the messages. Skip any that cannot be processed, and do not mark as acknowledged.
|
||||
for (const message of receivedMessages) {
|
||||
console.log(`Received message: ${message.message.data}`);
|
||||
const data = JSON.parse(message.message.data);
|
||||
console.log(`Received message: ${message.message?.data}`);
|
||||
const data = JSON.parse(message.message?.data as string) as {
|
||||
breachName: string;
|
||||
hashPrefix: string;
|
||||
hashSuffixes: string[];
|
||||
};
|
||||
|
||||
if (!(data.breachName && data.hashPrefix && data.hashSuffixes)) {
|
||||
console.error(
|
||||
|
|
@ -143,7 +171,14 @@ export async function poll(subClient, receivedMessages) {
|
|||
|
||||
subClient.acknowledge({
|
||||
subscription: formattedSubscription,
|
||||
ackIds: [message.ackId],
|
||||
ackIds:
|
||||
typeof message.ackId === "string"
|
||||
? [message.ackId]
|
||||
: /* c8 ignore next 4 */
|
||||
// When porting this code to TypeScript, the undefined/null case
|
||||
// wasn't dealt with, so presumably our messages always have an
|
||||
// ackId:
|
||||
message.ackId,
|
||||
});
|
||||
|
||||
continue;
|
||||
|
|
@ -157,7 +192,9 @@ export async function poll(subClient, receivedMessages) {
|
|||
|
||||
const subscribers = await getSubscribersByHashes(hashes);
|
||||
const emailAddresses = await getEmailAddressesByHashes(hashes);
|
||||
const recipients = subscribers.concat(emailAddresses);
|
||||
const recipients: Array<
|
||||
SubscriberRow | (SubscriberRow & EmailAddressRow)
|
||||
> = subscribers.concat(emailAddresses);
|
||||
|
||||
console.info(EmailTemplateType.Notification, {
|
||||
breachAlertName: breachAlert.Name,
|
||||
|
|
@ -165,7 +202,7 @@ export async function poll(subClient, receivedMessages) {
|
|||
});
|
||||
|
||||
const utmCampaignId = "breach-alert";
|
||||
const notifiedRecipients = [];
|
||||
const notifiedRecipients: string[] = [];
|
||||
|
||||
for (const recipient of recipients) {
|
||||
console.info("notify", { recipient });
|
||||
|
|
@ -175,7 +212,11 @@ export async function poll(subClient, receivedMessages) {
|
|||
// Get subscriber ID from:
|
||||
// - `subscriber_id`: if `email_addresses` record
|
||||
// - `id`: if `subscribers` record
|
||||
const subscriberId = recipient.subscriber_id ?? recipient.id;
|
||||
/* c8 ignore next 4 */
|
||||
// TODO: Add unit test when changing this code:
|
||||
const subscriberId = hasEmailAddressAttached(recipient)
|
||||
? recipient.subscriber_id
|
||||
: recipient.id;
|
||||
if (notifiedSubs.includes(subscriberId)) {
|
||||
console.info("Subscriber already notified, skipping: ", subscriberId);
|
||||
continue;
|
||||
|
|
@ -189,7 +230,7 @@ export async function poll(subClient, receivedMessages) {
|
|||
: [];
|
||||
/* c8 ignore stop */
|
||||
|
||||
const availableLanguages = process.env.SUPPORTED_LOCALES.split(",");
|
||||
const availableLanguages = process.env.SUPPORTED_LOCALES!.split(",");
|
||||
const supportedLocales = negotiateLanguages(
|
||||
requestedLanguage,
|
||||
availableLanguages,
|
||||
|
|
@ -246,7 +287,7 @@ export async function poll(subClient, receivedMessages) {
|
|||
breachId,
|
||||
data.recipientEmail,
|
||||
);
|
||||
} catch (e) {
|
||||
} catch (e: any) {
|
||||
console.error("Failed to mark email as notified: ", e);
|
||||
throw new Error(e);
|
||||
}
|
||||
|
|
@ -260,7 +301,14 @@ export async function poll(subClient, receivedMessages) {
|
|||
|
||||
subClient.acknowledge({
|
||||
subscription: formattedSubscription,
|
||||
ackIds: [message.ackId],
|
||||
ackIds:
|
||||
typeof message.ackId === "string"
|
||||
? [message.ackId]
|
||||
: /* c8 ignore next 4 */
|
||||
// When porting this code to TypeScript, the undefined/null case
|
||||
// wasn't dealt with, so presumably our messages always have an
|
||||
// ackId:
|
||||
message.ackId,
|
||||
});
|
||||
/* c8 ignore start */
|
||||
} catch (error) {
|
||||
|
|
@ -281,6 +329,16 @@ async function pullMessages() {
|
|||
sslCreds: grpc.credentials.createInsecure(),
|
||||
};
|
||||
}
|
||||
// These env vars are always set in tests:
|
||||
/* c8 ignore next 8 */
|
||||
if (!projectId) {
|
||||
throw new Error("Environment variable [$GCP_PUBSUB_PROJECT_ID] not set.");
|
||||
}
|
||||
if (!subscriptionName) {
|
||||
throw new Error(
|
||||
"Environment variable [$GCP_PUBSUB_SUBSCRIPTION_NAME] not set.",
|
||||
);
|
||||
}
|
||||
|
||||
const subClient = new pubsub.v1.SubscriberClient(options);
|
||||
|
||||
|
|
@ -297,14 +355,14 @@ async function pullMessages() {
|
|||
maxMessages,
|
||||
});
|
||||
|
||||
return [subClient, response.receivedMessages];
|
||||
return [subClient, response.receivedMessages] as const;
|
||||
}
|
||||
async function init() {
|
||||
await initFluentBundles();
|
||||
await initEmail();
|
||||
|
||||
const [subClient, receivedMessages] = await pullMessages();
|
||||
await poll(subClient, receivedMessages);
|
||||
await poll(subClient, receivedMessages ?? []);
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV !== "test") {
|
||||
|
|
@ -331,3 +389,9 @@ if (process.env.NODE_ENV !== "test") {
|
|||
});
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
function hasEmailAddressAttached(
|
||||
row: SubscriberRow | (SubscriberRow & EmailAddressRow),
|
||||
): row is SubscriberRow & EmailAddressRow {
|
||||
return typeof (row as EmailAddressRow).subscriber_id !== "undefined";
|
||||
}
|
||||
|
|
@ -229,31 +229,24 @@ async function loadBreachesIntoApp(app) {
|
|||
/**
|
||||
* Get addresses and language from either subscribers or email_addresses fields:
|
||||
*
|
||||
* @param {*} recipient
|
||||
* @param {import('knex/types/tables').SubscriberRow | (import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow)} recipient
|
||||
* @returns
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getAddressesAndLanguageForEmail(recipient) {
|
||||
const {
|
||||
all_emails_to_primary: allEmailsToPrimary,
|
||||
email: breachedEmail,
|
||||
primary_email: primaryEmail,
|
||||
signup_language: signupLanguage
|
||||
} = recipient
|
||||
|
||||
if (breachedEmail) {
|
||||
if (hasEmailAddressAttached(recipient)) {
|
||||
return {
|
||||
breachedEmail,
|
||||
recipientEmail: allEmailsToPrimary ? primaryEmail : breachedEmail,
|
||||
signupLanguage
|
||||
breachedEmail: recipient.email,
|
||||
recipientEmail: recipient.all_emails_to_primary ? recipient.primary_email : recipient.email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
breachedEmail: primaryEmail,
|
||||
recipientEmail: primaryEmail,
|
||||
signupLanguage
|
||||
breachedEmail: recipient.primary_email,
|
||||
recipientEmail: recipient.primary_email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
|
@ -407,6 +400,14 @@ async function deleteSubscribedHash(sha1) {
|
|||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {import('knex/types/tables').SubscriberRow} subscriberRow
|
||||
* @returns {subscriberRow is import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow}
|
||||
*/
|
||||
function hasEmailAddressAttached(subscriberRow) {
|
||||
return typeof (/** @type {import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow} */ (subscriberRow)).email === "string";
|
||||
}
|
||||
|
||||
export {
|
||||
req,
|
||||
kAnonReq,
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче