Merge pull request #4617 from mozilla/mntor-2700-send-email

Add cronjob for sending first data broker removal fixed email (MNTOR-2700)

package.json

@@ -20,6 +20,7 @@
     "e2e": "playwright test src/e2e/",
     "e2e:debug": "playwright test src/e2e/ --ui",
     "e2e:smoke": "playwright test src/e2e/ --grep @smoke",
+    "cron:first-data-broker-removal-fixed": "node dist/scripts/cronjobs/firstDataBrokerRemovalFixed.js",
     "cron:monthly-activity": "node dist/scripts/cronjobs/monthlyActivity.js",
     "cron:breach-alerts": "node src/scripts/emailBreachAlerts.js",
     "cron:db-delete-unverified-subscribers": "node scripts/delete-unverified-subscribers.js",

src/db/migrations/20240601114803_add_first_broker_removal_email_sent.js

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function up (knex) {
+  return knex.schema.table("subscribers", table => {
+    table.boolean("first_broker_removal_email_sent").defaultTo(false);
+    table.index("first_broker_removal_email_sent");
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function down (knex) {
+  return knex.schema.table("subscribers", table => {
+    table.dropIndex("first_broker_removal_email_sent")
+    table.dropColumn("first_broker_removal_email_sent");
+  });
+}

src/db/tables/featureFlags.ts

@@ -43,7 +43,8 @@ export type FeatureFlagName =
   | "UpdatedEmailPreferencesOption"
   | "MonthlyActivityEmail"
   | "CancellationFlow"
-  | "ConfirmCancellation";
+  | "ConfirmCancellation"
+  | "FirstDataBrokerRemovalFixedEmail";
 
 /**
  * @param options

src/db/tables/subscribers.js

@@ -291,6 +291,60 @@ async function deleteResolutionsWithEmail (id, email) {
 }
 /* c8 ignore stop */
 
+/**
+ * @returns {Promise<import("knex/types/tables").SubscriberRow[]>}
+ */
+// Not covered by tests; mostly side-effects. See test-coverage.md#mock-heavy
+/* c8 ignore start */
+async function getPotentialSubscribersWaitingForFirstDataBrokerRemovalFixedEmail() {
+  // I'm explicitly referencing the type here, so that these lines of code will
+  // show up as errors when we remove it from the flag list:
+  /** @type {import("./featureFlags.js").FeatureFlagName} */
+  const featureFlagName = "FirstDataBrokerRemovalFixedEmail";
+  // Interactions with the `feature_flags` table would generally go in the
+  // `src/db/tables/featureFlags` module. However, since that module is already
+  // written in TypeScript, it can't be loaded in pre-TypeScript cron jobs,
+  // which currently still import from the subscribers module. Hence, we've
+  // inlined this until https://mozilla-hub.atlassian.net/browse/MNTOR-3077 is fixed.
+  const flag = (await knex("feature_flags")
+      .first()
+      .where("name", featureFlagName)
+      // The `.andWhereNull` alias doesn't seem to exist:
+      // https://github.com/knex/knex/issues/1881#issuecomment-275433906
+      .whereNull("deleted_at"));
+
+  if (!flag?.is_enabled || !flag?.modified_at) {
+    return [];
+  }
+
+  let query = knex("subscribers")
+    .select()
+    // Only send to Plus users...
+    .whereRaw(
+      `(fxa_profile_json->'subscriptions')::jsonb \\? ?`,
+      MONITOR_PREMIUM_CAPABILITY,
+    )
+    // ...with an OneRep account...
+    .whereNotNull("onerep_profile_id")
+    // ...who haven’t received the email...
+    .andWhere("first_broker_removal_email_sent", false)
+    // ...and signed up after the feature flag `FirstDataBrokerRemovalFixedEmail`
+    // has been enabled last.
+    .andWhere("created_at", ">=", flag.modified_at);
+
+  if (Array.isArray(flag.allow_list) && flag.allow_list.length > 0) {
+    // If the feature flag has an allowlist, only send to users on that list.
+    // The `.andWhereIn` alias doesn't exist:
+    // https://github.com/knex/knex/issues/1881#issuecomment-275433906
+    query = query.whereIn("primary_email", flag.allow_list)
+  }
+
+  const rows = await query;
+
+  return rows;
+}
+/* c8 ignore stop */
+
 /**
  * @param {Partial<{ plusOnly: boolean; limit: number; }>} options
  * @returns {Promise<import("knex/types/tables").SubscriberRow[]>}
@@ -306,7 +360,7 @@ async function getSubscribersWaitingForMonthlyEmail (options = {}) {
   // `src/db/tables/featureFlags` module. However, since that module is already
   // written in TypeScript, it can't be loaded in pre-TypeScript cron jobs,
   // which currently still import from the subscribers module. Hence, we've
-  // inlined this for now.
+  // inlined this until https://mozilla-hub.atlassian.net/browse/MNTOR-3077 is fixed.
   const flag = (await knex("feature_flags")
       .first()
       .where("name", featureFlagName)
@@ -392,6 +446,29 @@ async function updateMonthlyEmailOptout (token) {
 }
 /* c8 ignore stop */
 
+/**
+ * @param {import("knex/types/tables").SubscriberRow} subscriber
+ */
+// Not covered by tests; mostly side-effects. See test-coverage.md#mock-heavy
+/* c8 ignore start */
+async function markFirstDataBrokerRemovalFixedEmailAsJustSent (subscriber) {
+  const affectedSubscribers = await knex("subscribers")
+    .update({
+      first_broker_removal_email_sent: true,
+      // @ts-ignore knex.fn.now() results in it being set to a date,
+      // even if it's not typed as a JS date object:
+      updated_at: knex.fn.now(),
+    })
+    .where("primary_email", subscriber.primary_email)
+    .andWhere("id", subscriber.id)
+    .returning("*");
+
+  if (affectedSubscribers.length !== 1) {
+    throw new Error(`Attempted to mark 1 user as having just been sent the first data broker removal fixed email, but instead found [${affectedSubscribers.length}] matching its ID and email address.`);
+  }
+}
+
+/* c8 ignore stop */
 /**
  * @param {import("knex/types/tables").SubscriberRow} subscriber
  */
@@ -525,9 +602,11 @@ export {
   setAllEmailsToPrimary,
   setMonthlyMonitorReport,
   setBreachResolution,
+  getPotentialSubscribersWaitingForFirstDataBrokerRemovalFixedEmail,
   getSubscribersWaitingForMonthlyEmail,
   updateMonthlyEmailTimestamp,
   updateMonthlyEmailOptout,
+  markFirstDataBrokerRemovalFixedEmailAsJustSent,
   markMonthlyActivityEmailAsJustSent,
   deleteUnverifiedSubscribers,
   deleteSubscriber,

src/emails/templates/firstDataBrokerRemovalFixed/FirstDataBrokerRemovalFixed.stories.tsx

@@ -32,7 +32,7 @@ export const FirstDataBrokerRemovalFixedStory: Story = {
     data: {
       dataBrokerName: "Data broker name",
       dataBrokerLink: "https://monitor.mozilla.org/",
-      removalDate: "5/31/2024",
+      removalDate: new Date("5/31/2024"),
     },
   },
 };

src/emails/templates/firstDataBrokerRemovalFixed/FirstDataBrokerRemovalFixed.tsx

@@ -8,12 +8,11 @@ import { EmailFooter } from "../EmailFooter";
 import { EmailHeader } from "../EmailHeader";
 
 export type Props = {
-  removalDate: Date;
   l10n: ExtendedReactLocalization;
   data: {
     dataBrokerName: string;
     dataBrokerLink: string;
-    removalDate: string;
+    removalDate: Date;
   };
 };
 

src/knex-tables.d.ts

@@ -133,6 +133,7 @@ declare module "knex/types/tables" {
     db_migration_2: null | unknown;
     onerep_profile_id: null | number;
     email_addresses: SubscriberEmail[];
+    first_broker_removal_email_sent: boolean;
   }
   type SubscriberOptionalColumns = Extract<
     keyof SubscriberRow,
@@ -153,6 +154,7 @@ declare module "knex/types/tables" {
     | "db_migration_2"
     | "onerep_profile_id"
     | "email_addresses"
+    | "first_broker_removal_email_sent"
   >;
   type SubscriberAutoInsertedColumns = Extract<
     keyof SubscriberRow,

src/scripts/cronjobs/firstDataBrokerRemovalFixed.tsx

@@ -0,0 +1,134 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import React from "react";
+import { OnerepScanResultRow, SubscriberRow } from "knex/types/tables";
+import {
+  getPotentialSubscribersWaitingForFirstDataBrokerRemovalFixedEmail,
+  markFirstDataBrokerRemovalFixedEmailAsJustSent,
+} from "../../db/tables/subscribers";
+import { initEmail, sendEmail } from "../../utils/email";
+import { renderEmail } from "../../emails/renderEmail";
+import { FirstDataBrokerRemovalFixed } from "../../emails/templates/firstDataBrokerRemovalFixed/FirstDataBrokerRemovalFixed";
+import { getEmailL10n } from "../../app/functions/l10n/cronjobs";
+import { sanitizeSubscriberRow } from "../../app/functions/server/sanitize";
+import { refreshStoredScanResults } from "../../app/functions/server/refreshStoredScanResults";
+import { getLatestOnerepScanResults } from "../../db/tables/onerep_scans";
+
+type SubscriberFirstRemovedScanResult = {
+  subscriber: SubscriberRow;
+  firstRemovedScanResult: OnerepScanResultRow;
+};
+
+function isFulfilledResult(
+  result: PromiseSettledResult<SubscriberFirstRemovedScanResult | undefined>,
+): result is PromiseFulfilledResult<SubscriberFirstRemovedScanResult> {
+  return typeof result !== "undefined" && result.status === "fulfilled";
+}
+
+void run();
+
+async function run() {
+  const batchSize = Number.parseInt(
+    process.env.FIRST_DATA_BROKER_REMOVAL_FIXED_EMAIL_BATCH_SIZE ?? "10",
+    10,
+  );
+  if (Number.isNaN(batchSize)) {
+    throw new Error(
+      `Could not send first data broker removal fixed emails, because the env var FIRST_DATA_BROKER_REMOVAL_FIXED_EMAIL_BATCH_SIZE has a non-numeric value: [${process.env.FIRST_DATA_BROKER_REMOVAL_FIXED_EMAIL_BATCH_SIZE}].`,
+    );
+  }
+  const potentialSubscribersToEmail =
+    await getPotentialSubscribersWaitingForFirstDataBrokerRemovalFixedEmail();
+
+  const subscribersToEmailWithData = (
+    await Promise.allSettled(
+      potentialSubscribersToEmail.map(async (subscriber) => {
+        try {
+          // OneRep suggested not relying on webhooks, but instead to fetch the latest
+          // data from their API. Thus, let's refresh the data in our DB in real-time:
+          if (subscriber.onerep_profile_id !== null) {
+            await refreshStoredScanResults(subscriber.onerep_profile_id);
+          }
+          const latestScan = await getLatestOnerepScanResults(
+            subscriber.onerep_profile_id,
+          );
+
+          let firstRemovedScanResult = null;
+          for (const scanResult of latestScan.results) {
+            // Consider a scan result if:
+            if (
+              // The scan result is not manually resolved...
+              !scanResult.manually_resolved &&
+              // ...the scan has been removed...
+              scanResult.status === "removed" &&
+              // ...and scan result has been created ealier than the currently
+              // selected `firstRemovedScanResult`.
+              firstRemovedScanResult &&
+              scanResult.created_at.getTime() <
+                firstRemovedScanResult.created_at.getTime()
+            ) {
+              firstRemovedScanResult = scanResult;
+            }
+          }
+
+          if (!firstRemovedScanResult) {
+            return;
+          }
+
+          return { subscriber, firstRemovedScanResult };
+        } catch (_error) {
+          console.error(
+            `An error ocurred while attemting to get the first removed scan result for subscriber: ${subscriber.id}`,
+          );
+        }
+      }),
+    )
+  )
+    .filter(isFulfilledResult)
+    .slice(0, batchSize);
+
+  await initEmail();
+
+  await Promise.allSettled(
+    subscribersToEmailWithData.map((data) => {
+      return sendFirstDataBrokerRemovalFixedActivityEmail(
+        data.value.subscriber,
+        data.value.firstRemovedScanResult,
+      );
+    }),
+  );
+  console.log(
+    `[${new Date(Date.now()).toISOString()}] Sent [${subscribersToEmailWithData.length}] first data broker removal fixed emails.`,
+  );
+}
+
+async function sendFirstDataBrokerRemovalFixedActivityEmail(
+  subscriber: SubscriberRow,
+  scanResult: OnerepScanResultRow,
+) {
+  const sanitizedSubscriber = sanitizeSubscriberRow(subscriber);
+  const l10n = getEmailL10n(sanitizedSubscriber);
+
+  let subject = l10n.getString("email-first-broker-removal-fixed-subject");
+
+  // Update the first-data-broker-removal-fixed-email date *first*,
+  // so that if something goes wrong, we don't keep resending the email.
+  await markFirstDataBrokerRemovalFixedEmailAsJustSent(subscriber);
+
+  await sendEmail(
+    sanitizedSubscriber.primary_email,
+    subject,
+    renderEmail(
+      <FirstDataBrokerRemovalFixed
+        data={{
+          dataBrokerName: scanResult.data_broker,
+          dataBrokerLink: scanResult.link,
+          removalDate: scanResult.updated_at,
+        }}
+        l10n={l10n}
+      />,
+    ),
+  );
+}