Migrate utils/hibp.js to TypeScript
This removes the `Breach` type in functions/universal/breaches,
which was created when first introducing TypeScript and the flow
of data was still unclear, but by now had overlap with other types
and no clear provenance.
Instead, there are now three breach-related types, that represent
where the data came from:
- HibpGetBreachesResponse: this is an array of breach elements as
returned from the HIBP API, unprocessed.
Properties are in PascalCase, so are a
breach's data classes.
- BreachRow: this is a breach's data as stored in our database,
along with some data we added to it, such as a favicon
URL. Properties are snake_case, and data classes are
lowercased and kebab-cased by the
formatDataClassesArray function.
- HibpLikeDbBreach: this is a breach's data fetched from the
database, but stored in an object meant to look
like the ones in HibpGetBreachesResponse. In
other words, it contains the same data as
BreachRow (including lowercased, kebab-cased
data classes), but on PascalCase properties.
The latter is somewhat of a historical artefact, because we used
to try to load breaches from our database, then if our database
didn't contain any breaches yet, fetch them live from the HIBP API
and continue working with that.
We no longer do that: now, even after fetching them from the HIBP
API, we do a new query to get them from the database and process
them into HibpLikeDbBreach, so that we can assume a consisent data
structure everywhere we work with breaches.
This commit is contained in:
Vincent
Родитель
630c687124
Коммит
319434001d
|
|
@ -179,8 +179,10 @@ export function createRandomHibpListing(
|
|||
];
|
||||
return {
|
||||
AddedDate: addedDate,
|
||||
BreachDate: breachDate.toISOString(),
|
||||
DataClasses: faker.helpers.arrayElements(possibleDataClasses),
|
||||
BreachDate: breachDate,
|
||||
DataClasses: faker.helpers.arrayElements(possibleDataClasses) as Array<
|
||||
(typeof BreachDataTypes)[keyof typeof BreachDataTypes]
|
||||
>,
|
||||
Description: faker.lorem.sentence(),
|
||||
Domain: faker.internet.domainName(),
|
||||
Id: faker.number.int(),
|
||||
|
|
|
|||
|
|
@ -13,10 +13,9 @@ import { BreachLogo } from "../../../../components/server/BreachLogo";
|
|||
import { getLocale } from "../../../../functions/universal/getLocale";
|
||||
import { useHasRenderedClientSide } from "../../../../hooks/useHasRenderedClientSide";
|
||||
import { memo, useMemo, useState, useTransition } from "react";
|
||||
import { Breach } from "../../../../functions/universal/breach";
|
||||
|
||||
export type Props = {
|
||||
allBreaches: Array<HibpLikeDbBreach | Breach>;
|
||||
allBreaches: HibpLikeDbBreach[];
|
||||
};
|
||||
export const BreachIndexView = (props: Props) => {
|
||||
const l10n = useL10n();
|
||||
|
|
@ -75,7 +74,7 @@ const FilterForm = (props: { onChange: (newValue: string) => void }) => {
|
|||
};
|
||||
|
||||
function matchesFilter(
|
||||
breach: Breach | HibpLikeDbBreach,
|
||||
breach: HibpLikeDbBreach,
|
||||
filterTerm: string,
|
||||
l10n: ExtendedReactLocalization,
|
||||
): boolean {
|
||||
|
|
@ -95,7 +94,7 @@ function matchesFilter(
|
|||
// time the filter changes, like the current page, is exactly what React isn't
|
||||
// great at. memo() alleviates it somewhat, though.
|
||||
const BreachCard = memo(
|
||||
(props: { breach: HibpLikeDbBreach | Breach; isVisible: boolean }) => {
|
||||
(props: { breach: HibpLikeDbBreach; isVisible: boolean }) => {
|
||||
const l10n = useL10n();
|
||||
const locale = getLocale(l10n);
|
||||
// Performance profiling shows that formatting the date is pretty expensive,
|
||||
|
|
|
|||
|
|
@ -6,10 +6,7 @@ import { NextResponse } from "next/server";
|
|||
import { logger } from "../../../../functions/server/logging";
|
||||
import mockAllBreaches from "../mockData/mockAllBreaches.json";
|
||||
import { errorIfProduction } from "../../../utils/errorThrower";
|
||||
import { Breach } from "../../../../functions/universal/breach";
|
||||
import { HibpLikeDbBreach } from "../../../../../utils/hibp";
|
||||
|
||||
type BreachesListResponse = (Breach | HibpLikeDbBreach)[];
|
||||
import { HibpGetBreachesResponse } from "../../../../../utils/hibp";
|
||||
|
||||
export function GET() {
|
||||
const prodError = errorIfProduction();
|
||||
|
|
@ -17,5 +14,5 @@ export function GET() {
|
|||
|
||||
logger.info("Mock endpoint: /breaches");
|
||||
|
||||
return NextResponse.json(mockAllBreaches.data as BreachesListResponse);
|
||||
return NextResponse.json(mockAllBreaches.data as HibpGetBreachesResponse);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,11 +6,7 @@ import { NextRequest, NextResponse } from "next/server";
|
|||
import { logger } from "../../../../../../functions/server/logging";
|
||||
import { errorIfProduction } from "../../../../../utils/errorThrower";
|
||||
import { getBreachesForHash } from "../../../config/defaults";
|
||||
|
||||
type BreachedAccountResponse = {
|
||||
hashSuffix: string;
|
||||
websites: string[];
|
||||
}[];
|
||||
import { BreachedAccountResponse } from "../../../../../../../utils/hibp";
|
||||
|
||||
export function GET(
|
||||
_: NextRequest,
|
||||
|
|
@ -26,7 +22,7 @@ export function GET(
|
|||
|
||||
const data: BreachedAccountResponse = [
|
||||
{
|
||||
hashSuffix: "", //hibp.js ignores hashSuffix if a mock endpoint is used.
|
||||
hashSuffix: "", //hibp.ts ignores hashSuffix if a mock endpoint is used.
|
||||
websites: breachesList,
|
||||
},
|
||||
];
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import {
|
|||
} from "../../../db/tables/subscribers.js";
|
||||
import { addSubscriber } from "../../../db/tables/emailAddresses.js";
|
||||
import { getBreaches } from "../../functions/server/getBreaches";
|
||||
import { getBreachesForEmail } from "../../../utils/hibp.js";
|
||||
import { getBreachesForEmail } from "../../../utils/hibp";
|
||||
import { getSha1, refreshOAuthTokens } from "../../../utils/fxa.js";
|
||||
import {
|
||||
getEmailCtaDashboardHref,
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@
|
|||
|
||||
import Image from "next/image";
|
||||
import styles from "./BreachLogo.module.scss";
|
||||
import { HibpLikeDbBreach, WithFaviconUrl } from "../../../utils/hibp";
|
||||
import { Breach } from "../../functions/universal/breach";
|
||||
import { HibpLikeDbBreach } from "../../../utils/hibp";
|
||||
|
||||
/**
|
||||
* @param props
|
||||
|
|
@ -15,7 +14,7 @@ import { Breach } from "../../functions/universal/breach";
|
|||
*/
|
||||
|
||||
export type Props = {
|
||||
breach: HibpLikeDbBreach | Breach;
|
||||
breach: HibpLikeDbBreach;
|
||||
};
|
||||
|
||||
// The <BreachLogo> component is currently a bit troublesome to test because it
|
||||
|
|
@ -23,7 +22,7 @@ export type Props = {
|
|||
// can add a unit test when we convert it to take SubscriberBreaches.
|
||||
/* c8 ignore start */
|
||||
export function BreachLogo(props: Props) {
|
||||
if (hasFavIconUrl(props.breach)) {
|
||||
if (props.breach.FaviconUrl) {
|
||||
return (
|
||||
<Image
|
||||
src={props.breach.FaviconUrl}
|
||||
|
|
@ -90,12 +89,3 @@ function getColorForName(name: string) {
|
|||
|
||||
return logoColors[charValue % logoColors.length];
|
||||
}
|
||||
|
||||
// We don't explicitly test for favicon URLs, because we have mocked out lazy-
|
||||
// loaded images, which makes them hard to test:
|
||||
/* c8 ignore next 5 */
|
||||
function hasFavIconUrl(
|
||||
breach: HibpLikeDbBreach | Breach,
|
||||
): breach is Required<WithFaviconUrl> & HibpLikeDbBreach {
|
||||
return typeof (breach as HibpLikeDbBreach).FaviconUrl === "string";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,13 +4,13 @@
|
|||
|
||||
import { getL10n } from "../l10n/serverComponents";
|
||||
import AppConstants from "../../../appConstants.js";
|
||||
import { Breach, BreachDataTypes } from "../universal/breach";
|
||||
import { BreachDataTypes } from "../universal/breach";
|
||||
|
||||
/**
|
||||
* TODO: Map from google doc: https://docs.google.com/document/d/1KoItFsTYVIBInIG2YmA7wSxkKS4vti_X0A0td_yaHVM/edit#
|
||||
* Hardcoded map of breach resolution data types
|
||||
*
|
||||
* @type { Record<keyof BreachDataTypes, { priority: number, header: string, body?: string, applicableCountryCodes?: string[] }> }
|
||||
* @type { Record<keyof typeof BreachDataTypes, { priority: number, header: string, body?: string, applicableCountryCodes?: string[] }> }
|
||||
*/
|
||||
const breachResolutionDataTypes = {
|
||||
[BreachDataTypes.Passwords]: {
|
||||
|
|
@ -98,7 +98,9 @@ function appendBreachResolutionChecklist(
|
|||
const { verifiedEmails } = userBreachData;
|
||||
|
||||
for (const { breaches } of verifiedEmails) {
|
||||
breaches.forEach((b: Breach) => {
|
||||
// Old untyped code, adding type defitions now isn't worth the effort:
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
breaches.forEach((b: any) => {
|
||||
const dataClasses = b.DataClasses;
|
||||
const blockList = (AppConstants.HIBP_BREACH_DOMAIN_BLOCKLIST ?? "").split(
|
||||
",",
|
||||
|
|
@ -124,7 +126,7 @@ function appendBreachResolutionChecklist(
|
|||
transUnionLink:
|
||||
'<a href="https://www.transunion.com/credit-freeze" target="_blank">TransUnion</a>',
|
||||
};
|
||||
(b as any).breachChecklist = getResolutionRecsPerBreach(
|
||||
b.breachChecklist = getResolutionRecsPerBreach(
|
||||
dataClasses,
|
||||
args,
|
||||
options,
|
||||
|
|
|
|||
|
|
@ -5,40 +5,28 @@
|
|||
import { logger } from "./logging";
|
||||
import {
|
||||
HibpLikeDbBreach,
|
||||
formatDataClassesArray,
|
||||
getAllBreachesFromDb,
|
||||
req,
|
||||
} from "../../../utils/hibp.js";
|
||||
fetchHibpBreaches,
|
||||
} from "../../../utils/hibp";
|
||||
import { upsertBreaches } from "../../../db/tables/breaches.js";
|
||||
import { Breach } from "../universal/breach.js";
|
||||
|
||||
let breaches: Array<Breach | HibpLikeDbBreach>;
|
||||
let breaches: Array<HibpLikeDbBreach>;
|
||||
|
||||
export async function getBreaches() {
|
||||
export async function getBreaches(): Promise<HibpLikeDbBreach[]> {
|
||||
if (breaches) {
|
||||
return breaches;
|
||||
}
|
||||
breaches = await getAllBreachesFromDb();
|
||||
logger.debug(
|
||||
"loadBreachesIntoApp",
|
||||
`loaded breaches from database: ${breaches.length}`,
|
||||
);
|
||||
logger.debug(`loaded breaches from database: ${breaches.length}`);
|
||||
|
||||
// if "breaches" table does not return results, fall back to HIBP request
|
||||
if (breaches?.length < 1) {
|
||||
const breachesResponse = (await req("/breaches")) as Breach[];
|
||||
logger.debug(
|
||||
"loadBreachesIntoApp",
|
||||
`loaded breaches from HIBP: ${breachesResponse.length}`,
|
||||
);
|
||||
|
||||
for (const breach of breachesResponse) {
|
||||
breach.DataClasses = formatDataClassesArray(breach.DataClasses);
|
||||
breaches.push(breach);
|
||||
}
|
||||
const breachesResponse = await fetchHibpBreaches();
|
||||
logger.debug(`loaded breaches from HIBP: ${breachesResponse.length}`);
|
||||
|
||||
// sync the "breaches" table with the latest from HIBP
|
||||
await upsertBreaches(breaches);
|
||||
await upsertBreaches(breachesResponse);
|
||||
breaches = await getAllBreachesFromDb();
|
||||
}
|
||||
|
||||
return breaches;
|
||||
|
|
|
|||
|
|
@ -6,29 +6,6 @@ import { DataClassEffected } from "../../../utils/subscriberBreaches";
|
|||
|
||||
// TODO: Move pure functions that operate on breaches to this file
|
||||
|
||||
export interface Breach {
|
||||
AddedDate: string;
|
||||
BreachDate: string;
|
||||
DataClasses: Array<string>;
|
||||
Description: string;
|
||||
Domain: string;
|
||||
Id: number;
|
||||
IsFabricated: boolean;
|
||||
IsMalware: boolean;
|
||||
IsResolved?: boolean;
|
||||
IsRetired: boolean;
|
||||
IsSensitive: boolean;
|
||||
IsSpamList: boolean;
|
||||
IsVerified: boolean;
|
||||
LogoPath: string;
|
||||
ModifiedDate: string;
|
||||
Name: string;
|
||||
PwnCount: number;
|
||||
recencyIndex: number;
|
||||
ResolutionsChecked: Array<string>;
|
||||
Title: string;
|
||||
}
|
||||
|
||||
export const BreachDataTypes = {
|
||||
Passwords: "passwords",
|
||||
Email: "email-addresses",
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import { formatDataClassesArray } from "../../utils/hibp";
|
||||
import createDbConnection from "../connect.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
|
@ -40,7 +41,7 @@ async function getAllBreachesCount() {
|
|||
* Upsert breaches into "breaches" table
|
||||
* Skip inserting when 'name' field (unique) has a conflict
|
||||
*
|
||||
* @param {any[]} hibpBreaches breaches array from HIBP API
|
||||
* @param {import("../../utils/hibp.js").HibpGetBreachesResponse} hibpBreaches breaches array from HIBP API
|
||||
* @returns
|
||||
*/
|
||||
// Not covered by tests; mostly side-effects. See test-coverage.md#mock-heavy
|
||||
|
|
@ -60,8 +61,8 @@ async function upsertBreaches(hibpBreaches) {
|
|||
modified_date: breach.ModifiedDate,
|
||||
pwn_count: breach.PwnCount,
|
||||
description: breach.Description,
|
||||
logo_path: breach.LogoPath,
|
||||
data_classes: breach.DataClasses,
|
||||
logo_path: /** @type {RegExpExecArray} */(/[^/]*$/.exec(breach.LogoPath))[0],
|
||||
data_classes: formatDataClassesArray(breach.DataClasses),
|
||||
is_verified: breach.IsVerified,
|
||||
is_fabricated: breach.IsFabricated,
|
||||
is_sensitive: breach.IsSensitive,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import createDbConnection from "../connect.js";
|
||||
import { subscribeHash } from '../../utils/hibp.js'
|
||||
import { subscribeHash } from '../../utils/hibp'
|
||||
import { getSha1 } from '../../utils/fxa.js'
|
||||
import { getSubscriberByEmail, updateFxAData } from './subscribers.js'
|
||||
import {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,10 @@ import { ISO8601DateString } from "./utils/parse";
|
|||
import { StateAbbr } from "./utils/states";
|
||||
import { RemovalStatus } from "./app/functions/universal/scanResult";
|
||||
import { BreachDataTypes } from "./app/functions/universal/breach";
|
||||
import type {
|
||||
formatDataClassesArray,
|
||||
HibpGetBreachesResponse,
|
||||
} from "./utils/hibp";
|
||||
|
||||
// See https://knexjs.org/guide/#typescript
|
||||
declare module "knex/types/tables" {
|
||||
|
|
@ -188,25 +192,24 @@ declare module "knex/types/tables" {
|
|||
|
||||
interface BreachRow {
|
||||
id: number;
|
||||
name: string;
|
||||
title: string;
|
||||
domain: null | string;
|
||||
name: HibpGetBreachesResponse[number]["Name"];
|
||||
title: HibpGetBreachesResponse[number]["Title"];
|
||||
domain: HibpGetBreachesResponse[number]["Domain"];
|
||||
breach_date: Date;
|
||||
/** Note: added_date is provided by HIBP; this is not the equivalent to created_at in other tables */
|
||||
added_date: Date;
|
||||
/** Note: modified_date is provided by HIBP; this is not the equivalent to updated_at in other tables */
|
||||
modified_date: Date;
|
||||
pwn_count: number;
|
||||
description: null | string;
|
||||
pwn_count: HibpGetBreachesResponse[number]["PwnCount"];
|
||||
description: null | HibpGetBreachesResponse[number]["Description"];
|
||||
logo_path: string;
|
||||
// TODO: Verify if Knex can actually parse this into a `string[]`
|
||||
data_classes: unknown;
|
||||
is_verified: boolean;
|
||||
is_fabricated: boolean;
|
||||
is_sensitive: boolean;
|
||||
is_retired: boolean;
|
||||
is_spam_list: boolean;
|
||||
is_malware: boolean;
|
||||
data_classes: ReturnType<typeof formatDataClassesArray>;
|
||||
is_verified: HibpGetBreachesResponse[number]["IsVerified"];
|
||||
is_fabricated: HibpGetBreachesResponse[number]["IsFabricated"];
|
||||
is_sensitive: HibpGetBreachesResponse[number]["IsSensitive"];
|
||||
is_retired: HibpGetBreachesResponse[number]["IsRetired"];
|
||||
is_spam_list: HibpGetBreachesResponse[number]["IsSpamList"];
|
||||
is_malware: HibpGetBreachesResponse[number]["IsMalware"];
|
||||
favicon_url: null | string;
|
||||
}
|
||||
type BreachOptionalColumns = Extract<
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ jest.mock("../../utils/email.js", () => {
|
|||
};
|
||||
});
|
||||
|
||||
jest.mock("../../utils/hibp.js", () => {
|
||||
jest.mock("../../utils/hibp", () => {
|
||||
return {
|
||||
getAddressesAndLanguageForEmail: jest.fn(() => {
|
||||
return {
|
||||
|
|
@ -191,7 +191,7 @@ test("processes valid messages", async () => {
|
|||
(typeof emailMod)["sendEmail"]
|
||||
>;
|
||||
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -283,7 +283,7 @@ test("skipping email when subscriber id exists in email_notifications table", as
|
|||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -338,7 +338,7 @@ test("throws an error when addEmailNotification fails", async () => {
|
|||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
@ -397,7 +397,7 @@ test("throws an error when markEmailAsNotified fails", async () => {
|
|||
// I'm respecting Chesterton's Fence and leaving them in place for now:
|
||||
jest.spyOn(console, "info").mockImplementation(() => undefined);
|
||||
const { sendEmail } = await import("../../utils/email.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp.js");
|
||||
const mockedUtilsHibp: any = jest.requireMock("../../utils/hibp");
|
||||
mockedUtilsHibp.getBreachByName.mockReturnValue({
|
||||
IsVerified: true,
|
||||
Domain: "test1",
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import {
|
|||
getBreachByName,
|
||||
getAllBreachesFromDb,
|
||||
knexHibp,
|
||||
} from "../../utils/hibp.js";
|
||||
} from "../../utils/hibp";
|
||||
|
||||
const SENTRY_SLUG = "cron-breach-alerts";
|
||||
|
||||
|
|
@ -121,6 +121,16 @@ export async function poll(
|
|||
|
||||
const { breachName, hashPrefix, hashSuffixes } = data;
|
||||
const breachAlert = getBreachByName(breaches, breachName);
|
||||
// Check added to old code for type safety, but we've been assuming
|
||||
// getBreachByName will always find a breach here without tests so far, so
|
||||
// apparently that's been working well enough:
|
||||
/* c8 ignore next 6 */
|
||||
if (!breachAlert) {
|
||||
console.error(
|
||||
"HIBP breach notification: couldn't find the breach to notify about.",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const {
|
||||
IsVerified,
|
||||
|
|
|
|||
|
|
@ -12,14 +12,13 @@
|
|||
import { readdir } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import Sentry from "@sentry/nextjs";
|
||||
import { req, formatDataClassesArray } from "../../utils/hibp.js";
|
||||
import { fetchHibpBreaches, HibpGetBreachesResponse } from "../../utils/hibp";
|
||||
import {
|
||||
getAllBreaches,
|
||||
upsertBreaches,
|
||||
updateBreachFaviconUrl,
|
||||
} from "../../db/tables/breaches.js";
|
||||
import { uploadToS3 } from "../s3.js";
|
||||
import type { Breach } from "../../app/functions/universal/breach.js";
|
||||
|
||||
const SENTRY_SLUG = "cron-sync-breaches";
|
||||
|
||||
|
|
@ -33,7 +32,7 @@ const checkInId = Sentry.captureCheckIn({
|
|||
status: "in_progress",
|
||||
});
|
||||
|
||||
export async function getBreachIcons(breaches: Breach[]) {
|
||||
export async function getBreachIcons(breaches: HibpGetBreachesResponse) {
|
||||
// make logofolder if it doesn't exist
|
||||
const logoFolder = os.tmpdir();
|
||||
console.log(`Logo folder: ${logoFolder}`);
|
||||
|
|
@ -83,30 +82,27 @@ export async function getBreachIcons(breaches: Breach[]) {
|
|||
}
|
||||
|
||||
// Get breaches and upserts to DB
|
||||
const breachesResponse: Breach[] = await req("/breaches");
|
||||
const breaches: Breach[] = [];
|
||||
const breachesResponse = await fetchHibpBreaches();
|
||||
const seen = new Set();
|
||||
for (const breach of breachesResponse) {
|
||||
breach.DataClasses = formatDataClassesArray(breach.DataClasses);
|
||||
breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)![0];
|
||||
breaches.push(breach);
|
||||
breachesResponse.forEach((breach) => {
|
||||
seen.add(breach.Name + breach.BreachDate);
|
||||
|
||||
// sanity check: corrupt data structure
|
||||
if (!isValidBreach(breach))
|
||||
if (!isValidBreach(breach)) {
|
||||
throw new Error(
|
||||
"Breach data structure is not valid: " + JSON.stringify(breach),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
console.log("Breaches found: ", breaches.length);
|
||||
console.log("Breaches found: ", breachesResponse.length);
|
||||
console.log("Unique breaches based on Name + BreachDate", seen.size);
|
||||
|
||||
// sanity check: no duplicate breaches with Name + BreachDate
|
||||
if (seen.size !== breaches.length) {
|
||||
if (seen.size !== breachesResponse.length) {
|
||||
throw new Error("Breaches contain duplicates. Stopping script...");
|
||||
} else {
|
||||
await upsertBreaches(breaches);
|
||||
await upsertBreaches(breachesResponse);
|
||||
|
||||
// get
|
||||
const result = await getAllBreaches();
|
||||
|
|
@ -116,7 +112,7 @@ if (seen.size !== breaches.length) {
|
|||
);
|
||||
}
|
||||
|
||||
await getBreachIcons(breaches);
|
||||
await getBreachIcons(breachesResponse);
|
||||
|
||||
Sentry.captureCheckIn({
|
||||
checkInId,
|
||||
|
|
@ -131,7 +127,7 @@ setTimeout(process.exit, 1000);
|
|||
* @param breach breach object from HIBP
|
||||
* @returns Boolean is it a valid breach
|
||||
*/
|
||||
function isValidBreach(breach: Breach) {
|
||||
function isValidBreach(breach: HibpGetBreachesResponse[number]) {
|
||||
return (
|
||||
breach.Name !== undefined &&
|
||||
breach.BreachDate !== undefined &&
|
||||
|
|
|
|||
|
|
@ -35,10 +35,9 @@
|
|||
|
||||
import AppConstants from "../../appConstants";
|
||||
import * as HIBP from "../../utils/hibp";
|
||||
import type { Breach } from "../../app/functions/universal/breach";
|
||||
|
||||
type RemoteSettingsBreach = Pick<
|
||||
Breach,
|
||||
HIBP.HibpGetBreachesResponse[number],
|
||||
"Name" | "Domain" | "BreachDate" | "PwnCount" | "AddedDate" | "DataClasses"
|
||||
>;
|
||||
|
||||
|
|
@ -48,7 +47,9 @@ const FX_RS_RECORDS = `${FX_RS_COLLECTION}/records`;
|
|||
const FX_RS_WRITER_USER = AppConstants.FX_REMOTE_SETTINGS_WRITER_USER;
|
||||
const FX_RS_WRITER_PASS = AppConstants.FX_REMOTE_SETTINGS_WRITER_PASS;
|
||||
|
||||
async function whichBreachesAreNotInRemoteSettingsYet(breaches: Breach[]) {
|
||||
async function whichBreachesAreNotInRemoteSettingsYet(
|
||||
breaches: HIBP.HibpGetBreachesResponse,
|
||||
) {
|
||||
const response = await fetch(FX_RS_RECORDS, {
|
||||
headers: {
|
||||
Authorization: `Basic ${Buffer.from(FX_RS_WRITER_USER + ":" + FX_RS_WRITER_PASS).toString("base64")}`,
|
||||
|
|
@ -56,7 +57,9 @@ async function whichBreachesAreNotInRemoteSettingsYet(breaches: Breach[]) {
|
|||
});
|
||||
const fxRSRecords = await response.json();
|
||||
const remoteSettingsBreachesSet = new Set(
|
||||
fxRSRecords.body.data.map((b: Breach) => b.Name),
|
||||
fxRSRecords.body.data.map(
|
||||
(b: HIBP.HibpGetBreachesResponse[number]) => b.Name,
|
||||
),
|
||||
);
|
||||
|
||||
return breaches.filter(({ Name }) => !remoteSettingsBreachesSet.has(Name));
|
||||
|
|
@ -99,8 +102,8 @@ if (
|
|||
}
|
||||
|
||||
(async () => {
|
||||
const allHibpBreaches = (await HIBP.req("/breaches")) as { body: Breach[] };
|
||||
const verifiedSiteBreaches = allHibpBreaches.body.filter((breach) => {
|
||||
const allHibpBreaches = await HIBP.fetchHibpBreaches();
|
||||
const verifiedSiteBreaches = allHibpBreaches.filter((breach) => {
|
||||
return (
|
||||
!breach.IsRetired &&
|
||||
!breach.IsSpamList &&
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import createDbConnection from "../db/connect.js";
|
||||
const knex = createDbConnection();
|
||||
|
||||
import { subscribeHash } from "../utils/hibp.js";
|
||||
import { subscribeHash } from "../utils/hibp";
|
||||
import { getSha1 } from "../utils/fxa.js";
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import { getUserEmails } from '../db/tables/emailAddresses.js'
|
||||
import { getBreachesForEmail, getFilteredBreaches } from './hibp.js'
|
||||
import { getBreachesForEmail, getFilteredBreaches } from './hibp'
|
||||
import { getSha1 } from './fxa.js'
|
||||
import { filterBreachDataTypes } from './breachResolution.js'
|
||||
import { captureMessage } from "@sentry/node";
|
||||
|
|
@ -95,7 +95,7 @@ function addRecencyIndex(foundBreaches) {
|
|||
/**
|
||||
* @typedef {{
|
||||
* email: string;
|
||||
* breaches: import('./hibp.js').HibpLikeDbBreach[];
|
||||
* breaches: import('./hibp').HibpLikeDbBreach[];
|
||||
* id: number;
|
||||
* primary: boolean;
|
||||
* verified: boolean;
|
||||
|
|
@ -106,7 +106,7 @@ function addRecencyIndex(foundBreaches) {
|
|||
/**
|
||||
* TODO: deprecate with MNTOR-2021
|
||||
*
|
||||
* @param {{ user: any; email: any; recordId: any; recordVerified: any; allBreaches: import('./hibp.js').HibpLikeDbBreach[]; }} options
|
||||
* @param {{ user: any; email: any; recordId: any; recordVerified: any; allBreaches: import('./hibp').HibpLikeDbBreach[]; }} options
|
||||
* @returns {Promise<BundledVerifiedEmails>}
|
||||
*/
|
||||
async function bundleVerifiedEmails(options) {
|
||||
|
|
|
|||
|
|
@ -1,424 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import AppConstants from '../appConstants.js'
|
||||
import { getAllBreaches, upsertBreaches, knex } from '../db/tables/breaches.js'
|
||||
import { InternalServerError } from '../utils/error.js'
|
||||
import { getMessage } from '../utils/fluent.js'
|
||||
import { isUsingMockHIBPEndpoint } from '../app/functions/universal/mock.ts'
|
||||
const { HIBP_THROTTLE_MAX_TRIES, HIBP_THROTTLE_DELAY, HIBP_API_ROOT, HIBP_KANON_API_ROOT, HIBP_KANON_API_TOKEN } = AppConstants
|
||||
|
||||
|
||||
// TODO: fix hardcode
|
||||
const HIBP_USER_AGENT = 'monitor/1.0.0'
|
||||
// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
|
||||
// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
|
||||
const RENAMED_BREACHES = ['covve']
|
||||
const RENAMED_BREACHES_MAP = {
|
||||
covve: 'db8151dd'
|
||||
}
|
||||
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function _addStandardOptions(options = {}) {
|
||||
const hibpOptions = {
|
||||
headers: {
|
||||
'User-Agent': HIBP_USER_AGENT
|
||||
},
|
||||
...options
|
||||
}
|
||||
return hibpOptions
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {string} url
|
||||
* @param {any | undefined} reqOptions
|
||||
* @param tryCount
|
||||
* @returns {Promise<any>}
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function _throttledFetch(url, reqOptions, tryCount = 1) {
|
||||
try {
|
||||
const response = await fetch(url, reqOptions)
|
||||
if (response.ok) return await response.json()
|
||||
|
||||
switch (response.status) {
|
||||
case 404:
|
||||
// 404 can mean "no results", return undefined response
|
||||
console.info('_throttledFetch', { err: 'Error 404, not going to retry. TryCount: ' + tryCount })
|
||||
return undefined
|
||||
case 429:
|
||||
console.info('_throttledFetch', { err: 'Error 429, tryCount: ' + tryCount })
|
||||
// @ts-ignore TODO: Explicitly parse into a number
|
||||
if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {
|
||||
throw new InternalServerError(getMessage('error-hibp-throttled'))
|
||||
} else {
|
||||
tryCount++
|
||||
// @ts-ignore HIBP_THROTTLE_DELAY should be defined
|
||||
await new Promise(resolve => setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount))
|
||||
return await _throttledFetch(url, reqOptions, tryCount)
|
||||
}
|
||||
default:
|
||||
console.error(await response.text())
|
||||
throw new InternalServerError(`bad response: ${response.status}`)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('_throttledFetch', { err })
|
||||
throw new InternalServerError(getMessage('error-hibp-connect'))
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {string} path
|
||||
* @param options
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function req(path, options = {}) {
|
||||
const url = `${HIBP_API_ROOT}${path}`
|
||||
const reqOptions = _addStandardOptions(options)
|
||||
try {
|
||||
return await _throttledFetch(url, reqOptions)
|
||||
} catch (ex) {
|
||||
console.error(ex)
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {string} path
|
||||
* @param options
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function kAnonReq(path, options = {}) {
|
||||
// Construct HIBP url and standard headers
|
||||
const url = `${HIBP_KANON_API_ROOT}${path}`
|
||||
options = {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "*/*",
|
||||
"Hibp-Enterprise-Api-Key": HIBP_KANON_API_TOKEN
|
||||
},
|
||||
...options
|
||||
}
|
||||
|
||||
const reqOptions = _addStandardOptions(options)
|
||||
try {
|
||||
return await _throttledFetch(url, reqOptions)
|
||||
} catch (ex) {
|
||||
console.error(ex)
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Sanitize data classes
|
||||
* ie. "Email Addresses" -> "email-addresses"
|
||||
*
|
||||
* @param {any[]} dataClasses
|
||||
* @returns Array sanitized data classes array
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function formatDataClassesArray(dataClasses) {
|
||||
return dataClasses.map(dataClass =>
|
||||
dataClass.toLowerCase()
|
||||
.replace(/[^-a-z0-9]/g, '-')
|
||||
.replace(/-{2,}/g, '-')
|
||||
.replace(/(^-|-$)/g, '')
|
||||
)
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* The type `HibpLikeDbBreach` is roughly the same as the data we receive from
|
||||
* HIBP, except that we added a `FaviconUrl`, that AddedDate and ModifiedData
|
||||
* are Date objects, and that a couple of fields are not available (because we
|
||||
* do not store them in our database, at the time of writing).
|
||||
*
|
||||
* @typedef {{ FaviconUrl?: string }} WithFaviconUrl
|
||||
* @typedef {WithFaviconUrl & Omit<import('../app/functions/universal/breach.ts').Breach, "recencyIndex" | "ResolutionsChecked" | "AddedDate" | "ModifiedDate"> & { AddedDate: Date; ModifiedDate: Date; }} HibpLikeDbBreach
|
||||
*/
|
||||
|
||||
/**
|
||||
* Get all breaches from the database table "breaches",
|
||||
* sanitize it, and return a javascript array
|
||||
*
|
||||
* @returns {Promise<HibpLikeDbBreach[]>} formatted all breaches array
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function getAllBreachesFromDb() {
|
||||
/**
|
||||
* @type {any[]}
|
||||
*/
|
||||
let dbBreaches = []
|
||||
try {
|
||||
dbBreaches = await getAllBreaches()
|
||||
} catch (e) {
|
||||
console.error('getAllBreachesFromDb', 'No breaches exist in the database: ' + e)
|
||||
return dbBreaches
|
||||
}
|
||||
|
||||
// TODO: we can do some filtering here for the most commonly used fields
|
||||
// TODO: change field names to camel case
|
||||
return dbBreaches.map(breach => ({
|
||||
Id: breach.id,
|
||||
Name: breach.name,
|
||||
Title: breach.title,
|
||||
Domain: breach.domain,
|
||||
BreachDate: breach.breach_date,
|
||||
AddedDate: breach.added_date,
|
||||
ModifiedDate: breach.modified_date,
|
||||
PwnCount: breach.pwn_count,
|
||||
Description: breach.description,
|
||||
LogoPath: breach.logo_path,
|
||||
DataClasses: breach.data_classes,
|
||||
IsVerified: breach.is_verified,
|
||||
IsFabricated: breach.is_fabricated,
|
||||
IsSensitive: breach.is_sensitive,
|
||||
IsRetired: breach.is_retired,
|
||||
IsSpamList: breach.is_spam_list,
|
||||
IsMalware: breach.is_malware,
|
||||
FaviconUrl: breach.favicon_url,
|
||||
}))
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {{ locals: { breachLogoMap: Map<any, any>; breaches: any[]; breachesLoadedDateTime: number; }; }} app
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function loadBreachesIntoApp(app) {
|
||||
// attempt to fetch breaches from the "breaches" database table
|
||||
const breaches = await getAllBreachesFromDb()
|
||||
console.debug('loadBreachesIntoApp', `loaded breaches from database: ${breaches.length}`)
|
||||
|
||||
// if "breaches" table does not return results, fall back to HIBP request
|
||||
if (breaches?.length < 1) {
|
||||
const breachesResponse = await req('/breaches')
|
||||
console.debug('loadBreachesIntoApp', `loaded breaches from HIBP: ${breachesResponse.length}`)
|
||||
|
||||
for (const breach of breachesResponse) {
|
||||
breach.DataClasses = formatDataClassesArray(breach.DataClasses)
|
||||
// @ts-ignore The result should be set
|
||||
breach.LogoPath = /[^/]*$/.exec(breach.LogoPath)[0]
|
||||
breaches.push(breach)
|
||||
}
|
||||
|
||||
// sync the "breaches" table with the latest from HIBP
|
||||
await upsertBreaches(breaches)
|
||||
}
|
||||
// This will be replaced by a map with the breach logos when
|
||||
// `downloadBreachIcons` resolves, but by setting it to an empty Map first,
|
||||
// we don't delay the server start - we just won't have breach logos yet.
|
||||
app.locals.breachLogoMap = new Map()
|
||||
app.locals.breaches = breaches
|
||||
app.locals.breachesLoadedDateTime = Date.now()
|
||||
console.info('done-loading-breaches', 'great success 👍')
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Get addresses and language from either subscribers or email_addresses fields:
|
||||
*
|
||||
* @param {import('knex/types/tables').SubscriberRow | (import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow)} recipient
|
||||
* @returns
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getAddressesAndLanguageForEmail(recipient) {
|
||||
if (hasEmailAddressAttached(recipient)) {
|
||||
return {
|
||||
breachedEmail: recipient.email,
|
||||
recipientEmail: recipient.all_emails_to_primary ? recipient.primary_email : recipient.email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
breachedEmail: recipient.primary_email,
|
||||
recipientEmail: recipient.primary_email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Filter breaches that we would not like to show.
|
||||
*
|
||||
* @param {any[]} breaches
|
||||
* @returns {any[]} filteredBreaches
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getFilteredBreaches(breaches) {
|
||||
return breaches.filter(breach => (
|
||||
!breach.IsRetired &&
|
||||
!breach.IsSpamList &&
|
||||
!breach.IsFabricated &&
|
||||
breach.IsVerified &&
|
||||
breach.Domain !== ''
|
||||
))
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* A range of hashes can be searched by passing the hash prefix in a GET request:
|
||||
* GET /breachedaccount/range/[hash prefix]
|
||||
*
|
||||
* @param {string} sha1 first 6 chars of email sha1
|
||||
* @param {any[]} allBreaches
|
||||
* @param {boolean} includeSensitive
|
||||
* @param {boolean} filterBreaches
|
||||
* @returns
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function getBreachesForEmail(sha1, allBreaches, includeSensitive = false, filterBreaches = true) {
|
||||
let foundBreaches = []
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase()
|
||||
const path = `/range/search/${sha1Prefix}`
|
||||
|
||||
const response = await kAnonReq(path)
|
||||
if (!response || (response && response.length < 1)) {
|
||||
console.error("failed_kAnonReq_call: no response or empty response")
|
||||
return []
|
||||
}
|
||||
if (isUsingMockHIBPEndpoint()) {
|
||||
let mockDataBreaches = response[0];
|
||||
return allBreaches.filter(breach => mockDataBreaches.websites.includes(breach.Name)).sort((a, b) => {
|
||||
// @ts-ignore TODO: Turn dates into a number
|
||||
return new Date(b.AddedDate) - new Date(a.AddedDate)
|
||||
})
|
||||
}
|
||||
// Parse response body, format:
|
||||
// [
|
||||
// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
|
||||
// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
|
||||
// ]
|
||||
for (const breachedAccount of response) {
|
||||
if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
|
||||
foundBreaches = allBreaches.filter(breach => breachedAccount.websites.includes(breach.Name))
|
||||
if (filterBreaches) {
|
||||
foundBreaches = getFilteredBreaches(foundBreaches)
|
||||
}
|
||||
|
||||
// NOTE: DO NOT CHANGE THIS SORT LOGIC
|
||||
// We store breach resolutions by recency indices,
|
||||
// so that our DB does not contain any part of any user's list of accounts
|
||||
foundBreaches.sort((a, b) => {
|
||||
// @ts-ignore TODO: Turn dates into a number
|
||||
return new Date(b.AddedDate) - new Date(a.AddedDate)
|
||||
})
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (includeSensitive) {
|
||||
return foundBreaches
|
||||
}
|
||||
return foundBreaches.filter(
|
||||
breach => !breach.IsSensitive
|
||||
)
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {any[]} allBreaches
|
||||
* @param {string} breachName
|
||||
* @returns {HibpLikeDbBreach}
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getBreachByName(allBreaches, breachName) {
|
||||
breachName = breachName.toLowerCase()
|
||||
if (RENAMED_BREACHES.includes(breachName)) {
|
||||
// @ts-ignore Converted from regular JS
|
||||
breachName = RENAMED_BREACHES_MAP[breachName]
|
||||
}
|
||||
const foundBreach = allBreaches.find(breach => breach.Name.toLowerCase() === breachName)
|
||||
return foundBreach
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* A range can be subscribed for callbacks with the following request:
|
||||
* POST /range/subscribe
|
||||
* {
|
||||
* hashPrefix:"[hash prefix]"
|
||||
* }
|
||||
* There are two possible response codes that can be returned:
|
||||
* 1. HTTP 201: New range subscription has been created
|
||||
* 2. HTTP 200: Range subscription already exists
|
||||
*
|
||||
* @param {string} sha1 sha1 of the email being subscribed
|
||||
* @returns 200 or 201 response codes
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function subscribeHash(sha1) {
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase()
|
||||
const path = '/range/subscribe'
|
||||
const options = {
|
||||
Method: 'POST',
|
||||
Body: { hashPrefix: sha1Prefix }
|
||||
}
|
||||
|
||||
return await kAnonReq(path, options)
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* A range subscription can be deleted with the following request:
|
||||
* DELETE /range/[hash prefix]
|
||||
*
|
||||
* There is one possible response code that can be returned:
|
||||
* HTTP 200: Range subscription successfully deleted
|
||||
*
|
||||
* @param {string} sha1 sha1 of the email being subscribed
|
||||
* @returns 200 response codes
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function deleteSubscribedHash(sha1) {
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase()
|
||||
const path = `/range/${sha1Prefix}`
|
||||
const options = {
|
||||
Method: 'DELETE'
|
||||
}
|
||||
|
||||
return await kAnonReq(path, options)
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* @param {import('knex/types/tables').SubscriberRow} subscriberRow
|
||||
* @returns {subscriberRow is import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow}
|
||||
*/
|
||||
function hasEmailAddressAttached(subscriberRow) {
|
||||
return typeof (/** @type {import('knex/types/tables').SubscriberRow & import('knex/types/tables').EmailAddressRow} */ (subscriberRow)).email === "string";
|
||||
}
|
||||
|
||||
export {
|
||||
req,
|
||||
kAnonReq,
|
||||
formatDataClassesArray,
|
||||
loadBreachesIntoApp,
|
||||
getAddressesAndLanguageForEmail,
|
||||
getBreachesForEmail,
|
||||
getBreachByName,
|
||||
getAllBreachesFromDb,
|
||||
getFilteredBreaches,
|
||||
subscribeHash,
|
||||
deleteSubscribedHash,
|
||||
knex as knexHibp
|
||||
}
|
||||
|
|
@ -0,0 +1,443 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import AppConstants from "../appConstants.js";
|
||||
import { getAllBreaches, knex } from "../db/tables/breaches.js";
|
||||
import { InternalServerError } from "./error.js";
|
||||
import { getMessage } from "./fluent.js";
|
||||
import { isUsingMockHIBPEndpoint } from "../app/functions/universal/mock.ts";
|
||||
import { BreachRow, EmailAddressRow, SubscriberRow } from "knex/types/tables";
|
||||
import { ISO8601DateString } from "./parse.js";
|
||||
import { HibpBreachDataTypes } from "../app/functions/universal/breach.ts";
|
||||
const {
|
||||
HIBP_THROTTLE_MAX_TRIES,
|
||||
HIBP_THROTTLE_DELAY,
|
||||
HIBP_API_ROOT,
|
||||
HIBP_KANON_API_ROOT,
|
||||
HIBP_KANON_API_TOKEN,
|
||||
} = AppConstants;
|
||||
|
||||
// TODO: fix hardcode
|
||||
const HIBP_USER_AGENT = "monitor/1.0.0";
|
||||
// When HIBP "re-names" a breach, it keeps its old 'Name' value but gets a new 'Title'
|
||||
// We use 'Name' in Firefox (via Remote Settings), so we have to maintain our own mapping of re-named breaches.
|
||||
const RENAMED_BREACHES = ["covve"];
|
||||
const RENAMED_BREACHES_MAP = {
|
||||
covve: "db8151dd",
|
||||
};
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function _addStandardOptions(options = {}) {
|
||||
const hibpOptions = {
|
||||
headers: {
|
||||
"User-Agent": HIBP_USER_AGENT,
|
||||
},
|
||||
...options,
|
||||
};
|
||||
return hibpOptions;
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function _throttledFetch(
|
||||
url: string,
|
||||
reqOptions?: RequestInit,
|
||||
tryCount = 1,
|
||||
) {
|
||||
try {
|
||||
const response = await fetch(url, reqOptions);
|
||||
if (response.ok) return (await response.json()) as unknown;
|
||||
|
||||
switch (response.status) {
|
||||
case 404:
|
||||
// 404 can mean "no results", return undefined response
|
||||
console.info("_throttledFetch", {
|
||||
err: "Error 404, not going to retry. TryCount: " + tryCount,
|
||||
});
|
||||
return undefined;
|
||||
case 429:
|
||||
console.info("_throttledFetch", {
|
||||
err: "Error 429, tryCount: " + tryCount,
|
||||
});
|
||||
// @ts-ignore TODO: Explicitly parse into a number
|
||||
if (tryCount >= HIBP_THROTTLE_MAX_TRIES) {
|
||||
throw new InternalServerError(getMessage("error-hibp-throttled"));
|
||||
} else {
|
||||
tryCount++;
|
||||
await new Promise((resolve) =>
|
||||
// @ts-ignore HIBP_THROTTLE_DELAY should be defined
|
||||
setTimeout(resolve, HIBP_THROTTLE_DELAY * tryCount),
|
||||
);
|
||||
return await _throttledFetch(url, reqOptions, tryCount);
|
||||
}
|
||||
default:
|
||||
console.error(await response.text());
|
||||
throw new InternalServerError(`bad response: ${response.status}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("_throttledFetch", { err });
|
||||
throw new InternalServerError(getMessage("error-hibp-connect"));
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function hibpApiFetch(path: string, options = {}) {
|
||||
const url = `${HIBP_API_ROOT}${path}`;
|
||||
const reqOptions = _addStandardOptions(options);
|
||||
try {
|
||||
return await _throttledFetch(url, reqOptions);
|
||||
} catch (ex) {
|
||||
console.error(ex);
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
export type HibpGetBreachesResponse = Array<{
|
||||
Name: string;
|
||||
Title: string;
|
||||
Domain: string;
|
||||
AddedDate: ISO8601DateString;
|
||||
BreachDate: ISO8601DateString;
|
||||
DataClasses: Array<keyof HibpBreachDataTypes>;
|
||||
Description: string;
|
||||
LogoPath: string;
|
||||
IsFabricated: boolean;
|
||||
IsMalware: boolean;
|
||||
IsRetired: boolean;
|
||||
IsSensitive: boolean;
|
||||
IsSpamList: boolean;
|
||||
IsVerified: boolean;
|
||||
ModifiedDate: ISO8601DateString;
|
||||
PwnCount: number;
|
||||
}>;
|
||||
|
||||
export async function fetchHibpBreaches(): Promise<HibpGetBreachesResponse> {
|
||||
return (await hibpApiFetch("/breaches")) as HibpGetBreachesResponse;
|
||||
}
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function kAnonReq(path: string, options = {}) {
|
||||
// Construct HIBP url and standard headers
|
||||
const url = `${HIBP_KANON_API_ROOT}${path}`;
|
||||
options = {
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "*/*",
|
||||
"Hibp-Enterprise-Api-Key": HIBP_KANON_API_TOKEN,
|
||||
},
|
||||
...options,
|
||||
};
|
||||
|
||||
const reqOptions = _addStandardOptions(options);
|
||||
try {
|
||||
return await _throttledFetch(url, reqOptions);
|
||||
} catch (ex) {
|
||||
console.error(ex);
|
||||
}
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Sanitize data classes
|
||||
* ie. "Email Addresses" -> "email-addresses"
|
||||
*
|
||||
* @returns Array sanitized data classes array
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function formatDataClassesArray(dataClasses: Array<keyof HibpBreachDataTypes>) {
|
||||
return dataClasses.map((dataClass) =>
|
||||
dataClass
|
||||
.toLowerCase()
|
||||
// Replace non-alphanumeric characters, except for dashes, by dashes:
|
||||
.replace(/[^-a-z0-9]/g, "-")
|
||||
// Replace 2 or more consecutive dashes by a single dash:
|
||||
.replace(/-{2,}/g, "-")
|
||||
// Remove dashes at the start or end of the data class:
|
||||
.replace(/(^-|-$)/g, ""),
|
||||
) as Array<HibpBreachDataTypes[keyof HibpBreachDataTypes]>;
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* The type `HibpLikeDbBreach` is roughly the same as the data we receive from
|
||||
* HIBP, except that we added a `FaviconUrl` and ID, that AddedDate,
|
||||
* ModifiedDate and BreachDate are Date objects, and that a couple of fields are
|
||||
* not available (because we do not store them in our database, at the time of
|
||||
* writing).
|
||||
*
|
||||
* Presumably, our code used to be set up in such a way that it was supposed to
|
||||
* work with objects that were either returned from the HIBP API, _or_ from our
|
||||
* database - but the latter doesn't capitalise the properties.
|
||||
*/
|
||||
export type HibpLikeDbBreach = {
|
||||
Id: number;
|
||||
Name: BreachRow["name"];
|
||||
Title: BreachRow["title"];
|
||||
Domain: BreachRow["domain"];
|
||||
BreachDate: BreachRow["breach_date"];
|
||||
AddedDate: BreachRow["added_date"];
|
||||
ModifiedDate: BreachRow["modified_date"];
|
||||
PwnCount: BreachRow["pwn_count"];
|
||||
Description: BreachRow["description"];
|
||||
LogoPath: BreachRow["logo_path"];
|
||||
DataClasses: BreachRow["data_classes"];
|
||||
IsVerified: BreachRow["is_verified"];
|
||||
IsFabricated: BreachRow["is_fabricated"];
|
||||
IsSensitive: BreachRow["is_sensitive"];
|
||||
IsRetired: BreachRow["is_retired"];
|
||||
IsSpamList: BreachRow["is_spam_list"];
|
||||
IsMalware: BreachRow["is_malware"];
|
||||
FaviconUrl?: BreachRow["favicon_url"];
|
||||
};
|
||||
|
||||
/**
|
||||
* Get all breaches from the database table "breaches",
|
||||
* sanitize it, and return a javascript array
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function getAllBreachesFromDb(): Promise<HibpLikeDbBreach[]> {
|
||||
let dbBreaches: BreachRow[] = [];
|
||||
try {
|
||||
dbBreaches = await getAllBreaches();
|
||||
} catch (e) {
|
||||
console.error(
|
||||
"getAllBreachesFromDb",
|
||||
"No breaches exist in the database: " + (e as string),
|
||||
);
|
||||
return [];
|
||||
}
|
||||
|
||||
// TODO: we can do some filtering here for the most commonly used fields
|
||||
// TODO: change field names to camel case
|
||||
return dbBreaches.map(
|
||||
(breach) =>
|
||||
({
|
||||
Id: breach.id,
|
||||
Name: breach.name,
|
||||
Title: breach.title,
|
||||
Domain: breach.domain,
|
||||
BreachDate: breach.breach_date,
|
||||
AddedDate: breach.added_date,
|
||||
ModifiedDate: breach.modified_date,
|
||||
PwnCount: breach.pwn_count,
|
||||
Description: breach.description,
|
||||
LogoPath: breach.logo_path,
|
||||
DataClasses: breach.data_classes,
|
||||
IsVerified: breach.is_verified,
|
||||
IsFabricated: breach.is_fabricated,
|
||||
IsSensitive: breach.is_sensitive,
|
||||
IsRetired: breach.is_retired,
|
||||
IsSpamList: breach.is_spam_list,
|
||||
IsMalware: breach.is_malware,
|
||||
FaviconUrl: breach.favicon_url,
|
||||
}) as HibpLikeDbBreach,
|
||||
);
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Get addresses and language from either subscribers or email_addresses fields:
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getAddressesAndLanguageForEmail(
|
||||
recipient: SubscriberRow | (SubscriberRow & EmailAddressRow),
|
||||
) {
|
||||
if (hasEmailAddressAttached(recipient)) {
|
||||
return {
|
||||
breachedEmail: recipient.email,
|
||||
recipientEmail: recipient.all_emails_to_primary
|
||||
? recipient.primary_email
|
||||
: recipient.email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
breachedEmail: recipient.primary_email,
|
||||
recipientEmail: recipient.primary_email,
|
||||
signupLanguage: recipient.signup_language,
|
||||
};
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* Filter breaches that we would not like to show.
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getFilteredBreaches(breaches: HibpLikeDbBreach[]): HibpLikeDbBreach[] {
|
||||
return breaches.filter(
|
||||
(breach) =>
|
||||
!breach.IsRetired &&
|
||||
!breach.IsSpamList &&
|
||||
!breach.IsFabricated &&
|
||||
breach.IsVerified &&
|
||||
breach.Domain !== "",
|
||||
);
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
export type BreachedAccountResponse = Array<{
|
||||
hashSuffix: string;
|
||||
websites: string[];
|
||||
}>;
|
||||
|
||||
/**
|
||||
* A range of hashes can be searched by passing the hash prefix in a GET request:
|
||||
* GET /breachedaccount/range/[hash prefix]
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function getBreachesForEmail(
|
||||
sha1: string,
|
||||
allBreaches: HibpLikeDbBreach[],
|
||||
includeSensitive = false,
|
||||
filterBreaches = true,
|
||||
): Promise<HibpLikeDbBreach[]> {
|
||||
let foundBreaches: HibpLikeDbBreach[] = [];
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase();
|
||||
const path = `/range/search/${sha1Prefix}`;
|
||||
|
||||
const response = (await kAnonReq(path)) as
|
||||
| BreachedAccountResponse
|
||||
| undefined;
|
||||
if (!response || (response && response.length < 1)) {
|
||||
console.error("failed_kAnonReq_call: no response or empty response");
|
||||
return [];
|
||||
}
|
||||
if (isUsingMockHIBPEndpoint()) {
|
||||
const mockDataBreaches = response[0];
|
||||
return allBreaches
|
||||
.filter((breach) => mockDataBreaches.websites.includes(breach.Name))
|
||||
.sort((a, b) => {
|
||||
// @ts-ignore TODO: Turn dates into a number
|
||||
return new Date(b.AddedDate) - new Date(a.AddedDate);
|
||||
});
|
||||
}
|
||||
// Parse response body, format:
|
||||
// [
|
||||
// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
|
||||
// {"hashSuffix":<suffix>,"websites":[<breach1Name>,...]},
|
||||
// ]
|
||||
for (const breachedAccount of response) {
|
||||
if (sha1.toUpperCase() === sha1Prefix + breachedAccount.hashSuffix) {
|
||||
foundBreaches = allBreaches.filter((breach) =>
|
||||
breachedAccount.websites.includes(breach.Name),
|
||||
);
|
||||
if (filterBreaches) {
|
||||
foundBreaches = getFilteredBreaches(foundBreaches);
|
||||
}
|
||||
|
||||
// NOTE: DO NOT CHANGE THIS SORT LOGIC
|
||||
// We store breach resolutions by recency indices,
|
||||
// so that our DB does not contain any part of any user's list of accounts
|
||||
foundBreaches.sort((a, b) => {
|
||||
// @ts-ignore TODO: Turn dates into a number
|
||||
return new Date(b.AddedDate) - new Date(a.AddedDate);
|
||||
});
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (includeSensitive) {
|
||||
return foundBreaches;
|
||||
}
|
||||
return foundBreaches.filter((breach) => !breach.IsSensitive);
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
function getBreachByName(allBreaches: HibpLikeDbBreach[], breachName: string) {
|
||||
breachName = breachName.toLowerCase();
|
||||
if (RENAMED_BREACHES.includes(breachName)) {
|
||||
// @ts-ignore Converted from regular JS
|
||||
breachName = RENAMED_BREACHES_MAP[breachName];
|
||||
}
|
||||
const foundBreach = allBreaches.find(
|
||||
(breach) => breach.Name.toLowerCase() === breachName,
|
||||
);
|
||||
return foundBreach;
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* A range can be subscribed for callbacks with the following request:
|
||||
* POST /range/subscribe
|
||||
* {
|
||||
* hashPrefix:"[hash prefix]"
|
||||
* }
|
||||
* There are two possible response codes that can be returned:
|
||||
* 1. HTTP 201: New range subscription has been created
|
||||
* 2. HTTP 200: Range subscription already exists
|
||||
*
|
||||
* @param sha1 sha1 of the email being subscribed
|
||||
* @returns 200 or 201 response codes
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function subscribeHash(sha1: string) {
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase();
|
||||
const path = "/range/subscribe";
|
||||
const options = {
|
||||
Method: "POST",
|
||||
Body: { hashPrefix: sha1Prefix },
|
||||
};
|
||||
|
||||
return await kAnonReq(path, options);
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
/**
|
||||
* A range subscription can be deleted with the following request:
|
||||
* DELETE /range/[hash prefix]
|
||||
*
|
||||
* There is one possible response code that can be returned:
|
||||
* HTTP 200: Range subscription successfully deleted
|
||||
*
|
||||
* @param {string} sha1 sha1 of the email being subscribed
|
||||
* @returns 200 response codes
|
||||
*/
|
||||
// TODO: Add unit test when changing this code:
|
||||
/* c8 ignore start */
|
||||
async function deleteSubscribedHash(sha1: string) {
|
||||
const sha1Prefix = sha1.slice(0, 6).toUpperCase();
|
||||
const path = `/range/${sha1Prefix}`;
|
||||
const options = {
|
||||
Method: "DELETE",
|
||||
};
|
||||
|
||||
return await kAnonReq(path, options);
|
||||
}
|
||||
/* c8 ignore stop */
|
||||
|
||||
function hasEmailAddressAttached(
|
||||
subscriberRow: SubscriberRow,
|
||||
): subscriberRow is SubscriberRow & EmailAddressRow {
|
||||
return (
|
||||
typeof (subscriberRow as SubscriberRow & EmailAddressRow).email === "string"
|
||||
);
|
||||
}
|
||||
|
||||
export {
|
||||
kAnonReq,
|
||||
formatDataClassesArray,
|
||||
getAddressesAndLanguageForEmail,
|
||||
getBreachesForEmail,
|
||||
getBreachByName,
|
||||
getAllBreachesFromDb,
|
||||
getFilteredBreaches,
|
||||
subscribeHash,
|
||||
deleteSubscribedHash,
|
||||
knex as knexHibp,
|
||||
};
|
||||
|
|
@ -3,16 +3,15 @@
|
|||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
import { SubscriberRow } from "knex/types/tables";
|
||||
import { getBreachesForEmail } from "./hibp";
|
||||
import { getBreachesForEmail, HibpLikeDbBreach } from "./hibp";
|
||||
import { getSubBreaches } from "./subscriberBreaches";
|
||||
import { getUserEmails } from "../db/tables/emailAddresses";
|
||||
import { Breach } from "../app/functions/universal/breach";
|
||||
|
||||
jest.mock("../db/tables/emailAddresses.js", () => ({
|
||||
getUserEmails: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock("./hibp.js", () => ({
|
||||
jest.mock("./hibp", () => ({
|
||||
getBreachesForEmail: jest.fn(),
|
||||
}));
|
||||
|
||||
|
|
@ -91,15 +90,15 @@ const subscriber: SubscriberRow = {
|
|||
first_broker_removal_email_sent: false,
|
||||
};
|
||||
|
||||
const allBreaches: Breach[] = [
|
||||
const allBreaches: HibpLikeDbBreach[] = [
|
||||
{
|
||||
Id: 627,
|
||||
Name: "Youku",
|
||||
Title: "Youku",
|
||||
Domain: "youku.com",
|
||||
BreachDate: "2016-12-01T08:00:00.000Z",
|
||||
AddedDate: "2017-04-15T11:02:35.000Z",
|
||||
ModifiedDate: "2017-04-15T11:02:35.000Z",
|
||||
BreachDate: new Date("2016-12-01T08:00:00.000Z"),
|
||||
AddedDate: new Date("2017-04-15T11:02:35.000Z"),
|
||||
ModifiedDate: new Date("2017-04-15T11:02:35.000Z"),
|
||||
PwnCount: 91890110,
|
||||
Description:
|
||||
'In late 2016, the online Chinese video service <a href="http://www.youku.com" target="_blank" rel="noopener">Youku</a> suffered a data breach. The incident exposed 92 million unique user accounts and corresponding MD5 password hashes. The data was contributed to Have I Been Pwned courtesy of rip@creep.im.',
|
||||
|
|
@ -111,30 +110,26 @@ const allBreaches: Breach[] = [
|
|||
IsRetired: false,
|
||||
IsSpamList: false,
|
||||
IsMalware: false,
|
||||
recencyIndex: 1,
|
||||
ResolutionsChecked: [],
|
||||
},
|
||||
{
|
||||
Id: 638,
|
||||
Name: "Zynga",
|
||||
Title: "Zynga",
|
||||
Domain: "zynga.com",
|
||||
BreachDate: "2019-09-01T07:00:00.000Z",
|
||||
AddedDate: "2019-12-19T04:54:45.000Z",
|
||||
ModifiedDate: "2020-01-11T00:41:51.000Z",
|
||||
BreachDate: new Date("2019-09-01T07:00:00.000Z"),
|
||||
AddedDate: new Date("2019-12-19T04:54:45.000Z"),
|
||||
ModifiedDate: new Date("2020-01-11T00:41:51.000Z"),
|
||||
PwnCount: 172869660,
|
||||
Description:
|
||||
'In September 2019, game developer <a href="https://www.cnet.com/news/words-with-friends-hack-reportedly-exposes-data-of-more-than-200m-players/" target="_blank" rel="noopener">Zynga (the creator of Words with Friends) suffered a data breach</a>. The incident exposed 173M unique email addresses alongside usernames and passwords stored as salted SHA-1 hashes. The data was provided to HIBP by <a href="https://dehashed.com/" target="_blank" rel="noopener">dehashed.com</a>.',
|
||||
LogoPath: "Zynga.png",
|
||||
DataClasses: ["email-addresses", "passwords", "phone-numbers", "usernames"],
|
||||
DataClasses: ["email-addresses", "passwords", "phone-numbers"],
|
||||
IsVerified: true,
|
||||
IsFabricated: false,
|
||||
IsSensitive: false,
|
||||
IsRetired: false,
|
||||
IsSpamList: false,
|
||||
IsMalware: false,
|
||||
recencyIndex: 2,
|
||||
ResolutionsChecked: [],
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -157,7 +152,7 @@ const breachesWithNoneResolved = [
|
|||
Domain: "something",
|
||||
DataClasses: ["email-addresses", "passwords", "something else"],
|
||||
},
|
||||
];
|
||||
] as HibpLikeDbBreach[];
|
||||
|
||||
const breachesWithOneResolved = [
|
||||
{
|
||||
|
|
@ -178,7 +173,7 @@ const breachesWithOneResolved = [
|
|||
Domain: "something",
|
||||
DataClasses: ["email-addresses", "passwords", "something else"],
|
||||
},
|
||||
];
|
||||
] as HibpLikeDbBreach[];
|
||||
|
||||
const breachesWithOneResolvedSsn = [
|
||||
{
|
||||
|
|
@ -204,7 +199,7 @@ const breachesWithOneResolvedSsn = [
|
|||
"something else",
|
||||
],
|
||||
},
|
||||
];
|
||||
] as HibpLikeDbBreach[];
|
||||
|
||||
describe("getSubBreaches", () => {
|
||||
it("summarises which dataClasses and emails are breached for the given user", async () => {
|
||||
|
|
@ -458,9 +453,9 @@ describe("getSubBreaches", () => {
|
|||
...breach,
|
||||
// Make sure the found breaches have ISO 8601 date strings, rather than
|
||||
// Date objects:
|
||||
BreachDate: "2016-12-01T08:00:00.000Z",
|
||||
AddedDate: "2017-04-15T11:02:35.000Z",
|
||||
ModifiedDate: "2017-04-15T11:02:35.000Z",
|
||||
BreachDate: "2016-12-01T08:00:00.000Z" as unknown as Date,
|
||||
AddedDate: "2017-04-15T11:02:35.000Z" as unknown as Date,
|
||||
ModifiedDate: "2017-04-15T11:02:35.000Z" as unknown as Date,
|
||||
})),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ import {
|
|||
SubscriberRow,
|
||||
} from "knex/types/tables";
|
||||
import { getUserEmails } from "../db/tables/emailAddresses.js";
|
||||
import { HibpLikeDbBreach, getBreachesForEmail } from "./hibp.js";
|
||||
import { HibpLikeDbBreach, getBreachesForEmail } from "./hibp";
|
||||
import { getSha1 } from "./fxa.js";
|
||||
import { parseIso8601Datetime } from "./parse.js";
|
||||
import {
|
||||
Breach,
|
||||
BreachDataTypes,
|
||||
HibpBreachDataTypes,
|
||||
ResolutionRelevantBreachDataTypes,
|
||||
|
|
@ -27,11 +26,11 @@ export interface SubscriberBreach {
|
|||
breachDate: Date;
|
||||
dataClasses: Array<HibpBreachDataTypes[keyof HibpBreachDataTypes]>;
|
||||
resolvedDataClasses: Array<HibpBreachDataTypes[keyof HibpBreachDataTypes]>;
|
||||
description: string;
|
||||
description: string | null;
|
||||
domain: string;
|
||||
id: number;
|
||||
isResolved?: boolean;
|
||||
favIconUrl: string;
|
||||
favIconUrl: string | null;
|
||||
modifiedDate: Date;
|
||||
name: string;
|
||||
title: string;
|
||||
|
|
@ -73,7 +72,7 @@ function filterBreachDataTypes(
|
|||
*/
|
||||
export async function getSubBreaches(
|
||||
subscriber: SubscriberRow,
|
||||
allBreaches: (Breach | HibpLikeDbBreach)[],
|
||||
allBreaches: HibpLikeDbBreach[],
|
||||
countryCode: string,
|
||||
) {
|
||||
const uniqueBreaches: SubscriberBreachMap = {};
|
||||
|
|
@ -137,7 +136,7 @@ export async function getSubBreaches(
|
|||
|
||||
// `allBreaches` is generally the return value of `getBreaches`, which
|
||||
// either loads breaches from the database, or fetches them from the HIBP
|
||||
// API. In the former csae, `AddedDate`, `BreachDate` and `ModifiedDate`
|
||||
// API. In the former case, `AddedDate`, `BreachDate` and `ModifiedDate`
|
||||
// are Date objects, but in the latter case, they are ISO 8601 date
|
||||
// strings. Thus, we normalise that to always be a Date object.
|
||||
const subscriberBreach: SubscriberBreach = {
|
||||
|
|
@ -149,7 +148,7 @@ export async function getSubBreaches(
|
|||
description: breach.Description,
|
||||
domain: breach.Domain,
|
||||
isResolved: isBreachResolved(dataClassesEffected, resolvedDataClasses),
|
||||
favIconUrl: breach.FaviconUrl,
|
||||
favIconUrl: breach.FaviconUrl ?? null,
|
||||
modifiedDate: normalizeDate(breach.ModifiedDate),
|
||||
name: breach.Name,
|
||||
title: breach.Title,
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче