Merge branch 'main' into dependabot/npm_and_yarn/knex-3.1.0
This commit is contained in:
Robert Helmer
GitHub
Коммит
d4e3463cef
|
|
@ -27,11 +27,16 @@
|
|||
},
|
||||
"[css]": {
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.fixAll.stylelint": true
|
||||
"source.fixAll.stylelint": "explicit"
|
||||
}
|
||||
},
|
||||
"gitlens.advanced.blame.customArguments": [
|
||||
"--ignore-revs-file",
|
||||
".git-blame-ignore-revs"
|
||||
],
|
||||
"[javascript][typescript][typescriptreact]": {
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.fixAll.eslint": "explicit"
|
||||
}
|
||||
},
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
Firefox Monitor notifies users when their credentials have been compromised in a data breach.
|
||||
|
||||
This code is for the monitor.firefox.com service & website.
|
||||
This code is for the monitor.mozilla.org service & website.
|
||||
|
||||
Breach data is powered by [haveibeenpwned.com](https://haveibeenpwned.com/).
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ const nextConfig = {
|
|||
protocol: "https",
|
||||
hostname: "monitor.firefox.com",
|
||||
},
|
||||
{
|
||||
protocol: "https",
|
||||
hostname: "monitor.mozilla.org",
|
||||
},
|
||||
{
|
||||
protocol: "https",
|
||||
hostname: "firefoxusercontent.com",
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -48,8 +48,8 @@
|
|||
"npm": "10.2.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.465.0",
|
||||
"@aws-sdk/lib-storage": "^3.465.0",
|
||||
"@aws-sdk/client-s3": "^3.470.0",
|
||||
"@aws-sdk/lib-storage": "^3.470.0",
|
||||
"@fluent/bundle": "^0.18.0",
|
||||
"@fluent/langneg": "^0.7.0",
|
||||
"@fluent/react": "^0.15.2",
|
||||
|
|
@ -58,9 +58,9 @@
|
|||
"@grpc/grpc-js": "1.9.13",
|
||||
"@leeoniya/ufuzzy": "^1.0.13",
|
||||
"@mozilla/glean": "3.0.0",
|
||||
"@sentry/nextjs": "^7.84.0",
|
||||
"@sentry/nextjs": "^7.86.0",
|
||||
"@sentry/node": "^7.58.1",
|
||||
"@sentry/tracing": "^7.84.0",
|
||||
"@sentry/tracing": "^7.86.0",
|
||||
"@types/jsdom": "^21.1.5",
|
||||
"@types/node": "^20.10.0",
|
||||
"@types/react": "^18.2.38",
|
||||
|
|
|
|||
|
|
@ -1,51 +0,0 @@
|
|||
'use strict'
|
||||
|
||||
const Knex = require('knex')
|
||||
const knexConfig = require('../db/knexfile')
|
||||
const knex = Knex(knexConfig)
|
||||
|
||||
const HIBP = require('../hibp')
|
||||
const getSha1 = require('../sha1-utils')
|
||||
|
||||
/**
|
||||
* OBSOLETE
|
||||
* One-off script to lowercase email addresses of subscribers before hashing
|
||||
*/
|
||||
async function subscribeLowercaseHashToHIBP (emailAddress) {
|
||||
const lowerCasedEmail = emailAddress.toLowerCase()
|
||||
const lowerCasedSha1 = getSha1(lowerCasedEmail)
|
||||
await HIBP.subscribeHash(lowerCasedSha1)
|
||||
return lowerCasedSha1
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const subRecordsWithUpperChars = await knex.select('id', 'primary_email').from('subscribers')
|
||||
.whereRaw('primary_email != lower(primary_email)')
|
||||
const subsWithUpperCount = subRecordsWithUpperChars.length
|
||||
console.log(`found ${subsWithUpperCount} subscribers records with primary_email != lower(primary_email). fixing ...`)
|
||||
for (const subRecord of subRecordsWithUpperChars) {
|
||||
const lowerCasedSha1 = await subscribeLowercaseHashToHIBP(subRecord.primary_email)
|
||||
await knex('subscribers')
|
||||
.update({
|
||||
primary_sha1: lowerCasedSha1
|
||||
})
|
||||
.where('id', subRecord.id)
|
||||
console.log(`fixed subscribers record ID: ${subRecord.id}`)
|
||||
}
|
||||
|
||||
const emailRecordsWithUpperChars = await knex.select('id', 'email').from('email_addresses')
|
||||
.whereRaw('email != lower(email)')
|
||||
const emailsWithUpperCount = emailRecordsWithUpperChars.length
|
||||
console.log(`found ${emailsWithUpperCount} email_addresses records with email != lower(email)`)
|
||||
for (const emailRecord of emailRecordsWithUpperChars) {
|
||||
const lowerCasedSha1 = await subscribeLowercaseHashToHIBP(emailRecord.email)
|
||||
await knex('email_addresses')
|
||||
.update({
|
||||
sha1: lowerCasedSha1
|
||||
})
|
||||
.where('id', emailRecord.id)
|
||||
console.log(`fixed email_addresses record ID: ${emailRecord.id}`)
|
||||
}
|
||||
console.log('done.')
|
||||
process.exit()
|
||||
})()
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
'use strict'
|
||||
|
||||
const Knex = require('knex')
|
||||
const knexConfig = require('../db/knexfile')
|
||||
const knex = Knex(knexConfig)
|
||||
|
||||
const HIBP = require('../hibp')
|
||||
const getSha1 = require('../sha1-utils')
|
||||
|
||||
/**
|
||||
* OBSOLETE
|
||||
* One-off script to find all non-lowercase email addresses
|
||||
* and lowercase them before hashing
|
||||
*/
|
||||
async function subscribeLowercaseHashToHIBP (emailAddress) {
|
||||
const lowerCasedEmail = emailAddress.toLowerCase()
|
||||
const lowerCasedSha1 = getSha1(lowerCasedEmail)
|
||||
await HIBP.subscribeHash(lowerCasedSha1)
|
||||
return lowerCasedSha1
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const chunkSize = process.argv[2]
|
||||
console.log(`subscribing lower-cased hashes in ${chunkSize}-sized chunks`)
|
||||
|
||||
const subRecordsThatNeedFixing = await knex('subscribers').count().whereRaw('primary_email != lower(primary_email)')
|
||||
const subsWithUpperCount = subRecordsThatNeedFixing[0].count
|
||||
console.log(`found ${subsWithUpperCount} subscribers records with primary_email != lower(primary_email). fixing ...`)
|
||||
|
||||
let subRecordsFixed = 0
|
||||
let subPrevMaxId = 0
|
||||
while (subRecordsFixed < subsWithUpperCount) {
|
||||
console.log(`working on chunk where id > ${subPrevMaxId} ...`)
|
||||
const subRecordsWithUpperCharsChunk = await knex.select('id', 'primary_email').from('subscribers')
|
||||
.where('id', '>', subPrevMaxId)
|
||||
.whereRaw('primary_email != lower(primary_email)')
|
||||
.orderBy('id', 'asc')
|
||||
.limit(chunkSize)
|
||||
for (const subRecord of subRecordsWithUpperCharsChunk) {
|
||||
await subscribeLowercaseHashToHIBP(subRecord.primary_email)
|
||||
subPrevMaxId = subRecord.id
|
||||
subRecordsFixed++
|
||||
console.log(`subscribed lower-case address hash for subscribers record ID: ${subRecord.id}`)
|
||||
}
|
||||
}
|
||||
|
||||
const emailRecordsThatNeedFixing = await knex('email_addresses').count().whereRaw('email != lower(email)')
|
||||
const emailWithUpperCount = emailRecordsThatNeedFixing[0].count
|
||||
console.log(`found ${emailWithUpperCount} email_address records with email != lower(email). fixing ...`)
|
||||
|
||||
let emailRecordsFixed = 0
|
||||
let emailPrevMaxId = 0
|
||||
while (emailRecordsFixed < emailWithUpperCount) {
|
||||
console.log(`working on chunk where id > ${emailPrevMaxId} ...`)
|
||||
const emailRecordsWithUpperChars = await knex.select('id', 'email').from('email_addresses')
|
||||
.where('id', '>', emailPrevMaxId)
|
||||
.whereRaw('email != lower(email)')
|
||||
.orderBy('id', 'asc')
|
||||
.limit(chunkSize)
|
||||
for (const emailRecord of emailRecordsWithUpperChars) {
|
||||
await subscribeLowercaseHashToHIBP(emailRecord.email)
|
||||
emailPrevMaxId = emailRecord.id
|
||||
emailRecordsFixed++
|
||||
console.log(`fixed email_addresses record ID: ${emailRecord.id}`)
|
||||
}
|
||||
}
|
||||
console.log('done.')
|
||||
process.exit()
|
||||
})()
|
||||
|
|
@ -49,7 +49,8 @@ Sentry.init({
|
|||
function getEnvironment() {
|
||||
if (
|
||||
document.location.origin === "https://monitor.firefox.com" ||
|
||||
document.location.origin === "https://monitor.mozilla.com"
|
||||
document.location.origin === "https://monitor.mozilla.com" ||
|
||||
document.location.origin === "https://monitor.mozilla.org"
|
||||
) {
|
||||
return "production";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ export const DashboardTopBanner = (props: DashboardTopBannerProps) => {
|
|||
const isShowFixed = props.tabType === "fixed";
|
||||
|
||||
const chartDataKey = isShowFixed
|
||||
? "inProgressFixedSanitizedDataPoints"
|
||||
? "fixedSanitizedDataPoints"
|
||||
: "unresolvedSanitizedDataPoints";
|
||||
|
||||
const chartData: [string, number][] = props.bannerData[chartDataKey].map(
|
||||
|
|
|
|||
|
|
@ -165,7 +165,7 @@ function getSecurityRecommendationsByType({
|
|||
elems: {
|
||||
link_to_info: (
|
||||
<a
|
||||
href="https://www.mozilla.org/products/vpn/?entrypoint_experiment=vpn-refresh-pricing&entrypoint_variation=1"
|
||||
href="https://www.mozilla.org/products/vpn/?utm_medium=monitor&utm_source=security-reco&utm_campaign=ip-breach&utm_content=global"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import {
|
|||
getDashboardSummary,
|
||||
getDataPointReduction,
|
||||
DataPoints,
|
||||
dataClassKeyMap,
|
||||
} from "./dashboard";
|
||||
import { SubscriberBreach } from "../../../utils/subscriberBreaches";
|
||||
import { RemovalStatus, RemovalStatusMap } from "../universal/scanResult";
|
||||
|
|
@ -488,9 +489,9 @@ describe("getExposureReduction", () => {
|
|||
unresolvedDataPoints: testExposure,
|
||||
inProgressDataPoints: testExposure,
|
||||
fixedDataPoints: testExposure,
|
||||
inProgressFixedDataPoints: testExposure,
|
||||
manuallyResolvedDataBrokerDataPoints: testExposure,
|
||||
unresolvedSanitizedDataPoints: [],
|
||||
inProgressFixedSanitizedDataPoints: [],
|
||||
fixedSanitizedDataPoints: [],
|
||||
dataBreachUnresolvedNum: 0,
|
||||
dataBreachResolvedNum: 0,
|
||||
};
|
||||
|
|
@ -533,9 +534,9 @@ describe("getExposureReduction", () => {
|
|||
unresolvedDataPoints: testExposure,
|
||||
inProgressDataPoints: testExposure,
|
||||
fixedDataPoints: testExposure,
|
||||
inProgressFixedDataPoints: testExposure,
|
||||
manuallyResolvedDataBrokerDataPoints: testExposure,
|
||||
unresolvedSanitizedDataPoints: [],
|
||||
inProgressFixedSanitizedDataPoints: [],
|
||||
fixedSanitizedDataPoints: [],
|
||||
dataBreachUnresolvedNum: 0,
|
||||
dataBreachResolvedNum: 0,
|
||||
};
|
||||
|
|
@ -572,13 +573,11 @@ describe("getDashboardSummary", () => {
|
|||
},
|
||||
);
|
||||
|
||||
summary.inProgressFixedSanitizedDataPoints.forEach(
|
||||
(inProgressFixedSanitizedExposure) => {
|
||||
Object.values(inProgressFixedSanitizedExposure).forEach((count) => {
|
||||
summary.fixedSanitizedDataPoints.forEach((fixedSanitizedExposure) => {
|
||||
Object.values(fixedSanitizedExposure).forEach((count) => {
|
||||
expect(count).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
it("gets breaches only summary", () => {
|
||||
|
|
@ -663,20 +662,16 @@ describe("getDashboardSummary", () => {
|
|||
expect(summary.dataBreachTotalDataPointsNum).toBe(0);
|
||||
expect(summary.dataBreachFixedDataPointsNum).toBe(0);
|
||||
expect(summary.dataBrokerTotalNum).toBe(4);
|
||||
expect(summary.inProgressFixedDataPoints.emailAddresses).toBe(
|
||||
summary.inProgressDataPoints.emailAddresses +
|
||||
expect(summary.fixedDataPoints.emailAddresses).toBe(
|
||||
summary.fixedDataPoints.emailAddresses,
|
||||
);
|
||||
expect(summary.inProgressFixedDataPoints.phoneNumbers).toBe(
|
||||
summary.inProgressDataPoints.phoneNumbers +
|
||||
expect(summary.fixedDataPoints.phoneNumbers).toBe(
|
||||
summary.fixedDataPoints.phoneNumbers,
|
||||
);
|
||||
expect(summary.inProgressFixedDataPoints.addresses).toBe(
|
||||
summary.inProgressDataPoints.addresses +
|
||||
expect(summary.fixedDataPoints.addresses).toBe(
|
||||
summary.fixedDataPoints.addresses,
|
||||
);
|
||||
expect(summary.inProgressFixedDataPoints.familyMembers).toBe(
|
||||
summary.inProgressDataPoints.familyMembers +
|
||||
expect(summary.fixedDataPoints.familyMembers).toBe(
|
||||
summary.fixedDataPoints.familyMembers,
|
||||
);
|
||||
expect(summary.unresolvedDataPoints.emailAddresses).toBe(0);
|
||||
|
|
@ -749,7 +744,7 @@ describe("getDashboardSummary", () => {
|
|||
expect(summary.dataBrokerManuallyResolvedDataPointsNum).toBe(12);
|
||||
});
|
||||
|
||||
it("inProgressFixedSanitizedExposures counts manually resolved exposures", () => {
|
||||
it("fixedSanitizedExposures counts manually resolved exposures", () => {
|
||||
const combinedScannedResults = [
|
||||
...unresolvedScannedResults,
|
||||
...manuallyResolvedScannedResults,
|
||||
|
|
@ -773,30 +768,34 @@ describe("getDashboardSummary", () => {
|
|||
];
|
||||
const summary = getDashboardSummary(combinedScannedResults, []);
|
||||
noNegativeCounts(summary);
|
||||
console.log(
|
||||
"yay ",
|
||||
JSON.stringify(summary.inProgressFixedSanitizedDataPoints),
|
||||
);
|
||||
expect(summary.inProgressFixedSanitizedDataPoints).toEqual(
|
||||
expect(summary.fixedSanitizedDataPoints).toEqual(
|
||||
expectedSanitizedExposures,
|
||||
);
|
||||
});
|
||||
|
||||
it("inProgressFixedExposures counts manually resolved exposures", () => {
|
||||
it("fixedSanitizedDataPoints counts manually resolved exposures", () => {
|
||||
const combinedScannedResults = [
|
||||
...unresolvedScannedResults,
|
||||
...manuallyResolvedScannedResults,
|
||||
];
|
||||
const summary = getDashboardSummary(combinedScannedResults, []);
|
||||
noNegativeCounts(summary);
|
||||
expect(summary.manuallyResolvedDataBrokerDataPoints.passwords).toBe(
|
||||
summary.inProgressFixedDataPoints.passwords,
|
||||
const getSanitizedDataPoint = (
|
||||
dataPoint: (typeof dataClassKeyMap)[keyof typeof dataClassKeyMap],
|
||||
) => {
|
||||
const sanitizedDataPoint = summary.fixedSanitizedDataPoints.find(
|
||||
(fixedData) => dataPoint in fixedData,
|
||||
);
|
||||
expect(summary.manuallyResolvedDataBrokerDataPoints.pins).toBe(
|
||||
summary.inProgressFixedDataPoints.pins,
|
||||
return sanitizedDataPoint?.[dataPoint] ?? 0;
|
||||
};
|
||||
expect(summary.manuallyResolvedDataBrokerDataPoints.emailAddresses).toBe(
|
||||
getSanitizedDataPoint(dataClassKeyMap.emailAddresses),
|
||||
);
|
||||
expect(summary.manuallyResolvedDataBrokerDataPoints.phoneNumbers).toBe(
|
||||
summary.inProgressFixedDataPoints.phoneNumbers,
|
||||
getSanitizedDataPoint(dataClassKeyMap.phoneNumbers),
|
||||
);
|
||||
expect(summary.manuallyResolvedDataBrokerDataPoints.addresses).toBe(
|
||||
getSanitizedDataPoint(dataClassKeyMap.addresses),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -65,16 +65,14 @@ export interface DashboardSummary {
|
|||
fixedDataPoints: DataPoints;
|
||||
/** manually resolved data broker data points separated by data classes */
|
||||
manuallyResolvedDataBrokerDataPoints: DataPoints;
|
||||
/** in-progress & resolved/removed data points separated by data classes */
|
||||
inProgressFixedDataPoints: DataPoints;
|
||||
|
||||
/** sanitized all data points for frontend display */
|
||||
unresolvedSanitizedDataPoints: SanitizedDataPoints;
|
||||
/** sanitized resolved/removed data points for frontend display */
|
||||
inProgressFixedSanitizedDataPoints: SanitizedDataPoints;
|
||||
/** sanitized resolved and removed data points for frontend display */
|
||||
fixedSanitizedDataPoints: SanitizedDataPoints;
|
||||
}
|
||||
|
||||
const dataClassKeyMap: Record<string, string> = {
|
||||
export const dataClassKeyMap: Record<string, string> = {
|
||||
emailAddresses: "email-addresses",
|
||||
phoneNumbers: "phone-numbers",
|
||||
|
||||
|
|
@ -185,23 +183,8 @@ export function getDashboardSummary(
|
|||
securityQuestions: 0,
|
||||
bankAccountNumbers: 0,
|
||||
},
|
||||
inProgressFixedDataPoints: {
|
||||
emailAddresses: 0,
|
||||
phoneNumbers: 0,
|
||||
addresses: 0,
|
||||
familyMembers: 0,
|
||||
|
||||
// data breaches
|
||||
socialSecurityNumbers: 0,
|
||||
ipAddresses: 0,
|
||||
passwords: 0,
|
||||
creditCardNumbers: 0,
|
||||
pins: 0,
|
||||
securityQuestions: 0,
|
||||
bankAccountNumbers: 0,
|
||||
},
|
||||
unresolvedSanitizedDataPoints: [],
|
||||
inProgressFixedSanitizedDataPoints: [],
|
||||
fixedSanitizedDataPoints: [],
|
||||
};
|
||||
|
||||
// calculate broker summary from scanned results
|
||||
|
|
@ -395,17 +378,6 @@ export function getDashboardSummary(
|
|||
{} as DataPoints,
|
||||
);
|
||||
|
||||
// count fixed, in-progress, and manually resolved (data brokers) data points
|
||||
summary.inProgressFixedDataPoints = Object.keys(
|
||||
summary.fixedDataPoints,
|
||||
).reduce((a, k) => {
|
||||
a[k as keyof DataPoints] =
|
||||
summary.fixedDataPoints[k as keyof DataPoints] +
|
||||
summary.inProgressDataPoints[k as keyof DataPoints] +
|
||||
summary.manuallyResolvedDataBrokerDataPoints[k as keyof DataPoints];
|
||||
return a;
|
||||
}, {} as DataPoints);
|
||||
|
||||
// sanitize unresolved data points
|
||||
summary.unresolvedSanitizedDataPoints = sanitizeDataPoints(
|
||||
summary.unresolvedDataPoints,
|
||||
|
|
@ -417,12 +389,21 @@ export function getDashboardSummary(
|
|||
isBreachesOnly,
|
||||
);
|
||||
|
||||
// sanitize fixed + inprogress data points
|
||||
summary.inProgressFixedSanitizedDataPoints = sanitizeDataPoints(
|
||||
summary.inProgressFixedDataPoints,
|
||||
// count fixed and manually resolved (data brokers) data points
|
||||
const dataBrokerFixedManuallyResolved = Object.keys(
|
||||
summary.fixedDataPoints,
|
||||
).reduce((a, k) => {
|
||||
a[k as keyof DataPoints] =
|
||||
summary.fixedDataPoints[k as keyof DataPoints] +
|
||||
summary.manuallyResolvedDataBrokerDataPoints[k as keyof DataPoints];
|
||||
return a;
|
||||
}, {} as DataPoints);
|
||||
|
||||
// sanitize fixed and removed data points
|
||||
summary.fixedSanitizedDataPoints = sanitizeDataPoints(
|
||||
dataBrokerFixedManuallyResolved,
|
||||
summary.dataBreachFixedDataPointsNum +
|
||||
summary.dataBrokerAutoFixedDataPointsNum +
|
||||
summary.dataBrokerInProgressDataPointsNum +
|
||||
summary.dataBrokerManuallyResolvedDataPointsNum,
|
||||
isBreachesOnly,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ export const ENV_URLS = {
|
|||
local: "http://localhost:6060",
|
||||
heroku: "https://fx-breach-alerts.herokuapp.com",
|
||||
stage: "https://stage.firefoxmonitor.nonprod.cloudops.mozgcp.net",
|
||||
prod: "https://monitor.firefox.com",
|
||||
prod: "https://monitor.mozilla.org",
|
||||
};
|
||||
|
||||
export const setEnvVariables = (email: string) => {
|
||||
|
|
|
|||
|
|
@ -1,122 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
|
||||
* with the goal of deprecating the column
|
||||
*/
|
||||
|
||||
import createDbConnection from "../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../utils/breaches.js";
|
||||
import { setBreachResolution } from "../db/tables/subscribers.js";
|
||||
import { BreachDataTypes } from "../utils/breach-resolution.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 50; // with millions of records, we have to load a few at a time
|
||||
let offset = 0; // looping through all records with offset
|
||||
let subscribersArr = [];
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past, convert those
|
||||
// records into the new v2 format
|
||||
do {
|
||||
console.log(
|
||||
`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`,
|
||||
);
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.limit(LIMIT)
|
||||
.offset(offset);
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
let { breaches_resolved: v1, breach_resolution: v2 } = subscriber;
|
||||
console.debug({ v1 });
|
||||
console.debug({ v2 });
|
||||
|
||||
let isV2Changed = false; // use a boolean to track if v2 has been changed, only upsert if so
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
const subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails));
|
||||
|
||||
for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
|
||||
console.debug({ email });
|
||||
console.debug({ resolvedRecencyIndices });
|
||||
for (const recencyIndex of resolvedRecencyIndices) {
|
||||
console.debug({ recencyIndex });
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(ve) => ve.email === email,
|
||||
)[0] || {};
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
console.debug({ subBreach });
|
||||
|
||||
if (!subBreach || !subBreach.DataClasses) {
|
||||
console.warn(
|
||||
`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if email does not exist in v2, we need to add it to the object
|
||||
// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
|
||||
if (!v2) v2 = {};
|
||||
if (!v2[email]) {
|
||||
v2[email] = {
|
||||
[recencyIndex]: {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses || [
|
||||
BreachDataTypes.General,
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
isV2Changed = true;
|
||||
}
|
||||
if (v2[email][recencyIndex]?.isResolved) {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`,
|
||||
);
|
||||
v2[email][recencyIndex] = {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses,
|
||||
};
|
||||
isV2Changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
if (isV2Changed) {
|
||||
await setBreachResolution(subscriber, v2);
|
||||
}
|
||||
}
|
||||
offset += LIMIT;
|
||||
} while (subscribersArr.length === LIMIT);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Reaching the end of the table, offset ended at", offset);
|
||||
process.exit();
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
import createDbConnection from "../db/connect.js";
|
||||
const knex = createDbConnection();
|
||||
|
||||
import { subscribeHash } from "../utils/hibp.js";
|
||||
import { getSha1 } from "../utils/fxa.js";
|
||||
|
||||
/**
|
||||
* MNTOR-2469: One-off script to lowercase email addresses of subscribers and emails before hashing
|
||||
*/
|
||||
async function subscribeLowerCaseHashToHIBP(emailAddress) {
|
||||
const lowerCasedEmail = emailAddress.toLowerCase();
|
||||
const lowerCasedSha1 = getSha1(lowerCasedEmail);
|
||||
await subscribeHash(lowerCasedSha1);
|
||||
return lowerCasedSha1;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const chunkSize = 1000;
|
||||
console.log(
|
||||
`Fixing lower-cased emails / sub to hashes in ${chunkSize}-sized chunks`,
|
||||
);
|
||||
|
||||
console.log(`/** Subscribers Table **/`);
|
||||
const subRecordsThatNeedFixing = await knex("subscribers")
|
||||
.count()
|
||||
.whereRaw("primary_email != lower(primary_email)");
|
||||
const subsWithUpperCount = subRecordsThatNeedFixing[0].count;
|
||||
console.log(
|
||||
`found ${subsWithUpperCount} subscribers records with primary_email != lower(primary_email)`,
|
||||
);
|
||||
|
||||
let subRecordsFixed = 0;
|
||||
let subPrevMaxId = 0;
|
||||
while (subRecordsFixed < subsWithUpperCount) {
|
||||
console.log(
|
||||
`working on ${chunkSize}-sized chunk where id > ${subPrevMaxId} ...`,
|
||||
);
|
||||
const subRecordsWithUpperCharsChunk = await knex
|
||||
.select("id", "primary_email")
|
||||
.from("subscribers")
|
||||
.where("id", ">", subPrevMaxId)
|
||||
.whereRaw("primary_email != lower(primary_email)")
|
||||
.orderBy("id", "asc")
|
||||
.limit(chunkSize);
|
||||
for (const subRecord of subRecordsWithUpperCharsChunk) {
|
||||
console.log(`Fixing record: ${subRecord.id}`);
|
||||
const trx = await knex.transaction();
|
||||
try {
|
||||
// update sha1 and sub to HIBP with the new sha1
|
||||
const lowerCasedSha1 = await subscribeLowerCaseHashToHIBP(
|
||||
subRecord.primary_email,
|
||||
);
|
||||
await knex("subscribers")
|
||||
.update({
|
||||
primary_sha1: lowerCasedSha1,
|
||||
})
|
||||
.where("id", subRecord.id)
|
||||
.transacting(trx);
|
||||
|
||||
// update primary email to lowercase
|
||||
await knex("subscribers")
|
||||
.update({
|
||||
primary_email: subRecord.primary_email.toLowerCase(),
|
||||
})
|
||||
.where("id", subRecord.id)
|
||||
.transacting(trx);
|
||||
await trx.commit();
|
||||
console.log(`fixed subscribers record ID: ${subRecord.id}`);
|
||||
} catch (e) {
|
||||
await trx.rollback();
|
||||
console.error(`Error fixing record: ${subRecord.id}`, e);
|
||||
}
|
||||
|
||||
subPrevMaxId = subRecord.id;
|
||||
subRecordsFixed++;
|
||||
console.log(
|
||||
`subscribed lower-case address hash for subscribers record ID: ${subRecord.id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixing Emails
|
||||
*/
|
||||
console.log(`/** Emails Table **/`);
|
||||
|
||||
const emailRecordsThatNeedFixing = await knex("email_addresses")
|
||||
.count()
|
||||
.whereRaw("email != lower(email)");
|
||||
const emailWithUpperCount = emailRecordsThatNeedFixing[0].count;
|
||||
console.log(
|
||||
`found ${emailWithUpperCount} email_address records with email != lower(email). fixing ...`,
|
||||
);
|
||||
|
||||
let emailRecordsFixed = 0;
|
||||
let emailPrevMaxId = 0;
|
||||
while (emailRecordsFixed < emailWithUpperCount) {
|
||||
console.log(`working on chunk where id > ${emailPrevMaxId} ...`);
|
||||
const emailRecordsWithUpperChars = await knex
|
||||
.select("id", "email")
|
||||
.from("email_addresses")
|
||||
.where("id", ">", emailPrevMaxId)
|
||||
.whereRaw("email != lower(email)")
|
||||
.orderBy("id", "asc")
|
||||
.limit(chunkSize);
|
||||
for (const emailRecord of emailRecordsWithUpperChars) {
|
||||
console.log(`Fixing record: ${emailRecord.id}`);
|
||||
const trx = await knex.transaction();
|
||||
try {
|
||||
// update sha1 and sub to HIBP with the new sha1
|
||||
const lowerCasedSha1 = await subscribeLowerCaseHashToHIBP(
|
||||
emailRecord.email,
|
||||
);
|
||||
await knex("email_addresses")
|
||||
.update({
|
||||
sha1: lowerCasedSha1,
|
||||
})
|
||||
.where("id", emailRecord.id)
|
||||
.transacting(trx);
|
||||
|
||||
// update primary email to lowercase
|
||||
await knex("email_addresses")
|
||||
.update({
|
||||
email: emailRecord.email.toLowerCase(),
|
||||
})
|
||||
.where("id", emailRecord.id)
|
||||
.transacting(trx);
|
||||
await trx.commit();
|
||||
console.log(`fixed email_addresses record ID: ${emailRecord.id}`);
|
||||
} catch (e) {
|
||||
await trx.rollback();
|
||||
console.error(
|
||||
`Error fixing email_addresses record: ${emailRecord.id}`,
|
||||
e,
|
||||
);
|
||||
}
|
||||
|
||||
emailPrevMaxId = emailRecord.id;
|
||||
emailRecordsFixed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log("done.");
|
||||
process.exit();
|
||||
})();
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
|
||||
* with the goal of deprecating the column
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../../utils/breaches.js";
|
||||
import { BreachDataTypes } from "../../utils/breach-resolution.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 1000; // with millions of records, we have to load a few at a time
|
||||
let subscribersArr = [];
|
||||
|
||||
/**
|
||||
* Batch update
|
||||
*
|
||||
* @param {*} updateCollection
|
||||
*/
|
||||
const batchUpdate = async (updateCollection) => {
|
||||
const trx = await knex.transaction();
|
||||
try {
|
||||
await Promise.all(
|
||||
updateCollection.map((tuple) => {
|
||||
const { user, updatedBreachesResolution } = tuple;
|
||||
return knex("subscribers")
|
||||
.where("id", user.id)
|
||||
.update({
|
||||
breach_resolution: updatedBreachesResolution,
|
||||
})
|
||||
.transacting(trx);
|
||||
}),
|
||||
);
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
await trx.rollback();
|
||||
console.error("batch update failed!!");
|
||||
console.log({ updateCollection });
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
const selectAndLockResolutions = async () => {
|
||||
const trx = await knex.transaction();
|
||||
let subscribersArr = [];
|
||||
try {
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.whereNull("db_migration_1")
|
||||
.limit(LIMIT)
|
||||
.orderBy("updated_at", "desc")
|
||||
.transacting(trx)
|
||||
.forUpdate();
|
||||
|
||||
// update the lock
|
||||
await Promise.all(
|
||||
subscribersArr.map((sub) => {
|
||||
const { id } = sub;
|
||||
return knex("subscribers")
|
||||
.where("id", id)
|
||||
.update({
|
||||
db_migration_1: true,
|
||||
})
|
||||
.transacting(trx);
|
||||
}),
|
||||
);
|
||||
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
await trx.rollback();
|
||||
console.log("select & mark rows failed!! first row:");
|
||||
console.log({ first: subscribersArr[0] });
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
return subscribersArr;
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past, convert those
|
||||
// records into the new v2 format
|
||||
let failedToSelect = true;
|
||||
while (failedToSelect) {
|
||||
try {
|
||||
subscribersArr = await selectAndLockResolutions();
|
||||
failedToSelect = false;
|
||||
} catch (e) {
|
||||
failedToSelect = true;
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
const updateCollection = [];
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
let { breaches_resolved: v1, breach_resolution: v2 } = subscriber;
|
||||
let isV2Changed = false; // use a boolean to track if v2 has been changed, only upsert if so
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
let subscriberBreachesEmail;
|
||||
try {
|
||||
subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error("Cannot fetch subscriber breaches at the moment: ", e);
|
||||
continue;
|
||||
}
|
||||
// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
|
||||
|
||||
for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
|
||||
// console.debug({ email })
|
||||
// console.debug({ resolvedRecencyIndices })
|
||||
for (const recencyIndex of resolvedRecencyIndices) {
|
||||
// console.debug({ recencyIndex })
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(e) => e.email === email,
|
||||
)[0] || {};
|
||||
// console.debug({ ve })
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
// console.debug({ subBreach })
|
||||
|
||||
if (!subBreach || !subBreach.DataClasses) {
|
||||
console.warn(
|
||||
`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if email does not exist in v2, we need to add it to the object
|
||||
// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
|
||||
if (!v2) v2 = {};
|
||||
if (!v2[email]) {
|
||||
v2[email] = {
|
||||
[recencyIndex]: {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses || [
|
||||
BreachDataTypes.General,
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
isV2Changed = true;
|
||||
}
|
||||
if (v2[email][recencyIndex]?.isResolved) {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`,
|
||||
);
|
||||
v2[email][recencyIndex] = {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses,
|
||||
};
|
||||
isV2Changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
if (isV2Changed) {
|
||||
console.log("upsert for subscriber: ", subscriber.primary_email);
|
||||
updateCollection.push({ user: subscriber, updatedBreachesResolution: v2 });
|
||||
}
|
||||
}
|
||||
await batchUpdate(updateCollection);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Script finished");
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -1,181 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* Execute after `deprecateBreachResolve.js`
|
||||
* The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
|
||||
* For backwards compatibility, all converted `breach_resolution` fields will have a boolean
|
||||
* `useBreachId: true/false`
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../../utils/breaches.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 1000; // with millions of records, we have to load a few at a time
|
||||
let subscribersArr = [];
|
||||
|
||||
const selectAndLockResolutions = async () => {
|
||||
const trx = await knex.transaction();
|
||||
let subscribers = [];
|
||||
try {
|
||||
subscribers = await knex
|
||||
.select("id", "primary_email", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breach_resolution")
|
||||
.whereNull("db_migration_2")
|
||||
.limit(LIMIT)
|
||||
.orderBy("updated_at", "desc")
|
||||
.transacting(trx)
|
||||
.forUpdate();
|
||||
|
||||
// update the lock
|
||||
await Promise.all(
|
||||
subscribers.map((sub) => {
|
||||
const { id } = sub;
|
||||
return knex("subscribers")
|
||||
.where("id", id)
|
||||
.update({
|
||||
db_migration_2: true,
|
||||
})
|
||||
.transacting(trx);
|
||||
}),
|
||||
);
|
||||
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
await trx.rollback();
|
||||
console.error("select & mark rows failed!! first row:");
|
||||
console.log({ first: subscribers[0] });
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
return subscribers;
|
||||
};
|
||||
|
||||
/**
|
||||
* Batch update
|
||||
*
|
||||
* @param {*} updateCollection
|
||||
*/
|
||||
const batchUpdate = async (updateCollection) => {
|
||||
const trx = await knex.transaction();
|
||||
try {
|
||||
await Promise.all(
|
||||
updateCollection.map((tuple) => {
|
||||
const { user, updatedBreachesResolution } = tuple;
|
||||
return knex("subscribers")
|
||||
.where("id", user.id)
|
||||
.update({
|
||||
breach_resolution: updatedBreachesResolution,
|
||||
})
|
||||
.transacting(trx);
|
||||
}),
|
||||
);
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
await trx.rollback();
|
||||
console.error("batch update failed!!");
|
||||
console.log({ updateCollection });
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
// Script begins here
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past,
|
||||
// replace recency index with breach id
|
||||
|
||||
let failedToSelect = true;
|
||||
while (failedToSelect) {
|
||||
try {
|
||||
subscribersArr = await selectAndLockResolutions();
|
||||
failedToSelect = false;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
const updateCollection = [];
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
const { breach_resolution: v2 } = subscriber;
|
||||
// console.debug({ v2 })
|
||||
|
||||
// if useBreachId is set, skip because this breach_resolution has already been worked on
|
||||
if (v2.useBreachId) {
|
||||
console.log(
|
||||
"Skipping since `useBreachId` is set already, this breach resolution is already converted",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const newResolutions = {};
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
let subscriberBreachesEmail;
|
||||
try {
|
||||
subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
} catch (e) {
|
||||
console.error("Cannot fetch subscriber breaches at the moment: ", e);
|
||||
continue;
|
||||
}
|
||||
// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
|
||||
|
||||
for (const email in v2) {
|
||||
// console.debug({ email })
|
||||
const resolutions = v2[email];
|
||||
// console.debug({ resolutions })
|
||||
newResolutions[email] = {};
|
||||
|
||||
for (const recencyIndex in resolutions) {
|
||||
// console.debug({ recencyIndex })
|
||||
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(ve) => ve.email === email,
|
||||
)[0] || {};
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
const breachName = subBreach?.Name;
|
||||
console.debug({ breachName });
|
||||
|
||||
// find breach id for the breach
|
||||
const breachId = allBreaches.find((b) => b.Name === breachName)?.Id;
|
||||
newResolutions[email][breachId] = v2[email][recencyIndex];
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
newResolutions.useBreachId = true;
|
||||
updateCollection.push({
|
||||
user: subscriber,
|
||||
updatedBreachesResolution: newResolutions,
|
||||
});
|
||||
}
|
||||
|
||||
await batchUpdate(updateCollection);
|
||||
|
||||
console.log("Reaching the end of the table");
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to clean up some of the failed records during db migration on 3/28/23
|
||||
*/
|
||||
|
||||
import createDbConnection from "../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../utils/breaches.js";
|
||||
import { setBreachResolution } from "../db/tables/subscribers.js";
|
||||
import mozlog from "../utils/log.js";
|
||||
const log = mozlog("script.migrationCleanup");
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 3000;
|
||||
let subscribersArr = [];
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
log.info(
|
||||
"breach_count",
|
||||
"breaches loaded successfully! ",
|
||||
allBreaches.length,
|
||||
);
|
||||
|
||||
const count = await knex
|
||||
.from("subscribers")
|
||||
.whereRaw("NOT ((breach_resolution)::jsonb \\? 'useBreachId')")
|
||||
.count("*");
|
||||
|
||||
log.info("total_to_be_executed", count[0]);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past,
|
||||
// replace recency index with breach id
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.orderBy("updated_at", "desc")
|
||||
.whereRaw("NOT ((breach_resolution)::jsonb \\? 'useBreachId')")
|
||||
.limit(LIMIT);
|
||||
|
||||
log.info("job", `Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
const { breach_resolution: v2 } = subscriber;
|
||||
// console.debug({ v2 })
|
||||
|
||||
// if useBreachId is set, skip because this breach_resolution has already been worked on
|
||||
if (v2.useBreachId) {
|
||||
log.warn(
|
||||
"job",
|
||||
"Skipping since `useBreachId` is set already, this breach resolution is already converted",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
const newResolutions = {};
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
const subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
|
||||
|
||||
for (const email in v2) {
|
||||
// console.debug({ email })
|
||||
const resolutions = v2[email];
|
||||
// console.debug({ resolutions })
|
||||
newResolutions[email] = {};
|
||||
|
||||
for (const recencyIndex in resolutions) {
|
||||
console.debug({ recencyIndex });
|
||||
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(ve) => ve.email === email,
|
||||
)[0] || {};
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
const breachName = subBreach?.Name;
|
||||
console.debug({ breachName });
|
||||
|
||||
// find breach id for the breach
|
||||
const breachId = allBreaches.find((b) => b.Name === breachName)?.Id;
|
||||
log.info("job", { breachId });
|
||||
newResolutions[email][breachId] = v2[email][recencyIndex];
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
newResolutions.useBreachId = true;
|
||||
await setBreachResolution(subscriber, newResolutions);
|
||||
}
|
||||
}
|
||||
|
||||
// breaking out of do..while loop
|
||||
log.info("job", "Reaching the end of the table");
|
||||
process.exit();
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
|
||||
* with the goal of deprecating the column
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../../utils/breaches.js";
|
||||
import { BreachDataTypes } from "../../utils/breach-resolution.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 1000; // with millions of records, we have to load a few at a time
|
||||
let offset = 0; // looping through all records with offset
|
||||
let subscribersArr = [];
|
||||
|
||||
let CAP = 5000; // cap the experiment
|
||||
if (process.argv.length > 2) {
|
||||
CAP = process.argv[2];
|
||||
console.log("using cap passed in: ", CAP);
|
||||
}
|
||||
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past, convert those
|
||||
// records into the new v2 format
|
||||
do {
|
||||
console.log(
|
||||
`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`,
|
||||
);
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.limit(LIMIT)
|
||||
.offset(offset)
|
||||
.orderBy("updated_at", "desc");
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
let { breaches_resolved: v1, breach_resolution: v2 } = subscriber;
|
||||
let isV2Changed = false; // use a boolean to track if v2 has been changed, only upsert if so
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
const subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
|
||||
|
||||
for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
|
||||
// console.debug({ email })
|
||||
// console.debug({ resolvedRecencyIndices })
|
||||
for (const recencyIndex of resolvedRecencyIndices) {
|
||||
// console.debug({ recencyIndex })
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(e) => e.email === email,
|
||||
)[0] || {};
|
||||
// console.debug({ ve })
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
// console.debug({ subBreach })
|
||||
|
||||
if (!subBreach || !subBreach.DataClasses) {
|
||||
console.warn(
|
||||
`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if email does not exist in v2, we need to add it to the object
|
||||
// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
|
||||
if (!v2) v2 = {};
|
||||
if (!v2[email]) {
|
||||
v2[email] = {
|
||||
[recencyIndex]: {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses || [
|
||||
BreachDataTypes.General,
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
isV2Changed = true;
|
||||
}
|
||||
if (v2[email][recencyIndex]?.isResolved) {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`,
|
||||
);
|
||||
v2[email][recencyIndex] = {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses,
|
||||
};
|
||||
isV2Changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
if (isV2Changed) {
|
||||
console.log("upsert for subscriber: ", subscriber.primary_email);
|
||||
}
|
||||
}
|
||||
offset += LIMIT;
|
||||
} while (subscribersArr.length === LIMIT && offset <= CAP);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Reaching the end of the table, offset ended at", offset);
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to convert all `subscriber.breaches_resolved` to `subscriber.breaches_resolution`
|
||||
* with the goal of deprecating the column
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
import { getAllEmailsAndBreaches } from "../../utils/breaches.js";
|
||||
import { BreachDataTypes } from "../../utils/breach-resolution.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 1000; // with millions of records, we have to load a few at a time
|
||||
let CAP = 5000; // cap the experiment
|
||||
if (process.argv.length > 2) {
|
||||
CAP = process.argv[2];
|
||||
console.log("using cap passed in: ", CAP);
|
||||
}
|
||||
let offset = 0; // looping through all records with offset
|
||||
let subscribersArr = [];
|
||||
|
||||
/**
|
||||
* Batch update
|
||||
*
|
||||
* @param {*} updateCollection
|
||||
*/
|
||||
const batchUpdate = async (updateCollection) => {
|
||||
const trx = await knex.transaction();
|
||||
try {
|
||||
await Promise.all(
|
||||
updateCollection.map((tuple) => {
|
||||
const { user, updatedBreachesResolution } = tuple;
|
||||
return knex("subscribers")
|
||||
.where("id", user.id)
|
||||
.update({
|
||||
breach_resolution: updatedBreachesResolution,
|
||||
})
|
||||
.transacting(trx);
|
||||
}),
|
||||
);
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
await trx.rollback();
|
||||
console.error("batch update failed!!");
|
||||
console.log({ updateCollection });
|
||||
console.error(error);
|
||||
}
|
||||
};
|
||||
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past, convert those
|
||||
// records into the new v2 format
|
||||
do {
|
||||
console.log(
|
||||
`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`,
|
||||
);
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.limit(LIMIT)
|
||||
.offset(offset)
|
||||
.orderBy("updated_at", "desc");
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
const updateCollection = [];
|
||||
|
||||
for (const subscriber of subscribersArr) {
|
||||
let { breaches_resolved: v1, breach_resolution: v2 } = subscriber;
|
||||
let isV2Changed = false; // use a boolean to track if v2 has been changed, only upsert if so
|
||||
|
||||
// fetch subscriber all breaches / email
|
||||
const subscriberBreachesEmail = await getAllEmailsAndBreaches(
|
||||
subscriber,
|
||||
allBreaches,
|
||||
);
|
||||
// console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))
|
||||
|
||||
for (const [email, resolvedRecencyIndices] of Object.entries(v1)) {
|
||||
// console.debug({ email })
|
||||
// console.debug({ resolvedRecencyIndices })
|
||||
for (const recencyIndex of resolvedRecencyIndices) {
|
||||
// console.debug({ recencyIndex })
|
||||
// find subscriber's relevant recency index breach information
|
||||
const ve =
|
||||
subscriberBreachesEmail.verifiedEmails?.filter(
|
||||
(e) => e.email === email,
|
||||
)[0] || {};
|
||||
// console.debug({ ve })
|
||||
const subBreach =
|
||||
ve.breaches?.filter(
|
||||
(b) => Number(b.recencyIndex) === Number(recencyIndex),
|
||||
)[0] || null;
|
||||
// console.debug({ subBreach })
|
||||
|
||||
if (!subBreach || !subBreach.DataClasses) {
|
||||
console.warn(
|
||||
`SKIP: Cannot find subscribers breach and data types - recency: ${recencyIndex} email: ${email}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// if email does not exist in v2, we need to add it to the object
|
||||
// format: {email: { recencyIndex: { isResolved: true, resolutionsChecked: [DataTypes]}}}
|
||||
if (!v2) v2 = {};
|
||||
if (!v2[email]) {
|
||||
v2[email] = {
|
||||
[recencyIndex]: {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses || [
|
||||
BreachDataTypes.General,
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
isV2Changed = true;
|
||||
}
|
||||
if (v2[email][recencyIndex]?.isResolved) {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} exists in v2 and is resolved, no changes`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`recencyIndex ${recencyIndex} either does not exist or is not resolved, overwriting`,
|
||||
);
|
||||
v2[email][recencyIndex] = {
|
||||
isResolved: true,
|
||||
resolutionsChecked: subBreach?.DataClasses,
|
||||
};
|
||||
isV2Changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if v2 is changed, if so, upsert the new v2
|
||||
if (isV2Changed) {
|
||||
console.log("upsert for subscriber: ", subscriber.primary_email);
|
||||
updateCollection.push({
|
||||
user: subscriber,
|
||||
updatedBreachesResolution: v2,
|
||||
});
|
||||
}
|
||||
}
|
||||
await batchUpdate(updateCollection);
|
||||
offset += LIMIT;
|
||||
} while (subscribersArr.length === LIMIT && offset <= CAP);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Reaching the end of the table, offset ended at", offset);
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to benchmark pure read with limit set as 100
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 100; // with millions of records, we have to load a few at a time
|
||||
let offset = 0; // looping through all records with offset
|
||||
let subscribersArr = [];
|
||||
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
// find all subscribers who resolved any breaches in the past, convert those
|
||||
// records into the new v2 format
|
||||
do {
|
||||
console.log(
|
||||
`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`,
|
||||
);
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.limit(LIMIT)
|
||||
.offset(offset);
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
|
||||
offset += LIMIT;
|
||||
} while (subscribersArr.length === LIMIT);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Reaching the end of the table, offset ended at", offset);
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
/* This Source Code Form is subject to the terms of the Mozilla Public
|
||||
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
||||
|
||||
/**
|
||||
* Executes once
|
||||
* The purpose of the script is to benchmark pure read with limit set as 1000
|
||||
*/
|
||||
|
||||
import createDbConnection from "../../db/connect.js";
|
||||
import { getAllBreachesFromDb } from "../../utils/hibp.js";
|
||||
|
||||
const knex = createDbConnection();
|
||||
|
||||
const LIMIT = 1000; // with millions of records, we have to load a few at a time
|
||||
let CAP = 1500000;
|
||||
if (process.argv.length > 2) {
|
||||
CAP = process.argv[2];
|
||||
console.log("using cap passed in: ", CAP);
|
||||
}
|
||||
let offset = 0; // looping through all records with offset
|
||||
let subscribersArr = [];
|
||||
|
||||
// load all breaches for ref
|
||||
const allBreaches = await getAllBreachesFromDb();
|
||||
if (allBreaches && allBreaches.length > 0)
|
||||
console.log("breaches loaded successfully! ", allBreaches.length);
|
||||
|
||||
const startTime = Date.now();
|
||||
console.log(`Start time is: ${startTime}`);
|
||||
do {
|
||||
console.log(
|
||||
`Converting breaches_resolved to breach_resolution - start: ${offset} limit: ${LIMIT}`,
|
||||
);
|
||||
subscribersArr = await knex
|
||||
.select("id", "primary_email", "breaches_resolved", "breach_resolution")
|
||||
.from("subscribers")
|
||||
.whereNotNull("breaches_resolved")
|
||||
.limit(LIMIT)
|
||||
.offset(offset)
|
||||
.orderBy("updated_at", "desc");
|
||||
|
||||
console.log(`Loaded # of subscribers: ${subscribersArr.length}`);
|
||||
|
||||
offset += LIMIT;
|
||||
} while (subscribersArr.length === LIMIT && offset <= CAP);
|
||||
|
||||
// breaking out of do..while loop
|
||||
console.log("Reaching the end of the table, offset ended at", offset);
|
||||
const endTime = Date.now();
|
||||
console.log(`End time is: ${endTime}`);
|
||||
console.log("Diff is: ", endTime - startTime);
|
||||
process.exit();
|
||||
|
|
@ -421,7 +421,6 @@ describe("getSubBreaches", () => {
|
|||
|
||||
const subBreaches = await getSubBreaches(subscriber, []);
|
||||
expect(subBreaches.length).toEqual(1);
|
||||
console.log(JSON.stringify(subBreaches));
|
||||
expect(subBreaches[0].isResolved).toBe(false);
|
||||
});
|
||||
// MNTOR-2125
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче