update hibp to use real range API

app-constants.js

@@ -18,6 +18,8 @@ const kEnvironmentVariables = [
   "OAUTH_PROFILE_URI",
   "OAUTH_CLIENT_ID",
   "OAUTH_CLIENT_SECRET",
+  "HIBP_STAGE_API_ROOT",
+  "HIBP_STAGE_API_TOKEN",
   "HIBP_STUB_API_ROOT",
   "HIBP_API_ROOT",
   "HIBP_API_TOKEN",

hbs-helpers.js

@@ -0,0 +1,22 @@
+"use strict";
+
+const HBSHelpers = {
+  init(hbs) {
+
+    hbs.registerHelper("prettyDate", function(date) {
+      const jsDate = new Date(date);
+      return jsDate.toLocaleDateString("en-US", {year: "numeric", month: "long", day: "numeric"});
+    });
+
+    hbs.registerHelper("breachDataClasses", function(dataClasses) {
+      return dataClasses.join(", ");
+    });
+
+    hbs.registerHelper("localeString", function(input) {
+      return input.toLocaleString();
+    });
+
+  },
+};
+
+module.exports = HBSHelpers;

hibp.js

@@ -17,7 +17,7 @@ const HIBP = {
 
     const sha1 = getSha1(email);
     const sha1Prefix = sha1.slice(0, 6);
-    const url = `${AppConstants.HIBP_STUB_API_ROOT}/breachedaccount/range/${sha1Prefix}`;
+    const url = `${AppConstants.HIBP_STAGE_API_ROOT}/range/${sha1Prefix}?code=${encodeURIComponent(AppConstants.HIBP_STAGE_API_TOKEN)}`;
     const headers = {
       "User-Agent": HIBP_USER_AGENT,
     };
@@ -25,17 +25,15 @@ const HIBP = {
     console.info(`Fetching ${url}...`);
 
     try {
-      const response = await got(url, {headers});
+      const response = await got(url, {headers, json: true});
       // Parse response body, format:
-      // {breachedAccount1sha1}:{breach1Name}[,{breach2Name},...]
-      // {breachedAccount2sha1}:{breach3Name}[,{breach4Name},...]
-      for (const breachedAccount of response.body.trim().split("\n")) {
-        const sliceIndex = breachedAccount.indexOf(":");
-        const breachedSha1 = breachedAccount.slice(0, sliceIndex);
-        const breachNamesStr = breachedAccount.slice(sliceIndex + 1);
-        const breachNames = breachNamesStr.split(",");
-        if (sha1 === breachedSha1) {
-          foundBreaches = await DBUtils.getBreachesByNames(breachNames);
+      // [
+      //   {"HashSuffix":<suffix>,"Websites":[<breach1Name>,...]},
+      //   {"HashSuffix":<suffix>,"Websites":[<breach1Name>,...]},
+      // ]
+      for (const breachedAccount of response.body) {
+        if (sha1.toUpperCase() === sha1Prefix + breachedAccount.HashSuffix) {
+          foundBreaches = await DBUtils.getBreachesByNames(breachedAccount.Websites);
         }
       }
     } catch (error) {

scripts/get-hashsets.js

@@ -1,100 +0,0 @@
-"use strict";
-
-const request = require("request");
-const S3 = require("aws-sdk/clients/s3");
-
-const AppConstants = require("../app-constants");
-// Disable eslint here till this file gets updated to use DBUtils.
-// eslint-disable-next-line node/no-missing-require
-const models = require("../db/models");
-const pkg = require("../package.json");
-
-const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
-const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
-
-const s3 = new S3();
-const BREACH_HASHSET_BUCKET_NAME = "mozilla.breach_alerts.stage.breach_hashsets";
-
-
-function getBreachHashset(breach) {
-  /*
-   * HIBP Breach, Object.keys(breach):
-   * [ 'Title', 'Name', 'Domain', 'BreachDate', 'AddedDate', 'ModifiedDate', 'PwnCount', 'Description',
-   *   'DataClasses', 'IsVerified', 'IsFabricated', 'IsSensitive', 'IsActive', 'IsRetired', 'IsSpamList',
-   *   'LogoType' ]
-   * See https://haveibeenpwned.com/API/v2#BreachModel for more
-   */
-  const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${breach.Name}`;
-  const headers = {
-    "User-Agent": HIBP_USER_AGENT,
-    "Authorization": HIBP_AUTH,
-  };
-  const hashsetRequestObject = {
-    url,
-    headers,
-  };
-
-  console.log(`Fetching ${url}...`);
-
-  request(hashsetRequestObject, async (error, response, body) => {
-    if (response.statusCode === 200) {
-      console.log("Uploading to S3 ...");
-      const uploadParams = {
-        Bucket: BREACH_HASHSET_BUCKET_NAME,
-        Key: `${breach.Name}.zip`,
-        Body: body,
-      };
-      s3.upload(uploadParams, (err, data) => {
-        if (err) {
-          console.error(`err: ${err}`);
-          throw(err);
-        }
-        console.log(`Uploaded to ${data.Location}`);
-      });
-    }
-  });
-}
-
-async function handleBreachesResponse(error, response, body) {
-  if (error) {
-    console.error(error);
-    process.exit(1);
-  }
-
-  try {
-    const breachesJSON = JSON.parse(body);
-
-    for (const breach of breachesJSON) {
-      models.Breach.findOrCreate({where: {
-        name: breach.Name,
-      }}).spread((ourBreach, created) => {
-        if (breach.IsActive && breach.IsVerified && breach.DataClasses.includes("Email addresses")) {
-          console.log(`Active, verified breach with email addresses: ${breach.Name}. Checking if we have the latest data ...`);
-          if (created || Date(ourBreach.updatedAt) < Date(breach.ModifiedDate)) {
-            console.log("New breach, or breach modified since last update. Getting Hashset ...");
-            getBreachHashset(breach);
-          } else {
-            console.log("Breach not modified since last update. Done.");
-          }
-        }
-      });
-    }
-  } catch (err) {
-    console.error(err);
-    throw(err);
-  }
-}
-
-function getBreaches() {
-  const breachesRequestObject = {
-    url: `${AppConstants.HIBP_API_ROOT}/breaches`,
-    headers: {
-      "User-Agent": HIBP_USER_AGENT,
-    },
-  };
-  request(breachesRequestObject, handleBreachesResponse);
-}
-
-models.sequelize.sync().then(()=>{
-  getBreaches();
-});

server.js

@@ -7,6 +7,7 @@ const hbs = require("express-hbs");
 const sessions = require("client-sessions");
 
 const EmailUtils = require("./email-utils");
+const HBSHelpers = require("./hbs-helpers");
 
 const HIBPRoutes = require("./routes/hibp-stubs");
 const HomeRoutes = require("./routes/home");
@@ -24,6 +25,7 @@ app.engine("hbs", hbs.express4({
 }));
 app.set("view engine", "hbs");
 app.set("views", __dirname + "/views");
+HBSHelpers.init(hbs);
 
 app.use(sessions({
   cookieName: "session",

views/scan.hbs

@@ -63,10 +63,11 @@
         <div class="flex">
           <h4>{{ name }}</h4><br>
           <p>
-            <span class="bold">Breach date:</span> {{ meta.date }}<br>
-            <span class="bold">Compromised data:</span> {{ meta.dataClasses }}<br>
-            <span class="bold">Compromised accounts:</span> {{ meta.acCount }}<br>
-            {{ meta.description }}
+            <span class="bold">Breach date:</span> {{prettyDate meta.BreachDate }}<br>
+            <span class="bold">Compromised data:</span> {{breachDataClasses meta.DataClasses }}<br>
+            <span class="bold">Compromised accounts:</span> {{localeString meta.PwnCount }}<br>
+            {{! TODO: make sure meta.Description is a safe string so we can use triple-{{{ }}
+            {{{ meta.Description }}}
           </p>
         </div>
       </div>
@@ -83,7 +84,7 @@
     <div class="cell">
       <p><span class="bold">Change your password</span> on these sites and anywhere else you’ve used the same password.</p>
       <p>Make your answers to <span class="bold">security questions</span> just as strong as your passwords.</p>
-      <p><span class="bold">Use password managers</span> like 1Password, LastPass, or Dashlane to generate strong passwords for you, remember them for you, and !ll them into websites so you don’t have type them in.</p>
+      <p><span class="bold">Use password managers</span> like 1Password, LastPass, or Dashlane to generate strong passwords for you, remember them for you, and fill them into websites so you don’t have type them in.</p>
       <p>Sign up for alerts from Firefox Monitor to learn sooner about your compromised accounts.</p>
     </div>
     <div class="cell"></div>