Use sequelize for hashset script

.env-dist

@@ -26,3 +26,5 @@ PGPASSWORD=""
 PGDATABASE="blurts"
 PGHOST="localhost"
 PGPORT=5432
+
+DATABASE_URL="postgres://postgres@localhost:5432/blurts"

.eslintignore

@@ -1 +1,2 @@
 public/js/vendor/*.js
+db/migrations

.sequelizerc

@@ -0,0 +1,7 @@
+const path = require("path");
+
+module.exports = {
+  "config": path.resolve("db", "config", "config.json"),
+  "migrations-path": path.resolve("db", "migrations"),
+  "models-path": path.resolve("db", "models"),
+}

app-constants.js

@@ -21,6 +21,7 @@ const kEnvironmentVariables = [
   "HIBP_API_TOKEN",
   "AWS_ACCESS_KEY_ID",
   "AWS_SECRET_ACCESS_KEY",
+  "DATABASE_URL",
 ];
 
 const AppConstants = { };

db/config/config.json

@@ -0,0 +1,12 @@
+{
+  "development": {
+    "use_env_variable": "DATABASE_URL"
+  },
+  "test": {
+    "use_env_variable": "DATABASE_URL"
+  },
+  "production": {
+    "use_env_variable": "DATABASE_URL",
+    "logging": false
+  }
+}

db/migrations/20180308192638-create-user.js

@@ -0,0 +1,30 @@
+'use strict';
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.createTable('Users', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER
+      },
+      email: {
+        type: Sequelize.STRING
+      },
+      sha1: {
+        type: Sequelize.STRING
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      }
+    });
+  },
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable('Users');
+  }
+};

db/migrations/20180308192818-create-breach.js

@@ -0,0 +1,27 @@
+'use strict';
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.createTable('Breaches', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER
+      },
+      name: {
+        type: Sequelize.STRING
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      }
+    });
+  },
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable('Breaches');
+  }
+};

db/migrations/20180308194022-create-breached-user.js

@@ -0,0 +1,35 @@
+'use strict';
+module.exports = {
+  up: (queryInterface, Sequelize) => {
+    return queryInterface.createTable('BreachedUsers', {
+      id: {
+        allowNull: false,
+        autoIncrement: true,
+        primaryKey: true,
+        type: Sequelize.INTEGER
+      },
+      userId: {
+        type: Sequelize.INTEGER,
+        references: { model: 'Users', key: 'id' },
+      },
+      breachId: {
+        type: Sequelize.INTEGER,
+        references: { model: 'Breaches', key: 'id' },
+      },
+      notified: {
+        type: Sequelize.DATE
+      },
+      createdAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      },
+      updatedAt: {
+        allowNull: false,
+        type: Sequelize.DATE
+      }
+    });
+  },
+  down: (queryInterface, Sequelize) => {
+    return queryInterface.dropTable('BreachedUsers');
+  }
+};

db/models/breach.js

@@ -0,0 +1,10 @@
+"use strict";
+module.exports = (sequelize, DataTypes) => {
+  const Breach = sequelize.define("Breach", {
+    name: DataTypes.STRING,
+  }, {});
+  Breach.associate = function(models) {
+    Breach.hasMany(models.BreachedUser);
+  };
+  return Breach;
+};

db/models/breacheduser.js

@@ -0,0 +1,13 @@
+"use strict";
+module.exports = (sequelize, DataTypes) => {
+  const BreachedUser = sequelize.define("BreachedUser", {
+    userId: DataTypes.INTEGER,
+    breachId: DataTypes.INTEGER,
+    notified: DataTypes.DATE,
+  }, {});
+  BreachedUser.associate = function(models) {
+    BreachedUser.belongsTo(models.User);
+    BreachedUser.belongsTo(models.Breach);
+  };
+  return BreachedUser;
+};

db/models/index.js

@@ -0,0 +1,38 @@
+"use strict";
+
+const fs        = require("fs");
+const path      = require("path");
+const Sequelize = require("sequelize");
+const basename  = path.basename(__filename);
+const env       = process.env.NODE_ENV || "development"; // eslint-disable-line no-process-env
+const config    = require(__dirname + "/../config/config.json")[env];
+const db        = {};
+
+let sequelize;
+
+if (config.use_env_variable) {
+  sequelize = new Sequelize(process.env[config.use_env_variable], config); // eslint-disable-line no-process-env
+} else {
+  sequelize = new Sequelize(config.database, config.username, config.password, config);
+}
+
+fs
+  .readdirSync(__dirname)
+  .filter(file => {
+    return (file.indexOf(".") !== 0) && (file !== basename) && (file.slice(-3) === ".js");
+  })
+  .forEach(file => {
+    const model = sequelize["import"](path.join(__dirname, file));
+    db[model.name] = model;
+  });
+
+Object.keys(db).forEach(modelName => {
+  if (db[modelName].associate) {
+    db[modelName].associate(db);
+  }
+});
+
+db.sequelize = sequelize;
+db.Sequelize = Sequelize;
+
+module.exports = db;

db/models/user.js

@@ -0,0 +1,11 @@
+"use strict";
+module.exports = (sequelize, DataTypes) => {
+  const User = sequelize.define("User", {
+    email: DataTypes.STRING,
+    sha1: DataTypes.STRING,
+  }, {});
+  User.associate = function(models) {
+    User.hasMany(models.BreachedUser);
+  };
+  return User;
+};

package.json

@@ -17,7 +17,9 @@
     "nodemailer": "^4.6.0",
     "pg": "^7.4.1",
     "popsicle": "^9.2.0",
-    "request": "^2.83.0"
+    "request": "^2.83.0",
+    "sequelize": "^4.35.2",
+    "sequelize-cli": "^4.0.0"
   },
   "devDependencies": {
     "coveralls": "^3.0.0",

scripts/get-hashsets.js

@@ -1,21 +1,20 @@
 "use strict";
 
-const fs = require("fs");
 const request = require("request");
 const S3 = require("aws-sdk/clients/s3");
 
-const AppConstants = require("../app-constants").init();
+const AppConstants = require("../app-constants");
+const models = require("../db/models");
 const pkg = require("../package.json");
 
 const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
 const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
 
 const s3 = new S3();
-const BREACH_HASHSET_DIR = "../breach_hashsets";
 const BREACH_HASHSET_BUCKET_NAME = "mozilla.breach_alerts.stage.breach_hashsets";
 
 
-async function getBreachHashset(breach) {
+function getBreachHashset(breach) {
   /*
    * HIBP Breach, Object.keys(breach):
    * [ 'Title', 'Name', 'Domain', 'BreachDate', 'AddedDate', 'ModifiedDate', 'PwnCount', 'Description',
@@ -23,37 +22,35 @@ async function getBreachHashset(breach) {
    *   'LogoType' ]
    * See https://haveibeenpwned.com/API/v2#BreachModel for more
    */
-  if (breach.IsActive && breach.IsVerified && breach.DataClasses.includes("Email addresses")) {
+  const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${breach.Name}`;
-    const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${breach.Name}`;
+  const headers = {
-    const headers = {
+    "User-Agent": HIBP_USER_AGENT,
-      "User-Agent": HIBP_USER_AGENT,
+    "Authorization": HIBP_AUTH,
-      "Authorization": HIBP_AUTH,
+  };
-    };
+  const hashsetRequestObject = {
-    const hashsetRequestObject = {
+    url,
-      url,
+    headers,
-      headers,
+  };
-    };
 
-    console.log(`Active, verified breach with email addresses: ${breach.Name}`);
+  console.log(`Fetching ${url}...`);
-    console.log(`Fetching ${url}...`);
 
-    request(hashsetRequestObject, async (error, response, body) => {
+  request(hashsetRequestObject, async (error, response, body) => {
-      if (response.statusCode === 200) {
+    if (response.statusCode === 200) {
-        console.log("Uploading to S3 ...");
+      console.log("Uploading to S3 ...");
-        const uploadParams = {
+      const uploadParams = {
-          Bucket: BREACH_HASHSET_BUCKET_NAME,
+        Bucket: BREACH_HASHSET_BUCKET_NAME,
-          Key: `${breach.Name}.zip`,
+        Key: `${breach.Name}.zip`,
-          Body: body,
+        Body: body,
-        };
+      };
-        try {
+      s3.upload(uploadParams, (err, data) => {
-          const uploadData = await s3.upload(uploadParams);
+        if (err) {
-          console.log(`Uploaded to ${uploadData.Location}`);
-        } catch (err) {
           console.error(`err: ${err}`);
+          throw(err);
         }
-      }
+        console.log(`Uploaded to ${data.Location}`);
-    });
+      });
-  }
+    }
+  });
 }
 
 async function handleBreachesResponse(error, response, body) {
@@ -67,20 +64,24 @@ async function handleBreachesResponse(error, response, body) {
   try {
     const breachesJSON = JSON.parse(body);
 
-    const listData = await s3.listObjects({Bucket: BREACH_HASHSET_BUCKET_NAME});
-    console.log(`listData: ${listData}`);
-
-    const bucketData = await s3.createBucket({Bucket: BREACH_HASHSET_BUCKET_NAME});
-    console.log(`bucketData: ${bucketData}`);
-
-    if (!fs.existsSync(BREACH_HASHSET_DIR)) {
-      fs.mkdirSync(BREACH_HASHSET_DIR);
-    }
     for (const breach of breachesJSON) {
-      getBreachHashset(breach);
+      models.Breach.findOrCreate({where: {
+        name: breach.Name,
+      }}).spread((ourBreach, created) => {
+        if (breach.IsActive && breach.IsVerified && breach.DataClasses.includes("Email addresses")) {
+          console.log(`Active, verified breach with email addresses: ${breach.Name}. Checking if we have the latest data ...`);
+          if (created || Date(ourBreach.updatedAt) < Date(breach.ModifiedDate)) {
+            console.log("New breach, or breach modified since last update. Getting Hashset ...");
+            getBreachHashset(breach);
+          } else {
+            console.log("Breach not modified since last update. Done.");
+          }
+        }
+      });
     }
   } catch (err) {
     console.error(err);
+    throw(err);
   }
 }
 
@@ -94,4 +95,6 @@ function getBreaches() {
   request(breachesRequestObject, handleBreachesResponse);
 }
 
-getBreaches();
+models.sequelize.sync().then(()=>{
+  getBreaches();
+});