Replace Sequelize with Objection+Knex, update make-breach-with-emails.js

db/config/config.json

@@ -1,12 +0,0 @@
-{
-  "development": {
-    "use_env_variable": "DATABASE_URL"
-  },
-  "test": {
-    "use_env_variable": "DATABASE_URL"
-  },
-  "production": {
-    "use_env_variable": "DATABASE_URL",
-    "logging": false
-  }
-}

db/knexfile.js

@@ -0,0 +1,23 @@
+module.exports = {
+  development: {
+    client: "postgresql",
+    connection: {
+      database: "blurts",
+    },
+    pool: {
+      min: 2,
+      max: 10,
+    },
+  },
+
+  production: {
+    client: "postgresql",
+    connection: {
+      database: "blurts",
+    },
+    pool: {
+      min: 2,
+      max: 10,
+    },
+  },
+};

db/migrations/20180308192818-create-breach.js

@@ -1,27 +0,0 @@
-'use strict';
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('Breaches', {
-      id: {
-        allowNull: false,
-        autoIncrement: true,
-        primaryKey: true,
-        type: Sequelize.INTEGER
-      },
-      name: {
-        type: Sequelize.STRING
-      },
-      createdAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      }
-    });
-  },
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('Breaches');
-  }
-};

db/migrations/20180327165814-create-email-hash.js

@@ -1,30 +0,0 @@
-'use strict';
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('EmailHashes', {
-      id: {
-        allowNull: false,
-        autoIncrement: true,
-        primaryKey: true,
-        type: Sequelize.INTEGER
-      },
-      SubscriberId: {
-        type: Sequelize.INTEGER,
-      },
-      sha1: {
-        type: Sequelize.STRING
-      },
-      createdAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      }
-    });
-  },
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('EmailHashes');
-  }
-};

db/migrations/20180327170759-create-subscriber.js

@@ -1,30 +0,0 @@
-'use strict';
-module.exports = {
-  up: (queryInterface, Sequelize) => {
-    return queryInterface.createTable('Subscribers', {
-      id: {
-        allowNull: false,
-        autoIncrement: true,
-        primaryKey: true,
-        type: Sequelize.INTEGER
-      },
-      email: {
-        type: Sequelize.STRING
-      },
-      verificationToken: {
-        type: Sequelize.STRING
-      },
-      createdAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      },
-      updatedAt: {
-        allowNull: false,
-        type: Sequelize.DATE
-      }
-    });
-  },
-  down: (queryInterface, Sequelize) => {
-    return queryInterface.dropTable('Subscribers');
-  }
-};

db/migrations/initial_schema.js

@@ -0,0 +1,35 @@
+"use strict";
+
+exports.up = knex => {
+  return knex.schema
+    .createTable("email_hashes", table => {
+      table.increments("id").primary();
+      table.string("sha1").unique();
+      table.string("email").unique();
+    })
+    .createTable("breaches", table => {
+      table.increments("id").primary();
+      table.string("name").unique();
+      table.string("meta");
+    })
+    .createTable("breached_hashes", table => {
+      table
+        .integer("sha1_id")
+        .unsigned()
+        .references("id")
+        .inTable("email_hashes");
+      table
+        .integer("breach_id")
+        .unsigned()
+        .references("id")
+        .inTable("breaches");
+    });
+};
+
+exports.down = knex => {
+  return knex.schema
+    .dropTableIfExists("subscribers")
+    .dropTableIfExists("email_hashes")
+    .dropTableIfExists("breaches")
+    .dropTableIfExists("breached_hashes");
+};

db/models/breach.js

@@ -1,13 +1,42 @@
-"use strict";
+'use strict';
 
-module.exports = (sequelize, DataTypes) => {
-  const Breach = sequelize.define("Breach", {
-    name: DataTypes.STRING,
-  }, {});
+const Model = require("objection").Model;
 
-  Breach.associate = function(models) {
-    Breach.belongsToMany(models.EmailHash, { through: "BreachedHashes" });
-  };
+class Breach extends Model {
+  // Table name is the only required property.
+  static get tableName() {
+    return "breaches";
+  }
 
-  return Breach;
-};
+/*
+  static get jsonSchema() {
+    return {
+      type: "object",
+      required: [],
+
+      properties: {
+        id: { type: "integer" },
+      }
+    };
+  }
+*/
+
+  static get relationMappings() {
+    return {
+      email_hashes: {
+        relation: Model.ManyToManyRelation,
+        modelClass: `${__dirname}/EmailHash`,
+        join: {
+          from: "breaches.id",
+          through: {
+            from: "breached_hashes.breach_id",
+            to: "breached_hashes.sha1_id",
+          },
+          to: "email_hashes.id",
+        },
+      },
+    };
+  }
+}
+
+module.exports = Breach;

db/models/emailhash.js

@@ -1,13 +1,42 @@
-"use strict";
+'use strict';
 
-module.exports = (sequelize, DataTypes) => {
-  const EmailHash = sequelize.define("EmailHash", {
-    sha1: DataTypes.STRING,
-  }, {});
+const Model = require("objection").Model;
 
-  EmailHash.associate = function(models) {
-    EmailHash.belongsToMany(models.Breach, { through: "BreachedHashes" });
-  };
+class EmailHash extends Model {
+  // Table name is the only required property.
+  static get tableName() {
+    return "email_hashes";
+  }
 
-  return EmailHash;
-};
+/*
+  static get jsonSchema() {
+    return {
+      type: "object",
+      required: [],
+
+      properties: {
+        id: { type: "integer" },
+      }
+    };
+  }
+*/
+
+  static get relationMappings() {
+    return {
+      breaches: {
+        relation: Model.ManyToManyRelation,
+        modelClass: __dirname + "/Breach",
+        join: {
+          from: "email_hashes.id",
+          through: {
+            from: "breached_hashes.sha1_id",
+            to: "breached_hashes.breach_id",
+          },
+          to: "breaches.id",
+        },
+      },
+    };
+  }
+}
+
+module.exports = EmailHash;

db/models/index.js

@@ -1,38 +0,0 @@
-"use strict";
-
-const fs        = require("fs");
-const path      = require("path");
-const Sequelize = require("sequelize");
-const basename  = path.basename(__filename);
-const env       = process.env.NODE_ENV || "development"; // eslint-disable-line no-process-env
-const config    = require(__dirname + "/../config/config.json")[env];
-const db        = {};
-
-let sequelize;
-
-if (config.use_env_variable) {
-  sequelize = new Sequelize(process.env[config.use_env_variable], config); // eslint-disable-line no-process-env
-} else {
-  sequelize = new Sequelize(config.database, config.username, config.password, config);
-}
-
-fs
-  .readdirSync(__dirname)
-  .filter(file => {
-    return (file.indexOf(".") !== 0) && (file !== basename) && (file.slice(-3) === ".js");
-  })
-  .forEach(file => {
-    const model = sequelize["import"](path.join(__dirname, file));
-    db[model.name] = model;
-  });
-
-Object.keys(db).forEach(modelName => {
-  if (db[modelName].associate) {
-    db[modelName].associate(db);
-  }
-});
-
-db.sequelize = sequelize;
-db.Sequelize = Sequelize;
-
-module.exports = db;

db/models/subscriber.js

@@ -1,33 +1,38 @@
-"use strict";
+'use strict';
 
-const crypto = require("crypto");
+const Model = require("objection").Model;
 
-module.exports = (sequelize, DataTypes) => {
+class Subscriber extends Model {
+  // Table name is the only required property.
+  static get tableName() {
+    return "subscribers";
+  }
 
-  const EmailHash = sequelize.import("./emailhash");
+/*
+  static get jsonSchema() {
+    return {
+      type: "object",
+      required: [],
 
-  const Subscriber = sequelize.define("Subscriber", {
-    email: {
-      type: DataTypes.STRING,
-      validate: { isEmail: true },
-    },
-    verificationToken: {
-      type: DataTypes.STRING,
-      defaultValue: function () {
-        return crypto.randomBytes(40).toString("hex");
+      properties: {
+        id: { type: "integer" },
+      }
+    };
+  }
+*/
+
+  static get relationMappings() {
+    return {
+      sha1: {
+        relation: Model.BelongsToOneRelation,
+        modelClass: __dirname + "/EmailHash",
+        join: {
+          from: "subscribers.sha1_id",
+          to: "email_hashes.id",
+        },
       },
-    },
-  }, {});
+    };
+  }
+}
 
-  Subscriber.associate = function() {
-    Subscriber.hasOne(EmailHash);
-  };
-
-  Subscriber.prototype.saveSha1 = async function() {
-    const sha1 = crypto.createHash("sha1").update(this.email).digest("hex");
-    const emailHash = await EmailHash.findOrCreate( { where: { sha1 }});
-    await this.setEmailHash(emailHash.id);
-  };
-
-  return Subscriber;
-};
+module.exports = Subscriber;

db/utils.js

@@ -0,0 +1,76 @@
+"use strict";
+
+const Breach = require("./models/Breach");
+const EmailHash = require("./models/EmailHash");
+
+const getSha1 = require("../sha1-utils");
+
+const DBUtils = {
+  async createBreach(name, meta) {
+    try {
+    return await Breach
+      .query()
+      .insert({ name, meta });
+    } catch(e) {}
+  },
+
+  async deleteBreach(id) {
+    await Breach.query().deleteById(id);
+  },
+
+  // Used internally, ideally should not be called by consumers.
+  async _addEmailHash(sha1, email) {
+    // Check if an entry exists
+    const existingEntries = await EmailHash
+      .query()
+      .where("sha1", sha1);
+
+    // If not, add it and return.
+    if (!existingEntries.length) {
+      return await EmailHash
+        .query()
+        .insert({ sha1, email });
+    }
+
+    // Entry existed, patch the email value if supplied.
+    if (email) {
+      return await existingEntries[0]
+        .$query()
+        .patch({ email })
+        .returning('*'); // Postgres trick to return the updated row as model.
+    }
+
+    return existingEntries[0];
+  },
+
+  async addSubscriber(email) {
+    const sha1 = getSha1(email);
+    return await this._addEmailHash(sha1, email);
+  },
+
+  async addBreachedHash(breachName, sha1) {
+    console.log(`Adding ${sha1} to ${breachName}`);
+    const addedEmailHash = await this._addEmailHash(sha1);
+    console.log(`Added email hash id: ${addedEmailHash.id}`);
+    const breachesByName = await Breach
+      .query()
+      .where("name", breachName);
+    console.log(`Got ${breachesByName.length} breaches for that name`);
+    await breachesByName[0]
+      .$relatedQuery("email_hashes")
+      .relate(addedEmailHash.id);
+  },
+
+  async getBreachesForHash(sha1) {
+    console.log(`Finding EmailHash entry for ${sha1}`);
+    const emailHashesBySha1 = await EmailHash
+      .query()
+      .where("sha1", sha1);
+    console.log(`Found ${emailHashesBySha1.length} entries`);
+    return await emailHashesBySha1[0]
+      .$relatedQuery("breaches")
+      .orderBy("name");
+  }
+};
+
+module.exports = DBUtils;

models/db.js

@@ -0,0 +1,49 @@
+"use strict";
+
+require("dotenv").load();
+
+const pg = require("pg");
+
+function assertTestingEnvironment() {
+  // eslint-disable-next-line no-process-env
+  if (!process.env.TESTING_ENVIRONMENT) {
+    throw new Error("Attempting to run database setup without TESTING_ENVIRONMENT set, exiting.");
+  }
+}
+
+const DBUtils = {
+  async setupUsersTable() {
+    assertTestingEnvironment();
+
+    // Use PG* env vars to configure client.
+    const client = new pg.Client();
+    try {
+      await client.connect();
+      await client.query("DROP TABLE IF EXISTS users;");
+      await client.query("CREATE TABLE users ( email VARCHAR(320) UNIQUE );");
+    } finally {
+      await client.end();
+    }
+  },
+
+  async setupTempUsersTable() {
+    assertTestingEnvironment();
+
+    // Use PG* env vars to configure client.
+    const client = new pg.Client();
+    try {
+      await client.connect();
+      await client.query("DROP TABLE IF EXISTS users_temp;");
+      await client.query(`CREATE TABLE users_temp (
+                            email VARCHAR(320) UNIQUE,
+                            token VARCHAR(80),
+                            time_added TIMESTAMP DEFAULT NOW()
+                          );`);
+
+    } finally {
+      await client.end();
+    }
+  },
+};
+
+module.exports = DBUtils;

package.json

@@ -15,11 +15,12 @@
     "express": "^4.16.2",
     "express-hbs": "^1.0.4",
     "hbs": "^4.0.1",
+    "knex": "^0.14.6",
     "nodemailer": "^4.6.0",
+    "objection": "^1.1.6",
     "pg": "^7.4.1",
     "popsicle": "^9.2.0",
     "request": "^2.85.0",
-    "sequelize": "^4.35.2",
     "sequelize-cli": "^4.0.0"
   },
   "devDependencies": {
@@ -51,6 +52,7 @@
     "lint": "npm-run-all lint:*",
     "lint:js": "eslint .",
     "lint:nsp": "nsp check",
+    "migrate": "knex migrate:latest --knexfile db/knexfile.js",
     "pretest": "npm run lint",
     "get-hashsets": "node scripts/get-hashsets",
     "start": "node server.js",

server.js

@@ -5,12 +5,18 @@ const AppConstants = require("./app-constants");
 const express = require("express");
 const hbs = require("express-hbs");
 const sessions = require("client-sessions");
+const Knex = require("knex");
+const knexConfig = require('./db/knexfile');
+const { Model } = require("objection");
 
 const EmailUtils = require("./email-utils");
 const BaseRoutes = require("./routes/home");
 const OAuthRoutes = require("./routes/oauth");
 const UserRoutes = require("./routes/user");
 
+const knex = Knex(knexConfig.development);
+Model.knex(knex);
+
 const app = express();
 app.use(express.static("public"));
 

sha1-utils.js

@@ -0,0 +1,9 @@
+"use strict";
+
+const crypto = require("crypto");
+
+function getSha1(email) {
+  return crypto.createHash("sha1").update(email).digest("hex");
+};
+
+module.exports = getSha1;

tests/fixtures/make-breach-with-emails.js

@@ -1,43 +1,46 @@
 "use strict";
 
 require("dotenv").load();
+const Knex = require("knex");
+const knexConfig = require('../../db/knexfile');
+const { Model } = require("objection");
 
-const models = require("../../db/models");
-const crypto = require("crypto");
+const DBUtils = require("../../db/utils.js")
+const getSha1 = require("../../sha1-utils");
+
+const knex = Knex(knexConfig.development);
+Model.knex(knex);
 
 const sampleBreaches = [
   {
     name: "Test Breach 1",
+    meta: { },
     emails: [ "test1@test.com", "test2@test.com" ],
   },
   {
     name: "Test Breach 2",
+    meta: { },
     emails: [ "test2@test.com", "test3@test.com" ],
   },
   {
     name: "Test Breach 3",
+    meta: { },
     emails: [ "test3@test.com", "test1@test.com" ],
   },
 ];
 
-models.sequelize.sync().then(async () => {
+(async () => {
   for (const sB of sampleBreaches) {
-    const [breach] = await models.Breach.findOrCreate({ where: { name: sB.name }});
+    await DBUtils.createBreach(sB.name, sB.meta);
     for (const e of sB.emails) {
-      const [emailHash] = await models.EmailHash.findOrCreate({where: { sha1: getSha1(e) }});
-      await emailHash.addBreach(breach);
+      await DBUtils.addBreachedHash(sB.name, getSha1(e));
     }
   }
 
   const testEmail = "test1@test.com";
-  const emailHash = await models.EmailHash.findOne({ where: { sha1: getSha1(testEmail) }});
-  const foundBreaches = (await emailHash.getBreaches()).map(aBreach => aBreach.dataValues.name);
+  const foundBreaches = await DBUtils.getBreachesForHash(getSha1(testEmail));
   console.log(`\n\n${testEmail} was found in the following breaches:\n`);
-  console.log(foundBreaches);
+  console.log(foundBreaches.map(b => b.name));
   // eslint-disable-next-line no-process-exit
   process.exit();
-});
-
-function getSha1(email) {
-  return crypto.createHash("sha1").update(email).digest("hex");
-}
+})();