Use sequelize for hashset script
This commit is contained in:
Родитель
faf23821c1
Коммит
0115eec7e9
|
@ -26,3 +26,5 @@ PGPASSWORD=""
|
|||
PGDATABASE="blurts"
|
||||
PGHOST="localhost"
|
||||
PGPORT=5432
|
||||
|
||||
DATABASE_URL="postgres://postgres@localhost:5432/blurts"
|
||||
|
|
|
@ -1 +1,2 @@
|
|||
public/js/vendor/*.js
|
||||
db/migrations
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
const path = require("path");
|
||||
|
||||
module.exports = {
|
||||
"config": path.resolve("db", "config", "config.json"),
|
||||
"migrations-path": path.resolve("db", "migrations"),
|
||||
"models-path": path.resolve("db", "models"),
|
||||
}
|
|
@ -21,6 +21,7 @@ const kEnvironmentVariables = [
|
|||
"HIBP_API_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"DATABASE_URL",
|
||||
];
|
||||
|
||||
const AppConstants = { };
|
||||
|
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"development": {
|
||||
"use_env_variable": "DATABASE_URL"
|
||||
},
|
||||
"test": {
|
||||
"use_env_variable": "DATABASE_URL"
|
||||
},
|
||||
"production": {
|
||||
"use_env_variable": "DATABASE_URL",
|
||||
"logging": false
|
||||
}
|
||||
}
|
|
@ -0,0 +1,30 @@
|
|||
'use strict';
|
||||
module.exports = {
|
||||
up: (queryInterface, Sequelize) => {
|
||||
return queryInterface.createTable('Users', {
|
||||
id: {
|
||||
allowNull: false,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
type: Sequelize.INTEGER
|
||||
},
|
||||
email: {
|
||||
type: Sequelize.STRING
|
||||
},
|
||||
sha1: {
|
||||
type: Sequelize.STRING
|
||||
},
|
||||
createdAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
},
|
||||
updatedAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
}
|
||||
});
|
||||
},
|
||||
down: (queryInterface, Sequelize) => {
|
||||
return queryInterface.dropTable('Users');
|
||||
}
|
||||
};
|
|
@ -0,0 +1,27 @@
|
|||
'use strict';
|
||||
module.exports = {
|
||||
up: (queryInterface, Sequelize) => {
|
||||
return queryInterface.createTable('Breaches', {
|
||||
id: {
|
||||
allowNull: false,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
type: Sequelize.INTEGER
|
||||
},
|
||||
name: {
|
||||
type: Sequelize.STRING
|
||||
},
|
||||
createdAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
},
|
||||
updatedAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
}
|
||||
});
|
||||
},
|
||||
down: (queryInterface, Sequelize) => {
|
||||
return queryInterface.dropTable('Breaches');
|
||||
}
|
||||
};
|
|
@ -0,0 +1,35 @@
|
|||
'use strict';
|
||||
module.exports = {
|
||||
up: (queryInterface, Sequelize) => {
|
||||
return queryInterface.createTable('BreachedUsers', {
|
||||
id: {
|
||||
allowNull: false,
|
||||
autoIncrement: true,
|
||||
primaryKey: true,
|
||||
type: Sequelize.INTEGER
|
||||
},
|
||||
userId: {
|
||||
type: Sequelize.INTEGER,
|
||||
references: { model: 'Users', key: 'id' },
|
||||
},
|
||||
breachId: {
|
||||
type: Sequelize.INTEGER,
|
||||
references: { model: 'Breaches', key: 'id' },
|
||||
},
|
||||
notified: {
|
||||
type: Sequelize.DATE
|
||||
},
|
||||
createdAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
},
|
||||
updatedAt: {
|
||||
allowNull: false,
|
||||
type: Sequelize.DATE
|
||||
}
|
||||
});
|
||||
},
|
||||
down: (queryInterface, Sequelize) => {
|
||||
return queryInterface.dropTable('BreachedUsers');
|
||||
}
|
||||
};
|
|
@ -0,0 +1,10 @@
|
|||
"use strict";
|
||||
module.exports = (sequelize, DataTypes) => {
|
||||
const Breach = sequelize.define("Breach", {
|
||||
name: DataTypes.STRING,
|
||||
}, {});
|
||||
Breach.associate = function(models) {
|
||||
Breach.hasMany(models.BreachedUser);
|
||||
};
|
||||
return Breach;
|
||||
};
|
|
@ -0,0 +1,13 @@
|
|||
"use strict";
|
||||
module.exports = (sequelize, DataTypes) => {
|
||||
const BreachedUser = sequelize.define("BreachedUser", {
|
||||
userId: DataTypes.INTEGER,
|
||||
breachId: DataTypes.INTEGER,
|
||||
notified: DataTypes.DATE,
|
||||
}, {});
|
||||
BreachedUser.associate = function(models) {
|
||||
BreachedUser.belongsTo(models.User);
|
||||
BreachedUser.belongsTo(models.Breach);
|
||||
};
|
||||
return BreachedUser;
|
||||
};
|
|
@ -0,0 +1,38 @@
|
|||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const Sequelize = require("sequelize");
|
||||
const basename = path.basename(__filename);
|
||||
const env = process.env.NODE_ENV || "development"; // eslint-disable-line no-process-env
|
||||
const config = require(__dirname + "/../config/config.json")[env];
|
||||
const db = {};
|
||||
|
||||
let sequelize;
|
||||
|
||||
if (config.use_env_variable) {
|
||||
sequelize = new Sequelize(process.env[config.use_env_variable], config); // eslint-disable-line no-process-env
|
||||
} else {
|
||||
sequelize = new Sequelize(config.database, config.username, config.password, config);
|
||||
}
|
||||
|
||||
fs
|
||||
.readdirSync(__dirname)
|
||||
.filter(file => {
|
||||
return (file.indexOf(".") !== 0) && (file !== basename) && (file.slice(-3) === ".js");
|
||||
})
|
||||
.forEach(file => {
|
||||
const model = sequelize["import"](path.join(__dirname, file));
|
||||
db[model.name] = model;
|
||||
});
|
||||
|
||||
Object.keys(db).forEach(modelName => {
|
||||
if (db[modelName].associate) {
|
||||
db[modelName].associate(db);
|
||||
}
|
||||
});
|
||||
|
||||
db.sequelize = sequelize;
|
||||
db.Sequelize = Sequelize;
|
||||
|
||||
module.exports = db;
|
|
@ -0,0 +1,11 @@
|
|||
"use strict";
|
||||
module.exports = (sequelize, DataTypes) => {
|
||||
const User = sequelize.define("User", {
|
||||
email: DataTypes.STRING,
|
||||
sha1: DataTypes.STRING,
|
||||
}, {});
|
||||
User.associate = function(models) {
|
||||
User.hasMany(models.BreachedUser);
|
||||
};
|
||||
return User;
|
||||
};
|
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
@ -17,7 +17,9 @@
|
|||
"nodemailer": "^4.6.0",
|
||||
"pg": "^7.4.1",
|
||||
"popsicle": "^9.2.0",
|
||||
"request": "^2.83.0"
|
||||
"request": "^2.83.0",
|
||||
"sequelize": "^4.35.2",
|
||||
"sequelize-cli": "^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"coveralls": "^3.0.0",
|
||||
|
|
|
@ -1,21 +1,20 @@
|
|||
"use strict";
|
||||
|
||||
const fs = require("fs");
|
||||
const request = require("request");
|
||||
const S3 = require("aws-sdk/clients/s3");
|
||||
|
||||
const AppConstants = require("../app-constants").init();
|
||||
const AppConstants = require("../app-constants");
|
||||
const models = require("../db/models");
|
||||
const pkg = require("../package.json");
|
||||
|
||||
const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
|
||||
const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
|
||||
|
||||
const s3 = new S3();
|
||||
const BREACH_HASHSET_DIR = "../breach_hashsets";
|
||||
const BREACH_HASHSET_BUCKET_NAME = "mozilla.breach_alerts.stage.breach_hashsets";
|
||||
|
||||
|
||||
async function getBreachHashset(breach) {
|
||||
function getBreachHashset(breach) {
|
||||
/*
|
||||
* HIBP Breach, Object.keys(breach):
|
||||
* [ 'Title', 'Name', 'Domain', 'BreachDate', 'AddedDate', 'ModifiedDate', 'PwnCount', 'Description',
|
||||
|
@ -23,37 +22,35 @@ async function getBreachHashset(breach) {
|
|||
* 'LogoType' ]
|
||||
* See https://haveibeenpwned.com/API/v2#BreachModel for more
|
||||
*/
|
||||
if (breach.IsActive && breach.IsVerified && breach.DataClasses.includes("Email addresses")) {
|
||||
const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${breach.Name}`;
|
||||
const headers = {
|
||||
"User-Agent": HIBP_USER_AGENT,
|
||||
"Authorization": HIBP_AUTH,
|
||||
};
|
||||
const hashsetRequestObject = {
|
||||
url,
|
||||
headers,
|
||||
};
|
||||
const url = `${AppConstants.HIBP_API_ROOT}/enterprisesubscriber/hashset/${breach.Name}`;
|
||||
const headers = {
|
||||
"User-Agent": HIBP_USER_AGENT,
|
||||
"Authorization": HIBP_AUTH,
|
||||
};
|
||||
const hashsetRequestObject = {
|
||||
url,
|
||||
headers,
|
||||
};
|
||||
|
||||
console.log(`Active, verified breach with email addresses: ${breach.Name}`);
|
||||
console.log(`Fetching ${url}...`);
|
||||
console.log(`Fetching ${url}...`);
|
||||
|
||||
request(hashsetRequestObject, async (error, response, body) => {
|
||||
if (response.statusCode === 200) {
|
||||
console.log("Uploading to S3 ...");
|
||||
const uploadParams = {
|
||||
Bucket: BREACH_HASHSET_BUCKET_NAME,
|
||||
Key: `${breach.Name}.zip`,
|
||||
Body: body,
|
||||
};
|
||||
try {
|
||||
const uploadData = await s3.upload(uploadParams);
|
||||
console.log(`Uploaded to ${uploadData.Location}`);
|
||||
} catch (err) {
|
||||
request(hashsetRequestObject, async (error, response, body) => {
|
||||
if (response.statusCode === 200) {
|
||||
console.log("Uploading to S3 ...");
|
||||
const uploadParams = {
|
||||
Bucket: BREACH_HASHSET_BUCKET_NAME,
|
||||
Key: `${breach.Name}.zip`,
|
||||
Body: body,
|
||||
};
|
||||
s3.upload(uploadParams, (err, data) => {
|
||||
if (err) {
|
||||
console.error(`err: ${err}`);
|
||||
throw(err);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
console.log(`Uploaded to ${data.Location}`);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function handleBreachesResponse(error, response, body) {
|
||||
|
@ -67,20 +64,24 @@ async function handleBreachesResponse(error, response, body) {
|
|||
try {
|
||||
const breachesJSON = JSON.parse(body);
|
||||
|
||||
const listData = await s3.listObjects({Bucket: BREACH_HASHSET_BUCKET_NAME});
|
||||
console.log(`listData: ${listData}`);
|
||||
|
||||
const bucketData = await s3.createBucket({Bucket: BREACH_HASHSET_BUCKET_NAME});
|
||||
console.log(`bucketData: ${bucketData}`);
|
||||
|
||||
if (!fs.existsSync(BREACH_HASHSET_DIR)) {
|
||||
fs.mkdirSync(BREACH_HASHSET_DIR);
|
||||
}
|
||||
for (const breach of breachesJSON) {
|
||||
getBreachHashset(breach);
|
||||
models.Breach.findOrCreate({where: {
|
||||
name: breach.Name,
|
||||
}}).spread((ourBreach, created) => {
|
||||
if (breach.IsActive && breach.IsVerified && breach.DataClasses.includes("Email addresses")) {
|
||||
console.log(`Active, verified breach with email addresses: ${breach.Name}. Checking if we have the latest data ...`);
|
||||
if (created || Date(ourBreach.updatedAt) < Date(breach.ModifiedDate)) {
|
||||
console.log("New breach, or breach modified since last update. Getting Hashset ...");
|
||||
getBreachHashset(breach);
|
||||
} else {
|
||||
console.log("Breach not modified since last update. Done.");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err);
|
||||
throw(err);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -94,4 +95,6 @@ function getBreaches() {
|
|||
request(breachesRequestObject, handleBreachesResponse);
|
||||
}
|
||||
|
||||
getBreaches();
|
||||
models.sequelize.sync().then(()=>{
|
||||
getBreaches();
|
||||
});
|
||||
|
|
Загрузка…
Ссылка в новой задаче