mozilla
/
blurts-server
зеркало из https://github.com/mozilla/blurts-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

start code to load hashsets into aws

This commit is contained in:
groovecoder 2018-02-14 14:10:59 -06:00
Родитель 2c1f31d1cf
Коммит 6c92ef1a93
6 изменённых файлов: 237 добавлений и 402 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
.env-dist
Просмотреть файл

@ -17,3 +17,6 @@ OAUTH_TOKEN_URI="https://oauth-stable.dev.lcip.org/v1/token"
HIBP_API_ROOT="https://stage.haveibeenpwned.com/api/v2/"
HIBP_API_TOKEN=""
AWS_ACCESS_KEY_ID=""
AWS_SECRET_ACCESS_KEY=""

4
README.md
Просмотреть файл

@ -94,10 +94,10 @@ source it via the `.env` file, or set it directly:
export HIBP_API_TOKEN="<HIBP-API-TOKEN>"
```
With the `HIBP_API_TOKEN` set, run the `get_hashsets.js` script:
With the `HIBP_API_TOKEN` set, run the `get-hashsets.js` script:
```sh
node get_hashsets.js
npm run scripts/get-hashsets.js
```
This will download the `.zip` files into `breach_hashsets/` directory.

2
app-constants.js
Просмотреть файл

@ -19,6 +19,8 @@ const kEnvironmentVariables = [
"OAUTH_CLIENT_SECRET",
"HIBP_API_ROOT",
"HIBP_API_TOKEN",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
];
const AppConstants = {

586
package-lock.json сгенерированный
Просмотреть файл

@ -143,18 +143,27 @@
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"async": {
"version": "0.9.2",
"resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
"integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=",
"dev": true,
"optional": true
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sdk": {
"version": "2.198.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.198.0.tgz",
"integrity": "sha1-IkUxEzfdyETMNME/P887fqhD9DQ=",
"requires": {
"buffer": "4.9.1",
"events": "1.1.1",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.1.0",
"xml2js": "0.4.17",
"xmlbuilder": "4.2.1"
}
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@ -206,6 +215,11 @@
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"base64-js": {
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.3.tgz",
"integrity": "sha512-MsAhsUW1GxCdgYSO6tAfZrNapmUKk7mWx/k5mFY/A1gBtkaCaNapTg+FExCw1r9yeaZhqx/xPg43xgTFH6KL5w=="
},
"bcrypt-pbkdf": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
@ -215,41 +229,6 @@
"tweetnacl": "0.14.5"
}
},
"bl": {
"version": "0.9.5",
"resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz",
"integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=",
"dev": true,
"requires": {
"readable-stream": "1.0.34"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
"dev": true
},
"readable-stream": {
"version": "1.0.34",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
"dev": true,
"requires": {
"core-util-is": "1.0.2",
"inherits": "2.0.3",
"isarray": "0.0.1",
"string_decoder": "0.10.31"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
"dev": true
}
}
},
"body-parser": {
"version": "1.18.2",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
@ -285,6 +264,16 @@
"concat-map": "0.0.1"
}
},
"buffer": {
"version": "4.9.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
"integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
"requires": {
"base64-js": "1.2.3",
"ieee754": "1.1.8",
"isarray": "1.0.0"
}
},
"builtin-modules": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
@ -497,201 +486,6 @@
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
"dev": true
},
"codecov.io": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/codecov.io/-/codecov.io-0.0.1.tgz",
"integrity": "sha1-JeorCV4enqEYcr36WEIRgTDfeLE=",
"dev": true,
"requires": {
"request": "2.42.0",
"urlgrey": "0.4.0"
},
"dependencies": {
"asn1": {
"version": "0.1.11",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz",
"integrity": "sha1-VZvhg3bQik7E2+gId9J4GGObLfc=",
"dev": true,
"optional": true
},
"assert-plus": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz",
"integrity": "sha1-7nQAlBMALYTOxyGcasgRgS5yMWA=",
"dev": true,
"optional": true
},
"aws-sign2": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz",
"integrity": "sha1-xXED96F/wDfwLXwuZLYC6iI/fWM=",
"dev": true,
"optional": true
},
"boom": {
"version": "0.4.2",
"resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz",
"integrity": "sha1-emNune1O/O+xnO9JR6PGffrukRs=",
"dev": true,
"requires": {
"hoek": "0.9.1"
}
},
"caseless": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.6.0.tgz",
"integrity": "sha1-gWfBq4OX+1u5X5bSjlqBxQ8kesQ=",
"dev": true
},
"combined-stream": {
"version": "0.0.7",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz",
"integrity": "sha1-ATfmV7qlp1QcV6w3rF/AfXO03B8=",
"dev": true,
"optional": true,
"requires": {
"delayed-stream": "0.0.5"
}
},
"cryptiles": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-0.2.2.tgz",
"integrity": "sha1-7ZH/HxetE9N0gohZT4pIoNJvMlw=",
"dev": true,
"optional": true,
"requires": {
"boom": "0.4.2"
}
},
"delayed-stream": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz",
"integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=",
"dev": true,
"optional": true
},
"forever-agent": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz",
"integrity": "sha1-bQ4JxJIflKJ/Y9O0nF/v8epMUTA=",
"dev": true
},
"form-data": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-0.1.4.tgz",
"integrity": "sha1-kavXiKupcCsaq/qLwBAxoqyeOxI=",
"dev": true,
"optional": true,
"requires": {
"async": "0.9.2",
"combined-stream": "0.0.7",
"mime": "1.2.11"
}
},
"hawk": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-1.1.1.tgz",
"integrity": "sha1-h81JH5tG5OKurKM1QWdmiF0tHtk=",
"dev": true,
"optional": true,
"requires": {
"boom": "0.4.2",
"cryptiles": "0.2.2",
"hoek": "0.9.1",
"sntp": "0.2.4"
}
},
"hoek": {
"version": "0.9.1",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-0.9.1.tgz",
"integrity": "sha1-PTIkYrrfB3Fup+uFuviAec3c5QU=",
"dev": true
},
"http-signature": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz",
"integrity": "sha1-T72sEyVZqoMjEh5UB3nAoBKyfmY=",
"dev": true,
"optional": true,
"requires": {
"asn1": "0.1.11",
"assert-plus": "0.1.5",
"ctype": "0.5.3"
}
},
"mime": {
"version": "1.2.11",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.2.11.tgz",
"integrity": "sha1-WCA+7Ybjpe8XrtK32evUfwpg3RA=",
"dev": true,
"optional": true
},
"mime-types": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-1.0.2.tgz",
"integrity": "sha1-mVrhOSq4r/y/yyZB3QVOlDwNXc4=",
"dev": true
},
"node-uuid": {
"version": "1.4.8",
"resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz",
"integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc=",
"dev": true
},
"oauth-sign": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.4.0.tgz",
"integrity": "sha1-8ilW8x6nFRqCHl8vsywRPK2Ln2k=",
"dev": true,
"optional": true
},
"qs": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/qs/-/qs-1.2.2.tgz",
"integrity": "sha1-GbV/8k3CqZzh+L32r82ln472H4g=",
"dev": true
},
"request": {
"version": "2.42.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.42.0.tgz",
"integrity": "sha1-VyvQFIk4VkBArHqxSLlkI6BjMEo=",
"dev": true,
"requires": {
"aws-sign2": "0.5.0",
"bl": "0.9.5",
"caseless": "0.6.0",
"forever-agent": "0.5.2",
"form-data": "0.1.4",
"hawk": "1.1.1",
"http-signature": "0.10.1",
"json-stringify-safe": "5.0.1",
"mime-types": "1.0.2",
"node-uuid": "1.4.8",
"oauth-sign": "0.4.0",
"qs": "1.2.2",
"stringstream": "0.0.5",
"tough-cookie": "2.3.3",
"tunnel-agent": "0.4.3"
}
},
"sntp": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-0.2.4.tgz",
"integrity": "sha1-+4hfGLDzqtGJ+CSGJTa87ux1CQA=",
"dev": true,
"optional": true,
"requires": {
"hoek": "0.9.1"
}
},
"tunnel-agent": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz",
"integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=",
"dev": true
}
}
},
"color-convert": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
@ -780,7 +574,7 @@
"requires": {
"js-yaml": "3.10.0",
"lcov-parse": "0.0.10",
"log-driver": "1.2.6",
"log-driver": "1.2.7",
"minimist": "1.2.0",
"request": "2.83.0"
}
@ -814,13 +608,6 @@
}
}
},
"ctype": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz",
"integrity": "sha1-gsGMJGH3QRTvFsE1IkrQuRRMoS8=",
"dev": true,
"optional": true
},
"cvss": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/cvss/-/cvss-1.0.2.tgz",
@ -1162,19 +949,13 @@
"split": "0.3.3",
"stream-combiner": "0.0.4",
"through": "2.3.8"
},
"dependencies": {
"split": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz",
"integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
"dev": true,
"requires": {
"through": "2.3.8"
}
}
}
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
},
"execa": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
@ -1214,7 +995,7 @@
"on-finished": "2.3.0",
"parseurl": "1.3.2",
"path-to-regexp": "0.1.7",
"proxy-addr": "2.0.2",
"proxy-addr": "2.0.3",
"qs": "6.5.1",
"range-parser": "1.2.0",
"safe-buffer": "5.1.1",
@ -1612,6 +1393,11 @@
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
"integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
},
"ieee754": {
"version": "1.1.8",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz",
"integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q="
},
"ignore": {
"version": "3.3.7",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.7.tgz",
@ -1668,9 +1454,9 @@
"dev": true
},
"ipaddr.js": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.5.2.tgz",
"integrity": "sha1-1LUFvemUaYfM8PxY2QEP+WB+P6A="
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz",
"integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs="
},
"is-arrayish": {
"version": "0.2.1",
@ -1789,6 +1575,11 @@
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"js-tokens": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
@ -1886,15 +1677,23 @@
}
},
"load-json-file": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
"integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
"integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
"dev": true,
"requires": {
"graceful-fs": "4.1.11",
"parse-json": "2.2.0",
"pify": "2.3.0",
"parse-json": "4.0.0",
"pify": "3.0.0",
"strip-bom": "3.0.0"
},
"dependencies": {
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"dev": true
}
}
},
"locate-path": {
@ -1910,17 +1709,13 @@
"lodash": {
"version": "4.17.5",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz",
"integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==",
"dev": true
"integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw=="
},
"log-driver": {
"version": "1.2.6",
"resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.6.tgz",
"integrity": "sha512-iUHz4WAGsXwUmL1UergWrkFD2iTUrGLMsQDRYUWtS9FI+wSyM76vIL+THQt7vrQq5fZDGdrPSCFUfIlqII28tg==",
"dev": true,
"requires": {
"codecov.io": "0.0.1"
}
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz",
"integrity": "sha512-U7KCmLdqsGHBLeWqYlFA0V0Sl6P08EE1ZrmA9cxjUE0WVqT9qnyVDPz1kzpFEP0jdJuFnasWIfSd7fsaNXkpbg==",
"dev": true
},
"lru-cache": {
"version": "4.1.1",
@ -2107,54 +1902,6 @@
"requires": {
"color-convert": "1.9.1"
}
},
"load-json-file": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
"integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
"dev": true,
"requires": {
"graceful-fs": "4.1.11",
"parse-json": "4.0.0",
"pify": "3.0.0",
"strip-bom": "3.0.0"
}
},
"parse-json": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
"integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
"dev": true,
"requires": {
"error-ex": "1.3.1",
"json-parse-better-errors": "1.0.1"
}
},
"path-type": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
"integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
"dev": true,
"requires": {
"pify": "3.0.0"
}
},
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"dev": true
},
"read-pkg": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
"integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
"dev": true,
"requires": {
"load-json-file": "4.0.0",
"normalize-package-data": "2.4.0",
"path-type": "3.0.0"
}
}
}
},
@ -3796,9 +3543,9 @@
"dev": true
},
"object-inspect": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.3.0.tgz",
"integrity": "sha512-OHHnLgLNXpM++GnJRyyhbr2bwl3pPVm4YvaraHrRvDt/N3r+s/gDVHciA7EJBTkijKXj61ssgSAikq1fb0IBRg==",
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.5.0.tgz",
"integrity": "sha512-UmOFbHbwvv+XHj7BerrhVq+knjceBdkvU5AriwLMvhv2qi+e7DJzxfBeFpILEjVzCp+xA+W/pIf06RGPWlZNfw==",
"dev": true
},
"object-keys": {
@ -3895,12 +3642,13 @@
"dev": true
},
"parse-json": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
"integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=",
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
"integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
"dev": true,
"requires": {
"error-ex": "1.3.1"
"error-ex": "1.3.1",
"json-parse-better-errors": "1.0.1"
}
},
"parseurl": {
@ -3944,12 +3692,20 @@
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"path-type": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz",
"integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
"integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
"dev": true,
"requires": {
"pify": "2.3.0"
"pify": "3.0.0"
},
"dependencies": {
"pify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
"integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=",
"dev": true
}
}
},
"pause-stream": {
@ -4022,12 +3778,12 @@
"dev": true
},
"proxy-addr": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.2.tgz",
"integrity": "sha1-ZXFQT0e7mI7IGAJT+F3X4UlSvew=",
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz",
"integrity": "sha512-jQTChiCJteusULxjBp8+jftSQE5Obdl3k4cnmLA6WXtK6XFuWRnvVL7aCiBqaLPM8c4ph0S4tKna8XvmIwEnXQ==",
"requires": {
"forwarded": "0.1.2",
"ipaddr.js": "1.5.2"
"ipaddr.js": "1.6.0"
}
},
"ps-tree": {
@ -4046,15 +3802,20 @@
"dev": true
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
},
"qs": {
"version": "6.5.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
"integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
},
"querystring": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
},
"range-parser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
@ -4072,14 +3833,14 @@
}
},
"read-pkg": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz",
"integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
"integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
"dev": true,
"requires": {
"load-json-file": "2.0.0",
"load-json-file": "4.0.0",
"normalize-package-data": "2.4.0",
"path-type": "2.0.0"
"path-type": "3.0.0"
}
},
"read-pkg-up": {
@ -4090,6 +3851,49 @@
"requires": {
"find-up": "2.1.0",
"read-pkg": "2.0.0"
},
"dependencies": {
"load-json-file": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
"integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
"dev": true,
"requires": {
"graceful-fs": "4.1.11",
"parse-json": "2.2.0",
"pify": "2.3.0",
"strip-bom": "3.0.0"
}
},
"parse-json": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
"integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=",
"dev": true,
"requires": {
"error-ex": "1.3.1"
}
},
"path-type": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz",
"integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=",
"dev": true,
"requires": {
"pify": "2.3.0"
}
},
"read-pkg": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz",
"integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=",
"dev": true,
"requires": {
"load-json-file": "2.0.0",
"normalize-package-data": "2.4.0",
"path-type": "2.0.0"
}
}
}
},
"readable-stream": {
@ -4132,7 +3936,7 @@
"stringstream": "0.0.5",
"tough-cookie": "2.3.3",
"tunnel-agent": "0.6.0",
"uuid": "3.2.1"
"uuid": "3.1.0"
}
},
"require-directory": {
@ -4229,6 +4033,11 @@
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"semver": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz",
@ -4356,9 +4165,9 @@
"dev": true
},
"split": {
"version": "0.2.10",
"resolved": "https://registry.npmjs.org/split/-/split-0.2.10.tgz",
"integrity": "sha1-Zwl8YB1pfOE2j0GPBs0gHPBSGlc=",
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz",
"integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=",
"dev": true,
"requires": {
"through": "2.3.8"
@ -4542,9 +4351,9 @@
}
},
"tape": {
"version": "4.8.0",
"resolved": "https://registry.npmjs.org/tape/-/tape-4.8.0.tgz",
"integrity": "sha512-TWILfEnvO7I8mFe35d98F6T5fbLaEtbFTG/lxWvid8qDfFTxt19EBijWmB4j3+Hoh5TfHE2faWs73ua+EphuBA==",
"version": "4.9.0",
"resolved": "https://registry.npmjs.org/tape/-/tape-4.9.0.tgz",
"integrity": "sha512-j0jO9BiScfqtPBb9QmPLL0qvxXMz98xjkMb7x8lKipFlJZwNJkqkWPou+NU4V6T9RnVh1kuSthLE8gLrN8bBfw==",
"dev": true,
"requires": {
"deep-equal": "1.0.1",
@ -4555,8 +4364,8 @@
"has": "1.0.1",
"inherits": "2.0.3",
"minimist": "1.2.0",
"object-inspect": "1.3.0",
"resolve": "1.4.0",
"object-inspect": "1.5.0",
"resolve": "1.5.0",
"resumer": "0.0.0",
"string.prototype.trim": "1.1.2",
"through": "2.3.8"
@ -4573,15 +4382,6 @@
"resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz",
"integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=",
"dev": true
},
"resolve": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.4.0.tgz",
"integrity": "sha512-aW7sVKPufyHqOmyyLzg/J+8606v5nevBgaliIlV7nUpVMsDnoBGV/cbSLNjZAg9q0Cfd/+easKVKQ8vOu8fn1Q==",
"dev": true,
"requires": {
"path-parse": "1.0.5"
}
}
}
},
@ -4648,6 +4448,13 @@
"integrity": "sha1-C2GKVWW23qkL80JdBNVe3EdadWE=",
"requires": {
"punycode": "1.4.1"
},
"dependencies": {
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
}
}
},
"tunnel-agent": {
@ -4692,31 +4499,13 @@
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"urlgrey": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/urlgrey/-/urlgrey-0.4.0.tgz",
"integrity": "sha1-8GU1cED7NcOzEdTl3DZITZbb6gY=",
"dev": true,
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"tape": "2.3.0"
},
"dependencies": {
"tape": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/tape/-/tape-2.3.0.tgz",
"integrity": "sha1-Df7scJIn+8yRcKvn8EaWKycUMds=",
"dev": true,
"requires": {
"deep-equal": "0.1.2",
"defined": "0.0.0",
"inherits": "2.0.3",
"jsonify": "0.0.0",
"resumer": "0.0.0",
"split": "0.2.10",
"stream-combiner": "0.0.4",
"through": "2.3.8"
}
}
"punycode": "1.3.2",
"querystring": "0.2.0"
}
},
"util-deprecate": {
@ -4730,9 +4519,9 @@
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"uuid": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
"integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
"integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
},
"validate-npm-package-license": {
"version": "3.0.1",
@ -4857,6 +4646,23 @@
"mkdirp": "0.5.1"
}
},
"xml2js": {
"version": "0.4.17",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.17.tgz",
"integrity": "sha1-F76T6q4/O3eTWceVtBlwWogX6Gg=",
"requires": {
"sax": "1.2.1",
"xmlbuilder": "4.2.1"
}
},
"xmlbuilder": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz",
"integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=",
"requires": {
"lodash": "4.17.5"
}
},
"xtend": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-2.1.2.tgz",

3
package.json
Просмотреть файл

@ -7,6 +7,7 @@
"url": "https://github.com/mozilla/blurts-server/issues"
},
"dependencies": {
"aws-sdk": "^2.194.0",
"body-parser": "^1.18.2",
"client-oauth2": "^4.1.0",
"client-sessions": "^0.8.0",
@ -41,11 +42,11 @@
"url": "git+https://github.com/mozilla/blurts-server.git"
},
"scripts": {
"get_hashsets": "node get_hashsets",
"lint": "npm-run-all lint:*",
"lint:js": "eslint .",
"lint:nsp": "nsp check",
"pretest": "npm run lint",
"get-hashsets": "node scripts/get-hashsets",
"start": "node server.js",
"test": "nyc tape tests/**/test*.js | faucet && nyc report --reporter=text-lcov | coveralls"
}

41
get_hashsets.js → scripts/get-hashsets.js
Просмотреть файл

@ -1,19 +1,21 @@
"use strict";
const AppConstants = require("./app-constants").init();
const fs = require("fs");
const path = require("path");
const request = require("request");
const S3 = require("aws-sdk/clients/s3");
const pkg = require("./package.json");
const AppConstants = require("../app-constants").init();
const pkg = require("../package.json");
const HIBP_AUTH = `Bearer ${AppConstants.HIBP_API_TOKEN}`;
const HIBP_USER_AGENT = `${pkg.name}/${pkg.version}`;
const BREACH_HASHSET_DIR = "breach_hashsets";
const s3 = new S3();
const BREACH_HASHSET_DIR = "../breach_hashsets";
const BREACH_HASHSET_BUCKET_NAME = "mozilla.breach_alerts.stage.breach_hashsets";
function getBreachHashset(breach) {
async function getBreachHashset(breach) {
/*
* HIBP Breach, Object.keys(breach):
* [ 'Title', 'Name', 'Domain', 'BreachDate', 'AddedDate', 'ModifiedDate', 'PwnCount', 'Description',
@ -31,15 +33,30 @@ function getBreachHashset(breach) {
url,
headers,
};
const BREACH_HASHSET_ZIP = path.join(BREACH_HASHSET_DIR, `${breach.Name}.zip`);
console.log(`Active, verified breach with email addresses: ${breach.Name}`);
console.log(`Fetching ${url}...`);
request(hashsetRequestObject).pipe(fs.createWriteStream(BREACH_HASHSET_ZIP));
request(hashsetRequestObject, async (error, response, body) => {
if (response.statusCode === 200) {
console.log("Uploading to S3 ...");
const uploadParams = {
Bucket: BREACH_HASHSET_BUCKET_NAME,
Key: `${breach.Name}.zip`,
Body: body,
};
try {
const uploadData = await s3.upload(uploadParams);
console.log(`Uploaded to ${uploadData.Location}`);
} catch (err) {
console.error(`err: ${err}`);
}
}
});
}
}
function handleBreachesResponse(error, response, body) {
async function handleBreachesResponse(error, response, body) {
if (error) {
console.error(error);
// We can `process.exit()` here since it's a CLI script.
@ -50,6 +67,12 @@ function handleBreachesResponse(error, response, body) {
try {
const breachesJSON = JSON.parse(body);
const listData = await s3.listObjects({Bucket: BREACH_HASHSET_BUCKET_NAME});
console.log(`listData: ${listData}`);
const bucketData = await s3.createBucket({Bucket: BREACH_HASHSET_BUCKET_NAME});
console.log(`bucketData: ${bucketData}`);
if (!fs.existsSync(BREACH_HASHSET_DIR)) {
fs.mkdirSync(BREACH_HASHSET_DIR);
}