diff --git a/.gitignore b/.gitignore index be60d7d..b632c89 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ lib # test files input/ output/ +.vscode/launch.json diff --git a/package-lock.json b/package-lock.json index 7d6a535..ccf8374 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "migrate-to-graph", - "version": "0.0.2", + "version": "0.2.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -267,9 +267,9 @@ "dev": true }, "@types/bluebird": { - "version": "3.5.24", - "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.24.tgz", - "integrity": "sha512-YeQoDpq4Lm8ppSBqAnAeF/xy1cYp/dMTif2JFcvmAbETMRlvKHT2iLcWu+WyYiJO3b3Ivokwo7EQca/xfLVJmg==", + "version": "3.5.27", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.27.tgz", + "integrity": "sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==", "dev": true }, "@types/continuation-local-storage": { @@ -299,6 +299,15 @@ "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", "dev": true }, + "@types/flush-write-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/flush-write-stream/-/flush-write-stream-1.0.0.tgz", + "integrity": "sha512-vABEdzyrFxWkHeoLnaxc9NILYA7BWlY2BHQifHDVSRUZbWVuPSB22nzygWeWmjA9en2OzTx6waY7YPs+m9yzmg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/fs-extra": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.4.tgz", @@ -320,9 +329,9 @@ "dev": true }, "@types/lodash": { - "version": "4.14.117", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.117.tgz", - "integrity": "sha512-xyf2m6tRbz8qQKcxYZa7PA4SllYcay+eh25DN3jmNYY6gSTL7Htc/bttVdkqj2wfJGbeWlQiX8pIyJpKU+tubw==", + "version": "4.14.134", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.134.tgz", + "integrity": "sha512-2/O0khFUCFeDlbi7sZ7ZFRCcT812fAeOLm7Ev4KbwASkZ575TDrDcY7YyaoHdTOzKcNbfiwLYZqPmoC4wadrsw==", "dev": true }, "@types/mssql": { @@ -350,9 +359,9 @@ } }, "@types/sequelize": { - "version": "4.27.29", - "resolved": "https://registry.npmjs.org/@types/sequelize/-/sequelize-4.27.29.tgz", - "integrity": "sha512-VER50QSlqcjioIlmr2b4SeKWC8CLAVMkju9qnxScqu+5laPsPNS9+58HAmnkUUWsm8LFUDqH9oVJkdtS19h2zQ==", + "version": "4.28.3", + "resolved": "https://registry.npmjs.org/@types/sequelize/-/sequelize-4.28.3.tgz", + "integrity": "sha512-Z+gbWsLmGnqnAz91I8Wtfbe+4D1aUyQ4aawYfohpvar4Azr8aSNwzS1rFQYO2dk6zcClt5k9n0qCek/YAq4WNw==", "dev": true, "requires": { "@types/bluebird": "*", @@ -386,9 +395,9 @@ } }, "@types/validator": { - "version": "9.4.2", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-9.4.2.tgz", - "integrity": "sha512-v6H2QH+oXVdLKp9keOJi5LQSt6X5/XIOtK1YmbCzvkAT2kHW9WyQkixit9w1UgJpBGrDCqqCZlQ+Qucpmsf8hA==", + "version": "10.11.1", + "resolved": "https://registry.npmjs.org/@types/validator/-/validator-10.11.1.tgz", + "integrity": "sha512-bVhLqvb+5xUNWRFnuuecRVISTvsG6AdhrB2kb/tChgtuTTqARqlQ3rLhOPy8cINZEUB8PkR+goyWF6fWxg4iSw==", "dev": true }, "JSONStream": { @@ -511,6 +520,15 @@ "normalize-path": "^2.1.1" } }, + "append-transform": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz", + "integrity": "sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==", + "dev": true, + "requires": { + "default-require-extensions": "^2.0.0" + } + }, "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", @@ -830,9 +848,9 @@ } }, "bluebird": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.1.tgz", + "integrity": "sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg==" }, "brace-expansion": { "version": "1.1.11", @@ -1124,6 +1142,12 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz", "integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg==" }, + "compare-versions": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.4.0.tgz", + "integrity": "sha512-tK69D7oNXXqUW3ZNo/z7NXTEz22TCF0pTE+YF9cxvaAM9XnkLo1fV621xCLrRR6aevJlKxExkss0vWqUCUpqdg==", + "dev": true + }, "component-emitter": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", @@ -1290,6 +1314,15 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, + "default-require-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", + "integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=", + "dev": true, + "requires": { + "strip-bom": "^3.0.0" + } + }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -1713,6 +1746,16 @@ "bser": "^2.0.0" } }, + "fileset": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/fileset/-/fileset-2.0.3.tgz", + "integrity": "sha1-jnVIqW08wjJ+5eZ0FocjozO7oqA=", + "dev": true, + "requires": { + "glob": "^7.0.3", + "minimatch": "^3.0.3" + } + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -1771,6 +1814,27 @@ "resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz", "integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q==" }, + "flush-write-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-2.0.0.tgz", + "integrity": "sha512-uXClqPxT4xW0lcdSBheb2ObVU+kuqUk3Jk64EwieirEXZx9XUrVwp/JuBfKAWaM4T5Td/VL7QLDWPXp/MvGm/g==", + "requires": { + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -1850,7 +1914,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -1871,12 +1936,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1891,17 +1958,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -2018,7 +2088,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -2030,6 +2101,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2044,6 +2116,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2051,12 +2124,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -2075,6 +2150,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -2155,7 +2231,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -2167,6 +2244,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -2252,7 +2330,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -2288,6 +2367,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -2307,6 +2387,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -2350,12 +2431,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -2399,9 +2482,9 @@ } }, "generic-pool": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.4.2.tgz", - "integrity": "sha512-H7cUpwCQSiJmAHM4c/aFu6fUfrhWXW1ncyh8ftxEPMu6AiYkHw9K8br720TGPZJbk5eOH2bynjZD1yPvdDAmag==" + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz", + "integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w==" }, "get-caller-file": { "version": "1.0.3", @@ -2498,9 +2581,9 @@ "dev": true }, "handlebars": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz", - "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.3.tgz", + "integrity": "sha512-B0W4A2U1ww3q7VVthTKfh+epHx+q4mCt6iK+zEAzbMBpWQAwxCeKxEGpj/1oQTpzPXDNSOG7hmG14TsISH50yw==", "requires": { "neo-async": "^2.6.0", "optimist": "^0.6.1", @@ -2962,13 +3045,19 @@ }, "istanbul-api": { "version": "2.1.1", - "resolved": "", + "resolved": "https://registry.npmjs.org/istanbul-api/-/istanbul-api-2.1.1.tgz", + "integrity": "sha512-kVmYrehiwyeBAk/wE71tW6emzLiHGjYIiDrc8sfyty4F8M02/lrgXSm+R1kXysmF20zArvmZXjlE/mg24TVPJw==", "dev": true, "requires": { "async": "^2.6.1", + "compare-versions": "^3.2.1", + "fileset": "^2.0.3", "istanbul-lib-coverage": "^2.0.3", + "istanbul-lib-hook": "^2.0.3", "istanbul-lib-instrument": "^3.1.0", + "istanbul-lib-report": "^2.0.4", "istanbul-lib-source-maps": "^3.0.2", + "istanbul-reports": "^2.1.1", "js-yaml": "^3.12.0", "make-dir": "^1.3.0", "minimatch": "^3.0.4", @@ -2981,6 +3070,15 @@ "integrity": "sha512-dKWuzRGCs4G+67VfW9pBFFz2Jpi4vSp/k7zBcJ888ofV5Mi1g5CUML5GvMvV6u9Cjybftu+E8Cgp+k0dI1E5lw==", "dev": true }, + "istanbul-lib-hook": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz", + "integrity": "sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA==", + "dev": true, + "requires": { + "append-transform": "^1.0.0" + } + }, "istanbul-lib-instrument": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.1.0.tgz", @@ -2996,6 +3094,56 @@ "semver": "^5.5.0" } }, + "istanbul-lib-report": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz", + "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==", + "dev": true, + "requires": { + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "supports-color": "^6.1.0" + }, + "dependencies": { + "istanbul-lib-coverage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", + "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==", + "dev": true + }, + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "semver": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", + "dev": true + }, + "supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + } + } + }, "istanbul-lib-source-maps": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.2.tgz", @@ -3026,6 +3174,15 @@ } } }, + "istanbul-reports": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.6.tgz", + "integrity": "sha512-SKi4rnMyLBKe0Jy2uUdx28h8oG7ph2PPuQPvIAh31d+Ci+lSiEu4C+h3oBPuJ9+mPKhOyW0M8gY4U5NM1WLeXA==", + "dev": true, + "requires": { + "handlebars": "^4.1.2" + } + }, "jest": { "version": "24.1.0", "resolved": "https://registry.npmjs.org/jest/-/jest-24.1.0.tgz", @@ -4250,9 +4407,9 @@ } }, "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" }, "lodash.sortby": { "version": "4.7.0", @@ -4417,7 +4574,7 @@ }, "minimist": { "version": "0.0.10", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=" }, "minipass": { @@ -4445,9 +4602,9 @@ } }, "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", "requires": { "for-in": "^1.0.2", "is-extendable": "^1.0.1" @@ -4479,14 +4636,14 @@ } }, "moment": { - "version": "2.22.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz", - "integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y=" + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, "moment-timezone": { - "version": "0.5.21", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.21.tgz", - "integrity": "sha512-j96bAh4otsgj3lKydm3K7kdtA3iKf2m6MY2iSYCzCm5a1zmHo1g+aK3068dDEeocLZQIS9kU8bsdQHLqEvgW0A==", + "version": "0.5.27", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.27.tgz", + "integrity": "sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw==", "requires": { "moment": ">= 2.9.0" } @@ -5562,16 +5719,16 @@ "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4=" }, "sequelize": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.41.0.tgz", - "integrity": "sha512-6zCYxjPriUBgLQ5dN4ZxC5eadUtRu2tHFoBiSU9vNGb6G3f4bZM6vFpJu0DhxeeyhodTAXqY1GaBpZzT3PXB/Q==", + "version": "4.44.3", + "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.44.3.tgz", + "integrity": "sha512-r2A4EVDKRCcABcZhY4ItvbcosvMJKpQMooxg/S8ouRFrZzqMPQ9O2thOUfgW59q8ZcEa5ccNeqwg15MCciqPMg==", "requires": { "bluebird": "^3.5.0", "cls-bluebird": "^2.1.0", "debug": "^3.1.0", "depd": "^1.1.0", "dottie": "^2.0.0", - "generic-pool": "^3.4.0", + "generic-pool": "3.5.0", "inflection": "1.12.0", "lodash": "^4.17.1", "moment": "^2.20.0", @@ -5594,9 +5751,9 @@ } }, "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" } } }, @@ -5606,9 +5763,9 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" }, "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", "requires": { "extend-shallow": "^2.0.1", "is-extendable": "^0.1.1", @@ -5648,9 +5805,9 @@ "dev": true }, "shimmer": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz", - "integrity": "sha512-xTCx2vohXC2EWWDqY/zb4+5Mu28D+HYNSOuFzsyRDRvI/e1ICb69afwaUwfjr+25ZXldbOLyp+iDUZHq8UnTag==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" }, "signal-exit": { "version": "3.0.2", @@ -6604,19 +6761,19 @@ } }, "uglify-js": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz", - "integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==", + "version": "3.6.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.3.tgz", + "integrity": "sha512-KfQUgOqTkLp2aZxrMbCuKCDGW9slFYu2A23A36Gs7sGzTLcRBDORdOi5E21KWHFIfkY8kzgi/Pr1cXCh0yIp5g==", "optional": true, "requires": { - "commander": "~2.20.0", + "commander": "~2.20.3", "source-map": "~0.6.1" }, "dependencies": { "commander": { - "version": "2.20.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz", - "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==", + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "optional": true } } @@ -6637,35 +6794,14 @@ "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=" }, "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } + "set-value": "^2.0.1" } }, "universalify": { @@ -6766,9 +6902,9 @@ } }, "validator": { - "version": "10.8.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-10.8.0.tgz", - "integrity": "sha512-mXqMxfCh5NLsVgYVKl9WvnHNDPCcbNppHSPPowu0VjtSsGWVY+z8hJF44edLR1nbLNzi3jYoYsIl8KZpioIk6g==" + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz", + "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw==" }, "verror": { "version": "1.10.0", @@ -6871,9 +7007,9 @@ } }, "wkx": { - "version": "0.4.5", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.5.tgz", - "integrity": "sha512-01dloEcJZAJabLO5XdcRgqdKpmnxS0zIT02LhkdWOZX2Zs2tPM6hlZ4XG9tWaWur1Qd1OO4kJxUbe2+5BofvnA==", + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", + "integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==", "requires": { "@types/node": "*" } diff --git a/package.json b/package.json index 8532bbb..b15be04 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "migrate-to-graph", - "version": "0.0.2", + "version": "0.2.0", "description": "A tool to migrate existing database to a graph database", "license": "MIT", "repository": "https://github.com/Microsoft/MigrateToGraph", @@ -42,9 +42,10 @@ "commander": "^2.19.0", "convert-hrtime": "^2.0.0", "ejs": "^2.6.1", + "flush-write-stream": "^2.0.0", "fs-extra": "^7.0.0", "gremlin": "^3.4.0", - "handlebars": "^4.1.2", + "handlebars": "^4.4.3", "json-source-map": "^0.4.0", "jsonlint": "^1.6.3", "knex": "^0.16.3", @@ -53,7 +54,7 @@ "parallel-transform": "^1.1.0", "pg": "^7.6.0", "pg-hstore": "^2.3.2", - "sequelize": "^4.41.0", + "sequelize": "^4.44.3", "single-line-log": "^1.1.2", "sqlite3": "^4.0.3", "through2": "^3.0.1", @@ -63,6 +64,7 @@ "@types/async": "^2.0.50", "@types/convert-hrtime": "^2.0.0", "@types/ejs": "^2.6.3", + "@types/flush-write-stream": "^1.0.0", "@types/fs-extra": "^5.0.4", "@types/jest": "^23.3.7", "@types/mssql": "^4.0.8", diff --git a/src/cli.ts b/src/cli.ts index f2eea25..0d70a5e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -6,7 +6,7 @@ import { sqlToGraphCmd, runCmd, } from './index'; -import { streamJSONCmd } from './commands/stream'; +import { streamCmd } from './commands/stream'; program .version('0.0.1') @@ -15,11 +15,11 @@ program console.log('Executing run command'); runCmd(configFile); }); -//Will be added once complete -// program.command('stream ').action((configFile: string) => { -// console.log('Executing run command'); -// streamJSONCmd(configFile); -// }); + +program.command('stream ').action((configFile: string) => { + console.log('Executing stream command'); + streamCmd(configFile); +}); program .command('jsontogremlin ') .action((inputFile: string, templateFile: string, outputFile: string) => { diff --git a/src/commands/run.ts b/src/commands/run.ts index 7d42dbc..45423a9 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -5,13 +5,17 @@ import { getInputConnector, getOutputConnector } from '../loaders/io-loader'; import { Transformer } from '../transformer/transformer'; import * as graphSchema from '../schema/graph-schema.json'; import { GraphInfo } from '../models/graph-model'; +import convertHrtime = require('convert-hrtime'); export function runCmd(configFile: string) { + const timer = process.hrtime(); const config = fs.readJSONSync(configFile); run(config, (err: any) => { if (err) { console.log(err.message); } + const timeTaken = convertHrtime(process.hrtime(timer)).seconds; + console.log(`process completed in ${timeTaken} seconds`); }); } diff --git a/src/commands/stream.ts b/src/commands/stream.ts index 7f45d57..d70677b 100644 --- a/src/commands/stream.ts +++ b/src/commands/stream.ts @@ -1,38 +1,34 @@ -import { jsonStreamReader } from '../stream/reader'; +import { jsonStreamReader, getStreamReader } from '../stream/reader'; import * as through2 from 'through2'; import * as transform from 'parallel-transform'; import { Transformer } from '../transformer/transformer'; import * as schema from '../schema/graph-schema.json'; import * as fs from 'fs-extra'; +import { gremlinCmdStreamWriter, getStreamWriter } from '../stream/writer'; +import { tranformToGraphInfo } from '../stream/transform'; +import convertHrtime = require('convert-hrtime'); -export function streamJSONCmd(configFile: string) { - let config = fs.readJSONSync(configFile); - streamJSON(config, () => console.log('stream ended')); +export function streamCmd(configFile: any) { + const timer = process.hrtime(); + const config = fs.readJSONSync(configFile); + stream(config, (err: Error) => { + if (err) { + console.log(err); + } + const timeTaken = convertHrtime(process.hrtime(timer)).seconds; + console.log(`process completed in ${timeTaken} seconds`); + }); } -export function streamJSON(config: any, callback: any) { - return jsonStreamReader(config.input) +export function stream(config: any, callback: any) { + return getStreamReader(config.input.type, config.input.config) + .pipe(tranformToGraphInfo(config.transform.config)) .pipe( - through2.obj(function(chunck, enc, callback) { - const transformer = new Transformer(config.transform); - transformer.transformInput( - [chunck], - schema, - (err: any, result: any) => { - if (!err) { - this.push(result); - callback(); - } - } - ); + transform(config.output.config.batchSize, (data, cb) => { + cb(null, data); }) ) - .pipe( - transform(config.output.batchSize, (data, callback) => { - setTimeout(() => callback(null, JSON.stringify(data) + '\n\n'), 2000); - }) - ) - .pipe(process.stdout) - .on('error', err => console.error(err)) - .on('end', callback); + .pipe(getStreamWriter(config.output.type, config.output.config)) + .on('error', (err: Error) => console.error(err)) + .on('finish', callback); } diff --git a/src/connectors/gremlin-connector.ts b/src/connectors/gremlin-connector.ts index dbb4943..eb80091 100644 --- a/src/connectors/gremlin-connector.ts +++ b/src/connectors/gremlin-connector.ts @@ -23,9 +23,9 @@ export class GremlinConnector implements OutputConnector { const endpoint = `wss://${config.host}:${config.port}/gremlin`; this.client = new Gremlin.driver.Client(endpoint, { authenticator, - traversalsource: 'g', - rejectUnauthorized: true, mimeType: 'application/vnd.gremlin-v2.0+json', + rejectUnauthorized: true, + traversalsource: 'g', }); this.batchSize = config.batchSize @@ -53,13 +53,6 @@ export class GremlinConnector implements OutputConnector { public addVertices(vertices: Vertex[], callback: any) { const timer = process.hrtime(); this.addToGraph(Etype.vertex, vertices, (err: any) => { - if (!err) { - console.log('\nFinished adding vertices'); - const timeTaken = convertHrtime(process.hrtime(timer)).seconds; - console.log( - `Added ${vertices.length} vertices in ${timeTaken} seconds` - ); - } callback(err); }); } @@ -67,11 +60,6 @@ export class GremlinConnector implements OutputConnector { public addEdges(edges: Edge[], callback: any) { const timer = process.hrtime(); this.addToGraph(Etype.edge, edges, (err: any) => { - if (!err) { - console.log('\nFinished adding edges'); - const timeTaken = convertHrtime(process.hrtime(timer)).seconds; - console.log(`Added ${edges.length} edges in ${timeTaken} seconds`); - } callback(err); }); } @@ -116,16 +104,12 @@ export class GremlinConnector implements OutputConnector { const retryableIterator = this.getRetryable( this.vertexEdgeIterator.bind(this) ); - let completedCnt = 0; async.eachOfLimit( arr as Vertex[] & Edge[], this.batchSize, (value, key, cb) => { retryableIterator(type, value, (err: any) => { - if (!err) { - log(`Added(${type}): ${++completedCnt}/${arr.length}`); - } cb(err); }); }, @@ -140,9 +124,8 @@ export class GremlinConnector implements OutputConnector { async.waterfall( [ - (cb: any) => this.checkExists(type, id, cb), - (res: boolean, cb: any) => { - const command = this.getCommand(type, value, res); + (cb: any) => { + const command = this.getCommand(type, value, this.upsert); async.asyncify(() => this.client.submit(command))(cb); }, ], @@ -169,15 +152,15 @@ export class GremlinConnector implements OutputConnector { private getCommand( type: Etype, value: Vertex | Edge, - update: boolean = false + upsert: boolean = false ): string { if (type === Etype.vertex) { - return update - ? GraphHelper.getUpdateVertexQuery(value as Vertex) + return upsert + ? GraphHelper.getUpsertVertexQuery(value as Vertex) : GraphHelper.getAddVertexQuery(value as Vertex); } else { - return update - ? GraphHelper.getUpdateEdgeQuery(value as Edge) + return upsert && value.id != null + ? GraphHelper.getUpsertEdgeQuery(value as Edge) : GraphHelper.getAddEdgeQuery(value as Edge); } } diff --git a/src/helpers/graphHelper.ts b/src/helpers/graphHelper.ts index a11fece..3e96ac7 100644 --- a/src/helpers/graphHelper.ts +++ b/src/helpers/graphHelper.ts @@ -7,6 +7,16 @@ export function getAddVertexQuery(vertexObj: Vertex): string { return query + getIdQuery(vertexObj) + getPropertiesQuery(vertexObj); } +export function getUpsertVertexQuery(vertexObj: Vertex): string { + const id = escapeSingleQuote(vertexObj.id); + const query = `g.V().has('${ + vertexObj.label + }','id','${id}').fold().coalesce(unfold(),addV('${ + vertexObj.label + }').property('id','${id}'))`; + return query + getPropertiesQuery(vertexObj); +} + export function getUpdateVertexQuery(vertexObj: Vertex): string { const id = escapeSingleQuote(vertexObj.id); const query = `g.V().hasId('${id}')`; @@ -19,6 +29,18 @@ export function getUpdateEdgeQuery(edgeObj: Edge): string { return query + getPropertiesQuery(edgeObj); } +export function getUpsertEdgeQuery(edgeObj: Edge): string { + const id = escapeSingleQuote(edgeObj.id); + const query = `g.E().has('${ + edgeObj.label + }','id','${id}').fold().coalesce(unfold(),g.V().has('id','${ + edgeObj.from + }').addE('${edgeObj.label}').to(g.V().has('id','${ + edgeObj.to + }')).property('id','${id}'))`; + return query + getPropertiesQuery(edgeObj); +} + export function getAddEdgeQuery(edgeObj: Edge): string { const from = escapeSingleQuote(edgeObj.from); const to = escapeSingleQuote(edgeObj.to); diff --git a/src/index.ts b/src/index.ts index 31de509..8b2dded 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,3 +2,4 @@ export * from './commands/jsonToGraph'; export * from './commands/jsonToGremlin'; export * from './commands/sqlToGraph'; export * from './commands/run'; +export * from './commands/stream'; diff --git a/src/schema/graph-schema.json b/src/schema/graph-schema.json index 2f63c9c..b082a8a 100644 --- a/src/schema/graph-schema.json +++ b/src/schema/graph-schema.json @@ -59,21 +59,21 @@ "properties": { "edges": { "items": { - "$ref": "#/definitions/Edge" + "oneOf": [ + {"$ref": "#/definitions/Edge"}, + { "type": "string", "pattern": "^<%(.*)%>$"} + ] }, - "oneOf": [ - { "type": "array"}, - { "type": "string", "pattern": "^<%(.*)%>$"} - ] + "type": "array" }, "vertices": { "items": { - "$ref": "#/definitions/Vertex" + "oneOf": [ + { "$ref": "#/definitions/Vertex"}, + { "type": "string", "pattern": "^<%(.*)%>$"} + ] }, - "oneOf": [ - { "type": "array"}, - { "type": "string", "pattern": "^<%(.*)%>$"} - ] + "type": "array" } }, "required": [ diff --git a/src/stream/reader.ts b/src/stream/reader.ts index 1902e52..4e25d37 100644 --- a/src/stream/reader.ts +++ b/src/stream/reader.ts @@ -3,6 +3,17 @@ import * as fs from 'fs-extra'; import * as JSONStream from 'JSONStream'; import * as stream from 'stream'; +export function getStreamReader(type: string, config: any) { + switch (type) { + case 'sql': + return sqlStreamReader(config); + case 'json': + return jsonStreamReader(config); + default: + throw new Error(`Invalid input type ${type}`); + } +} + export function sqlStreamReader(config: any) { const client = knex({ client: config.dialect, diff --git a/src/stream/transform.ts b/src/stream/transform.ts index e318b82..0fb7880 100644 --- a/src/stream/transform.ts +++ b/src/stream/transform.ts @@ -1,37 +1,41 @@ -import { Transformer } from "../transformer/transformer"; -import * as fs from "fs-extra"; -import * as schema from "../schema/graph-schema.json"; -import * as ejs from "ejs"; -import * as through2 from "through2"; -import * as Ajv from "ajv"; -import { ajvErrorLint } from "../utils/ajvErrorLint"; -import * as jsonlint from "jsonlint"; +import { Transformer } from '../transformer/transformer'; +import * as fs from 'fs-extra'; +import * as schema from '../schema/graph-schema.json'; +import * as ejs from 'ejs'; +import * as through2 from 'through2'; +import * as Ajv from 'ajv'; +import { ajvErrorLint } from '../utils/ajvErrorLint'; +import * as jsonlint from 'jsonlint'; +import { GraphInfo } from '../models/graph-model'; export function tranformToGraphInfo(config: any) { return through2.obj((chunck, enc, callback) => { const renderedTemplate = ejsTransform(config, chunck); - const json = jsonlint.parse(renderedTemplate); - if (validateJSON(json, config.schema)) { - callback(null,json); + const json: GraphInfo = jsonlint.parse(renderedTemplate); + if (validateJSON(json, schema)) { + callback(null, json); } }); } function ejsTransform(config: any, obj: any) { - const template = getTemplate(config); + let template: string = getTemplate(config); + const strt = /"<%/g; + const end = /%>"/g; + template = template.replace(strt, '<%').replace(end, '%>'); const opts: ejs.Options = { - escape: str => JSON.stringify(str) + escape: str => JSON.stringify(str), }; return ejs.render(template, { obj }, opts) as string; } function getTemplate(config: any) { if (config.templatePath) { - return fs.readFileSync(config.templatePath, { encoding: "utf-8" }); + return fs.readFileSync(config.templatePath, { encoding: 'utf-8' }); } else if (config.template) { return config.template; } else { - throw new Error("template not specified in transform config"); + throw new Error('template not specified in transform config'); } } @@ -44,7 +48,7 @@ function validateJSON(json: any, schema: object): boolean { validator.errors![0] as Ajv.ErrorObject, validator.errorsText() ); - throw new Error("Schema validation error: \n" + output); + throw new Error('Schema validation error: \n' + output); } return valid; } diff --git a/src/stream/writer.ts b/src/stream/writer.ts new file mode 100644 index 0000000..b2bc373 --- /dev/null +++ b/src/stream/writer.ts @@ -0,0 +1,52 @@ +import * as writer from 'flush-write-stream'; +import * as fs from 'fs-extra'; +import * as through2 from 'through2'; +import { GraphInfo } from '../models/graph-model'; +import * as GraphHelper from '../helpers/graphHelper'; +import * as Gremlin from 'gremlin'; +import { GremlinConnector } from '../connectors/gremlin-connector'; + +export function getStreamWriter(type: string, config: any) { + switch (type) { + case 'gremlinCmd': + return gremlinCmdStreamWriter(config); + case 'gremlinDb': + return gremlindbStreamWriter(config); + } +} + +export function gremlinCmdStreamWriter(config: any) { + const fileWriteStream = fs.createWriteStream(config.filePath, { + encoding: 'utf-8', + flags: 'w', + }); + return writer.obj( + (chunck: GraphInfo, enc, callback) => { + chunck.vertices.forEach(vertex => { + const cmd = GraphHelper.getAddVertexQuery(vertex); + fileWriteStream.write(cmd + '\n'); + }); + chunck.edges.forEach(edge => { + const cmd = GraphHelper.getAddEdgeQuery(edge); + fileWriteStream.write(cmd + '\n'); + }); + callback(); + }, + callback => { + fileWriteStream.end(callback()); + } + ); +} + +export function gremlindbStreamWriter(config: any) { + const gremlinConnector = new GremlinConnector(config); + return writer.obj( + (chunck: GraphInfo, enc, callback) => { + gremlinConnector.createGraph(chunck, callback); + }, + callback => { + gremlinConnector.closeConnection(); + callback(); + } + ); +} diff --git a/src/test.js b/src/test.js deleted file mode 100644 index 50eb757..0000000 --- a/src/test.js +++ /dev/null @@ -1,12 +0,0 @@ -let ejs = require('ejs'); -let template = require('../input/template.json'); -console.log(template); -template = JSON.stringify(template).replace(new RegExp('"<%','g'),'<%').replace(new RegExp('%>"','g'),'%>'); - - -console.log(ejs.render(template,{obj:{name:'abcy',id:121}},{escape:(str) => { - console.log(str); - return JSON.stringify(str); -}})); - - diff --git a/tslint.json b/tslint.json index 77347c1..68231dd 100644 --- a/tslint.json +++ b/tslint.json @@ -8,7 +8,8 @@ "interface-name":false, "no-shadowed-variable": false, "prefer-conditional-expression": false, - "ordered-imports": false + "ordered-imports": false, + "object-literal-sort-keys":false }, "defaultSeverity": "warn" }