Feature/stream support (#2)
* added gremlincmd stream writer * fixed streaming logic * fixed upsert query in gremlinDb output * Fixed vulnerability issues and fixed schema * removed comment
This commit is contained in:
Abbas Cyclewala
GitHub
Родитель
385a7ef870
Коммит
19e90667d6
|
|
@ -66,3 +66,4 @@ lib
|
|||
# test files
|
||||
input/
|
||||
output/
|
||||
.vscode/launch.json
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "migrate-to-graph",
|
||||
"version": "0.0.2",
|
||||
"version": "0.2.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
|
|
@ -267,9 +267,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"@types/bluebird": {
|
||||
"version": "3.5.24",
|
||||
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.24.tgz",
|
||||
"integrity": "sha512-YeQoDpq4Lm8ppSBqAnAeF/xy1cYp/dMTif2JFcvmAbETMRlvKHT2iLcWu+WyYiJO3b3Ivokwo7EQca/xfLVJmg==",
|
||||
"version": "3.5.27",
|
||||
"resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.27.tgz",
|
||||
"integrity": "sha512-6BmYWSBea18+tSjjSC3QIyV93ZKAeNWGM7R6aYt1ryTZXrlHF+QLV0G2yV0viEGVyRkyQsWfMoJ0k/YghBX5sQ==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/continuation-local-storage": {
|
||||
|
|
@ -299,6 +299,15 @@
|
|||
"integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/flush-write-stream": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/flush-write-stream/-/flush-write-stream-1.0.0.tgz",
|
||||
"integrity": "sha512-vABEdzyrFxWkHeoLnaxc9NILYA7BWlY2BHQifHDVSRUZbWVuPSB22nzygWeWmjA9en2OzTx6waY7YPs+m9yzmg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/fs-extra": {
|
||||
"version": "5.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.4.tgz",
|
||||
|
|
@ -320,9 +329,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"@types/lodash": {
|
||||
"version": "4.14.117",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.117.tgz",
|
||||
"integrity": "sha512-xyf2m6tRbz8qQKcxYZa7PA4SllYcay+eh25DN3jmNYY6gSTL7Htc/bttVdkqj2wfJGbeWlQiX8pIyJpKU+tubw==",
|
||||
"version": "4.14.134",
|
||||
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.134.tgz",
|
||||
"integrity": "sha512-2/O0khFUCFeDlbi7sZ7ZFRCcT812fAeOLm7Ev4KbwASkZ575TDrDcY7YyaoHdTOzKcNbfiwLYZqPmoC4wadrsw==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/mssql": {
|
||||
|
|
@ -350,9 +359,9 @@
|
|||
}
|
||||
},
|
||||
"@types/sequelize": {
|
||||
"version": "4.27.29",
|
||||
"resolved": "https://registry.npmjs.org/@types/sequelize/-/sequelize-4.27.29.tgz",
|
||||
"integrity": "sha512-VER50QSlqcjioIlmr2b4SeKWC8CLAVMkju9qnxScqu+5laPsPNS9+58HAmnkUUWsm8LFUDqH9oVJkdtS19h2zQ==",
|
||||
"version": "4.28.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/sequelize/-/sequelize-4.28.3.tgz",
|
||||
"integrity": "sha512-Z+gbWsLmGnqnAz91I8Wtfbe+4D1aUyQ4aawYfohpvar4Azr8aSNwzS1rFQYO2dk6zcClt5k9n0qCek/YAq4WNw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/bluebird": "*",
|
||||
|
|
@ -386,9 +395,9 @@
|
|||
}
|
||||
},
|
||||
"@types/validator": {
|
||||
"version": "9.4.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/validator/-/validator-9.4.2.tgz",
|
||||
"integrity": "sha512-v6H2QH+oXVdLKp9keOJi5LQSt6X5/XIOtK1YmbCzvkAT2kHW9WyQkixit9w1UgJpBGrDCqqCZlQ+Qucpmsf8hA==",
|
||||
"version": "10.11.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/validator/-/validator-10.11.1.tgz",
|
||||
"integrity": "sha512-bVhLqvb+5xUNWRFnuuecRVISTvsG6AdhrB2kb/tChgtuTTqARqlQ3rLhOPy8cINZEUB8PkR+goyWF6fWxg4iSw==",
|
||||
"dev": true
|
||||
},
|
||||
"JSONStream": {
|
||||
|
|
@ -511,6 +520,15 @@
|
|||
"normalize-path": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"append-transform": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz",
|
||||
"integrity": "sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"default-require-extensions": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
|
||||
|
|
@ -830,9 +848,9 @@
|
|||
}
|
||||
},
|
||||
"bluebird": {
|
||||
"version": "3.5.2",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz",
|
||||
"integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg=="
|
||||
"version": "3.7.1",
|
||||
"resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.1.tgz",
|
||||
"integrity": "sha512-DdmyoGCleJnkbp3nkbxTLJ18rjDsE4yCggEwKNXkeV123sPNfOCYeDoeuOY+F2FrSjO1YXcTU+dsy96KMy+gcg=="
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
|
|
@ -1124,6 +1142,12 @@
|
|||
"resolved": "https://registry.npmjs.org/commander/-/commander-2.19.0.tgz",
|
||||
"integrity": "sha512-6tvAOO+D6OENvRAh524Dh9jcfKTYDQAqvqezbCW82xj5X0pSrcpxtvRKHLG0yBY6SD7PSDrJaj+0AiOcKVd1Xg=="
|
||||
},
|
||||
"compare-versions": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.4.0.tgz",
|
||||
"integrity": "sha512-tK69D7oNXXqUW3ZNo/z7NXTEz22TCF0pTE+YF9cxvaAM9XnkLo1fV621xCLrRR6aevJlKxExkss0vWqUCUpqdg==",
|
||||
"dev": true
|
||||
},
|
||||
"component-emitter": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
|
||||
|
|
@ -1290,6 +1314,15 @@
|
|||
"integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
|
||||
"dev": true
|
||||
},
|
||||
"default-require-extensions": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz",
|
||||
"integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"strip-bom": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"define-properties": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
|
||||
|
|
@ -1713,6 +1746,16 @@
|
|||
"bser": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"fileset": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/fileset/-/fileset-2.0.3.tgz",
|
||||
"integrity": "sha1-jnVIqW08wjJ+5eZ0FocjozO7oqA=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"glob": "^7.0.3",
|
||||
"minimatch": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"fill-range": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
|
||||
|
|
@ -1771,6 +1814,27 @@
|
|||
"resolved": "https://registry.npmjs.org/flagged-respawn/-/flagged-respawn-1.0.1.tgz",
|
||||
"integrity": "sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q=="
|
||||
},
|
||||
"flush-write-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-uXClqPxT4xW0lcdSBheb2ObVU+kuqUk3Jk64EwieirEXZx9XUrVwp/JuBfKAWaM4T5Td/VL7QLDWPXp/MvGm/g==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.1.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"readable-stream": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
|
||||
"integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"for-in": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
|
||||
|
|
@ -1850,7 +1914,8 @@
|
|||
"ansi-regex": {
|
||||
"version": "2.1.1",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"aproba": {
|
||||
"version": "1.2.0",
|
||||
|
|
@ -1871,12 +1936,14 @@
|
|||
"balanced-match": {
|
||||
"version": "1.0.0",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
|
|
@ -1891,17 +1958,20 @@
|
|||
"code-point-at": {
|
||||
"version": "1.1.0",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"console-control-strings": {
|
||||
"version": "1.1.0",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"core-util-is": {
|
||||
"version": "1.0.2",
|
||||
|
|
@ -2018,7 +2088,8 @@
|
|||
"inherits": {
|
||||
"version": "2.0.3",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"ini": {
|
||||
"version": "1.3.5",
|
||||
|
|
@ -2030,6 +2101,7 @@
|
|||
"version": "1.0.0",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"number-is-nan": "^1.0.0"
|
||||
}
|
||||
|
|
@ -2044,6 +2116,7 @@
|
|||
"version": "3.0.4",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
|
|
@ -2051,12 +2124,14 @@
|
|||
"minimist": {
|
||||
"version": "0.0.8",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.3.5",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.0"
|
||||
|
|
@ -2075,6 +2150,7 @@
|
|||
"version": "0.5.1",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"minimist": "0.0.8"
|
||||
}
|
||||
|
|
@ -2155,7 +2231,8 @@
|
|||
"number-is-nan": {
|
||||
"version": "1.0.1",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"object-assign": {
|
||||
"version": "4.1.1",
|
||||
|
|
@ -2167,6 +2244,7 @@
|
|||
"version": "1.4.0",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
|
|
@ -2252,7 +2330,8 @@
|
|||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"safer-buffer": {
|
||||
"version": "2.1.2",
|
||||
|
|
@ -2288,6 +2367,7 @@
|
|||
"version": "1.0.2",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"code-point-at": "^1.0.0",
|
||||
"is-fullwidth-code-point": "^1.0.0",
|
||||
|
|
@ -2307,6 +2387,7 @@
|
|||
"version": "3.0.1",
|
||||
"bundled": true,
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"ansi-regex": "^2.0.0"
|
||||
}
|
||||
|
|
@ -2350,12 +2431,14 @@
|
|||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.0.3",
|
||||
"bundled": true,
|
||||
"dev": true
|
||||
"dev": true,
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -2399,9 +2482,9 @@
|
|||
}
|
||||
},
|
||||
"generic-pool": {
|
||||
"version": "3.4.2",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.4.2.tgz",
|
||||
"integrity": "sha512-H7cUpwCQSiJmAHM4c/aFu6fUfrhWXW1ncyh8ftxEPMu6AiYkHw9K8br720TGPZJbk5eOH2bynjZD1yPvdDAmag=="
|
||||
"version": "3.5.0",
|
||||
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.5.0.tgz",
|
||||
"integrity": "sha512-dEkxmX+egB2o4NR80c/q+xzLLzLX+k68/K8xv81XprD+Sk7ZtP14VugeCz+fUwv5FzpWq40pPtAkzPRqT8ka9w=="
|
||||
},
|
||||
"get-caller-file": {
|
||||
"version": "1.0.3",
|
||||
|
|
@ -2498,9 +2581,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"handlebars": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
|
||||
"integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
|
||||
"version": "4.4.3",
|
||||
"resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.3.tgz",
|
||||
"integrity": "sha512-B0W4A2U1ww3q7VVthTKfh+epHx+q4mCt6iK+zEAzbMBpWQAwxCeKxEGpj/1oQTpzPXDNSOG7hmG14TsISH50yw==",
|
||||
"requires": {
|
||||
"neo-async": "^2.6.0",
|
||||
"optimist": "^0.6.1",
|
||||
|
|
@ -2962,13 +3045,19 @@
|
|||
},
|
||||
"istanbul-api": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-api/-/istanbul-api-2.1.1.tgz",
|
||||
"integrity": "sha512-kVmYrehiwyeBAk/wE71tW6emzLiHGjYIiDrc8sfyty4F8M02/lrgXSm+R1kXysmF20zArvmZXjlE/mg24TVPJw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"async": "^2.6.1",
|
||||
"compare-versions": "^3.2.1",
|
||||
"fileset": "^2.0.3",
|
||||
"istanbul-lib-coverage": "^2.0.3",
|
||||
"istanbul-lib-hook": "^2.0.3",
|
||||
"istanbul-lib-instrument": "^3.1.0",
|
||||
"istanbul-lib-report": "^2.0.4",
|
||||
"istanbul-lib-source-maps": "^3.0.2",
|
||||
"istanbul-reports": "^2.1.1",
|
||||
"js-yaml": "^3.12.0",
|
||||
"make-dir": "^1.3.0",
|
||||
"minimatch": "^3.0.4",
|
||||
|
|
@ -2981,6 +3070,15 @@
|
|||
"integrity": "sha512-dKWuzRGCs4G+67VfW9pBFFz2Jpi4vSp/k7zBcJ888ofV5Mi1g5CUML5GvMvV6u9Cjybftu+E8Cgp+k0dI1E5lw==",
|
||||
"dev": true
|
||||
},
|
||||
"istanbul-lib-hook": {
|
||||
"version": "2.0.7",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz",
|
||||
"integrity": "sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"append-transform": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"istanbul-lib-instrument": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.1.0.tgz",
|
||||
|
|
@ -2996,6 +3094,56 @@
|
|||
"semver": "^5.5.0"
|
||||
}
|
||||
},
|
||||
"istanbul-lib-report": {
|
||||
"version": "2.0.8",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz",
|
||||
"integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"istanbul-lib-coverage": "^2.0.5",
|
||||
"make-dir": "^2.1.0",
|
||||
"supports-color": "^6.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"istanbul-lib-coverage": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz",
|
||||
"integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==",
|
||||
"dev": true
|
||||
},
|
||||
"make-dir": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz",
|
||||
"integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"pify": "^4.0.1",
|
||||
"semver": "^5.6.0"
|
||||
}
|
||||
},
|
||||
"pify": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
|
||||
"integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==",
|
||||
"dev": true
|
||||
},
|
||||
"semver": {
|
||||
"version": "5.7.0",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
|
||||
"integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==",
|
||||
"dev": true
|
||||
},
|
||||
"supports-color": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz",
|
||||
"integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"has-flag": "^3.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"istanbul-lib-source-maps": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.2.tgz",
|
||||
|
|
@ -3026,6 +3174,15 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"istanbul-reports": {
|
||||
"version": "2.2.6",
|
||||
"resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.6.tgz",
|
||||
"integrity": "sha512-SKi4rnMyLBKe0Jy2uUdx28h8oG7ph2PPuQPvIAh31d+Ci+lSiEu4C+h3oBPuJ9+mPKhOyW0M8gY4U5NM1WLeXA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"handlebars": "^4.1.2"
|
||||
}
|
||||
},
|
||||
"jest": {
|
||||
"version": "24.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jest/-/jest-24.1.0.tgz",
|
||||
|
|
@ -4250,9 +4407,9 @@
|
|||
}
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.11",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
|
||||
"integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
|
||||
"version": "4.17.15",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
|
||||
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
|
||||
},
|
||||
"lodash.sortby": {
|
||||
"version": "4.7.0",
|
||||
|
|
@ -4417,7 +4574,7 @@
|
|||
},
|
||||
"minimist": {
|
||||
"version": "0.0.10",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
|
||||
"resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz",
|
||||
"integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8="
|
||||
},
|
||||
"minipass": {
|
||||
|
|
@ -4445,9 +4602,9 @@
|
|||
}
|
||||
},
|
||||
"mixin-deep": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
|
||||
"integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
|
||||
"version": "1.3.2",
|
||||
"resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
|
||||
"integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
|
||||
"requires": {
|
||||
"for-in": "^1.0.2",
|
||||
"is-extendable": "^1.0.1"
|
||||
|
|
@ -4479,14 +4636,14 @@
|
|||
}
|
||||
},
|
||||
"moment": {
|
||||
"version": "2.22.2",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.22.2.tgz",
|
||||
"integrity": "sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y="
|
||||
"version": "2.24.0",
|
||||
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
|
||||
"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
|
||||
},
|
||||
"moment-timezone": {
|
||||
"version": "0.5.21",
|
||||
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.21.tgz",
|
||||
"integrity": "sha512-j96bAh4otsgj3lKydm3K7kdtA3iKf2m6MY2iSYCzCm5a1zmHo1g+aK3068dDEeocLZQIS9kU8bsdQHLqEvgW0A==",
|
||||
"version": "0.5.27",
|
||||
"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.27.tgz",
|
||||
"integrity": "sha512-EIKQs7h5sAsjhPCqN6ggx6cEbs94GK050254TIJySD1bzoM5JTYDwAU1IoVOeTOL6Gm27kYJ51/uuvq1kIlrbw==",
|
||||
"requires": {
|
||||
"moment": ">= 2.9.0"
|
||||
}
|
||||
|
|
@ -5562,16 +5719,16 @@
|
|||
"integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4="
|
||||
},
|
||||
"sequelize": {
|
||||
"version": "4.41.0",
|
||||
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.41.0.tgz",
|
||||
"integrity": "sha512-6zCYxjPriUBgLQ5dN4ZxC5eadUtRu2tHFoBiSU9vNGb6G3f4bZM6vFpJu0DhxeeyhodTAXqY1GaBpZzT3PXB/Q==",
|
||||
"version": "4.44.3",
|
||||
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-4.44.3.tgz",
|
||||
"integrity": "sha512-r2A4EVDKRCcABcZhY4ItvbcosvMJKpQMooxg/S8ouRFrZzqMPQ9O2thOUfgW59q8ZcEa5ccNeqwg15MCciqPMg==",
|
||||
"requires": {
|
||||
"bluebird": "^3.5.0",
|
||||
"cls-bluebird": "^2.1.0",
|
||||
"debug": "^3.1.0",
|
||||
"depd": "^1.1.0",
|
||||
"dottie": "^2.0.0",
|
||||
"generic-pool": "^3.4.0",
|
||||
"generic-pool": "3.5.0",
|
||||
"inflection": "1.12.0",
|
||||
"lodash": "^4.17.1",
|
||||
"moment": "^2.20.0",
|
||||
|
|
@ -5594,9 +5751,9 @@
|
|||
}
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
|
||||
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -5606,9 +5763,9 @@
|
|||
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
|
||||
},
|
||||
"set-value": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
|
||||
"integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
|
||||
"integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
|
||||
"requires": {
|
||||
"extend-shallow": "^2.0.1",
|
||||
"is-extendable": "^0.1.1",
|
||||
|
|
@ -5648,9 +5805,9 @@
|
|||
"dev": true
|
||||
},
|
||||
"shimmer": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.0.tgz",
|
||||
"integrity": "sha512-xTCx2vohXC2EWWDqY/zb4+5Mu28D+HYNSOuFzsyRDRvI/e1ICb69afwaUwfjr+25ZXldbOLyp+iDUZHq8UnTag=="
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz",
|
||||
"integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw=="
|
||||
},
|
||||
"signal-exit": {
|
||||
"version": "3.0.2",
|
||||
|
|
@ -6604,19 +6761,19 @@
|
|||
}
|
||||
},
|
||||
"uglify-js": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.0.tgz",
|
||||
"integrity": "sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==",
|
||||
"version": "3.6.3",
|
||||
"resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.6.3.tgz",
|
||||
"integrity": "sha512-KfQUgOqTkLp2aZxrMbCuKCDGW9slFYu2A23A36Gs7sGzTLcRBDORdOi5E21KWHFIfkY8kzgi/Pr1cXCh0yIp5g==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"commander": "~2.20.0",
|
||||
"commander": "~2.20.3",
|
||||
"source-map": "~0.6.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"commander": {
|
||||
"version": "2.20.0",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
|
||||
"integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
|
||||
"version": "2.20.3",
|
||||
"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
|
||||
"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
|
|
@ -6637,35 +6794,14 @@
|
|||
"integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag="
|
||||
},
|
||||
"union-value": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
|
||||
"integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
|
||||
"integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
|
||||
"requires": {
|
||||
"arr-union": "^3.1.0",
|
||||
"get-value": "^2.0.6",
|
||||
"is-extendable": "^0.1.1",
|
||||
"set-value": "^0.4.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"extend-shallow": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
|
||||
"integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
|
||||
"requires": {
|
||||
"is-extendable": "^0.1.0"
|
||||
}
|
||||
},
|
||||
"set-value": {
|
||||
"version": "0.4.3",
|
||||
"resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
|
||||
"integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
|
||||
"requires": {
|
||||
"extend-shallow": "^2.0.1",
|
||||
"is-extendable": "^0.1.1",
|
||||
"is-plain-object": "^2.0.1",
|
||||
"to-object-path": "^0.3.0"
|
||||
}
|
||||
}
|
||||
"set-value": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"universalify": {
|
||||
|
|
@ -6766,9 +6902,9 @@
|
|||
}
|
||||
},
|
||||
"validator": {
|
||||
"version": "10.8.0",
|
||||
"resolved": "https://registry.npmjs.org/validator/-/validator-10.8.0.tgz",
|
||||
"integrity": "sha512-mXqMxfCh5NLsVgYVKl9WvnHNDPCcbNppHSPPowu0VjtSsGWVY+z8hJF44edLR1nbLNzi3jYoYsIl8KZpioIk6g=="
|
||||
"version": "10.11.0",
|
||||
"resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
|
||||
"integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
|
||||
},
|
||||
"verror": {
|
||||
"version": "1.10.0",
|
||||
|
|
@ -6871,9 +7007,9 @@
|
|||
}
|
||||
},
|
||||
"wkx": {
|
||||
"version": "0.4.5",
|
||||
"resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.5.tgz",
|
||||
"integrity": "sha512-01dloEcJZAJabLO5XdcRgqdKpmnxS0zIT02LhkdWOZX2Zs2tPM6hlZ4XG9tWaWur1Qd1OO4kJxUbe2+5BofvnA==",
|
||||
"version": "0.4.8",
|
||||
"resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",
|
||||
"integrity": "sha512-ikPXMM9IR/gy/LwiOSqWlSL3X/J5uk9EO2hHNRXS41eTLXaUFEVw9fn/593jW/tE5tedNg8YjT5HkCa4FqQZyQ==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "migrate-to-graph",
|
||||
"version": "0.0.2",
|
||||
"version": "0.2.0",
|
||||
"description": "A tool to migrate existing database to a graph database",
|
||||
"license": "MIT",
|
||||
"repository": "https://github.com/Microsoft/MigrateToGraph",
|
||||
|
|
@ -42,9 +42,10 @@
|
|||
"commander": "^2.19.0",
|
||||
"convert-hrtime": "^2.0.0",
|
||||
"ejs": "^2.6.1",
|
||||
"flush-write-stream": "^2.0.0",
|
||||
"fs-extra": "^7.0.0",
|
||||
"gremlin": "^3.4.0",
|
||||
"handlebars": "^4.1.2",
|
||||
"handlebars": "^4.4.3",
|
||||
"json-source-map": "^0.4.0",
|
||||
"jsonlint": "^1.6.3",
|
||||
"knex": "^0.16.3",
|
||||
|
|
@ -53,7 +54,7 @@
|
|||
"parallel-transform": "^1.1.0",
|
||||
"pg": "^7.6.0",
|
||||
"pg-hstore": "^2.3.2",
|
||||
"sequelize": "^4.41.0",
|
||||
"sequelize": "^4.44.3",
|
||||
"single-line-log": "^1.1.2",
|
||||
"sqlite3": "^4.0.3",
|
||||
"through2": "^3.0.1",
|
||||
|
|
@ -63,6 +64,7 @@
|
|||
"@types/async": "^2.0.50",
|
||||
"@types/convert-hrtime": "^2.0.0",
|
||||
"@types/ejs": "^2.6.3",
|
||||
"@types/flush-write-stream": "^1.0.0",
|
||||
"@types/fs-extra": "^5.0.4",
|
||||
"@types/jest": "^23.3.7",
|
||||
"@types/mssql": "^4.0.8",
|
||||
|
|
|
|||
12
src/cli.ts
12
src/cli.ts
|
|
@ -6,7 +6,7 @@ import {
|
|||
sqlToGraphCmd,
|
||||
runCmd,
|
||||
} from './index';
|
||||
import { streamJSONCmd } from './commands/stream';
|
||||
import { streamCmd } from './commands/stream';
|
||||
|
||||
program
|
||||
.version('0.0.1')
|
||||
|
|
@ -15,11 +15,11 @@ program
|
|||
console.log('Executing run command');
|
||||
runCmd(configFile);
|
||||
});
|
||||
//Will be added once complete
|
||||
// program.command('stream <configFile>').action((configFile: string) => {
|
||||
// console.log('Executing run command');
|
||||
// streamJSONCmd(configFile);
|
||||
// });
|
||||
|
||||
program.command('stream <configFile>').action((configFile: string) => {
|
||||
console.log('Executing stream command');
|
||||
streamCmd(configFile);
|
||||
});
|
||||
program
|
||||
.command('jsontogremlin <inputFile> <templateFile> <outputFile>')
|
||||
.action((inputFile: string, templateFile: string, outputFile: string) => {
|
||||
|
|
|
|||
|
|
@ -5,13 +5,17 @@ import { getInputConnector, getOutputConnector } from '../loaders/io-loader';
|
|||
import { Transformer } from '../transformer/transformer';
|
||||
import * as graphSchema from '../schema/graph-schema.json';
|
||||
import { GraphInfo } from '../models/graph-model';
|
||||
import convertHrtime = require('convert-hrtime');
|
||||
|
||||
export function runCmd(configFile: string) {
|
||||
const timer = process.hrtime();
|
||||
const config = fs.readJSONSync(configFile);
|
||||
run(config, (err: any) => {
|
||||
if (err) {
|
||||
console.log(err.message);
|
||||
}
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(`process completed in ${timeTaken} seconds`);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +1,34 @@
|
|||
import { jsonStreamReader } from '../stream/reader';
|
||||
import { jsonStreamReader, getStreamReader } from '../stream/reader';
|
||||
import * as through2 from 'through2';
|
||||
import * as transform from 'parallel-transform';
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
import * as schema from '../schema/graph-schema.json';
|
||||
import * as fs from 'fs-extra';
|
||||
import { gremlinCmdStreamWriter, getStreamWriter } from '../stream/writer';
|
||||
import { tranformToGraphInfo } from '../stream/transform';
|
||||
import convertHrtime = require('convert-hrtime');
|
||||
|
||||
export function streamJSONCmd(configFile: string) {
|
||||
let config = fs.readJSONSync(configFile);
|
||||
streamJSON(config, () => console.log('stream ended'));
|
||||
export function streamCmd(configFile: any) {
|
||||
const timer = process.hrtime();
|
||||
const config = fs.readJSONSync(configFile);
|
||||
stream(config, (err: Error) => {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
}
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(`process completed in ${timeTaken} seconds`);
|
||||
});
|
||||
}
|
||||
|
||||
export function streamJSON(config: any, callback: any) {
|
||||
return jsonStreamReader(config.input)
|
||||
export function stream(config: any, callback: any) {
|
||||
return getStreamReader(config.input.type, config.input.config)
|
||||
.pipe(tranformToGraphInfo(config.transform.config))
|
||||
.pipe(
|
||||
through2.obj(function(chunck, enc, callback) {
|
||||
const transformer = new Transformer(config.transform);
|
||||
transformer.transformInput(
|
||||
[chunck],
|
||||
schema,
|
||||
(err: any, result: any) => {
|
||||
if (!err) {
|
||||
this.push(result);
|
||||
callback();
|
||||
}
|
||||
}
|
||||
);
|
||||
transform(config.output.config.batchSize, (data, cb) => {
|
||||
cb(null, data);
|
||||
})
|
||||
)
|
||||
.pipe(
|
||||
transform(config.output.batchSize, (data, callback) => {
|
||||
setTimeout(() => callback(null, JSON.stringify(data) + '\n\n'), 2000);
|
||||
})
|
||||
)
|
||||
.pipe(process.stdout)
|
||||
.on('error', err => console.error(err))
|
||||
.on('end', callback);
|
||||
.pipe(getStreamWriter(config.output.type, config.output.config))
|
||||
.on('error', (err: Error) => console.error(err))
|
||||
.on('finish', callback);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,9 +23,9 @@ export class GremlinConnector implements OutputConnector {
|
|||
const endpoint = `wss://${config.host}:${config.port}/gremlin`;
|
||||
this.client = new Gremlin.driver.Client(endpoint, {
|
||||
authenticator,
|
||||
traversalsource: 'g',
|
||||
rejectUnauthorized: true,
|
||||
mimeType: 'application/vnd.gremlin-v2.0+json',
|
||||
rejectUnauthorized: true,
|
||||
traversalsource: 'g',
|
||||
});
|
||||
|
||||
this.batchSize = config.batchSize
|
||||
|
|
@ -53,13 +53,6 @@ export class GremlinConnector implements OutputConnector {
|
|||
public addVertices(vertices: Vertex[], callback: any) {
|
||||
const timer = process.hrtime();
|
||||
this.addToGraph(Etype.vertex, vertices, (err: any) => {
|
||||
if (!err) {
|
||||
console.log('\nFinished adding vertices');
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(
|
||||
`Added ${vertices.length} vertices in ${timeTaken} seconds`
|
||||
);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
|
@ -67,11 +60,6 @@ export class GremlinConnector implements OutputConnector {
|
|||
public addEdges(edges: Edge[], callback: any) {
|
||||
const timer = process.hrtime();
|
||||
this.addToGraph(Etype.edge, edges, (err: any) => {
|
||||
if (!err) {
|
||||
console.log('\nFinished adding edges');
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(`Added ${edges.length} edges in ${timeTaken} seconds`);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
|
@ -116,16 +104,12 @@ export class GremlinConnector implements OutputConnector {
|
|||
const retryableIterator = this.getRetryable(
|
||||
this.vertexEdgeIterator.bind(this)
|
||||
);
|
||||
let completedCnt = 0;
|
||||
async.eachOfLimit(
|
||||
arr as Vertex[] & Edge[],
|
||||
|
||||
this.batchSize,
|
||||
(value, key, cb) => {
|
||||
retryableIterator(type, value, (err: any) => {
|
||||
if (!err) {
|
||||
log(`Added(${type}): ${++completedCnt}/${arr.length}`);
|
||||
}
|
||||
cb(err);
|
||||
});
|
||||
},
|
||||
|
|
@ -140,9 +124,8 @@ export class GremlinConnector implements OutputConnector {
|
|||
|
||||
async.waterfall(
|
||||
[
|
||||
(cb: any) => this.checkExists(type, id, cb),
|
||||
(res: boolean, cb: any) => {
|
||||
const command = this.getCommand(type, value, res);
|
||||
(cb: any) => {
|
||||
const command = this.getCommand(type, value, this.upsert);
|
||||
async.asyncify(() => this.client.submit(command))(cb);
|
||||
},
|
||||
],
|
||||
|
|
@ -169,15 +152,15 @@ export class GremlinConnector implements OutputConnector {
|
|||
private getCommand(
|
||||
type: Etype,
|
||||
value: Vertex | Edge,
|
||||
update: boolean = false
|
||||
upsert: boolean = false
|
||||
): string {
|
||||
if (type === Etype.vertex) {
|
||||
return update
|
||||
? GraphHelper.getUpdateVertexQuery(value as Vertex)
|
||||
return upsert
|
||||
? GraphHelper.getUpsertVertexQuery(value as Vertex)
|
||||
: GraphHelper.getAddVertexQuery(value as Vertex);
|
||||
} else {
|
||||
return update
|
||||
? GraphHelper.getUpdateEdgeQuery(value as Edge)
|
||||
return upsert && value.id != null
|
||||
? GraphHelper.getUpsertEdgeQuery(value as Edge)
|
||||
: GraphHelper.getAddEdgeQuery(value as Edge);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,16 @@ export function getAddVertexQuery(vertexObj: Vertex): string {
|
|||
return query + getIdQuery(vertexObj) + getPropertiesQuery(vertexObj);
|
||||
}
|
||||
|
||||
export function getUpsertVertexQuery(vertexObj: Vertex): string {
|
||||
const id = escapeSingleQuote(vertexObj.id);
|
||||
const query = `g.V().has('${
|
||||
vertexObj.label
|
||||
}','id','${id}').fold().coalesce(unfold(),addV('${
|
||||
vertexObj.label
|
||||
}').property('id','${id}'))`;
|
||||
return query + getPropertiesQuery(vertexObj);
|
||||
}
|
||||
|
||||
export function getUpdateVertexQuery(vertexObj: Vertex): string {
|
||||
const id = escapeSingleQuote(vertexObj.id);
|
||||
const query = `g.V().hasId('${id}')`;
|
||||
|
|
@ -19,6 +29,18 @@ export function getUpdateEdgeQuery(edgeObj: Edge): string {
|
|||
return query + getPropertiesQuery(edgeObj);
|
||||
}
|
||||
|
||||
export function getUpsertEdgeQuery(edgeObj: Edge): string {
|
||||
const id = escapeSingleQuote(edgeObj.id);
|
||||
const query = `g.E().has('${
|
||||
edgeObj.label
|
||||
}','id','${id}').fold().coalesce(unfold(),g.V().has('id','${
|
||||
edgeObj.from
|
||||
}').addE('${edgeObj.label}').to(g.V().has('id','${
|
||||
edgeObj.to
|
||||
}')).property('id','${id}'))`;
|
||||
return query + getPropertiesQuery(edgeObj);
|
||||
}
|
||||
|
||||
export function getAddEdgeQuery(edgeObj: Edge): string {
|
||||
const from = escapeSingleQuote(edgeObj.from);
|
||||
const to = escapeSingleQuote(edgeObj.to);
|
||||
|
|
|
|||
|
|
@ -2,3 +2,4 @@ export * from './commands/jsonToGraph';
|
|||
export * from './commands/jsonToGremlin';
|
||||
export * from './commands/sqlToGraph';
|
||||
export * from './commands/run';
|
||||
export * from './commands/stream';
|
||||
|
|
|
|||
|
|
@ -59,21 +59,21 @@
|
|||
"properties": {
|
||||
"edges": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/Edge"
|
||||
"oneOf": [
|
||||
{"$ref": "#/definitions/Edge"},
|
||||
{ "type": "string", "pattern": "^<%(.*)%>$"}
|
||||
]
|
||||
},
|
||||
"oneOf": [
|
||||
{ "type": "array"},
|
||||
{ "type": "string", "pattern": "^<%(.*)%>$"}
|
||||
]
|
||||
"type": "array"
|
||||
},
|
||||
"vertices": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/Vertex"
|
||||
"oneOf": [
|
||||
{ "$ref": "#/definitions/Vertex"},
|
||||
{ "type": "string", "pattern": "^<%(.*)%>$"}
|
||||
]
|
||||
},
|
||||
"oneOf": [
|
||||
{ "type": "array"},
|
||||
{ "type": "string", "pattern": "^<%(.*)%>$"}
|
||||
]
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
|
|||
|
|
@ -3,6 +3,17 @@ import * as fs from 'fs-extra';
|
|||
import * as JSONStream from 'JSONStream';
|
||||
import * as stream from 'stream';
|
||||
|
||||
export function getStreamReader(type: string, config: any) {
|
||||
switch (type) {
|
||||
case 'sql':
|
||||
return sqlStreamReader(config);
|
||||
case 'json':
|
||||
return jsonStreamReader(config);
|
||||
default:
|
||||
throw new Error(`Invalid input type ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function sqlStreamReader(config: any) {
|
||||
const client = knex({
|
||||
client: config.dialect,
|
||||
|
|
|
|||
|
|
@ -1,37 +1,41 @@
|
|||
import { Transformer } from "../transformer/transformer";
|
||||
import * as fs from "fs-extra";
|
||||
import * as schema from "../schema/graph-schema.json";
|
||||
import * as ejs from "ejs";
|
||||
import * as through2 from "through2";
|
||||
import * as Ajv from "ajv";
|
||||
import { ajvErrorLint } from "../utils/ajvErrorLint";
|
||||
import * as jsonlint from "jsonlint";
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as schema from '../schema/graph-schema.json';
|
||||
import * as ejs from 'ejs';
|
||||
import * as through2 from 'through2';
|
||||
import * as Ajv from 'ajv';
|
||||
import { ajvErrorLint } from '../utils/ajvErrorLint';
|
||||
import * as jsonlint from 'jsonlint';
|
||||
import { GraphInfo } from '../models/graph-model';
|
||||
|
||||
export function tranformToGraphInfo(config: any) {
|
||||
return through2.obj((chunck, enc, callback) => {
|
||||
const renderedTemplate = ejsTransform(config, chunck);
|
||||
const json = jsonlint.parse(renderedTemplate);
|
||||
if (validateJSON(json, config.schema)) {
|
||||
callback(null,json);
|
||||
const json: GraphInfo = jsonlint.parse(renderedTemplate);
|
||||
if (validateJSON(json, schema)) {
|
||||
callback(null, json);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function ejsTransform(config: any, obj: any) {
|
||||
const template = getTemplate(config);
|
||||
let template: string = getTemplate(config);
|
||||
const strt = /"<%/g;
|
||||
const end = /%>"/g;
|
||||
template = template.replace(strt, '<%').replace(end, '%>');
|
||||
const opts: ejs.Options = {
|
||||
escape: str => JSON.stringify(str)
|
||||
escape: str => JSON.stringify(str),
|
||||
};
|
||||
return ejs.render(template, { obj }, opts) as string;
|
||||
}
|
||||
|
||||
function getTemplate(config: any) {
|
||||
if (config.templatePath) {
|
||||
return fs.readFileSync(config.templatePath, { encoding: "utf-8" });
|
||||
return fs.readFileSync(config.templatePath, { encoding: 'utf-8' });
|
||||
} else if (config.template) {
|
||||
return config.template;
|
||||
} else {
|
||||
throw new Error("template not specified in transform config");
|
||||
throw new Error('template not specified in transform config');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -44,7 +48,7 @@ function validateJSON(json: any, schema: object): boolean {
|
|||
validator.errors![0] as Ajv.ErrorObject,
|
||||
validator.errorsText()
|
||||
);
|
||||
throw new Error("Schema validation error: \n" + output);
|
||||
throw new Error('Schema validation error: \n' + output);
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import * as writer from 'flush-write-stream';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as through2 from 'through2';
|
||||
import { GraphInfo } from '../models/graph-model';
|
||||
import * as GraphHelper from '../helpers/graphHelper';
|
||||
import * as Gremlin from 'gremlin';
|
||||
import { GremlinConnector } from '../connectors/gremlin-connector';
|
||||
|
||||
export function getStreamWriter(type: string, config: any) {
|
||||
switch (type) {
|
||||
case 'gremlinCmd':
|
||||
return gremlinCmdStreamWriter(config);
|
||||
case 'gremlinDb':
|
||||
return gremlindbStreamWriter(config);
|
||||
}
|
||||
}
|
||||
|
||||
export function gremlinCmdStreamWriter(config: any) {
|
||||
const fileWriteStream = fs.createWriteStream(config.filePath, {
|
||||
encoding: 'utf-8',
|
||||
flags: 'w',
|
||||
});
|
||||
return writer.obj(
|
||||
(chunck: GraphInfo, enc, callback) => {
|
||||
chunck.vertices.forEach(vertex => {
|
||||
const cmd = GraphHelper.getAddVertexQuery(vertex);
|
||||
fileWriteStream.write(cmd + '\n');
|
||||
});
|
||||
chunck.edges.forEach(edge => {
|
||||
const cmd = GraphHelper.getAddEdgeQuery(edge);
|
||||
fileWriteStream.write(cmd + '\n');
|
||||
});
|
||||
callback();
|
||||
},
|
||||
callback => {
|
||||
fileWriteStream.end(callback());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
export function gremlindbStreamWriter(config: any) {
|
||||
const gremlinConnector = new GremlinConnector(config);
|
||||
return writer.obj(
|
||||
(chunck: GraphInfo, enc, callback) => {
|
||||
gremlinConnector.createGraph(chunck, callback);
|
||||
},
|
||||
callback => {
|
||||
gremlinConnector.closeConnection();
|
||||
callback();
|
||||
}
|
||||
);
|
||||
}
|
||||
12
src/test.js
12
src/test.js
|
|
@ -1,12 +0,0 @@
|
|||
let ejs = require('ejs');
|
||||
let template = require('../input/template.json');
|
||||
console.log(template);
|
||||
template = JSON.stringify(template).replace(new RegExp('"<%','g'),'<%').replace(new RegExp('%>"','g'),'%>');
|
||||
|
||||
|
||||
console.log(ejs.render(template,{obj:{name:'abcy',id:121}},{escape:(str) => {
|
||||
console.log(str);
|
||||
return JSON.stringify(str);
|
||||
}}));
|
||||
|
||||
|
||||
|
|
@ -8,7 +8,8 @@
|
|||
"interface-name":false,
|
||||
"no-shadowed-variable": false,
|
||||
"prefer-conditional-expression": false,
|
||||
"ordered-imports": false
|
||||
"ordered-imports": false,
|
||||
"object-literal-sort-keys":false
|
||||
},
|
||||
"defaultSeverity": "warn"
|
||||
}
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче