initial commit
This commit is contained in:
abbas
Родитель
3f9ec00d86
Коммит
f8f7d078b6
|
|
@ -0,0 +1,12 @@
|
|||
# http://editorconfig.org
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
|
@ -37,7 +37,7 @@ node_modules/
|
|||
jspm_packages/
|
||||
|
||||
# TypeScript v1 declaration files
|
||||
typings/
|
||||
# typings/
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
|
@ -59,3 +59,6 @@ typings/
|
|||
|
||||
# next.js build output
|
||||
.next
|
||||
|
||||
# generated files
|
||||
lib
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# gitignore
|
||||
|
||||
coverage/
|
||||
node_modules/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.nyc_output
|
||||
lib
|
||||
lib_test
|
||||
input/
|
||||
output/
|
||||
|
||||
# npmignore
|
||||
|
||||
src/
|
||||
__tests__/
|
||||
.vscode/
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
{
|
||||
// Use IntelliSense to learn about possible Node.js debug attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"name": "Debug file",
|
||||
"program": "${workspaceRoot}/lib/cli.js",
|
||||
"args": [],
|
||||
"cwd": "${workspaceRoot}",
|
||||
"sourceMaps": true,
|
||||
"smartStep": true,
|
||||
"preLaunchTask": "build",
|
||||
"outFiles": [
|
||||
"${workspaceRoot}/lib/*.js"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "node",
|
||||
"request": "launch",
|
||||
"name": "Debug test",
|
||||
"program": "${workspaceRoot}/node_modules/jest/bin/jest.js",
|
||||
"args": [
|
||||
"--findRelatedTests",
|
||||
"${relativeFile}"
|
||||
],
|
||||
"cwd": "${workspaceRoot}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"typescript.tsdk": "node_modules/typescript/lib/"
|
||||
}
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
{
|
||||
// See https://go.microsoft.com/fwlink/?LinkId=733558
|
||||
// for the documentation about the tasks.json format
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Run current file",
|
||||
"command": "ts-node ${relativeFile}",
|
||||
"type": "shell",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "clean",
|
||||
"script": "clean",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "build",
|
||||
"script": "build",
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
},
|
||||
"problemMatcher":[]
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "format",
|
||||
"script": "format",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "coverage",
|
||||
"script": "coverage",
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "test",
|
||||
"script": "test",
|
||||
"group": {
|
||||
"kind": "test",
|
||||
"isDefault": true
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"label": "lint",
|
||||
"script": "lint",
|
||||
"problemMatcher": [
|
||||
"$eslint-stylish"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Contributing
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
||||
|
||||
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
222
README.md
222
README.md
|
|
@ -1,14 +1,216 @@
|
|||
# Migrate To Graph
|
||||
This tool allows you to migrate existing database to a graph database
|
||||
|
||||
# Contributing
|
||||
Conversions supported:
|
||||
- json to gremlin
|
||||
- json to graph
|
||||
- sql to graph
|
||||
|
||||
This project welcomes contributions and suggestions. Most contributions require you to agree to a
|
||||
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
|
||||
the rights to use your contribution. For details, visit https://cla.microsoft.com.
|
||||
## Installation (CLI)
|
||||
> npm i -g migrate-to-graph
|
||||
|
||||
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
|
||||
a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
|
||||
provided by the bot. You will only need to do this once across all repos using our CLA.
|
||||
## Usage (CLI)
|
||||
Usage: migrate-to-graph [options] [command]
|
||||
|
||||
This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
|
||||
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
|
||||
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
|
||||
Options:
|
||||
|
||||
-V, --version output the version number
|
||||
-h, --help output usage information
|
||||
|
||||
Commands:
|
||||
|
||||
jsontogremlin <inputFile> <templateFile> <outputFile>
|
||||
jsontograph <inputFile> <templateFile> <graphConfigFile>
|
||||
sqltograph <sqlConfigFile> <query> <templateFile> <graphConfigFile>
|
||||
|
||||
## Installation (Lib)
|
||||
> npm i --save migrate-to-graph
|
||||
|
||||
## Usage (Lib)
|
||||
```js
|
||||
var graphtool = require('migrate-to-graph');
|
||||
var result = graphtool.jsonToGraph(json,template);
|
||||
|
||||
//or ES6
|
||||
import {jsonToGraph} from 'migrate-to-graph';
|
||||
```
|
||||
|
||||
### SQL Config File
|
||||
```json5
|
||||
{
|
||||
"dialect":"mssql", //dialect to use, 'mysql'|'sqlite'|'postgres'|'mssql'
|
||||
"username": "test",
|
||||
"password": "password",
|
||||
"host": "server",
|
||||
"database": "database",
|
||||
"options": {
|
||||
"encrypt": true //set to true if you need encryption
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Graph Config File
|
||||
```json5
|
||||
{
|
||||
"host":"server",
|
||||
"password":"password",
|
||||
"user": "username",
|
||||
"port": "443",
|
||||
"ssl": true,
|
||||
"batchSize": 10, //No. of gremlin queries to execute in parallel (Default: 10)
|
||||
"upsert": false //Set to true if you want to upsert vertices or edges (Default: false)
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
<b>Note: </b>For Azure cosmos graph DB , user is '/dbs/{dbName}/colls/{collectionName}' and password is its secretKey
|
||||
|
||||
## Template
|
||||
To transform data to a graph, you need to transform the data into vertex and edge format.
|
||||
|
||||
Using a template you can convert a single data object into one/many vertexes and edges
|
||||
|
||||
We use handlebars to convert input to vertex / edge format
|
||||
|
||||
### Example
|
||||
|
||||
template:
|
||||
|
||||
```hbs
|
||||
{
|
||||
"vertices":[
|
||||
{
|
||||
"id": "{{myId}}",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "{{myName}}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "{{myFriendId}}",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "{{myFriendName}}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges":[
|
||||
{
|
||||
"label": "friend",
|
||||
"from": "{{myId}}",
|
||||
"to": "{{myFriendId}}",
|
||||
"properties": {
|
||||
"value" : {{friendshipLvl}}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
<b>Note: </b> You can specify as many vertices and edges as you want as long as it transforms to Vertex-Edge format
|
||||
|
||||
Input Data(a single entity from array of data):
|
||||
|
||||
```json
|
||||
{
|
||||
"myId": "1",
|
||||
"myName": "abc",
|
||||
"myFriendId": "2",
|
||||
"myFriendName": "xyz",
|
||||
"friendshipLvl": 3
|
||||
}
|
||||
```
|
||||
|
||||
Transformed Data:
|
||||
|
||||
```json
|
||||
{
|
||||
"vertices":[
|
||||
{
|
||||
"id": "1",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "abc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "xyz"
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges":[
|
||||
{
|
||||
"label": "friend",
|
||||
"from": "1",
|
||||
"to": "2",
|
||||
"properties": {
|
||||
"value" : 3
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Vertex-Edge Format
|
||||
This is a custom format inspired from the way Azure Cosmos Graph DB stores data. We use this format to convert it to gremlin queries so you need to provide a template which transforms to vertex-edge format
|
||||
|
||||
Model for Vertex and Edge
|
||||
```ts
|
||||
export interface Vertex {
|
||||
id: string;
|
||||
label: string; //label for the vertex
|
||||
type: 'vertex';
|
||||
properties: {
|
||||
[key: string]: any; //Represents all the properties you wish to add to the vertex
|
||||
};
|
||||
}
|
||||
|
||||
export interface Edge {
|
||||
id?: string;
|
||||
label: string; //label for the edge
|
||||
type: 'edge';
|
||||
to: string; //id of vertex from which you want the edge to start
|
||||
from: string; //id of vertex to which you want the edge to end
|
||||
properties?: {
|
||||
[key: string]: any; //Represents all the properties you wish to add to the edge
|
||||
};
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
Vertex-Edge Format expects you specify an array of vertices and edges
|
||||
|
||||
```json
|
||||
{
|
||||
"vertices":[
|
||||
{
|
||||
"id": "1",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "abc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "2",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "xyz"
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges":[
|
||||
{
|
||||
"label": "friend",
|
||||
"from": "1",
|
||||
"to": "2",
|
||||
"properties": {
|
||||
"value" : 3
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import { Transformer } from '../../src/transformer/transformer';
|
||||
import * as graphSchema from '../../src/schema/graph-schema.json';
|
||||
|
||||
const template = `{
|
||||
"vertices":[
|
||||
{
|
||||
"id": "{{myId}}",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "{{myName}}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "{{myFriendId}}",
|
||||
"label": "vertexLabel",
|
||||
"properties":{
|
||||
"name": "{{myFriendName}}"
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges":[
|
||||
{
|
||||
"label": "friend",
|
||||
"from": "{{myId}}",
|
||||
"to": "{{myFriendId}}",
|
||||
"properties": {
|
||||
"value" : {{friendshipLvl}}
|
||||
}
|
||||
}
|
||||
]
|
||||
}`;
|
||||
|
||||
describe('Given transformer is initialised', () => {
|
||||
let transformer: Transformer;
|
||||
beforeAll(() => {
|
||||
transformer = new Transformer({});
|
||||
});
|
||||
|
||||
it('when parseTemplate is called it should return the parsed data', () => {
|
||||
const result = transformer.parseTemplate(`{{hello}}`, { hello: 'world' });
|
||||
expect(result).toEqual('world');
|
||||
});
|
||||
|
||||
it('when parseTemplate is called it with guid helper it should return the parsed data', () => {
|
||||
const guidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
const result = transformer.parseTemplate(`{{$guid}}`, {});
|
||||
expect(result).toMatch(guidRegex);
|
||||
});
|
||||
|
||||
it('when transformJSON is called it should transform JSON', () => {
|
||||
const data = [
|
||||
{
|
||||
myId: '1',
|
||||
myName: 'abc',
|
||||
myFriendId: '2',
|
||||
myFriendName: 'xyz',
|
||||
friendshipLvl: 3,
|
||||
},
|
||||
];
|
||||
|
||||
const expectedResult = {
|
||||
vertices: [
|
||||
{
|
||||
id: '1',
|
||||
label: 'vertexLabel',
|
||||
properties: {
|
||||
name: 'abc',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: '2',
|
||||
label: 'vertexLabel',
|
||||
properties: {
|
||||
name: 'xyz',
|
||||
},
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
from: '1',
|
||||
label: 'friend',
|
||||
to: '2',
|
||||
properties: {
|
||||
value: 3,
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = transformer.transformJSON(template, data, graphSchema);
|
||||
|
||||
expect(result).toEqual(expectedResult);
|
||||
});
|
||||
|
||||
it('when validateJSON is called and json does not match the schema, it should throw error', () => {
|
||||
const json = { vertices: [{ test: 'data' }] };
|
||||
expect(() => transformer.validateJSON(json, graphSchema)).toThrow();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"resolveJsonModule": true
|
||||
}
|
||||
}
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -0,0 +1,94 @@
|
|||
{
|
||||
"name": "migrate-to-graph",
|
||||
"version": "0.0.1",
|
||||
"description": "A tool to migrate existing database to a graph database",
|
||||
"license": "MIT",
|
||||
"repository": "https://github.com/Microsoft/MigrateToGraph",
|
||||
"author": {
|
||||
"name": "Microsoft Corporation"
|
||||
},
|
||||
"keywords": [
|
||||
"graph",
|
||||
"gremlin",
|
||||
"database",
|
||||
"migrate",
|
||||
"graph-database",
|
||||
"migration-tool",
|
||||
"cosmosdb"
|
||||
],
|
||||
"files": [
|
||||
"lib"
|
||||
],
|
||||
"main": "lib/index.js",
|
||||
"typings": "lib/index.d.ts",
|
||||
"scripts": {
|
||||
"clean": "rimraf lib && rimraf coverage",
|
||||
"gen-schema": "typescript-json-schema tsconfig.json GraphInfo --required -o src/schema/graph-schema.json",
|
||||
"format": "prettier --write \"{src,__tests__}/**/*.ts\" --single-quote --trailing-comma es5",
|
||||
"lint": "tslint --force --format verbose \"src/**/*.ts\"",
|
||||
"prepublishOnly": "npm run build",
|
||||
"prebuild": "npm run clean && npm run format && npm run lint && echo Using TypeScript && tsc --version",
|
||||
"build": "tsc --pretty",
|
||||
"test": "jest",
|
||||
"coverage": "jest --coverage",
|
||||
"watch": "npm run build -- --watch",
|
||||
"watch:test": "jest --watch",
|
||||
"start": "ts-node ./src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "^6.5.4",
|
||||
"async": "^2.6.1",
|
||||
"commander": "^2.19.0",
|
||||
"convert-hrtime": "^2.0.0",
|
||||
"fs-extra": "^7.0.0",
|
||||
"gremlin": "^2.7.0",
|
||||
"handlebars": "^4.0.12",
|
||||
"json-source-map": "^0.4.0",
|
||||
"jsonlint": "^1.6.3",
|
||||
"mssql": "^4.2.2",
|
||||
"mysql2": "^1.6.1",
|
||||
"pg": "^7.6.0",
|
||||
"pg-hstore": "^2.3.2",
|
||||
"sequelize": "^4.41.0",
|
||||
"single-line-log": "^1.1.2",
|
||||
"sqlite3": "^4.0.3",
|
||||
"uuid": "^3.3.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/async": "^2.0.50",
|
||||
"@types/convert-hrtime": "^2.0.0",
|
||||
"@types/fs-extra": "^5.0.4",
|
||||
"@types/handlebars": "^4.0.38",
|
||||
"@types/jest": "^23.3.7",
|
||||
"@types/mssql": "^4.0.8",
|
||||
"@types/node": "^8.10.36",
|
||||
"@types/sequelize": "^4.27.29",
|
||||
"@types/single-line-log": "^1.1.0",
|
||||
"@types/uuid": "^3.4.3",
|
||||
"coveralls": "^3.0.2",
|
||||
"jest": "^23.6.0",
|
||||
"prettier": "^1.14.3",
|
||||
"rimraf": "^2.0.0",
|
||||
"ts-jest": "^23.6.0",
|
||||
"ts-node": "^7.0.1",
|
||||
"tslint": "^5.0.0",
|
||||
"tslint-config-prettier": "^1.1.0",
|
||||
"typescript": "^3.1.3",
|
||||
"typescript-json-schema": "^0.34.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
},
|
||||
"bin": "./lib/cli.js",
|
||||
"jest": {
|
||||
"transform": {
|
||||
".(ts)": "ts-jest"
|
||||
},
|
||||
"testRegex": "(/__tests__/.*|\\.(test|spec))\\.(ts|js)$",
|
||||
"moduleFileExtensions": [
|
||||
"ts",
|
||||
"js"
|
||||
],
|
||||
"testEnvironment": "node"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
#!/usr/bin/env node --no-warnings
|
||||
import * as program from 'commander';
|
||||
import {
|
||||
jsonToGraphCmd,
|
||||
jsonToGremlinCmd,
|
||||
sqlToGraphCmd,
|
||||
runCmd,
|
||||
} from './index';
|
||||
|
||||
program
|
||||
.version('0.0.1')
|
||||
.command('run <configFile>')
|
||||
.action((configFile: string) => {
|
||||
console.log('Executing run command');
|
||||
runCmd(configFile);
|
||||
});
|
||||
program
|
||||
.command('jsontogremlin <inputFile> <templateFile> <outputFile>')
|
||||
.action((inputFile: string, templateFile: string, outputFile: string) => {
|
||||
console.log('Executing JSON to Gremlin command');
|
||||
jsonToGremlinCmd(inputFile, templateFile, outputFile);
|
||||
});
|
||||
program
|
||||
.command('jsontograph <inputFile> <templateFile> <graphConfigFile>')
|
||||
.action(
|
||||
(inputFile: string, templateFile: string, graphConfigFile: string) => {
|
||||
console.log('Executing JSON to Graph command');
|
||||
jsonToGraphCmd(inputFile, templateFile, graphConfigFile);
|
||||
}
|
||||
);
|
||||
|
||||
program
|
||||
.command(
|
||||
'sqltograph <sqlConfigFile> <query> <templateFile> <graphConfigFile>'
|
||||
)
|
||||
.action(
|
||||
(
|
||||
sqlConfigFile: string,
|
||||
query: string,
|
||||
templateFile: string,
|
||||
graphConfigFile: string
|
||||
) => {
|
||||
console.log('Executing SQL to Graph command');
|
||||
sqlToGraphCmd(sqlConfigFile, query, templateFile, graphConfigFile);
|
||||
}
|
||||
);
|
||||
|
||||
program.on('command:*', () => {
|
||||
console.error(
|
||||
'Invalid command: %s\nSee --help for a list of available commands.',
|
||||
program.args.join(' ')
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
if (process.argv.length < 3) {
|
||||
program.help();
|
||||
}
|
||||
|
||||
program.parse(process.argv);
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import * as fs from 'fs-extra';
|
||||
import { GremlinConnector } from '../connectors/gremlin-connector';
|
||||
import * as graphSchema from '../schema/graph-schema.json';
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
|
||||
export function jsonToGraphCmd(
|
||||
inputFile: string,
|
||||
templateFile: string,
|
||||
graphConfigFile: string
|
||||
) {
|
||||
const inputJSON = fs.readJSONSync(inputFile) as any[];
|
||||
const template = fs.readFileSync(templateFile, { encoding: 'utf-8' });
|
||||
const graphConfig = fs.readJsonSync(graphConfigFile);
|
||||
jsonToGraph(inputJSON, template, graphConfig, err => {
|
||||
if (err) {
|
||||
console.log(err.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function jsonToGraph(
|
||||
inputJSON: any[],
|
||||
template: string,
|
||||
graphConfig: any,
|
||||
callback?: (err: any) => void
|
||||
): void {
|
||||
const transformer = new Transformer({});
|
||||
const result = transformer.transformJSON(template, inputJSON, graphSchema);
|
||||
const connector = new GremlinConnector(graphConfig);
|
||||
|
||||
connector.createGraph(result, (err: any) => {
|
||||
connector.closeConnection();
|
||||
if (callback) {
|
||||
callback(err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
import * as fs from 'fs-extra';
|
||||
import * as GraphHelper from '../helpers/graphHelper';
|
||||
import * as graphSchema from '../schema/graph-schema.json';
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
|
||||
export function jsonToGremlinCmd(
|
||||
inputFile: string,
|
||||
templateFile: string,
|
||||
outputFile: string
|
||||
) {
|
||||
const inputJSON = fs.readJSONSync(inputFile) as any[];
|
||||
const template = fs.readFileSync(templateFile, { encoding: 'utf-8' });
|
||||
try {
|
||||
const graphCmdList: string[] = jsonToGremlin(inputJSON, template);
|
||||
fs.writeFileSync(outputFile, graphCmdList.join('\n'));
|
||||
} catch (err) {
|
||||
console.log(err.message);
|
||||
}
|
||||
}
|
||||
|
||||
export function jsonToGremlin(inputJSON: any[], template: string): string[] {
|
||||
const transformer = new Transformer({});
|
||||
const result = transformer.transformJSON(template, inputJSON, graphSchema);
|
||||
const vertexCmdList: string[] = result.vertices.map(vertex =>
|
||||
GraphHelper.getAddVertexQuery(vertex)
|
||||
);
|
||||
const edgeCmdList: string[] = result.edges.map(edge =>
|
||||
GraphHelper.getAddEdgeQuery(edge)
|
||||
);
|
||||
return vertexCmdList.concat(edgeCmdList);
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import { RunConfig } from '../models/config-model';
|
||||
import * as async from 'async';
|
||||
import * as fs from 'fs-extra';
|
||||
import { getInputConnector, getOutputConnector } from '../loaders/io-loader';
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
import * as graphSchema from '../schema/graph-schema.json';
|
||||
import { GraphInfo } from '../models/graph-model';
|
||||
|
||||
export function runCmd(configFile: string) {
|
||||
const config = fs.readJSONSync(configFile);
|
||||
run(config, (err: any) => {
|
||||
if (err) {
|
||||
console.log(err.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function run(config: RunConfig, callback?: any) {
|
||||
const inputConnector = getInputConnector(
|
||||
config.input.type,
|
||||
config.input.config
|
||||
);
|
||||
const transformer = new Transformer(config.transform);
|
||||
const outputConnector = getOutputConnector(
|
||||
config.output.type,
|
||||
config.output.config
|
||||
);
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
(cb: any) => inputConnector.readInput(cb),
|
||||
(data: any[], cb: any) =>
|
||||
transformer.transformInput(data, graphSchema, cb),
|
||||
(data: GraphInfo, cb: any) => outputConnector.saveOutput(data, cb),
|
||||
],
|
||||
err => {
|
||||
if (callback) {
|
||||
callback(err);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import * as async from 'async';
|
||||
import * as fs from 'fs-extra';
|
||||
import { GremlinConnector } from '../connectors/gremlin-connector';
|
||||
import { SQLInputConnnector } from '../connectors/sql-input-connector';
|
||||
import * as graphSchema from '../schema/graph-schema.json';
|
||||
import { Transformer } from '../transformer/transformer';
|
||||
|
||||
export function sqlToGraphCmd(
|
||||
sqlConfigFile: string,
|
||||
query: string,
|
||||
templateFile: string,
|
||||
graphConfigFile: string
|
||||
) {
|
||||
const sqlConfig = fs.readJSONSync(sqlConfigFile);
|
||||
const template = fs.readFileSync(templateFile, { encoding: 'utf-8' });
|
||||
const graphConfig: any = fs.readJSONSync(graphConfigFile);
|
||||
sqlToGraph(sqlConfig, query, template, graphConfig, err => {
|
||||
if (err) {
|
||||
console.log(err.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function sqlToGraph(
|
||||
sqlConfig: any,
|
||||
query: string,
|
||||
template: string,
|
||||
graphConfig: any,
|
||||
callback?: (err: any) => void
|
||||
) {
|
||||
const sqlConnector = new SQLInputConnnector(sqlConfig);
|
||||
const graphConnector = new GremlinConnector(graphConfig);
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
(cb: any) => {
|
||||
sqlConnector.queryDatabase(query, cb);
|
||||
},
|
||||
(rows: any[], cb: any) => {
|
||||
const transformer = new Transformer({});
|
||||
const result = transformer.transformJSON(template, rows, graphSchema);
|
||||
graphConnector.createGraph(result, cb);
|
||||
},
|
||||
],
|
||||
err => {
|
||||
sqlConnector.closeConnection();
|
||||
graphConnector.closeConnection();
|
||||
if (callback) {
|
||||
callback(err);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
import * as async from 'async';
|
||||
import * as convertHrtime from 'convert-hrtime';
|
||||
import * as Gremlin from 'gremlin';
|
||||
import { stdout as log } from 'single-line-log';
|
||||
import * as GraphHelper from '../helpers/graphHelper';
|
||||
import { Edge, Etype, GraphInfo, Vertex } from '../models/graph-model';
|
||||
import { isNullOrUndefined } from 'util';
|
||||
import { OutputConnector } from '../models/connector-model';
|
||||
|
||||
export class GremlinConnector implements OutputConnector {
|
||||
private client: Gremlin.GremlinClient;
|
||||
private batchSize: number;
|
||||
private upsert: boolean;
|
||||
private retry: number;
|
||||
private defaultRetry = 3;
|
||||
private defaultBatchSize = 10;
|
||||
|
||||
constructor(config: any) {
|
||||
this.client = Gremlin.createClient(config.port, config.host, {
|
||||
password: config.password,
|
||||
session: false,
|
||||
ssl: true,
|
||||
user: config.user,
|
||||
});
|
||||
this.batchSize = config.batchSize
|
||||
? config.batchSize
|
||||
: this.defaultBatchSize;
|
||||
this.upsert = config.upsert;
|
||||
this.retry = config.retry ? config.retry : this.defaultRetry;
|
||||
}
|
||||
|
||||
public createGraph(graphInfo: GraphInfo, callback: any) {
|
||||
async.series(
|
||||
[
|
||||
(cb: any) => {
|
||||
this.addVertices(graphInfo.vertices, cb);
|
||||
},
|
||||
(cb: any) => {
|
||||
this.addEdges(graphInfo.edges, cb);
|
||||
},
|
||||
],
|
||||
err => callback(err)
|
||||
);
|
||||
}
|
||||
|
||||
public addVertices(vertices: Vertex[], callback: any) {
|
||||
const timer = process.hrtime();
|
||||
this.addToGraph(Etype.vertex, vertices, (err: any) => {
|
||||
if (!err) {
|
||||
console.log('\nFinished adding vertices');
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(
|
||||
`Added ${vertices.length} vertices in ${timeTaken} seconds`
|
||||
);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
public addEdges(edges: Edge[], callback: any) {
|
||||
const timer = process.hrtime();
|
||||
this.addToGraph(Etype.edge, edges, (err: any) => {
|
||||
if (!err) {
|
||||
console.log('\nFinished adding edges');
|
||||
const timeTaken = convertHrtime(process.hrtime(timer)).seconds;
|
||||
console.log(`Added ${edges.length} edges in ${timeTaken} seconds`);
|
||||
}
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
public saveOutput(data: GraphInfo, callback: any) {
|
||||
this.createGraph(data, (err: any) => {
|
||||
this.client.closeConnection();
|
||||
callback(err);
|
||||
});
|
||||
}
|
||||
|
||||
public closeConnection() {
|
||||
this.client.closeConnection();
|
||||
}
|
||||
|
||||
public checkExists(type: Etype, id: string, callback: any) {
|
||||
if (!this.upsert || isNullOrUndefined(id)) {
|
||||
callback(null, false);
|
||||
return;
|
||||
}
|
||||
let query: string;
|
||||
if (type === Etype.vertex) {
|
||||
query = `g.V().hasId('${id}').count()`;
|
||||
} else {
|
||||
query = `g.E().hasId('${id}').count()`;
|
||||
}
|
||||
this.client.execute(query, (err, res) => {
|
||||
let exists = false;
|
||||
if (res && res.length > 0 && res[0] > 0) {
|
||||
exists = true;
|
||||
}
|
||||
callback(err, exists);
|
||||
});
|
||||
}
|
||||
|
||||
private addToGraph(type: Etype, arr: Vertex[] | Edge[], callback: any) {
|
||||
const timer = process.hrtime();
|
||||
const retryableIterator = this.getRetryable(
|
||||
this.vertexEdgeIterator.bind(this)
|
||||
);
|
||||
let completedCnt = 0;
|
||||
async.eachOfLimit(
|
||||
arr as Vertex[] & Edge[],
|
||||
|
||||
this.batchSize,
|
||||
(value, key, cb) => {
|
||||
retryableIterator(type, value, (err: any) => {
|
||||
if (!err) {
|
||||
log(`Added(${type}): ${++completedCnt}/${arr.length}`);
|
||||
}
|
||||
cb(err);
|
||||
});
|
||||
},
|
||||
err => {
|
||||
callback(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private vertexEdgeIterator(type: Etype, value: Vertex | Edge, callback: any) {
|
||||
const id = value.properties ? value.properties.id : null;
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
(cb: any) => this.checkExists(type, id, cb),
|
||||
(res: boolean, cb: any) => {
|
||||
const command = this.getCommand(type, value, res);
|
||||
this.client.execute(command, cb);
|
||||
},
|
||||
],
|
||||
callback
|
||||
);
|
||||
}
|
||||
|
||||
private getRetryable(task: any) {
|
||||
if (this.retry > 0) {
|
||||
return async.retryable(
|
||||
{
|
||||
interval: retryCount => {
|
||||
return 500 * Math.pow(2, retryCount);
|
||||
},
|
||||
times: this.retry,
|
||||
},
|
||||
task
|
||||
);
|
||||
} else {
|
||||
return task;
|
||||
}
|
||||
}
|
||||
|
||||
private getCommand(
|
||||
type: Etype,
|
||||
value: Vertex | Edge,
|
||||
update: boolean = false
|
||||
): string {
|
||||
if (type === Etype.vertex) {
|
||||
return update
|
||||
? GraphHelper.getUpdateVertexQuery(value as Vertex)
|
||||
: GraphHelper.getAddVertexQuery(value as Vertex);
|
||||
} else {
|
||||
return update
|
||||
? GraphHelper.getUpdateEdgeQuery(value as Edge)
|
||||
: GraphHelper.getAddEdgeQuery(value as Edge);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
import { OutputConnector } from '../models/connector-model';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as async from 'async';
|
||||
import * as GraphHelper from '../helpers/graphHelper';
|
||||
import { GraphInfo } from '../models/graph-model';
|
||||
|
||||
export class GremlinCmdOutputConnector implements OutputConnector {
|
||||
private config: any;
|
||||
constructor(config: any) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public saveOutput(data: GraphInfo, callback: any): void {
|
||||
if (!this.config || !this.config.fileName) {
|
||||
callback(new Error('fileName not specified in output config'));
|
||||
}
|
||||
const fileName = this.config.fileName;
|
||||
const vertices = data.vertices.map(c => GraphHelper.getAddVertexQuery(c));
|
||||
const edges = data.edges.map(c => GraphHelper.getAddEdgeQuery(c));
|
||||
const commands = vertices.concat(edges);
|
||||
fs.writeFile(
|
||||
fileName,
|
||||
commands.join('\n'),
|
||||
{ encoding: 'utf-8' },
|
||||
callback
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
import { InputConnector } from '../models/connector-model';
|
||||
import * as fs from 'fs-extra';
|
||||
|
||||
export class JSONInputConnector implements InputConnector {
|
||||
private config: any;
|
||||
|
||||
constructor(config: any) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public readInput(callback: any): void {
|
||||
if (this.config.data) {
|
||||
callback(null, this.config.data);
|
||||
} else if (this.config.filePath) {
|
||||
fs.readJSON(this.config.filePath, { encoding: 'utf-8' }, callback);
|
||||
} else {
|
||||
callback(new Error('JSON filePath not provided in config'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import * as async from 'async';
|
||||
import * as Sequelize from 'sequelize';
|
||||
import { InputConnector } from '../models/connector-model';
|
||||
|
||||
export class SQLInputConnnector implements InputConnector {
|
||||
private connection: Sequelize.Sequelize;
|
||||
private query: string;
|
||||
constructor(config: any) {
|
||||
config.options = config.options || {};
|
||||
this.query = config.query;
|
||||
config.options.rowCollectionOnRequestCompletion = true;
|
||||
this.connection = new Sequelize(
|
||||
config.database,
|
||||
config.username,
|
||||
config.password,
|
||||
{
|
||||
dialect: config.dialect,
|
||||
dialectOptions: config.options,
|
||||
host: config.host,
|
||||
operatorsAliases: false,
|
||||
pool: {
|
||||
idle: 10000,
|
||||
max: 5,
|
||||
min: 0,
|
||||
},
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public queryDatabase(query: string, callback: any): void {
|
||||
this.connection
|
||||
.query(query, { raw: false, type: Sequelize.QueryTypes.SELECT })
|
||||
.then(
|
||||
response => {
|
||||
callback(null, response);
|
||||
},
|
||||
error => {
|
||||
callback(error);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
public readInput(callback: any): void {
|
||||
this.queryDatabase(this.query, (...args: any[]) => {
|
||||
this.closeConnection();
|
||||
callback(...args);
|
||||
});
|
||||
}
|
||||
|
||||
public closeConnection() {
|
||||
this.connection.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
import { Edge, Vertex } from '../models/graph-model';
|
||||
import { escapeSingleQuote } from '../utils/safeString';
|
||||
|
||||
export function getAddVertexQuery(vertexObj: Vertex): string {
|
||||
const label = escapeSingleQuote(vertexObj.label);
|
||||
const query = `g.addV('${label}')`;
|
||||
return query + getIdQuery(vertexObj) + getPropertiesQuery(vertexObj);
|
||||
}
|
||||
|
||||
export function getUpdateVertexQuery(vertexObj: Vertex): string {
|
||||
const id = escapeSingleQuote(vertexObj.properties.id);
|
||||
const query = `g.V().hasId('${id}')`;
|
||||
return query + getPropertiesQuery(vertexObj);
|
||||
}
|
||||
|
||||
export function getUpdateEdgeQuery(edgeObj: Edge): string {
|
||||
const id = escapeSingleQuote(edgeObj.properties.id);
|
||||
const query = `g.E().hasId('${id}')`;
|
||||
return query + getPropertiesQuery(edgeObj);
|
||||
}
|
||||
|
||||
export function getAddEdgeQuery(edgeObj: Edge): string {
|
||||
const from = escapeSingleQuote(edgeObj.from);
|
||||
const to = escapeSingleQuote(edgeObj.to);
|
||||
const label = escapeSingleQuote(edgeObj.label);
|
||||
const query =
|
||||
`g.V().has('id','${from}').addE('${label}')` +
|
||||
`.to(g.V().has('id','${to}'))`;
|
||||
|
||||
return query + getIdQuery(edgeObj) + getPropertiesQuery(edgeObj);
|
||||
}
|
||||
|
||||
export function getPropertiesQuery(obj: Vertex | Edge): string {
|
||||
let query = '';
|
||||
if (obj.properties) {
|
||||
for (const key of Object.keys(obj.properties)) {
|
||||
if (key === 'id') {
|
||||
continue;
|
||||
}
|
||||
let value = obj.properties[key];
|
||||
if (typeof value === 'string') {
|
||||
value = escapeSingleQuote(value);
|
||||
value = `'${value}'`;
|
||||
}
|
||||
const safeKey = escapeSingleQuote(key);
|
||||
query += `.property('${safeKey}',${value})`;
|
||||
}
|
||||
}
|
||||
return query;
|
||||
}
|
||||
|
||||
export function removeDuplicateVertexes(vertexes: Vertex[]) {
|
||||
const seen: { [key: string]: boolean } = {};
|
||||
return vertexes.filter(vertex => {
|
||||
return seen.hasOwnProperty(vertex.id) ? false : (seen[vertex.id] = true);
|
||||
});
|
||||
}
|
||||
|
||||
export function removeDuplicateEdges(edges: Edge[]) {
|
||||
const seen: { [key: string]: boolean } = {};
|
||||
return edges.filter(edge => {
|
||||
let edgeId = `${edge.label}-${edge.from}-${edge.to}`;
|
||||
if (edge.id) {
|
||||
edgeId = edge.id;
|
||||
}
|
||||
return seen.hasOwnProperty(edgeId) ? false : (seen[edgeId] = true);
|
||||
});
|
||||
}
|
||||
|
||||
function getIdQuery(obj: Vertex | Edge) {
|
||||
if (!obj.id) {
|
||||
return '';
|
||||
}
|
||||
return `.property('id','${obj.id}')`;
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
export * from './commands/jsonToGraph';
|
||||
export * from './commands/jsonToGremlin';
|
||||
export * from './commands/sqlToGraph';
|
||||
export * from './commands/run';
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
import {
|
||||
InputConnector,
|
||||
InputType,
|
||||
OutputType,
|
||||
} from '../models/connector-model';
|
||||
import { SQLInputConnnector } from '../connectors/sql-input-connector';
|
||||
import { JSONInputConnector } from '../connectors/json-input-connector';
|
||||
import { GremlinConnector } from '../connectors/gremlin-connector';
|
||||
import { GremlinCmdOutputConnector } from '../connectors/gremlinCmd-output-connector';
|
||||
|
||||
export function getInputConnector(
|
||||
type: InputType,
|
||||
config: any
|
||||
): InputConnector {
|
||||
switch (type) {
|
||||
case InputType.sql: {
|
||||
return new SQLInputConnnector(config);
|
||||
}
|
||||
case InputType.json: {
|
||||
return new JSONInputConnector(config);
|
||||
}
|
||||
default: {
|
||||
throw new Error('Invalid input type');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function getOutputConnector(type: OutputType, config: any) {
|
||||
switch (type) {
|
||||
case OutputType.gremlinGraph: {
|
||||
return new GremlinConnector(config);
|
||||
}
|
||||
case OutputType.gremlinCmd: {
|
||||
return new GremlinCmdOutputConnector(config);
|
||||
}
|
||||
default: {
|
||||
throw new Error('Invalid output type');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import { InputType, OutputType } from './connector-model';
|
||||
|
||||
export interface RunConfig {
|
||||
input: RunInputConfig;
|
||||
transform?: RunTransformConfigStringTemplate | RunTransformConfigTemplatePath;
|
||||
output: RunOutputConfig;
|
||||
}
|
||||
|
||||
interface RunInputConfig {
|
||||
type: InputType;
|
||||
config: any;
|
||||
}
|
||||
|
||||
interface RunOutputConfig {
|
||||
type: OutputType;
|
||||
config: any;
|
||||
}
|
||||
|
||||
export interface RunTransformConfigStringTemplate {
|
||||
template: string;
|
||||
}
|
||||
|
||||
export interface RunTransformConfigTemplatePath {
|
||||
templatePath: string;
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
export interface InputConnector {
|
||||
readInput(callback: any): void;
|
||||
}
|
||||
|
||||
export interface OutputConnector {
|
||||
saveOutput(data: any, callback: any): void;
|
||||
}
|
||||
|
||||
export enum InputType {
|
||||
sql = 'sql',
|
||||
json = 'json',
|
||||
}
|
||||
|
||||
export enum OutputType {
|
||||
gremlinGraph = 'gremlinGraph',
|
||||
gremlinCmd = 'gremlinCmd',
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
export interface Vertex {
|
||||
id: string;
|
||||
label: string;
|
||||
properties: {
|
||||
[key: string]: any;
|
||||
};
|
||||
}
|
||||
|
||||
export interface Edge {
|
||||
id?: string;
|
||||
label: string;
|
||||
to: string;
|
||||
from: string;
|
||||
properties?: {
|
||||
[key: string]: any;
|
||||
};
|
||||
}
|
||||
|
||||
export interface GraphInfo {
|
||||
vertices: Vertex[];
|
||||
edges: Edge[];
|
||||
}
|
||||
|
||||
export interface VertexEdgeArray {
|
||||
[index: number]: Vertex | Edge;
|
||||
}
|
||||
|
||||
// tslint:disable-next-line:interface-name
|
||||
export interface GraphCmd {
|
||||
vertexCmd: string[];
|
||||
edgeCmd: string[];
|
||||
}
|
||||
|
||||
export enum Etype {
|
||||
vertex = 'vertex',
|
||||
edge = 'edge',
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"Edge": {
|
||||
"properties": {
|
||||
"from": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"properties": {
|
||||
"additionalProperties": {
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"from",
|
||||
"label",
|
||||
"to"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"Vertex": {
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
"type": "string"
|
||||
},
|
||||
"properties": {
|
||||
"additionalProperties": {
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"id",
|
||||
"label",
|
||||
"properties"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"edges": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/Edge"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"vertices": {
|
||||
"items": {
|
||||
"$ref": "#/definitions/Vertex"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"edges",
|
||||
"vertices"
|
||||
],
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"definitions": {
|
||||
"InputType": {
|
||||
"enum": [
|
||||
"json",
|
||||
"sql"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"OutputType": {
|
||||
"enum": [
|
||||
"gremlinCmd",
|
||||
"gremlinGraph"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"RunInputConfig": {
|
||||
"properties": {
|
||||
"config": {
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/InputType"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"RunOutputConfig": {
|
||||
"properties": {
|
||||
"config": {
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/OutputType"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"RunTransformConfigStringTemplate": {
|
||||
"properties": {
|
||||
"template": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"RunTransformConfigTemplatePath": {
|
||||
"properties": {
|
||||
"templatePath": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"properties": {
|
||||
"input": {
|
||||
"$ref": "#/definitions/RunInputConfig"
|
||||
},
|
||||
"output": {
|
||||
"$ref": "#/definitions/RunOutputConfig"
|
||||
},
|
||||
"transform": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/RunTransformConfigStringTemplate"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/RunTransformConfigTemplatePath"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
import * as Ajv from 'ajv';
|
||||
import * as handlebars from 'handlebars';
|
||||
import * as jsonlint from 'jsonlint';
|
||||
import * as uuid from 'uuid';
|
||||
import * as fs from 'fs-extra';
|
||||
import * as GraphHelper from '../helpers/graphHelper';
|
||||
import { Edge, GraphInfo, Vertex } from '../models/graph-model';
|
||||
import { ajvErrorLint } from '../utils/ajvErrorLint';
|
||||
import {
|
||||
RunTransformConfigTemplatePath,
|
||||
RunTransformConfigStringTemplate,
|
||||
} from '../models/config-model';
|
||||
|
||||
export class Transformer {
|
||||
private validator: Ajv.Ajv;
|
||||
private config: RunTransformConfigStringTemplate &
|
||||
RunTransformConfigTemplatePath;
|
||||
|
||||
constructor(config?: any) {
|
||||
this.config = config;
|
||||
this.registerHelpers();
|
||||
this.validator = new Ajv({ jsonPointers: true });
|
||||
}
|
||||
|
||||
public parseTemplate(template: string, data: object): string {
|
||||
const compiledTemplate = this.getCompiledTemplate(template);
|
||||
return compiledTemplate(data);
|
||||
}
|
||||
|
||||
public transformJSON(
|
||||
template: string,
|
||||
jsonArray: object[],
|
||||
validationSchema?: object
|
||||
): GraphInfo {
|
||||
let vertices: Vertex[] = [];
|
||||
let edges: Edge[] = [];
|
||||
|
||||
const compiledTemplate = this.getCompiledTemplate(template);
|
||||
|
||||
jsonArray.forEach(doc => {
|
||||
const transformedDoc = compiledTemplate(doc);
|
||||
const result: GraphInfo = jsonlint.parse(transformedDoc);
|
||||
if (validationSchema) {
|
||||
this.validateJSON(result, validationSchema);
|
||||
}
|
||||
|
||||
vertices = vertices.concat(result.vertices);
|
||||
edges = edges.concat(result.edges);
|
||||
});
|
||||
|
||||
vertices = GraphHelper.removeDuplicateVertexes(vertices);
|
||||
edges = GraphHelper.removeDuplicateEdges(edges);
|
||||
return { vertices, edges };
|
||||
}
|
||||
|
||||
public transformInput(data: any[], schema: object, callback: any) {
|
||||
const template = this.getTemplateFromConfig();
|
||||
if (template) {
|
||||
const result = this.transformJSON(template, data, schema);
|
||||
callback(null, result);
|
||||
} else {
|
||||
callback(null, data);
|
||||
}
|
||||
}
|
||||
|
||||
public validateJSON(json: any, schema: object) {
|
||||
const valid = this.validator.validate(schema, json);
|
||||
if (!valid) {
|
||||
const output = ajvErrorLint(
|
||||
json,
|
||||
this.validator.errors![0] as Ajv.ErrorObject,
|
||||
this.validator.errorsText()
|
||||
);
|
||||
throw new Error('Schema validation error: \n' + output);
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
|
||||
private getTemplateFromConfig() {
|
||||
let template = '';
|
||||
if (this.config && this.config.template) {
|
||||
template = this.config.template;
|
||||
} else if (this.config && this.config.templatePath) {
|
||||
template = fs.readFileSync(this.config.templatePath, {
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
}
|
||||
return template;
|
||||
}
|
||||
|
||||
private registerHelpers(): void {
|
||||
handlebars.registerHelper('$guid', () => uuid.v4());
|
||||
handlebars.registerHelper('toJSON', object => {
|
||||
return new handlebars.SafeString(JSON.stringify(object));
|
||||
});
|
||||
}
|
||||
|
||||
private getCompiledTemplate(template: string) {
|
||||
return handlebars.compile(template);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
import * as async from 'async';
|
||||
declare module 'async' {
|
||||
export function retryable<T, E = Error>(
|
||||
opts:
|
||||
| number
|
||||
| { times: number; interval: number | ((retryCount: number) => number) },
|
||||
task: AsyncFunction<T, E>
|
||||
): AsyncFunction<T, E>;
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
/* tslint:disable */
|
||||
declare module 'gremlin' {
|
||||
import { EventEmitter } from 'events';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export interface Bindings {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface QueryObject {
|
||||
gremlin: string;
|
||||
bindings: Bindings;
|
||||
}
|
||||
|
||||
export class WebSocketGremlinConnection extends EventEmitter {
|
||||
open: boolean;
|
||||
sendMessage(message: string): void;
|
||||
handleMessage(message: string): void;
|
||||
}
|
||||
|
||||
export class MessageStream extends Readable {}
|
||||
|
||||
export class GremlinClient extends EventEmitter {
|
||||
closeConnection: () => void;
|
||||
port: number;
|
||||
host: string;
|
||||
options: ClientOptions;
|
||||
useSession: boolean;
|
||||
sessionId: string;
|
||||
connected: boolean;
|
||||
connection: WebSocketGremlinConnection;
|
||||
|
||||
execute<T>(
|
||||
script: QueryObject,
|
||||
callback: (err: Error | null, results: T[]) => void
|
||||
): void;
|
||||
execute<T>(
|
||||
script: string,
|
||||
callback: (err: Error | null, results: T[]) => void
|
||||
): void;
|
||||
execute<T>(
|
||||
script: string,
|
||||
bindings: Bindings,
|
||||
callback: (err: Error | null, results: T[]) => void
|
||||
): void;
|
||||
execute<T>(
|
||||
script: string,
|
||||
bindings: Bindings,
|
||||
message: any,
|
||||
callback: (err: Error | null, results: T[]) => void
|
||||
): void;
|
||||
|
||||
messageStream(script: QueryObject): MessageStream;
|
||||
messageStream(script: string): MessageStream;
|
||||
messageStream(script: string, bindings: Bindings): MessageStream;
|
||||
messageStream(
|
||||
script: string,
|
||||
bindings: Bindings,
|
||||
message: any
|
||||
): MessageStream;
|
||||
|
||||
readable(script: QueryObject): Readable;
|
||||
readable(script: string): Readable;
|
||||
readable(script: string, bindings: Bindings): Readable;
|
||||
readable(script: string, bindings: Bindings, message: any): Readable;
|
||||
}
|
||||
|
||||
export interface ClientOptions {
|
||||
/**
|
||||
* Whether to use sessions or not (default: `false`).
|
||||
*/
|
||||
session?: boolean;
|
||||
/**
|
||||
* The script engine to use on the server, see your gremlin-server.yaml file (default: `"gremlin-groovy"`).
|
||||
*/
|
||||
language?: string;
|
||||
/**
|
||||
* The name of the "operation" to execute based on the available OpProcessor (default: `"eval"`).
|
||||
*/
|
||||
op?: string;
|
||||
/**
|
||||
* The name of the OpProcessor to utilize (default: `""`).
|
||||
*/
|
||||
processor?: string;
|
||||
/**
|
||||
* Mime type of returned responses, depending on the serializer (default: `"application/json"`).
|
||||
*/
|
||||
accept?: string;
|
||||
/**
|
||||
* A custom URL connection path if connecting to a Gremlin server behind a WebSocket proxy.
|
||||
*/
|
||||
path?: string;
|
||||
|
||||
ssl?: boolean;
|
||||
|
||||
user?: string;
|
||||
|
||||
password?: string;
|
||||
}
|
||||
|
||||
export function createClient(
|
||||
port: number,
|
||||
host: string,
|
||||
options?: ClientOptions
|
||||
): GremlinClient;
|
||||
export function createClient(
|
||||
port: number,
|
||||
options?: ClientOptions
|
||||
): GremlinClient;
|
||||
export function createClient(options?: ClientOptions): GremlinClient;
|
||||
|
||||
export interface QueryPromise<T> extends Promise<T> {
|
||||
readonly query: string;
|
||||
}
|
||||
|
||||
export interface TemplateTag {
|
||||
<T>(gremlinChunks: TemplateStringsArray, ...values: any[]): QueryPromise<
|
||||
T[]
|
||||
>;
|
||||
}
|
||||
|
||||
export function makeTemplateTag(client: GremlinClient): TemplateTag;
|
||||
|
||||
export interface Functions {
|
||||
[key: string]: (...args: any[]) => QueryObject;
|
||||
}
|
||||
|
||||
export interface BoundFunctions {
|
||||
[key: string]: <T>(...args: any[]) => QueryPromise<T[]>;
|
||||
}
|
||||
|
||||
export function bindForClient(
|
||||
client: GremlinClient,
|
||||
functions: Functions
|
||||
): BoundFunctions;
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
declare module 'json-source-map' {
|
||||
export function stringify(
|
||||
value: any,
|
||||
replacer?: (key: string, value: any) => any,
|
||||
space?: string | number
|
||||
): { json: string; pointers: any };
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
declare module '*/package.json' {
|
||||
const version: string;
|
||||
export { version };
|
||||
}
|
||||
|
||||
declare module '*.json' {
|
||||
const value: any;
|
||||
export default value;
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
declare module 'jsonlint' {
|
||||
export function parse(input: string): any;
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
import { ErrorObject } from 'ajv';
|
||||
import * as jsonSrcMap from 'json-source-map';
|
||||
|
||||
export function ajvErrorLint(
|
||||
data: any,
|
||||
error: ErrorObject,
|
||||
errorText?: string
|
||||
) {
|
||||
const srcMap = jsonSrcMap.stringify(data, undefined, 2);
|
||||
const errorPtr = srcMap.pointers[error.dataPath];
|
||||
const jsonStr = srcMap.json.split('\n');
|
||||
for (let i = errorPtr.value.line; i <= errorPtr.valueEnd.line; i++) {
|
||||
jsonStr[i] = ' * ' + jsonStr[i];
|
||||
}
|
||||
return errorText + '\n' + jsonStr.join('\n');
|
||||
}
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
export function escapeSingleQuote(s: string): string {
|
||||
return s.replace(/'/g, "\\'");
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
import * as convertHrtime from 'convert-hrtime';
|
||||
|
||||
export function timeit(fn: () => void): convertHrtime.HRTime {
|
||||
const start = process.hrtime();
|
||||
fn();
|
||||
return convertHrtime(process.hrtime(process.hrtime()));
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"module": "commonjs",
|
||||
"moduleResolution": "node",
|
||||
"lib": [
|
||||
"esnext"
|
||||
],
|
||||
"target": "es2015",
|
||||
"strict": true,
|
||||
"resolveJsonModule": true,
|
||||
"strictNullChecks": false,
|
||||
"outDir": "./lib",
|
||||
"preserveConstEnums": true,
|
||||
"removeComments": true,
|
||||
"inlineSourceMap": true,
|
||||
"typeRoots": [
|
||||
"./node_modules/@types",
|
||||
"./src/typings"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"src/**/*"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"**/*-spec.ts"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"extends": [
|
||||
"tslint:latest",
|
||||
"tslint-config-prettier"
|
||||
],
|
||||
"rules":{
|
||||
"no-console":false,
|
||||
"interface-name":false,
|
||||
"no-shadowed-variable": false,
|
||||
"prefer-conditional-expression": false,
|
||||
"ordered-imports": false
|
||||
},
|
||||
"defaultSeverity": "warn"
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче