This commit is contained in:
Ken Chau 2020-05-25 13:01:22 -07:00
Родитель 8127e8bb1d
Коммит c68fdd3519
15 изменённых файлов: 4467 добавлений и 105 удалений

105
.gitignore поставляемый
Просмотреть файл

@ -1,104 +1,3 @@
# Logs
logs
node_modules
lib
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# TypeScript v1 declaration files
typings/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
# parcel-bundler cache (https://parceljs.org/)
.cache
# Next.js build output
.next
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and *not* Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port

3
.npmignore Normal file
Просмотреть файл

@ -0,0 +1,3 @@
src
tsconfig.json
yarn.lock

192
CHANGELOG.json Normal file
Просмотреть файл

@ -0,0 +1,192 @@
{
"name": "p-graph",
"entries": [
{
"date": "Tue, 05 May 2020 21:42:11 GMT",
"tag": "p-graph_v0.4.0",
"version": "0.4.0",
"comments": {
"none": [
{
"comment": "adding test before release",
"author": "kchau@microsoft.com",
"commit": "4acba927925474f330d00abcbdfb85b261185bec",
"package": "p-graph"
}
]
}
},
{
"date": "Tue, 05 May 2020 21:41:06 GMT",
"tag": "p-graph_v0.4.0",
"version": "0.4.0",
"comments": {
"patch": [
{
"comment": "fix typings",
"author": "kchau@microsoft.com",
"commit": "8c344d7555cece152271094fb19e0a2d454aab61",
"package": "p-graph"
}
],
"minor": [
{
"comment": "ripped out the dependency on p-queue",
"author": "kchau@microsoft.com",
"commit": "0471bdb2615e368c3d801798a255d603569ddb64",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 20:41:56 GMT",
"tag": "p-graph_v0.3.4",
"version": "0.3.4",
"comments": {
"patch": [
{
"comment": "process the args accurately",
"author": "kchau@microsoft.com",
"commit": "7fe3fabf056ce9f6c7d4ed5d756e86789602a6dd",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 19:12:59 GMT",
"tag": "p-graph_v0.3.3",
"version": "0.3.3",
"comments": {
"patch": [
{
"comment": "this lib is using esm default to publish, so it's a bit different in how to require",
"author": "kchau@microsoft.com",
"commit": "9a97d9f11a3da35ec6bc7f6230e51846d6bf73a8",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 18:26:56 GMT",
"tag": "p-graph_v0.3.2",
"version": "0.3.2",
"comments": {
"patch": [
{
"comment": "one more way of doing pGraph() call in types",
"author": "kchau@microsoft.com",
"commit": "7700bc7f9dea02d7be0dfa1020e5507eaf9d67d4",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 18:25:27 GMT",
"tag": "p-graph_v0.3.1",
"version": "0.3.1",
"comments": {
"patch": [
{
"comment": "one more way of doing pGraph() call in types",
"author": "kchau@microsoft.com",
"commit": "c1da50e3d23c4807ce8f486015ed33cb65f9f69d",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 18:21:34 GMT",
"tag": "p-graph_v0.3.0",
"version": "0.3.0",
"comments": {
"minor": [
{
"comment": "making public api work with 2 args with options",
"author": "kchau@microsoft.com",
"commit": "4198ea4a7d95ba40d423d4d98fba9e73f0f750e5",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 17:55:50 GMT",
"tag": "p-graph_v0.2.4",
"version": "0.2.4",
"comments": {
"patch": [
{
"comment": "removed the section of readme that is known not to work",
"author": "kchau@microsoft.com",
"commit": "e25641e2e34d952eb7ae30af79c3269d340ace23",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 17:39:19 GMT",
"tag": "p-graph_v0.2.3",
"version": "0.2.3",
"comments": {
"patch": [
{
"comment": "fixes the url",
"author": "kchau@microsoft.com",
"commit": "7c7ebfcd0ef6794ff2f6bffc9bfbd3888fe07447",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 17:37:29 GMT",
"tag": "p-graph_v0.2.2",
"version": "0.2.2",
"comments": {
"patch": [
{
"comment": "updates the readme.md and added a url to the package.json",
"author": "kchau@microsoft.com",
"commit": "b23b85b41f9adc3ebf8f6e1f936d25ca25640e89",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 16:30:46 GMT",
"tag": "p-graph_v0.2.1",
"version": "0.2.1",
"comments": {
"patch": [
{
"comment": "added types",
"author": "kchau@microsoft.com",
"commit": "560fba181433eafc55043be0c101613c3dff54fc",
"package": "p-graph"
}
]
}
},
{
"date": "Fri, 01 May 2020 16:15:38 GMT",
"tag": "p-graph_v0.2.0",
"version": "0.2.0",
"comments": {
"minor": [
{
"author": "kchau@microsoft.com",
"commit": "9ee9ae8897acc7213768680b37c1cb3b18234bb6",
"package": "p-graph"
}
]
}
}
]
}

97
CHANGELOG.md Normal file
Просмотреть файл

@ -0,0 +1,97 @@
# Change Log - p-graph
This log was last generated on Tue, 05 May 2020 21:41:06 GMT and should not be manually modified.
<!-- Start content -->
## 0.4.0
Tue, 05 May 2020 21:41:06 GMT
### Minor changes
- ripped out the dependency on p-queue (kchau@microsoft.com)
### Patches
- fix typings (kchau@microsoft.com)
## 0.3.4
Fri, 01 May 2020 20:41:56 GMT
### Patches
- process the args accurately (kchau@microsoft.com)
## 0.3.3
Fri, 01 May 2020 19:12:59 GMT
### Patches
- this lib is using esm default to publish, so it's a bit different in how to require (kchau@microsoft.com)
## 0.3.2
Fri, 01 May 2020 18:26:56 GMT
### Patches
- one more way of doing pGraph() call in types (kchau@microsoft.com)
## 0.3.1
Fri, 01 May 2020 18:25:27 GMT
### Patches
- one more way of doing pGraph() call in types (kchau@microsoft.com)
## 0.3.0
Fri, 01 May 2020 18:21:34 GMT
### Minor changes
- making public api work with 2 args with options (kchau@microsoft.com)
## 0.2.4
Fri, 01 May 2020 17:55:50 GMT
### Patches
- removed the section of readme that is known not to work (kchau@microsoft.com)
## 0.2.3
Fri, 01 May 2020 17:39:19 GMT
### Patches
- fixes the url (kchau@microsoft.com)
## 0.2.2
Fri, 01 May 2020 17:37:29 GMT
### Patches
- updates the readme.md and added a url to the package.json (kchau@microsoft.com)
## 0.2.1
Fri, 01 May 2020 16:30:46 GMT
### Patches
- added types (kchau@microsoft.com)
## 0.2.0
Fri, 01 May 2020 16:15:38 GMT
### Minor changes
- undefined (kchau@microsoft.com)

131
README.md
Просмотреть файл

@ -1,3 +1,134 @@
# p-graph
Run a promise graph with concurrency control.
## Install
```
$ npm install p-graph
```
## Usage
The p-graph library takes in a `graph` argument. To start, create a graph of functions that return promises (let's call them Run Functions), then run them through the pGraph API:
```js
const { default: pGraph } = require("p-graph"); // ES6 import also works: import pGraph from 'p-graph';
const putOnShirt = () => Promise.resolve("put on your shirt");
const putOnShorts = () => Promise.resolve("put on your shorts");
const putOnJacket = () => Promise.resolve("put on your jacket");
const putOnShoes = () => Promise.resolve("put on your shoes");
const tieShoes = () => Promise.resolve("tie your shoes");
const graph = [
[putOnShoes, tieShoes],
[putOnShirt, putOnJacket],
[putOnShorts, putOnJacket],
[putOnShorts, putOnShoes],
];
await pGraph(graph, { concurrency: 3 }).run(); // returns a promise that will resolve when all the tasks are done from this graph in order
```
### Ways to define a graph
1. Use a dependency array
```js
const putOnShirt = () => Promise.resolve("put on your shirt");
const putOnShorts = () => Promise.resolve("put on your shorts");
const putOnJacket = () => Promise.resolve("put on your jacket");
const putOnShoes = () => Promise.resolve("put on your shoes");
const tieShoes = () => Promise.resolve("tie your shoes");
const graph = [
[putOnShoes, tieShoes],
[putOnShirt, putOnJacket],
[putOnShorts, putOnJacket],
[putOnShorts, putOnShoes],
];
await pGraph(graph);
```
2. Use a dependency array with a list of named functions
```js
const funcs = new Map();
funcs.set("putOnShirt", () => Promise.resolve("put on your shirt"));
funcs.set("putOnShorts", () => Promise.resolve("put on your shorts"));
funcs.set("putOnJacket", () => Promise.resolve("put on your jacket"));
funcs.set("putOnShoes", () => Promise.resolve("put on your shoes"));
funcs.set("tieShoes", () => Promise.resolve("tie your shoes"));
const graph = [
[putOnShoes, tieShoes],
[putOnShirt, putOnJacket],
[putOnShorts, putOnJacket],
[putOnShorts, putOnShoes],
];
await pGraph(namedFunctions, graph);
```
3. Use a dependency map with a list of named functions
```js
const funcs = new Map();
funcs.set("putOnShirt", () => Promise.resolve("put on your shirt"));
funcs.set("putOnShorts", () => Promise.resolve("put on your shorts"));
funcs.set("putOnJacket", () => Promise.resolve("put on your jacket"));
funcs.set("putOnShoes", () => Promise.resolve("put on your shoes"));
funcs.set("tieShoes", () => Promise.resolve("tie your shoes"));
const depMap = new Map();
depMap.set(tieShoes, new Set([putOnShoes]));
depMap.set(putOnJacket, new Set([putOnShirt, putOnShorts]));
depMap.set(putOnShoes, new Set([putOnShorts]));
depMap.set(putOnShorts, new Set());
depMap.set(putOnShirt, new Set());
await pGraph(namedFunctions, graph);
```
### Using the ID as argument to the same function
In many cases, the jobs that need to run are the same where the only difference is the arguments for the function. In that case, you can treat the IDs as arguments as they are passed into the Run Function.
```ts
type Id = unknown;
type RunFunction = (id: Id) => Promise<unknown>;
```
As you can see, the ID can be anything. It will be passed as the argument for your Run Function. This is a good option if having a large number of functions inside a graph is prohibitive in memory sensitive situations.
```js
const funcs = new Map();
const thatImportantTask = (id) => Promise.resolve(id);
funcs.set("putOnShirt", thatImportantTask);
funcs.set("putOnShorts", thatImportantTask);
funcs.set("putOnJacket", thatImportantTask);
funcs.set("putOnShoes", thatImportantTask);
funcs.set("tieShoes", thatImportantTask);
```
## Scopes and filtering
After a graph are sent to the `pGraph` function, the graph is executed with the `run()` function. The `run()` takes in an argument that lets you filter which tasks to end. This allows you to run tasks up to a certain point in the graph.
```js
// graph is one of the three options up top
// depMap is the dependency map where the key is the ID for the Run Function
// - the ID CAN be the Run Function itself if graph is specified as the dependency array format
await pGraph(graph).run((depMap) => {
return [...depMap.keys()].filter((id) => id.startsWith("b"));
});
```
# Contributing

5
jest.config.js Normal file
Просмотреть файл

@ -0,0 +1,5 @@
module.exports = {
preset: "ts-jest",
modulePathIgnorePatterns: ["<rootDir>/lib"],
testEnvironment: "node",
};

25
package.json Normal file
Просмотреть файл

@ -0,0 +1,25 @@
{
"name": "p-graph",
"version": "0.4.0",
"license": "MIT",
"main": "lib/index.js",
"types": "lib/index.d.ts",
"scripts": {
"build": "tsc",
"start": "tsc -w --preserveWatchOutput",
"test": "jest",
"release": "yarn build && yarn test && beachball publish",
"check": "beachball check",
"change": "beachball change"
},
"repository": {
"url": "https://github.com/kenotron/p-graph"
},
"devDependencies": {
"@types/jest": "^25.2.1",
"beachball": "^1.31.0",
"jest": "^25.5.3",
"ts-jest": "^25.4.0",
"typescript": "^3.8.3"
}
}

49
src/PGraph.ts Normal file
Просмотреть файл

@ -0,0 +1,49 @@
import { NamedFunctions, DepGraphMap, ScopeFunction, Id } from "./types";
export class PGraph {
private promises: Map<Id, Promise<unknown>> = new Map();
namedFunctions: NamedFunctions;
graph: DepGraphMap;
constructor(namedFunctions, graph: DepGraphMap) {
this.namedFunctions = namedFunctions;
this.graph = graph;
this.promises = new Map();
}
/**
* Runs the promise graph with scoping
* @param scope
*/
run(scope?: ScopeFunction) {
const scopedPromises = scope
? scope(this.graph).map((id) => this.execute(id))
: [...this.graph.keys()].map((id) => this.execute(id));
return Promise.all(scopedPromises);
}
private execute(id: Id) {
if (this.promises.has(id)) {
return this.promises.get(id);
}
let execPromise: Promise<unknown> = Promise.resolve();
const deps = this.graph.get(id);
if (deps) {
execPromise = execPromise.then(() =>
Promise.all([...deps].map((depId) => this.execute(depId)))
);
}
execPromise = execPromise.then(() => this.namedFunctions.get(id)(id));
this.promises.set(id, execPromise);
return execPromise;
}
}

Просмотреть файл

@ -0,0 +1,42 @@
import { PGraph } from "../PGraph";
import { NamedFunctions, DepGraphMap } from "../types";
describe("PGraph", () => {
it("should allow a full graph to be created", async () => {
const fns: NamedFunctions = new Map();
const graph: DepGraphMap = new Map();
const mockFn = jest.fn((id) => Promise.resolve());
fns.set("fn1", mockFn);
fns.set("fn2", mockFn);
graph.set("fn1", new Set(["fn2"]));
const pGraph = new PGraph(fns, graph);
await pGraph.run();
expect(mockFn).toHaveBeenCalledTimes(2);
expect(mockFn).toHaveBeenNthCalledWith(1, "fn2");
expect(mockFn).toHaveBeenNthCalledWith(2, "fn1");
});
it("should throw when one of the promises threw", async () => {
const fns: NamedFunctions = new Map();
const graph: DepGraphMap = new Map();
const mockFn = jest.fn((id) => Promise.resolve());
const failFn = jest.fn((id) => {
throw new Error("expected failure");
});
fns.set("fn1", mockFn);
fns.set("fn2", mockFn);
fns.set("fail", failFn);
graph.set("fn1", new Set(["fn2", "fail"]));
const pGraph = new PGraph(fns, graph);
await pGraph.run();
});
});

Просмотреть файл

@ -0,0 +1,78 @@
import pGraph from "../index";
import { DepGraphArray } from "../types";
describe("Public API", () => {
let calls = [];
// Example graph from: https://www.npmjs.com/package/toposort
const putOnShirt = () =>
Promise.resolve("put on your shirt").then((v) => {
calls.push(v);
});
const putOnShorts = () =>
Promise.resolve("put on your shorts").then((v) => {
calls.push(v);
});
const putOnJacket = () =>
Promise.resolve("put on your jacket").then((v) => {
calls.push(v);
});
const putOnShoes = () =>
Promise.resolve("put on your shoes").then((v) => {
calls.push(v);
});
const tieShoes = () =>
Promise.resolve("tie your shoes").then((v) => {
calls.push(v);
});
beforeEach(() => {
calls = [];
});
it("should accept an array dep graph", async () => {
const graph: DepGraphArray = [
[putOnShoes, tieShoes],
[putOnShirt, putOnJacket],
[putOnShorts, putOnJacket],
[putOnShorts, putOnShoes],
];
await pGraph(graph).run();
expect(calls).toEqual([
"put on your shirt",
"put on your shorts",
"put on your jacket",
"put on your shoes",
"tie your shoes",
]);
});
it("should accept an array dep graph", async () => {
const graph: DepGraphArray = [
[putOnShoes, tieShoes],
[putOnShirt, putOnJacket],
[putOnShorts, putOnJacket],
[putOnShorts, putOnShoes],
];
await pGraph(graph).run();
expect(calls.indexOf("tie your shoes")).toBeGreaterThan(
calls.indexOf("put on your shoes")
);
expect(calls.indexOf("put on your jacket")).toBeGreaterThan(
calls.indexOf("put on your shirt")
);
expect(calls.indexOf("put on your jacket")).toBeGreaterThan(
calls.indexOf("put on your shorts")
);
expect(calls.indexOf("put on your shoes")).toBeGreaterThan(
calls.indexOf("put on your shorts")
);
});
});

31
src/depConverters.ts Normal file
Просмотреть файл

@ -0,0 +1,31 @@
import {
DepGraphArray,
NamedFunctions,
DepGraphMap,
RunFunction,
} from "./types";
export function depArrayToNamedFunctions(array: DepGraphArray) {
const namedFunctions: NamedFunctions = new Map();
// dependant depends on subject (Child depends on Parent means Child is dependent, Parent is subject)
for (const [subject, dependent] of array) {
namedFunctions.set(subject, subject as RunFunction);
namedFunctions.set(dependent, dependent as RunFunction);
}
return namedFunctions;
}
export function depArrayToMap(array: DepGraphArray) {
const graph: DepGraphMap = new Map();
// dependant depends on subject (Child depends on Parent means Child is dependent, Parent is subject)
for (const [subjectId, dependentId] of array) {
if (!graph.has(dependentId)) {
graph.set(dependentId, new Set([subjectId]));
} else {
graph.get(dependentId).add(subjectId);
}
}
return graph;
}

47
src/index.ts Normal file
Просмотреть файл

@ -0,0 +1,47 @@
import { DepGraphArray, NamedFunctions, DepGraphMap } from "./types";
import { PGraph } from "./PGraph";
import { depArrayToNamedFunctions, depArrayToMap } from "./depConverters";
function pGraph(namedFunctions: NamedFunctions, graph: DepGraphMap);
function pGraph(namedFunctions: NamedFunctions, graph: DepGraphArray);
function pGraph(graph: DepGraphArray);
function pGraph(...args: any[]) {
if (args.length < 1 || args.length > 2) {
throw new Error("Incorrect number of arguments");
}
let namedFunctions: NamedFunctions;
let graph: DepGraphMap;
if (args.length === 1) {
if (!Array.isArray(args[0])) {
throw new Error(
"Unexpected graph definition format. Expecting graph in the form of [()=>Promise, ()=>Promise][]"
);
}
const depArray = args[0] as DepGraphArray;
namedFunctions = depArrayToNamedFunctions(depArray);
graph = depArrayToMap(depArray);
} else if (args.length === 2) {
if (Array.isArray(args[0])) {
const depArray = args[0] as DepGraphArray;
namedFunctions = depArrayToNamedFunctions(depArray);
graph = depArrayToMap(depArray);
} else if (args[0] instanceof Map && Array.isArray(args[1])) {
const depArray = args[1] as DepGraphArray;
namedFunctions = args[0];
graph = depArrayToMap(depArray);
} else if (args[0] instanceof Map && args[1] instanceof Map) {
namedFunctions = args[0];
graph = args[1];
} else {
throw new Error("Unexpected arguments");
}
}
return new PGraph(namedFunctions, graph);
}
export default pGraph;

6
src/types.ts Normal file
Просмотреть файл

@ -0,0 +1,6 @@
export type RunFunction = (id: Id) => Promise<unknown>;
export type Id = string | number | RunFunction;
export type NamedFunctions = Map<Id, RunFunction>;
export type DepGraphMap = Map<Id, Set<Id>>;
export type ScopeFunction = (graph: DepGraphMap) => Id[];
export type DepGraphArray = [Id, Id][];

13
tsconfig.json Normal file
Просмотреть файл

@ -0,0 +1,13 @@
{
"compilerOptions": {
"target": "ES2017",
"declaration": true,
"module": "CommonJS",
"moduleResolution": "node",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"lib": ["ES2017"],
"outDir": "lib"
},
"include": ["src"]
}

3744
yarn.lock Normal file

Разница между файлами не показана из-за своего большого размера Загрузить разницу