Non-blocking save asset metadata. (#910)

* Non-blocking save asset metadata.

* Implement queue map.

* Add withQueueMap decorator.
This commit is contained in:
SimoTw 2021-04-09 11:56:47 +08:00 коммит произвёл GitHub
Родитель 592e84496e
Коммит ffeb09de1d
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
9 изменённых файлов: 409 добавлений и 7 удалений

Просмотреть файл

@ -57,7 +57,7 @@
"compile": "tsc",
"build": "node ./scripts/dump_git_info.js && react-scripts build",
"react-start": "node ./scripts/dump_git_info.js && react-scripts start",
"test": "react-scripts test --env=jsdom --silent",
"test": "react-scripts test --env=jsdom",
"eject": "react-scripts eject",
"webpack:dev": "webpack --config ./config/webpack.dev.js",
"webpack:prod": "webpack --config ./config/webpack.prod.js",
@ -108,6 +108,7 @@
"enzyme-adapter-react-16": "^1.15.1",
"eslint-utils": "^1.4.3",
"foreman": "^3.0.1",
"jest-enzyme": "^7.1.2",
"kind-of": "^6.0.3",
"mime": "^2.4.6",
"minimist": "^1.2.2",

Просмотреть файл

@ -0,0 +1,16 @@
export type Args = any[];
export interface IQueue {
queue: Args[];
isLooping: boolean;
promise?: Promise<void>;
}
export class Queue implements IQueue {
queue: Args[];
isLooping: boolean;
constructor() {
this.queue = [];
this.isLooping = false;
}
}

Просмотреть файл

@ -0,0 +1,114 @@
import QueueMap from "./queueMap";
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
describe("QueueMap", () => {
test("dequeueUntilLast", () => {
const queueMap = new QueueMap();
const queueId = "1";
const a = ["a", 1];
const b = ["b", 2];
queueMap.enque(queueId, a);
queueMap.enque(queueId, b);
queueMap.dequeueUntilLast(queueId);
const { queue } = queueMap.getQueueById(queueId);
expect([b]).toEqual(queue);
})
test("call enque while looping items in the queue", async () => {
const queueMap = new QueueMap();
const mockWrite = jest.fn();
const queueId = "1";
const sleepThenReturn = ms => async (...params) => {
await mockWrite(...params);
await sleep(ms);
}
const a = ["a", 1];
const b = ["b", 2];
const c = ["c", 3];
const d = ["d", 4];
const expected = [b, d]
queueMap.enque(queueId, a);
queueMap.enque(queueId, b);
queueMap.on(queueId, sleepThenReturn(1000), params => params);
queueMap.enque(queueId, c);
queueMap.enque(queueId, d);
await sleep(2000);
expect(mockWrite.mock.calls.length).toBe(2);
expect([mockWrite.mock.calls[0], mockWrite.mock.calls[1]]).toEqual(expected);
})
test("prevent call on twice.", async () => {
const queueMap = new QueueMap();
const queueId = "1";
const mockWrite = jest.fn();
const sleepThenReturn = ms => async (...params) => {
await mockWrite(...params);
await sleep(ms);
}
const a = ["a", 1];
const b = ["b", 2];
const c = ["c", 3];
const d = ["d", 4];
const expected = [b, d]
queueMap.enque(queueId, a);
queueMap.enque(queueId, b);
queueMap.on(queueId, sleepThenReturn(1000), params => params);
queueMap.enque(queueId, c);
queueMap.on(queueId, sleepThenReturn(1000), params => params);
queueMap.enque(queueId, d);
await sleep(2000);
expect(mockWrite.mock.calls.length).toBe(2);
expect([mockWrite.mock.calls[0], mockWrite.mock.calls[1]]).toEqual(expected);
})
test("read last element.", async () => {
const queueMap = new QueueMap();
const queueId = "1";
const f = jest.fn();
const sleepThenReturn = ms => async (...params) => {
await f(...params);
await sleep(ms);
}
const a = ["a", 1];
const b = ["b", 2];
const c = ["c", 3];
const d = ["d", 4];
queueMap.enque(queueId, a);
queueMap.enque(queueId, b);
queueMap.on(queueId, sleepThenReturn(1000), params => params);
queueMap.enque(queueId, c);
queueMap.enque(queueId, d);
expect(queueMap.getLast(queueId)).toEqual(d);
})
test("delete after write finished", async () => {
const mockCallback = jest.fn();
const mockWrite = jest.fn();
const queueMap = new QueueMap();
const queueId = "1";
const mockAsync = ms => async (...params) => {
await mockWrite(...params);
await sleep(ms);
}
const a = ["a", 1];
const b = ["b", 2];
const c = ["c", 3];
const d = ["d", 4];
queueMap.enque(queueId, a);
queueMap.enque(queueId, b);
queueMap.on(queueId, mockAsync(1000));
queueMap.enque(queueId, c);
queueMap.enque(queueId, d);
const args = [a, b];
queueMap.callAfterLoop(queueId, mockCallback, args);
await sleep(3000);
expect(mockCallback.mock.calls.length).toBe(1);
expect(mockCallback.mock.calls[0]).toEqual(args);
})
test("can call callback finished", async () => {
const mockCallback = jest.fn();
const queueMap = new QueueMap();
const queueId = "1";
queueMap.callAfterLoop(queueId, mockCallback);
expect(mockCallback.mock.calls.length).toBe(1);
})
})

Просмотреть файл

@ -0,0 +1,123 @@
import { Args, IQueue, Queue } from "./queue";
interface IQueueMap {
[id: string]: IQueue;
}
export default class QueueMap {
queueById: IQueueMap;
constructor() {
this.queueById = {};
}
/**
* Get a copy of IQueueMap from QueueMap
* @return QueueMap - IQueueMap
*/
getMap = (): IQueueMap => {
return { ...this.queueById };
}
/**
* Get the IQueue by id. Create a new IQueue while get null.
* @param id - id of the queue
* @return IQueue
*/
getQueueById = (id: string): IQueue => {
if (!this.queueById.hasOwnProperty(id)) {
this.queueById[id] = new Queue();
}
return this.queueById[id];
}
/**
* Find a queue by id, then enqueue an object into the queue.
* @param id - id of the queue
* @param args - list of argument
*/
enque = (id: string, args: Args) => {
const { queue } = this.getQueueById(id);
queue.push(args);
}
/**
* @param id - id of the queue
* @return - dequeued object
*/
dequeue = (id: string): Args => {
const { queue } = this.getQueueById(id);
return queue.shift();
}
/**
* Find a queue by id then dequeue. Then clear objects before the last one.
* @param id - id of the queue
* @return - dequeue object
*/
dequeueUntilLast = (id: string): Args => {
let ret = [];
const { queue } = this.getQueueById(id);
while (queue.length > 1) {
ret = queue.shift();
}
return ret;
}
/** Find and return the last element in the queue
* @param id - id of the queue
* @return last element in the queue
*/
getLast = (id: string): Args => {
const { queue } = this.getQueueById(id);
if (queue.length) {
return queue[queue.length - 1];
}
return [];
}
/**
* loop to use last element as parameters to call async method.
* will prevent this function call while the queue is already looping by another function.
* @param id - id of the queue
* @param method - async method to call
* @param paramsHandler - process dequeue object to method parameters
* @param errorHandler - handle async method error
*/
on = (id: string, method: (...args: any[]) => void, paramsHandler = (params) => params, errorHandler = console.error) => {
const q = this.getQueueById(id);
const loop = async () => {
q.isLooping = true;
while (q.queue.length) {
this.dequeueUntilLast(id);
const args = this.getLast(id);
const params = args.map(paramsHandler);
try {
await method(...params);
} catch (err) {
errorHandler(err);
}
this.dequeue(id);
}
q.isLooping = false;
}
if (q.isLooping === false) {
q.promise = loop();
}
}
/**
* call the callback function after loop finished
* @param id - id of the queue
* @param callback - callback after loop finished
* @param args - callback arguments
*/
callAfterLoop = async (id: string, callback: (...args: any[]) => void, args: Args = []) => {
const q = this.getQueueById(id);
if (q.promise) {
await q.promise;
}
await callback(...args);
}
}
export const queueMap = new QueueMap();

Просмотреть файл

@ -0,0 +1,44 @@
import { IStorageProvider } from "../../providers/storage/storageProviderFactory";
import { constants } from "../constants";
import { queueMap } from "./queueMap";
// tslint:disable-next-line
export function withQueueMap<T extends { new(...args: any[]): IStorageProvider }>(constructor: T) {
return class extends constructor {
isQueuedFile = (filePath: string = ""): boolean => {
return filePath.endsWith(constants.labelFileExtension);
}
writeText = async (filePath: string, contents: string): Promise<void> => {
const parentWriteText = super.writeText.bind(this);
if (this.isQueuedFile(filePath)) {
queueMap.enque(filePath, [filePath, contents]);
queueMap.on(filePath, parentWriteText);
return;
}
return await parentWriteText(filePath, contents);
}
readText = async (filePath: string, ignoreNotFound?: boolean): Promise<string> => {
const parentReadText = super.readText.bind(this);
if (this.isQueuedFile(filePath)) {
const args = queueMap.getLast(filePath);
if (args.length >= 2) {
const contents = args[1] || "";
return (async () => contents)()
}
}
return parentReadText(filePath, ignoreNotFound);
}
deleteFile = async (filePath: string, ignoreNotFound?: boolean, ignoreForbidden?: boolean) => {
// Expect this function is not called too often or may cause race with readText.
const parentDeleteFile = super.deleteFile.bind(this);
if (this.isQueuedFile(filePath)) {
await queueMap.callAfterLoop(filePath, parentDeleteFile, [filePath, ignoreNotFound, ignoreForbidden])
return;
}
parentDeleteFile(filePath, ignoreNotFound, ignoreForbidden);
}
}
}

Просмотреть файл

@ -7,6 +7,7 @@ import { AppError, AssetState, AssetType, ErrorCode, IAsset, StorageType, ILabel
import { throwUnhandledRejectionForEdge } from "../../react/components/common/errorHandler/errorHandler";
import { AssetService } from "../../services/assetService";
import { IStorageProvider } from "./storageProviderFactory";
import {withQueueMap} from "../../common/queueMap/withQueueMap"
/**
* Options for Azure Cloud Storage
@ -21,6 +22,7 @@ export interface IAzureCloudStorageOptions {
/**
* Storage Provider for Azure Blob Storage
*/
@withQueueMap
export class AzureBlobStorage implements IStorageProvider {
/**
* Storage type

Просмотреть файл

@ -286,9 +286,9 @@ export function saveAssetMetadata(
return async (dispatch: Dispatch) => {
const assetService = new AssetService(project);
const savedMetadata = await assetService.save(newAssetMetadata);
dispatch(saveAssetMetadataAction(savedMetadata));
return { ...savedMetadata };
assetService.save(newAssetMetadata);
dispatch(saveAssetMetadataAction(newAssetMetadata));
return { ...newAssetMetadata };
};
}

Просмотреть файл

@ -268,7 +268,6 @@ export class AssetService {
this.project.sourceConnection.providerOptions,
);
}
return this.storageProviderInstance;
}
@ -745,7 +744,7 @@ export class AssetService {
* @param project to get assets and connect to file system.
* @returns updated project
*/
public static checkAndUpdateSchema = async(project: IProject): Promise<IProject> => {
public static checkAndUpdateSchema = async (project: IProject): Promise<IProject> => {
let shouldAssetsUpdate = false;
let updatedProject;
const { assets } = project;

105
yarn.lock
Просмотреть файл

@ -1797,6 +1797,13 @@
dependencies:
"@types/node" "*"
"@types/cheerio@^0.22.22":
version "0.22.28"
resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.28.tgz#90808aabb44fec40fa2950f4c72351e3e4eb065b"
integrity sha512-ehUMGSW5IeDxJjbru4awKYMlKGmo1wSSGUVqXtYwlgmUM8X1a0PZttEIm6yEY7vHsY/hh6iPnklF213G0UColw==
dependencies:
"@types/node" "*"
"@types/color-name@^1.1.1":
version "1.1.1"
resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0"
@ -3627,6 +3634,11 @@ cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3:
inherits "^2.0.1"
safe-buffer "^5.0.1"
circular-json-es6@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/circular-json-es6/-/circular-json-es6-2.0.2.tgz#e4f4a093e49fb4b6aba1157365746112a78bd344"
integrity sha512-ODYONMMNb3p658Zv+Pp+/XPa5s6q7afhz3Tzyvo+VRh9WIrJ64J76ZC4GQxnlye/NesTn09jvOiuE8+xxfpwhQ==
class-utils@^0.3.5:
version "0.3.6"
resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463"
@ -4438,6 +4450,13 @@ deep-diff@^0.3.5:
resolved "https://registry.yarnpkg.com/deep-diff/-/deep-diff-0.3.8.tgz#c01de63efb0eec9798801d40c7e0dae25b582c84"
integrity sha1-wB3mPvsO7JeYgB1Ax+Da4ltYLIQ=
deep-equal-ident@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/deep-equal-ident/-/deep-equal-ident-1.1.1.tgz#06f4b89e53710cd6cea4a7781c7a956642de8dc9"
integrity sha1-BvS4nlNxDNbOpKd4HHqVZkLejck=
dependencies:
lodash.isequal "^3.0"
deep-equal@^1.0.1, deep-equal@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.1.1.tgz#b5c98c942ceffaf7cb051e24e1434a25a2e6076a"
@ -4966,6 +4985,14 @@ enzyme-adapter-utils@^1.13.0:
prop-types "^15.7.2"
semver "^5.7.1"
enzyme-matchers@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/enzyme-matchers/-/enzyme-matchers-7.1.2.tgz#d80530a61f22d28bb993dd7588abba38bd4de282"
integrity sha512-03WqAg2XDl7id9rARIO97HQ1JIw9F2heJ3R4meGu/13hx0ULTDEgl0E67MGl2Uq1jq1DyRnJfto1/VSzskdV5A==
dependencies:
circular-json-es6 "^2.0.1"
deep-equal-ident "^1.1.1"
enzyme-shallow-equal@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/enzyme-shallow-equal/-/enzyme-shallow-equal-1.0.1.tgz#7afe03db3801c9b76de8440694096412a8d9d49e"
@ -4974,6 +5001,15 @@ enzyme-shallow-equal@^1.0.1:
has "^1.0.3"
object-is "^1.0.2"
enzyme-to-json@^3.3.0:
version "3.6.1"
resolved "https://registry.yarnpkg.com/enzyme-to-json/-/enzyme-to-json-3.6.1.tgz#d60740950bc7ca6384dfe6fe405494ec5df996bc"
integrity sha512-15tXuONeq5ORoZjV/bUo2gbtZrN2IH+Z6DvL35QmZyKHgbY1ahn6wcnLd9Xv9OjiwbAXiiP8MRZwbZrCv1wYNg==
dependencies:
"@types/cheerio" "^0.22.22"
lodash "^4.17.15"
react-is "^16.12.0"
enzyme@^3.10.0:
version "3.11.0"
resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.11.0.tgz#71d680c580fe9349f6f5ac6c775bc3e6b7a79c28"
@ -7387,6 +7423,13 @@ jest-each@^24.9.0:
jest-util "^24.9.0"
pretty-format "^24.9.0"
jest-environment-enzyme@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/jest-environment-enzyme/-/jest-environment-enzyme-7.1.2.tgz#4561f26a719e8e87ce8c9a6d3f540a92663ba8d5"
integrity sha512-3tfaYAzO7qZSRrv+srQnfK16Vu5XwH/pHi8FpoqSHjKKngbHzXf7aBCBuWh8y3w0OtknHRfDMFrC60Khj+g1hA==
dependencies:
jest-environment-jsdom "^24.0.0"
jest-environment-jsdom-fourteen@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/jest-environment-jsdom-fourteen/-/jest-environment-jsdom-fourteen-1.0.1.tgz#4cd0042f58b4ab666950d96532ecb2fc188f96fb"
@ -7399,7 +7442,7 @@ jest-environment-jsdom-fourteen@1.0.1:
jest-util "^24.0.0"
jsdom "^14.1.0"
jest-environment-jsdom@^24.9.0:
jest-environment-jsdom@^24.0.0, jest-environment-jsdom@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-24.9.0.tgz#4b0806c7fc94f95edb369a69cc2778eec2b7375b"
integrity sha512-Zv9FV9NBRzLuALXjvRijO2351DRQeLYXtpD4xNvfoVFw21IOKNhZAEUKcbiEtjTkm2GsJ3boMVgkaR7rN8qetA==
@ -7422,6 +7465,15 @@ jest-environment-node@^24.9.0:
jest-mock "^24.9.0"
jest-util "^24.9.0"
jest-enzyme@^7.1.2:
version "7.1.2"
resolved "https://registry.yarnpkg.com/jest-enzyme/-/jest-enzyme-7.1.2.tgz#91a10b2d3be1b56c0d65b34286e5bdc41ab4ba3d"
integrity sha512-j+jkph3t5hGBS12eOldpfsnERYRCHi4c/0KWPMnqRPoJJXvCpLIc5th1MHl0xDznQDXVU0AHUXg3rqMrf8vGpA==
dependencies:
enzyme-matchers "^7.1.2"
enzyme-to-json "^3.3.0"
jest-environment-enzyme "^7.1.2"
jest-get-type@^24.9.0:
version "24.9.0"
resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-24.9.0.tgz#1684a0c8a50f2e4901b6644ae861f579eed2ef0e"
@ -8128,6 +8180,25 @@ locate-path@^5.0.0:
dependencies:
p-locate "^4.1.0"
lodash._baseisequal@^3.0.0:
version "3.0.7"
resolved "https://registry.yarnpkg.com/lodash._baseisequal/-/lodash._baseisequal-3.0.7.tgz#d8025f76339d29342767dcc887ce5cb95a5b51f1"
integrity sha1-2AJfdjOdKTQnZ9zIh85cuVpbUfE=
dependencies:
lodash.isarray "^3.0.0"
lodash.istypedarray "^3.0.0"
lodash.keys "^3.0.0"
lodash._bindcallback@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e"
integrity sha1-5THCdkTPi1epnhftlbNcdIeJOS4=
lodash._getnative@^3.0.0:
version "3.9.1"
resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5"
integrity sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U=
lodash._reinterpolate@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d"
@ -8143,6 +8214,24 @@ lodash.flattendeep@^4.4.0:
resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2"
integrity sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=
lodash.isarguments@^3.0.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
lodash.isarray@^3.0.0:
version "3.0.4"
resolved "https://registry.yarnpkg.com/lodash.isarray/-/lodash.isarray-3.0.4.tgz#79e4eb88c36a8122af86f844aa9bcd851b5fbb55"
integrity sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U=
lodash.isequal@^3.0:
version "3.0.4"
resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-3.0.4.tgz#1c35eb3b6ef0cd1ff51743e3ea3cf7fdffdacb64"
integrity sha1-HDXrO27wzR/1F0Pj6jz3/f/ay2Q=
dependencies:
lodash._baseisequal "^3.0.0"
lodash._bindcallback "^3.0.0"
lodash.isequal@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
@ -8163,6 +8252,20 @@ lodash.isplainobject@^4.0.6:
resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
integrity sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=
lodash.istypedarray@^3.0.0:
version "3.0.6"
resolved "https://registry.yarnpkg.com/lodash.istypedarray/-/lodash.istypedarray-3.0.6.tgz#c9a477498607501d8e8494d283b87c39281cef62"
integrity sha1-yaR3SYYHUB2OhJTSg7h8OSgc72I=
lodash.keys@^3.0.0:
version "3.1.2"
resolved "https://registry.yarnpkg.com/lodash.keys/-/lodash.keys-3.1.2.tgz#4dbc0472b156be50a0b286855d1bd0b0c656098a"
integrity sha1-TbwEcrFWvlCgsoaFXRvQsMZWCYo=
dependencies:
lodash._getnative "^3.0.0"
lodash.isarguments "^3.0.0"
lodash.isarray "^3.0.0"
lodash.memoize@^4.1.2:
version "4.1.2"
resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe"