feat: Active Learning Updates (#778)
Adds new active learning form Moves active learning settings from project settings to here Refactored and created activeLearningService
This commit is contained in:
Wallace Breza
Родитель
a2ef52c7a4
Коммит
921dbac155
|
|
@ -0,0 +1 @@
|
|||
[{"name":"/m/01g317","id":1,"displayName":"person"},{"name":"/m/0199g","id":2,"displayName":"bicycle"},{"name":"/m/0k4j","id":3,"displayName":"car"},{"name":"/m/04_sv","id":4,"displayName":"motorcycle"},{"name":"/m/05czz6l","id":5,"displayName":"airplane"},{"name":"/m/01bjv","id":6,"displayName":"bus"},{"name":"/m/07jdr","id":7,"displayName":"train"},{"name":"/m/07r04","id":8,"displayName":"truck"},{"name":"/m/019jd","id":9,"displayName":"boat"},{"name":"/m/015qff","id":10,"displayName":"traffic light"},{"name":"/m/01pns0","id":11,"displayName":"fire hydrant"},{"name":"/m/02pv19","id":13,"displayName":"stop sign"},{"name":"/m/015qbp","id":14,"displayName":"parking meter"},{"name":"/m/0cvnqh","id":15,"displayName":"bench"},{"name":"/m/015p6","id":16,"displayName":"bird"},{"name":"/m/01yrx","id":17,"displayName":"cat"},{"name":"/m/0bt9lr","id":18,"displayName":"dog"},{"name":"/m/03k3r","id":19,"displayName":"horse"},{"name":"/m/07bgp","id":20,"displayName":"sheep"},{"name":"/m/01xq0k1","id":21,"displayName":"cow"},{"name":"/m/0bwd_0j","id":22,"displayName":"elephant"},{"name":"/m/01dws","id":23,"displayName":"bear"},{"name":"/m/0898b","id":24,"displayName":"zebra"},{"name":"/m/03bk1","id":25,"displayName":"giraffe"},{"name":"/m/01940j","id":27,"displayName":"backpack"},{"name":"/m/0hnnb","id":28,"displayName":"umbrella"},{"name":"/m/080hkjn","id":31,"displayName":"handbag"},{"name":"/m/01rkbr","id":32,"displayName":"tie"},{"name":"/m/01s55n","id":33,"displayName":"suitcase"},{"name":"/m/02wmf","id":34,"displayName":"frisbee"},{"name":"/m/071p9","id":35,"displayName":"skis"},{"name":"/m/06__v","id":36,"displayName":"snowboard"},{"name":"/m/018xm","id":37,"displayName":"sports ball"},{"name":"/m/02zt3","id":38,"displayName":"kite"},{"name":"/m/03g8mr","id":39,"displayName":"baseball bat"},{"name":"/m/03grzl","id":40,"displayName":"baseball glove"},{"name":"/m/06_fw","id":41,"displayName":"skateboard"},{"name":"/m/019w40","id":42,"displayName":"surfboard"},{"name":"/m/0dv9c","id":43,"displayName":"tennis racket"},{"name":"/m/04dr76w","id":44,"displayName":"bottle"},{"name":"/m/09tvcd","id":46,"displayName":"wine glass"},{"name":"/m/08gqpm","id":47,"displayName":"cup"},{"name":"/m/0dt3t","id":48,"displayName":"fork"},{"name":"/m/04ctx","id":49,"displayName":"knife"},{"name":"/m/0cmx8","id":50,"displayName":"spoon"},{"name":"/m/04kkgm","id":51,"displayName":"bowl"},{"name":"/m/09qck","id":52,"displayName":"banana"},{"name":"/m/014j1m","id":53,"displayName":"apple"},{"name":"/m/0l515","id":54,"displayName":"sandwich"},{"name":"/m/0cyhj_","id":55,"displayName":"orange"},{"name":"/m/0hkxq","id":56,"displayName":"broccoli"},{"name":"/m/0fj52s","id":57,"displayName":"carrot"},{"name":"/m/01b9xk","id":58,"displayName":"hot dog"},{"name":"/m/0663v","id":59,"displayName":"pizza"},{"name":"/m/0jy4k","id":60,"displayName":"donut"},{"name":"/m/0fszt","id":61,"displayName":"cake"},{"name":"/m/01mzpv","id":62,"displayName":"chair"},{"name":"/m/02crq1","id":63,"displayName":"couch"},{"name":"/m/03fp41","id":64,"displayName":"potted plant"},{"name":"/m/03ssj5","id":65,"displayName":"bed"},{"name":"/m/04bcr3","id":67,"displayName":"dining table"},{"name":"/m/09g1w","id":70,"displayName":"toilet"},{"name":"/m/07c52","id":72,"displayName":"tv"},{"name":"/m/01c648","id":73,"displayName":"laptop"},{"name":"/m/020lf","id":74,"displayName":"mouse"},{"name":"/m/0qjjc","id":75,"displayName":"remote"},{"name":"/m/01m2v","id":76,"displayName":"keyboard"},{"name":"/m/050k8","id":77,"displayName":"cell phone"},{"name":"/m/0fx9l","id":78,"displayName":"microwave"},{"name":"/m/029bxz","id":79,"displayName":"oven"},{"name":"/m/01k6s3","id":80,"displayName":"toaster"},{"name":"/m/0130jx","id":81,"displayName":"sink"},{"name":"/m/040b_t","id":82,"displayName":"refrigerator"},{"name":"/m/0bt_c3","id":84,"displayName":"book"},{"name":"/m/01x3z","id":85,"displayName":"clock"},{"name":"/m/02s195","id":86,"displayName":"vase"},{"name":"/m/01lsmm","id":87,"displayName":"scissors"},{"name":"/m/0kmg4","id":88,"displayName":"teddy bear"},{"name":"/m/03wvsk","id":89,"displayName":"hair drier"},{"name":"/m/012xff","id":90,"displayName":"toothbrush"}]
|
||||
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Двоичный файл не отображается.
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -20,3 +20,5 @@ linux:
|
|||
- snap
|
||||
publish: null
|
||||
electronVersion: 3.0.13
|
||||
extraFiles:
|
||||
- "cocoSSDModel"
|
||||
|
|
|
|||
|
|
@ -1017,6 +1017,55 @@
|
|||
"loader-utils": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-1.0.3.tgz",
|
||||
"integrity": "sha512-tF6GcjO2KBYlPPiS7o4X+D3oASXJcWAYaZA13GCYp5cXAui0ncHxpC85kmNQlp2HEVmcE82BJz/1uUtkNxxQpw==",
|
||||
"requires": {
|
||||
"@tensorflow/tfjs-converter": "1.0.3",
|
||||
"@tensorflow/tfjs-core": "1.0.3",
|
||||
"@tensorflow/tfjs-data": "1.0.3",
|
||||
"@tensorflow/tfjs-layers": "1.0.3"
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs-converter": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-1.0.3.tgz",
|
||||
"integrity": "sha512-vrGvVrPekhTOwMGsomcpcjw0ZUep6xhI8DQQoFXHjBcprt9bFO2hHMdAmYpqafcJ7KVMylbK4h2LJrsBI2zDgQ=="
|
||||
},
|
||||
"@tensorflow/tfjs-core": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.0.3.tgz",
|
||||
"integrity": "sha512-2UbjMQkmrykIIZuoRfmDPrtWm+6fdQRlYLCUJdiOIooeu/q4nye587HM1qKcdZosGPZTW6VvX+4VIVieYn5i0A==",
|
||||
"requires": {
|
||||
"@types/seedrandom": "2.4.27",
|
||||
"@types/webgl-ext": "0.0.30",
|
||||
"@types/webgl2": "0.0.4",
|
||||
"seedrandom": "2.4.3"
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs-data": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-1.0.3.tgz",
|
||||
"integrity": "sha512-WFjYU2pWNZ0TZaJ7rN18GD/wOTVe6rBGxvSwZxIhEVIbwKXaKXFa9V4aGp4QBG9AXHIA89SjmGSGPxfsC015hQ==",
|
||||
"requires": {
|
||||
"@types/node-fetch": "^2.1.2",
|
||||
"node-fetch": "~2.1.2",
|
||||
"seedrandom": "~2.4.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-fetch": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz",
|
||||
"integrity": "sha1-q4hOjn5X44qUR1POxwb3iNF2i7U="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@tensorflow/tfjs-layers": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-1.0.3.tgz",
|
||||
"integrity": "sha512-7VdvQb0ft7TrWAbBy7HI+p420KX9rblYYACZ7/BzvzsikfEOdEL90WxrZDjZ167rYN/KvqC/haGcmhW/dYU3MA=="
|
||||
},
|
||||
"@types/axios": {
|
||||
"version": "0.14.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz",
|
||||
|
|
@ -1087,8 +1136,15 @@
|
|||
"@types/node": {
|
||||
"version": "10.12.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.7.tgz",
|
||||
"integrity": "sha512-Zh5Z4kACfbeE8aAOYh9mqotRxaZMro8MbBQtR8vEXOMiZo2rGEh2LayJijKdlu48YnS6y2EFU/oo2NCe5P6jGw==",
|
||||
"dev": true
|
||||
"integrity": "sha512-Zh5Z4kACfbeE8aAOYh9mqotRxaZMro8MbBQtR8vEXOMiZo2rGEh2LayJijKdlu48YnS6y2EFU/oo2NCe5P6jGw=="
|
||||
},
|
||||
"@types/node-fetch": {
|
||||
"version": "2.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.1.7.tgz",
|
||||
"integrity": "sha512-TZozHCDVrs0Aj1B9ZR5F4Q9MknDNcVd+hO5lxXOCzz07ELBey6s1gMUSZHUYHlPfRFKJFXiTnNuD7ePiI6S4/g==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/prop-types": {
|
||||
"version": "15.5.8",
|
||||
|
|
@ -1230,6 +1286,11 @@
|
|||
"redux": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"@types/seedrandom": {
|
||||
"version": "2.4.27",
|
||||
"resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz",
|
||||
"integrity": "sha1-nbVjk33YaRX2kJK8QyWdL0hXjkE="
|
||||
},
|
||||
"@types/snapsvg": {
|
||||
"version": "0.4.35",
|
||||
"resolved": "https://registry.npmjs.org/@types/snapsvg/-/snapsvg-0.4.35.tgz",
|
||||
|
|
@ -1243,6 +1304,16 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.2.tgz",
|
||||
"integrity": "sha512-42zEJkBpNfMEAvWR5WlwtTH22oDzcMjFsL9gDGExwF8X8WvAiw7Vwop7hPw03QT8TKfec83LwbHj6SvpqM4ELQ=="
|
||||
},
|
||||
"@types/webgl-ext": {
|
||||
"version": "0.0.30",
|
||||
"resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz",
|
||||
"integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg=="
|
||||
},
|
||||
"@types/webgl2": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz",
|
||||
"integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw=="
|
||||
},
|
||||
"@webassemblyjs/ast": {
|
||||
"version": "1.7.6",
|
||||
"resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.7.6.tgz",
|
||||
|
|
@ -9380,6 +9451,17 @@
|
|||
"requires": {
|
||||
"node-fetch": "^1.0.1",
|
||||
"whatwg-fetch": ">=0.10.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-fetch": {
|
||||
"version": "1.7.3",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
|
||||
"integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
|
||||
"requires": {
|
||||
"encoding": "^0.1.11",
|
||||
"is-stream": "^1.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"isstream": {
|
||||
|
|
@ -10150,6 +10232,11 @@
|
|||
"topo": "2.x.x"
|
||||
}
|
||||
},
|
||||
"jpeg-js": {
|
||||
"version": "0.3.4",
|
||||
"resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.3.4.tgz",
|
||||
"integrity": "sha512-6IzjQxvnlT8UlklNmDXIJMWxijULjqGrzgqc0OG7YadZdvm7KPQ1j0ehmQQHckgEWOfgpptzcnWgESovxudpTA=="
|
||||
},
|
||||
"jquery": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/jquery/-/jquery-3.3.1.tgz",
|
||||
|
|
@ -11261,13 +11348,9 @@
|
|||
}
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "1.7.3",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
|
||||
"integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
|
||||
"requires": {
|
||||
"encoding": "^0.1.11",
|
||||
"is-stream": "^1.0.1"
|
||||
}
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.3.0.tgz",
|
||||
"integrity": "sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA=="
|
||||
},
|
||||
"node-forge": {
|
||||
"version": "0.7.5",
|
||||
|
|
@ -16377,6 +16460,11 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"seedrandom": {
|
||||
"version": "2.4.3",
|
||||
"resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz",
|
||||
"integrity": "sha1-JDhQTa0zkXMUv/GKxNeU8W1qrsw="
|
||||
},
|
||||
"select-hose": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"main": "build/main.js",
|
||||
"dependencies": {
|
||||
"@azure/storage-blob": "^10.3.0",
|
||||
"@tensorflow/tfjs": "^1.0.3",
|
||||
"@types/snapsvg": "^0.4.35",
|
||||
"axios": "^0.18.0",
|
||||
"bootstrap": "^4.1.3",
|
||||
|
|
@ -23,8 +24,10 @@
|
|||
"crypto-js": "^3.1.9-1",
|
||||
"dotenv": "^7.0.0",
|
||||
"google-protobuf": "^3.6.1",
|
||||
"jpeg-js": "^0.3.4",
|
||||
"lodash": "^4.17.11",
|
||||
"md5.js": "^1.3.5",
|
||||
"node-fetch": "^2.3.0",
|
||||
"node-int64": "^0.4.0",
|
||||
"rc-align": "^2.4.5",
|
||||
"rc-checkbox": "^2.1.6",
|
||||
|
|
|
|||
|
|
@ -10,17 +10,7 @@ describe("Html File Reader", () => {
|
|||
|
||||
beforeEach(() => {
|
||||
assetTestCache.clear();
|
||||
|
||||
document.createElement = jest.fn((elementType) => {
|
||||
switch (elementType) {
|
||||
case "img":
|
||||
return mockImage();
|
||||
case "video":
|
||||
return mockVideo();
|
||||
case "canvas":
|
||||
return mockCanvas();
|
||||
}
|
||||
});
|
||||
MockFactory.mockElement(assetTestCache);
|
||||
});
|
||||
|
||||
it("Resolves promise after successfully reading file", async () => {
|
||||
|
|
@ -234,67 +224,4 @@ describe("Html File Reader", () => {
|
|||
await expect(HtmlFileReader.getAssetFrameImage(videoErrorFrame)).rejects.not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
const mockImage = jest.fn(() => {
|
||||
const element: any = {
|
||||
naturalWidth: 0,
|
||||
naturalHeight: 0,
|
||||
onload: jest.fn(),
|
||||
};
|
||||
|
||||
setImmediate(() => {
|
||||
const asset = assetTestCache.get(element.src);
|
||||
element.naturalWidth = asset.size.width;
|
||||
element.naturalHeight = asset.size.height;
|
||||
|
||||
element.onload();
|
||||
});
|
||||
|
||||
return element;
|
||||
});
|
||||
|
||||
const mockVideo = jest.fn(() => {
|
||||
const element: any = {
|
||||
src: "",
|
||||
duration: 0,
|
||||
currentTime: 0,
|
||||
videoWidth: 0,
|
||||
videoHeight: 0,
|
||||
onloadedmetadata: jest.fn(),
|
||||
onseeked: jest.fn(),
|
||||
onerror: jest.fn(),
|
||||
};
|
||||
|
||||
setImmediate(() => {
|
||||
const asset = assetTestCache.get(element.src);
|
||||
if (asset.name.toLowerCase().indexOf("error") > -1) {
|
||||
element.onerror("An error occurred loading the video");
|
||||
} else {
|
||||
element.videoWidth = asset.size.width;
|
||||
element.videoHeight = asset.size.height;
|
||||
element.currentTime = asset.timestamp;
|
||||
element.onloadedmetadata();
|
||||
element.onseeked();
|
||||
}
|
||||
});
|
||||
|
||||
return element;
|
||||
});
|
||||
|
||||
const mockCanvas = jest.fn(() => {
|
||||
const canvas: any = {
|
||||
width: 0,
|
||||
height: 0,
|
||||
getContext: jest.fn(() => {
|
||||
return {
|
||||
drawImage: jest.fn(),
|
||||
};
|
||||
}),
|
||||
toBlob: jest.fn((callback) => {
|
||||
callback(new Blob(["Binary image data"]));
|
||||
}),
|
||||
};
|
||||
|
||||
return canvas;
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -143,6 +143,7 @@ export const english: IAppStrings = {
|
|||
warnings: {
|
||||
existingName: "Tag name already exists. Choose another name",
|
||||
emptyName: "Cannot have an empty tag name",
|
||||
unknownTagName: "Unknown",
|
||||
},
|
||||
toolbar: {
|
||||
add: "Add new tag",
|
||||
|
|
@ -231,6 +232,7 @@ export const english: IAppStrings = {
|
|||
nextAsset: "Next Asset",
|
||||
saveProject: "Save Project",
|
||||
exportProject: "Export Project",
|
||||
activeLearning: "Active Learning",
|
||||
},
|
||||
videoPlayer: {
|
||||
previousTaggedFrame: {
|
||||
|
|
@ -275,9 +277,8 @@ export const english: IAppStrings = {
|
|||
messages: {
|
||||
enforceTaggedRegions: {
|
||||
title: "Invalid region(s) detected",
|
||||
// tslint:disable-next-line:max-line-length
|
||||
description: "1 or more regions have not been tagged. Ensure all regions are tagged before \
|
||||
continuing to next asset.",
|
||||
continuing to next asset.",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -391,6 +392,40 @@ export const english: IAppStrings = {
|
|||
},
|
||||
activeLearning: {
|
||||
title: "Active Learning",
|
||||
form: {
|
||||
properties: {
|
||||
modelPathType: {
|
||||
title: "Model Provider",
|
||||
description: "Where to load the training model from",
|
||||
options: {
|
||||
preTrained: "Pre-trained Coco SSD",
|
||||
customFilePath: "Custom (File path)",
|
||||
customWebUrl: "Custom (Url)",
|
||||
},
|
||||
},
|
||||
autoDetect: {
|
||||
title: "Auto Detect",
|
||||
description: "Whether or not to automatically make predictions as you navigate between assets",
|
||||
},
|
||||
modelPath: {
|
||||
title: "Model path",
|
||||
description: "Select a model from your local file system",
|
||||
},
|
||||
modelUrl: {
|
||||
title: "Model URL",
|
||||
description: "Load your model from a public web URL",
|
||||
},
|
||||
predictTag: {
|
||||
title: "Predict Tag",
|
||||
description: "Whether or not to automatically include tags in predictions",
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
loadingModel: "Loading active learning model...",
|
||||
errorLoadModel: "Error loading active learning model",
|
||||
saveSuccess: "Successfully saved active learning settings",
|
||||
},
|
||||
},
|
||||
profile: {
|
||||
settings: "Profile Settings",
|
||||
|
|
@ -444,5 +479,10 @@ export const english: IAppStrings = {
|
|||
title: "Error exporting project",
|
||||
message: "Project is missing export format. Please select an export format in the export setting page.",
|
||||
},
|
||||
activeLearningPredictionError: {
|
||||
title: "Active Learning Error",
|
||||
message: "An error occurred while predicting regions in the current asset. \
|
||||
Please verify your active learning configuration and try again",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -60,8 +60,8 @@ export const spanish: IAppStrings = {
|
|||
},
|
||||
securityTokens: {
|
||||
title: "Tokens de seguridad",
|
||||
// tslint:disable-next-line:max-line-length
|
||||
description: "Los tokens de seguridad se utilizan para cifrar datos confidenciales dentro de la configuración del proyecto",
|
||||
description: "Los tokens de seguridad se utilizan para cifrar datos confidenciales \
|
||||
dentro de la configuración del proyecto",
|
||||
},
|
||||
version: {
|
||||
description: "Versión:",
|
||||
|
|
@ -144,6 +144,7 @@ export const spanish: IAppStrings = {
|
|||
warnings: {
|
||||
existingName: "Nombre de etiqueta ya existe. Elige otro nombre",
|
||||
emptyName: "El nombre de etiqueta no puede ser vacío",
|
||||
unknownTagName: "Desconocido",
|
||||
},
|
||||
toolbar: {
|
||||
add: "Agregar nueva etiqueta",
|
||||
|
|
@ -233,6 +234,7 @@ export const spanish: IAppStrings = {
|
|||
nextAsset: "Siguiente activo",
|
||||
saveProject: "Guardar Proyecto",
|
||||
exportProject: "Exprtar Proyecto",
|
||||
activeLearning: "Aprendizaje Activo",
|
||||
},
|
||||
videoPlayer: {
|
||||
previousTaggedFrame: {
|
||||
|
|
@ -278,8 +280,8 @@ export const spanish: IAppStrings = {
|
|||
messages: {
|
||||
enforceTaggedRegions: {
|
||||
title: "Las regiones no válidas detectadas",
|
||||
// tslint:disable-next-line:max-line-length
|
||||
description: "1 o más regiones no se han etiquetado. Por favor, etiquete todas las regiones antes de continuar con el siguiente activo.",
|
||||
description: "1 o más regiones no se han etiquetado. \
|
||||
Por favor, etiquete todas las regiones antes de continuar con el siguiente activo.",
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -393,6 +395,41 @@ export const spanish: IAppStrings = {
|
|||
},
|
||||
activeLearning: {
|
||||
title: "Aprendizaje Activo",
|
||||
form: {
|
||||
properties: {
|
||||
modelPathType: {
|
||||
title: "Proveedor del modelo",
|
||||
description: "Fuente desde la cual cargar el modelo",
|
||||
options: {
|
||||
preTrained: "SSD de coco pre-entrenado",
|
||||
customFilePath: "Personalizado (ruta de archivo)",
|
||||
customWebUrl: "Personalizado (URL)",
|
||||
},
|
||||
},
|
||||
autoDetect: {
|
||||
title: "Detección automática",
|
||||
description: "Si desea o no realizar automáticamente predicciones a \
|
||||
medida que navega entre activos",
|
||||
},
|
||||
modelPath: {
|
||||
title: "Ruta de modelo",
|
||||
description: "Seleccione un modelo de su sistema de archivos local",
|
||||
},
|
||||
modelUrl: {
|
||||
title: "URL del modelo",
|
||||
description: "Cargue el modelo desde una URL web pública",
|
||||
},
|
||||
predictTag: {
|
||||
title: "Predecir etiqueta",
|
||||
description: "Si se incluirán o no automáticamente las etiquetas en las predicciones",
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
loadingModel: "Cargando modelo...",
|
||||
errorLoadModel: "Error al cargar el modelo",
|
||||
saveSuccess: "La configuración de aprendizaje activa se ha guardada correctamente",
|
||||
},
|
||||
},
|
||||
profile: {
|
||||
settings: "Configuración de Perfíl",
|
||||
|
|
@ -448,5 +485,10 @@ export const spanish: IAppStrings = {
|
|||
message: `Proyecto falta el formato de exportación. Seleccione un formato de exportación en la página
|
||||
de configuración de exportación.`,
|
||||
},
|
||||
activeLearningPredictionError: {
|
||||
title: "Error de aprendizaje",
|
||||
message: "Se ha producido un error al predecir regiones en el activo actual. \
|
||||
Compruebe la configuración de aprendizaje activa y vuelva a intentarlo",
|
||||
},
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import {
|
|||
AssetState, AssetType, IApplicationState, IAppSettings, IAsset, IAssetMetadata,
|
||||
IConnection, IExportFormat, IProject, ITag, StorageType, ISecurityToken,
|
||||
EditorMode, IAppError, IProjectVideoSettings, ErrorCode,
|
||||
IPoint, IRegion, RegionType,
|
||||
IPoint, IRegion, RegionType, ModelPathType,
|
||||
} from "../models/applicationState";
|
||||
import { IV1Project, IV1Region } from "../models/v1Models";
|
||||
import { ExportAssetState } from "../providers/export/exportProvider";
|
||||
|
|
@ -33,6 +33,7 @@ import { SelectionMode } from "vott-ct/lib/js/CanvasTools/Interface/ISelectorSet
|
|||
import { IKeyboardBindingProps } from "../react/components/common/keyboardBinding/keyboardBinding";
|
||||
import { KeyEventType } from "../react/components/common/keyboardManager/keyboardManager";
|
||||
import { IKeyboardRegistrations } from "../react/components/common/keyboardManager/keyboardRegistrationManager";
|
||||
import { IActiveLearningPageProps } from "../react/components/pages/activeLearning/activeLearningPage";
|
||||
|
||||
export default class MockFactory {
|
||||
|
||||
|
|
@ -283,6 +284,13 @@ export default class MockFactory {
|
|||
targetConnection: connection,
|
||||
tags: MockFactory.createTestTags(tagCount),
|
||||
videoSettings: MockFactory.createVideoSettings(),
|
||||
activeLearningSettings: {
|
||||
modelPathType: ModelPathType.Coco,
|
||||
modelPath: "",
|
||||
modelUrl: "",
|
||||
autoDetect: false,
|
||||
predictTag: false,
|
||||
},
|
||||
autoSave: true,
|
||||
};
|
||||
}
|
||||
|
|
@ -886,6 +894,21 @@ export default class MockFactory {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates fake IActiveLearningPageProps
|
||||
* @param projectId Current project ID
|
||||
*/
|
||||
public static activeLearningProps(projectId?: string): IActiveLearningPageProps {
|
||||
return {
|
||||
actions: (projectActions as any) as IProjectActions,
|
||||
history: MockFactory.history(),
|
||||
location: MockFactory.location(),
|
||||
match: MockFactory.match(projectId, "active-learning"),
|
||||
project: null,
|
||||
recentProjects: MockFactory.createTestProjects(),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates fake IEditorPageProps
|
||||
* @param projectId Current project ID
|
||||
|
|
@ -1012,6 +1035,93 @@ export default class MockFactory {
|
|||
};
|
||||
}
|
||||
|
||||
public static mockElement(assetTestCache: Map<string, IAsset>) {
|
||||
document.createElement = jest.fn((elementType) => {
|
||||
switch (elementType) {
|
||||
case "img":
|
||||
const mockImage = MockFactory.mockImage(assetTestCache);
|
||||
return mockImage();
|
||||
case "video":
|
||||
const mockVideo = MockFactory.mockVideo(assetTestCache);
|
||||
return mockVideo();
|
||||
case "canvas":
|
||||
const mockCanvas = MockFactory.mockCanvas();
|
||||
return mockCanvas();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static mockImage(assetTestCache: Map<string, IAsset>) {
|
||||
return jest.fn(() => {
|
||||
const element: any = {
|
||||
naturalWidth: 0,
|
||||
naturalHeight: 0,
|
||||
onload: jest.fn(),
|
||||
};
|
||||
|
||||
setImmediate(() => {
|
||||
const asset = assetTestCache.get(element.src);
|
||||
if (asset) {
|
||||
element.naturalWidth = asset.size.width;
|
||||
element.naturalHeight = asset.size.height;
|
||||
}
|
||||
|
||||
element.onload();
|
||||
});
|
||||
|
||||
return element;
|
||||
});
|
||||
}
|
||||
|
||||
public static mockVideo(assetTestCache: Map<string, IAsset>) {
|
||||
return jest.fn(() => {
|
||||
const element: any = {
|
||||
src: "",
|
||||
duration: 0,
|
||||
currentTime: 0,
|
||||
videoWidth: 0,
|
||||
videoHeight: 0,
|
||||
onloadedmetadata: jest.fn(),
|
||||
onseeked: jest.fn(),
|
||||
onerror: jest.fn(),
|
||||
};
|
||||
|
||||
setImmediate(() => {
|
||||
const asset = assetTestCache.get(element.src);
|
||||
if (asset.name.toLowerCase().indexOf("error") > -1) {
|
||||
element.onerror("An error occurred loading the video");
|
||||
} else {
|
||||
element.videoWidth = asset.size.width;
|
||||
element.videoHeight = asset.size.height;
|
||||
element.currentTime = asset.timestamp;
|
||||
element.onloadedmetadata();
|
||||
element.onseeked();
|
||||
}
|
||||
});
|
||||
|
||||
return element;
|
||||
});
|
||||
}
|
||||
|
||||
public static mockCanvas() {
|
||||
return jest.fn(() => {
|
||||
const canvas: any = {
|
||||
width: 800,
|
||||
height: 600,
|
||||
getContext: jest.fn(() => {
|
||||
return {
|
||||
drawImage: jest.fn(),
|
||||
};
|
||||
}),
|
||||
toBlob: jest.fn((callback) => {
|
||||
callback(new Blob(["Binary image data"]));
|
||||
}),
|
||||
};
|
||||
|
||||
return canvas;
|
||||
});
|
||||
}
|
||||
|
||||
private static pageProps(projectId: string, method: string) {
|
||||
return {
|
||||
project: null,
|
||||
|
|
@ -1093,5 +1203,4 @@ export default class MockFactory {
|
|||
return StorageType.Other;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ export interface IAppStrings {
|
|||
warnings: {
|
||||
existingName: string;
|
||||
emptyName: string;
|
||||
unknownTagName: string;
|
||||
}
|
||||
};
|
||||
connections: {
|
||||
|
|
@ -230,6 +231,7 @@ export interface IAppStrings {
|
|||
nextAsset: string;
|
||||
saveProject: string;
|
||||
exportProject: string;
|
||||
activeLearning: string;
|
||||
}
|
||||
videoPlayer: {
|
||||
nextTaggedFrame: {
|
||||
|
|
@ -387,6 +389,40 @@ export interface IAppStrings {
|
|||
};
|
||||
activeLearning: {
|
||||
title: string;
|
||||
form: {
|
||||
properties: {
|
||||
modelPathType: {
|
||||
title: string,
|
||||
description: string,
|
||||
options: {
|
||||
preTrained: string,
|
||||
customFilePath: string,
|
||||
customWebUrl: string,
|
||||
},
|
||||
},
|
||||
autoDetect: {
|
||||
title: string,
|
||||
description: string,
|
||||
},
|
||||
predictTag: {
|
||||
title: string,
|
||||
description: string,
|
||||
},
|
||||
modelPath: {
|
||||
title: string,
|
||||
description: string,
|
||||
},
|
||||
modelUrl: {
|
||||
title: string,
|
||||
description: string,
|
||||
},
|
||||
},
|
||||
}
|
||||
messages: {
|
||||
loadingModel: string;
|
||||
errorLoadModel: string;
|
||||
saveSuccess: string;
|
||||
}
|
||||
};
|
||||
profile: {
|
||||
settings: string;
|
||||
|
|
@ -403,6 +439,7 @@ export interface IAppStrings {
|
|||
importError: IErrorMetadata,
|
||||
pasteRegionTooBigError: IErrorMetadata,
|
||||
exportFormatNotFound: IErrorMetadata,
|
||||
activeLearningPredictionError: IErrorMetadata,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { ExportAssetState } from "../providers/export/exportProvider";
|
||||
import { IAssetPreviewSettings } from "../react/components/common/assetPreview/assetPreview";
|
||||
|
||||
/**
|
||||
* @name - Application State
|
||||
|
|
@ -49,6 +50,7 @@ export enum ErrorCode {
|
|||
ExportFormatNotFound = "exportFormatNotFound",
|
||||
PasteRegionTooBig = "pasteRegionTooBig",
|
||||
OverloadedKeyBinding = "overloadedKeyBinding",
|
||||
ActiveLearningPredictionError = "activeLearningPredictionError",
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -112,6 +114,7 @@ export interface IProject {
|
|||
targetConnection: IConnection;
|
||||
exportFormat: IExportFormat;
|
||||
videoSettings: IProjectVideoSettings;
|
||||
activeLearningSettings: IActiveLearningSettings;
|
||||
autoSave: boolean;
|
||||
assets?: { [index: string]: IAsset };
|
||||
lastVisitedAssetId?: string;
|
||||
|
|
@ -198,6 +201,44 @@ export interface IProjectVideoSettings {
|
|||
frameExtractionRate: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @name - Model Path Type
|
||||
* @description - Defines the mechanism to load the TF.js model for Active Learning
|
||||
* @member Coco - Specifies the default/generic pre-trained Coco-SSD model
|
||||
* @member File - Specifies to load a custom model from filesystem
|
||||
* @member Url - Specifies to load a custom model from a web server
|
||||
*/
|
||||
export enum ModelPathType {
|
||||
Coco = "coco",
|
||||
File = "file",
|
||||
Url = "url",
|
||||
}
|
||||
|
||||
/**
|
||||
* Properties for additional project settings
|
||||
* @member activeLearningSettings - Active Learning settings
|
||||
*/
|
||||
export interface IAdditionalPageSettings extends IAssetPreviewSettings {
|
||||
activeLearningSettings: IActiveLearningSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
* @name - Active Learning Settings for the project
|
||||
* @description - Defines the active learning settings within a VoTT project
|
||||
* @member modelPathType - Model loading type ["coco", "file", "url"]
|
||||
* @member modelPath - Local filesystem path to the TF.js model
|
||||
* @member modelUrl - Web url to the TF.js model
|
||||
* @member autoDetect - Flag for automatically call the model while opening a new asset
|
||||
* @member predictTag - Flag to predict also the tag name other than the rectangle coordinates only
|
||||
*/
|
||||
export interface IActiveLearningSettings {
|
||||
modelPathType: ModelPathType;
|
||||
modelPath?: string;
|
||||
modelUrl?: string;
|
||||
autoDetect: boolean;
|
||||
predictTag: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* @name - Asset Video Settings
|
||||
* @description - Defines the settings for video assets
|
||||
|
|
@ -231,6 +272,7 @@ export interface IAsset {
|
|||
format?: string;
|
||||
timestamp?: number;
|
||||
parent?: IAsset;
|
||||
predicted?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
jest.mock("../storage/localFileSystemProxy");
|
||||
import { LocalFileSystemProxy } from "../storage/localFileSystemProxy";
|
||||
import { ElectronProxyHandler } from "./electronProxyHandler";
|
||||
import * as tf from "@tensorflow/tfjs";
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const modelJson = require("../../../cocoSSDModel/model.json");
|
||||
|
||||
describe("Load default model from filesystem with TF io.IOHandler", () => {
|
||||
it("Check file system proxy is correctly called", async () => {
|
||||
const storageProviderMock = LocalFileSystemProxy as jest.Mock<LocalFileSystemProxy>;
|
||||
storageProviderMock.mockClear();
|
||||
|
||||
storageProviderMock.prototype.readText = jest.fn((fileName) => {
|
||||
return Promise.resolve(JSON.stringify(modelJson));
|
||||
});
|
||||
|
||||
storageProviderMock.prototype.readBinary = jest.fn((fileName) => {
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
|
||||
const handler = new ElectronProxyHandler("folder");
|
||||
try {
|
||||
const model = await tf.loadGraphModel(handler);
|
||||
} catch (_) {
|
||||
// fully loading TF model fails as it has to load also weights
|
||||
}
|
||||
|
||||
expect(LocalFileSystemProxy.prototype.readText).toBeCalledWith("/model.json");
|
||||
|
||||
// Coco SSD Lite default embedded model has 5 weights matrix
|
||||
expect(LocalFileSystemProxy.prototype.readBinary).toBeCalledTimes(5);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
import * as tfc from "@tensorflow/tfjs-core";
|
||||
import { LocalFileSystemProxy, ILocalFileSystemProxyOptions } from "../../providers/storage/localFileSystemProxy";
|
||||
|
||||
export class ElectronProxyHandler implements tfc.io.IOHandler {
|
||||
protected readonly provider: LocalFileSystemProxy;
|
||||
|
||||
constructor(folderPath: string) {
|
||||
const options: ILocalFileSystemProxyOptions = { folderPath };
|
||||
this.provider = new LocalFileSystemProxy(options);
|
||||
}
|
||||
|
||||
public async load(): Promise<tfc.io.ModelArtifacts> {
|
||||
const modelJSON = JSON.parse(await this.provider.readText("/model.json"));
|
||||
|
||||
const modelArtifacts: tfc.io.ModelArtifacts = {
|
||||
modelTopology: modelJSON.modelTopology,
|
||||
};
|
||||
|
||||
if (modelJSON.weightsManifest != null) {
|
||||
const [weightSpecs, weightData] =
|
||||
await this.loadWeights(modelJSON.weightsManifest);
|
||||
modelArtifacts.weightSpecs = weightSpecs;
|
||||
modelArtifacts.weightData = weightData;
|
||||
}
|
||||
|
||||
return modelArtifacts;
|
||||
}
|
||||
|
||||
public async loadClasses(): Promise<JSON> {
|
||||
const json = await this.provider.readText("/classes.json");
|
||||
return json ? JSON.parse(json) : null;
|
||||
}
|
||||
|
||||
private async loadWeights(weightsManifest: tfc.io.WeightsManifestConfig)
|
||||
: Promise<[tfc.io.WeightsManifestEntry[], ArrayBuffer]> {
|
||||
const buffers: Buffer[] = [];
|
||||
const weightSpecs: tfc.io.WeightsManifestEntry[] = [];
|
||||
|
||||
for (const group of weightsManifest) {
|
||||
for (const shardName of group.paths) {
|
||||
const buffer = await this.provider.readBinary("/" + shardName);
|
||||
buffers.push(buffer);
|
||||
}
|
||||
weightSpecs.push(...group.weights);
|
||||
}
|
||||
|
||||
return [weightSpecs, this.toArrayBuffer(buffers)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Buffer or an Array of Buffers to an ArrayBuffer.
|
||||
*
|
||||
* If the input is an Array of Buffers, they will be concatenated in the
|
||||
* specified order to form the output ArrayBuffer.
|
||||
*/
|
||||
private toArrayBuffer(buf: Buffer | Buffer[]): ArrayBuffer {
|
||||
if (Array.isArray(buf)) {
|
||||
// An Array of Buffers.
|
||||
let totalLength = 0;
|
||||
for (const buffer of buf) {
|
||||
totalLength += buffer.length;
|
||||
}
|
||||
|
||||
const ab = new ArrayBuffer(totalLength);
|
||||
const view = new Uint8Array(ab);
|
||||
let pos = 0;
|
||||
for (const buffer of buf) {
|
||||
pos += buffer.copy(view, pos);
|
||||
}
|
||||
return ab;
|
||||
} else {
|
||||
// A single Buffer. Return a copy of the underlying ArrayBuffer slice.
|
||||
return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
import * as tf from "@tensorflow/tfjs";
|
||||
jest.mock("../storage/localFileSystemProxy");
|
||||
import { LocalFileSystemProxy } from "../storage/localFileSystemProxy";
|
||||
import { ObjectDetection, DetectedObject } from "./objectDetection";
|
||||
import { strings } from "../../common/strings";
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const modelJson = require("../../../cocoSSDModel/model.json");
|
||||
|
||||
describe("Load an Object Detection model", () => {
|
||||
it("Load model from file system using proxy", async () => {
|
||||
const storageProviderMock = LocalFileSystemProxy as jest.Mock<LocalFileSystemProxy>;
|
||||
storageProviderMock.mockClear();
|
||||
|
||||
storageProviderMock.prototype.readText = jest.fn((fileName) => {
|
||||
return Promise.resolve(JSON.stringify(modelJson));
|
||||
});
|
||||
|
||||
storageProviderMock.prototype.readBinary = jest.fn((fileName) => {
|
||||
return Promise.resolve([]);
|
||||
});
|
||||
|
||||
const model = new ObjectDetection();
|
||||
|
||||
try {
|
||||
await model.load("path");
|
||||
} catch (_) {
|
||||
// fully loading TF model fails has it has to load also weights
|
||||
}
|
||||
|
||||
expect(LocalFileSystemProxy.prototype.readText).toBeCalledWith("/model.json");
|
||||
|
||||
// Coco SSD Lite default embedded model has 5 weights matrix
|
||||
expect(LocalFileSystemProxy.prototype.readBinary).toBeCalledTimes(5);
|
||||
|
||||
// Modal not properly loaded as readBinary mock is not really loading the weights
|
||||
expect(model.loaded).toBeFalsy();
|
||||
|
||||
const noDetection = await model.detect(null);
|
||||
expect(noDetection.length).toEqual(0);
|
||||
|
||||
model.dispose();
|
||||
});
|
||||
|
||||
it("Load model from http url", async () => {
|
||||
window.fetch = jest.fn().mockImplementation((url, o) => {
|
||||
if (url === "http://url/model.json") {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
json: () => modelJson,
|
||||
});
|
||||
} else {
|
||||
return Promise.resolve({
|
||||
ok: true,
|
||||
data: () => [],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const model = new ObjectDetection();
|
||||
|
||||
expect(model.load("http://url")).rejects.not.toBeNull();
|
||||
expect(window.fetch).toBeCalledTimes(1);
|
||||
|
||||
// Modal not properly loaded as readBinary mock is not really loading the weights
|
||||
expect(model.loaded).toBeFalsy();
|
||||
|
||||
const noDetection = await model.detect(null);
|
||||
expect(noDetection.length).toEqual(0);
|
||||
|
||||
model.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("Test Detection on Fake Model", () => {
|
||||
beforeEach(() => {
|
||||
spyOn(tf, "loadGraphModel").and.callFake(() => {
|
||||
const model = {
|
||||
executeAsync:
|
||||
(x: tf.Tensor) => [tf.ones([1, 1917, 90]), tf.ones([1, 1917, 1, 4])],
|
||||
};
|
||||
|
||||
return model;
|
||||
});
|
||||
});
|
||||
|
||||
it("ObjectDetection detect method should generate output", async () => {
|
||||
const model = new ObjectDetection();
|
||||
await model.load("path");
|
||||
|
||||
const x = tf.zeros([227, 227, 3]) as tf.Tensor3D;
|
||||
|
||||
const data = await model.detect(x, 1);
|
||||
|
||||
expect(data).toEqual([{bbox: [227, 227, 0, 0], class: strings.tags.warnings.unknownTagName, score: 1}]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Test predictImage on Fake Model", () => {
|
||||
beforeEach(() => {
|
||||
spyOn(tf, "loadGraphModel").and.callFake(() => {
|
||||
const model = {
|
||||
executeAsync:
|
||||
(x: tf.Tensor) => [tf.ones([1, 1917, 90]), tf.ones([1, 1917, 1, 4])],
|
||||
};
|
||||
|
||||
return model;
|
||||
});
|
||||
});
|
||||
|
||||
it("predictImage on a fake image", async () => {
|
||||
const model = new ObjectDetection();
|
||||
await model.load("path");
|
||||
|
||||
const x = tf.zeros([227, 227, 3]) as tf.Tensor3D;
|
||||
const regions = await model.predictImage(x, false, 1, 1);
|
||||
|
||||
expect(regions.length).toEqual(20);
|
||||
expect(regions[0].boundingBox.left).toEqual(227);
|
||||
expect(regions[0].boundingBox.top).toEqual(227);
|
||||
expect(regions[0].boundingBox.width).toEqual(0);
|
||||
expect(regions[0].boundingBox.height).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
import axios from "axios";
|
||||
import * as shortid from "shortid";
|
||||
import * as tf from "@tensorflow/tfjs";
|
||||
import { ElectronProxyHandler } from "./electronProxyHandler";
|
||||
import { IRegion, RegionType } from "../../models/applicationState";
|
||||
import { strings } from "../../common/strings";
|
||||
|
||||
// tslint:disable-next-line:interface-over-type-literal
|
||||
export type DetectedObject = {
|
||||
bbox: [number, number, number, number]; // [x, y, width, height]
|
||||
class: string;
|
||||
score: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Defines supported data types supported by Tensorflow JS
|
||||
*/
|
||||
export type ImageObject = tf.Tensor3D | ImageData | HTMLImageElement | HTMLCanvasElement | HTMLVideoElement;
|
||||
|
||||
/**
|
||||
* Object Dectection loads active learning models and predicts regions
|
||||
*/
|
||||
export class ObjectDetection {
|
||||
private modelLoaded: boolean = false;
|
||||
|
||||
get loaded(): boolean {
|
||||
return this.modelLoaded;
|
||||
}
|
||||
|
||||
private model: tf.GraphModel;
|
||||
private jsonClasses: JSON;
|
||||
|
||||
/**
|
||||
* Dispose the tensors allocated by the model. You should call this when you
|
||||
* are done with the model.
|
||||
*/
|
||||
public dispose() {
|
||||
if (this.model) {
|
||||
this.model.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a TensorFlow.js Object Detection model from file: or http URL.
|
||||
* @param modelFolderPath file: or http URL to the model
|
||||
*/
|
||||
public async load(modelFolderPath: string) {
|
||||
try {
|
||||
if (modelFolderPath.toLowerCase().startsWith("http://") ||
|
||||
modelFolderPath.toLowerCase().startsWith("https://")) {
|
||||
this.model = await tf.loadGraphModel(modelFolderPath + "/model.json");
|
||||
|
||||
const response = await axios.get(modelFolderPath + "/classes.json");
|
||||
this.jsonClasses = JSON.parse(JSON.stringify(response.data));
|
||||
} else {
|
||||
const handler = new ElectronProxyHandler(modelFolderPath);
|
||||
this.model = await tf.loadGraphModel(handler);
|
||||
this.jsonClasses = await handler.loadClasses();
|
||||
}
|
||||
|
||||
// Warmup the model.
|
||||
const result = await this.model.executeAsync(tf.zeros([1, 300, 300, 3])) as tf.Tensor[];
|
||||
result.forEach(async (t) => await t.data());
|
||||
result.forEach(async (t) => t.dispose());
|
||||
this.modelLoaded = true;
|
||||
} catch (err) {
|
||||
this.modelLoaded = false;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Predict Regions from an HTMLImageElement returning list of IRegion.
|
||||
* @param image ImageObject to be used for prediction
|
||||
* @param predictTag Flag indicates if predict only region bounding box of tag too.
|
||||
* @param xRatio Width compression ratio between the HTMLImageElement and the original image.
|
||||
* @param yRatio Height compression ratio between the HTMLImageElement and the original image.
|
||||
*/
|
||||
public async predictImage(image: ImageObject, predictTag: boolean, xRatio: number, yRatio: number)
|
||||
: Promise<IRegion[]> {
|
||||
const regions: IRegion[] = [];
|
||||
|
||||
const predictions = await this.detect(image);
|
||||
predictions.forEach((prediction) => {
|
||||
const left = Math.max(0, prediction.bbox[0] * xRatio);
|
||||
const top = Math.max(0, prediction.bbox[1] * yRatio);
|
||||
const width = Math.max(0, prediction.bbox[2] * xRatio);
|
||||
const height = Math.max(0, prediction.bbox[3] * yRatio);
|
||||
|
||||
regions.push({
|
||||
id: shortid.generate(),
|
||||
type: RegionType.Rectangle,
|
||||
tags: predictTag ? [prediction.class] : [],
|
||||
boundingBox: {
|
||||
left,
|
||||
top,
|
||||
width,
|
||||
height,
|
||||
},
|
||||
points: [{
|
||||
x: left,
|
||||
y: top,
|
||||
},
|
||||
{
|
||||
x: left + width,
|
||||
y: top,
|
||||
},
|
||||
{
|
||||
x: left + width,
|
||||
y: top + height,
|
||||
},
|
||||
{
|
||||
x: left,
|
||||
y: top + height,
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect objects for an image returning a list of bounding boxes with
|
||||
* associated class and score.
|
||||
*
|
||||
* @param img The image to detect objects from. Can be a tensor or a DOM
|
||||
* element image, video, or canvas.
|
||||
* @param maxNumBoxes The maximum number of bounding boxes of detected
|
||||
* objects. There can be multiple objects of the same class, but at different
|
||||
* locations. Defaults to 20.
|
||||
*
|
||||
*/
|
||||
public async detect(img: ImageObject, maxNumBoxes: number = 20): Promise<DetectedObject[]> {
|
||||
if (this.model) {
|
||||
return this.infer(img, maxNumBoxes);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Infers through the model.
|
||||
*
|
||||
* @param img The image to classify. Can be a tensor or a DOM element image,
|
||||
* video, or canvas.
|
||||
* @param maxNumBoxes The maximum number of bounding boxes of detected
|
||||
* objects. There can be multiple objects of the same class, but at different
|
||||
* locations. Defaults to 20.
|
||||
*/
|
||||
private async infer(img: ImageObject, maxNumBoxes: number = 20): Promise<DetectedObject[]> {
|
||||
const batched = tf.tidy(() => {
|
||||
if (!(img instanceof tf.Tensor)) {
|
||||
img = tf.browser.fromPixels(img);
|
||||
}
|
||||
// Reshape to a single-element batch so we can pass it to executeAsync.
|
||||
return img.expandDims(0);
|
||||
});
|
||||
const height = batched.shape[1];
|
||||
const width = batched.shape[2];
|
||||
|
||||
// model returns two tensors:
|
||||
// 1. box classification score with shape of [1, 1917, 90]
|
||||
// 2. box location with shape of [1, 1917, 1, 4]
|
||||
// where 1917 is the number of box detectors, 90 is the number of classes.
|
||||
// and 4 is the four coordinates of the box.
|
||||
const result = await this.model.executeAsync(batched) as tf.Tensor[];
|
||||
|
||||
const scores = result[0].dataSync() as Float32Array;
|
||||
const boxes = result[1].dataSync() as Float32Array;
|
||||
|
||||
// clean the webgl tensors
|
||||
batched.dispose();
|
||||
tf.dispose(result);
|
||||
|
||||
const [maxScores, classes] = this.calculateMaxScores(scores, result[0].shape[1], result[0].shape[2]);
|
||||
|
||||
const prevBackend = tf.getBackend();
|
||||
// run post process in cpu
|
||||
tf.setBackend("cpu");
|
||||
const indexTensor = tf.tidy(() => {
|
||||
const boxes2 = tf.tensor2d(boxes, [result[1].shape[1], result[1].shape[3]]);
|
||||
return tf.image.nonMaxSuppression(boxes2, maxScores, maxNumBoxes, 0.5, 0.5);
|
||||
});
|
||||
|
||||
const indexes = indexTensor.dataSync() as Float32Array;
|
||||
indexTensor.dispose();
|
||||
|
||||
// restore previous backend
|
||||
tf.setBackend(prevBackend);
|
||||
|
||||
return this.buildDetectedObjects(width, height, boxes, maxScores, indexes, classes);
|
||||
}
|
||||
|
||||
private buildDetectedObjects(
|
||||
width: number, height: number, boxes: Float32Array, scores: number[],
|
||||
indexes: Float32Array, classes: number[]): DetectedObject[] {
|
||||
const count = indexes.length;
|
||||
const objects: DetectedObject[] = [];
|
||||
|
||||
for (let i = 0; i < count; i++) {
|
||||
const bbox = [];
|
||||
for (let j = 0; j < 4; j++) {
|
||||
bbox[j] = boxes[indexes[i] * 4 + j];
|
||||
}
|
||||
const minY = bbox[0] * height;
|
||||
const minX = bbox[1] * width;
|
||||
const maxY = bbox[2] * height;
|
||||
const maxX = bbox[3] * width;
|
||||
bbox[0] = minX;
|
||||
bbox[1] = minY;
|
||||
bbox[2] = maxX - minX;
|
||||
bbox[3] = maxY - minY;
|
||||
objects.push({
|
||||
bbox: bbox as [number, number, number, number],
|
||||
class: this.getClass(i, indexes, classes),
|
||||
score: scores[indexes[i]],
|
||||
});
|
||||
}
|
||||
|
||||
return objects;
|
||||
}
|
||||
|
||||
private getClass(index: number, indexes: Float32Array, classes: number[]): string {
|
||||
if (this.jsonClasses && index < indexes.length && indexes[index] < classes.length) {
|
||||
const classId = classes[indexes[index]] - 1;
|
||||
const classObject = this.jsonClasses[classId];
|
||||
|
||||
return classObject ? classObject.displayName : strings.tags.warnings.unknownTagName;
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
private calculateMaxScores(
|
||||
scores: Float32Array, numBoxes: number,
|
||||
numClasses: number): [number[], number[]] {
|
||||
const maxes = [];
|
||||
const classes = [];
|
||||
for (let i = 0; i < numBoxes; i++) {
|
||||
let max = Number.MIN_VALUE;
|
||||
let index = -1;
|
||||
for (let j = 0; j < numClasses; j++) {
|
||||
if (scores[i * numClasses + j] > max) {
|
||||
max = scores[i * numClasses + j];
|
||||
index = j;
|
||||
}
|
||||
}
|
||||
maxes[i] = max;
|
||||
classes[i] = index;
|
||||
}
|
||||
return [maxes, classes];
|
||||
}
|
||||
}
|
||||
|
|
@ -12,7 +12,8 @@ export class ImageAsset extends React.Component<IAssetProps> {
|
|||
<img ref={this.image}
|
||||
src={this.props.asset.path}
|
||||
onLoad={this.onLoad}
|
||||
onError={this.props.onError} />);
|
||||
onError={this.props.onError}
|
||||
crossOrigin="anonymous" />);
|
||||
}
|
||||
|
||||
private onLoad = () => {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ export class TFRecordAsset extends React.Component<IAssetProps, ITFRecordState>
|
|||
<img ref={this.image}
|
||||
src={this.state.tfRecordImage64}
|
||||
onLoad={this.onLoad}
|
||||
onError={this.onError} />
|
||||
onError={this.onError}
|
||||
crossOrigin="anonymous" />
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,7 +73,8 @@ export class VideoAsset extends React.Component<IVideoAssetProps> {
|
|||
height="100%"
|
||||
autoPlay={autoPlay}
|
||||
src={videoPath}
|
||||
onError={this.props.onError}>
|
||||
onError={this.props.onError}
|
||||
crossOrigin="anonymous">
|
||||
<BigPlayButton position="center" />
|
||||
{autoPlay &&
|
||||
<ControlBar autoHide={false}>
|
||||
|
|
|
|||
|
|
@ -5,9 +5,7 @@ import ExternalPicker, { IExternalPickerProps, IExternalPickerState, FilterOpera
|
|||
import MockFactory from "../../../../common/mockFactory";
|
||||
|
||||
describe("External Picker", () => {
|
||||
const onChangeHandler = jest.fn(() => {
|
||||
console.log("hi");
|
||||
});
|
||||
const onChangeHandler = jest.fn();
|
||||
const defaultProps = createProps({
|
||||
id: "my-custom-control",
|
||||
value: "",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"modelPathType": {
|
||||
"type": "string",
|
||||
"title": "${strings.activeLearning.form.properties.modelPathType.title}",
|
||||
"description": "${strings.activeLearning.form.properties.modelPathType.description}",
|
||||
"enum": [
|
||||
"coco",
|
||||
"file",
|
||||
"url"
|
||||
],
|
||||
"default": "coco",
|
||||
"enumNames": [
|
||||
"${strings.activeLearning.form.properties.modelPathType.options.preTrained}",
|
||||
"${strings.activeLearning.form.properties.modelPathType.options.customFilePath}",
|
||||
"${strings.activeLearning.form.properties.modelPathType.options.customWebUrl}"
|
||||
]
|
||||
},
|
||||
"autoDetect": {
|
||||
"title": "${strings.activeLearning.form.properties.autoDetect.title}",
|
||||
"description": "${strings.activeLearning.form.properties.autoDetect.description}",
|
||||
"type": "boolean"
|
||||
},
|
||||
"predictTag": {
|
||||
"title": " ${strings.activeLearning.form.properties.predictTag.title}",
|
||||
"description": "${strings.activeLearning.form.properties.predictTag.description}",
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"modelPathType": {
|
||||
"oneOf": [
|
||||
{
|
||||
"properties": {
|
||||
"modelPathType": {
|
||||
"enum": [
|
||||
"coco"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"required": [
|
||||
"modelPath"
|
||||
],
|
||||
"properties": {
|
||||
"modelPathType": {
|
||||
"enum": [
|
||||
"file"
|
||||
]
|
||||
},
|
||||
"modelPath": {
|
||||
"title": "${strings.activeLearning.form.properties.modelPath.title}",
|
||||
"description": "${strings.activeLearning.form.properties.modelPath.description}",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"required": [
|
||||
"modelUrl"
|
||||
],
|
||||
"properties": {
|
||||
"modelPathType": {
|
||||
"enum": [
|
||||
"url"
|
||||
]
|
||||
},
|
||||
"modelUrl": {
|
||||
"title": "${strings.activeLearning.form.properties.modelUrl.title}",
|
||||
"description": "${strings.activeLearning.form.properties.modelUrl.description}",
|
||||
"default": "http://",
|
||||
"pattern": "^https?\\\\://[a-zA-Z0-9\\\\-\\\\.]+\\\\.[a-zA-Z]{2,3}(/\\\\S*)?$",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
import React from "react";
|
||||
import { IActiveLearningFormProps, ActiveLearningForm, IActiveLearningFormState } from "./activeLearningForm";
|
||||
import { ReactWrapper, mount } from "enzyme";
|
||||
import { ModelPathType, IActiveLearningSettings } from "../../../../models/applicationState";
|
||||
import Form from "react-jsonschema-form";
|
||||
|
||||
describe("Active Learning Form", () => {
|
||||
const onChangeHandler = jest.fn();
|
||||
const onSubmitHandler = jest.fn();
|
||||
const onCancelHandler = jest.fn();
|
||||
const defaultProps: IActiveLearningFormProps = {
|
||||
settings: {
|
||||
modelPathType: ModelPathType.Coco,
|
||||
modelPath: null,
|
||||
modelUrl: null,
|
||||
autoDetect: false,
|
||||
predictTag: true,
|
||||
},
|
||||
onChange: onChangeHandler,
|
||||
onSubmit: onSubmitHandler,
|
||||
onCancel: onCancelHandler,
|
||||
};
|
||||
|
||||
function createComponent(props?: IActiveLearningFormProps)
|
||||
: ReactWrapper<IActiveLearningFormProps, IActiveLearningFormState> {
|
||||
props = props || defaultProps;
|
||||
return mount(<ActiveLearningForm {...props} />);
|
||||
}
|
||||
|
||||
it("renders a dynamic json schema form with default props", () => {
|
||||
const wrapper = createComponent();
|
||||
expect(wrapper.find(Form).exists()).toBe(true);
|
||||
expect(wrapper.state().formData).toEqual(defaultProps.settings);
|
||||
});
|
||||
|
||||
it("sets formData state when loaded with different props", () => {
|
||||
const props: IActiveLearningFormProps = {
|
||||
...defaultProps,
|
||||
settings: {
|
||||
modelPathType: ModelPathType.Url,
|
||||
modelUrl: "https://myserver.com/myModel",
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
},
|
||||
};
|
||||
|
||||
const wrapper = createComponent(props);
|
||||
expect(wrapper.state().formData).toEqual(props.settings);
|
||||
});
|
||||
|
||||
it("updates form data when the props change", () => {
|
||||
const wrapper = createComponent();
|
||||
|
||||
const newSettings: IActiveLearningSettings = {
|
||||
modelPathType: ModelPathType.Url,
|
||||
modelUrl: "https://myserver.com/myModel",
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
};
|
||||
|
||||
wrapper.setProps({ settings: newSettings });
|
||||
expect(wrapper.state().formData).toEqual(newSettings);
|
||||
});
|
||||
|
||||
it("sets formData state when form changes", () => {
|
||||
const wrapper = createComponent();
|
||||
const formData: IActiveLearningSettings = {
|
||||
modelPathType: ModelPathType.Url,
|
||||
modelUrl: "https://myserver.com/myModel",
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
};
|
||||
|
||||
// Set type to URL
|
||||
wrapper.find(Form).props().onChange({ formData: { modelPathType: ModelPathType.Url } });
|
||||
// Set the remaining settings
|
||||
wrapper.find(Form).props().onChange({ formData });
|
||||
expect(wrapper.state().formData).toEqual(formData);
|
||||
expect(onChangeHandler).toBeCalledWith(formData);
|
||||
});
|
||||
|
||||
it("submits form data to the registered submit handler", () => {
|
||||
const wrapper = createComponent();
|
||||
wrapper.find(Form).props().onSubmit({ formData: defaultProps.settings });
|
||||
|
||||
expect(onSubmitHandler).toBeCalledWith(defaultProps.settings);
|
||||
});
|
||||
|
||||
it("raises the cancel event and called registered handler", () => {
|
||||
const wrapper = createComponent();
|
||||
wrapper.find(".btn-cancel").simulate("click");
|
||||
|
||||
expect(onCancelHandler).toBeCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
import React from "react";
|
||||
import Form, { ISubmitEvent, IChangeEvent, Widget } from "react-jsonschema-form";
|
||||
import { IActiveLearningSettings, ModelPathType } from "../../../../models/applicationState";
|
||||
import { strings, addLocValues } from "../../../../common/strings";
|
||||
import CustomFieldTemplate from "../../common/customField/customFieldTemplate";
|
||||
import LocalFolderPicker from "../../common/localFolderPicker/localFolderPicker";
|
||||
import { CustomWidget } from "../../common/customField/customField";
|
||||
import Checkbox from "rc-checkbox";
|
||||
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const formSchema = addLocValues(require("./activeLearningForm.json"));
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const uiSchema = addLocValues(require("./activeLearningForm.ui.json"));
|
||||
|
||||
export interface IActiveLearningFormProps extends React.Props<ActiveLearningForm> {
|
||||
settings: IActiveLearningSettings;
|
||||
onSubmit: (settings: IActiveLearningSettings) => void;
|
||||
onChange?: (settings: IActiveLearningSettings) => void;
|
||||
onCancel?: () => void;
|
||||
}
|
||||
|
||||
export interface IActiveLearningFormState {
|
||||
classNames: string[];
|
||||
formData: IActiveLearningSettings;
|
||||
uiSchema: any;
|
||||
formSchema: any;
|
||||
}
|
||||
|
||||
export class ActiveLearningForm extends React.Component<IActiveLearningFormProps, IActiveLearningFormState> {
|
||||
public state: IActiveLearningFormState = {
|
||||
classNames: ["needs-validation"],
|
||||
uiSchema: { ...uiSchema },
|
||||
formSchema: { ...formSchema },
|
||||
formData: {
|
||||
...this.props.settings,
|
||||
},
|
||||
};
|
||||
|
||||
private widgets = {
|
||||
localFolderPicker: (LocalFolderPicker as any) as Widget,
|
||||
checkbox: CustomWidget(Checkbox, (props) => ({
|
||||
checked: props.value,
|
||||
onChange: (value) => props.onChange(value.target.checked),
|
||||
disabled: props.disabled,
|
||||
})),
|
||||
};
|
||||
|
||||
public componentDidUpdate(prevProps: Readonly<IActiveLearningFormProps>) {
|
||||
if (this.props.settings !== prevProps.settings) {
|
||||
this.setState({ formData: this.props.settings });
|
||||
}
|
||||
}
|
||||
|
||||
public render() {
|
||||
return (
|
||||
<Form
|
||||
className={this.state.classNames.join(" ")}
|
||||
showErrorList={false}
|
||||
liveValidate={true}
|
||||
noHtml5Validate={true}
|
||||
FieldTemplate={CustomFieldTemplate}
|
||||
widgets={this.widgets}
|
||||
schema={this.state.formSchema}
|
||||
uiSchema={this.state.uiSchema}
|
||||
formData={this.state.formData}
|
||||
onChange={this.onFormChange}
|
||||
onSubmit={this.onFormSubmit}>
|
||||
<div>
|
||||
<button className="btn btn-success mr-1" type="submit">{strings.projectSettings.save}</button>
|
||||
<button className="btn btn-secondary btn-cancel"
|
||||
type="button"
|
||||
onClick={this.onFormCancel}>{strings.common.cancel}</button>
|
||||
</div>
|
||||
</Form>
|
||||
);
|
||||
}
|
||||
|
||||
private onFormChange = (changeEvent: IChangeEvent<IActiveLearningSettings>): void => {
|
||||
let updatedSettings = changeEvent.formData;
|
||||
|
||||
if (changeEvent.formData.modelPathType !== this.state.formData.modelPathType) {
|
||||
updatedSettings = {
|
||||
...changeEvent.formData,
|
||||
modelPath: null,
|
||||
modelUrl: null,
|
||||
};
|
||||
}
|
||||
|
||||
this.setState({
|
||||
formData: updatedSettings,
|
||||
}, () => {
|
||||
if (this.props.onChange) {
|
||||
this.props.onChange(updatedSettings);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private onFormSubmit = (args: ISubmitEvent<IActiveLearningSettings>): void => {
|
||||
const settings: IActiveLearningSettings = {
|
||||
...args.formData,
|
||||
};
|
||||
|
||||
this.setState({ formData: settings });
|
||||
this.props.onSubmit(settings);
|
||||
}
|
||||
|
||||
private onFormCancel = (): void => {
|
||||
if (this.props.onCancel) {
|
||||
this.props.onCancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"modelPath": {
|
||||
"ui:widget": "localFolderPicker"
|
||||
},
|
||||
"predictTag": {
|
||||
"ui:widget": "checkbox"
|
||||
},
|
||||
"autoDetect": {
|
||||
"ui:widget": "checkbox"
|
||||
},
|
||||
"ui:order": [
|
||||
"modelPathType",
|
||||
"*",
|
||||
"predictTag",
|
||||
"autoDetect"
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
import React from "react";
|
||||
import ActiveLearningPage, { IActiveLearningPageProps, IActiveLearningPageState } from "./activeLearningPage";
|
||||
import { ReactWrapper, mount } from "enzyme";
|
||||
import { Provider } from "react-redux";
|
||||
import { BrowserRouter as Router } from "react-router-dom";
|
||||
import createReduxStore from "../../../../redux/store/store";
|
||||
import MockFactory from "../../../../common/mockFactory";
|
||||
import { ActiveLearningForm } from "./activeLearningForm";
|
||||
import { IActiveLearningSettings, ModelPathType } from "../../../../models/applicationState";
|
||||
jest.mock("../../../../services/projectService");
|
||||
import ProjectService from "../../../../services/projectService";
|
||||
import { toast } from "react-toastify";
|
||||
import { strings } from "../../../../common/strings";
|
||||
|
||||
describe("Active Learning Page", () => {
|
||||
function createComponent(store, props: IActiveLearningPageProps): ReactWrapper {
|
||||
return mount(
|
||||
<Provider store={store}>
|
||||
<Router>
|
||||
<ActiveLearningPage {...props} />
|
||||
</Router>
|
||||
</Provider>,
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
toast.success = jest.fn(() => 2);
|
||||
});
|
||||
|
||||
it("renders and loads settings from props", () => {
|
||||
const testProject = MockFactory.createTestProject("TestProject");
|
||||
const store = createReduxStore(MockFactory.initialState({
|
||||
currentProject: testProject,
|
||||
}));
|
||||
|
||||
const props = MockFactory.activeLearningProps();
|
||||
const wrapper = createComponent(store, props);
|
||||
|
||||
const activeLearningPage = wrapper
|
||||
.find(ActiveLearningPage)
|
||||
.childAt(0) as ReactWrapper<IActiveLearningPageProps, IActiveLearningPageState>;
|
||||
|
||||
expect(activeLearningPage.state().settings).toEqual(testProject.activeLearningSettings);
|
||||
expect(wrapper.find(ActiveLearningForm).props().settings).toEqual(testProject.activeLearningSettings);
|
||||
});
|
||||
|
||||
it("updates active learning settings if project changes", () => {
|
||||
const store = createReduxStore(MockFactory.initialState());
|
||||
const props = MockFactory.activeLearningProps();
|
||||
|
||||
const testProject = props.recentProjects[0];
|
||||
const wrapper = createComponent(store, props);
|
||||
|
||||
const activeLearningPage = wrapper
|
||||
.find(ActiveLearningPage)
|
||||
.childAt(0) as ReactWrapper<IActiveLearningPageProps, IActiveLearningPageState>;
|
||||
|
||||
expect(activeLearningPage.state().settings).toEqual(testProject.activeLearningSettings);
|
||||
expect(wrapper.find(ActiveLearningForm).props().settings).toEqual(testProject.activeLearningSettings);
|
||||
});
|
||||
|
||||
it("saves the active learning settings when the form is submitted", async () => {
|
||||
const testProject = MockFactory.createTestProject("TestProject");
|
||||
const activeLearningSettings: IActiveLearningSettings = {
|
||||
...testProject.activeLearningSettings,
|
||||
modelPathType: ModelPathType.Url,
|
||||
modelUrl: "http://myserver.com/custommodel",
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
};
|
||||
|
||||
const store = createReduxStore(MockFactory.initialState({
|
||||
currentProject: testProject,
|
||||
}));
|
||||
|
||||
const projectServiceMock = ProjectService as jest.Mocked<typeof ProjectService>;
|
||||
projectServiceMock.prototype.load = jest.fn((project) => Promise.resolve(project));
|
||||
projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
|
||||
|
||||
const props = MockFactory.activeLearningProps();
|
||||
const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
|
||||
saveProjectSpy.mockClear();
|
||||
|
||||
const wrapper = createComponent(store, props);
|
||||
|
||||
const activeLearningForm = wrapper.find(ActiveLearningForm);
|
||||
activeLearningForm.props().onSubmit(activeLearningSettings);
|
||||
|
||||
await MockFactory.flushUi();
|
||||
|
||||
expect(saveProjectSpy).toBeCalledWith(expect.objectContaining({
|
||||
...testProject,
|
||||
activeLearningSettings,
|
||||
}));
|
||||
|
||||
expect(toast.success).toBeCalledWith(strings.activeLearning.messages.saveSuccess);
|
||||
expect(props.history.goBack).toBeCalled();
|
||||
});
|
||||
|
||||
it("returns to the previous page when the form is cancelled", async () => {
|
||||
const testProject = MockFactory.createTestProject("TestProject");
|
||||
const store = createReduxStore(MockFactory.initialState({
|
||||
currentProject: testProject,
|
||||
}));
|
||||
|
||||
const props = MockFactory.activeLearningProps();
|
||||
const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
|
||||
saveProjectSpy.mockClear();
|
||||
|
||||
const wrapper = createComponent(store, props);
|
||||
|
||||
wrapper.find(ActiveLearningForm).props().onCancel();
|
||||
|
||||
await MockFactory.flushUi();
|
||||
expect(props.history.goBack).toBeCalled();
|
||||
expect(saveProjectSpy).not.toBeCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
import React from "react";
|
||||
import { connect } from "react-redux";
|
||||
import { RouteComponentProps } from "react-router";
|
||||
import { bindActionCreators } from "redux";
|
||||
import { IActiveLearningSettings, IProject, IApplicationState } from "../../../../models/applicationState";
|
||||
import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
|
||||
import { strings } from "../../../../common/strings";
|
||||
import { ActiveLearningForm } from "./activeLearningForm";
|
||||
import { toast } from "react-toastify";
|
||||
|
||||
export interface IActiveLearningPageProps extends RouteComponentProps, React.Props<ActiveLearningPage> {
|
||||
project: IProject;
|
||||
recentProjects: IProject[];
|
||||
actions: IProjectActions;
|
||||
}
|
||||
|
||||
export interface IActiveLearningPageState {
|
||||
settings: IActiveLearningSettings;
|
||||
}
|
||||
|
||||
function mapStateToProps(state: IApplicationState) {
|
||||
return {
|
||||
project: state.currentProject,
|
||||
recentProjects: state.recentProjects,
|
||||
};
|
||||
}
|
||||
|
||||
function mapDispatchToProps(dispatch) {
|
||||
return {
|
||||
actions: bindActionCreators(projectActions, dispatch),
|
||||
};
|
||||
}
|
||||
|
||||
@connect(mapStateToProps, mapDispatchToProps)
|
||||
export default class ActiveLearningPage extends React.Component<IActiveLearningPageProps, IActiveLearningPageState> {
|
||||
public state: IActiveLearningPageState = {
|
||||
settings: this.props.project ? this.props.project.activeLearningSettings : null,
|
||||
};
|
||||
|
||||
public async componentDidMount() {
|
||||
const projectId = this.props.match.params["projectId"];
|
||||
// If we are creating a new project check to see if there is a partial
|
||||
// project already created in local storage
|
||||
if (!this.props.project && projectId) {
|
||||
const projectToLoad = this.props.recentProjects.find((project) => project.id === projectId);
|
||||
if (projectToLoad) {
|
||||
await this.props.actions.loadProject(projectToLoad);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public componentDidUpdate(prevProps: Readonly<IActiveLearningPageProps>) {
|
||||
if (prevProps.project !== this.props.project) {
|
||||
this.setState({ settings: this.props.project.activeLearningSettings });
|
||||
}
|
||||
}
|
||||
|
||||
public render() {
|
||||
return (
|
||||
<div className="project-settings-page">
|
||||
<div className="project-settings-page-settings m-3">
|
||||
<h3>
|
||||
<i className="fas fa-graduation-cap" />
|
||||
<span className="px-2">
|
||||
{strings.activeLearning.title}
|
||||
</span>
|
||||
</h3>
|
||||
<div className="m-3">
|
||||
<ActiveLearningForm
|
||||
settings={this.state.settings}
|
||||
onSubmit={this.onFormSubmit}
|
||||
onCancel={this.onFormCancel} />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
private onFormSubmit = async (settings: IActiveLearningSettings): Promise<void> => {
|
||||
const updatedProject: IProject = {
|
||||
...this.props.project,
|
||||
activeLearningSettings: settings,
|
||||
};
|
||||
|
||||
await this.props.actions.saveProject(updatedProject);
|
||||
toast.success(strings.activeLearning.messages.saveSuccess);
|
||||
this.props.history.goBack();
|
||||
}
|
||||
|
||||
private onFormCancel = (): void => {
|
||||
this.props.history.goBack();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
import React from "react";
|
||||
|
||||
export default class ActiveLearningPage extends React.Component {
|
||||
public render() {
|
||||
return (
|
||||
<div>ActiveLearningPage</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -45,6 +45,7 @@ describe("Editor Canvas", () => {
|
|||
const canvasProps: ICanvasProps = {
|
||||
selectedAsset: getAssetMetadata(),
|
||||
onAssetMetadataChanged: jest.fn(),
|
||||
onCanvasRendered: jest.fn(),
|
||||
editorMode: EditorMode.Rectangle,
|
||||
selectionMode: SelectionMode.RECT,
|
||||
project: MockFactory.createTestProject(),
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ export interface ICanvasProps extends React.Props<Canvas> {
|
|||
children?: ReactElement<AssetPreview>;
|
||||
onAssetMetadataChanged?: (assetMetadata: IAssetMetadata) => void;
|
||||
onSelectedRegionsChanged?: (regions: IRegion[]) => void;
|
||||
onCanvasRendered?: (canvas: HTMLCanvasElement) => void;
|
||||
}
|
||||
|
||||
export interface ICanvasState {
|
||||
|
|
@ -455,6 +456,11 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
|
|||
private setContentSource = async (contentSource: ContentSource) => {
|
||||
try {
|
||||
await this.editor.addContentSource(contentSource as any);
|
||||
|
||||
if (this.props.onCanvasRendered) {
|
||||
const canvas = this.canvasZone.current.querySelector("canvas");
|
||||
this.props.onCanvasRendered(canvas);
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn(e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import EditorPage, { IEditorPageProps, IEditorPageState } from "./editorPage";
|
|||
import MockFactory from "../../../../common/mockFactory";
|
||||
import {
|
||||
IApplicationState, IAssetMetadata, IProject,
|
||||
EditorMode, IAsset, AssetState, AssetType, ISize,
|
||||
EditorMode, IAsset, AssetState, ISize, IActiveLearningSettings, ModelPathType,
|
||||
} from "../../../../models/applicationState";
|
||||
import { AssetProviderFactory } from "../../../../providers/storage/assetProviderFactory";
|
||||
import createReduxStore from "../../../../redux/store/store";
|
||||
|
|
@ -31,6 +31,9 @@ import EditorSideBar from "./editorSideBar";
|
|||
import Alert from "../../common/alert/alert";
|
||||
import registerMixins from "../../../../registerMixins";
|
||||
import { TagInput } from "../../common/tagInput/tagInput";
|
||||
import { EditorToolbar } from "./editorToolbar";
|
||||
import { ToolbarItem } from "../../toolbar/toolbarItem";
|
||||
import { ActiveLearningService } from "../../../../services/activeLearningService";
|
||||
|
||||
function createComponent(store, props: IEditorPageProps): ReactWrapper<IEditorPageProps, IEditorPageState, EditorPage> {
|
||||
return mount(
|
||||
|
|
@ -60,9 +63,20 @@ describe("Editor Page Component", () => {
|
|||
let assetServiceMock: jest.Mocked<typeof AssetService> = null;
|
||||
let projectServiceMock: jest.Mocked<typeof ProjectService> = null;
|
||||
|
||||
const electronMock = {
|
||||
remote: {
|
||||
app: {
|
||||
getAppPath: jest.fn(() => ""),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const testAssets: IAsset[] = MockFactory.createTestAssets(5);
|
||||
|
||||
beforeAll(() => {
|
||||
registerToolbar();
|
||||
window["require"] = jest.fn(() => electronMock);
|
||||
|
||||
const editorMock = Editor as any;
|
||||
editorMock.prototype.addContentSource = jest.fn(() => Promise.resolve());
|
||||
editorMock.prototype.scaleRegionToSourceSize = jest.fn((regionData: any) => regionData);
|
||||
|
|
@ -334,45 +348,6 @@ describe("Editor Page Component", () => {
|
|||
expect(saveProjectSpy).toBeCalledWith(expect.objectContaining(partialProject));
|
||||
});
|
||||
|
||||
describe("Editor Page Component Forcing Tag Scenario", () => {
|
||||
it("Detect new Tag from asset metadata when selecting the Asset", async () => {
|
||||
const getAssetMetadataMock = assetServiceMock.prototype.getAssetMetadata as jest.Mock;
|
||||
getAssetMetadataMock.mockImplementationOnce((asset) => {
|
||||
const assetMetadata: IAssetMetadata = {
|
||||
asset: { ...asset },
|
||||
regions: [{ ...MockFactory.createTestRegion(), tags: ["NEWTAG"] }],
|
||||
version: appInfo.version,
|
||||
};
|
||||
return Promise.resolve(assetMetadata);
|
||||
});
|
||||
|
||||
// create test project and asset
|
||||
const testProject = MockFactory.createTestProject("TestProject");
|
||||
|
||||
// mock store and props
|
||||
const store = createStore(testProject, true);
|
||||
const props = MockFactory.editorPageProps(testProject.id);
|
||||
|
||||
const saveProjectSpy = jest.spyOn(props.actions, "saveProject");
|
||||
|
||||
// create mock editor page
|
||||
createComponent(store, props);
|
||||
|
||||
const partialProjectToBeSaved = {
|
||||
id: testProject.id,
|
||||
name: testProject.name,
|
||||
tags: expect.arrayContaining([{
|
||||
name: "NEWTAG",
|
||||
color: expect.any(String),
|
||||
}]),
|
||||
};
|
||||
|
||||
await MockFactory.flushUi();
|
||||
|
||||
expect(saveProjectSpy).toBeCalledWith(expect.objectContaining(partialProjectToBeSaved));
|
||||
});
|
||||
});
|
||||
|
||||
it("When an image is updated the asset metadata is updated", async () => {
|
||||
const testProject = MockFactory.createTestProject("TestProject");
|
||||
const store = createStore(testProject, true);
|
||||
|
|
@ -498,7 +473,6 @@ describe("Editor Page Component", () => {
|
|||
const removeAllRegionsConfirm = jest.fn();
|
||||
|
||||
beforeAll(() => {
|
||||
registerToolbar();
|
||||
const clipboard = (navigator as any).clipboard;
|
||||
if (!(clipboard && clipboard.writeText)) {
|
||||
(navigator as any).clipboard = {
|
||||
|
|
@ -826,6 +800,72 @@ describe("Editor Page Component", () => {
|
|||
}));
|
||||
});
|
||||
});
|
||||
|
||||
describe("Active Learning", async () => {
|
||||
let wrapper: ReactWrapper;
|
||||
let editorPage: ReactWrapper<IEditorPageProps, IEditorPageState>;
|
||||
const activeLearningMock = ActiveLearningService as jest.Mocked<typeof ActiveLearningService>;
|
||||
|
||||
async function beforeActiveLearningTest(activeLearningSettings?: IActiveLearningSettings) {
|
||||
document.querySelector = MockFactory.mockCanvas();
|
||||
activeLearningMock.prototype.isModelLoaded = jest.fn(() => true);
|
||||
activeLearningMock.prototype.predictRegions = jest.fn((canvas, assetMetadtata) => {
|
||||
return Promise.resolve({
|
||||
...assetMetadtata,
|
||||
predicted: true,
|
||||
});
|
||||
});
|
||||
const project = MockFactory.createTestProject();
|
||||
|
||||
if (activeLearningSettings) {
|
||||
project.activeLearningSettings = activeLearningSettings;
|
||||
}
|
||||
|
||||
const store = createReduxStore({
|
||||
...MockFactory.initialState(),
|
||||
currentProject: project,
|
||||
});
|
||||
|
||||
wrapper = createComponent(store, MockFactory.editorPageProps());
|
||||
await waitForSelectedAsset(wrapper);
|
||||
wrapper.update();
|
||||
editorPage = wrapper.find(EditorPage).childAt(0);
|
||||
}
|
||||
|
||||
it("predicts regions when auto detect has been enabled", async () => {
|
||||
const activeLearningSettings: IActiveLearningSettings = {
|
||||
modelPathType: ModelPathType.Coco,
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
};
|
||||
|
||||
await beforeActiveLearningTest(activeLearningSettings);
|
||||
|
||||
editorPage.find(Canvas).props().onCanvasRendered(document.createElement("canvas"));
|
||||
expect(activeLearningMock.prototype.predictRegions).toBeCalled();
|
||||
});
|
||||
|
||||
it("predicts regions when toolbar item is selected", async () => {
|
||||
await beforeActiveLearningTest();
|
||||
|
||||
const toolbarItem = {
|
||||
props: {
|
||||
name: ToolbarItemName.ActiveLearning,
|
||||
},
|
||||
};
|
||||
|
||||
const selectedAsset = editorPage.state().selectedAsset;
|
||||
wrapper.find(EditorToolbar).props().onToolbarItemSelected(toolbarItem as ToolbarItem);
|
||||
|
||||
await MockFactory.flushUi();
|
||||
|
||||
expect(activeLearningMock.prototype.predictRegions).toBeCalledWith(expect.anything(), selectedAsset);
|
||||
expect(assetServiceMock.prototype.save).toBeCalledWith({
|
||||
...selectedAsset,
|
||||
predicted: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
function createStore(project: IProject, setCurrentProject: boolean = false): Store<any, AnyAction> {
|
||||
|
|
|
|||
|
|
@ -10,14 +10,14 @@ import { strings } from "../../../../common/strings";
|
|||
import {
|
||||
AssetState, AssetType, EditorMode, IApplicationState,
|
||||
IAppSettings, IAsset, IAssetMetadata, IProject, IRegion,
|
||||
ISize, ITag,
|
||||
ISize, ITag, IAdditionalPageSettings, AppError, ErrorCode,
|
||||
} from "../../../../models/applicationState";
|
||||
import { IToolbarItemRegistration, ToolbarItemFactory } from "../../../../providers/toolbar/toolbarItemFactory";
|
||||
import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions";
|
||||
import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
|
||||
import { ToolbarItemName } from "../../../../registerToolbar";
|
||||
import { AssetService } from "../../../../services/assetService";
|
||||
import { AssetPreview, IAssetPreviewSettings } from "../../common/assetPreview/assetPreview";
|
||||
import { AssetPreview } from "../../common/assetPreview/assetPreview";
|
||||
import { KeyboardBinding } from "../../common/keyboardBinding/keyboardBinding";
|
||||
import { KeyEventType } from "../../common/keyboardManager/keyboardManager";
|
||||
import { TagInput } from "../../common/tagInput/tagInput";
|
||||
|
|
@ -29,8 +29,8 @@ import EditorSideBar from "./editorSideBar";
|
|||
import { EditorToolbar } from "./editorToolbar";
|
||||
import Alert from "../../common/alert/alert";
|
||||
import Confirm from "../../common/confirm/confirm";
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const tagColors = require("../../common/tagColors.json");
|
||||
import { ActiveLearningService } from "../../../../services/activeLearningService";
|
||||
import { toast } from "react-toastify";
|
||||
|
||||
/**
|
||||
* Properties for Editor Page
|
||||
|
|
@ -64,7 +64,7 @@ export interface IEditorPageState {
|
|||
/** The child assets used for nest asset typs */
|
||||
childAssets?: IAsset[];
|
||||
/** Additional settings for asset previews */
|
||||
additionalSettings?: IAssetPreviewSettings;
|
||||
additionalSettings?: IAdditionalPageSettings;
|
||||
/** Most recently selected tag */
|
||||
selectedTag: string;
|
||||
/** Tags locked for region labeling */
|
||||
|
|
@ -101,7 +101,6 @@ function mapDispatchToProps(dispatch) {
|
|||
*/
|
||||
@connect(mapStateToProps, mapDispatchToProps)
|
||||
export default class EditorPage extends React.Component<IEditorPageProps, IEditorPageState> {
|
||||
|
||||
public state: IEditorPageState = {
|
||||
selectedTag: null,
|
||||
lockedTags: [],
|
||||
|
|
@ -109,12 +108,16 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
assets: [],
|
||||
childAssets: [],
|
||||
editorMode: EditorMode.Rectangle,
|
||||
additionalSettings: { videoSettings: (this.props.project) ? this.props.project.videoSettings : null },
|
||||
additionalSettings: {
|
||||
videoSettings: (this.props.project) ? this.props.project.videoSettings : null,
|
||||
activeLearningSettings: (this.props.project) ? this.props.project.activeLearningSettings : null,
|
||||
},
|
||||
thumbnailSize: this.props.appSettings.thumbnailSize || { width: 175, height: 155 },
|
||||
isValid: true,
|
||||
showInvalidRegionWarning: false,
|
||||
};
|
||||
|
||||
private activeLearningService: ActiveLearningService = null;
|
||||
private loadingProjectAssets: boolean = false;
|
||||
private toolbarItems: IToolbarItemRegistration[] = ToolbarItemFactory.getToolbarItems();
|
||||
private canvas: RefObject<Canvas> = React.createRef();
|
||||
|
|
@ -129,6 +132,8 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
const project = this.props.recentProjects.find((project) => project.id === projectId);
|
||||
await this.props.actions.loadProject(project);
|
||||
}
|
||||
|
||||
this.activeLearningService = new ActiveLearningService(this.props.project.activeLearningSettings);
|
||||
}
|
||||
|
||||
public async componentDidUpdate(prevProps: Readonly<IEditorPageProps>) {
|
||||
|
|
@ -143,6 +148,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
this.setState({
|
||||
additionalSettings: {
|
||||
videoSettings: (this.props.project) ? this.props.project.videoSettings : null,
|
||||
activeLearningSettings: (this.props.project) ? this.props.project.activeLearningSettings : null,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -211,6 +217,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
ref={this.canvas}
|
||||
selectedAsset={this.state.selectedAsset}
|
||||
onAssetMetadataChanged={this.onAssetMetadataChanged}
|
||||
onCanvasRendered={this.onCanvasRendered}
|
||||
onSelectedRegionsChanged={this.onSelectedRegionsChanged}
|
||||
editorMode={this.state.editorMode}
|
||||
selectionMode={this.state.selectionMode}
|
||||
|
|
@ -479,6 +486,17 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
this.setState({ childAssets, assets, isValid: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Raised when the asset binary has been painted onto the canvas tools rendering canvas
|
||||
*/
|
||||
private onCanvasRendered = async (canvas: HTMLCanvasElement) => {
|
||||
// When active learning auto-detect is enabled
|
||||
// run predictions when asset changes
|
||||
if (this.props.project.activeLearningSettings.autoDetect && !this.state.selectedAsset.asset.predicted) {
|
||||
await this.predictRegions(canvas);
|
||||
}
|
||||
}
|
||||
|
||||
private onSelectedRegionsChanged = (selectedRegions: IRegion[]) => {
|
||||
this.setState({ selectedRegions });
|
||||
}
|
||||
|
|
@ -540,6 +558,41 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
case ToolbarItemName.RemoveAllRegions:
|
||||
this.canvas.current.confirmRemoveAllRegions();
|
||||
break;
|
||||
case ToolbarItemName.ActiveLearning:
|
||||
await this.predictRegions();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private predictRegions = async (canvas?: HTMLCanvasElement) => {
|
||||
canvas = canvas || document.querySelector("canvas");
|
||||
if (!canvas) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Load the configured ML model
|
||||
if (!this.activeLearningService.isModelLoaded()) {
|
||||
let toastId: number = null;
|
||||
try {
|
||||
toastId = toast.info(strings.activeLearning.messages.loadingModel, { autoClose: false });
|
||||
await this.activeLearningService.ensureModelLoaded();
|
||||
} catch (e) {
|
||||
toast.error(strings.activeLearning.messages.errorLoadModel);
|
||||
return;
|
||||
} finally {
|
||||
toast.dismiss(toastId);
|
||||
}
|
||||
}
|
||||
|
||||
// Predict and add regions to current asset
|
||||
try {
|
||||
const updatedAssetMetadata = await this.activeLearningService
|
||||
.predictRegions(canvas, this.state.selectedAsset);
|
||||
|
||||
await this.onAssetMetadataChanged(updatedAssetMetadata);
|
||||
this.setState({ selectedAsset: updatedAssetMetadata });
|
||||
} catch (e) {
|
||||
throw new AppError(ErrorCode.ActiveLearningPredictionError, "Error predicting regions");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -579,7 +632,6 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
}
|
||||
|
||||
const assetMetadata = await this.props.actions.loadAssetMetadata(this.props.project, asset);
|
||||
await this.updateProjectTagsFromAsset(assetMetadata);
|
||||
|
||||
try {
|
||||
if (!assetMetadata.asset.size) {
|
||||
|
|
@ -597,32 +649,6 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
|
|||
});
|
||||
}
|
||||
|
||||
private async updateProjectTagsFromAsset(asset: IAssetMetadata) {
|
||||
const assetTags = new Set();
|
||||
asset.regions.forEach((region) => region.tags.forEach((tag) => assetTags.add(tag)));
|
||||
|
||||
const newTags: ITag[] = this.props.project.tags ? [...this.props.project.tags] : [];
|
||||
let updateTags = false;
|
||||
|
||||
assetTags.forEach((tag) => {
|
||||
if (!this.props.project.tags || this.props.project.tags.length === 0 ||
|
||||
!this.props.project.tags.find((projectTag) => tag === projectTag.name)) {
|
||||
newTags.push({
|
||||
name: tag,
|
||||
color: tagColors[newTags.length % tagColors.length],
|
||||
});
|
||||
updateTags = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (updateTags) {
|
||||
asset.asset.state = AssetState.Tagged;
|
||||
const newProject = { ...this.props.project, tags: newTags };
|
||||
await this.props.actions.saveAssetMetadata(newProject, asset);
|
||||
await this.props.actions.saveProject(newProject);
|
||||
}
|
||||
}
|
||||
|
||||
private loadProjectAssets = async (): Promise<void> => {
|
||||
if (this.loadingProjectAssets || this.state.assets.length > 0) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -116,7 +116,6 @@ export default class ExportForm extends React.Component<IExportFormProps, IExpor
|
|||
if (providerType !== this.state.providerName) {
|
||||
this.bindForm(args.formData, true);
|
||||
} else {
|
||||
console.log(args.formData);
|
||||
this.bindForm(args.formData, false);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import React from "react";
|
||||
|
||||
export default class ProfileSettingsPage extends React.Component {
|
||||
public render() {
|
||||
return (
|
||||
<div>ProfileSettingsPage</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import React from "react";
|
||||
import Form, { FormValidation, ISubmitEvent, IChangeEvent } from "react-jsonschema-form";
|
||||
import Form, { FormValidation, ISubmitEvent, IChangeEvent, Widget } from "react-jsonschema-form";
|
||||
import { ITagsInputProps, TagEditorModal, TagsInput } from "vott-react";
|
||||
import { addLocValues, strings } from "../../../../common/strings";
|
||||
import { IConnection, IProject, ITag, IAppSettings } from "../../../../models/applicationState";
|
||||
|
|
@ -10,6 +10,7 @@ import CustomFieldTemplate from "../../common/customField/customFieldTemplate";
|
|||
import { ISecurityTokenPickerProps, SecurityTokenPicker } from "../../common/securityTokenPicker/securityTokenPicker";
|
||||
import "vott-react/dist/css/tagsInput.css";
|
||||
import { IConnectionProviderPickerProps } from "../../common/connectionProviderPicker/connectionProviderPicker";
|
||||
import LocalFolderPicker from "../../common/localFolderPicker/localFolderPicker";
|
||||
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const formSchema = addLocValues(require("./projectForm.json"));
|
||||
|
|
@ -51,6 +52,10 @@ export interface IProjectFormState {
|
|||
* @description - Form for editing or creating VoTT projects
|
||||
*/
|
||||
export default class ProjectForm extends React.Component<IProjectFormProps, IProjectFormState> {
|
||||
private widgets = {
|
||||
localFolderPicker: (LocalFolderPicker as any) as Widget,
|
||||
};
|
||||
|
||||
private tagsInput: React.RefObject<TagsInput>;
|
||||
private tagEditorModal: React.RefObject<TagEditorModal>;
|
||||
|
||||
|
|
@ -95,6 +100,7 @@ export default class ProjectForm extends React.Component<IProjectFormProps, IPro
|
|||
FieldTemplate={CustomFieldTemplate}
|
||||
validate={this.onFormValidate}
|
||||
fields={this.fields()}
|
||||
widgets={this.widgets}
|
||||
schema={this.state.formSchema}
|
||||
uiSchema={this.state.uiSchema}
|
||||
formData={this.state.formData}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import MainContentRouter from "./mainContentRouter";
|
|||
import HomePage, { IHomePageProps } from "./../pages/homepage/homePage";
|
||||
import SettingsPage from "./../pages/appSettings/appSettingsPage";
|
||||
import ConnectionsPage from "./../pages/connections/connectionsPage";
|
||||
import ProfilePage from "./../pages/profileSettingsPage";
|
||||
import { IApplicationState } from "./../../../models/applicationState";
|
||||
|
||||
describe("Main Content Router", () => {
|
||||
|
|
@ -43,7 +42,6 @@ describe("Main Content Router", () => {
|
|||
expect(pathMap["/"]).toBe(HomePage);
|
||||
expect(pathMap["/settings"]).toBe(SettingsPage);
|
||||
expect(pathMap["/connections"]).toBe(ConnectionsPage);
|
||||
expect(pathMap["/profile"]).toBe(ProfilePage);
|
||||
});
|
||||
|
||||
it("renders a redirect when no route is matched", () => {
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
import React from "react";
|
||||
import { Switch, Route, Redirect } from "react-router-dom";
|
||||
import { Switch, Route } from "react-router-dom";
|
||||
import HomePage from "../pages/homepage/homePage";
|
||||
import ActiveLearningPage from "../pages/activeLearningPage";
|
||||
import ActiveLearningPage from "../pages/activeLearning/activeLearningPage";
|
||||
import AppSettingsPage from "../pages/appSettings/appSettingsPage";
|
||||
import ConnectionPage from "../pages/connections/connectionsPage";
|
||||
import EditorPage from "../pages/editorPage/editorPage";
|
||||
import ExportPage from "../pages/export/exportPage";
|
||||
import ProjectSettingsPage from "../pages/projectSettings/projectSettingsPage";
|
||||
import ProfileSettingsPage from "../pages/profileSettingsPage";
|
||||
|
||||
/**
|
||||
* @name - Main Content Router
|
||||
|
|
@ -19,7 +18,6 @@ export default function MainContentRouter() {
|
|||
<Switch>
|
||||
<Route path="/" exact component={HomePage} />
|
||||
<Route path="/settings" component={AppSettingsPage} />
|
||||
<Route path="/profile" component={ProfileSettingsPage} />
|
||||
<Route path="/connections/:connectionId" component={ConnectionPage} />
|
||||
<Route path="/connections" exact component={ConnectionPage} />
|
||||
<Route path="/projects/:projectId/edit" component={EditorPage} />
|
||||
|
|
|
|||
|
|
@ -16,6 +16,6 @@ describe("Sidebar Component", () => {
|
|||
expect(wrapper).not.toBeNull();
|
||||
|
||||
const links = wrapper.find("ul li");
|
||||
expect(links.length).toEqual(6);
|
||||
expect(links.length).toEqual(7);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -39,7 +39,16 @@ export default function Sidebar({ project }) {
|
|||
<ConditionalNavLink disabled={!projectId}
|
||||
title={strings.export.title}
|
||||
to={`/projects/${projectId}/export`}>
|
||||
<i className="fas fa-external-link-square-alt"></i></ConditionalNavLink></li>
|
||||
<i className="fas fa-external-link-square-alt"></i>
|
||||
</ConditionalNavLink>
|
||||
</li>
|
||||
<li>
|
||||
<ConditionalNavLink disabled={!projectId}
|
||||
title={strings.activeLearning.title}
|
||||
to={`/projects/${projectId}/active-learning`}>
|
||||
<i className="fas fa-graduation-cap"></i>
|
||||
</ConditionalNavLink>
|
||||
</li>
|
||||
<li>
|
||||
<NavLink title={strings.connections.title}
|
||||
to={`/connections`}><i className="fas fa-plug"></i></NavLink>
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import ProjectService from "../../services/projectService";
|
|||
jest.mock("../../services/assetService");
|
||||
import { AssetService } from "../../services/assetService";
|
||||
import { ExportProviderFactory } from "../../providers/export/exportProviderFactory";
|
||||
import { ExportAssetState, IExportProvider } from "../../providers/export/exportProvider";
|
||||
import { IExportProvider } from "../../providers/export/exportProvider";
|
||||
import { IApplicationState, IProject } from "../../models/applicationState";
|
||||
import initialState from "../store/initialState";
|
||||
import { appInfo } from "../../common/appInfo";
|
||||
|
|
@ -87,40 +87,6 @@ describe("Project Redux Actions", () => {
|
|||
expect(result.version).toEqual(appInfo.version);
|
||||
});
|
||||
|
||||
it("Save Project action on new project correctly add default export format", async () => {
|
||||
projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
|
||||
|
||||
const skeletonProject = MockFactory.createTestProject("TestProject");
|
||||
const project = {
|
||||
...skeletonProject,
|
||||
exportFormat: null,
|
||||
};
|
||||
|
||||
const result = await projectActions.saveProject(project)(store.dispatch, store.getState);
|
||||
|
||||
expect(result.exportFormat).toEqual({
|
||||
providerType: "vottJson",
|
||||
providerOptions: {
|
||||
assetState: ExportAssetState.Visited,
|
||||
includeImages: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("Save Project action on new project correctly set tags to empty if none created", async () => {
|
||||
projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
|
||||
|
||||
const skeletonProject = MockFactory.createTestProject("TestProject");
|
||||
const project = {
|
||||
...skeletonProject,
|
||||
tags: null,
|
||||
};
|
||||
|
||||
const result = await projectActions.saveProject(project)(store.dispatch, store.getState);
|
||||
|
||||
expect(result.tags).toEqual([]);
|
||||
});
|
||||
|
||||
it("Save Project action does not override existing export format", async () => {
|
||||
projectServiceMock.prototype.save = jest.fn((project) => Promise.resolve(project));
|
||||
|
||||
|
|
|
|||
|
|
@ -80,24 +80,7 @@ export function saveProject(project: IProject)
|
|||
throw new AppError(ErrorCode.SecurityTokenNotFound, "Security Token Not Found");
|
||||
}
|
||||
|
||||
const defaultExportProviderOptions: IVottJsonExportProviderOptions = {
|
||||
assetState: ExportAssetState.Visited,
|
||||
includeImages: true,
|
||||
};
|
||||
|
||||
const defaultExportFormat: IExportFormat = {
|
||||
providerType: "vottJson",
|
||||
providerOptions: defaultExportProviderOptions,
|
||||
};
|
||||
|
||||
const newProject = {
|
||||
...project,
|
||||
version: appInfo.version,
|
||||
exportFormat: project.exportFormat || defaultExportFormat,
|
||||
tags: project.tags || [],
|
||||
};
|
||||
|
||||
const savedProject = await projectService.save(newProject, projectToken);
|
||||
const savedProject = await projectService.save(project, projectToken);
|
||||
dispatch(saveProjectAction(savedProject));
|
||||
|
||||
// Reload project after save actions
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import _ from "lodash";
|
||||
import { reducer } from "./currentProjectReducer";
|
||||
import { IProject, IAssetMetadata, AssetState } from "../../models/applicationState";
|
||||
import { IProject, IAssetMetadata, AssetState, ITag } from "../../models/applicationState";
|
||||
import MockFactory from "../../common/mockFactory";
|
||||
import {
|
||||
loadProjectAction,
|
||||
|
|
@ -50,7 +50,7 @@ describe("Current Project Reducer", () => {
|
|||
expect(result).toEqual(currentProject);
|
||||
});
|
||||
|
||||
it("Updating connection used by current project is updated in curren project", () => {
|
||||
it("Updating connection used by current project is updated in current project", () => {
|
||||
const currentProject = MockFactory.createTestProject("1");
|
||||
const state: IProject = currentProject;
|
||||
|
||||
|
|
@ -113,6 +113,29 @@ describe("Current Project Reducer", () => {
|
|||
expect(result.assets[testAssets[0].id]).toEqual(assetMetadata.asset);
|
||||
});
|
||||
|
||||
it("Appends new tags to project when saving asset contains new tags", () => {
|
||||
const state: IProject = MockFactory.createTestProject("TestProject");
|
||||
const testAssets = MockFactory.createTestAssets();
|
||||
|
||||
const expectedTag: ITag = {
|
||||
name: "NEWTAG",
|
||||
color: expect.any(String),
|
||||
};
|
||||
|
||||
const assetMetadata = MockFactory.createTestAssetMetadata(
|
||||
testAssets[0],
|
||||
[MockFactory.createTestRegion("Region 1", [expectedTag.name])],
|
||||
);
|
||||
|
||||
const action = saveAssetMetadataAction(assetMetadata);
|
||||
const result = reducer(state, action);
|
||||
expect(result).not.toBe(state);
|
||||
expect(result.tags).toEqual([
|
||||
...state.tags,
|
||||
expectedTag,
|
||||
]);
|
||||
});
|
||||
|
||||
it("Unknown action performs a noop", () => {
|
||||
const state: IProject = MockFactory.createTestProject("TestProject");
|
||||
const action = anyOtherAction();
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import _ from "lodash";
|
||||
import { ActionTypes } from "../actions/actionTypes";
|
||||
import { IProject } from "../../models/applicationState";
|
||||
import { IProject, ITag } from "../../models/applicationState";
|
||||
import { AnyAction } from "../actions/actionCreators";
|
||||
// tslint:disable-next-line:no-var-requires
|
||||
const tagColors = require("../../react/components/common/tagColors.json");
|
||||
|
||||
/**
|
||||
* Reducer for project. Actions handled:
|
||||
|
|
@ -38,6 +40,31 @@ export const reducer = (state: IProject = null, action: AnyAction): IProject =>
|
|||
const updatedAssets = { ...state.assets } || {};
|
||||
updatedAssets[action.payload.asset.id] = { ...action.payload.asset };
|
||||
|
||||
const assetTags = new Set();
|
||||
action.payload.regions.forEach((region) => region.tags.forEach((tag) => assetTags.add(tag)));
|
||||
|
||||
const newTags: ITag[] = state.tags ? [...state.tags] : [];
|
||||
let updateTags = false;
|
||||
|
||||
assetTags.forEach((tag) => {
|
||||
if (!state.tags || state.tags.length === 0 ||
|
||||
!state.tags.find((projectTag) => tag === projectTag.name)) {
|
||||
newTags.push({
|
||||
name: tag,
|
||||
color: tagColors[newTags.length % tagColors.length],
|
||||
});
|
||||
updateTags = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (updateTags) {
|
||||
return {
|
||||
...state,
|
||||
tags: newTags,
|
||||
assets: updatedAssets,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
assets: updatedAssets,
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ export enum ToolbarItemName {
|
|||
NextAsset = "navigateNextAsset",
|
||||
SaveProject = "saveProject",
|
||||
ExportProject = "exportProject",
|
||||
ActiveLearning = "activeLearning",
|
||||
}
|
||||
|
||||
export enum ToolbarItemGroup {
|
||||
|
|
@ -102,6 +103,15 @@ export default function registerToolbar() {
|
|||
accelerators: ["CmdOrCtrl+Delete", "CmdOrCtrl+Backspace"],
|
||||
});
|
||||
|
||||
ToolbarItemFactory.register({
|
||||
name: ToolbarItemName.ActiveLearning,
|
||||
tooltip: strings.editorPage.toolbar.activeLearning,
|
||||
icon: "fas fa-graduation-cap",
|
||||
group: ToolbarItemGroup.Canvas,
|
||||
type: ToolbarItemType.Action,
|
||||
accelerators: ["CmdOrCtrl+D", "CmdOrCtrl+d"],
|
||||
});
|
||||
|
||||
ToolbarItemFactory.register({
|
||||
name: ToolbarItemName.PreviousAsset,
|
||||
tooltip: strings.editorPage.toolbar.previousAsset,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
import { ActiveLearningService } from "./activeLearningService";
|
||||
import { IActiveLearningSettings, ModelPathType, IAssetMetadata, AssetState } from "../models/applicationState";
|
||||
import MockFactory from "../common/mockFactory";
|
||||
import { appInfo } from "../common/appInfo";
|
||||
import { ObjectDetection } from "../providers/activeLearning/objectDetection";
|
||||
|
||||
describe("Active Learning Service", () => {
|
||||
const objectDetectionMock = ObjectDetection as jest.Mocked<typeof ObjectDetection>;
|
||||
const defaultSettings: IActiveLearningSettings = {
|
||||
modelPathType: ModelPathType.Coco,
|
||||
autoDetect: true,
|
||||
predictTag: true,
|
||||
};
|
||||
|
||||
let activeLearningService: ActiveLearningService = null;
|
||||
|
||||
const electronMock = {
|
||||
remote: {
|
||||
app: {
|
||||
getAppPath: jest.fn(),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
window["require"] = jest.fn(() => electronMock);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
activeLearningService = new ActiveLearningService(defaultSettings);
|
||||
objectDetectionMock.prototype.load = jest.fn(() => Promise.resolve());
|
||||
objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve([]));
|
||||
});
|
||||
|
||||
it("Predicts new regions to the asset metadata", async () => {
|
||||
objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve(expectedRegions));
|
||||
|
||||
const expectedRegions = MockFactory.createTestRegions(2);
|
||||
const canvas = MockFactory.mockCanvas()();
|
||||
const asset = MockFactory.createTestAsset("TestAsset", AssetState.Visited);
|
||||
const assetMetadata: IAssetMetadata = {
|
||||
asset: {
|
||||
...asset,
|
||||
state: AssetState.Tagged,
|
||||
},
|
||||
regions: [],
|
||||
version: appInfo.version,
|
||||
};
|
||||
|
||||
const updatedAssetMetadata = await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
|
||||
expect(updatedAssetMetadata).toEqual({
|
||||
asset: {
|
||||
...assetMetadata.asset,
|
||||
predicted: true,
|
||||
},
|
||||
regions: expectedRegions,
|
||||
version: appInfo.version,
|
||||
});
|
||||
});
|
||||
|
||||
it("Predicts non matching regions to the asset metadata", async () => {
|
||||
objectDetectionMock.prototype.predictImage = jest.fn(() => Promise.resolve(expectedRegions));
|
||||
|
||||
const uniqueRegion = MockFactory.createTestRegion("UniqueRegion", ["tag1", "tag2"]);
|
||||
const expectedRegions = MockFactory.createTestRegions(4);
|
||||
const canvas = MockFactory.mockCanvas()();
|
||||
const asset = MockFactory.createTestAsset("TestAsset", AssetState.Visited);
|
||||
const assetMetadata: IAssetMetadata = {
|
||||
asset: {
|
||||
...asset,
|
||||
state: AssetState.Tagged,
|
||||
},
|
||||
regions: [
|
||||
uniqueRegion,
|
||||
expectedRegions[0],
|
||||
expectedRegions[1],
|
||||
],
|
||||
version: appInfo.version,
|
||||
};
|
||||
|
||||
const updatedAssetMetadata = await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
|
||||
expect(updatedAssetMetadata).toEqual({
|
||||
asset: {
|
||||
...assetMetadata.asset,
|
||||
predicted: true,
|
||||
},
|
||||
regions: [
|
||||
uniqueRegion,
|
||||
...expectedRegions,
|
||||
],
|
||||
version: appInfo.version,
|
||||
});
|
||||
});
|
||||
|
||||
it("ensures the underlying object detection model is only loaded 1 time", async () => {
|
||||
const canvas = MockFactory.mockCanvas()();
|
||||
const assetMetadata: IAssetMetadata = {
|
||||
asset: MockFactory.createTestAsset("TestAsset", AssetState.Visited),
|
||||
regions: [],
|
||||
version: appInfo.version,
|
||||
};
|
||||
|
||||
await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
await activeLearningService.predictRegions(canvas, assetMetadata);
|
||||
expect(objectDetectionMock.prototype.load).toBeCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fails if constructor requirements aren't satisfied", () => {
|
||||
expect(() => new ActiveLearningService(null)).toThrow();
|
||||
});
|
||||
|
||||
it("fails if method requirements aren't satisfied", () => {
|
||||
const service = new ActiveLearningService(defaultSettings);
|
||||
expect(service.predictRegions(null, null)).rejects.not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
import { IAssetMetadata, ModelPathType, IActiveLearningSettings, AssetState } from "../models/applicationState";
|
||||
import { ObjectDetection } from "../providers/activeLearning/objectDetection";
|
||||
import Guard from "../common/guard";
|
||||
import { isElectron } from "../common/hostProcess";
|
||||
import { Env } from "../common/environment";
|
||||
|
||||
export class ActiveLearningService {
|
||||
private objectDetection: ObjectDetection;
|
||||
private modelLoaded: boolean = false;
|
||||
|
||||
constructor(private settings: IActiveLearningSettings) {
|
||||
Guard.null(settings);
|
||||
this.objectDetection = new ObjectDetection();
|
||||
}
|
||||
|
||||
public isModelLoaded() {
|
||||
return this.modelLoaded;
|
||||
}
|
||||
|
||||
public async predictRegions(canvas: HTMLCanvasElement, assetMetadata: IAssetMetadata): Promise<IAssetMetadata> {
|
||||
Guard.null(canvas);
|
||||
Guard.null(assetMetadata);
|
||||
|
||||
// If the canvas or asset are invalid return asset metadata
|
||||
if (!(canvas.width && canvas.height && assetMetadata.asset && assetMetadata.asset.size)) {
|
||||
return assetMetadata;
|
||||
}
|
||||
|
||||
await this.ensureModelLoaded();
|
||||
|
||||
const xRatio = assetMetadata.asset.size.width / canvas.width;
|
||||
const yRatio = assetMetadata.asset.size.height / canvas.height;
|
||||
const predictedRegions = await this.objectDetection.predictImage(
|
||||
canvas,
|
||||
this.settings.predictTag,
|
||||
xRatio,
|
||||
yRatio,
|
||||
);
|
||||
|
||||
const updatedRegions = [...assetMetadata.regions];
|
||||
predictedRegions.forEach((prediction) => {
|
||||
const matchingRegion = updatedRegions.find((region) => {
|
||||
return region.boundingBox
|
||||
&& region.boundingBox.left === prediction.boundingBox.left
|
||||
&& region.boundingBox.top === prediction.boundingBox.top
|
||||
&& region.boundingBox.width === prediction.boundingBox.width
|
||||
&& region.boundingBox.height === prediction.boundingBox.height;
|
||||
});
|
||||
|
||||
if (updatedRegions.length === 0 || !matchingRegion) {
|
||||
updatedRegions.push(prediction);
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
...assetMetadata,
|
||||
regions: updatedRegions,
|
||||
asset: {
|
||||
...assetMetadata.asset,
|
||||
state: updatedRegions.length > 0 ? AssetState.Tagged : AssetState.Visited,
|
||||
predicted: true,
|
||||
},
|
||||
} as IAssetMetadata;
|
||||
}
|
||||
|
||||
public async ensureModelLoaded(): Promise<void> {
|
||||
if (this.modelLoaded) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
await this.loadModel();
|
||||
this.modelLoaded = true;
|
||||
}
|
||||
|
||||
private async loadModel() {
|
||||
let modelPath = "";
|
||||
if (this.settings.modelPathType === ModelPathType.Coco) {
|
||||
if (isElectron()) {
|
||||
const appPath = this.getAppPath();
|
||||
|
||||
if (Env.get() !== "production") {
|
||||
modelPath = appPath + "/cocoSSDModel";
|
||||
} else {
|
||||
modelPath = appPath + "/../../cocoSSDModel";
|
||||
}
|
||||
} else {
|
||||
modelPath = "https://vott.blob.core.windows.net/coco-ssd-model";
|
||||
}
|
||||
} else if (this.settings.modelPathType === ModelPathType.File) {
|
||||
if (isElectron()) {
|
||||
modelPath = this.settings.modelPath;
|
||||
}
|
||||
} else {
|
||||
modelPath = this.settings.modelUrl;
|
||||
}
|
||||
|
||||
await this.objectDetection.load(modelPath);
|
||||
}
|
||||
|
||||
private getAppPath = () => {
|
||||
const remote = (window as any).require("electron").remote as Electron.Remote;
|
||||
return remote.app.getAppPath();
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import shortid from "shortid";
|
|||
import {
|
||||
IProject, ITag, IConnection, AppError, ErrorCode,
|
||||
IAssetMetadata, IRegion, RegionType, AssetState, IFileInfo,
|
||||
IAsset, AssetType,
|
||||
IAsset, AssetType, ModelPathType,
|
||||
} from "../models/applicationState";
|
||||
import { IV1Project, IV1Region } from "../models/v1Models";
|
||||
import packageJson from "../../package.json";
|
||||
|
|
@ -66,6 +66,7 @@ export default class ImportService implements IImportService {
|
|||
videoSettings: {
|
||||
frameExtractionRate: originalProject.framerate ? Number(originalProject.framerate) : 15,
|
||||
},
|
||||
activeLearningSettings: null,
|
||||
autoSave: true,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,16 @@ import _ from "lodash";
|
|||
import ProjectService, { IProjectService } from "./projectService";
|
||||
import MockFactory from "../common/mockFactory";
|
||||
import { StorageProviderFactory } from "../providers/storage/storageProviderFactory";
|
||||
import { IProject, IExportFormat, ISecurityToken, AssetState } from "../models/applicationState";
|
||||
import {
|
||||
IProject, IExportFormat, ISecurityToken,
|
||||
AssetState, IActiveLearningSettings, ModelPathType,
|
||||
} from "../models/applicationState";
|
||||
import { constants } from "../common/constants";
|
||||
import { ExportProviderFactory } from "../providers/export/exportProviderFactory";
|
||||
import { generateKey } from "../common/crypto";
|
||||
import { encryptProject } from "../common/utils";
|
||||
import { encryptProject, decryptProject } from "../common/utils";
|
||||
import { ExportAssetState } from "../providers/export/exportProvider";
|
||||
import { IVottJsonExportProviderOptions } from "../providers/export/vottJson";
|
||||
|
||||
describe("Project Service", () => {
|
||||
let projectSerivce: IProjectService = null;
|
||||
|
|
@ -76,6 +81,45 @@ describe("Project Service", () => {
|
|||
expect.any(String));
|
||||
});
|
||||
|
||||
it("sets default export settings when not defined", async () => {
|
||||
testProject.exportFormat = null;
|
||||
const result = await projectSerivce.save(testProject, securityToken);
|
||||
|
||||
const vottJsonExportProviderOptions: IVottJsonExportProviderOptions = {
|
||||
assetState: ExportAssetState.Visited,
|
||||
includeImages: true,
|
||||
};
|
||||
|
||||
const expectedExportFormat: IExportFormat = {
|
||||
providerType: "vottJson",
|
||||
providerOptions: vottJsonExportProviderOptions,
|
||||
};
|
||||
|
||||
const decryptedProject = decryptProject(result, securityToken);
|
||||
|
||||
expect(decryptedProject.exportFormat).toEqual(expectedExportFormat);
|
||||
});
|
||||
|
||||
it("sets default active learning setting when not defined", async () => {
|
||||
testProject.activeLearningSettings = null;
|
||||
const result = await projectSerivce.save(testProject, securityToken);
|
||||
|
||||
const activeLearningSettings: IActiveLearningSettings = {
|
||||
autoDetect: false,
|
||||
predictTag: true,
|
||||
modelPathType: ModelPathType.Coco,
|
||||
};
|
||||
|
||||
expect(result.activeLearningSettings).toEqual(activeLearningSettings);
|
||||
});
|
||||
|
||||
it("initializes tags to empty array if not defined", async () => {
|
||||
testProject.tags = null;
|
||||
const result = await projectSerivce.save(testProject, securityToken);
|
||||
|
||||
expect(result.tags).toEqual([]);
|
||||
});
|
||||
|
||||
it("Save calls configured export provider save when defined", async () => {
|
||||
testProject.exportFormat = {
|
||||
providerType: "azureCustomVision",
|
||||
|
|
|
|||
|
|
@ -1,12 +1,17 @@
|
|||
import _ from "lodash";
|
||||
import shortid from "shortid";
|
||||
import { StorageProviderFactory } from "../providers/storage/storageProviderFactory";
|
||||
import { IProject, ISecurityToken, AppError, ErrorCode, AssetState } from "../models/applicationState";
|
||||
import {
|
||||
IProject, ISecurityToken, AppError,
|
||||
ErrorCode, ModelPathType, IActiveLearningSettings,
|
||||
} from "../models/applicationState";
|
||||
import Guard from "../common/guard";
|
||||
import { constants } from "../common/constants";
|
||||
import { ExportProviderFactory } from "../providers/export/exportProviderFactory";
|
||||
import { decryptProject, encryptProject } from "../common/utils";
|
||||
import packageJson from "../../package.json";
|
||||
import { ExportAssetState } from "../providers/export/exportProvider";
|
||||
import { IExportFormat } from "vott-react";
|
||||
|
||||
/**
|
||||
* Functions required for a project service
|
||||
|
|
@ -20,6 +25,20 @@ export interface IProjectService {
|
|||
isDuplicate(project: IProject, projectList: IProject[]): boolean;
|
||||
}
|
||||
|
||||
const defaultActiveLearningSettings: IActiveLearningSettings = {
|
||||
autoDetect: false,
|
||||
predictTag: true,
|
||||
modelPathType: ModelPathType.Coco,
|
||||
};
|
||||
|
||||
const defaultExportOptions: IExportFormat = {
|
||||
providerType: "vottJson",
|
||||
providerOptions: {
|
||||
assetState: ExportAssetState.Visited,
|
||||
includeImages: true,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* @name - Project Service
|
||||
* @description - Functions for dealing with projects
|
||||
|
|
@ -35,7 +54,23 @@ export default class ProjectService implements IProjectService {
|
|||
|
||||
try {
|
||||
const loadedProject = decryptProject(project, securityToken);
|
||||
return Promise.resolve(loadedProject);
|
||||
|
||||
// Ensure tags is always initialized to an array
|
||||
if (!loadedProject.tags) {
|
||||
loadedProject.tags = [];
|
||||
}
|
||||
|
||||
// Initialize active learning settings if they don't exist
|
||||
if (!loadedProject.activeLearningSettings) {
|
||||
loadedProject.activeLearningSettings = defaultActiveLearningSettings;
|
||||
}
|
||||
|
||||
// Initialize export settings if they don't exist
|
||||
if (!loadedProject.exportFormat) {
|
||||
loadedProject.exportFormat = defaultExportOptions;
|
||||
}
|
||||
|
||||
return Promise.resolve({ ...loadedProject });
|
||||
} catch (e) {
|
||||
const error = new AppError(ErrorCode.ProjectInvalidSecurityToken, "Error decrypting project settings");
|
||||
return Promise.reject(error);
|
||||
|
|
@ -54,6 +89,21 @@ export default class ProjectService implements IProjectService {
|
|||
project.id = shortid.generate();
|
||||
}
|
||||
|
||||
// Ensure tags is always initialized to an array
|
||||
if (!project.tags) {
|
||||
project.tags = [];
|
||||
}
|
||||
|
||||
// Initialize active learning settings if they don't exist
|
||||
if (!project.activeLearningSettings) {
|
||||
project.activeLearningSettings = defaultActiveLearningSettings;
|
||||
}
|
||||
|
||||
// Initialize export settings if they don't exist
|
||||
if (!project.exportFormat) {
|
||||
project.exportFormat = defaultExportOptions;
|
||||
}
|
||||
|
||||
project.version = packageJson.version;
|
||||
|
||||
const storageProvider = StorageProviderFactory.createFromConnection(project.targetConnection);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче