diff --git a/.gitignore b/.gitignore index bd37d72a..82e15428 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ secrets.sh # complexity reports es6-src/ report/ +debug.log diff --git a/src/common/localization/en-us.ts b/src/common/localization/en-us.ts index f8b31624..7fe04331 100644 --- a/src/common/localization/en-us.ts +++ b/src/common/localization/en-us.ts @@ -202,6 +202,14 @@ export const english: IAppStrings = { downloadScript: "Analyze with python script", defaultLocalFileInput: "Browse for a file...", defaultURLInput: "Paste or type URL...", + editAndUploadToTrainingSet: "Edit & upload to training set", + editAndUploadToTrainingSetNotify: "by clicking on this button, this form will be added to this project, where you can edit these labels.", + editAndUploadToTrainingSetNotify2: "We are adding this file to your training set, where you could edit the labels and re-train the model.", + uploadInPrgoress: "Upload in progress...", + confirmDuplicatedAssetName: { + title: "Asset name exists", + message: "Asset with name '${name}' exists in project, override?" + } }, recentModelsView: { header: "Select a model to analyze with", @@ -434,6 +442,8 @@ export const english: IAppStrings = { subIMenuItems: { runOcrOnCurrentDocument: "Run OCR on current document", runOcrOnAllDocuments: "Run OCR on all documents", + runAutoLabelingCurrentDocument: "Run AutoLabeling on current document", + noPredictModelOnProject: "Predict model not avaliable, please train the model first.", } } } diff --git a/src/common/localization/es-cl.ts b/src/common/localization/es-cl.ts index fd3dcc74..9d3818a6 100644 --- a/src/common/localization/es-cl.ts +++ b/src/common/localization/es-cl.ts @@ -201,6 +201,14 @@ export const spanish: IAppStrings = { downloadScript: "Analizar con script python", defaultLocalFileInput: "Busca un archivo...", defaultURLInput: "Pegar o escribir URL...", + editAndUploadToTrainingSet: "Editar y cargar al conjunto de entrenamiento", + editAndUploadToTrainingSetNotify: "Al hacer clic en este botón, este formulario se agregará al Blob de Azure Storage para este proyecto, donde puede editar estas etiquetas.", + editAndUploadToTrainingSetNotify2: "Estamos agregando este archivo a su conjunto de entrenamiento, donde puede editar las etiquetas y volver a entrenar el modelo.", + uploadInPrgoress: "carga en curso...", + confirmDuplicatedAssetName: { + title: "El nombre del activo existe", + message: "El activo con el nombre '${name}' existe en el proyecto, ¿anularlo?" + } }, recentModelsView: { header: "Seleccionar modelo para analizar con", @@ -435,6 +443,8 @@ export const spanish: IAppStrings = { subIMenuItems: { runOcrOnCurrentDocument: "Ejecutar OCR en el documento actual", runOcrOnAllDocuments: "Ejecute OCR en todos los documentos", + runAutoLabelingCurrentDocument: "Ejecutar AutoLabeling en el documento actual", + noPredictModelOnProject: "Predecir modelo no disponible, entrene el modelo primero.", } } } diff --git a/src/common/mockFactory.ts b/src/common/mockFactory.ts index 88c2ce81..6986d9dc 100644 --- a/src/common/mockFactory.ts +++ b/src/common/mockFactory.ts @@ -475,6 +475,7 @@ export default class MockFactory { saveProject: jest.fn(() => Promise.resolve()), deleteProject: jest.fn(() => Promise.resolve()), closeProject: jest.fn(() => Promise.resolve()), + addAssetToProject: jest.fn(() => Promise.resolve()), deleteAsset: jest.fn(() => Promise.resolve()), loadAssets: jest.fn(() => Promise.resolve()), loadAssetMetadata: jest.fn(() => Promise.resolve()), diff --git a/src/common/strings.ts b/src/common/strings.ts index 9f3b0f03..e543dc84 100644 --- a/src/common/strings.ts +++ b/src/common/strings.ts @@ -200,6 +200,14 @@ export interface IAppStrings { downloadScript: string, defaultLocalFileInput: string, defaultURLInput: string, + editAndUploadToTrainingSet: string, + editAndUploadToTrainingSetNotify: string, + editAndUploadToTrainingSetNotify2: string, + uploadInPrgoress: string, + confirmDuplicatedAssetName: { + title: string, + message: string + }, }; recentModelsView: { header: string; @@ -277,7 +285,7 @@ export interface IAppStrings { messages: { saveSuccess: string, deleteSuccess: string, - doNotAllowDuplicateNames:string, + doNotAllowDuplicateNames: string, }, imageCorsWarning: string, blobCorsWarning: string, @@ -429,6 +437,8 @@ export interface IAppStrings { subIMenuItems: { runOcrOnCurrentDocument: string, runOcrOnAllDocuments: string, + runAutoLabelingCurrentDocument: string, + noPredictModelOnProject: string, } } } diff --git a/src/models/applicationState.ts b/src/models/applicationState.ts index 1d5f4eed..6661a5e6 100644 --- a/src/models/applicationState.ts +++ b/src/models/applicationState.ts @@ -67,6 +67,7 @@ export interface IProviderOptions { export interface IAppSettings { securityTokens: ISecurityToken[], thumbnailSize?: ISize, + hideUploadingOption?: boolean; } /** diff --git a/src/react/components/pages/editorPage/canvas.tsx b/src/react/components/pages/editorPage/canvas.tsx index f8d0dd19..26ec4fdf 100644 --- a/src/react/components/pages/editorPage/canvas.tsx +++ b/src/react/components/pages/editorPage/canvas.tsx @@ -35,6 +35,8 @@ import { constants } from "../../../../common/constants"; import { CanvasCommandBar } from "./canvasCommandBar"; import { TooltipHost, ITooltipHostStyles } from "@fluentui/react"; import { IAppSettings } from '../../../../models/applicationState'; +import { AutoLabelingStatus, PredictService } from "../../../../services/predictService"; +import { AssetService } from "../../../../services/assetService"; import { strings } from "../../../../common/strings"; @@ -55,6 +57,7 @@ export interface ICanvasProps extends React.Props { onSelectedRegionsChanged?: (regions: IRegion[]) => void; onCanvasRendered?: (canvas: HTMLCanvasElement) => void; onRunningOCRStatusChanged?: (isRunning: boolean) => void; + onRunningAutoLabelingStatusChanged?: (isRunning: boolean) => void; onTagChanged?: (oldTag: ITag, newTag: ITag) => void; runOcrForAllDocs?: (runForAllDocs: boolean) => void; onAssetDeleted?: () => void; @@ -75,6 +78,7 @@ export interface ICanvasState { errorTitle?: string; errorMessage: string; ocrStatus: OcrStatus; + autoLableingStatus: AutoLabelingStatus; layers: any; tableIconTooltip: any; hoveringFeature: string; @@ -134,8 +138,9 @@ export default class Canvas extends React.Component isError: false, errorMessage: undefined, ocrStatus: OcrStatus.done, - layers: {text: true, tables: true, checkboxes: true, label: true, drawnRegions: true}, - tableIconTooltip: { display: "none", width: 0, height: 0, top: 0, left: 0}, + autoLableingStatus: AutoLabelingStatus.none, + layers: { text: true, tables: true, checkboxes: true, label: true, drawnRegions: true }, + tableIconTooltip: { display: "none", width: 0, height: 0, top: 0, left: 0 }, hoveringFeature: null, groupSelectMode: false, drawRegionMode: false, @@ -188,7 +193,7 @@ export default class Canvas extends React.Component pdfFile: null, imageUri: null, tiffImages: [], - layers: {text: true, tables: true, checkboxes: true, label: true, drawnRegions: true}, + layers: { text: true, tables: true, checkboxes: true, label: true, drawnRegions: true }, }, async () => { const asset = this.state.currentAsset.asset; await this.loadImage(); @@ -223,12 +228,12 @@ export default class Canvas extends React.Component return (
", ".", - "{", "[", "}", "]", "+", "-", "/", "=", "_", "?"]} - handler={this.handleKeyDown} + displayName={"Delete region"} + key={"Delete"} + keyEventType={KeyEventType.KeyDown} + accelerators={["Escape", "Alt+Backspace", "Shift", "Delete", "Backspace", "<", ",", ">", ".", + "{", "[", "}", "]", "+", "-", "/", "=", "_", "?"]} + handler={this.handleKeyDown} /> handleRunOcr={this.runOcr} handleAssetDeleted={this.props.onAssetDeleted} handleRunOcrForAllDocuments={this.runOcrForAllDocuments} + handleRunAutoLabelingOnCurrentDocument={this.runAutoLabelingOnCurrentDocument} connectionType={this.props.project.sourceConnection.providerType} handleToggleDrawRegionMode={this.handleToggleDrawRegionMode} drawRegionMode={this.state.drawRegionMode} + project={this.props.project} parentPage={strings.editorPage.title} /> /> @@ -299,32 +306,40 @@ export default class Canvas extends React.Component onClick={this.handleTableIconFeatureSelect} /> - { this.shouldShowPreviousPageButton() && + {this.shouldShowPreviousPageButton() && } - { this.shouldShowNextPageButton() && + {this.shouldShowNextPageButton() && } - { this.shouldShowMultiPageIndicator() && + {this.shouldShowMultiPageIndicator() &&

Page {this.state.currentPage} of {this.state.numPages}

} - { this.state.ocrStatus !== OcrStatus.done && + {this.state.ocrStatus !== OcrStatus.done &&
- + +
+
+ } + {this.state.autoLableingStatus === AutoLabelingStatus.running && +
+
+ +
} @@ -343,10 +358,28 @@ export default class Canvas extends React.Component } private runOcrForAllDocuments = () => { - this.setState({ocrStatus: OcrStatus.runningOCR}) + this.setState({ ocrStatus: OcrStatus.runningOCR }) this.props.runOcrForAllDocs(true); } + private runAutoLabelingOnCurrentDocument = async () => { + try { + this.setAutoLabelingStatus(AutoLabelingStatus.running); + const asset = this.state.currentAsset.asset; + const assetPath = asset.path; + const predictService = new PredictService(this.props.project); + const result = await predictService.getPrediction(assetPath); + + const assetService = new AssetService(this.props.project); + await assetService.uploadAssetPredictResult(asset, result); + const assetMetadata = await assetService.getAssetMetadata(asset); + await this.props.onAssetMetadataChanged(assetMetadata); + } + finally { + this.setAutoLabelingStatus(AutoLabelingStatus.done); + } + } + public updateSize() { this.imageMap.updateSize(); } @@ -405,8 +438,8 @@ export default class Canvas extends React.Component const newTag = { ...tag, documentCount: 1, - type : fieldType, - format : FieldFormat.NotSpecified, + type: fieldType, + format: FieldFormat.NotSpecified, } as ITag; this.props.onTagChanged(tag, newTag); } @@ -572,7 +605,7 @@ export default class Canvas extends React.Component this.imageMap.removeAllDrawnLabelFeatures(); this.addLabelledDataToLayer(regions.filter( (region) => region.tags[0] !== undefined && - region.pageNumber === this.state.currentPage)); + region.pageNumber === this.state.currentPage)); } this.setState({ currentAsset, @@ -695,9 +728,9 @@ export default class Canvas extends React.Component state: "rest", }); - const iconTR = [coordinates[0][0] - 5, coordinates[0][1] ]; + const iconTR = [coordinates[0][0] - 5, coordinates[0][1]]; const iconTL = [iconTR[0] - 31.5, iconTR[1]]; - const iconBL = [iconTR[0] , iconTR[1] - 29.5]; + const iconBL = [iconTR[0], iconTR[1] - 29.5]; const iconBR = [iconTR[0] - 31.5, iconTR[1] - 29.5]; tableFeatures["iconBorder"] = new Feature({ @@ -883,7 +916,7 @@ export default class Canvas extends React.Component image: new Icon({ opacity: 0.3, scale: this.imageMap && this.imageMap.getResolutionForZoom(3) ? - this.imageMap.getResolutionForZoom(3) / resolution : 1, + this.imageMap.getResolutionForZoom(3) / resolution : 1, anchor: [.95, 0.15], anchorXUnits: "fraction", anchorYUnits: "fraction", @@ -895,7 +928,7 @@ export default class Canvas extends React.Component image: new Icon({ opacity: 1, scale: this.imageMap && this.imageMap.getResolutionForZoom(3) ? - this.imageMap.getResolutionForZoom(3) / resolution : 1, + this.imageMap.getResolutionForZoom(3) / resolution : 1, anchor: [.95, 0.15], anchorXUnits: "fraction", anchorYUnits: "fraction", @@ -977,8 +1010,7 @@ export default class Canvas extends React.Component if (category === FeatureCategory.DrawnRegion || (category === FeatureCategory.Label && this.state.currentAsset.regions - .find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion)) - { + .find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion)) { selectedRegions.forEach((region) => { if (region?.category !== FeatureCategory.DrawnRegion) { this.removeFromSelectedRegions(region.id) @@ -987,14 +1019,14 @@ export default class Canvas extends React.Component } else if (category === FeatureCategory.Checkbox || (category === FeatureCategory.Label && this.state.currentAsset.regions - .find((r) => r.id === regionId).category === FeatureCategory.Checkbox)) { - selectedRegions.forEach((region) => this.removeFromSelectedRegions(region.id)); + .find((r) => r.id === regionId).category === FeatureCategory.Checkbox)) { + selectedRegions.forEach((region) => this.removeFromSelectedRegions(region.id)); } else if (category === FeatureCategory.Text || (category === FeatureCategory.Label && this.state.currentAsset.regions - .find((r) => r.id === regionId).category === FeatureCategory.Text)) { - selectedRegions.filter((region) => region.category === FeatureCategory.Checkbox || - region.category === FeatureCategory.DrawnRegion) - .forEach((region) => this.removeFromSelectedRegions(region.id)); + .find((r) => r.id === regionId).category === FeatureCategory.Text)) { + selectedRegions.filter((region) => region.category === FeatureCategory.Checkbox || + region.category === FeatureCategory.DrawnRegion) + .forEach((region) => this.removeFromSelectedRegions(region.id)); } } @@ -1014,7 +1046,7 @@ export default class Canvas extends React.Component const iRegionId = this.getIndexOfSelectedRegionIndex(regionId); if (iRegionId >= 0) { const region = this.getSelectedRegions().find((r) => r.id === regionId); - if (region && region.tags && region.tags.length === 0 ) { + if (region && region.tags && region.tags.length === 0) { this.onRegionDelete(regionId); } this.selectedRegionIds.splice(iRegionId, 1); @@ -1025,9 +1057,9 @@ export default class Canvas extends React.Component } private addToSelectedRegions = (regionId: string, - text: string, - polygon: number[], - regionCategory: FeatureCategory) => { + text: string, + polygon: number[], + regionCategory: FeatureCategory) => { let selectedRegion; if (this.isRegionSelected(regionId)) { // skip if it's already existed in selected regions @@ -1041,7 +1073,7 @@ export default class Canvas extends React.Component if (this.selectedRegionIds.includes(regionId)) { return; } - } else { + } else { const regionBoundingBox = this.convertToRegionBoundingBox(polygon); const regionPoints = this.convertToRegionPoints(polygon); selectedRegion = { @@ -1105,6 +1137,13 @@ export default class Canvas extends React.Component } }); } + private setAutoLabelingStatus = (autoLableingStatus: AutoLabelingStatus) => { + this.setState({ autoLableingStatus }, () => { + if (this.props.onRunningAutoLabelingStatusChanged) { + this.props.onRunningAutoLabelingStatusChanged(autoLableingStatus === AutoLabelingStatus.running); + } + }) + } private runOcr = () => { this.loadOcr(true); @@ -1161,7 +1200,7 @@ export default class Canvas extends React.Component private loadPdfFile = async (assetId, url) => { try { - const pdf = await pdfjsLib.getDocument({url, cMapUrl, cMapPacked: true}).promise; + const pdf = await pdfjsLib.getDocument({ url, cMapUrl, cMapPacked: true }).promise; // Fetch current page if (assetId === this.state.currentAsset.asset.id) { await this.loadPdfPage(assetId, pdf, this.state.currentPage); @@ -1247,7 +1286,7 @@ export default class Canvas extends React.Component private convertLabelDataToRegions = (labelData: ILabelData): IRegion[] => { const regions = []; - if (labelData.labels) { + if (labelData && labelData.labels) { labelData.labels.forEach((label) => { if (label.value) { label.value.forEach((formRegion) => { @@ -1339,13 +1378,13 @@ export default class Canvas extends React.Component const top = Math.min(...yAxisValues); const right = Math.max(...xAxisValues); const bottom = Math.max(...yAxisValues); - return([left, top, right, top, right, bottom, left, bottom]); + return ([left, top, right, top, right, bottom, left, bottom]); } private convertToRegionPoints = (polygon: number[]) => { const points = []; for (let i = 0; i < polygon.length; i += 2) { - points.push({x: polygon[i], y: polygon[i + 1]}); + points.push({ x: polygon[i], y: polygon[i + 1] }); } return points; } @@ -1555,7 +1594,7 @@ export default class Canvas extends React.Component private getBoundingBoxTextFromRegion = (formRegion: IFormRegion, boundingBoxIndex: number) => { // get value from formRegion.text - const regionValues = formRegion.text.split(" "); + const regionValues = formRegion.text && formRegion.text.split(" "); if (regionValues && regionValues.length > boundingBoxIndex) { return regionValues[boundingBoxIndex]; } @@ -1675,7 +1714,7 @@ export default class Canvas extends React.Component // 2. Avoid rebuilding order index when users switch back and forth between pages. const ocrs = this.state.ocr; const ocrReadResults = (ocrs.recognitionResults || (ocrs.analyzeResult && ocrs.analyzeResult.readResults)); - const ocrPageResults = (ocrs.recognitionResults || (ocrs.analyzeResult && ocrs.analyzeResult.pageResults)); + const ocrPageResults = (ocrs.recognitionResults || (ocrs.analyzeResult && ocrs.analyzeResult.pageResults)); const imageExtent = this.imageMap.getImageExtent(); ocrReadResults.forEach((ocr) => { const ocrExtent = [0, 0, ocr.width, ocr.height]; @@ -1888,12 +1927,12 @@ export default class Canvas extends React.Component const newLayers = Object.assign({}, this.state.layers); newLayers[layer] = !newLayers[layer]; this.setState({ - layers : newLayers, + layers: newLayers, }); } private handleTableToolTipChange = async (display: string, width: number, height: number, top: number, - left: number, rows: number, columns: number, featureID: string) => { + left: number, rows: number, columns: number, featureID: string) => { if (!this.imageMap) { return; } @@ -1902,23 +1941,23 @@ export default class Canvas extends React.Component this.imageMap.getTableBorderFeatureByID(featureID).set("state", "hovering"); this.imageMap.getTableIconFeatureByID(featureID).set("state", "hovering"); } else if (featureID === null && this.state.hoveringFeature && - this.imageMap.getTableBorderFeatureByID(this.state.hoveringFeature).get("state") !== "selected") { + this.imageMap.getTableBorderFeatureByID(this.state.hoveringFeature).get("state") !== "selected") { this.imageMap.getTableBorderFeatureByID(this.state.hoveringFeature).set("state", "rest"); this.imageMap.getTableIconFeatureByID(this.state.hoveringFeature).set("state", "rest"); } const newTableIconTooltip = { display, - width, - height, - top, - left, - rows, - columns, - }; + width, + height, + top, + left, + rows, + columns, + }; this.setState({ - tableIconTooltip : newTableIconTooltip, - hoveringFeature: featureID, - }); + tableIconTooltip: newTableIconTooltip, + hoveringFeature: featureID, + }); } private redrawAllFeatures = () => { @@ -2028,7 +2067,7 @@ export default class Canvas extends React.Component private addDrawnRegionFeatureProps = (feature) => { const featureCoordinates = feature.getGeometry().getCoordinates()[0]; - const {featureId, boundingBox} = this.getFeatureIDAndBoundingBox(featureCoordinates); + const { featureId, boundingBox } = this.getFeatureIDAndBoundingBox(featureCoordinates); feature.setProperties({ id: featureId, text: "", @@ -2093,7 +2132,7 @@ export default class Canvas extends React.Component polygonPoints.push(boundingBox[i + 1] / ocrHeight); } const featureId = this.createRegionIdFromBoundingBox(polygonPoints, ocrPage); - return {featureId, boundingBox} + return { featureId, boundingBox } } private modifySelectedRegion = (existingRegionId, newRegionId) => { @@ -2124,7 +2163,7 @@ export default class Canvas extends React.Component const originalFeatureId = feature.getId(); const featureCoordinates = feature.getGeometry().getCoordinates()[0]; if (this.imageMap.modifyStartFeatureCoordinates[originalFeatureId] !== featureCoordinates.join(",")) { - const {featureId, boundingBox} = this.getFeatureIDAndBoundingBox(featureCoordinates); + const { featureId, boundingBox } = this.getFeatureIDAndBoundingBox(featureCoordinates); feature.setProperties({ id: featureId, boundingbox: boundingBox, diff --git a/src/react/components/pages/editorPage/canvasCommandBar.scss b/src/react/components/pages/editorPage/canvasCommandBar.scss new file mode 100644 index 00000000..c7bbcae6 --- /dev/null +++ b/src/react/components/pages/editorPage/canvasCommandBar.scss @@ -0,0 +1,3 @@ +.ms-ContextualMenu-link.is-disabled { + color: gray; +} diff --git a/src/react/components/pages/editorPage/canvasCommandBar.tsx b/src/react/components/pages/editorPage/canvasCommandBar.tsx index 1459f53f..2d4e9d70 100644 --- a/src/react/components/pages/editorPage/canvasCommandBar.tsx +++ b/src/react/components/pages/editorPage/canvasCommandBar.tsx @@ -4,10 +4,14 @@ import { ICustomizations, Customizer } from "@fluentui/react/lib/Utilities"; import { getDarkGreyTheme } from "../../../../common/themes"; import { strings } from '../../../../common/strings'; import { ContextualMenuItemType } from "@fluentui/react"; +import { IProject } from "../../../../models/applicationState"; +import "./canvasCommandBar.scss"; interface ICanvasCommandBarProps { handleZoomIn: () => void; handleZoomOut: () => void; + handleRunAutoLabelingOnCurrentDocument?: () => void; + project: IProject; handleRotateImage: (degrees: number) => void; handleRunOcr?: () => void; handleRunOcrForAllDocuments?: () => void; @@ -60,16 +64,16 @@ export const CanvasCommandBar: React.FunctionComponent = isChecked: props.layers["checkboxes"], onClick: () => props.handleLayerChange("checkboxes"), }, - // { - // key: "DrawnRegions", - // text: strings.editorPage.canvas.canvasCommandBar.items.layers.subMenuItems.drawnRegions, - // canCheck: true, - // iconProps: { iconName: "AddField" }, - // isChecked: props.layers["drawnRegions"], - // className: props.drawRegionMode ? "disabled" : "", - // onClick: () => props.handleLayerChange("drawnRegions"), - // disabled: props.drawRegionMode - // }, + // { + // key: "DrawnRegions", + // text: strings.editorPage.canvas.canvasCommandBar.items.layers.subMenuItems.drawnRegions, + // canCheck: true, + // iconProps: { iconName: "AddField" }, + // isChecked: props.layers["drawnRegions"], + // className: props.drawRegionMode ? "disabled" : "", + // onClick: () => props.handleLayerChange("drawnRegions"), + // disabled: props.drawRegionMode + // }, { key: "Label", text: strings.editorPage.canvas.canvasCommandBar.items.layers.subMenuItems.labels, @@ -159,6 +163,17 @@ export const CanvasCommandBar: React.FunctionComponent = iconProps: { iconName: "Documentation" }, onClick: () => props.handleRunOcrForAllDocuments(), }, + { + key: "runAutoLabelingCurrentDocument", + text: strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.runAutoLabelingCurrentDocument, + iconProps: { iconName: "Tag" }, + disabled: !props.project.predictModelId, + title: props.project.predictModelId ? "" : + strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.noPredictModelOnProject, + onClick: () => { + props.handleRunAutoLabelingOnCurrentDocument(); + }, + }, { key: 'divider_1', itemType: ContextualMenuItemType.Divider, diff --git a/src/react/components/pages/editorPage/editorPage.tsx b/src/react/components/pages/editorPage/editorPage.tsx index 253b383d..02e7a3ad 100644 --- a/src/react/components/pages/editorPage/editorPage.tsx +++ b/src/react/components/pages/editorPage/editorPage.tsx @@ -18,7 +18,7 @@ import { import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions"; import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions"; import IAppTitleActions, * as appTitleActions from "../../../../redux/actions/appTitleActions"; -import {AssetPreview, ContentSource} from "../../common/assetPreview/assetPreview"; +import { AssetPreview, ContentSource } from "../../common/assetPreview/assetPreview"; import { KeyboardBinding } from "../../common/keyboardBinding/keyboardBinding"; import { KeyEventType } from "../../common/keyboardManager/keyboardManager"; import { TagInput } from "../../common/tagInput/tagInput"; @@ -37,6 +37,8 @@ import PreventLeaving from "../../common/preventLeaving/preventLeaving"; import { Spinner, SpinnerSize } from "@fluentui/react/lib/Spinner"; import { getPrimaryGreenTheme, getPrimaryRedTheme } from "../../../../common/themes"; import { toast } from "react-toastify"; +import { PredictService } from "../../../../services/predictService"; +import { AssetService } from "../../../../services/assetService"; /** * Properties for Editor Page @@ -87,6 +89,7 @@ export interface IEditorPageState { isRunningOCRs?: boolean; /** Whether OCR is running in the main canvas */ isCanvasRunningOCR?: boolean; + isCanvasRunningAutoLabeling?: boolean; isError?: boolean; errorTitle?: string; errorMessage?: string; @@ -178,7 +181,7 @@ export default class EditorPage extends React.Component this.loadOcrForNotVisited()} - disabled={this.state.isRunningOCRs}> + disabled={this.isBusy()}> {this.state.isRunningOCRs ?
- this.resizeCanvas()}> + this.resizeCanvas()}>
-
+
{selectedAsset && + > } -
+
@@ -352,6 +356,9 @@ export default class EditorPage extends React.Component +
); } @@ -498,11 +505,11 @@ export default class EditorPage extends React.Component item.id === asset.id); if (assetIndex > -1) { const assets = [...this.state.assets]; - const item = {...assets[assetIndex]}; + const item = { ...assets[assetIndex] }; item.cachedImage = (contentSource as HTMLImageElement).src; assets[assetIndex] = item; - this.setState({assets}); + this.setState({ assets }); } } @@ -631,6 +638,9 @@ export default class EditorPage extends React.Component { + return this.state.isRunningOCRs || this.state.isCanvasRunningOCR || this.state.isCanvasRunningAutoLabeling; + } public loadOcrForNotVisited = async (runForAll?: boolean) => { - if (this.state.isRunningOCRs) { + if (this.isBusy()) { return; } const { project } = this.props; @@ -774,17 +787,19 @@ export default class EditorPage extends React.Component { - this.setState({hoveredLabel: label}); + this.setState({ hoveredLabel: label }); } private onLabelLeave = (label: ILabel) => { - this.setState({hoveredLabel: null}); + this.setState({ hoveredLabel: null }); } private onCanvasRunningOCRStatusChanged = (isCanvasRunningOCR: boolean) => { this.setState({ isCanvasRunningOCR }); } - + private onCanvasRunningAutoLabelingStatusChanged = (isCanvasRunningAutoLabeling: boolean) => { + this.setState({ isCanvasRunningAutoLabeling }); + } private onFocused = () => { this.loadProjectAssets(); } diff --git a/src/react/components/pages/predict/predictPage.tsx b/src/react/components/pages/predict/predictPage.tsx index 305db532..83dff4f2 100644 --- a/src/react/components/pages/predict/predictPage.tsx +++ b/src/react/components/pages/predict/predictPage.tsx @@ -1,45 +1,46 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { + DefaultButton, Dropdown, FontIcon, IconButton, IDropdownOption, + ISelection, PrimaryButton, Selection, + SelectionMode, Separator, Spinner, SpinnerSize, TextField +} from "@fluentui/react"; +import axios from "axios"; +import _ from "lodash"; +import { Feature } from "ol"; +import Polygon from "ol/geom/Polygon"; +import Fill from "ol/style/Fill"; +import Stroke from "ol/style/Stroke"; +import Style from "ol/style/Style"; +import pdfjsLib from "pdfjs-dist"; import React from "react"; import { connect } from "react-redux"; import { RouteComponentProps } from "react-router-dom"; import { bindActionCreators } from "redux"; +import url from "url"; +import { constants } from "../../../../common/constants"; +import HtmlFileReader from "../../../../common/htmlFileReader"; +import { interpolate, strings } from "../../../../common/strings"; import { - FontIcon, Selection, PrimaryButton, Spinner, SpinnerSize, IconButton, TextField, IDropdownOption, - Dropdown, DefaultButton, Separator, ISelection, SelectionMode -} from "@fluentui/react"; -import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions"; + getGreenWithWhiteBackgroundTheme, getPrimaryGreenTheme, getPrimaryWhiteTheme, + getRightPaneDefaultButtonTheme +} from "../../../../common/themes"; +import { loadImageToCanvas, parseTiffData, renderTiffToCanvas } from "../../../../common/utils"; +import { AppError, ErrorCode, IApplicationState, IAppSettings, IConnection, ImageMapParent, IProject, IRecentModel } from "../../../../models/applicationState"; import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions"; import IAppTitleActions, * as appTitleActions from "../../../../redux/actions/appTitleActions"; -import "./predictPage.scss"; -import { - IApplicationState, IConnection, IProject, IAppSettings, AppError, ErrorCode, IRecentModel, ImageMapParent, -} from "../../../../models/applicationState"; -import { ImageMap } from "../../common/imageMap/imageMap"; -import Style from "ol/style/Style"; -import Stroke from "ol/style/Stroke"; -import Fill from "ol/style/Fill"; -import PredictResult from "./predictResult"; -import _ from "lodash"; -import pdfjsLib from "pdfjs-dist"; -import Alert from "../../common/alert/alert"; -import url from "url"; -import HtmlFileReader from "../../../../common/htmlFileReader"; -import { Feature } from "ol"; -import Polygon from "ol/geom/Polygon"; -import { strings, interpolate } from "../../../../common/strings"; -import PreventLeaving from "../../common/preventLeaving/preventLeaving"; +import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions"; import ServiceHelper from "../../../../services/serviceHelper"; -import { parseTiffData, renderTiffToCanvas, loadImageToCanvas } from "../../../../common/utils"; -import { constants } from "../../../../common/constants"; -import { getPrimaryGreenTheme, getPrimaryWhiteTheme, - getGreenWithWhiteBackgroundTheme, - getRightPaneDefaultButtonTheme} from "../../../../common/themes"; -import axios from "axios"; -import { IAnalyzeModelInfo } from './predictResult'; -import RecentModelsView from "./recentModelsView"; import { getAppInsights } from '../../../../services/telemetryService'; +import Alert from "../../common/alert/alert"; +import Confirm from "../../common/confirm/confirm"; +import { ImageMap } from "../../common/imageMap/imageMap"; +import PreventLeaving from "../../common/preventLeaving/preventLeaving"; +import "./predictPage.scss"; +import PredictResult, { IAnalyzeModelInfo } from "./predictResult"; +import RecentModelsView from "./recentModelsView"; +import { UploadToTrainingSetView } from "./uploadToTrainingSetView"; import { CanvasCommandBar } from "../editorPage/canvasCommandBar"; pdfjsLib.GlobalWorkerOptions.workerSrc = constants.pdfjsWorkerSrc(pdfjsLib.version); @@ -83,6 +84,7 @@ export interface IPredictPageState { highlightedField: string; modelList: IModel[]; modelOption: string; + confirmDuplicatedAssetNameMessage?: string; imageAngle: number; } @@ -149,6 +151,8 @@ export default class PredictPage extends React.Component = React.createRef(); + private duplicateAssetNameConfirm: React.RefObject = React.createRef(); public async componentDidMount() { const projectId = this.props.match.params["projectId"]; @@ -180,7 +184,7 @@ export default class PredictPage extends React.Component {this.setState({showRecentModelsView: true})}} + onClick={() => { this.setState({ showRecentModelsView: true }) }} disabled={!mostRecentModel || browseFileDisabled} />
-
-
-
- {strings.predict.downloadScript} -
- +
+
+
+ {strings.predict.downloadScript} +
+
or
{strings.predict.uploadFile}
-
Image source
+
Image source
- { this.state.sourceOption === "localFile" && + {this.state.sourceOption === "localFile" && } - { this.state.sourceOption === "localFile" && + {this.state.sourceOption === "localFile" && } - { this.state.sourceOption === "localFile" && + {this.state.sourceOption === "localFile" && } - { this.state.sourceOption === "url" && + {this.state.sourceOption === "url" && } - { this.state.sourceOption === "url" && + {this.state.sourceOption === "url" &&
- +
{this.state.isFetching &&
@@ -415,21 +419,33 @@ export default class PredictPage extends React.Component } + { (Object.keys(predictions).length === 0 && this.state.predictRun) &&
No field can be extracted.
} +
} - : + : }
@@ -467,76 +483,76 @@ export default class PredictPage extends React.Component { if (this.state.inputedFileURL === strings.predict.defaultURLInput) { - this.setState({inputedFileURL: ""}); + this.setState({ inputedFileURL: "" }); } } private setInputedFileURL = (event) => { - this.setState({inputedFileURL: event.target.value}); + this.setState({ inputedFileURL: event.target.value }); } private getFileFromURL = () => { - this.setState({isFetching: true}); - fetch(this.state.inputedFileURL, { headers: {Accept: "application/pdf, image/jpeg, image/png, image/tiff"}}) - .then((response) => { - if (!response.ok) { + this.setState({ isFetching: true }); + fetch(this.state.inputedFileURL, { headers: { Accept: "application/pdf, image/jpeg, image/png, image/tiff" } }) + .then((response) => { + if (!response.ok) { + this.setState({ + isFetching: false, + shouldShowAlert: true, + alertTitle: "Failed to fetch", + alertMessage: response.status.toString() + " " + response.statusText, + isPredicting: false, + }); + return; + } + const contentType = response.headers.get("Content-Type"); + if (!["application/pdf", "image/jpeg", "image/png", "image/tiff"].includes(contentType)) { + this.setState({ + isFetching: false, + shouldShowAlert: true, + alertTitle: "Content-Type not supported", + alertMessage: "Content-Type " + contentType + " not supported", + isPredicting: false, + }); + return; + } + response.blob().then((blob) => { + const fileAsURL = new URL(this.state.inputedFileURL); + const fileName = fileAsURL.pathname.split("/").pop(); + const file = new File([blob], fileName, { type: contentType }); + this.setState({ + fetchedFileURL: this.state.inputedFileURL, + isFetching: false, + fileLabel: fileName, + currPage: 1, + analyzeResult: {}, + fileChanged: true, + file, + predictRun: false, + }, () => { + if (this.imageMap) { + this.imageMap.removeAllFeatures(); + } + }); + }).catch((error) => { + this.setState({ + isFetching: false, + shouldShowAlert: true, + alertTitle: "Invalid data", + alertMessage: error, + isPredicting: false, + }); + return; + }); + }).catch(() => { this.setState({ isFetching: false, shouldShowAlert: true, - alertTitle: "Failed to fetch", - alertMessage: response.status.toString() + " " + response.statusText, - isPredicting: false, - }); - return; - } - const contentType = response.headers.get("Content-Type"); - if (![ "application/pdf", "image/jpeg", "image/png", "image/tiff"].includes(contentType)) { - this.setState({ - isFetching: false, - shouldShowAlert: true, - alertTitle: "Content-Type not supported", - alertMessage: "Content-Type " + contentType + " not supported", - isPredicting: false, - }); - return; - } - response.blob().then((blob) => { - const fileAsURL = new URL(this.state.inputedFileURL); - const fileName = fileAsURL.pathname.split("/").pop(); - const file = new File([blob], fileName, {type: contentType}); - this.setState({ - fetchedFileURL: this.state.inputedFileURL, - isFetching: false, - fileLabel: fileName, - currPage: 1, - analyzeResult: {}, - fileChanged: true, - file, - predictRun: false, - }, () => { - if (this.imageMap) { - this.imageMap.removeAllFeatures(); - } - }); - }).catch((error) => { - this.setState({ - isFetching: false, - shouldShowAlert: true, - alertTitle: "Invalid data", - alertMessage: error, - isPredicting: false, + alertTitle: "Fetch failed", + alertMessage: "Network error or Cross-Origin Resource Sharing (CORS) is not configured server-side", }); return; }); - }).catch(() => { - this.setState({ - isFetching: false, - shouldShowAlert: true, - alertTitle: "Fetch failed", - alertMessage: "Network error or Cross-Origin Resource Sharing (CORS) is not configured server-side", - }); - return; - }); } private selectSource = (event, option) => { @@ -587,7 +603,7 @@ export default class PredictPage extends React.Component ); @@ -616,7 +632,7 @@ export default class PredictPage extends React.Component ); } else { @@ -631,6 +647,7 @@ export default class PredictPage extends React.Component |||/gi, (matched: string) => { - switch (matched) { - case "": - return endpointURL; - case "": - return apiKey; - case "": - return modelID; - case "": - return constants.apiVersion; - } - }); + switch (matched) { + case "": + return endpointURL; + case "": + return apiKey; + case "": + return modelID; + case "": + return constants.apiVersion; + } + }); const fileURL = window.URL.createObjectURL( new Blob([analyzeScript])); const fileLink = document.createElement("a"); @@ -813,7 +830,7 @@ export default class PredictPage extends React.Component { const typedArray = new Uint8Array(e.target.result); - const loadingTask = pdfjsLib.getDocument({data: typedArray, cMapUrl, cMapPacked: true}); + const loadingTask = pdfjsLib.getDocument({ data: typedArray, cMapUrl, cMapPacked: true }); loadingTask.promise.then((pdf) => { this.currPdf = pdf; this.loadPdfPage(pdf, this.state.currPage); @@ -1056,7 +1073,41 @@ export default class PredictPage extends React.Component { // no operation } - + private onAddAssetToProjectClick = async () => { + if (this.state.file) { + // this.props.project.assets + const fileName = `${this.props.project.folderPath}/${decodeURIComponent(this.state.file.name)}`; + const asset = Object.values(this.props.project.assets).find(asset => asset.name === fileName); + if (asset) { + const confirmDuplicatedAssetNameMessage = interpolate(strings.predict.confirmDuplicatedAssetName.message, { name: decodeURI(this.state.file.name) }); + this.setState({ + confirmDuplicatedAssetNameMessage + }); + this.duplicateAssetNameConfirm.current.open(); + } + else { + this.onAddAssetToProjectConfirm(); + } + } + } + private onAddAssetToProjectConfirm = async () => { + if (this.props.appSettings.hideUploadingOption) { + this.uploadToTrainingSetView.current.open(); + await this.onAddAssetToProject(); + this.uploadToTrainingSetView.current.close(); + } else { + this.uploadToTrainingSetView.current.open(); + } + } + private onAddAssetToProject = async () => { + if (this.state.file) { + const fileData = new Buffer(await this.state.file.arrayBuffer()); + const readResults: any = this.state.analyzeResult; + const fileName = decodeURIComponent(this.state.file.name).split("/").pop(); + await this.props.actions.addAssetToProject(this.props.project, fileName, fileData, readResults); + this.props.history.push(`/projects/${this.props.project.id}/edit`); + } + } private onPredictionClick = (predictedItem: any) => { const targetPage = predictedItem.page; if (Number.isInteger(targetPage) && targetPage !== this.state.currPage) { @@ -1092,12 +1143,12 @@ export default class PredictPage extends React.Component { const selectedIndex = this.getSelectedIndex(); if (selectedIndex !== this.state.selectionIndexTracker) { - this.setState({selectionIndexTracker: selectedIndex}) + this.setState({ selectionIndexTracker: selectedIndex }) } } private handleRecentModelsViewClose = () => { - this.setState({showRecentModelsView: false}); + this.setState({ showRecentModelsView: false }); const selectedIndex = this.getSelectedIndex(); if (selectedIndex !== this.state.selectedRecentModelIndex) { this.selectionHandler.setIndexSelected(this.state.selectedRecentModelIndex, true, true); @@ -1121,22 +1172,22 @@ export default class PredictPage extends React.Component { - const status = err.response.status; - if (status === 401) { - this.setState({ - couldNotGetRecentModel: true, - shouldShowAlert: true, - alertTitle: "Failed to get recent model", - alertMessage: "Permission denied. Check API key", - }); - } else { - this.setState({ - couldNotGetRecentModel: true, - }); - } - }) + { headers: { [constants.apiKeyHeader]: this.props.project.apiKey as string } }) + .catch((err) => { + const status = err.response.status; + if (status === 401) { + this.setState({ + couldNotGetRecentModel: true, + shouldShowAlert: true, + alertTitle: "Failed to get recent model", + alertMessage: "Permission denied. Check API key", + }); + } else { + this.setState({ + couldNotGetRecentModel: true, + }); + } + }) } catch { this.setState({ couldNotGetRecentModel: true, @@ -1175,7 +1226,7 @@ export default class PredictPage extends React.Component void; onPredictionClick?: (item: any) => void; onPredictionMouseEnter?: (item: any) => void; onPredictionMouseLeave?: (item: any) => void; @@ -46,6 +48,13 @@ export default class PredictResult extends React.Component
Prediction results
+ +
+
+ { + if (this.props.onAddAssetToProject) { + this.props.onAddAssetToProject(); + } + } private triggerDownload = (): void => { const { analyzeResult } = this.props; const predictionData = JSON.stringify(this.sanitizeData(analyzeResult)); @@ -189,23 +203,23 @@ export default class PredictResult extends React.Component Promise; + showOption: boolean; +} +interface IUploadToTrainingSetViewState { + hideModal: boolean; + isLoading: boolean; +} +export class UploadToTrainingSetView extends React.Component{ + constructor(props) { + super(props); + this.state = { + hideModal: true, + isLoading: !props.showOption, + }; + this.open = this.open.bind(this); + this.close = this.close.bind(this); + this.onConfirm = this.onConfirm.bind(this); + } + open() { + this.setState({ + hideModal: false, + isLoading: !this.props.showOption + }); + } + close() { + this.setState({ hideModal: true }); + } + async onConfirm() { + this.setState({ isLoading: true }); + if (this.props.onConfirm) { + await this.props.onConfirm(); + } + this.close(); + } + render() { + const dark: ICustomizations = { + settings: { + theme: getDarkGreyTheme(), + }, + scopedSettings: {}, + }; + const notifyMessage = this.props.showOption ? strings.predict.editAndUploadToTrainingSetNotify : strings.predict.editAndUploadToTrainingSetNotify2; + + return ( + <> + + +

Notice: {notifyMessage}

+
+ {this.state.isLoading ? +
+ +
: +
+ + +
+ } +
+ + + + ) + } +} diff --git a/src/redux/actions/actionCreators.ts b/src/redux/actions/actionCreators.ts index b5591807..a557152c 100644 --- a/src/redux/actions/actionCreators.ts +++ b/src/redux/actions/actionCreators.ts @@ -24,6 +24,7 @@ import { IUpdateProjectTagAction, IUpdateProjectTagsFromFilesAction, IUpdateTagDocumentCount, + IAddAssetToProjectAction, } from "./projectActions"; import { IShowAppErrorAction, @@ -83,6 +84,7 @@ export type AnyAction = IOtherAction | IDeleteConnectionAction | ILoadConnectionAction | ISaveConnectionAction | + IAddAssetToProjectAction| IDeleteConnectionAction | ILoadProjectAction | ICloseProjectAction | diff --git a/src/redux/actions/actionTypes.ts b/src/redux/actions/actionTypes.ts index 7edaae1f..600defa8 100644 --- a/src/redux/actions/actionTypes.ts +++ b/src/redux/actions/actionTypes.ts @@ -12,6 +12,7 @@ export enum ActionTypes { // Projects LOAD_PROJECT_SUCCESS = "LOAD_PROJECT_SUCCESS", SAVE_PROJECT_SUCCESS = "SAVE_PROJECT_SUCCESS", + ADD_ASSET_TO_PROJECT_SUCCESS = "ADD_ASSET_TO_PROJECT_SUCCESS", DELETE_PROJECT_SUCCESS = "DELETE_PROJECT_SUCCESS", CLOSE_PROJECT_SUCCESS = "CLOSE_PROJECT_SUCCESS", LOAD_PROJECT_ASSETS_SUCCESS = "LOAD_PROJECT_ASSETS_SUCCESS", diff --git a/src/redux/actions/projectActions.ts b/src/redux/actions/projectActions.ts index b8c685c4..2aeb5f59 100644 --- a/src/redux/actions/projectActions.ts +++ b/src/redux/actions/projectActions.ts @@ -29,6 +29,7 @@ export default interface IProjectActions { saveProject(project: IProject, saveTags?: boolean, updateTagsFromFiles?: boolean): Promise; deleteProject(project: IProject): Promise; closeProject(): void; + addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): Promise; deleteAsset(project: IProject, assetMetadata: IAssetMetadata): Promise; loadAssets(project: IProject): Promise; loadAssetMetadata(project: IProject, asset: IAsset): Promise; @@ -64,7 +65,7 @@ export function loadProject(project: IProject, sharedToken?: ISecurityToken): ] })); } else if (existingToken.key !== sharedToken.key) { - const reason = interpolate(strings.shareProject.errors.tokenNameExist, {sharedTokenName: sharedToken.name}) + const reason = interpolate(strings.shareProject.errors.tokenNameExist, { sharedTokenName: sharedToken.name }) toast.error(reason, { autoClose: false, closeOnClick: false }); return null; } @@ -126,7 +127,7 @@ export function updateProjectTagsFromFiles(project: IProject, asset?: string): ( } export function updatedAssetMetadata(project: IProject, - assetDocumentCountDifference: any): (dispatch: Dispatch) => Promise { + assetDocumentCountDifference: any): (dispatch: Dispatch) => Promise { return async (dispatch: Dispatch) => { const projectService = new ProjectService(); const updatedProject = await projectService.updatedAssetMetadata(project, assetDocumentCountDifference); @@ -169,7 +170,22 @@ export function closeProject(): (dispatch: Dispatch) => void { dispatch({ type: ActionTypes.CLOSE_PROJECT_SUCCESS }); }; } +/** + * add asset, ocr data, labels to project storage. + */ +export function addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): (dispatch: Dispatch) => Promise { + return async (dispatch: Dispatch) => { + const assetService = new AssetService(project); + await assetService.uploadBuffer(fileName, buffer); + const assets = await assetService.getAssets(); + const assetName = project.folderPath ? `${project.folderPath}/${fileName}` : fileName; + const asset = assets.find(a => a.name === assetName); + await assetService.uploadAssetPredictResult(asset, analyzeResult); + dispatch(addAssetToProjectAction(asset)); + return asset; + }; +} /** * Dispatches Delete Asset action */ @@ -351,6 +367,12 @@ export interface IDeleteProjectAction extends IPayloadAction { type: ActionTypes.DELETE_PROJECT_SUCCESS; } +/** + * Add asset to project action type + */ +export interface IAddAssetToProjectAction extends IPayloadAction { + type: ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS; +} /** * Load project assets action type */ @@ -409,6 +431,10 @@ export const saveProjectAction = createPayloadAction(ActionT * Instance of Delete Project action */ export const deleteProjectAction = createPayloadAction(ActionTypes.DELETE_PROJECT_SUCCESS); +/** + * Instance of Add Asset to Project action + */ +export const addAssetToProjectAction = createPayloadAction(ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS); /** * Instance of Load Project Assets action */ diff --git a/src/redux/reducers/applicationReducer.ts b/src/redux/reducers/applicationReducer.ts index 786cd232..810e9920 100644 --- a/src/redux/reducers/applicationReducer.ts +++ b/src/redux/reducers/applicationReducer.ts @@ -18,6 +18,8 @@ export const reducer = (state: IAppSettings = null, action: AnyAction): IAppSett return { ...action.payload }; case ActionTypes.ENSURE_SECURITY_TOKEN_SUCCESS: return { ...action.payload }; + case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS: + return { ...state, hideUploadingOption: true }; default: return state; } diff --git a/src/redux/reducers/currentProjectReducer.ts b/src/redux/reducers/currentProjectReducer.ts index 5b30bf02..56ed1962 100644 --- a/src/redux/reducers/currentProjectReducer.ts +++ b/src/redux/reducers/currentProjectReducer.ts @@ -25,6 +25,8 @@ export const reducer = (state: IProject = null, action: AnyAction): IProject => return null; case ActionTypes.LOAD_PROJECT_SUCCESS: return { ...action.payload }; + case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS: + return { ...state, lastVisitedAssetId: action.payload.id }; case ActionTypes.LOAD_ASSET_METADATA_SUCCESS: if (!state) { return state; diff --git a/src/services/assetService.ts b/src/services/assetService.ts index a6e22e49..7c1f5e59 100644 --- a/src/services/assetService.ts +++ b/src/services/assetService.ts @@ -23,7 +23,7 @@ const supportedImageFormats = { interface IMime { types: string[]; - pattern: (number|undefined)[]; + pattern: (number | undefined)[]; } // tslint:disable number-literal-format @@ -62,6 +62,91 @@ const mimeBytesNeeded: number = (Math.max(...imageMimes.map((m) => m.pattern.len * @description - Functions for dealing with project assets */ export class AssetService { + private getOcrFromAnalyzeResult(analyzeResult: any) { + return _.get(analyzeResult, "analyzeResult.readResults", []); + } + async uploadAssetPredictResult(asset: IAsset, readResults: any): Promise { + const getBoundingBox = (pageIndex, arr: number[]) => { + const ocrForCurrentPage: any = this.getOcrFromAnalyzeResult(readResults)[pageIndex - 1]; + const ocrExtent = [0, 0, ocrForCurrentPage.width, ocrForCurrentPage.height]; + const ocrWidth = ocrExtent[2] - ocrExtent[0]; + const ocrHeight = ocrExtent[3] - ocrExtent[1]; + const result = []; + for (let i = 0; i < arr.length; i += 2) { + result.push([ + (arr[i] / ocrWidth), + (arr[i + 1] / ocrHeight), + ]); + } + return result; + }; + const getLabelValues = (field: any) => { + return field.elements.map((path: string) => { + const pathArr = path.split('/').slice(1); + const word = pathArr.reduce((obj: any, key: string) => obj[key], { ...readResults.analyzeResult }); + return { + page: field.page, + text: word.text || word.state, + confidence: word.confidence, + boundingBoxes: [getBoundingBox(field.page, word.boundingBox)] + }; + }); + }; + const labels = []; + readResults.analyzeResult.documentResults + .map(result => Object.keys(result.fields) + .filter(key => result.fields[key]) + .map(key => ( + { + label: key, + key: null, + value: getLabelValues(result.fields[key]) + }))).forEach(items => { + labels.push(...items); + }); + + if (labels.length > 0) { + const fileName = decodeURIComponent(asset.name).split('/').pop(); + const labelData: ILabelData = { + document: fileName, + labels + }; + const metadata = { + ...await this.getAssetMetadata(asset), + labelData + }; + metadata.asset.state = AssetState.Tagged; + + const ocrData = JSON.parse(JSON.stringify(readResults)); + delete ocrData.analyzeResult.documentResults; + if (ocrData.analyzeResult.errors) { + delete ocrData.analyzeResult.errors; + } + const ocrFileName = `${asset.name}${constants.ocrFileExtension}`; + await Promise.all([ + this.save(metadata), + this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2)) + ]); + } + else { + const ocrData = { ...readResults }; + delete ocrData.analyzeResult.documentResults; + if (ocrData.analyzeResult.errors) { + delete ocrData.analyzeResult.errors; + } + const labelFileName = decodeURIComponent(`${asset.name}${constants.labelFileExtension}`); + const ocrFileName = decodeURIComponent(`${asset.name}${constants.ocrFileExtension}`); + try { + await Promise.all([ + this.storageProvider.deleteFile(labelFileName, true, true), + this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2)) + ]); + } + catch{ + return; + } + } + } /** * Create IAsset from filePath * @param filePath - filepath of asset @@ -230,13 +315,20 @@ export class AssetService { return asset; }) } else { - return assets.map((asset) => { + return assets.map((asset) => { asset.name = decodeURIComponent(asset.name); return asset; }).filter((asset) => this.isInExactFolderPath(asset.name, folderPath)); } } - + public async uploadBuffer(name: string, buffer: Buffer) { + const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name; + await this.storageProvider.writeBinary(path, buffer); + } + public async uploadText(name: string, contents: string) { + const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name; + await this.storageProvider.writeText(path, contents); + } /** * Delete asset * @param metadata - Metadata for asset diff --git a/src/services/predictService.ts b/src/services/predictService.ts new file mode 100644 index 00000000..13b400da --- /dev/null +++ b/src/services/predictService.ts @@ -0,0 +1,81 @@ +import _ from "lodash"; +import url from 'url'; +import { constants } from "../common/constants"; +import { interpolate, strings } from "../common/strings"; +import { AppError, ErrorCode, IProject } from "../models/applicationState"; +import ServiceHelper from "./serviceHelper"; + +export enum AutoLabelingStatus { + none, + running, + done +} +export class PredictService { + + constructor(private project: IProject) { + } + public async getPrediction(fileUrl: string): Promise { + const modelID = this.project.predictModelId; + if (!modelID) { + throw new AppError( + ErrorCode.PredictWithoutTrainForbidden, + strings.errors.predictWithoutTrainForbidden.message, + strings.errors.predictWithoutTrainForbidden.title); + } + const endpointURL = url.resolve( + this.project.apiUriBase, + `${constants.apiModelsPath}/${modelID}/analyze?includeTextDetails=true`, + ); + + const headers = { "Content-Type": "application/json", "cache-control": "no-cache" }; + const body = { source: fileUrl }; + + try { + const response = await ServiceHelper.postWithAutoRetry(endpointURL, body, { headers }, this.project.apiKey as string); + const operationLocation = response.headers["operation-location"]; + + return this.poll(() => + ServiceHelper.getWithAutoRetry( + operationLocation, { headers }, this.project.apiKey as string), 120000, 500); + + + } catch (err) { + if (err.response.status === 404) { + throw new AppError( + ErrorCode.ModelNotFound, + interpolate(strings.errors.modelNotFound.message, { modelID }) + ); + } else { + ServiceHelper.handleServiceError(err); + } + } + } + private poll = (func, timeout, interval): Promise => { + const endTime = Number(new Date()) + (timeout || 10000); + interval = interval || 100; + + const checkSucceeded = (resolve, reject) => { + const ajax = func(); + ajax.then((response) => { + if (response.data.status.toLowerCase() === constants.statusCodeSucceeded) { + resolve(response.data); + // prediction response from API + console.log("raw data", JSON.parse(response.request.response)); + } else if (response.data.status.toLowerCase() === constants.statusCodeFailed) { + reject(_.get( + response, + "data.analyzeResult.errors[0].errorMessage", + "Generic error during prediction")); + } else if (Number(new Date()) < endTime) { + // If the request isn't succeeded and the timeout hasn't elapsed, go again + setTimeout(checkSucceeded, interval, resolve, reject); + } else { + // Didn't succeeded after too much time, reject + reject("Timed out, please try other file."); + } + }); + }; + + return new Promise(checkSucceeded); + } +}