Yongbing chen/human in the loop (#517)

* upload analyze file to assets

* move spinner label to strings

* adjust labels

* issuefix

* typo

* adjust getLabelValues

* typo

* remove option bar when upload asset in the future runs

* feature:  "Auto Labeling"

* move AutoLabelingStatus to predictService.ts

* update ocr when auto labeling

(cherry picked from commit 42438598d5d7a325e82bbad191ed483f44094439)

* fix tslint error

* handle formRegion.text undefined error

* decode asset name when upload asset

* add HITL document override confirm

* Merge branch 'yongbing-chen/human-in-the-loop' of https://github.com/microsoft/OCR-Form-Tools into yongbing-chen/human-in-the-loop

* disable autolabeling button when no predict model

* move runAutoLabelingOnCurrentDocument to canvas.tsx

* remove assetMetadata if no labels find after auto-labeling

* fix tslint check

* object deep clone

Co-authored-by: alex-krasn <64093224+alex-krasn@users.noreply.github.com>

.gitignore

@@ -37,3 +37,4 @@ secrets.sh
 # complexity reports
 es6-src/
 report/
+debug.log

src/common/localization/en-us.ts

@@ -202,6 +202,14 @@ export const english: IAppStrings = {
         downloadScript: "Analyze with python script",
         defaultLocalFileInput: "Browse for a file...",
         defaultURLInput: "Paste or type URL...",
+        editAndUploadToTrainingSet: "Edit & upload to training set",
+        editAndUploadToTrainingSetNotify: "by clicking on this button, this form will be added to this project, where you can edit these labels.",
+        editAndUploadToTrainingSetNotify2: "We are adding this file to your training set, where you could edit the labels and re-train the model.",
+        uploadInPrgoress: "Upload in progress...",
+        confirmDuplicatedAssetName: {
+            title: "Asset name exists",
+            message: "Asset with name '${name}' exists in project, override?"
+        }
     },
     recentModelsView: {
         header: "Select a model to analyze with",
@@ -434,6 +442,8 @@ export const english: IAppStrings = {
                         subIMenuItems: {
                             runOcrOnCurrentDocument: "Run OCR on current document",
                             runOcrOnAllDocuments: "Run OCR on all documents",
+                            runAutoLabelingCurrentDocument: "Run AutoLabeling on current document",
+                            noPredictModelOnProject: "Predict model not avaliable, please train the model first.",
                         }
                     }
                 }

src/common/localization/es-cl.ts

@@ -201,6 +201,14 @@ export const spanish: IAppStrings = {
         downloadScript: "Analizar con script python",
         defaultLocalFileInput: "Busca un archivo...",
         defaultURLInput: "Pegar o escribir URL...",
+        editAndUploadToTrainingSet: "Editar y cargar al conjunto de entrenamiento",
+        editAndUploadToTrainingSetNotify: "Al hacer clic en este botón, este formulario se agregará al Blob de Azure Storage para este proyecto, donde puede editar estas etiquetas.",
+        editAndUploadToTrainingSetNotify2: "Estamos agregando este archivo a su conjunto de entrenamiento, donde puede editar las etiquetas y volver a entrenar el modelo.",
+        uploadInPrgoress: "carga en curso...",
+        confirmDuplicatedAssetName: {
+            title: "El nombre del activo existe",
+            message: "El activo con el nombre '${name}' existe en el proyecto, ¿anularlo?"
+        }
     },
     recentModelsView: {
         header: "Seleccionar modelo para analizar con",
@@ -435,6 +443,8 @@ export const spanish: IAppStrings = {
                         subIMenuItems: {
                             runOcrOnCurrentDocument: "Ejecutar OCR en el documento actual",
                             runOcrOnAllDocuments: "Ejecute OCR en todos los documentos",
+                            runAutoLabelingCurrentDocument: "Ejecutar AutoLabeling en el documento actual",
+                            noPredictModelOnProject: "Predecir modelo no disponible, entrene el modelo primero.",
                         }
                     }
                 }

src/common/mockFactory.ts

@@ -475,6 +475,7 @@ export default class MockFactory {
             saveProject: jest.fn(() => Promise.resolve()),
             deleteProject: jest.fn(() => Promise.resolve()),
             closeProject: jest.fn(() => Promise.resolve()),
+            addAssetToProject: jest.fn(() => Promise.resolve()),
             deleteAsset: jest.fn(() => Promise.resolve()),
             loadAssets: jest.fn(() => Promise.resolve()),
             loadAssetMetadata: jest.fn(() => Promise.resolve()),

src/common/strings.ts

@@ -200,6 +200,14 @@ export interface IAppStrings {
         downloadScript: string,
         defaultLocalFileInput: string,
         defaultURLInput: string,
+        editAndUploadToTrainingSet: string,
+        editAndUploadToTrainingSetNotify: string,
+        editAndUploadToTrainingSetNotify2: string,
+        uploadInPrgoress: string,
+        confirmDuplicatedAssetName: {
+            title: string,
+            message: string
+        },
     };
     recentModelsView: {
         header: string;
@@ -429,6 +437,8 @@ export interface IAppStrings {
                         subIMenuItems: {
                             runOcrOnCurrentDocument: string,
                             runOcrOnAllDocuments: string,
+                            runAutoLabelingCurrentDocument: string,
+                            noPredictModelOnProject: string,
                         }
                     }
                 }

src/models/applicationState.ts

@@ -67,6 +67,7 @@ export interface IProviderOptions {
 export interface IAppSettings {
     securityTokens: ISecurityToken[],
     thumbnailSize?: ISize,
+    hideUploadingOption?: boolean;
 }
 
 /**

src/react/components/pages/editorPage/canvas.tsx

@@ -35,6 +35,8 @@ import { constants } from "../../../../common/constants";
 import { CanvasCommandBar } from "./canvasCommandBar";
 import { TooltipHost, ITooltipHostStyles } from "@fluentui/react";
 import { IAppSettings } from '../../../../models/applicationState';
+import { AutoLabelingStatus, PredictService } from "../../../../services/predictService";
+import { AssetService } from "../../../../services/assetService";
 import { strings } from "../../../../common/strings";
 
 
@@ -55,6 +57,7 @@ export interface ICanvasProps extends React.Props<Canvas> {
     onSelectedRegionsChanged?: (regions: IRegion[]) => void;
     onCanvasRendered?: (canvas: HTMLCanvasElement) => void;
     onRunningOCRStatusChanged?: (isRunning: boolean) => void;
+    onRunningAutoLabelingStatusChanged?: (isRunning: boolean) => void;
     onTagChanged?: (oldTag: ITag, newTag: ITag) => void;
     runOcrForAllDocs?: (runForAllDocs: boolean) => void;
     onAssetDeleted?: () => void;
@@ -75,6 +78,7 @@ export interface ICanvasState {
     errorTitle?: string;
     errorMessage: string;
     ocrStatus: OcrStatus;
+    autoLableingStatus: AutoLabelingStatus;
     layers: any;
     tableIconTooltip: any;
     hoveringFeature: string;
@@ -134,6 +138,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
         isError: false,
         errorMessage: undefined,
         ocrStatus: OcrStatus.done,
+        autoLableingStatus: AutoLabelingStatus.none,
         layers: { text: true, tables: true, checkboxes: true, label: true, drawnRegions: true },
         tableIconTooltip: { display: "none", width: 0, height: 0, top: 0, left: 0 },
         hoveringFeature: null,
@@ -246,9 +251,11 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
                     handleRunOcr={this.runOcr}
                     handleAssetDeleted={this.props.onAssetDeleted}
                     handleRunOcrForAllDocuments={this.runOcrForAllDocuments}
+                    handleRunAutoLabelingOnCurrentDocument={this.runAutoLabelingOnCurrentDocument}
                     connectionType={this.props.project.sourceConnection.providerType}
                     handleToggleDrawRegionMode={this.handleToggleDrawRegionMode}
                     drawRegionMode={this.state.drawRegionMode}
+                    project={this.props.project}
                     parentPage={strings.editorPage.title}
                 />
                 <ImageMap
@@ -328,6 +335,14 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
                         </div>
                     </div>
                 }
+                {this.state.autoLableingStatus === AutoLabelingStatus.running &&
+                    <div className="canvas-ocr-loading">
+                        <div className="canvas-ocr-loading-spinner">
+                            <Label className="p-0" ></Label>
+                            <Spinner size={SpinnerSize.large} label="Running Auto Labeling..." ariaLive="assertive" labelPosition="right" />
+                        </div>
+                    </div>
+                }
                 <Alert
                     show={this.state.isError}
                     title={this.state.errorTitle || "Error"}
@@ -347,6 +362,24 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
         this.props.runOcrForAllDocs(true);
     }
 
+    private runAutoLabelingOnCurrentDocument = async () => {
+        try {
+            this.setAutoLabelingStatus(AutoLabelingStatus.running);
+            const asset = this.state.currentAsset.asset;
+            const assetPath = asset.path;
+            const predictService = new PredictService(this.props.project);
+            const result = await predictService.getPrediction(assetPath);
+
+            const assetService = new AssetService(this.props.project);
+            await assetService.uploadAssetPredictResult(asset, result);
+            const assetMetadata = await assetService.getAssetMetadata(asset);
+            await this.props.onAssetMetadataChanged(assetMetadata);
+        }
+        finally {
+            this.setAutoLabelingStatus(AutoLabelingStatus.done);
+        }
+    }
+
     public updateSize() {
         this.imageMap.updateSize();
     }
@@ -977,8 +1010,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
 
         if (category === FeatureCategory.DrawnRegion ||
             (category === FeatureCategory.Label && this.state.currentAsset.regions
-            .find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion))
+                .find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion)) {
-        {
             selectedRegions.forEach((region) => {
                 if (region?.category !== FeatureCategory.DrawnRegion) {
                     this.removeFromSelectedRegions(region.id)
@@ -1105,6 +1137,13 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
             }
         });
     }
+    private setAutoLabelingStatus = (autoLableingStatus: AutoLabelingStatus) => {
+        this.setState({ autoLableingStatus }, () => {
+            if (this.props.onRunningAutoLabelingStatusChanged) {
+                this.props.onRunningAutoLabelingStatusChanged(autoLableingStatus === AutoLabelingStatus.running);
+            }
+        })
+    }
 
     private runOcr = () => {
         this.loadOcr(true);
@@ -1247,7 +1286,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
     private convertLabelDataToRegions = (labelData: ILabelData): IRegion[] => {
         const regions = [];
 
-        if (labelData.labels) {
+        if (labelData && labelData.labels) {
             labelData.labels.forEach((label) => {
                 if (label.value) {
                     label.value.forEach((formRegion) => {
@@ -1555,7 +1594,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
 
     private getBoundingBoxTextFromRegion = (formRegion: IFormRegion, boundingBoxIndex: number) => {
         // get value from formRegion.text
-        const regionValues = formRegion.text.split(" ");
+        const regionValues = formRegion.text && formRegion.text.split(" ");
         if (regionValues && regionValues.length > boundingBoxIndex) {
             return regionValues[boundingBoxIndex];
         }

src/react/components/pages/editorPage/canvasCommandBar.scss

@@ -0,0 +1,3 @@
+.ms-ContextualMenu-link.is-disabled {
+    color: gray;
+}

src/react/components/pages/editorPage/canvasCommandBar.tsx

@@ -4,10 +4,14 @@ import { ICustomizations, Customizer } from "@fluentui/react/lib/Utilities";
 import { getDarkGreyTheme } from "../../../../common/themes";
 import { strings } from '../../../../common/strings';
 import { ContextualMenuItemType } from "@fluentui/react";
+import { IProject } from "../../../../models/applicationState";
+import "./canvasCommandBar.scss";
 
 interface ICanvasCommandBarProps {
     handleZoomIn: () => void;
     handleZoomOut: () => void;
+    handleRunAutoLabelingOnCurrentDocument?: () => void;
+    project: IProject;
     handleRotateImage: (degrees: number) => void;
     handleRunOcr?: () => void;
     handleRunOcrForAllDocuments?: () => void;
@@ -159,6 +163,17 @@ export const CanvasCommandBar: React.FunctionComponent<ICanvasCommandBarProps> =
                         iconProps: { iconName: "Documentation" },
                         onClick: () => props.handleRunOcrForAllDocuments(),
                     },
+                    {
+                        key: "runAutoLabelingCurrentDocument",
+                        text: strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.runAutoLabelingCurrentDocument,
+                        iconProps: { iconName: "Tag" },
+                        disabled: !props.project.predictModelId,
+                        title: props.project.predictModelId ? "" :
+                            strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.noPredictModelOnProject,
+                        onClick: () => {
+                            props.handleRunAutoLabelingOnCurrentDocument();
+                        },
+                    },
                     {
                         key: 'divider_1',
                         itemType: ContextualMenuItemType.Divider,

src/react/components/pages/editorPage/editorPage.tsx

@@ -37,6 +37,8 @@ import PreventLeaving from "../../common/preventLeaving/preventLeaving";
 import { Spinner, SpinnerSize } from "@fluentui/react/lib/Spinner";
 import { getPrimaryGreenTheme, getPrimaryRedTheme } from "../../../../common/themes";
 import { toast } from "react-toastify";
+import { PredictService } from "../../../../services/predictService";
+import { AssetService } from "../../../../services/assetService";
 
 /**
  * Properties for Editor Page
@@ -87,6 +89,7 @@ export interface IEditorPageState {
     isRunningOCRs?: boolean;
     /** Whether OCR is running in the main canvas */
     isCanvasRunningOCR?: boolean;
+    isCanvasRunningAutoLabeling?: boolean;
     isError?: boolean;
     errorTitle?: string;
     errorMessage?: string;
@@ -178,7 +181,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
 
     public render() {
         const { project } = this.props;
-        const { assets, selectedAsset, isRunningOCRs, isCanvasRunningOCR } = this.state;
+        const { assets, selectedAsset, isRunningOCRs, isCanvasRunningOCR, isCanvasRunningAutoLabeling } = this.state;
 
         const labels = (selectedAsset &&
             selectedAsset.labelData &&
@@ -222,7 +225,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
                                 className="editor-page-sidebar-run-ocr"
                                 type="button"
                                 onClick={() => this.loadOcrForNotVisited()}
-                                disabled={this.state.isRunningOCRs}>
+                                disabled={this.isBusy()}>
                                 {this.state.isRunningOCRs ?
                                     <div>
                                         <Spinner
@@ -263,6 +266,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
                                             onCanvasRendered={this.onCanvasRendered}
                                             onSelectedRegionsChanged={this.onSelectedRegionsChanged}
                                             onRunningOCRStatusChanged={this.onCanvasRunningOCRStatusChanged}
+                                            onRunningAutoLabelingStatusChanged={this.onCanvasRunningAutoLabelingStatusChanged}
                                             onTagChanged={this.onTagChanged}
                                             onAssetDeleted={this.confirmDocumentDeleted}
                                             editorMode={this.state.editorMode}
@@ -352,6 +356,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
                     when={isRunningOCRs || isCanvasRunningOCR}
                     message={"An OCR operation is currently in progress, are you sure you want to leave?"}
                 />
+                <PreventLeaving
+                    when={isCanvasRunningAutoLabeling}
+                    message={"An AutoLabeling option is currently in progress, are you sure you want to leave?"} />
             </div>
         );
     }
@@ -631,6 +638,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
             this.setState({ showInvalidRegionWarning: true });
             return;
         }
+        if (this.state.isCanvasRunningAutoLabeling) {
+            return;
+        }
 
         const assetMetadata = await this.props.actions.loadAssetMetadata(this.props.project, asset);
 
@@ -687,9 +697,12 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
             throw Error(error);
         }
     }
+    private isBusy = (): boolean => {
+        return this.state.isRunningOCRs || this.state.isCanvasRunningOCR || this.state.isCanvasRunningAutoLabeling;
+    }
 
     public loadOcrForNotVisited = async (runForAll?: boolean) => {
-        if (this.state.isRunningOCRs) {
+        if (this.isBusy()) {
             return;
         }
         const { project } = this.props;
@@ -784,7 +797,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
     private onCanvasRunningOCRStatusChanged = (isCanvasRunningOCR: boolean) => {
         this.setState({ isCanvasRunningOCR });
     }
-
+    private onCanvasRunningAutoLabelingStatusChanged = (isCanvasRunningAutoLabeling: boolean) => {
+        this.setState({ isCanvasRunningAutoLabeling });
+    }
     private onFocused = () => {
         this.loadProjectAssets();
     }

src/react/components/pages/predict/predictPage.tsx

@@ -1,45 +1,46 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+import {
+    DefaultButton, Dropdown, FontIcon, IconButton, IDropdownOption,
+    ISelection, PrimaryButton, Selection,
+    SelectionMode, Separator, Spinner, SpinnerSize, TextField
+} from "@fluentui/react";
+import axios from "axios";
+import _ from "lodash";
+import { Feature } from "ol";
+import Polygon from "ol/geom/Polygon";
+import Fill from "ol/style/Fill";
+import Stroke from "ol/style/Stroke";
+import Style from "ol/style/Style";
+import pdfjsLib from "pdfjs-dist";
 import React from "react";
 import { connect } from "react-redux";
 import { RouteComponentProps } from "react-router-dom";
 import { bindActionCreators } from "redux";
+import url from "url";
+import { constants } from "../../../../common/constants";
+import HtmlFileReader from "../../../../common/htmlFileReader";
+import { interpolate, strings } from "../../../../common/strings";
 import {
-    FontIcon, Selection, PrimaryButton, Spinner, SpinnerSize, IconButton, TextField, IDropdownOption,
+    getGreenWithWhiteBackgroundTheme, getPrimaryGreenTheme, getPrimaryWhiteTheme,
-    Dropdown, DefaultButton, Separator, ISelection, SelectionMode
+    getRightPaneDefaultButtonTheme
-} from "@fluentui/react";
+} from "../../../../common/themes";
-import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
+import { loadImageToCanvas, parseTiffData, renderTiffToCanvas } from "../../../../common/utils";
+import { AppError, ErrorCode, IApplicationState, IAppSettings, IConnection, ImageMapParent, IProject, IRecentModel } from "../../../../models/applicationState";
 import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions";
 import IAppTitleActions, * as appTitleActions from "../../../../redux/actions/appTitleActions";
-import "./predictPage.scss";
+import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
-import {
-    IApplicationState, IConnection, IProject, IAppSettings, AppError, ErrorCode, IRecentModel, ImageMapParent,
-} from "../../../../models/applicationState";
-import { ImageMap } from "../../common/imageMap/imageMap";
-import Style from "ol/style/Style";
-import Stroke from "ol/style/Stroke";
-import Fill from "ol/style/Fill";
-import PredictResult from "./predictResult";
-import _ from "lodash";
-import pdfjsLib from "pdfjs-dist";
-import Alert from "../../common/alert/alert";
-import url from "url";
-import HtmlFileReader from "../../../../common/htmlFileReader";
-import { Feature } from "ol";
-import Polygon from "ol/geom/Polygon";
-import { strings, interpolate } from "../../../../common/strings";
-import PreventLeaving from "../../common/preventLeaving/preventLeaving";
 import ServiceHelper from "../../../../services/serviceHelper";
-import { parseTiffData, renderTiffToCanvas, loadImageToCanvas } from "../../../../common/utils";
-import { constants } from "../../../../common/constants";
-import { getPrimaryGreenTheme, getPrimaryWhiteTheme,
-         getGreenWithWhiteBackgroundTheme,
-         getRightPaneDefaultButtonTheme} from "../../../../common/themes";
-import axios from "axios";
-import { IAnalyzeModelInfo } from './predictResult';
-import RecentModelsView from "./recentModelsView";
 import { getAppInsights } from '../../../../services/telemetryService';
+import Alert from "../../common/alert/alert";
+import Confirm from "../../common/confirm/confirm";
+import { ImageMap } from "../../common/imageMap/imageMap";
+import PreventLeaving from "../../common/preventLeaving/preventLeaving";
+import "./predictPage.scss";
+import PredictResult, { IAnalyzeModelInfo } from "./predictResult";
+import RecentModelsView from "./recentModelsView";
+import { UploadToTrainingSetView } from "./uploadToTrainingSetView";
 import { CanvasCommandBar } from "../editorPage/canvasCommandBar";
 
 pdfjsLib.GlobalWorkerOptions.workerSrc = constants.pdfjsWorkerSrc(pdfjsLib.version);
@@ -83,6 +84,7 @@ export interface IPredictPageState {
     highlightedField: string;
     modelList: IModel[];
     modelOption: string;
+    confirmDuplicatedAssetNameMessage?: string;
     imageAngle: number;
 }
 
@@ -149,6 +151,8 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
     private currPdf: any;
     private tiffImages: any[];
     private imageMap: ImageMap;
+    private uploadToTrainingSetView: React.RefObject<UploadToTrainingSetView> = React.createRef();
+    private duplicateAssetNameConfirm: React.RefObject<Confirm> = React.createRef();
 
     public async componentDidMount() {
         const projectId = this.props.match.params["projectId"];
@@ -415,17 +419,29 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
                                                     page={this.state.currPage}
                                                     tags={this.props.project.tags}
                                                     downloadResultLabel={this.state.fileLabel}
+                                                    onAddAssetToProject={this.onAddAssetToProjectClick}
                                                     onPredictionClick={this.onPredictionClick}
                                                     onPredictionMouseEnter={this.onPredictionMouseEnter}
                                                     onPredictionMouseLeave={this.onPredictionMouseLeave}
                                                 />
                                             }
+                                            <UploadToTrainingSetView
+                                                showOption={!this.props.appSettings.hideUploadingOption}
+                                                ref={this.uploadToTrainingSetView}
+                                                onConfirm={this.onAddAssetToProject} />
                                             {
                                                 (Object.keys(predictions).length === 0 && this.state.predictRun) &&
                                                 <div>
                                                     No field can be extracted.
                                                 </div>
                                             }
+                                            <Confirm
+                                                ref={this.duplicateAssetNameConfirm}
+                                                title={strings.predict.confirmDuplicatedAssetName.title}
+                                                message={this.state.confirmDuplicatedAssetNameMessage}
+                                                onConfirm={this.onAddAssetToProjectConfirm}
+                                                confirmButtonTheme={getPrimaryGreenTheme()}
+                                            />
                                         </div>
                                     </>
                                 }
@@ -631,6 +647,7 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
                     handleZoomIn={this.handleCanvasZoomIn}
                     handleZoomOut={this.handleCanvasZoomOut}
                     handleRotateImage={this.handleRotateCanvas}
+                    project={this.props.project}
                     parentPage={"predict"}
                 />
                 <ImageMap
@@ -1056,7 +1073,41 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
     private noOp = () => {
         // no operation
     }
-
+    private onAddAssetToProjectClick = async () => {
+        if (this.state.file) {
+            // this.props.project.assets
+            const fileName = `${this.props.project.folderPath}/${decodeURIComponent(this.state.file.name)}`;
+            const asset = Object.values(this.props.project.assets).find(asset => asset.name === fileName);
+            if (asset) {
+                const confirmDuplicatedAssetNameMessage = interpolate(strings.predict.confirmDuplicatedAssetName.message, { name: decodeURI(this.state.file.name) });
+                this.setState({
+                    confirmDuplicatedAssetNameMessage
+                });
+                this.duplicateAssetNameConfirm.current.open();
+            }
+            else {
+                this.onAddAssetToProjectConfirm();
+            }
+        }
+    }
+    private onAddAssetToProjectConfirm = async () => {
+        if (this.props.appSettings.hideUploadingOption) {
+            this.uploadToTrainingSetView.current.open();
+            await this.onAddAssetToProject();
+            this.uploadToTrainingSetView.current.close();
+        } else {
+            this.uploadToTrainingSetView.current.open();
+        }
+    }
+    private onAddAssetToProject = async () => {
+        if (this.state.file) {
+            const fileData = new Buffer(await this.state.file.arrayBuffer());
+            const readResults: any = this.state.analyzeResult;
+            const fileName = decodeURIComponent(this.state.file.name).split("/").pop();
+            await this.props.actions.addAssetToProject(this.props.project, fileName, fileData, readResults);
+            this.props.history.push(`/projects/${this.props.project.id}/edit`);
+        }
+    }
     private onPredictionClick = (predictedItem: any) => {
         const targetPage = predictedItem.page;
         if (Number.isInteger(targetPage) && targetPage !== this.state.currPage) {

src/react/components/pages/predict/predictResult.tsx

@@ -7,6 +7,7 @@ import "./predictResult.scss";
 import { getPrimaryGreenTheme } from "../../../../common/themes";
 import { PrimaryButton } from "@fluentui/react";
 import PredictModelInfo from './predictModelInfo';
+import { strings } from "../../../../common/strings";
 
 export interface IAnalyzeModelInfo {
     docType: string,
@@ -21,6 +22,7 @@ export interface IPredictResultProps {
     page: number;
     tags: ITag[];
     downloadResultLabel: string;
+    onAddAssetToProject?: () => void;
     onPredictionClick?: (item: any) => void;
     onPredictionMouseEnter?: (item: any) => void;
     onPredictionMouseLeave?: (item: any) => void;
@@ -46,6 +48,13 @@ export default class PredictResult extends React.Component<IPredictResultProps,
             <div>
                 <div className="container-items-center container-space-between results-container">
                     <h5 className="results-header">Prediction results</h5>
+
+                </div>
+                <div className="container-items-center container-space-between">
+                    <PrimaryButton
+                        theme={getPrimaryGreenTheme()}
+                        onClick={this.onAddAssetToProject}
+                        text={strings.predict.editAndUploadToTrainingSet} />
                     <PrimaryButton
                         className="align-self-end keep-button-80px"
                         theme={getPrimaryGreenTheme()}
@@ -142,6 +151,11 @@ export default class PredictResult extends React.Component<IPredictResultProps,
         return data;
     }
 
+    private onAddAssetToProject = async () => {
+        if (this.props.onAddAssetToProject) {
+            this.props.onAddAssetToProject();
+        }
+    }
     private triggerDownload = (): void => {
         const { analyzeResult } = this.props;
         const predictionData = JSON.stringify(this.sanitizeData(analyzeResult));

src/react/components/pages/predict/uploadToTrainingSetView.scss

@@ -0,0 +1,3 @@
+.upload-to-training-set-modal {
+    max-width: 34em;
+}

src/react/components/pages/predict/uploadToTrainingSetView.tsx

@@ -0,0 +1,90 @@
+import { Customizer, ICustomizations, Modal, PrimaryButton, Spinner, SpinnerSize } from '@fluentui/react';
+import React from 'react';
+import { strings } from "../../../../common/strings";
+import { getDarkGreyTheme, getDefaultDarkTheme, getPrimaryGreenTheme, getPrimaryGreyTheme } from '../../../../common/themes';
+import './uploadToTrainingSetView.scss';
+
+interface IUploadToTrainingSetViewProp {
+    onConfirm?: () => Promise<void>;
+    showOption: boolean;
+}
+interface IUploadToTrainingSetViewState {
+    hideModal: boolean;
+    isLoading: boolean;
+}
+export class UploadToTrainingSetView extends React.Component<IUploadToTrainingSetViewProp, IUploadToTrainingSetViewState>{
+    constructor(props) {
+        super(props);
+        this.state = {
+            hideModal: true,
+            isLoading: !props.showOption,
+        };
+        this.open = this.open.bind(this);
+        this.close = this.close.bind(this);
+        this.onConfirm = this.onConfirm.bind(this);
+    }
+    open() {
+        this.setState({
+            hideModal: false,
+            isLoading: !this.props.showOption
+        });
+    }
+    close() {
+        this.setState({ hideModal: true });
+    }
+    async onConfirm() {
+        this.setState({ isLoading: true });
+        if (this.props.onConfirm) {
+            await this.props.onConfirm();
+        }
+        this.close();
+    }
+    render() {
+        const dark: ICustomizations = {
+            settings: {
+                theme: getDarkGreyTheme(),
+            },
+            scopedSettings: {},
+        };
+        const notifyMessage = this.props.showOption ? strings.predict.editAndUploadToTrainingSetNotify : strings.predict.editAndUploadToTrainingSetNotify2;
+
+        return (
+            <>
+                <Customizer {...dark}>
+                    <Modal
+                        isOpen={!this.state.hideModal}
+                        isModeless={false}
+                        containerClassName="modal-container upload-to-training-set-modal"
+                        scrollableContentClassName="scrollable-content"
+                    >
+                        <h4>Notice: <small>{notifyMessage}</small></h4>
+                        <div className="modal-buttons-container mt-4">
+                            {this.state.isLoading ?
+                                <div>
+                                    <Spinner
+                                        label={strings.predict.uploadInPrgoress}
+                                        ariaLive="assertive"
+                                        labelPosition="right"
+                                        theme={getDefaultDarkTheme()}
+                                        size={SpinnerSize.large} />
+                                </div> :
+                                <div>
+                                    <PrimaryButton
+                                        className="mr-3"
+                                        text={strings.predict.editAndUploadToTrainingSet}
+                                        theme={getPrimaryGreenTheme()}
+                                        onClick={this.onConfirm} />
+                                    <PrimaryButton
+                                        className="modal-cancel"
+                                        theme={getPrimaryGreyTheme()}
+                                        onClick={this.close}
+                                        text="Cancel" />
+                                </div>
+                            }
+                        </div>
+                    </Modal>
+                </Customizer>
+            </>
+        )
+    }
+}

src/redux/actions/actionCreators.ts

@@ -24,6 +24,7 @@ import {
     IUpdateProjectTagAction,
     IUpdateProjectTagsFromFilesAction,
     IUpdateTagDocumentCount,
+    IAddAssetToProjectAction,
 } from "./projectActions";
 import {
     IShowAppErrorAction,
@@ -83,6 +84,7 @@ export type AnyAction = IOtherAction |
     IDeleteConnectionAction |
     ILoadConnectionAction |
     ISaveConnectionAction |
+    IAddAssetToProjectAction|
     IDeleteConnectionAction |
     ILoadProjectAction |
     ICloseProjectAction |

src/redux/actions/actionTypes.ts

@@ -12,6 +12,7 @@ export enum ActionTypes {
     // Projects
     LOAD_PROJECT_SUCCESS = "LOAD_PROJECT_SUCCESS",
     SAVE_PROJECT_SUCCESS = "SAVE_PROJECT_SUCCESS",
+    ADD_ASSET_TO_PROJECT_SUCCESS = "ADD_ASSET_TO_PROJECT_SUCCESS",
     DELETE_PROJECT_SUCCESS = "DELETE_PROJECT_SUCCESS",
     CLOSE_PROJECT_SUCCESS = "CLOSE_PROJECT_SUCCESS",
     LOAD_PROJECT_ASSETS_SUCCESS = "LOAD_PROJECT_ASSETS_SUCCESS",

src/redux/actions/projectActions.ts

@@ -29,6 +29,7 @@ export default interface IProjectActions {
     saveProject(project: IProject, saveTags?: boolean, updateTagsFromFiles?: boolean): Promise<IProject>;
     deleteProject(project: IProject): Promise<void>;
     closeProject(): void;
+    addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): Promise<IAsset>;
     deleteAsset(project: IProject, assetMetadata: IAssetMetadata): Promise<void>;
     loadAssets(project: IProject): Promise<IAsset[]>;
     loadAssetMetadata(project: IProject, asset: IAsset): Promise<IAssetMetadata>;
@@ -169,7 +170,22 @@ export function closeProject(): (dispatch: Dispatch) => void {
         dispatch({ type: ActionTypes.CLOSE_PROJECT_SUCCESS });
     };
 }
+/**
+ * add asset, ocr data, labels to project storage.
+ */
+export function addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): (dispatch: Dispatch) => Promise<IAsset> {
+    return async (dispatch: Dispatch) => {
+        const assetService = new AssetService(project);
+        await assetService.uploadBuffer(fileName, buffer);
+        const assets = await assetService.getAssets();
+        const assetName = project.folderPath ? `${project.folderPath}/${fileName}` : fileName;
+        const asset = assets.find(a => a.name === assetName);
 
+        await assetService.uploadAssetPredictResult(asset, analyzeResult);
+        dispatch(addAssetToProjectAction(asset));
+        return asset;
+    };
+}
 /**
  * Dispatches Delete Asset action
  */
@@ -351,6 +367,12 @@ export interface IDeleteProjectAction extends IPayloadAction<string, IProject> {
     type: ActionTypes.DELETE_PROJECT_SUCCESS;
 }
 
+/**
+ * Add asset to project action type
+ */
+export interface IAddAssetToProjectAction extends IPayloadAction<string, IAsset> {
+    type: ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS;
+}
 /**
  * Load project assets action type
  */
@@ -409,6 +431,10 @@ export const saveProjectAction = createPayloadAction<ISaveProjectAction>(ActionT
  * Instance of Delete Project action
  */
 export const deleteProjectAction = createPayloadAction<IDeleteProjectAction>(ActionTypes.DELETE_PROJECT_SUCCESS);
+/**
+ * Instance of Add Asset to Project action
+ */
+export const addAssetToProjectAction = createPayloadAction<IAddAssetToProjectAction>(ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS);
 /**
  * Instance of Load Project Assets action
  */

src/redux/reducers/applicationReducer.ts

@@ -18,6 +18,8 @@ export const reducer = (state: IAppSettings = null, action: AnyAction): IAppSett
             return { ...action.payload };
         case ActionTypes.ENSURE_SECURITY_TOKEN_SUCCESS:
             return { ...action.payload };
+        case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS:
+            return { ...state, hideUploadingOption: true };
         default:
             return state;
     }

src/redux/reducers/currentProjectReducer.ts

@@ -25,6 +25,8 @@ export const reducer = (state: IProject = null, action: AnyAction): IProject =>
             return null;
         case ActionTypes.LOAD_PROJECT_SUCCESS:
             return { ...action.payload };
+        case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS:
+            return { ...state, lastVisitedAssetId: action.payload.id };
         case ActionTypes.LOAD_ASSET_METADATA_SUCCESS:
             if (!state) {
                 return state;

src/services/assetService.ts

@@ -62,6 +62,91 @@ const mimeBytesNeeded: number = (Math.max(...imageMimes.map((m) => m.pattern.len
  * @description - Functions for dealing with project assets
  */
 export class AssetService {
+    private getOcrFromAnalyzeResult(analyzeResult: any) {
+        return _.get(analyzeResult, "analyzeResult.readResults", []);
+    }
+    async uploadAssetPredictResult(asset: IAsset, readResults: any): Promise<void> {
+        const getBoundingBox = (pageIndex, arr: number[]) => {
+            const ocrForCurrentPage: any = this.getOcrFromAnalyzeResult(readResults)[pageIndex - 1];
+            const ocrExtent = [0, 0, ocrForCurrentPage.width, ocrForCurrentPage.height];
+            const ocrWidth = ocrExtent[2] - ocrExtent[0];
+            const ocrHeight = ocrExtent[3] - ocrExtent[1];
+            const result = [];
+            for (let i = 0; i < arr.length; i += 2) {
+                result.push([
+                    (arr[i] / ocrWidth),
+                    (arr[i + 1] / ocrHeight),
+                ]);
+            }
+            return result;
+        };
+        const getLabelValues = (field: any) => {
+            return field.elements.map((path: string) => {
+                const pathArr = path.split('/').slice(1);
+                const word = pathArr.reduce((obj: any, key: string) => obj[key], { ...readResults.analyzeResult });
+                return {
+                    page: field.page,
+                    text: word.text || word.state,
+                    confidence: word.confidence,
+                    boundingBoxes: [getBoundingBox(field.page, word.boundingBox)]
+                };
+            });
+        };
+        const labels = [];
+        readResults.analyzeResult.documentResults
+            .map(result => Object.keys(result.fields)
+                .filter(key => result.fields[key])
+                .map<ILabel>(key => (
+                    {
+                        label: key,
+                        key: null,
+                        value: getLabelValues(result.fields[key])
+                    }))).forEach(items => {
+                        labels.push(...items);
+                    });
+
+        if (labels.length > 0) {
+            const fileName = decodeURIComponent(asset.name).split('/').pop();
+            const labelData: ILabelData = {
+                document: fileName,
+                labels
+            };
+            const metadata = {
+                ...await this.getAssetMetadata(asset),
+                labelData
+            };
+            metadata.asset.state = AssetState.Tagged;
+
+            const ocrData = JSON.parse(JSON.stringify(readResults));
+            delete ocrData.analyzeResult.documentResults;
+            if (ocrData.analyzeResult.errors) {
+                delete ocrData.analyzeResult.errors;
+            }
+            const ocrFileName = `${asset.name}${constants.ocrFileExtension}`;
+            await Promise.all([
+                this.save(metadata),
+                this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2))
+            ]);
+        }
+        else {
+            const ocrData = { ...readResults };
+            delete ocrData.analyzeResult.documentResults;
+            if (ocrData.analyzeResult.errors) {
+                delete ocrData.analyzeResult.errors;
+            }
+            const labelFileName = decodeURIComponent(`${asset.name}${constants.labelFileExtension}`);
+            const ocrFileName = decodeURIComponent(`${asset.name}${constants.ocrFileExtension}`);
+            try {
+                await Promise.all([
+                    this.storageProvider.deleteFile(labelFileName, true, true),
+                    this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2))
+                ]);
+            }
+            catch{
+                return;
+            }
+        }
+    }
     /**
      * Create IAsset from filePath
      * @param filePath - filepath of asset
@@ -236,7 +321,14 @@ export class AssetService {
             }).filter((asset) => this.isInExactFolderPath(asset.name, folderPath));
         }
     }
-
+    public async uploadBuffer(name: string, buffer: Buffer) {
+        const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name;
+        await this.storageProvider.writeBinary(path, buffer);
+    }
+    public async uploadText(name: string, contents: string) {
+        const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name;
+        await this.storageProvider.writeText(path, contents);
+    }
     /**
      * Delete asset
      * @param metadata - Metadata for asset

src/services/predictService.ts

@@ -0,0 +1,81 @@
+import _ from "lodash";
+import url from 'url';
+import { constants } from "../common/constants";
+import { interpolate, strings } from "../common/strings";
+import { AppError, ErrorCode, IProject } from "../models/applicationState";
+import ServiceHelper from "./serviceHelper";
+
+export enum AutoLabelingStatus {
+    none,
+    running,
+    done
+}
+export class PredictService {
+
+    constructor(private project: IProject) {
+    }
+    public async getPrediction(fileUrl: string): Promise<any> {
+        const modelID = this.project.predictModelId;
+        if (!modelID) {
+            throw new AppError(
+                ErrorCode.PredictWithoutTrainForbidden,
+                strings.errors.predictWithoutTrainForbidden.message,
+                strings.errors.predictWithoutTrainForbidden.title);
+        }
+        const endpointURL = url.resolve(
+            this.project.apiUriBase,
+            `${constants.apiModelsPath}/${modelID}/analyze?includeTextDetails=true`,
+        );
+
+        const headers = { "Content-Type": "application/json", "cache-control": "no-cache" };
+        const body = { source: fileUrl };
+
+        try {
+            const response = await ServiceHelper.postWithAutoRetry(endpointURL, body, { headers }, this.project.apiKey as string);
+            const operationLocation = response.headers["operation-location"];
+
+            return this.poll(() =>
+            ServiceHelper.getWithAutoRetry(
+                operationLocation, { headers }, this.project.apiKey as string), 120000, 500);
+
+
+        } catch (err) {
+            if (err.response.status === 404) {
+                throw new AppError(
+                    ErrorCode.ModelNotFound,
+                    interpolate(strings.errors.modelNotFound.message, { modelID })
+                );
+            } else {
+                ServiceHelper.handleServiceError(err);
+            }
+        }
+    }
+    private poll = (func, timeout, interval): Promise<any> => {
+        const endTime = Number(new Date()) + (timeout || 10000);
+        interval = interval || 100;
+
+        const checkSucceeded = (resolve, reject) => {
+            const ajax = func();
+            ajax.then((response) => {
+                if (response.data.status.toLowerCase() === constants.statusCodeSucceeded) {
+                    resolve(response.data);
+                    // prediction response from API
+                    console.log("raw data", JSON.parse(response.request.response));
+                } else if (response.data.status.toLowerCase() === constants.statusCodeFailed) {
+                    reject(_.get(
+                        response,
+                        "data.analyzeResult.errors[0].errorMessage",
+                        "Generic error during prediction"));
+                } else if (Number(new Date()) < endTime) {
+                    // If the request isn't succeeded and the timeout hasn't elapsed, go again
+                    setTimeout(checkSucceeded, interval, resolve, reject);
+                } else {
+                    // Didn't succeeded after too much time, reject
+                    reject("Timed out, please try other file.");
+                }
+            });
+        };
+
+        return new Promise(checkSucceeded);
+    }
+}