Yongbing chen/human in the loop (#517)

* upload analyze file to assets

* move spinner label to strings

* adjust labels

* issuefix

* typo

* adjust getLabelValues

* typo

* remove option bar when upload asset in the future runs

* feature:  "Auto Labeling"

* move AutoLabelingStatus to predictService.ts

* update ocr when auto labeling

(cherry picked from commit 42438598d5d7a325e82bbad191ed483f44094439)

* fix tslint error

* handle formRegion.text undefined error

* decode asset name when upload asset

* add HITL document override confirm

* Merge branch 'yongbing-chen/human-in-the-loop' of https://github.com/microsoft/OCR-Form-Tools into yongbing-chen/human-in-the-loop

* disable autolabeling button when no predict model

* move runAutoLabelingOnCurrentDocument to canvas.tsx

* remove assetMetadata if no labels find after auto-labeling

* fix tslint check

* object deep clone

Co-authored-by: alex-krasn <64093224+alex-krasn@users.noreply.github.com>
This commit is contained in:
Alex Chen 2020-09-11 02:20:33 +08:00 коммит произвёл GitHub
Родитель b92b73bb80
Коммит be9d564815
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
21 изменённых файлов: 733 добавлений и 264 удалений

1
.gitignore поставляемый
Просмотреть файл

@ -37,3 +37,4 @@ secrets.sh
# complexity reports
es6-src/
report/
debug.log

Просмотреть файл

@ -202,6 +202,14 @@ export const english: IAppStrings = {
downloadScript: "Analyze with python script",
defaultLocalFileInput: "Browse for a file...",
defaultURLInput: "Paste or type URL...",
editAndUploadToTrainingSet: "Edit & upload to training set",
editAndUploadToTrainingSetNotify: "by clicking on this button, this form will be added to this project, where you can edit these labels.",
editAndUploadToTrainingSetNotify2: "We are adding this file to your training set, where you could edit the labels and re-train the model.",
uploadInPrgoress: "Upload in progress...",
confirmDuplicatedAssetName: {
title: "Asset name exists",
message: "Asset with name '${name}' exists in project, override?"
}
},
recentModelsView: {
header: "Select a model to analyze with",
@ -434,6 +442,8 @@ export const english: IAppStrings = {
subIMenuItems: {
runOcrOnCurrentDocument: "Run OCR on current document",
runOcrOnAllDocuments: "Run OCR on all documents",
runAutoLabelingCurrentDocument: "Run AutoLabeling on current document",
noPredictModelOnProject: "Predict model not avaliable, please train the model first.",
}
}
}

Просмотреть файл

@ -201,6 +201,14 @@ export const spanish: IAppStrings = {
downloadScript: "Analizar con script python",
defaultLocalFileInput: "Busca un archivo...",
defaultURLInput: "Pegar o escribir URL...",
editAndUploadToTrainingSet: "Editar y cargar al conjunto de entrenamiento",
editAndUploadToTrainingSetNotify: "Al hacer clic en este botón, este formulario se agregará al Blob de Azure Storage para este proyecto, donde puede editar estas etiquetas.",
editAndUploadToTrainingSetNotify2: "Estamos agregando este archivo a su conjunto de entrenamiento, donde puede editar las etiquetas y volver a entrenar el modelo.",
uploadInPrgoress: "carga en curso...",
confirmDuplicatedAssetName: {
title: "El nombre del activo existe",
message: "El activo con el nombre '${name}' existe en el proyecto, ¿anularlo?"
}
},
recentModelsView: {
header: "Seleccionar modelo para analizar con",
@ -435,6 +443,8 @@ export const spanish: IAppStrings = {
subIMenuItems: {
runOcrOnCurrentDocument: "Ejecutar OCR en el documento actual",
runOcrOnAllDocuments: "Ejecute OCR en todos los documentos",
runAutoLabelingCurrentDocument: "Ejecutar AutoLabeling en el documento actual",
noPredictModelOnProject: "Predecir modelo no disponible, entrene el modelo primero.",
}
}
}

Просмотреть файл

@ -475,6 +475,7 @@ export default class MockFactory {
saveProject: jest.fn(() => Promise.resolve()),
deleteProject: jest.fn(() => Promise.resolve()),
closeProject: jest.fn(() => Promise.resolve()),
addAssetToProject: jest.fn(() => Promise.resolve()),
deleteAsset: jest.fn(() => Promise.resolve()),
loadAssets: jest.fn(() => Promise.resolve()),
loadAssetMetadata: jest.fn(() => Promise.resolve()),

Просмотреть файл

@ -200,6 +200,14 @@ export interface IAppStrings {
downloadScript: string,
defaultLocalFileInput: string,
defaultURLInput: string,
editAndUploadToTrainingSet: string,
editAndUploadToTrainingSetNotify: string,
editAndUploadToTrainingSetNotify2: string,
uploadInPrgoress: string,
confirmDuplicatedAssetName: {
title: string,
message: string
},
};
recentModelsView: {
header: string;
@ -429,6 +437,8 @@ export interface IAppStrings {
subIMenuItems: {
runOcrOnCurrentDocument: string,
runOcrOnAllDocuments: string,
runAutoLabelingCurrentDocument: string,
noPredictModelOnProject: string,
}
}
}

Просмотреть файл

@ -67,6 +67,7 @@ export interface IProviderOptions {
export interface IAppSettings {
securityTokens: ISecurityToken[],
thumbnailSize?: ISize,
hideUploadingOption?: boolean;
}
/**

Просмотреть файл

@ -35,6 +35,8 @@ import { constants } from "../../../../common/constants";
import { CanvasCommandBar } from "./canvasCommandBar";
import { TooltipHost, ITooltipHostStyles } from "@fluentui/react";
import { IAppSettings } from '../../../../models/applicationState';
import { AutoLabelingStatus, PredictService } from "../../../../services/predictService";
import { AssetService } from "../../../../services/assetService";
import { strings } from "../../../../common/strings";
@ -55,6 +57,7 @@ export interface ICanvasProps extends React.Props<Canvas> {
onSelectedRegionsChanged?: (regions: IRegion[]) => void;
onCanvasRendered?: (canvas: HTMLCanvasElement) => void;
onRunningOCRStatusChanged?: (isRunning: boolean) => void;
onRunningAutoLabelingStatusChanged?: (isRunning: boolean) => void;
onTagChanged?: (oldTag: ITag, newTag: ITag) => void;
runOcrForAllDocs?: (runForAllDocs: boolean) => void;
onAssetDeleted?: () => void;
@ -75,6 +78,7 @@ export interface ICanvasState {
errorTitle?: string;
errorMessage: string;
ocrStatus: OcrStatus;
autoLableingStatus: AutoLabelingStatus;
layers: any;
tableIconTooltip: any;
hoveringFeature: string;
@ -134,6 +138,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
isError: false,
errorMessage: undefined,
ocrStatus: OcrStatus.done,
autoLableingStatus: AutoLabelingStatus.none,
layers: { text: true, tables: true, checkboxes: true, label: true, drawnRegions: true },
tableIconTooltip: { display: "none", width: 0, height: 0, top: 0, left: 0 },
hoveringFeature: null,
@ -246,9 +251,11 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
handleRunOcr={this.runOcr}
handleAssetDeleted={this.props.onAssetDeleted}
handleRunOcrForAllDocuments={this.runOcrForAllDocuments}
handleRunAutoLabelingOnCurrentDocument={this.runAutoLabelingOnCurrentDocument}
connectionType={this.props.project.sourceConnection.providerType}
handleToggleDrawRegionMode={this.handleToggleDrawRegionMode}
drawRegionMode={this.state.drawRegionMode}
project={this.props.project}
parentPage={strings.editorPage.title}
/>
<ImageMap
@ -328,6 +335,14 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
</div>
</div>
}
{this.state.autoLableingStatus === AutoLabelingStatus.running &&
<div className="canvas-ocr-loading">
<div className="canvas-ocr-loading-spinner">
<Label className="p-0" ></Label>
<Spinner size={SpinnerSize.large} label="Running Auto Labeling..." ariaLive="assertive" labelPosition="right" />
</div>
</div>
}
<Alert
show={this.state.isError}
title={this.state.errorTitle || "Error"}
@ -347,6 +362,24 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
this.props.runOcrForAllDocs(true);
}
private runAutoLabelingOnCurrentDocument = async () => {
try {
this.setAutoLabelingStatus(AutoLabelingStatus.running);
const asset = this.state.currentAsset.asset;
const assetPath = asset.path;
const predictService = new PredictService(this.props.project);
const result = await predictService.getPrediction(assetPath);
const assetService = new AssetService(this.props.project);
await assetService.uploadAssetPredictResult(asset, result);
const assetMetadata = await assetService.getAssetMetadata(asset);
await this.props.onAssetMetadataChanged(assetMetadata);
}
finally {
this.setAutoLabelingStatus(AutoLabelingStatus.done);
}
}
public updateSize() {
this.imageMap.updateSize();
}
@ -977,8 +1010,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
if (category === FeatureCategory.DrawnRegion ||
(category === FeatureCategory.Label && this.state.currentAsset.regions
.find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion))
{
.find((r) => r.id === regionId).category === FeatureCategory.DrawnRegion)) {
selectedRegions.forEach((region) => {
if (region?.category !== FeatureCategory.DrawnRegion) {
this.removeFromSelectedRegions(region.id)
@ -1105,6 +1137,13 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
}
});
}
private setAutoLabelingStatus = (autoLableingStatus: AutoLabelingStatus) => {
this.setState({ autoLableingStatus }, () => {
if (this.props.onRunningAutoLabelingStatusChanged) {
this.props.onRunningAutoLabelingStatusChanged(autoLableingStatus === AutoLabelingStatus.running);
}
})
}
private runOcr = () => {
this.loadOcr(true);
@ -1247,7 +1286,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
private convertLabelDataToRegions = (labelData: ILabelData): IRegion[] => {
const regions = [];
if (labelData.labels) {
if (labelData && labelData.labels) {
labelData.labels.forEach((label) => {
if (label.value) {
label.value.forEach((formRegion) => {
@ -1555,7 +1594,7 @@ export default class Canvas extends React.Component<ICanvasProps, ICanvasState>
private getBoundingBoxTextFromRegion = (formRegion: IFormRegion, boundingBoxIndex: number) => {
// get value from formRegion.text
const regionValues = formRegion.text.split(" ");
const regionValues = formRegion.text && formRegion.text.split(" ");
if (regionValues && regionValues.length > boundingBoxIndex) {
return regionValues[boundingBoxIndex];
}

Просмотреть файл

@ -0,0 +1,3 @@
.ms-ContextualMenu-link.is-disabled {
color: gray;
}

Просмотреть файл

@ -4,10 +4,14 @@ import { ICustomizations, Customizer } from "@fluentui/react/lib/Utilities";
import { getDarkGreyTheme } from "../../../../common/themes";
import { strings } from '../../../../common/strings';
import { ContextualMenuItemType } from "@fluentui/react";
import { IProject } from "../../../../models/applicationState";
import "./canvasCommandBar.scss";
interface ICanvasCommandBarProps {
handleZoomIn: () => void;
handleZoomOut: () => void;
handleRunAutoLabelingOnCurrentDocument?: () => void;
project: IProject;
handleRotateImage: (degrees: number) => void;
handleRunOcr?: () => void;
handleRunOcrForAllDocuments?: () => void;
@ -159,6 +163,17 @@ export const CanvasCommandBar: React.FunctionComponent<ICanvasCommandBarProps> =
iconProps: { iconName: "Documentation" },
onClick: () => props.handleRunOcrForAllDocuments(),
},
{
key: "runAutoLabelingCurrentDocument",
text: strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.runAutoLabelingCurrentDocument,
iconProps: { iconName: "Tag" },
disabled: !props.project.predictModelId,
title: props.project.predictModelId ? "" :
strings.editorPage.canvas.canvasCommandBar.farItems.additionalActions.subIMenuItems.noPredictModelOnProject,
onClick: () => {
props.handleRunAutoLabelingOnCurrentDocument();
},
},
{
key: 'divider_1',
itemType: ContextualMenuItemType.Divider,

Просмотреть файл

@ -37,6 +37,8 @@ import PreventLeaving from "../../common/preventLeaving/preventLeaving";
import { Spinner, SpinnerSize } from "@fluentui/react/lib/Spinner";
import { getPrimaryGreenTheme, getPrimaryRedTheme } from "../../../../common/themes";
import { toast } from "react-toastify";
import { PredictService } from "../../../../services/predictService";
import { AssetService } from "../../../../services/assetService";
/**
* Properties for Editor Page
@ -87,6 +89,7 @@ export interface IEditorPageState {
isRunningOCRs?: boolean;
/** Whether OCR is running in the main canvas */
isCanvasRunningOCR?: boolean;
isCanvasRunningAutoLabeling?: boolean;
isError?: boolean;
errorTitle?: string;
errorMessage?: string;
@ -178,7 +181,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
public render() {
const { project } = this.props;
const { assets, selectedAsset, isRunningOCRs, isCanvasRunningOCR } = this.state;
const { assets, selectedAsset, isRunningOCRs, isCanvasRunningOCR, isCanvasRunningAutoLabeling } = this.state;
const labels = (selectedAsset &&
selectedAsset.labelData &&
@ -222,7 +225,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
className="editor-page-sidebar-run-ocr"
type="button"
onClick={() => this.loadOcrForNotVisited()}
disabled={this.state.isRunningOCRs}>
disabled={this.isBusy()}>
{this.state.isRunningOCRs ?
<div>
<Spinner
@ -263,6 +266,7 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
onCanvasRendered={this.onCanvasRendered}
onSelectedRegionsChanged={this.onSelectedRegionsChanged}
onRunningOCRStatusChanged={this.onCanvasRunningOCRStatusChanged}
onRunningAutoLabelingStatusChanged={this.onCanvasRunningAutoLabelingStatusChanged}
onTagChanged={this.onTagChanged}
onAssetDeleted={this.confirmDocumentDeleted}
editorMode={this.state.editorMode}
@ -352,6 +356,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
when={isRunningOCRs || isCanvasRunningOCR}
message={"An OCR operation is currently in progress, are you sure you want to leave?"}
/>
<PreventLeaving
when={isCanvasRunningAutoLabeling}
message={"An AutoLabeling option is currently in progress, are you sure you want to leave?"} />
</div>
);
}
@ -631,6 +638,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
this.setState({ showInvalidRegionWarning: true });
return;
}
if (this.state.isCanvasRunningAutoLabeling) {
return;
}
const assetMetadata = await this.props.actions.loadAssetMetadata(this.props.project, asset);
@ -687,9 +697,12 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
throw Error(error);
}
}
private isBusy = (): boolean => {
return this.state.isRunningOCRs || this.state.isCanvasRunningOCR || this.state.isCanvasRunningAutoLabeling;
}
public loadOcrForNotVisited = async (runForAll?: boolean) => {
if (this.state.isRunningOCRs) {
if (this.isBusy()) {
return;
}
const { project } = this.props;
@ -784,7 +797,9 @@ export default class EditorPage extends React.Component<IEditorPageProps, IEdito
private onCanvasRunningOCRStatusChanged = (isCanvasRunningOCR: boolean) => {
this.setState({ isCanvasRunningOCR });
}
private onCanvasRunningAutoLabelingStatusChanged = (isCanvasRunningAutoLabeling: boolean) => {
this.setState({ isCanvasRunningAutoLabeling });
}
private onFocused = () => {
this.loadProjectAssets();
}

Просмотреть файл

@ -1,45 +1,46 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import {
DefaultButton, Dropdown, FontIcon, IconButton, IDropdownOption,
ISelection, PrimaryButton, Selection,
SelectionMode, Separator, Spinner, SpinnerSize, TextField
} from "@fluentui/react";
import axios from "axios";
import _ from "lodash";
import { Feature } from "ol";
import Polygon from "ol/geom/Polygon";
import Fill from "ol/style/Fill";
import Stroke from "ol/style/Stroke";
import Style from "ol/style/Style";
import pdfjsLib from "pdfjs-dist";
import React from "react";
import { connect } from "react-redux";
import { RouteComponentProps } from "react-router-dom";
import { bindActionCreators } from "redux";
import url from "url";
import { constants } from "../../../../common/constants";
import HtmlFileReader from "../../../../common/htmlFileReader";
import { interpolate, strings } from "../../../../common/strings";
import {
FontIcon, Selection, PrimaryButton, Spinner, SpinnerSize, IconButton, TextField, IDropdownOption,
Dropdown, DefaultButton, Separator, ISelection, SelectionMode
} from "@fluentui/react";
import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
getGreenWithWhiteBackgroundTheme, getPrimaryGreenTheme, getPrimaryWhiteTheme,
getRightPaneDefaultButtonTheme
} from "../../../../common/themes";
import { loadImageToCanvas, parseTiffData, renderTiffToCanvas } from "../../../../common/utils";
import { AppError, ErrorCode, IApplicationState, IAppSettings, IConnection, ImageMapParent, IProject, IRecentModel } from "../../../../models/applicationState";
import IApplicationActions, * as applicationActions from "../../../../redux/actions/applicationActions";
import IAppTitleActions, * as appTitleActions from "../../../../redux/actions/appTitleActions";
import "./predictPage.scss";
import {
IApplicationState, IConnection, IProject, IAppSettings, AppError, ErrorCode, IRecentModel, ImageMapParent,
} from "../../../../models/applicationState";
import { ImageMap } from "../../common/imageMap/imageMap";
import Style from "ol/style/Style";
import Stroke from "ol/style/Stroke";
import Fill from "ol/style/Fill";
import PredictResult from "./predictResult";
import _ from "lodash";
import pdfjsLib from "pdfjs-dist";
import Alert from "../../common/alert/alert";
import url from "url";
import HtmlFileReader from "../../../../common/htmlFileReader";
import { Feature } from "ol";
import Polygon from "ol/geom/Polygon";
import { strings, interpolate } from "../../../../common/strings";
import PreventLeaving from "../../common/preventLeaving/preventLeaving";
import IProjectActions, * as projectActions from "../../../../redux/actions/projectActions";
import ServiceHelper from "../../../../services/serviceHelper";
import { parseTiffData, renderTiffToCanvas, loadImageToCanvas } from "../../../../common/utils";
import { constants } from "../../../../common/constants";
import { getPrimaryGreenTheme, getPrimaryWhiteTheme,
getGreenWithWhiteBackgroundTheme,
getRightPaneDefaultButtonTheme} from "../../../../common/themes";
import axios from "axios";
import { IAnalyzeModelInfo } from './predictResult';
import RecentModelsView from "./recentModelsView";
import { getAppInsights } from '../../../../services/telemetryService';
import Alert from "../../common/alert/alert";
import Confirm from "../../common/confirm/confirm";
import { ImageMap } from "../../common/imageMap/imageMap";
import PreventLeaving from "../../common/preventLeaving/preventLeaving";
import "./predictPage.scss";
import PredictResult, { IAnalyzeModelInfo } from "./predictResult";
import RecentModelsView from "./recentModelsView";
import { UploadToTrainingSetView } from "./uploadToTrainingSetView";
import { CanvasCommandBar } from "../editorPage/canvasCommandBar";
pdfjsLib.GlobalWorkerOptions.workerSrc = constants.pdfjsWorkerSrc(pdfjsLib.version);
@ -83,6 +84,7 @@ export interface IPredictPageState {
highlightedField: string;
modelList: IModel[];
modelOption: string;
confirmDuplicatedAssetNameMessage?: string;
imageAngle: number;
}
@ -149,6 +151,8 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
private currPdf: any;
private tiffImages: any[];
private imageMap: ImageMap;
private uploadToTrainingSetView: React.RefObject<UploadToTrainingSetView> = React.createRef();
private duplicateAssetNameConfirm: React.RefObject<Confirm> = React.createRef();
public async componentDidMount() {
const projectId = this.props.match.params["projectId"];
@ -415,17 +419,29 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
page={this.state.currPage}
tags={this.props.project.tags}
downloadResultLabel={this.state.fileLabel}
onAddAssetToProject={this.onAddAssetToProjectClick}
onPredictionClick={this.onPredictionClick}
onPredictionMouseEnter={this.onPredictionMouseEnter}
onPredictionMouseLeave={this.onPredictionMouseLeave}
/>
}
<UploadToTrainingSetView
showOption={!this.props.appSettings.hideUploadingOption}
ref={this.uploadToTrainingSetView}
onConfirm={this.onAddAssetToProject} />
{
(Object.keys(predictions).length === 0 && this.state.predictRun) &&
<div>
No field can be extracted.
</div>
}
<Confirm
ref={this.duplicateAssetNameConfirm}
title={strings.predict.confirmDuplicatedAssetName.title}
message={this.state.confirmDuplicatedAssetNameMessage}
onConfirm={this.onAddAssetToProjectConfirm}
confirmButtonTheme={getPrimaryGreenTheme()}
/>
</div>
</>
}
@ -631,6 +647,7 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
handleZoomIn={this.handleCanvasZoomIn}
handleZoomOut={this.handleCanvasZoomOut}
handleRotateImage={this.handleRotateCanvas}
project={this.props.project}
parentPage={"predict"}
/>
<ImageMap
@ -1056,7 +1073,41 @@ export default class PredictPage extends React.Component<IPredictPageProps, IPre
private noOp = () => {
// no operation
}
private onAddAssetToProjectClick = async () => {
if (this.state.file) {
// this.props.project.assets
const fileName = `${this.props.project.folderPath}/${decodeURIComponent(this.state.file.name)}`;
const asset = Object.values(this.props.project.assets).find(asset => asset.name === fileName);
if (asset) {
const confirmDuplicatedAssetNameMessage = interpolate(strings.predict.confirmDuplicatedAssetName.message, { name: decodeURI(this.state.file.name) });
this.setState({
confirmDuplicatedAssetNameMessage
});
this.duplicateAssetNameConfirm.current.open();
}
else {
this.onAddAssetToProjectConfirm();
}
}
}
private onAddAssetToProjectConfirm = async () => {
if (this.props.appSettings.hideUploadingOption) {
this.uploadToTrainingSetView.current.open();
await this.onAddAssetToProject();
this.uploadToTrainingSetView.current.close();
} else {
this.uploadToTrainingSetView.current.open();
}
}
private onAddAssetToProject = async () => {
if (this.state.file) {
const fileData = new Buffer(await this.state.file.arrayBuffer());
const readResults: any = this.state.analyzeResult;
const fileName = decodeURIComponent(this.state.file.name).split("/").pop();
await this.props.actions.addAssetToProject(this.props.project, fileName, fileData, readResults);
this.props.history.push(`/projects/${this.props.project.id}/edit`);
}
}
private onPredictionClick = (predictedItem: any) => {
const targetPage = predictedItem.page;
if (Number.isInteger(targetPage) && targetPage !== this.state.currPage) {

Просмотреть файл

@ -7,6 +7,7 @@ import "./predictResult.scss";
import { getPrimaryGreenTheme } from "../../../../common/themes";
import { PrimaryButton } from "@fluentui/react";
import PredictModelInfo from './predictModelInfo';
import { strings } from "../../../../common/strings";
export interface IAnalyzeModelInfo {
docType: string,
@ -21,6 +22,7 @@ export interface IPredictResultProps {
page: number;
tags: ITag[];
downloadResultLabel: string;
onAddAssetToProject?: () => void;
onPredictionClick?: (item: any) => void;
onPredictionMouseEnter?: (item: any) => void;
onPredictionMouseLeave?: (item: any) => void;
@ -46,6 +48,13 @@ export default class PredictResult extends React.Component<IPredictResultProps,
<div>
<div className="container-items-center container-space-between results-container">
<h5 className="results-header">Prediction results</h5>
</div>
<div className="container-items-center container-space-between">
<PrimaryButton
theme={getPrimaryGreenTheme()}
onClick={this.onAddAssetToProject}
text={strings.predict.editAndUploadToTrainingSet} />
<PrimaryButton
className="align-self-end keep-button-80px"
theme={getPrimaryGreenTheme()}
@ -142,6 +151,11 @@ export default class PredictResult extends React.Component<IPredictResultProps,
return data;
}
private onAddAssetToProject = async () => {
if (this.props.onAddAssetToProject) {
this.props.onAddAssetToProject();
}
}
private triggerDownload = (): void => {
const { analyzeResult } = this.props;
const predictionData = JSON.stringify(this.sanitizeData(analyzeResult));

Просмотреть файл

@ -0,0 +1,3 @@
.upload-to-training-set-modal {
max-width: 34em;
}

Просмотреть файл

@ -0,0 +1,90 @@
import { Customizer, ICustomizations, Modal, PrimaryButton, Spinner, SpinnerSize } from '@fluentui/react';
import React from 'react';
import { strings } from "../../../../common/strings";
import { getDarkGreyTheme, getDefaultDarkTheme, getPrimaryGreenTheme, getPrimaryGreyTheme } from '../../../../common/themes';
import './uploadToTrainingSetView.scss';
interface IUploadToTrainingSetViewProp {
onConfirm?: () => Promise<void>;
showOption: boolean;
}
interface IUploadToTrainingSetViewState {
hideModal: boolean;
isLoading: boolean;
}
export class UploadToTrainingSetView extends React.Component<IUploadToTrainingSetViewProp, IUploadToTrainingSetViewState>{
constructor(props) {
super(props);
this.state = {
hideModal: true,
isLoading: !props.showOption,
};
this.open = this.open.bind(this);
this.close = this.close.bind(this);
this.onConfirm = this.onConfirm.bind(this);
}
open() {
this.setState({
hideModal: false,
isLoading: !this.props.showOption
});
}
close() {
this.setState({ hideModal: true });
}
async onConfirm() {
this.setState({ isLoading: true });
if (this.props.onConfirm) {
await this.props.onConfirm();
}
this.close();
}
render() {
const dark: ICustomizations = {
settings: {
theme: getDarkGreyTheme(),
},
scopedSettings: {},
};
const notifyMessage = this.props.showOption ? strings.predict.editAndUploadToTrainingSetNotify : strings.predict.editAndUploadToTrainingSetNotify2;
return (
<>
<Customizer {...dark}>
<Modal
isOpen={!this.state.hideModal}
isModeless={false}
containerClassName="modal-container upload-to-training-set-modal"
scrollableContentClassName="scrollable-content"
>
<h4>Notice: <small>{notifyMessage}</small></h4>
<div className="modal-buttons-container mt-4">
{this.state.isLoading ?
<div>
<Spinner
label={strings.predict.uploadInPrgoress}
ariaLive="assertive"
labelPosition="right"
theme={getDefaultDarkTheme()}
size={SpinnerSize.large} />
</div> :
<div>
<PrimaryButton
className="mr-3"
text={strings.predict.editAndUploadToTrainingSet}
theme={getPrimaryGreenTheme()}
onClick={this.onConfirm} />
<PrimaryButton
className="modal-cancel"
theme={getPrimaryGreyTheme()}
onClick={this.close}
text="Cancel" />
</div>
}
</div>
</Modal>
</Customizer>
</>
)
}
}

Просмотреть файл

@ -24,6 +24,7 @@ import {
IUpdateProjectTagAction,
IUpdateProjectTagsFromFilesAction,
IUpdateTagDocumentCount,
IAddAssetToProjectAction,
} from "./projectActions";
import {
IShowAppErrorAction,
@ -83,6 +84,7 @@ export type AnyAction = IOtherAction |
IDeleteConnectionAction |
ILoadConnectionAction |
ISaveConnectionAction |
IAddAssetToProjectAction|
IDeleteConnectionAction |
ILoadProjectAction |
ICloseProjectAction |

Просмотреть файл

@ -12,6 +12,7 @@ export enum ActionTypes {
// Projects
LOAD_PROJECT_SUCCESS = "LOAD_PROJECT_SUCCESS",
SAVE_PROJECT_SUCCESS = "SAVE_PROJECT_SUCCESS",
ADD_ASSET_TO_PROJECT_SUCCESS = "ADD_ASSET_TO_PROJECT_SUCCESS",
DELETE_PROJECT_SUCCESS = "DELETE_PROJECT_SUCCESS",
CLOSE_PROJECT_SUCCESS = "CLOSE_PROJECT_SUCCESS",
LOAD_PROJECT_ASSETS_SUCCESS = "LOAD_PROJECT_ASSETS_SUCCESS",

Просмотреть файл

@ -29,6 +29,7 @@ export default interface IProjectActions {
saveProject(project: IProject, saveTags?: boolean, updateTagsFromFiles?: boolean): Promise<IProject>;
deleteProject(project: IProject): Promise<void>;
closeProject(): void;
addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): Promise<IAsset>;
deleteAsset(project: IProject, assetMetadata: IAssetMetadata): Promise<void>;
loadAssets(project: IProject): Promise<IAsset[]>;
loadAssetMetadata(project: IProject, asset: IAsset): Promise<IAssetMetadata>;
@ -169,7 +170,22 @@ export function closeProject(): (dispatch: Dispatch) => void {
dispatch({ type: ActionTypes.CLOSE_PROJECT_SUCCESS });
};
}
/**
* add asset, ocr data, labels to project storage.
*/
export function addAssetToProject(project: IProject, fileName: string, buffer: Buffer, analyzeResult: any): (dispatch: Dispatch) => Promise<IAsset> {
return async (dispatch: Dispatch) => {
const assetService = new AssetService(project);
await assetService.uploadBuffer(fileName, buffer);
const assets = await assetService.getAssets();
const assetName = project.folderPath ? `${project.folderPath}/${fileName}` : fileName;
const asset = assets.find(a => a.name === assetName);
await assetService.uploadAssetPredictResult(asset, analyzeResult);
dispatch(addAssetToProjectAction(asset));
return asset;
};
}
/**
* Dispatches Delete Asset action
*/
@ -351,6 +367,12 @@ export interface IDeleteProjectAction extends IPayloadAction<string, IProject> {
type: ActionTypes.DELETE_PROJECT_SUCCESS;
}
/**
* Add asset to project action type
*/
export interface IAddAssetToProjectAction extends IPayloadAction<string, IAsset> {
type: ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS;
}
/**
* Load project assets action type
*/
@ -409,6 +431,10 @@ export const saveProjectAction = createPayloadAction<ISaveProjectAction>(ActionT
* Instance of Delete Project action
*/
export const deleteProjectAction = createPayloadAction<IDeleteProjectAction>(ActionTypes.DELETE_PROJECT_SUCCESS);
/**
* Instance of Add Asset to Project action
*/
export const addAssetToProjectAction = createPayloadAction<IAddAssetToProjectAction>(ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS);
/**
* Instance of Load Project Assets action
*/

Просмотреть файл

@ -18,6 +18,8 @@ export const reducer = (state: IAppSettings = null, action: AnyAction): IAppSett
return { ...action.payload };
case ActionTypes.ENSURE_SECURITY_TOKEN_SUCCESS:
return { ...action.payload };
case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS:
return { ...state, hideUploadingOption: true };
default:
return state;
}

Просмотреть файл

@ -25,6 +25,8 @@ export const reducer = (state: IProject = null, action: AnyAction): IProject =>
return null;
case ActionTypes.LOAD_PROJECT_SUCCESS:
return { ...action.payload };
case ActionTypes.ADD_ASSET_TO_PROJECT_SUCCESS:
return { ...state, lastVisitedAssetId: action.payload.id };
case ActionTypes.LOAD_ASSET_METADATA_SUCCESS:
if (!state) {
return state;

Просмотреть файл

@ -62,6 +62,91 @@ const mimeBytesNeeded: number = (Math.max(...imageMimes.map((m) => m.pattern.len
* @description - Functions for dealing with project assets
*/
export class AssetService {
private getOcrFromAnalyzeResult(analyzeResult: any) {
return _.get(analyzeResult, "analyzeResult.readResults", []);
}
async uploadAssetPredictResult(asset: IAsset, readResults: any): Promise<void> {
const getBoundingBox = (pageIndex, arr: number[]) => {
const ocrForCurrentPage: any = this.getOcrFromAnalyzeResult(readResults)[pageIndex - 1];
const ocrExtent = [0, 0, ocrForCurrentPage.width, ocrForCurrentPage.height];
const ocrWidth = ocrExtent[2] - ocrExtent[0];
const ocrHeight = ocrExtent[3] - ocrExtent[1];
const result = [];
for (let i = 0; i < arr.length; i += 2) {
result.push([
(arr[i] / ocrWidth),
(arr[i + 1] / ocrHeight),
]);
}
return result;
};
const getLabelValues = (field: any) => {
return field.elements.map((path: string) => {
const pathArr = path.split('/').slice(1);
const word = pathArr.reduce((obj: any, key: string) => obj[key], { ...readResults.analyzeResult });
return {
page: field.page,
text: word.text || word.state,
confidence: word.confidence,
boundingBoxes: [getBoundingBox(field.page, word.boundingBox)]
};
});
};
const labels = [];
readResults.analyzeResult.documentResults
.map(result => Object.keys(result.fields)
.filter(key => result.fields[key])
.map<ILabel>(key => (
{
label: key,
key: null,
value: getLabelValues(result.fields[key])
}))).forEach(items => {
labels.push(...items);
});
if (labels.length > 0) {
const fileName = decodeURIComponent(asset.name).split('/').pop();
const labelData: ILabelData = {
document: fileName,
labels
};
const metadata = {
...await this.getAssetMetadata(asset),
labelData
};
metadata.asset.state = AssetState.Tagged;
const ocrData = JSON.parse(JSON.stringify(readResults));
delete ocrData.analyzeResult.documentResults;
if (ocrData.analyzeResult.errors) {
delete ocrData.analyzeResult.errors;
}
const ocrFileName = `${asset.name}${constants.ocrFileExtension}`;
await Promise.all([
this.save(metadata),
this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2))
]);
}
else {
const ocrData = { ...readResults };
delete ocrData.analyzeResult.documentResults;
if (ocrData.analyzeResult.errors) {
delete ocrData.analyzeResult.errors;
}
const labelFileName = decodeURIComponent(`${asset.name}${constants.labelFileExtension}`);
const ocrFileName = decodeURIComponent(`${asset.name}${constants.ocrFileExtension}`);
try {
await Promise.all([
this.storageProvider.deleteFile(labelFileName, true, true),
this.storageProvider.writeText(ocrFileName, JSON.stringify(ocrData, null, 2))
]);
}
catch{
return;
}
}
}
/**
* Create IAsset from filePath
* @param filePath - filepath of asset
@ -236,7 +321,14 @@ export class AssetService {
}).filter((asset) => this.isInExactFolderPath(asset.name, folderPath));
}
}
public async uploadBuffer(name: string, buffer: Buffer) {
const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name;
await this.storageProvider.writeBinary(path, buffer);
}
public async uploadText(name: string, contents: string) {
const path = this.project.folderPath ? `${this.project.folderPath}/${name}` : name;
await this.storageProvider.writeText(path, contents);
}
/**
* Delete asset
* @param metadata - Metadata for asset

Просмотреть файл

@ -0,0 +1,81 @@
import _ from "lodash";
import url from 'url';
import { constants } from "../common/constants";
import { interpolate, strings } from "../common/strings";
import { AppError, ErrorCode, IProject } from "../models/applicationState";
import ServiceHelper from "./serviceHelper";
export enum AutoLabelingStatus {
none,
running,
done
}
export class PredictService {
constructor(private project: IProject) {
}
public async getPrediction(fileUrl: string): Promise<any> {
const modelID = this.project.predictModelId;
if (!modelID) {
throw new AppError(
ErrorCode.PredictWithoutTrainForbidden,
strings.errors.predictWithoutTrainForbidden.message,
strings.errors.predictWithoutTrainForbidden.title);
}
const endpointURL = url.resolve(
this.project.apiUriBase,
`${constants.apiModelsPath}/${modelID}/analyze?includeTextDetails=true`,
);
const headers = { "Content-Type": "application/json", "cache-control": "no-cache" };
const body = { source: fileUrl };
try {
const response = await ServiceHelper.postWithAutoRetry(endpointURL, body, { headers }, this.project.apiKey as string);
const operationLocation = response.headers["operation-location"];
return this.poll(() =>
ServiceHelper.getWithAutoRetry(
operationLocation, { headers }, this.project.apiKey as string), 120000, 500);
} catch (err) {
if (err.response.status === 404) {
throw new AppError(
ErrorCode.ModelNotFound,
interpolate(strings.errors.modelNotFound.message, { modelID })
);
} else {
ServiceHelper.handleServiceError(err);
}
}
}
private poll = (func, timeout, interval): Promise<any> => {
const endTime = Number(new Date()) + (timeout || 10000);
interval = interval || 100;
const checkSucceeded = (resolve, reject) => {
const ajax = func();
ajax.then((response) => {
if (response.data.status.toLowerCase() === constants.statusCodeSucceeded) {
resolve(response.data);
// prediction response from API
console.log("raw data", JSON.parse(response.request.response));
} else if (response.data.status.toLowerCase() === constants.statusCodeFailed) {
reject(_.get(
response,
"data.analyzeResult.errors[0].errorMessage",
"Generic error during prediction"));
} else if (Number(new Date()) < endTime) {
// If the request isn't succeeded and the timeout hasn't elapsed, go again
setTimeout(checkSucceeded, interval, resolve, reject);
} else {
// Didn't succeeded after too much time, reject
reject("Timed out, please try other file.");
}
});
};
return new Promise(checkSucceeded);
}
}