diff --git a/.eslintrc.json b/.eslintrc.json index 756dc5491..7ccbeb9cc 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -36,7 +36,8 @@ ] } ], - "no-alert": ["error"] + "no-alert": ["error"], + "import/no-internal-modules": "off" } }, { diff --git a/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts b/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts index 905ed265f..0e93c4e0d 100644 --- a/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts +++ b/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts @@ -12,8 +12,8 @@ export const fridgeObjectDetection: IDataset = { features: [ [96.30899737412763], [95.32630225415797], - [1003762680516188], - [92000130390912], + [100.3762680516188], + [92.000130390912], [95.33849179841164] ], images: fridgeObjectDetectionImages, @@ -40,11 +40,11 @@ export const fridgeObjectDetection: IDataset = { ], [ [ - 2, 47.880287170410156, 1465001831054688, 212.69313049316406, + 2, 47.880287170410156, 146.5001831054688, 212.69313049316406, 513.9315795898438, 0.9914382696151733 ], [ - 4, 322.61370849609375, 1733768920898438, 455.5107421875, + 4, 322.61370849609375, 173.3768920898438, 455.5107421875, 498.9791564941406, 0.9787105917930603 ] ], @@ -54,8 +54,8 @@ export const fridgeObjectDetection: IDataset = { 412.123779296875, 0.9933412671089172 ], [ - 3, 52.82023239135742, 3065950927734375, 363.34197998046875, - 420689697265625, 0.979895830154419 + 3, 52.82023239135742, 306.5950927734375, 363.34197998046875, + 420.689697265625, 0.979895830154419 ] ], [ @@ -64,7 +64,7 @@ export const fridgeObjectDetection: IDataset = { 507.566650390625, 0.9822636246681213 ], [ - 4, 98.31806182861328, 172.11666870117188, 2242011108398438, + 4, 98.31806182861328, 172.11666870117188, 224.2011108398438, 483.5189208984375, 0.9667745232582092 ] ] diff --git a/libs/core-ui/src/lib/Interfaces/IDataset.ts b/libs/core-ui/src/lib/Interfaces/IDataset.ts index de1eae6be..700f69305 100644 --- a/libs/core-ui/src/lib/Interfaces/IDataset.ts +++ b/libs/core-ui/src/lib/Interfaces/IDataset.ts @@ -42,6 +42,7 @@ export interface IDataset { index?: string[]; object_detection_true_y?: number[][][]; object_detection_predicted_y?: number[][][]; + imageDimensions?: Array<[number, number]>; } // TODO Remove DatasetSummary when possible diff --git a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx index f7a35d3b7..f5cfc5d3e 100644 --- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx +++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx @@ -1,31 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { - ComboBox, - IComboBox, - IComboBoxOption, - Icon, - Image, - ImageFit, - List, - Panel, - PanelType, - FocusZone, - Stack, - Text, - Spinner, - Separator -} from "@fluentui/react"; -import { FluentUIStyles, IVisionListItem } from "@responsible-ai/core-ui"; +import * as FluentUI from "@fluentui/react"; +import { FluentUIStyles } from "@responsible-ai/core-ui"; import { localization } from "@responsible-ai/localization"; -import React from "react"; +import * as React from "react"; +import { CanvasTools } from "vott-ct"; -import { - generateSelectableObjectDetectionIndexes, - onRenderCell, - updateMetadata -} from "../utils/FlyoutUtils"; +import * as FlyoutStyles from "../utils/FlyoutUtils"; import { getJoinedLabelString } from "../utils/labelUtils"; import { @@ -33,34 +15,13 @@ import { explanationImage, explanationImageWidth } from "./Flyout.styles"; +import * as FlyoutODUtils from "./FlyoutObjectDetectionUtils"; -export interface IFlyoutProps { - explanations: Map>; - isOpen: boolean; - item: IVisionListItem | undefined; - loadingExplanation: boolean[][]; - otherMetadataFieldNames: string[]; - callback: () => void; - onChange: (item: IVisionListItem, index: number) => void; -} - -export interface IFlyoutState { - item: IVisionListItem | undefined; - metadata: Array> | undefined; - selectableObjectIndexes: IComboBoxOption[]; - odSelectedKey: string; -} - -const stackTokens = { - large: { childrenGap: "l2" }, - medium: { childrenGap: "l1" } -}; -const ExcessLabelLen = localization.InterpretVision.Dashboard.prefix.length; export class FlyoutObjectDetection extends React.Component< - IFlyoutProps, - IFlyoutState + FlyoutODUtils.IFlyoutProps, + FlyoutODUtils.IFlyoutState > { - public constructor(props: IFlyoutProps) { + public constructor(props: FlyoutODUtils.IFlyoutProps) { super(props); this.state = { item: undefined, @@ -69,30 +30,37 @@ export class FlyoutObjectDetection extends React.Component< selectableObjectIndexes: [] }; } + public componentDidMount(): void { const item = this.props.item; if (!item) { return; } const fieldNames = this.props.otherMetadataFieldNames; - const metadata = updateMetadata(item, fieldNames); - const selectableObjectIndexes = generateSelectableObjectDetectionIndexes( - localization.InterpretVision.Dashboard.prefix, - item - ); + const metadata = FlyoutStyles.updateMetadata(item, fieldNames); + const selectableObjectIndexes = + FlyoutStyles.generateSelectableObjectDetectionIndexes( + localization.InterpretVision.Dashboard.prefix, + item + ); this.setState({ item, metadata, selectableObjectIndexes }); } - public componentDidUpdate(prevProps: IFlyoutProps): void { + + public componentDidUpdate(prevProps: FlyoutODUtils.IFlyoutProps): void { if (prevProps !== this.props) { const item = this.props.item; if (!item) { return; } - const metadata = updateMetadata(item, this.props.otherMetadataFieldNames); - const selectableObjectIndexes = generateSelectableObjectDetectionIndexes( - localization.InterpretVision.Dashboard.prefix, - item + const metadata = FlyoutStyles.updateMetadata( + item, + this.props.otherMetadataFieldNames ); + const selectableObjectIndexes = + FlyoutStyles.generateSelectableObjectDetectionIndexes( + localization.InterpretVision.Dashboard.prefix, + item + ); this.setState({ item: this.props.item, metadata, @@ -100,6 +68,7 @@ export class FlyoutObjectDetection extends React.Component< }); } } + public render(): React.ReactNode { const { isOpen } = this.props; const item = this.state.item; @@ -109,43 +78,46 @@ export class FlyoutObjectDetection extends React.Component< const classNames = flyoutStyles(); const predictedY = getJoinedLabelString(item?.predictedY); const trueY = getJoinedLabelString(item?.trueY); + return ( - - + - - - - - - - + + + + + + - - + - - - + - - + + {predictedY !== trueY ? ( - @@ -166,9 +138,9 @@ export class FlyoutObjectDetection extends React.Component< localization.InterpretVision.Dashboard .titleBarError } - + ) : ( - @@ -176,71 +148,73 @@ export class FlyoutObjectDetection extends React.Component< localization.InterpretVision.Dashboard .titleBarSuccess } - + )} - - - - + + + + {localization.InterpretVision.Dashboard.indexLabel} {item?.index} - - - - + + + + {localization.InterpretVision.Dashboard.predictedY} {predictedY} - - - - + + + + {localization.InterpretVision.Dashboard.trueY} {trueY} - - - - - - - - - - - - - + + + + + + +
+ + + + + + + + + - - + + {localization.InterpretVision.Dashboard.panelInformation} - - - - + + + - - - - - - - - - + + + + + + + + + {localization.InterpretVision.Dashboard.panelExplanation} - - - + + + { - } - + {!this.props.loadingExplanation[item.index][ - +this.state.odSelectedKey.slice(ExcessLabelLen) + +this.state.odSelectedKey.slice( + FlyoutODUtils.ExcessLabelLen + ) ] ? ( - - + - + ) : ( - - + - + )} - - - - - - + + + + + + ); } private callbackWrapper = (): void => { @@ -286,15 +264,29 @@ export class FlyoutObjectDetection extends React.Component< callback(); }; private selectODChoiceFromDropdown = ( - _event: React.FormEvent, - item?: IComboBoxOption + _event: React.FormEvent, + item?: FluentUI.IComboBoxOption ): void => { if (typeof item?.key === "string") { this.setState({ odSelectedKey: item?.key }); if (this.state.item !== undefined) { - // Remove "Object: " from labels. We only want index - this.props.onChange(this.state.item, +item.key.slice(ExcessLabelLen)); + this.props.onChange( + this.state.item, + +item.key.slice(FlyoutODUtils.ExcessLabelLen) + ); } } }; + private readonly callbackRef = (editorCallback: HTMLDivElement): void => { + if (!editorCallback) { + return; + } + const editor = new CanvasTools.Editor(editorCallback); + const loadImage = async (): Promise => { + if (this.state.item) { + await FlyoutODUtils.loadImageFromBase64(this.state.item.image, editor); + } + }; + loadImage(); + }; } diff --git a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx new file mode 100644 index 000000000..d836ad80d --- /dev/null +++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { IComboBoxOption } from "@fluentui/react"; +import { IDataset, IVisionListItem } from "@responsible-ai/core-ui"; +import { localization } from "@responsible-ai/localization"; +import { Editor } from "vott-ct/lib/js/CanvasTools/CanvasTools.Editor"; + +export interface IFlyoutProps { + dataset: IDataset; + explanations: Map>; + isOpen: boolean; + item: IVisionListItem | undefined; + loadingExplanation: boolean[][]; + otherMetadataFieldNames: string[]; + callback: () => void; + onChange: (item: IVisionListItem, index: number) => void; +} + +export interface IFlyoutState { + item: IVisionListItem | undefined; + metadata: Array> | undefined; + selectableObjectIndexes: IComboBoxOption[]; + odSelectedKey: string; + editorCallback?: HTMLDivElement; +} + +export const stackTokens = { + large: { childrenGap: "l2" }, + medium: { childrenGap: "l1" } +}; +export const ExcessLabelLen = + localization.InterpretVision.Dashboard.prefix.length; + +export function loadImageFromBase64( + base64String: string, + editor: Editor +): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + image.addEventListener("load", (e) => { + editor.addContentSource(e.target as HTMLImageElement); + editor.AS.setSelectionMode(2); + resolve(image); + }); + image.addEventListener("error", () => { + reject(new Error("Failed to load image")); + }); + image.src = `data:image/jpg;base64,${base64String}`; + }); +} diff --git a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx index 3b5a3f730..4c15dbc82 100644 --- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx +++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx @@ -93,15 +93,18 @@ export class VisionExplanationDashboard extends React.Component< /> - + {this.state.panelOpen && ( + + )} ) : ( diff --git a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts index 828775d2a..fe64013c1 100644 --- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts +++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts @@ -74,7 +74,7 @@ export function preprocessData( const trueY = mapClassNames(dataSummary.true_y, classNames); const features = dataSummary.features?.map((featuresArr) => { - return featuresArr[0] as number; + return Number((featuresArr[0] as number).toFixed(2)); }); const fieldNames = dataSummary.feature_names; diff --git a/package.json b/package.json index f13cb37d6..3305c3c34 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ }, "dependencies": { "@fluentui/react": "8.58.0", + "canvas": "^2.11.2", "core-js": "^3.6.5", "d3-array": "^2.8.0", "d3-color": "^2.0.0", @@ -70,7 +71,8 @@ "react-router-dom": "^5.0.1", "regenerator-runtime": "0.13.7", "tslib": "^2.5.0", - "uuid": "^8.3.0" + "uuid": "^8.3.0", + "vott-ct": "^2.4.2-rc.0" }, "devDependencies": { "@babel/core": "7.11.1", diff --git a/responsibleai/responsibleai/_interfaces.py b/responsibleai/responsibleai/_interfaces.py index 05eaa9ebc..a7ac878fa 100644 --- a/responsibleai/responsibleai/_interfaces.py +++ b/responsibleai/responsibleai/_interfaces.py @@ -37,6 +37,7 @@ class Dataset: index: Optional[List[str]] object_detection_true_y: Optional[List] object_detection_predicted_y: Optional[List] + imageDimensions: Optional[List[List[int]]] class BoundedCoordinates: