[Image Explorer] CanvasTools Image Loading support for Object Detection (#2097)

* bbox vott template ckpt * canvastools image load ckpt * loading image from scratch ckpt * canvastools image loading support * coordinate fixes * regiondata call + path fix * code cleanup * image loading ckpt * callback image loading support * lint fixes * disabling internal imports for vott * file refactor lint fix * auto lint fixes * lint fixes * image dims arg for frontend bbox * lint fixes * lint fixes * image dimension support * canvas module added * comment fix * async image loading support * lint fixes * lint fixes * max-lines lint fixes * lint fixes
2023-06-13 18:07:13 -04:00 · 2023-06-13 18:07:13 -04:00 · f55e4be135
--- a/.eslintrc.json
+++ b/.eslintrc.json
@ -36,7 +36,8 @@
            ]
          }
        ],
-        "no-alert": ["error"]
+        "no-alert": ["error"],
+        "import/no-internal-modules": "off"
      }
    },
    {
--- a/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts
+++ b/apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts
@ -12,8 +12,8 @@ export const fridgeObjectDetection: IDataset = {
  features: [
    [96.30899737412763],
    [95.32630225415797],
-    [1003762680516188],
-    [92000130390912],
+    [100.3762680516188],
+    [92.000130390912],
    [95.33849179841164]
  ],
  images: fridgeObjectDetectionImages,
@ -40,11 +40,11 @@ export const fridgeObjectDetection: IDataset = {
    ],
    [
      [
-        2, 47.880287170410156, 1465001831054688, 212.69313049316406,
+        2, 47.880287170410156, 146.5001831054688, 212.69313049316406,
        513.9315795898438, 0.9914382696151733
      ],
      [
-        4, 322.61370849609375, 1733768920898438, 455.5107421875,
+        4, 322.61370849609375, 173.3768920898438, 455.5107421875,
        498.9791564941406, 0.9787105917930603
      ]
    ],
@ -54,8 +54,8 @@ export const fridgeObjectDetection: IDataset = {
        412.123779296875, 0.9933412671089172
      ],
      [
-        3, 52.82023239135742, 3065950927734375, 363.34197998046875,
-        420689697265625, 0.979895830154419
+        3, 52.82023239135742, 306.5950927734375, 363.34197998046875,
+        420.689697265625, 0.979895830154419
      ]
    ],
    [
@ -64,7 +64,7 @@ export const fridgeObjectDetection: IDataset = {
        507.566650390625, 0.9822636246681213
      ],
      [
-        4, 98.31806182861328, 172.11666870117188, 2242011108398438,
+        4, 98.31806182861328, 172.11666870117188, 224.2011108398438,
        483.5189208984375, 0.9667745232582092
      ]
    ]
--- a/libs/core-ui/src/lib/Interfaces/IDataset.ts
+++ b/libs/core-ui/src/lib/Interfaces/IDataset.ts
@ -42,6 +42,7 @@ export interface IDataset {
  index?: string[];
  object_detection_true_y?: number[][][];
  object_detection_predicted_y?: number[][][];
+  imageDimensions?: Array<[number, number]>;
 }

 // TODO Remove DatasetSummary when possible
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx
@ -1,31 +1,13 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.

-import {
-  ComboBox,
-  IComboBox,
-  IComboBoxOption,
-  Icon,
-  Image,
-  ImageFit,
-  List,
-  Panel,
-  PanelType,
-  FocusZone,
-  Stack,
-  Text,
-  Spinner,
-  Separator
-} from "@fluentui/react";
-import { FluentUIStyles, IVisionListItem } from "@responsible-ai/core-ui";
+import * as FluentUI from "@fluentui/react";
+import { FluentUIStyles } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
-import React from "react";
+import * as React from "react";
+import { CanvasTools } from "vott-ct";

-import {
-  generateSelectableObjectDetectionIndexes,
-  onRenderCell,
-  updateMetadata
-} from "../utils/FlyoutUtils";
+import * as FlyoutStyles from "../utils/FlyoutUtils";
 import { getJoinedLabelString } from "../utils/labelUtils";

 import {
@ -33,34 +15,13 @@ import {
  explanationImage,
  explanationImageWidth
 } from "./Flyout.styles";
+import * as FlyoutODUtils from "./FlyoutObjectDetectionUtils";

-export interface IFlyoutProps {
-  explanations: Map<number, Map<number, string>>;
-  isOpen: boolean;
-  item: IVisionListItem | undefined;
-  loadingExplanation: boolean[][];
-  otherMetadataFieldNames: string[];
-  callback: () => void;
-  onChange: (item: IVisionListItem, index: number) => void;
-}
-
-export interface IFlyoutState {
-  item: IVisionListItem | undefined;
-  metadata: Array<Array<string | number | boolean>> | undefined;
-  selectableObjectIndexes: IComboBoxOption[];
-  odSelectedKey: string;
-}
-
-const stackTokens = {
-  large: { childrenGap: "l2" },
-  medium: { childrenGap: "l1" }
-};
-const ExcessLabelLen = localization.InterpretVision.Dashboard.prefix.length;
 export class FlyoutObjectDetection extends React.Component<
-  IFlyoutProps,
-  IFlyoutState
+  FlyoutODUtils.IFlyoutProps,
+  FlyoutODUtils.IFlyoutState
 > {
-  public constructor(props: IFlyoutProps) {
+  public constructor(props: FlyoutODUtils.IFlyoutProps) {
    super(props);
    this.state = {
      item: undefined,
@ -69,30 +30,37 @@ export class FlyoutObjectDetection extends React.Component<
      selectableObjectIndexes: []
    };
  }
+
  public componentDidMount(): void {
    const item = this.props.item;
    if (!item) {
      return;
    }
    const fieldNames = this.props.otherMetadataFieldNames;
-    const metadata = updateMetadata(item, fieldNames);
-    const selectableObjectIndexes = generateSelectableObjectDetectionIndexes(
-      localization.InterpretVision.Dashboard.prefix,
-      item
-    );
+    const metadata = FlyoutStyles.updateMetadata(item, fieldNames);
+    const selectableObjectIndexes =
+      FlyoutStyles.generateSelectableObjectDetectionIndexes(
+        localization.InterpretVision.Dashboard.prefix,
+        item
+      );
    this.setState({ item, metadata, selectableObjectIndexes });
  }
-  public componentDidUpdate(prevProps: IFlyoutProps): void {
+
+  public componentDidUpdate(prevProps: FlyoutODUtils.IFlyoutProps): void {
    if (prevProps !== this.props) {
      const item = this.props.item;
      if (!item) {
        return;
      }
-      const metadata = updateMetadata(item, this.props.otherMetadataFieldNames);
-      const selectableObjectIndexes = generateSelectableObjectDetectionIndexes(
-        localization.InterpretVision.Dashboard.prefix,
-        item
+      const metadata = FlyoutStyles.updateMetadata(
+        item,
+        this.props.otherMetadataFieldNames
      );
+      const selectableObjectIndexes =
+        FlyoutStyles.generateSelectableObjectDetectionIndexes(
+          localization.InterpretVision.Dashboard.prefix,
+          item
+        );
      this.setState({
        item: this.props.item,
        metadata,
@ -100,6 +68,7 @@ export class FlyoutObjectDetection extends React.Component<
      });
    }
  }
+
  public render(): React.ReactNode {
    const { isOpen } = this.props;
    const item = this.state.item;
@ -109,43 +78,46 @@ export class FlyoutObjectDetection extends React.Component<
    const classNames = flyoutStyles();
    const predictedY = getJoinedLabelString(item?.predictedY);
    const trueY = getJoinedLabelString(item?.trueY);
+
    return (
-      <FocusZone>
-        <Panel
+      <FluentUI.FocusZone>
+        <FluentUI.Panel
          headerText={localization.InterpretVision.Dashboard.panelTitle}
          isOpen={isOpen}
          closeButtonAriaLabel="Close"
          onDismiss={this.callbackWrapper}
          isLightDismiss
-          type={PanelType.large}
+          type={FluentUI.PanelType.large}
          className={classNames.mainContainer}
        >
-          <Stack tokens={stackTokens.medium} horizontal>
-            <Stack>
-              <Stack.Item>
-                <Separator className={classNames.separator} />
-              </Stack.Item>
-              <Stack.Item>
-                <Stack
+          <FluentUI.Stack tokens={FlyoutODUtils.stackTokens.medium} horizontal>
+            <FluentUI.Stack>
+              <FluentUI.Stack.Item>
+                <FluentUI.Separator className={classNames.separator} />
+              </FluentUI.Stack.Item>
+              <FluentUI.Stack.Item>
+                <FluentUI.Stack
                  horizontal
-                  tokens={stackTokens.medium}
+                  tokens={FlyoutODUtils.stackTokens.medium}
                  horizontalAlign="space-around"
                  verticalAlign="center"
                >
-                  <Stack.Item>
-                    <Stack
-                      tokens={stackTokens.large}
+                  <FluentUI.Stack.Item>
+                    <FluentUI.Stack
+                      tokens={FlyoutODUtils.stackTokens.large}
                      horizontalAlign="start"
                      verticalAlign="start"
                    >
-                      <Stack
+                      <FluentUI.Stack
                        horizontal
                        tokens={{ childrenGap: "s1" }}
                        horizontalAlign="center"
                        verticalAlign="center"
                      >
-                        <Stack.Item className={classNames.iconContainer}>
-                          <Icon
+                        <FluentUI.Stack.Item
+                          className={classNames.iconContainer}
+                        >
+                          <FluentUI.Icon
                            iconName={
                              predictedY !== trueY ? "Cancel" : "Checkmark"
                            }
@ -155,10 +127,10 @@ export class FlyoutObjectDetection extends React.Component<
                                : classNames.successIcon
                            }
                          />
-                        </Stack.Item>
-                        <Stack.Item>
+                        </FluentUI.Stack.Item>
+                        <FluentUI.Stack.Item>
                          {predictedY !== trueY ? (
-                            <Text
+                            <FluentUI.Text
                              variant="large"
                              className={classNames.errorTitle}
                            >
@ -166,9 +138,9 @@ export class FlyoutObjectDetection extends React.Component<
                                localization.InterpretVision.Dashboard
                                  .titleBarError
                              }
-                            </Text>
+                            </FluentUI.Text>
                          ) : (
-                            <Text
+                            <FluentUI.Text
                              variant="large"
                              className={classNames.successTitle}
                            >
@ -176,71 +148,73 @@ export class FlyoutObjectDetection extends React.Component<
                                localization.InterpretVision.Dashboard
                                  .titleBarSuccess
                              }
-                            </Text>
+                            </FluentUI.Text>
                          )}
-                        </Stack.Item>
-                      </Stack>
-                      <Stack.Item>
-                        <Text variant="large">
+                        </FluentUI.Stack.Item>
+                      </FluentUI.Stack>
+                      <FluentUI.Stack.Item>
+                        <FluentUI.Text variant="large">
                          {localization.InterpretVision.Dashboard.indexLabel}
                          {item?.index}
-                        </Text>
-                      </Stack.Item>
-                      <Stack.Item>
-                        <Text variant="large">
+                        </FluentUI.Text>
+                      </FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item>
+                        <FluentUI.Text variant="large">
                          {localization.InterpretVision.Dashboard.predictedY}
                          {predictedY}
-                        </Text>
-                      </Stack.Item>
-                      <Stack.Item>
-                        <Text variant="large">
+                        </FluentUI.Text>
+                      </FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item>
+                        <FluentUI.Text variant="large">
                          {localization.InterpretVision.Dashboard.trueY}
                          {trueY}
-                        </Text>
-                      </Stack.Item>
-                    </Stack>
-                  </Stack.Item>
-                  <Stack.Item className={classNames.imageContainer}>
-                    <Image
-                      src={`data:image/jpg;base64,${item?.image}`}
-                      className={classNames.image}
-                      imageFit={ImageFit.contain}
-                    />
-                  </Stack.Item>
-                </Stack>
-              </Stack.Item>
-              <Stack.Item>
-                <Separator className={classNames.separator} />
-              </Stack.Item>
-              <Stack
+                        </FluentUI.Text>
+                      </FluentUI.Stack.Item>
+                    </FluentUI.Stack>
+                  </FluentUI.Stack.Item>
+                  <FluentUI.Stack.Item className={classNames.imageContainer}>
+                    <FluentUI.Stack.Item id="canvasToolsDiv">
+                      <FluentUI.Stack.Item id="selectionDiv">
+                        <div ref={this.callbackRef} id="editorDiv" />
+                      </FluentUI.Stack.Item>
+                    </FluentUI.Stack.Item>
+                  </FluentUI.Stack.Item>
+                </FluentUI.Stack>
+              </FluentUI.Stack.Item>
+              <FluentUI.Stack.Item>
+                <FluentUI.Separator className={classNames.separator} />
+              </FluentUI.Stack.Item>
+              <FluentUI.Stack
                tokens={{ childrenGap: "l2" }}
                className={classNames.sectionIndent}
              >
-                <Stack.Item>
-                  <Text variant="large" className={classNames.title}>
+                <FluentUI.Stack.Item>
+                  <FluentUI.Text variant="large" className={classNames.title}>
                    {localization.InterpretVision.Dashboard.panelInformation}
-                  </Text>
-                </Stack.Item>
-                <Stack.Item className={classNames.featureListContainer}>
-                  <List
+                  </FluentUI.Text>
+                </FluentUI.Stack.Item>
+                <FluentUI.Stack.Item
+                  className={classNames.featureListContainer}
+                >
+                  <FluentUI.List
                    items={this.state.metadata}
-                    onRenderCell={onRenderCell}
+                    onRenderCell={FlyoutStyles.onRenderCell}
                  />
-                </Stack.Item>
-              </Stack>
-            </Stack>
-            <Stack>
-              <Stack.Item>
-                <Separator className={classNames.separator} />
-              </Stack.Item>
-              <Stack.Item>
-                <Text variant="large" className={classNames.title}>
+                </FluentUI.Stack.Item>
+              </FluentUI.Stack>
+            </FluentUI.Stack>
+            <FluentUI.Stack>
+              <FluentUI.Stack.Item>
+                <FluentUI.Separator className={classNames.separator} />
+              </FluentUI.Stack.Item>
+              <FluentUI.Stack.Item>
+                <FluentUI.Text variant="large" className={classNames.title}>
                  {localization.InterpretVision.Dashboard.panelExplanation}
-                </Text>
-              </Stack.Item>
-              <Stack>
+                </FluentUI.Text>
+              </FluentUI.Stack.Item>
+              <FluentUI.Stack>
                {
-                  <ComboBox
+                  <FluentUI.ComboBox
                    id={localization.InterpretVision.Dashboard.objectSelect}
                    label={localization.InterpretVision.Dashboard.chooseObject}
                    onChange={this.selectODChoiceFromDropdown}
@ -250,34 +224,38 @@ export class FlyoutObjectDetection extends React.Component<
                    styles={FluentUIStyles.smallDropdownStyle}
                  />
                }
-                <Stack>
+                <FluentUI.Stack>
                  {!this.props.loadingExplanation[item.index][
-                    +this.state.odSelectedKey.slice(ExcessLabelLen)
+                    +this.state.odSelectedKey.slice(
+                      FlyoutODUtils.ExcessLabelLen
+                    )
                  ] ? (
-                    <Stack.Item>
-                      <Image
+                    <FluentUI.Stack.Item>
+                      <FluentUI.Image
                        src={`data:image/jpg;base64,${this.props.explanations
                          .get(item.index)
                          ?.get(
-                            +this.state.odSelectedKey.slice(ExcessLabelLen)
+                            +this.state.odSelectedKey.slice(
+                              FlyoutODUtils.ExcessLabelLen
+                            )
                          )}`}
                        width={explanationImageWidth}
                        style={explanationImage}
                      />
-                    </Stack.Item>
+                    </FluentUI.Stack.Item>
                  ) : (
-                    <Stack.Item>
-                      <Spinner
+                    <FluentUI.Stack.Item>
+                      <FluentUI.Spinner
                        label={`${localization.InterpretVision.Dashboard.loading} ${item?.index}`}
                      />
-                    </Stack.Item>
+                    </FluentUI.Stack.Item>
                  )}
-                </Stack>
-              </Stack>
-            </Stack>
-          </Stack>
-        </Panel>
-      </FocusZone>
+                </FluentUI.Stack>
+              </FluentUI.Stack>
+            </FluentUI.Stack>
+          </FluentUI.Stack>
+        </FluentUI.Panel>
+      </FluentUI.FocusZone>
    );
  }
  private callbackWrapper = (): void => {
@ -286,15 +264,29 @@ export class FlyoutObjectDetection extends React.Component<
    callback();
  };
  private selectODChoiceFromDropdown = (
-    _event: React.FormEvent<IComboBox>,
-    item?: IComboBoxOption
+    _event: React.FormEvent<FluentUI.IComboBox>,
+    item?: FluentUI.IComboBoxOption
  ): void => {
    if (typeof item?.key === "string") {
      this.setState({ odSelectedKey: item?.key });
      if (this.state.item !== undefined) {
-        // Remove "Object: " from labels. We only want index
-        this.props.onChange(this.state.item, +item.key.slice(ExcessLabelLen));
+        this.props.onChange(
+          this.state.item,
+          +item.key.slice(FlyoutODUtils.ExcessLabelLen)
+        );
      }
    }
  };
+  private readonly callbackRef = (editorCallback: HTMLDivElement): void => {
+    if (!editorCallback) {
+      return;
+    }
+    const editor = new CanvasTools.Editor(editorCallback);
+    const loadImage = async (): Promise<void> => {
+      if (this.state.item) {
+        await FlyoutODUtils.loadImageFromBase64(this.state.item.image, editor);
+      }
+    };
+    loadImage();
+  };
 }
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx
@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { IComboBoxOption } from "@fluentui/react";
+import { IDataset, IVisionListItem } from "@responsible-ai/core-ui";
+import { localization } from "@responsible-ai/localization";
+import { Editor } from "vott-ct/lib/js/CanvasTools/CanvasTools.Editor";
+
+export interface IFlyoutProps {
+  dataset: IDataset;
+  explanations: Map<number, Map<number, string>>;
+  isOpen: boolean;
+  item: IVisionListItem | undefined;
+  loadingExplanation: boolean[][];
+  otherMetadataFieldNames: string[];
+  callback: () => void;
+  onChange: (item: IVisionListItem, index: number) => void;
+}
+
+export interface IFlyoutState {
+  item: IVisionListItem | undefined;
+  metadata: Array<Array<string | number | boolean>> | undefined;
+  selectableObjectIndexes: IComboBoxOption[];
+  odSelectedKey: string;
+  editorCallback?: HTMLDivElement;
+}
+
+export const stackTokens = {
+  large: { childrenGap: "l2" },
+  medium: { childrenGap: "l1" }
+};
+export const ExcessLabelLen =
+  localization.InterpretVision.Dashboard.prefix.length;
+
+export function loadImageFromBase64(
+  base64String: string,
+  editor: Editor
+): Promise<HTMLImageElement> {
+  return new Promise((resolve, reject) => {
+    const image = new Image();
+    image.addEventListener("load", (e) => {
+      editor.addContentSource(e.target as HTMLImageElement);
+      editor.AS.setSelectionMode(2);
+      resolve(image);
+    });
+    image.addEventListener("error", () => {
+      reject(new Error("Failed to load image"));
+    });
+    image.src = `data:image/jpg;base64,${base64String}`;
+  });
+}
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboard.tsx
@ -93,15 +93,18 @@ export class VisionExplanationDashboard extends React.Component<
          />
        </Stack.Item>
        <Stack.Item>
-          <FlyoutObjectDetection
-            explanations={this.state.computedExplanations}
-            isOpen={this.state.panelOpen}
-            item={this.state.selectedItem}
-            loadingExplanation={this.state.loadingExplanation}
-            otherMetadataFieldNames={this.state.otherMetadataFieldNames}
-            callback={this.onPanelClose}
-            onChange={this.onItemSelectObjectDetection}
-          />
+          {this.state.panelOpen && (
+            <FlyoutObjectDetection
+              dataset={this.context.dataset}
+              explanations={this.state.computedExplanations}
+              isOpen={this.state.panelOpen}
+              item={this.state.selectedItem}
+              loadingExplanation={this.state.loadingExplanation}
+              otherMetadataFieldNames={this.state.otherMetadataFieldNames}
+              callback={this.onPanelClose}
+              onChange={this.onItemSelectObjectDetection}
+            />
+          )}
        </Stack.Item>
      </Stack>
    ) : (
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/VisionExplanationDashboardHelper.ts
@ -74,7 +74,7 @@ export function preprocessData(
  const trueY = mapClassNames(dataSummary.true_y, classNames);

  const features = dataSummary.features?.map((featuresArr) => {
-    return featuresArr[0] as number;
+    return Number((featuresArr[0] as number).toFixed(2));
  });

  const fieldNames = dataSummary.feature_names;
--- a/package.json
+++ b/package.json
@ -43,6 +43,7 @@
  },
  "dependencies": {
    "@fluentui/react": "8.58.0",
+    "canvas": "^2.11.2",
    "core-js": "^3.6.5",
    "d3-array": "^2.8.0",
    "d3-color": "^2.0.0",
@ -70,7 +71,8 @@
    "react-router-dom": "^5.0.1",
    "regenerator-runtime": "0.13.7",
    "tslib": "^2.5.0",
-    "uuid": "^8.3.0"
+    "uuid": "^8.3.0",
+    "vott-ct": "^2.4.2-rc.0"
  },
  "devDependencies": {
    "@babel/core": "7.11.1",
--- a/responsibleai/responsibleai/_interfaces.py
+++ b/responsibleai/responsibleai/_interfaces.py
@ -37,6 +37,7 @@ class Dataset:
    index: Optional[List[str]]
    object_detection_true_y: Optional[List]
    object_detection_predicted_y: Optional[List]
+    imageDimensions: Optional[List[List[int]]]


 class BoundedCoordinates: