fix images missing alternate text in RAI Vision dashboard for accessibility (#2463)

2023-12-18 17:31:51 -05:00 · 2023-12-18 17:31:51 -05:00 · 4f7bda6549
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/DataCharacteristics.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/DataCharacteristics.tsx
@ -15,6 +15,7 @@ import { IVisionListItem } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
 import React from "react";

+import { getAltTextForItem } from "../utils/getAltTextUtils";
 import { getFilteredDataFromSearch } from "../utils/getFilteredData";
 import { getJoinedLabelString } from "../utils/labelUtils";

@ -47,16 +48,14 @@ export class DataCharacteristics extends React.Component<
  private classNames = dataCharacteristicsStyles();
  public constructor(props: IDataCharacteristicsProps) {
    super(props);
+    const labelType = this.predOrIncorrectLabelType;
    const labelTypeDropdownOptions: IDropdownOption[] = [
-      {
-        key: this.predOrIncorrectLabelType,
-        text: this.predOrIncorrectLabelType
-      },
+      { key: labelType, text: labelType },
      { key: this.trueOrCorrectLabelType, text: this.trueOrCorrectLabelType }
    ];
    this.state = {
      ...defaultState,
-      labelType: this.predOrIncorrectLabelType,
+      labelType,
      labelTypeDropdownOptions
    };
  }
@ -75,6 +74,12 @@ export class DataCharacteristics extends React.Component<
    const predicted = this.state.labelType === this.predOrIncorrectLabelType;
    const items = predicted ? this.state.itemsPredicted : this.state.itemsTrue;
    const keys = sortKeys([...items.keys()], items, this.props.taskType);
+    const dropdownOptions = predicted
+      ? this.state.dropdownOptionsPredicted
+      : this.state.dropdownOptionsTrue;
+    const dropdownKeys = predicted
+      ? this.state.selectedKeysPredicted
+      : this.state.selectedKeysTrue;
    return (
      <FocusZone>
        <Stack tokens={stackTokens}>
@ -110,16 +115,8 @@ export class DataCharacteristics extends React.Component<
                        localization.InterpretVision.Dashboard
                          .labelVisibilityDropdown
                      }
-                      options={
-                        this.state.labelType === this.predOrIncorrectLabelType
-                          ? this.state.dropdownOptionsPredicted
-                          : this.state.dropdownOptionsTrue
-                      }
-                      selectedKeys={
-                        this.state.labelType === this.predOrIncorrectLabelType
-                          ? this.state.selectedKeysPredicted
-                          : this.state.selectedKeysTrue
-                      }
+                      options={dropdownOptions}
+                      selectedKeys={dropdownKeys}
                      onChange={this.onLabelVisibilitySelect}
                      multiSelect
                      ariaLabel={
@ -205,24 +202,27 @@ export class DataCharacteristics extends React.Component<
  private onRenderCell = (
    item?: IVisionListItem | undefined
  ): React.ReactElement => {
+    if (!item) {
+      return <div />;
+    }
    const imageDim = this.props.imageDim;
-    const predictedY = getJoinedLabelString(item?.predictedY);
-    const trueY = getJoinedLabelString(item?.trueY);
+    const predictedY = getJoinedLabelString(item.predictedY);
+    const trueY = getJoinedLabelString(item.trueY);
+    const indicator =
+      predictedY === trueY
+        ? this.classNames.successIndicator
+        : this.classNames.errorIndicator;
    const indicatorStyle = mergeStyles(
      this.classNames.indicator,
      { width: imageDim },
-      predictedY === trueY
-        ? this.classNames.successIndicator
-        : this.classNames.errorIndicator
+      indicator
    );
-    return !item ? (
-      <div />
-    ) : (
+    return (
      <Stack className={this.classNames.tile}>
        <Stack.Item style={{ height: imageDim, width: imageDim }}>
          <Image
-            alt={predictedY}
-            src={`data:image/jpg;base64,${item?.image}`}
+            alt={getAltTextForItem(item, this.props.taskType)}
+            src={`data:image/jpg;base64,${item.image}`}
            onClick={this.callbackWrapper(item)}
            className={this.classNames.image}
            style={{ display: "inline", height: imageDim }}
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/Flyout.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/Flyout.tsx
@ -18,6 +18,7 @@ import { IVisionListItem } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
 import React from "react";

+import { getImageAltText } from "../utils/getAltTextUtils";
 import { getJoinedLabelString } from "../utils/labelUtils";

 import { flyoutStyles } from "./Flyout.styles";
@ -196,6 +197,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                </Stack.Item>
                <Stack.Item className={classNames.imageContainer}>
                  <Image
+                    alt={getImageAltText(predictedY, trueY, item?.index)}
                    id={`flyoutImage_${item?.index}`}
                    src={`data:image/jpg;base64,${item?.image}`}
                    className={classNames.image}
@ -219,6 +221,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
              {!this.props.loadingExplanation[0][index] ? (
                <Stack.Item>
                  <Image
+                    alt={getImageAltText(predictedY, trueY, item?.index, true)}
                    src={`data:image/jpg;base64,${this.props.explanations
                      .get(0)
                      ?.get(index)}`}
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx
@ -1,13 +1,27 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.

-import * as FluentUI from "@fluentui/react";
+import {
+  ComboBox,
+  FocusZone,
+  IComboBox,
+  IComboBoxOption,
+  Image,
+  List,
+  Panel,
+  PanelType,
+  Separator,
+  Spinner,
+  Stack,
+  Text
+} from "@fluentui/react";
 import { FluentUIStyles } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
 import * as React from "react";
 import { CanvasTools } from "vott-ct";

 import * as FlyoutStyles from "../utils/FlyoutUtils";
+import { getObjectDetectionImageAltText } from "../utils/getAltTextUtils";
 import { getJoinedLabelString } from "../utils/labelUtils";

 import {
@ -76,114 +90,118 @@ export class FlyoutObjectDetection extends React.Component<
      return <div />;
    }
    const classNames = flyoutStyles();
-    const correctDetections = getJoinedLabelString(item?.odCorrect);
-    const incorrectDetections = getJoinedLabelString(item?.odIncorrect);
-
+    const odCorrect = item.odCorrect;
+    const odIncorrect = item.odIncorrect;
+    const correctDetections = getJoinedLabelString(odCorrect);
+    const incorrectDetections = getJoinedLabelString(odIncorrect);
+    const alt = getObjectDetectionImageAltText(
+      odCorrect,
+      odIncorrect,
+      item.index
+    );
    return (
-      <FluentUI.FocusZone>
-        <FluentUI.Panel
+      <FocusZone>
+        <Panel
          headerText={localization.InterpretVision.Dashboard.panelTitle}
          isOpen={isOpen}
          closeButtonAriaLabel="Close"
          onDismiss={this.callbackWrapper}
          isLightDismiss
-          type={FluentUI.PanelType.large}
+          type={PanelType.large}
          className={classNames.odFlyoutContainer}
        >
-          <FluentUI.Stack tokens={FlyoutODUtils.stackTokens.medium}>
-            <FluentUI.Stack tokens={FlyoutODUtils.stackTokens.medium}>
-              <FluentUI.Stack.Item>
-                <FluentUI.Separator className={classNames.separator} />
-              </FluentUI.Stack.Item>
-              <FluentUI.Stack.Item>
-                <FluentUI.Stack
+          <Stack tokens={FlyoutODUtils.stackTokens.medium}>
+            <Stack tokens={FlyoutODUtils.stackTokens.medium}>
+              <Stack.Item>
+                <Separator className={classNames.separator} />
+              </Stack.Item>
+              <Stack.Item>
+                <Stack
                  tokens={FlyoutODUtils.stackTokens.medium}
                  horizontalAlign="space-around"
                  verticalAlign="center"
                >
-                  <FluentUI.Stack.Item>
-                    <FluentUI.Stack
+                  <Stack.Item>
+                    <Stack
                      tokens={FlyoutODUtils.stackTokens.large}
                      horizontalAlign="start"
                      verticalAlign="start"
                    >
-                      <FluentUI.Stack
+                      <Stack
                        horizontal
                        tokens={FlyoutODUtils.stackTokens.medium}
                        horizontalAlign="center"
                        verticalAlign="center"
                      />
-                      <FluentUI.Stack.Item>
-                        <FluentUI.Text variant="large">
+                      <Stack.Item>
+                        <Text variant="large">
                          {localization.InterpretVision.Dashboard.indexLabel}
                          {item?.index}
-                        </FluentUI.Text>
-                      </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
-                        <FluentUI.Text variant="large">
+                        </Text>
+                      </Stack.Item>
+                      <Stack.Item>
+                        <Text variant="large">
                          {
                            localization.InterpretVision.Dashboard
                              .correctDetections
                          }
                          {correctDetections}
-                        </FluentUI.Text>
-                      </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
-                        <FluentUI.Text variant="large">
+                        </Text>
+                      </Stack.Item>
+                      <Stack.Item>
+                        <Text variant="large">
                          {
                            localization.InterpretVision.Dashboard
                              .incorrectDetections
                          }
                          {incorrectDetections}
-                        </FluentUI.Text>
-                      </FluentUI.Stack.Item>
-                    </FluentUI.Stack>
-                  </FluentUI.Stack.Item>
-                </FluentUI.Stack>
-              </FluentUI.Stack.Item>
-              <FluentUI.Stack.Item>
-                <FluentUI.Separator className={classNames.separator} />
-              </FluentUI.Stack.Item>
-              <FluentUI.Stack
+                        </Text>
+                      </Stack.Item>
+                    </Stack>
+                  </Stack.Item>
+                </Stack>
+              </Stack.Item>
+              <Stack.Item>
+                <Separator className={classNames.separator} />
+              </Stack.Item>
+              <Stack
                tokens={FlyoutODUtils.stackTokens.large}
                className={classNames.sectionIndent}
              >
-                <FluentUI.Stack.Item>
-                  <FluentUI.Text variant="large" className={classNames.title}>
+                <Stack.Item>
+                  <Text variant="large" className={classNames.title}>
                    {localization.InterpretVision.Dashboard.panelInformation}
-                  </FluentUI.Text>
-                </FluentUI.Stack.Item>
-                <FluentUI.Stack.Item
-                  className={classNames.featureListContainer}
-                >
-                  <FluentUI.List
+                  </Text>
+                </Stack.Item>
+                <Stack.Item className={classNames.featureListContainer}>
+                  <List
                    items={this.state.metadata}
                    onRenderCell={FlyoutStyles.onRenderCell}
                  />
-                </FluentUI.Stack.Item>
-              </FluentUI.Stack>
-              <FluentUI.Stack>
-                <FluentUI.Stack.Item className={classNames.imageContainer}>
-                  <FluentUI.Stack.Item id="canvasToolsDiv">
-                    <FluentUI.Stack.Item id="selectionDiv">
+                </Stack.Item>
+              </Stack>
+              <Stack>
+                <Stack.Item className={classNames.imageContainer}>
+                  <Stack.Item id="canvasToolsDiv">
+                    <Stack.Item id="selectionDiv">
                      <div ref={this.callbackRef} id="editorDiv" />
-                    </FluentUI.Stack.Item>
-                  </FluentUI.Stack.Item>
-                </FluentUI.Stack.Item>
-              </FluentUI.Stack>
-            </FluentUI.Stack>
-            <FluentUI.Stack>
-              <FluentUI.Stack.Item>
-                <FluentUI.Separator className={classNames.separator} />
-              </FluentUI.Stack.Item>
-              <FluentUI.Stack.Item>
-                <FluentUI.Text variant="large" className={classNames.title}>
+                    </Stack.Item>
+                  </Stack.Item>
+                </Stack.Item>
+              </Stack>
+            </Stack>
+            <Stack>
+              <Stack.Item>
+                <Separator className={classNames.separator} />
+              </Stack.Item>
+              <Stack.Item>
+                <Text variant="large" className={classNames.title}>
                  {localization.InterpretVision.Dashboard.panelExplanation}
-                </FluentUI.Text>
-              </FluentUI.Stack.Item>
-              <FluentUI.Stack>
+                </Text>
+              </Stack.Item>
+              <Stack>
                {
-                  <FluentUI.ComboBox
+                  <ComboBox
                    id={localization.InterpretVision.Dashboard.objectSelect}
                    label={localization.InterpretVision.Dashboard.chooseObject}
                    onChange={this.selectODChoiceFromDropdown}
@ -193,14 +211,15 @@ export class FlyoutObjectDetection extends React.Component<
                    styles={FluentUIStyles.smallDropdownStyle}
                  />
                }
-                <FluentUI.Stack>
+                <Stack>
                  {!this.props.loadingExplanation[item.index][
                    +this.state.odSelectedKey.slice(
                      FlyoutODUtils.ExcessLabelLen
                    )
                  ] && (
-                    <FluentUI.Stack.Item className={classNames.imageContainer}>
-                      <FluentUI.Image
+                    <Stack.Item className={classNames.imageContainer}>
+                      <Image
+                        alt={alt}
                        src={`data:image/jpg;base64,${this.props.explanations
                          .get(item.index)
                          ?.get(
@ -211,7 +230,7 @@ export class FlyoutObjectDetection extends React.Component<
                        width={explanationImageWidth}
                        style={explanationImage}
                      />
-                    </FluentUI.Stack.Item>
+                    </Stack.Item>
                  )}
                  {this.state.odSelectedKey !== "" &&
                    this.props.loadingExplanation[item.index][
@ -219,18 +238,18 @@ export class FlyoutObjectDetection extends React.Component<
                        FlyoutODUtils.ExcessLabelLen
                      )
                    ] && (
-                      <FluentUI.Stack.Item>
-                        <FluentUI.Spinner
+                      <Stack.Item>
+                        <Spinner
                          label={`${localization.InterpretVision.Dashboard.loading} ${item?.index}`}
                        />
-                      </FluentUI.Stack.Item>
+                      </Stack.Item>
                    )}
-                </FluentUI.Stack>
-              </FluentUI.Stack>
-            </FluentUI.Stack>
-          </FluentUI.Stack>
-        </FluentUI.Panel>
-      </FluentUI.FocusZone>
+                </Stack>
+              </Stack>
+            </Stack>
+          </Stack>
+        </Panel>
+      </FocusZone>
    );
  }
  private callbackWrapper = (): void => {
@ -239,8 +258,8 @@ export class FlyoutObjectDetection extends React.Component<
    callback();
  };
  private selectODChoiceFromDropdown = (
-    _event: React.FormEvent<FluentUI.IComboBox>,
-    item?: FluentUI.IComboBoxOption
+    _event: React.FormEvent<IComboBox>,
+    item?: IComboBoxOption
  ): void => {
    if (typeof item?.key === "string") {
      this.setState({ odSelectedKey: item?.key });
@ -259,10 +278,16 @@ export class FlyoutObjectDetection extends React.Component<
    const editor = new CanvasTools.Editor(editorCallback);
    const loadImage = async (): Promise<void> => {
      if (this.state.item) {
+        const item = this.state.item;
+        const altText = getObjectDetectionImageAltText(
+          item.odCorrect,
+          item.odIncorrect,
+          item.index
+        );
        // this.state.item.image is base64 encoded string
-        await FlyoutODUtils.loadImageFromBase64(this.state.item.image, editor);
+        await FlyoutODUtils.loadImageFromBase64(item.image, editor, altText);
        FlyoutODUtils.drawBoundingBoxes(
-          this.state.item,
+          item,
          editorCallback,
          editor,
          this.props.dataset
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetectionUtils.tsx
@ -36,7 +36,8 @@ export const ExcessLabelLen =

 export function loadImageFromBase64(
  base64String: string,
-  editor: Editor
+  editor: Editor,
+  altText: string
 ): Promise<HTMLImageElement> {
  return new Promise((resolve, reject) => {
    const image = new Image();
@ -49,6 +50,7 @@ export function loadImageFromBase64(
      reject(new Error("Failed to load image"));
    });
    image.src = `data:image/jpg;base64,${base64String}`;
+    image.alt = altText;
  });
 }

--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/ImageList.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/ImageList.tsx
@ -15,6 +15,7 @@ import { DatasetTaskType, IVisionListItem } from "@responsible-ai/core-ui";
 import React from "react";

 import { ISearchable } from "../Interfaces/ISearchable";
+import { getAltTextForItem } from "../utils/getAltTextUtils";
 import { getFilteredDataFromSearch } from "../utils/getFilteredData";
 import { getJoinedLabelString } from "../utils/labelUtils";

@ -105,11 +106,11 @@ export class ImageList extends React.Component<
    if (!item) {
      return;
    }
-    const itemPredY = item?.predictedY;
+    const itemPredY = item.predictedY;
    const predictedY = getJoinedLabelString(itemPredY);
-    const itemTrueY = item?.trueY;
+    const itemTrueY = item.trueY;
    const trueY = getJoinedLabelString(itemTrueY);
-    const alt = predictedY;
+    const alt = getAltTextForItem(item, this.props.taskType);
    const odAggregate = getJoinedLabelString(item?.odAggregate)?.split(", ");

    return (
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/TableList.tsx
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/TableList.tsx
@ -18,6 +18,7 @@ import { DatasetTaskType, IVisionListItem } from "@responsible-ai/core-ui";
 import { localization } from "@responsible-ai/localization";
 import React from "react";

+import { getAltTextForItem } from "../utils/getAltTextUtils";
 import { getFilteredDataFromSearch } from "../utils/getFilteredData";
 import { isItemPredTrueEqual } from "../utils/labelUtils";
 import { visionExplanationDashboardStyles } from "../VisionExplanationDashboard.styles";
@ -249,7 +250,6 @@ export class TableList extends React.Component<
    column?: IColumn | undefined
  ): React.ReactNode => {
    const classNames = visionExplanationDashboardStyles();
-
    let value =
      item && column && column.fieldName
        ? item[column.fieldName as keyof IVisionListItem]
@ -270,9 +270,10 @@ export class TableList extends React.Component<
        : "";
    return (
      <Stack horizontal tokens={{ childrenGap: "s1" }}>
-        {image ? (
+        {image && item ? (
          <Stack.Item>
            <Image
+              alt={getAltTextForItem(item, this.props.taskType)}
              className={classNames.tableListImage}
              src={`data:image/jpg;base64,${image}`}
              style={{ width: this.props.imageDim }}
--- a/libs/interpret-vision/src/lib/VisionExplanationDashboard/utils/getAltTextUtils.ts
+++ b/libs/interpret-vision/src/lib/VisionExplanationDashboard/utils/getAltTextUtils.ts
@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { IVisionListItem, DatasetTaskType } from "@responsible-ai/core-ui";
+
+import { getJoinedLabelString } from "./labelUtils";
+
+export function getAltTextForItem(
+  item: IVisionListItem,
+  taskType: string,
+  isExplanation?: boolean
+): string {
+  if (taskType === DatasetTaskType.ObjectDetection) {
+    return getObjectDetectionImageAltText(
+      item.odCorrect,
+      item.odIncorrect,
+      item.index,
+      isExplanation
+    );
+  }
+  return getImageAltText(
+    item.predictedY,
+    item.trueY,
+    item.index,
+    isExplanation
+  );
+}
+
+export function getImageAltText(
+  predictedY: string | string[],
+  trueY: string | string[],
+  index?: number,
+  isExplanation?: boolean
+): string {
+  let predictedYString = "predicted label";
+  if (Array.isArray(predictedY)) {
+    predictedYString += `s ${getJoinedLabelString(predictedY)}`;
+  } else {
+    predictedYString += ` ${predictedY}`;
+  }
+  let trueYString = "ground truth label";
+  if (Array.isArray(trueY)) {
+    trueYString += `s ${getJoinedLabelString(trueY)}`;
+  } else {
+    trueYString += ` ${trueY}`;
+  }
+  let headerString = isExplanation ? "Explanation of image" : "Image";
+  if (index !== undefined) {
+    headerString += ` with index ${index}`;
+  }
+  return `${headerString} and ${predictedYString} and ${trueYString}.`;
+}
+
+export function getObjectDetectionImageAltText(
+  odCorrect: string | string[],
+  odIncorrect: string | string[],
+  index?: number,
+  isExplanation?: boolean
+): string {
+  let headerString = isExplanation ? "Explanation of image" : "Image";
+  if (index !== undefined) {
+    headerString += ` with index ${index}`;
+  }
+  let odCorrectString = "correct object detection label";
+  let odIncorrectString = "incorrect object detection label";
+  if (Array.isArray(odCorrect)) {
+    odCorrectString += `s ${getJoinedLabelString(odCorrect)}`;
+  } else {
+    odCorrectString += ` ${odCorrect}`;
+  }
+  if (Array.isArray(odIncorrect)) {
+    odIncorrectString += `s ${getJoinedLabelString(odIncorrect)}`;
+  } else {
+    odIncorrectString += ` ${odIncorrect}`;
+  }
+
+  return `${headerString} and ${odCorrectString} and ${odIncorrectString}.`;
+}