move js outside from python (#118)

* add app

Signed-off-by: Ke Xu <xuke@microsoft.com>

* rename

Signed-off-by: Ke Xu <xuke@microsoft.com>

* add fetch

Signed-off-by: Ke Xu <xuke@microsoft.com>

* remove unused

Signed-off-by: Ke Xu <xuke@microsoft.com>

* fix build

Signed-off-by: Ke Xu <xuke@microsoft.com>

* remove old js

Signed-off-by: Ke Xu <xuke@microsoft.com>

* fix build

Signed-off-by: Ke Xu <xuke@microsoft.com>

* merge difference

Signed-off-by: Ke Xu <xuke@microsoft.com>

* fix

Signed-off-by: Ke Xu <xuke@microsoft.com>

* fix lint

Signed-off-by: Ke Xu <xuke@microsoft.com>

* remove comments

Signed-off-by: Ke Xu <xuke@microsoft.com>

* fix lint

Signed-off-by: Ke Xu <xuke@microsoft.com>

* remove supported parity keys as input parameter since they're predefined anyway

Signed-off-by: Roman Lutz <rolutz@microsoft.com>

* fix build

Signed-off-by: Ke Xu <xuke@microsoft.com>

* lintfix

Signed-off-by: Roman Lutz <rolutz@microsoft.com>

* fix flake

Signed-off-by: Ke Xu <xuke@microsoft.com>

Co-authored-by: Roman Lutz <rolutz@microsoft.com>

.eslintrc

@@ -148,10 +148,8 @@
           "demographic_parity_ratio",
           "equalized_odds_difference",
           "equalized_odds_ratio",
-          "error_rate_difference_binary_classification",
-          "error_rate_difference_regression",
-          "error_rate_ratio_binary_classification",
-          "error_rate_ratio_regression",
+          "error_rate_difference",
+          "error_rate_ratio",
           "error_y",
           "f1_score",
           "f1_score_min",

.gitignore

@@ -4,8 +4,7 @@
 /dist
 /tmp
 /out-tsc
-/raiwidgets/raiwidgets/js/dist/*
-/raiwidgets/raiwidgets/js/lib/*
+/raiwidgets/raiwidgets/widget
 
 # dependencies
 **/node_modules/

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: Current File",
+      "type": "python",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal"
+    }
+  ]
+}

apps/dashboard/src/app/App.tsx

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { generateRoute } from "@responsible-ai/core-ui";
 import _ from "lodash";
 import { ITheme } from "office-ui-fabric-react";
 import React from "react";
@@ -11,7 +12,6 @@ import { App as Interpret } from "../interpret/App";
 
 import { AppHeader } from "./AppHeader";
 import { applications, IApplications, applicationKeys } from "./applications";
-import { generateRoute } from "./generateRoute";
 import { IAppSetting, routeKey } from "./IAppSetting";
 import { languages } from "./languages";
 import { themes } from "./themes";

apps/dashboard/src/fairness/App.tsx

@@ -44,42 +44,6 @@ export class App extends React.Component<IAppProps> {
     "mean_absolute_error"
   ];
 
-  private static supportedBinaryClassificationParityKeys = [
-    "accuracy_score_difference",
-    "accuracy_score_min",
-    "accuracy_score_ratio",
-    "balanced_accuracy_min",
-    "demographic_parity_difference",
-    "demographic_parity_ratio",
-    "equalized_odds_difference",
-    "equalized_odds_ratio",
-    "error_rate_difference_binary_classification",
-    "error_rate_ratio_binary_classification",
-    "f1_score_min",
-    "false_negative_rate_difference",
-    "false_negative_rate_ratio",
-    "false_positive_rate_difference",
-    "false_positive_rate_ratio",
-    "precision_score_min",
-    "recall_score_min",
-    "true_positive_rate_difference",
-    "true_positive_rate_ratio",
-    "true_negative_rate_difference",
-    "true_negative_rate_ratio"
-  ];
-
-  private static supportedRegressionParityKeys = [
-    "mean_absolute_error_max",
-    "mean_squared_error_max",
-    "r2_score_min"
-  ];
-
-  private static supportedProbabilityParityKeys = [
-    "roc_auc_score_min",
-    "log_loss_max",
-    "mean_squared_error_max"
-  ];
-
   private static messages = {
     LocalExpAndTestReq: [{ displayText: "LocalExpAndTestReq" }],
     LocalOrGlobalAndTestReq: [{ displayText: "LocalOrGlobalAndTestReq" }],
@@ -93,14 +57,10 @@ export class App extends React.Component<IAppProps> {
       locale: this.props.language,
       requestMetrics: this.generateRandomMetrics.bind(this),
       stringParams: { contextualHelp: App.messages },
-      supportedBinaryClassificationParityKeys:
-        App.supportedBinaryClassificationParityKeys,
       supportedBinaryClassificationPerformanceKeys:
         App.supportedBinaryClassificationPerformanceKeys,
-      supportedProbabilityParityKeys: App.supportedProbabilityParityKeys,
       supportedProbabilityPerformanceKeys:
         App.supportedProbabilityPerformanceKeys,
-      supportedRegressionParityKeys: App.supportedRegressionParityKeys,
       supportedRegressionPerformanceKeys:
         App.supportedRegressionPerformanceKeys,
       theme: this.props.theme

apps/widget-e2e/.eslintrc

@@ -0,0 +1,27 @@
+{
+  "overrides": [
+    {
+      "files": ["src/plugins/index.js"],
+      "rules": {
+        "@typescript-eslint/no-var-requires": "off",
+        "no-undef": "off",
+        "filenames/no-index": "off"
+      }
+    },
+    {
+      "files": ["src/support/index.ts"],
+      "rules": {
+        "filenames/no-index": "off"
+      }
+    },
+    {
+      "files": ["src/support/commands.ts"],
+      "rules": {
+        "@typescript-eslint/no-namespace": "off",
+        "@typescript-eslint/naming-convention": "off"
+      }
+    }
+  ],
+  "extends": ["plugin:cypress/recommended", "../../.eslintrc"],
+  "ignorePatterns": ["!**/*"]
+}

apps/widget-e2e/cypress.json

@@ -0,0 +1,12 @@
+{
+  "fileServerFolder": ".",
+  "fixturesFolder": "./src/fixtures",
+  "integrationFolder": "./src/integration",
+  "modifyObstructiveCode": false,
+  "pluginsFile": "./src/plugins/index",
+  "supportFile": "./src/support/index.ts",
+  "video": true,
+  "videosFolder": "../../dist/cypress/apps/widget-e2e/videos",
+  "screenshotsFolder": "../../dist/cypress/apps/widget-e2e/screenshots",
+  "chromeWebSecurity": false
+}

apps/widget-e2e/src/fixtures/example.json

@@ -0,0 +1,4 @@
+{
+  "name": "Using fixtures to represent data",
+  "email": "hello@cypress.io"
+}

apps/widget-e2e/src/integration/App.spec.ts

@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+describe("widget", () => {
+  it("should successes", () => {
+    return;
+  });
+});

apps/widget-e2e/src/plugins/index.js

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// ***********************************************************
+// This example plugins/index.js can be used to load plugins
+//
+// You can change the location of this file or turn off loading
+// the plugins file with the 'pluginsFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/plugins-guide
+// ***********************************************************
+
+// This function is called when a project is opened or re-opened (e.g. due to
+// the project's config changing)
+
+const { preprocessTypescript } = require("@nrwl/cypress/plugins/preprocessor");
+
+module.exports = (on, config) => {
+  // `on` is used to hook into various events Cypress emits
+  // `config` is the resolved Cypress config
+
+  // Preprocess Typescript file using Nx helper
+  on("file:preprocessor", preprocessTypescript(config));
+};

apps/widget-e2e/src/support/commands.ts

@@ -0,0 +1,24 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+declare namespace Cypress {
+  interface Chainable<Subject> {
+    login(email: string, password: string): void;
+  }
+}
+//
+// -- This is a parent command --
+Cypress.Commands.add("login", (email, password) => {
+  console.log("Custom command example: Login", email, password);
+});
+//
+// -- This is a child command --
+// Cypress.Commands.add("drag", { prevSubject: 'element'}, (subject, options) => { ... })
+//
+//
+// -- This is a dual command --
+// Cypress.Commands.add("dismiss", { prevSubject: 'optional'}, (subject, options) => { ... })
+//
+//
+// -- This will overwrite an existing command --
+// Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })

apps/widget-e2e/src/support/index.ts

@@ -0,0 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// Import commands.js using ES2015 syntax:
+import "./commands";

apps/widget-e2e/tsconfig.e2e.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "sourceMap": false,
+    "outDir": "../../dist/out-tsc",
+    "allowJs": true,
+    "types": ["cypress", "node"],
+    "isolatedModules": false
+  },
+  "include": ["src/**/*.ts", "src/**/*.js"]
+}

apps/widget-e2e/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "compilerOptions": {
+    "allowJs": false
+  },
+  "references": [
+    {
+      "path": "./tsconfig.e2e.json"
+    }
+  ]
+}

apps/widget/.babelrc

@@ -0,0 +1,4 @@
+{
+  "presets": ["@nrwl/react/babel"],
+  "plugins": []
+}

apps/widget/.browserslistrc

@@ -0,0 +1,16 @@
+# This file is used by:
+# 1. autoprefixer to adjust CSS to support the below specified browsers
+# 2. babel preset-env to adjust included polyfills
+#
+# For additional information regarding the format and rule options, please see:
+# https://github.com/browserslist/browserslist#queries
+#
+# If you need to support different browsers in production, you may tweak the list below.
+
+last 1 Chrome version
+last 1 Firefox version
+last 2 Edge major versions
+last 2 Safari major version
+last 2 iOS major versions
+Firefox ESR
+not IE 9-11 # For IE 9-11 support, remove 'not'.

apps/widget/.eslintrc

@@ -0,0 +1,24 @@
+{
+  "extends": ["../../.eslintrc"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["main.tsx"],
+      "rules": {
+        "filenames/match-regex": "off"
+      }
+    },
+    {
+      "files": ["src/environments/*.*"],
+      "rules": {
+        "filenames/match-regex": ["error", "^environment(\\.[a-z]+)?$"]
+      }
+    },
+    {
+      "files": ["**/__mock_data__/*.ts"],
+      "rules": {
+        "max-lines": "off"
+      }
+    }
+  ]
+}

apps/widget/babel-jest.config.json

@@ -0,0 +1,14 @@
+{
+  "presets": [
+    [
+      "@babel/preset-env",
+      {
+        "targets": {
+          "node": "current"
+        }
+      }
+    ],
+    "@babel/preset-typescript",
+    "@babel/preset-react"
+  ]
+}

apps/widget/jest.config.js

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+module.exports = {
+  coverageDirectory: "../../coverage/apps/widget",
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
+  name: "widget",
+  preset: "../../jest.config.js",
+  transform: {
+    "^(?!.*\\.(js|jsx|ts|tsx|css|json)$)": "@nrwl/react/plugins/jest",
+    "^.+\\.[tj]sx?$": [
+      "babel-jest",
+      { configFile: "./babel-jest.config.json", cwd: __dirname }
+    ]
+  }
+};

apps/widget/src/app/App.tsx

@@ -0,0 +1,35 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import React from "react";
+import { Route, Switch, RouteComponentProps } from "react-router-dom";
+
+import { Fairness } from "./Fairness";
+import { IAppConfig } from "./IAppConfig";
+import { IFairnessRouteProps } from "./IFairnessRouteProps";
+
+export interface IAppState {
+  config: IAppConfig;
+}
+
+export class App extends React.Component<unknown, IAppState> {
+  public render(): React.ReactNode {
+    return (
+      <Switch>
+        <Route path={Fairness.route} render={this.renderFairness} exact />
+      </Switch>
+    );
+  }
+  public async componentDidMount(): Promise<void> {
+    const res = await (await fetch(new Request("/getconfig"))).json();
+    this.setState({ config: res });
+  }
+  private readonly renderFairness = (
+    props: RouteComponentProps<IFairnessRouteProps>
+  ): React.ReactNode => {
+    if (!this.state?.config) {
+      return "Loading";
+    }
+    return <Fairness {...props.match.params} {...this.state.config} />;
+  };
+}

apps/widget/src/app/Fairness.tsx

@@ -0,0 +1,87 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { generateRoute } from "@responsible-ai/core-ui";
+import {
+  FairnessWizardV2,
+  IMetricRequest,
+  IMetricResponse
+} from "@responsible-ai/fairness";
+import React from "react";
+
+import { IAppConfig } from "./IAppConfig";
+import { IFairnessRouteProps, routeKey } from "./IFairnessRouteProps";
+
+export interface IFairnessState {
+  fairnessConfig: any | undefined;
+}
+
+export type IFairnessProps = IFairnessRouteProps & IAppConfig;
+export class Fairness extends React.Component<IFairnessProps, IFairnessState> {
+  public static route = `/fairness/model${generateRoute(routeKey)}`;
+
+  public async componentDidMount(): Promise<void> {
+    const res = await (
+      await fetch(new Request(`/fairness/getmodel/${this.props.model}`))
+    ).json();
+    this.setState({ fairnessConfig: res });
+  }
+  public render(): React.ReactNode {
+    if (this.state?.fairnessConfig) {
+      return (
+        <FairnessWizardV2
+          dataSummary={{
+            classNames: this.state.fairnessConfig.classes,
+            featureNames: this.state.fairnessConfig.features
+          }}
+          testData={this.state.fairnessConfig.dataset}
+          predictedY={this.state.fairnessConfig.predicted_ys}
+          trueY={this.state.fairnessConfig.true_y}
+          modelNames={this.state.fairnessConfig.model_names}
+          precomputedMetrics={this.state.fairnessConfig.precomputedMetrics}
+          precomputedFeatureBins={
+            this.state.fairnessConfig.precomputedFeatureBins
+          }
+          customMetrics={this.state.fairnessConfig.customMetrics}
+          predictionType={this.state.fairnessConfig.predictionType}
+          supportedBinaryClassificationPerformanceKeys={
+            this.state.fairnessConfig.classification_methods
+          }
+          supportedRegressionPerformanceKeys={
+            this.state.fairnessConfig.regression_methods
+          }
+          supportedProbabilityPerformanceKeys={
+            this.state.fairnessConfig.probability_methods
+          }
+          locale={this.state.fairnessConfig.locale}
+          requestMetrics={this.requestMetrics}
+        />
+      );
+    }
+    return "Loading";
+  }
+
+  private readonly requestMetrics = (
+    postData: IMetricRequest
+  ): Promise<IMetricResponse> => {
+    return fetch(this.state.fairnessConfig.metricsUrl, {
+      body: JSON.stringify(postData),
+      headers: {
+        "Content-Type": "application/json"
+      },
+      method: "post"
+    })
+      .then((resp) => {
+        if (resp.status >= 200 && resp.status < 300) {
+          return resp.json();
+        }
+        return Promise.reject(new Error(resp.statusText));
+      })
+      .then((json) => {
+        if (json.error !== undefined) {
+          throw new Error(json.error);
+        }
+        return Promise.resolve(json.data);
+      });
+  };
+}

apps/widget/src/app/IAppConfig.ts

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export interface IAppConfig {
+  localUrl: string;
+}

apps/widget/src/app/IFairnessRouteProps.tsx

@@ -0,0 +1,7 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export const routeKey = ["model"] as const;
+export type IFairnessRouteProps = {
+  [key in typeof routeKey[number]]?: string;
+};

apps/widget/src/environments/environment.prod.ts

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export const environment = {
+  production: true
+};

apps/widget/src/environments/environment.ts

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+export const environment = {
+  production: false
+};

apps/widget/src/index.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <title>Widget</title>
+    <base href="/" />
+
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link rel="icon" type="image/x-icon" href="favicon.ico" />
+    <script src="https://cdn.plot.ly/plotly-latest.js"></script>
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

apps/widget/src/main.tsx

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import React from "react";
+import ReactDOM from "react-dom";
+import { BrowserRouter } from "react-router-dom";
+
+import { App } from "./app/App";
+
+ReactDOM.render(
+  <React.StrictMode>
+    <BrowserRouter>
+      <App />
+    </BrowserRouter>
+  </React.StrictMode>,
+  document.querySelector("#root")
+);

apps/widget/src/polyfills.ts

@@ -0,0 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import "core-js/stable";
+import "regenerator-runtime/runtime.js";

apps/widget/tsconfig.app.json

@@ -0,0 +1,13 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "types": ["node"]
+  },
+  "files": [
+    "../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+    "../../node_modules/@nrwl/react/typings/image.d.ts"
+  ],
+  "exclude": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.test.ts", "**/*.test.tsx"],
+  "include": ["**/*.js", "**/*.jsx", "**/*.ts", "**/*.tsx"]
+}

apps/widget/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {},
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.app.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

apps/widget/tsconfig.spec.json

@@ -0,0 +1,23 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": [
+    "**/*.test.ts",
+    "**/*.test.tsx",
+    "**/*.test.js",
+    "**/*.test.jsx",
+    "**/*.spec.ts",
+    "**/*.spec.tsx",
+    "**/*.spec.js",
+    "**/*.spec.jsx",
+    "**/*.d.ts"
+  ],
+  "files": [
+    "../../node_modules/@nrwl/react/typings/cssmodule.d.ts",
+    "../../node_modules/@nrwl/react/typings/image.d.ts"
+  ]
+}

libs/core-ui/src/index.ts

@@ -4,6 +4,7 @@
 export * from "./lib/util/Never";
 export * from "./lib/util/PartialRequired";
 export * from "./lib/util/array";
+export * from "./lib/util/generateRoute";
 export * from "./lib/util/initializeOfficeFabric";
 export * from "./lib/util/string";
 export * from "./lib/components/ExpandableText";

libs/fairness/src/lib/IFairnessProps.ts

@@ -73,9 +73,6 @@ export type IFairnessProps = IFairnessData & {
   supportedBinaryClassificationPerformanceKeys: string[];
   supportedRegressionPerformanceKeys: string[];
   supportedProbabilityPerformanceKeys: string[];
-  supportedBinaryClassificationParityKeys: string[];
-  supportedRegressionParityKeys: string[];
-  supportedProbabilityParityKeys: string[];
   shouldInitializeIcons?: boolean;
   iconUrl?: string;
   // The request hook

libs/fairness/src/lib/util/ParityMetrics.ts

@@ -3,12 +3,15 @@
 
 import { localization } from "@responsible-ai/localization";
 
+import { PredictionTypes } from "../IFairnessProps";
+
 export interface IParityOption {
   key: string;
   title: string;
   description?: string;
   parityMetric: string;
   parityMode: ParityModes;
+  supportedTasks: Set<PredictionTypes>;
 }
 
 export enum ParityModes {
@@ -25,6 +28,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "accuracy_score_difference",
     parityMetric: "accuracy_score",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.accuracyScoreDifference
   },
   accuracy_score_min: {
@@ -32,6 +36,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "accuracy_score_min",
     parityMetric: "accuracy_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.accuracyScoreMin
   },
   accuracy_score_ratio: {
@@ -39,6 +44,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "accuracy_score_ratio",
     parityMetric: "accuracy_score",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.accuracyScoreRatio
   },
   balanced_accuracy_score_min: {
@@ -47,6 +53,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "balanced_accuracy_score_min",
     parityMetric: "balanced_accuracy_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.balancedAccuracyScoreMin
   },
   demographic_parity_difference: {
@@ -55,6 +62,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "demographic_parity_difference",
     parityMetric: "selection_rate",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.demographicParityDifference
   },
   demographic_parity_ratio: {
@@ -63,6 +71,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "demographic_parity_ratio",
     parityMetric: "selection_rate",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.demographicParityRatio
   },
   equalized_odds_difference: {
@@ -71,6 +80,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "equalized_odds_difference",
     parityMetric: "", // combination of two metrics
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.equalizedOddsDifference
   },
   equalized_odds_ratio: {
@@ -78,36 +88,25 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "equalized_odds_ratio",
     parityMetric: "", // combination of two metrics
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.equalizedOddsRatio
   },
   // zero_one_loss is the error rate for binary classification, while
   // mean_absolute_error is the error rate for probabilistic classification and regression
-  error_rate_difference_binary_classification: {
+  error_rate_difference: {
     description: localization.Fairness.Metrics.errorRateDifferenceDescription,
-    key: "error_rate_difference_binary_classification",
+    key: "error_rate_difference",
     parityMetric: "zero_one_loss",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.errorRateDifference
   },
-  error_rate_difference_regression: {
-    description: localization.Fairness.Metrics.errorRateDifferenceDescription,
-    key: "error_rate_difference_regression",
-    parityMetric: "mean_absolute_error",
-    parityMode: ParityModes.Difference,
-    title: localization.Fairness.Metrics.errorRateDifference
-  },
-  error_rate_ratio_binary_classification: {
+  error_rate_ratio: {
     description: localization.Fairness.Metrics.errorRateRatioDescription,
-    key: "error_rate_ratio_binary_classification",
+    key: "error_rate_ratio",
     parityMetric: "zero_one_loss",
     parityMode: ParityModes.Ratio,
-    title: localization.Fairness.Metrics.errorRateRatio
-  },
-  error_rate_ratio_regression: {
-    description: localization.Fairness.Metrics.errorRateRatioDescription,
-    key: "error_rate_ratio_regression",
-    parityMetric: "mean_absolute_error",
-    parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.errorRateRatio
   },
   f1_score_min: {
@@ -115,6 +114,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "f1_score_min",
     parityMetric: "f1_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.f1ScoreMin
   },
   false_negative_rate_difference: {
@@ -123,6 +123,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "false_negative_rate_difference",
     parityMetric: "false_negative_over_total",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.falseNegativeRateDifference
   },
   false_negative_rate_ratio: {
@@ -131,6 +132,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "false_negative_rate_ratio",
     parityMetric: "false_negative_over_total",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.falseNegativeRateRatio
   },
   false_positive_rate_difference: {
@@ -139,6 +141,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "false_positive_rate_difference",
     parityMetric: "false_positive_over_total",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.falsePositiveRateDifference
   },
   false_positive_rate_ratio: {
@@ -147,6 +150,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "false_positive_rate_ratio",
     parityMetric: "false_positive_over_total",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.falsePositiveRateRatio
   },
   log_loss_max: {
@@ -154,6 +158,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "log_loss_max",
     parityMetric: "log_loss",
     parityMode: ParityModes.Max,
+    supportedTasks: new Set([PredictionTypes.Probability]),
     title: localization.Fairness.Metrics.logLossMax
   },
   mean_absolute_error_max: {
@@ -161,6 +166,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "mean_absolute_error_max",
     parityMetric: "mean_absolute_error",
     parityMode: ParityModes.Max,
+    supportedTasks: new Set([PredictionTypes.Regression]),
     title: localization.Fairness.Metrics.meanAbsoluteErrorMax
   },
   mean_squared_error_max: {
@@ -168,6 +174,10 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "mean_squared_error_max",
     parityMetric: "mean_squared_error",
     parityMode: ParityModes.Max,
+    supportedTasks: new Set([
+      PredictionTypes.Regression,
+      PredictionTypes.Probability
+    ]),
     title: localization.Fairness.Metrics.meanSquaredErrorMax
   },
   precision_score_min: {
@@ -175,6 +185,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "precision_score_min",
     parityMetric: "precision_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.precisionScoreMin
   },
   r2_score_min: {
@@ -182,6 +193,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "r2_score_min",
     parityMetric: "r2_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.Regression]),
     title: localization.Fairness.Metrics.r2ScoreMin
   },
   recall_score_min: {
@@ -189,6 +201,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "recall_score_min",
     parityMetric: "recall_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.recallScoreMin
   },
   roc_auc_score_min: {
@@ -196,6 +209,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "roc_auc_score_min",
     parityMetric: "roc_auc_score",
     parityMode: ParityModes.Min,
+    supportedTasks: new Set([PredictionTypes.Probability]),
     title: localization.Fairness.Metrics.ROCAUCScoreMin
   },
   true_negative_rate_difference: {
@@ -204,6 +218,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "true_negative_rate_difference",
     parityMetric: "recall_score",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.trueNegativeRateDifference
   },
   true_negative_rate_ratio: {
@@ -212,6 +227,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "true_negative_rate_ratio",
     parityMetric: "recall_score",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.trueNegativeRateRatio
   },
   true_positive_rate_difference: {
@@ -220,6 +236,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "true_positive_rate_difference",
     parityMetric: "recall_score",
     parityMode: ParityModes.Difference,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.truePositiveRateDifference
   },
   true_positive_rate_ratio: {
@@ -228,6 +245,7 @@ export const parityOptions: { [key: string]: IParityOption } = {
     key: "true_positive_rate_ratio",
     parityMetric: "recall_score",
     parityMode: ParityModes.Ratio,
+    supportedTasks: new Set([PredictionTypes.BinaryClassification]),
     title: localization.Fairness.Metrics.truePositiveRateRatio
   }
 };

libs/fairness/src/lib/v2/FairnessWizard.tsx

@@ -344,25 +344,11 @@ export class FairnessWizardV2 extends React.PureComponent<
   private getParityMetrics(
     fairnessContext: IRunTimeFairnessContext
   ): IParityOption[] {
-    if (
-      fairnessContext.modelMetadata.PredictionType ===
-      PredictionTypes.BinaryClassification
-    ) {
-      return this.props.supportedBinaryClassificationParityKeys.map(
-        (key) => parityOptions[key]
-      );
-    }
-    if (
-      fairnessContext.modelMetadata.PredictionType ===
-      PredictionTypes.Regression
-    ) {
-      return this.props.supportedRegressionParityKeys.map(
-        (key) => parityOptions[key]
-      );
-    }
-    return this.props.supportedProbabilityParityKeys.map(
-      (key) => parityOptions[key]
+    return Object.values(parityOptions).filter((parityOption) => {
+      return parityOption.supportedTasks.has(
+        fairnessContext.modelMetadata.PredictionType
       );
+    });
   }
 
   private readonly hideIntro = (): void => {

nx.json

@@ -41,6 +41,13 @@
     },
     "localization": {
       "tags": []
+    },
+    "widget": {
+      "tags": []
+    },
+    "widget-e2e": {
+      "tags": [],
+      "implicitDependencies": ["widget"]
     }
   }
 }

package.json

@@ -36,6 +36,7 @@
     "workspace-schematic": "nx workspace-schematic"
   },
   "dependencies": {
+    "document-register-element": "1.13.1",
     "eslint-plugin-sort-keys-fix": "^1.1.1",
     "jmespath": "^0.15.0",
     "localized-strings": "^0.2.4",
@@ -62,6 +63,7 @@
     "@nrwl/workspace": "10.2.0",
     "@rollup/plugin-json": "4.1.0",
     "@svgr/rollup": "5.4.0",
+    "@testing-library/react": "10.4.1",
     "@types/enzyme": "3.10.5",
     "@types/enzyme-adapter-react-16": "1.0.6",
     "@types/enzyme-to-json": "1.5.3",

raiwidgets/raiwidgets/fairness_dashboard.py

@@ -8,7 +8,7 @@
 from rai_core_flask import FlaskHelper  # , environment_detector
 from .fairness_metric_calculation import FairnessMetricModule
 
-from flask import jsonify, request
+from flask import jsonify, request, Response
 from IPython.display import display, HTML
 from jinja2 import Environment, PackageLoader
 import json
@@ -35,26 +35,48 @@ class FairnessDashboard(object):
     :param sensitive_feature_names: Feature names
     :type sensitive_feature_names: numpy.array or list[]
     """
-    env = Environment(loader=PackageLoader(__name__, 'templates'))
-    default_template = env.get_template("inlineDashboard.html")
+    env = Environment(loader=PackageLoader(__name__, 'widget'))
     _dashboard_js = None
     fairness_inputs = {}
     model_count = 0
     _service = None
 
-    @FlaskHelper.app.route('/')
+    @FlaskHelper.app.route('/widget/<path:path>')
+    def widget_static(path):
+        mimetypes = {
+            ".css": "text/css",
+            ".html": "text/html",
+            ".js": "application/javascript",
+        }
+        ext = os.path.splitext(path)[1]
+        mimetype = mimetypes.get(ext, "application/octet-stream")
+        return Response(load_widget_file(path), mimetype=mimetype)
+
+    @FlaskHelper.app.route('/getconfig')
+    def get_config():
+        burl = FairnessDashboard._service.env.base_url
+        ct = FairnessDashboard.model_count
+        return {
+            "local_url": f"{burl}/fairness/model/{ct}"
+        }
+
+    @FlaskHelper.app.route('/fairness')
     def list():
         return "No global list view supported at this time."
 
-    @FlaskHelper.app.route('/<id>')
+    @FlaskHelper.app.route('/fairness/model/<id>')
     def fairness_visual(id):
-        if id in FairnessDashboard.fairness_inputs:
-            return generate_inline_html(
-                FairnessDashboard.fairness_inputs[id], None)
-        else:
-            return "Unknown model id."
+        return load_widget_file("index.html")
 
-    @FlaskHelper.app.route('/<id>/metrics', methods=['POST'])
+    @FlaskHelper.app.route('/fairness/getmodel/<id>')
+    def fairness_get_model(id):
+        if id in FairnessDashboard.fairness_inputs:
+            model_data = json.dumps(FairnessDashboard.fairness_inputs[id])
+            return model_data
+        else:
+            return Response("Unknown model id.", status=404)
+
+    @FlaskHelper.app.route('/fairness/model/<id>/metrics', methods=['POST'])
     def fairness_metrics_calculation(id):
         try:
             data = request.get_json(force=True)
@@ -156,8 +178,6 @@ class FairnessDashboard(object):
                               "ignoring")
             fairness_input["features"] = sensitive_feature_names
 
-        self._load_local_js()
-
         if FairnessDashboard._service is None:
             try:
                 FairnessDashboard._service = FlaskHelper(port=port)
@@ -168,7 +188,9 @@ class FairnessDashboard(object):
         FairnessDashboard.model_count += 1
         model_count = FairnessDashboard.model_count
 
-        local_url = f"{FairnessDashboard._service.env.base_url}/{model_count}"
+        burl = FairnessDashboard._service.env.base_url
+
+        local_url = f"{burl}/fairness/model/{model_count}"
         metrics_url = f"{local_url}/metrics"
 
         fairness_input['metricsUrl'] = metrics_url
@@ -178,17 +200,11 @@ class FairnessDashboard(object):
 
         FairnessDashboard.fairness_inputs[str(model_count)] = fairness_input
 
-        html = generate_inline_html(fairness_input, local_url)
+        html = load_widget_file("index.html")
         # TODO https://github.com/microsoft/responsible-ai-widgets/issues/92
         # FairnessDashboard._service.env.display(html)
         display(HTML(html))
 
-    def _load_local_js(self):
-        script_path = os.path.dirname(os.path.abspath(__file__))
-        js_path = os.path.join(script_path, "static", "index.js")
-        with open(js_path, "r", encoding="utf-8") as f:
-            FairnessDashboard._dashboard_js = f.read()
-
     def _sanitize_data_shape(self, dataset):
         result = self._convert_to_list(dataset)
         # Dataset should be 2d, if not we need to map
@@ -210,10 +226,12 @@ class FairnessDashboard(object):
         return array
 
 
-def generate_inline_html(fairness_input, local_url):
-    return FairnessDashboard.default_template.render(
-        fairness_input=json.dumps(fairness_input),
-        main_js=FairnessDashboard._dashboard_js,
-        app_id='app_fairness',
-        local_url=local_url,
-        has_local_url=local_url is not None)
+def get_widget_path(path):
+    script_path = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(script_path, "widget", path)
+
+
+def load_widget_file(path):
+    js_path = get_widget_path(path)
+    with open(js_path, "r", encoding="utf-8") as f:
+        return f.read()

raiwidgets/raiwidgets/fairness_metric_calculation.py

@@ -55,11 +55,11 @@ class FairnessMetricModule:
                     "model_type": [],
                     "function": module.false_positive_rate_group_summary
                 },
-                "false_positive_over_total": {
+                "false_positive_rate": {
                     "model_type": [],
                     "function": module.false_positive_rate_group_summary
                 },
-                "false_negative_over_total": {
+                "false_negative_rate": {
                     "model_type": [],
                     "function": module.false_negative_rate_group_summary
                 },

raiwidgets/raiwidgets/js/.babelrc

@@ -1,7 +0,0 @@
-{
-  "presets": ["@babel/preset-env", "@babel/preset-react"],
-  "plugins": [
-    "@babel/plugin-proposal-class-properties",
-    "transform-regenerator"
-  ]
-}

raiwidgets/raiwidgets/js/main.js

@@ -1,71 +0,0 @@
-import React from "react";
-import ReactDOM from "react-dom";
-import axios from "axios";
-import "babel-polyfill";
-
-import { FairnessWizardV2 } from "@responsible-ai/fairness";
-
-const RenderDashboard = (divId, data) => {
-  let calculateMetrics = (postData) => {
-    if (data.withCredentials) {
-      var headers_data = {
-        Accept:
-          "application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
-        "Content-Type": "application/json"
-      };
-      axios.defaults.withCredentials = true;
-      var axios_options = { headers: headers_data, withCredentials: true };
-      return axios
-        .post(data.metricsUrl, JSON.stringify(postData), axios_options)
-        .then((response) => {
-          return response.data;
-        })
-        .catch(function (error) {
-          throw new Error(error);
-        });
-    } else {
-      return fetch(data.metricsUrl, {
-        method: "post",
-        body: JSON.stringify(postData),
-        headers: {
-          "Content-Type": "application/json"
-        }
-      })
-        .then((resp) => {
-          if (resp.status >= 200 && resp.status < 300) {
-            return resp.json();
-          }
-          return Promise.reject(new Error(resp.statusText));
-        })
-        .then((json) => {
-          if (json.error !== undefined) {
-            throw new Error(json.error);
-          }
-          return Promise.resolve(json.data);
-        });
-    }
-  };
-
-  ReactDOM.render(
-    <FairnessWizardV2
-      dataSummary={{ featureNames: data.features, classNames: data.classes }}
-      testData={data.dataset}
-      predictedY={data.predicted_ys}
-      trueY={data.true_y}
-      modelNames={data.model_names}
-      precomputedMetrics={data.precomputedMetrics}
-      precomputedFeatureBins={data.precomputedFeatureBins}
-      customMetrics={data.customMetrics}
-      predictionType={data.predictionType}
-      supportedBinaryClassificationAccuracyKeys={data.classification_methods}
-      supportedRegressionAccuracyKeys={data.regression_methods}
-      supportedProbabilityAccuracyKeys={data.probability_methods}
-      locale={data.locale}
-      key={new Date()}
-      requestMetrics={calculateMetrics}
-    />,
-    document.getElementById(divId)
-  );
-};
-
-export { RenderDashboard, FairnessWizardV2 };

raiwidgets/raiwidgets/js/package.json

@@ -1,38 +0,0 @@
-{
-  "name": "inline-fairness-dashboard",
-  "version": "1.0.0",
-  "description": "Host for component to be consumed as npm package",
-  "license": "MIT",
-  "author": "",
-  "main": "main.js",
-  "scripts": {
-    "build": "webpack",
-    "start": "start http://localhost:7777/ & webpack-dev-server",
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "dependencies": {
-    "@babel/core": "7.4.3",
-    "@babel/plugin-proposal-class-properties": "^7.4.0",
-    "@babel/plugin-transform-runtime": "7.11.5",
-    "@babel/preset-env": "7.4.3",
-    "@babel/preset-react": "7.0.0",
-    "@responsible-ai/fairness": "0.1.4",
-    "axios": "^0.19.2",
-    "babel-loader": "8.0.5",
-    "babel-polyfill": "^6.26.0",
-    "babel-preset-es2015": "6.24.1",
-    "babel-preset-react": "6.24.1",
-    "core-js": "3.6.5",
-    "css-loader": "2.1.1",
-    "lodash": "4.17.20",
-    "react": "16.13.1",
-    "react-dom": "16.13.1",
-    "style-loader": "0.23.1",
-    "webpack": "4.29.6",
-    "webpack-cli": "3.3.0",
-    "webpack-dev-server": "3.2.1"
-  },
-  "devDependencies": {
-    "babel-plugin-transform-regenerator": "^6.26.0"
-  }
-}

raiwidgets/raiwidgets/js/webpack.config.js

@@ -1,43 +0,0 @@
-var path = require("path");
-
-var config = {
-  entry: "./main.js",
-
-  output: {
-    path: path.resolve(__dirname, "..", "static"),
-    filename: "index.js",
-    library: "inline-fairness-dashboard",
-    libraryTarget: "umd",
-    umdNamedDefine: true
-  },
-  devtool: "source-map",
-  module: {
-    rules: [
-      { test: /\.css$/, use: ["style-loader", "css-loader"] },
-      {
-        test: /\.jsx?$/,
-        exclude: /node_modules/,
-        use: {
-          loader: "babel-loader"
-        }
-      },
-      {
-        test: /\.js?$/,
-        exclude: /node_modules/,
-        use: {
-          loader: "babel-loader"
-        }
-      }
-    ]
-  },
-
-  optimization: {
-    usedExports: true
-  },
-
-  node: {
-    fs: "empty"
-  }
-};
-
-module.exports = [config];

raiwidgets/raiwidgets/templates/inlineDashboard.html

@@ -1,32 +0,0 @@
-<!DOCTYPE html>
-<title>Fairness Dashboard</title>
-{% if has_local_url %}
-<a href="{{local_url}}" target="_blank">Open in new tab</a>
-{% endif %}
-<div style="height: 1000px;" id="{{app_id}}"></div>
-<script type="text/javascript"> {{ main_js }}; </script>
-<script defer type="text/javascript">
-  (function universalLoad(root, callback) {
-        if(typeof exports === 'object' && typeof module === 'object') {
-          // CommonJS2
-          var fairnessInline = require('inline-fairness-dashboard');
-          callback(fairnessInline);
-        } else if(typeof define === 'function' && define.amd) {
-          // AMD
-          require(['inline-fairness-dashboard'], function(fairnessInline) {
-            callback(fairnessInline);
-          });
-        } else if(typeof exports === 'object') {
-          // CommonJS
-          var fairnessInline = require('inline-fairness-dashboard');
-          callback(fairnessInline);
-        } else {
-          // Browser
-          callback(root['inline-fairness-dashboard']);
-        }
-      })(this, function(fairnessInline) {
-          var data = {{ fairness_input | safe }};
-          //data.predictionUrl = data.predictionUrl ? data.predictionUrl : undefined;
-          fairnessInline.RenderDashboard("{{app_id}}", data);
-      });
-</script>

raiwidgets/setup.py

@@ -35,9 +35,7 @@ setuptools.setup(
     include_package_data=True,
     package_data={
         '': [
-            'templates/inlineDashboard.html',
-            'static/index.js',
-            'static/index.js.map'
+            'widget/**'
         ]
     },
     zip_safe=False,

raiwidgets/tests/test.py

@@ -0,0 +1,62 @@
+# Copyright (c) Microsoft Corporation
+# Licensed under the MIT License.
+
+from raiwidgets import FairnessDashboard
+from sklearn.model_selection import train_test_split
+
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.linear_model import LogisticRegression
+import pandas as pd
+
+from sklearn.datasets import fetch_openml
+data = fetch_openml(data_id=1590, as_frame=True)
+X_raw = data.data
+Y = (data.target == '>50K') * 1
+X_raw
+
+
+A = X_raw["sex"]
+X = X_raw.drop(labels=['sex'], axis=1)
+X = pd.get_dummies(X)
+
+sc = StandardScaler()
+X_scaled = sc.fit_transform(X)
+X_scaled = pd.DataFrame(X_scaled, columns=X.columns)
+
+le = LabelEncoder()
+Y = le.fit_transform(Y)
+
+
+X_train,\
+    X_test,\
+    Y_train,\
+    Y_test,\
+    A_train,\
+    A_test = train_test_split(X_scaled,
+                              Y,
+                              A,
+                              test_size=0.2,
+                              random_state=0,
+                              stratify=Y)
+
+
+X_train = X_train.reset_index(drop=True)
+A_train = A_train.reset_index(drop=True)
+X_test = X_test.reset_index(drop=True)
+A_test = A_test.reset_index(drop=True)
+
+
+unmitigated_predictor = LogisticRegression(
+    solver='liblinear', fit_intercept=True)
+
+unmitigated_predictor.fit(X_train, Y_train)
+
+
+FairnessDashboard(sensitive_features=A_test, sensitive_feature_names=['sex'],
+                  y_true=Y_test,
+                  y_pred={
+                      "unmitigated": unmitigated_predictor.predict(X_test)
+})
+
+
+input("Press Enter to continue...")

workspace.json

@@ -390,6 +390,118 @@
           }
         }
       }
+    },
+    "widget": {
+      "root": "apps/widget",
+      "sourceRoot": "apps/widget/src",
+      "projectType": "application",
+      "schematics": {},
+      "architect": {
+        "build": {
+          "builder": "@nrwl/web:build",
+          "options": {
+            "baseHref": "/widget/",
+            "outputPath": "raiwidgets/raiwidgets/widget",
+            "index": "apps/widget/src/index.html",
+            "main": "apps/widget/src/main.tsx",
+            "polyfills": "apps/widget/src/polyfills.ts",
+            "tsConfig": "apps/widget/tsconfig.app.json",
+            "progress": false,
+            "memoryLimit": 4096,
+            "globals": [
+              {
+                "moduleId": "plotly.js",
+                "global": "Plotly"
+              }
+            ],
+            "assets": ["apps/widget/src/favicon.ico", "apps/widget/src/assets"],
+            "styles": [],
+            "scripts": [],
+            "webpackConfig": "./webpack.config.js"
+          },
+          "configurations": {
+            "production": {
+              "fileReplacements": [
+                {
+                  "replace": "apps/widget/src/environments/environment.ts",
+                  "with": "apps/widget/src/environments/environment.prod.ts"
+                }
+              ],
+              "optimization": true,
+              "outputHashing": "all",
+              "sourceMap": false,
+              "extractCss": true,
+              "namedChunks": false,
+              "extractLicenses": true,
+              "vendorChunk": false,
+              "budgets": [
+                {
+                  "type": "initial",
+                  "maximumWarning": "2mb",
+                  "maximumError": "5mb"
+                }
+              ]
+            }
+          }
+        },
+        "serve": {
+          "builder": "@nrwl/web:dev-server",
+          "options": {
+            "buildTarget": "widget:build"
+          },
+          "configurations": {
+            "production": {
+              "buildTarget": "widget:build:production"
+            }
+          }
+        },
+        "lint": {
+          "builder": "@nrwl/linter:lint",
+          "options": {
+            "linter": "eslint",
+            "tsConfig": [
+              "apps/widget/tsconfig.app.json",
+              "apps/widget/tsconfig.spec.json"
+            ],
+            "exclude": ["**/node_modules/**", "!apps/widget/**/*"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "apps/widget/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
+    },
+    "widget-e2e": {
+      "root": "apps/widget-e2e",
+      "sourceRoot": "apps/widget-e2e/src",
+      "projectType": "application",
+      "architect": {
+        "e2e": {
+          "builder": "@nrwl/cypress:cypress",
+          "options": {
+            "cypressConfig": "apps/widget-e2e/cypress.json",
+            "tsConfig": "apps/widget-e2e/tsconfig.e2e.json",
+            "devServerTarget": "widget:serve"
+          },
+          "configurations": {
+            "production": {
+              "devServerTarget": "widget:serve:production"
+            }
+          }
+        },
+        "lint": {
+          "builder": "@nrwl/linter:lint",
+          "options": {
+            "linter": "eslint",
+            "tsConfig": ["apps/widget-e2e/tsconfig.e2e.json"],
+            "exclude": ["**/node_modules/**", "!apps/widget-e2e/**/*"]
+          }
+        }
+      }
     }
   },
   "cli": {
@@ -408,16 +520,20 @@
     },
     "@nrwl/react": {
       "application": {
+        "style": "none",
         "linter": "eslint",
         "babel": true
       },
       "library": {
+        "style": "none",
         "linter": "eslint"
       },
       "storybook-configuration": {
         "linter": "eslint"
       },
-      "component": {}
+      "component": {
+        "style": "none"
+      }
     },
     "@nrwl/next": {
       "application": {

wrapped-flask/rai_core_flask/flask_helper.py

@@ -87,6 +87,7 @@ class FlaskHelper(object):
 
         server = WSGIServer((self.ip, self.port), self.app, log=devnull)
         self.app.config["server"] = server
+        self.app.config["CACHE_TYPE"] = "null"
         server.serve_forever()
 
         # Closes server on program exit, including freeing all sockets

yarn.lock

@@ -2136,6 +2136,28 @@
     "@svgr/plugin-svgo" "^5.4.0"
     loader-utils "^2.0.0"
 
+"@testing-library/dom@^7.17.1":
+  version "7.26.3"
+  resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-7.26.3.tgz#5554ee985f712d621bd676104b879f85d9a7a0ef"
+  integrity sha512-/1P6taENE/H12TofJaS3L1J28HnXx8ZFhc338+XPR5y1E3g5ttOgu86DsGnV9/n2iPrfJQVUZ8eiGYZGSxculw==
+  dependencies:
+    "@babel/code-frame" "^7.10.4"
+    "@babel/runtime" "^7.10.3"
+    "@types/aria-query" "^4.2.0"
+    aria-query "^4.2.2"
+    chalk "^4.1.0"
+    dom-accessibility-api "^0.5.1"
+    lz-string "^1.4.4"
+    pretty-format "^26.4.2"
+
+"@testing-library/react@10.4.1":
+  version "10.4.1"
+  resolved "https://registry.yarnpkg.com/@testing-library/react/-/react-10.4.1.tgz#d38dee4abab172c06f6cf8894c51190e6c503f61"
+  integrity sha512-QX31fRDGLnOdBYoQ95VEOYgRahaPfsI+toOaYhlvuGNFQrcagZv/KLWCIctRGB0h1PTsQt3JpLBbbLGM63yy5Q==
+  dependencies:
+    "@babel/runtime" "^7.10.3"
+    "@testing-library/dom" "^7.17.1"
+
 "@tootallnate/once@1":
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
@@ -2177,6 +2199,11 @@
   dependencies:
     "@turf/helpers" "6.x"
 
+"@types/aria-query@^4.2.0":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.0.tgz#14264692a9d6e2fa4db3df5e56e94b5e25647ac0"
+  integrity sha512-iIgQNzCm0v7QMhhe4Jjn9uRh+I6GoPmt03CbEtwx3ao8/EfoQcmgtqH4vQ5Db/lxiIGaWDv6nwvunuh0RyX0+A==
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7":
   version "7.1.9"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.9.tgz#77e59d438522a6fb898fa43dc3455c6e72f3963d"
@@ -4159,7 +4186,7 @@ chalk@^3.0.0:
     ansi-styles "^4.1.0"
     supports-color "^7.1.0"
 
-chalk@^4.0.0:
+chalk@^4.0.0, chalk@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.0.tgz#4e14870a618d9e2edd97dd8345fd9d9dc315646a"
   integrity sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==
@@ -5693,6 +5720,18 @@ doctrine@^3.0.0:
   dependencies:
     esutils "^2.0.2"
 
+document-register-element@1.13.1:
+  version "1.13.1"
+  resolved "https://registry.yarnpkg.com/document-register-element/-/document-register-element-1.13.1.tgz#dad8cb7be38e04ee3f56842e6cf81af46c1249ba"
+  integrity sha512-92ZyLDKg9j4rOll//NNXj25f+8rAzOkYsGJonhugKwXfeqH7bzs8Ucpvey0WzZ2ZzKdrvW9RnUw3UyOZ/uhBFw==
+  dependencies:
+    lightercollective "^0.1.0"
+
+dom-accessibility-api@^0.5.1:
+  version "0.5.4"
+  resolved "https://registry.yarnpkg.com/dom-accessibility-api/-/dom-accessibility-api-0.5.4.tgz#b06d059cdd4a4ad9a79275f9d414a5c126241166"
+  integrity sha512-TvrjBckDy2c6v6RLxPv5QXOnU+SmF9nBII5621Ve5fu6Z/BDrENurBEvlC1f44lKEUVqOpK4w9E5Idc5/EgkLQ==
+
 dom-serializer@0:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.2.2.tgz#1afb81f533717175d478655debc5e332d9f9bb51"
@@ -9817,6 +9856,11 @@ license-webpack-plugin@2.1.2:
     "@types/webpack-sources" "^0.1.5"
     webpack-sources "^1.2.0"
 
+lightercollective@^0.1.0:
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/lightercollective/-/lightercollective-0.1.0.tgz#70df102c530dcb8d0ccabfe6175a8d00d5f61300"
+  integrity sha512-J9tg5uraYoQKaWbmrzDDexbG6hHnMcWS1qLYgJSWE+mpA3U5OCSeMUhb+K55otgZJ34oFdR0ECvdIb3xuO5JOQ==
+
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -10078,6 +10122,11 @@ lru-cache@^6.0.0:
   dependencies:
     yallist "^4.0.0"
 
+lz-string@^1.4.4:
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
+  integrity sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY=
+
 magic-string@0.25.7, magic-string@^0.25.2:
   version "0.25.7"
   resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051"