Xuke/adde2e (#982)

* add e2e for widget notebook

.github/workflows/CI-notebook.yml

@@ -56,6 +56,9 @@ jobs:
       - name: Run notebook tests
         run: python -m pytest notebooks
 
+      - name: Run widget tests
+        run: yarn e2e-widget
+
       - name: Upload notebook test result
         if: always()
         uses: actions/upload-artifact@v2

.github/workflows/CI-python.yml

@@ -93,6 +93,10 @@ jobs:
           pytest --durations=10 --doctest-modules --junitxml=junit/test-results.xml --cov=${{ matrix.packageDirectory }} --cov-report=xml --cov-report=html
         working-directory: ${{ matrix.packageDirectory }}
 
+      - if: ${{ matrix.packageDirectory == 'raiwidgets' }}
+        name: Run widget tests
+        run: yarn e2e-widget
+
       - if: ${{ matrix.packageDirectory == 'raiwidgets' }}
         name: Run tests
         run: |

.github/workflows/release-rai.yml

@@ -109,6 +109,9 @@ jobs:
         run: pytest ./tests/
         working-directory: ${{ env.raiDirectory }}
 
+      - name: Run widget tests
+        run: yarn e2e-widget
+
       - name: Upload a raiwidgets build result
         uses: actions/upload-artifact@v2
         with:

.gitignore

@@ -58,6 +58,7 @@ Thumbs.db
 
 # test files
 apps/*-e2e/debug.log
+**/cypress.env.json
 
 # notebook artifacts
 /notebooks/*.csv
@@ -65,4 +66,6 @@ apps/*-e2e/debug.log
 
 # docs
 docs/build/*
-*/.mypy_cache/*
+*/.mypy_cache/*
+
+notebooks/responsibleaitoolbox-dashboard/*.py

apps/widget-e2e/.eslintrc.json

@@ -12,7 +12,8 @@
       "rules": {
         "@typescript-eslint/no-var-requires": "off",
         "no-undef": "off",
-        "filenames/no-index": "off"
+        "filenames/no-index": "off",
+        "unicorn/no-null": "off"
       }
     },
     {

apps/widget-e2e/src/integration/App.spec.ts

@@ -2,7 +2,11 @@
 // Licensed under the MIT License.
 
 describe("widget", () => {
-  it("should successes", () => {
+  it("should render", () => {
+    const hosts = Cypress.env().hosts;
+    cy.task("log", hosts);
+    cy.visit(hosts[0].host);
+    cy.get("#ModelAssessmentDashboard").should("exist");
     return;
   });
 });

apps/widget-e2e/src/plugins/index.js

@@ -22,4 +22,10 @@ module.exports = (on, config) => {
 
   // Preprocess Typescript file using Nx helper
   on("file:preprocessor", preprocessTypescript(config));
+  on("task", {
+    log(message) {
+      console.log(message);
+      return null;
+    }
+  });
 };

libs/model-assessment/src/lib/ModelAssessmentDashboard/ModelAssessmentDashboard.tsx

@@ -111,7 +111,10 @@ export class ModelAssessmentDashboard extends CohortBasedComponent<
           theme: this.props.theme
         }}
       >
-        <Stack className={modelAssessmentDashboardStyles.page}>
+        <Stack
+          id="ModelAssessmentDashboard"
+          className={modelAssessmentDashboardStyles.page}
+        >
           <MainMenu
             activeGlobalTabs={this.state.activeGlobalTabs}
             removeTab={this.removeTab}

package.json

@@ -15,12 +15,11 @@
     "auto-version": "node ./scripts/auto-version.js",
     "build": "nx build",
     "buildall": "cross-env NODE_OPTIONS=--max_old_space_size=6144 nx run-many --target=build --configuration=production --all",
-    "ci": "yarn install --frozen-lock-file && yarn testall --codeCoverage && yarn lintall && yarn buildall && yarn e2eall",
+    "ci": "yarn install --frozen-lock-file && yarn testall --codeCoverage && yarn lintall && yarn buildall && yarn e2e",
     "copytest": "node ./scripts/copyTest.js",
     "dep-graph": "nx dep-graph",
-    "e2e": "nx e2e",
-    "e2e-watch": "yarn e2e dashboard-e2e --watch",
-    "e2eall": "nx run-many --target=e2e --all",
+    "e2e": "nx e2e dashboard-e2e",
+    "e2e-widget": "node ./scripts/e2e-widget.js",
     "format": "nx format:write",
     "format:check": "nx format:check",
     "format:write": "nx format:write",

scripts/e2e-widget.js

@@ -0,0 +1,139 @@
+const { spawnSync, spawn } = require("child_process");
+const fs = require("fs");
+const path = require("path");
+const _ = require("lodash");
+const commander = require("commander");
+
+const baseDir = path.join(
+  __dirname,
+  "../notebooks/responsibleaitoolbox-dashboard"
+);
+const filePrefix = "responsibleaitoolbox";
+
+const hostReg = /^ModelAssessment started at (http:\/\/localhost:\d+)$/m;
+const timeout = 3600;
+
+/**
+ *
+ * @param {string} name
+ * @returns {Promise<string>}
+ */
+async function runNotebook(name) {
+  console.log(`Running ${name}`);
+  const timer = setTimeout(() => {
+    throw new Error(`${name} timeout.`);
+  }, timeout * 1000);
+  const nbProcess = spawn("python", ["-i", path.join(baseDir, name)]);
+  nbProcess.on("exit", () => {
+    throw new Error(`Failed to run notebook ${name}`);
+  });
+  return new Promise((resolve) => {
+    let stdout = "";
+    const handleOutput = (data) => {
+      const message = data.toString();
+      stdout += message;
+      console.log(message);
+      if (hostReg.test(stdout)) {
+        clearTimeout(timer);
+        resolve(hostReg.exec(stdout)[1]);
+      }
+    };
+    nbProcess.stdout.on("data", handleOutput);
+    nbProcess.stderr.on("data", handleOutput);
+    nbProcess.stdout.on("error", (error) => {
+      throw error;
+    });
+  });
+}
+
+function convertNotebook() {
+  console.log("Converting notebook");
+  const { status, stderr } = spawnSync(
+    "jupyter",
+    ["nbconvert", path.join(baseDir, `${filePrefix}*.ipynb`), "--to", "script"],
+    {
+      stdio: "inherit"
+    }
+  );
+  if (status) {
+    throw new Error(`Failed to convert notebook:\r\n\r\n${stderr}`);
+  }
+  console.log("Converted notebook\r\n");
+}
+/**
+ * @typedef {Object} Host
+ * @property {string} file
+ * @property {string} host
+ */
+/**
+ * @returns {Host[]}
+ */
+async function runNotebooks() {
+  const files = fs
+    .readdirSync(baseDir)
+    .filter((f) => f.startsWith(filePrefix) && f.endsWith(".py"));
+  const hosts = [];
+  for (const f of files) {
+    hosts.push({ file: f, host: await runNotebook(f) });
+  }
+  return hosts;
+}
+
+/**
+ *
+ * @param {Host[]} hosts
+ */
+function writeCypressSettings(hosts) {
+  fs.writeFileSync(
+    path.join(__dirname, "../apps/widget-e2e/cypress.env.json"),
+    JSON.stringify({
+      hosts
+    })
+  );
+}
+
+function e2e(watch) {
+  const nxPath = path.join(__dirname, "../node_modules/@nrwl/cli/bin/nx.js");
+  console.log("Running e2e");
+  const { status, stderr } = spawnSync(
+    "node",
+    [nxPath, "e2e", "widget-e2e", watch ? "--watch" : undefined],
+    {
+      stdio: "inherit",
+      cwd: path.join(__dirname, "..")
+    }
+  );
+  if (status) {
+    throw new Error(`Failed to run e2e:\r\n\r\n${stderr}`);
+  }
+  console.log("E2e finished\r\n");
+}
+
+async function main() {
+  commander
+    .option("-w, --watch", "Watch mode")
+    .parse(process.argv)
+    .outputHelp();
+  convertNotebook();
+  const hosts = await runNotebooks();
+  writeCypressSettings(hosts);
+  e2e(commander.opts().watch);
+  process.exit(0);
+}
+function onExit() {
+  console.log("Existing e2e");
+}
+async function onExitRequested() {
+  process.exit();
+}
+
+main();
+
+process.stdin.resume();
+process.on("SIGINT", onExitRequested);
+
+// catches "kill pid" (for example: nodemon restart)
+process.on("SIGUSR1", onExitRequested);
+process.on("SIGUSR2", onExitRequested);
+
+process.on("exit", onExit);

workspace.json

@@ -908,8 +908,7 @@
           "builder": "@nrwl/cypress:cypress",
           "options": {
             "cypressConfig": "apps/widget-e2e/cypress.json",
-            "tsConfig": "apps/widget-e2e/tsconfig.e2e.json",
-            "devServerTarget": "widget:serve"
+            "tsConfig": "apps/widget-e2e/tsconfig.e2e.json"
           },
           "configurations": {
             "production": {