2021-10-30 10:37:32 +03:00
|
|
|
const { spawnSync, spawn } = require("child_process");
|
|
|
|
const fs = require("fs");
|
|
|
|
const path = require("path");
|
|
|
|
const _ = require("lodash");
|
|
|
|
const commander = require("commander");
|
2022-06-24 07:51:07 +03:00
|
|
|
const { exit } = require("process");
|
2021-10-30 10:37:32 +03:00
|
|
|
|
2022-09-20 08:36:03 +03:00
|
|
|
const nxPath = path.join(__dirname, "../node_modules/@nrwl/cli/bin/nx.js");
|
2021-12-07 04:53:39 +03:00
|
|
|
const baseDir = path.join(__dirname, "../notebooks/responsibleaidashboard");
|
2022-01-07 23:24:24 +03:00
|
|
|
const filePrefix = "responsibleaidashboard-";
|
2022-02-10 02:00:48 +03:00
|
|
|
// Please add notebook name into 'fileNames' array only when you are adding e2e tests to that notebook.
|
2023-07-13 22:38:49 +03:00
|
|
|
// Keep this list in sync with .github/workflows/CI-e2e-notebooks.yml and/or .github/workflows/CI-e2e-notebooks-vision.yml
|
2022-01-07 23:24:24 +03:00
|
|
|
const fileNames = [
|
|
|
|
"responsibleaidashboard-census-classification-model-debugging",
|
2022-01-13 23:56:55 +03:00
|
|
|
"responsibleaidashboard-diabetes-regression-model-debugging",
|
2022-01-14 23:47:41 +03:00
|
|
|
"responsibleaidashboard-housing-classification-model-debugging",
|
2022-02-10 02:00:48 +03:00
|
|
|
"responsibleaidashboard-diabetes-decision-making",
|
|
|
|
"responsibleaidashboard-housing-decision-making",
|
2023-07-13 22:38:49 +03:00
|
|
|
"responsibleaidashboard-multiclass-dnn-model-debugging",
|
|
|
|
"responsibleaidashboard-fridge-object-detection-model-debugging"
|
2022-01-07 23:24:24 +03:00
|
|
|
];
|
2022-09-20 08:36:03 +03:00
|
|
|
const notebookHostReg = /^ResponsibleAI started at (http:\/\/localhost:\d+)$/m;
|
|
|
|
const serveHostReg = /Web Development Server is listening at\s+(.*)$/m;
|
2021-10-30 10:37:32 +03:00
|
|
|
const timeout = 3600;
|
|
|
|
|
2022-09-20 08:36:03 +03:00
|
|
|
/**
|
|
|
|
*
|
|
|
|
* @param {string} host
|
|
|
|
* @returns {Promise<string>}
|
|
|
|
*/
|
|
|
|
async function serve(host) {
|
|
|
|
console.log(`Running nx serve`);
|
|
|
|
const timer = setTimeout(() => {
|
|
|
|
throw new Error(`serve timeout.`);
|
|
|
|
}, timeout * 1000);
|
|
|
|
const nbProcess = spawn("node", [nxPath, "serve", "widget"], {
|
|
|
|
cwd: path.join(__dirname, ".."),
|
|
|
|
env: {
|
|
|
|
NX_based_url: host
|
|
|
|
}
|
|
|
|
});
|
|
|
|
nbProcess.on("exit", () => {
|
|
|
|
throw new Error(`Failed to run serve`);
|
|
|
|
});
|
|
|
|
return new Promise((resolve) => {
|
|
|
|
let stdout = "";
|
|
|
|
const handleOutput = (data) => {
|
|
|
|
const message = data.toString();
|
|
|
|
stdout += message;
|
|
|
|
console.log(message);
|
|
|
|
if (serveHostReg.test(stdout)) {
|
|
|
|
clearTimeout(timer);
|
|
|
|
resolve(serveHostReg.exec(stdout)[1]);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
nbProcess.stdout.on("data", handleOutput);
|
|
|
|
nbProcess.stderr.on("data", handleOutput);
|
|
|
|
nbProcess.stdout.on("error", (error) => {
|
|
|
|
throw error;
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2021-10-30 10:37:32 +03:00
|
|
|
/**
|
|
|
|
*
|
|
|
|
* @param {string} name
|
|
|
|
* @returns {Promise<string>}
|
|
|
|
*/
|
|
|
|
async function runNotebook(name) {
|
|
|
|
console.log(`Running ${name}`);
|
|
|
|
const timer = setTimeout(() => {
|
|
|
|
throw new Error(`${name} timeout.`);
|
|
|
|
}, timeout * 1000);
|
|
|
|
const nbProcess = spawn("python", ["-i", path.join(baseDir, name)]);
|
|
|
|
nbProcess.on("exit", () => {
|
|
|
|
throw new Error(`Failed to run notebook ${name}`);
|
|
|
|
});
|
|
|
|
return new Promise((resolve) => {
|
|
|
|
let stdout = "";
|
|
|
|
const handleOutput = (data) => {
|
|
|
|
const message = data.toString();
|
|
|
|
stdout += message;
|
|
|
|
console.log(message);
|
2022-09-20 08:36:03 +03:00
|
|
|
if (notebookHostReg.test(stdout)) {
|
2021-10-30 10:37:32 +03:00
|
|
|
clearTimeout(timer);
|
2022-09-20 08:36:03 +03:00
|
|
|
resolve(notebookHostReg.exec(stdout)[1]);
|
2021-10-30 10:37:32 +03:00
|
|
|
}
|
|
|
|
};
|
|
|
|
nbProcess.stdout.on("data", handleOutput);
|
|
|
|
nbProcess.stderr.on("data", handleOutput);
|
|
|
|
nbProcess.stdout.on("error", (error) => {
|
|
|
|
throw error;
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2022-06-24 07:51:07 +03:00
|
|
|
function addFlightsInFile(path, flights) {
|
|
|
|
if (!fs.existsSync(path)) {
|
|
|
|
throw new Error(`${path} does not exist.`);
|
|
|
|
}
|
|
|
|
let content = fs.readFileSync(path, { encoding: "utf-8" });
|
|
|
|
let startIndex = 0;
|
|
|
|
const dashboardConstructorCall = "ResponsibleAIDashboard(";
|
|
|
|
while (startIndex < content.length) {
|
|
|
|
startIndex = content.indexOf(dashboardConstructorCall, startIndex);
|
|
|
|
if (startIndex === -1) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
let dashboardArgsIndex = startIndex + dashboardConstructorCall.length;
|
|
|
|
let parenthesesBalance = 1;
|
|
|
|
while (parenthesesBalance > 0) {
|
|
|
|
if (content.at(dashboardArgsIndex) === "(") {
|
|
|
|
parenthesesBalance += 1;
|
|
|
|
} else if (content.at(dashboardArgsIndex) === ")") {
|
|
|
|
parenthesesBalance -= 1;
|
|
|
|
}
|
|
|
|
dashboardArgsIndex += 1;
|
|
|
|
}
|
|
|
|
content =
|
|
|
|
content.slice(0, dashboardArgsIndex - 1) +
|
|
|
|
`, feature_flights="${flights.split(",").join("&")}")` +
|
|
|
|
content.slice(dashboardArgsIndex);
|
|
|
|
startIndex = dashboardArgsIndex + 1;
|
|
|
|
}
|
|
|
|
console.log(`writing notebook with flights to ${path}`);
|
|
|
|
fs.writeFileSync(path, content, { encoding: "utf-8" });
|
|
|
|
}
|
|
|
|
|
2022-02-10 02:00:48 +03:00
|
|
|
function checkIfAllNotebooksHaveTests() {
|
2022-06-16 04:04:15 +03:00
|
|
|
console.log(`Checking if all notebooks under ${baseDir} have tests`);
|
2022-02-10 02:00:48 +03:00
|
|
|
const files = fs
|
|
|
|
.readdirSync(baseDir)
|
|
|
|
.filter((f) => f.startsWith(filePrefix) && f.endsWith(".ipynb"))
|
|
|
|
.map((f) => f.replace(".ipynb", ""));
|
|
|
|
const allNotebooksHaveTests = _.isEqual(_.sortBy(files), _.sortBy(fileNames));
|
|
|
|
if (!allNotebooksHaveTests) {
|
|
|
|
throw new Error(
|
2022-06-16 04:04:15 +03:00
|
|
|
`Some of the notebooks don't have tests. If a new notebook is added, Please add tests.`
|
2022-02-10 02:00:48 +03:00
|
|
|
);
|
|
|
|
}
|
|
|
|
console.log(`All notebooks have tests.`);
|
|
|
|
}
|
|
|
|
|
2022-06-24 07:51:07 +03:00
|
|
|
function convertNotebooks(notebook, flights) {
|
2022-06-16 04:04:15 +03:00
|
|
|
console.log("Converting notebooks");
|
2022-01-07 23:24:24 +03:00
|
|
|
for (var fileName of fileNames) {
|
2022-06-16 04:04:15 +03:00
|
|
|
if (notebook && fileName !== notebook) {
|
|
|
|
console.log(`Skipping ${fileName}. Looking for ${notebook} only.`);
|
|
|
|
continue;
|
|
|
|
}
|
2022-06-24 07:51:07 +03:00
|
|
|
if (flights) {
|
|
|
|
// flights were passed (not just -f without flights arg)
|
|
|
|
console.log(
|
|
|
|
`Converting notebook ${fileName} with flights ${flights.toString()}\r\n`
|
|
|
|
);
|
|
|
|
} else {
|
|
|
|
// no flights were passed
|
|
|
|
console.log(`Converting notebook ${fileName} with no flights.\r\n`);
|
|
|
|
}
|
2022-01-07 23:24:24 +03:00
|
|
|
const { status, stderr } = spawnSync(
|
|
|
|
"jupyter",
|
|
|
|
["nbconvert", path.join(baseDir, `${fileName}.ipynb`), "--to", "script"],
|
|
|
|
{
|
|
|
|
stdio: "inherit"
|
|
|
|
}
|
|
|
|
);
|
|
|
|
if (status) {
|
|
|
|
throw new Error(`Failed to convert notebook:\r\n\r\n${stderr}`);
|
2021-10-30 10:37:32 +03:00
|
|
|
}
|
2022-06-24 07:51:07 +03:00
|
|
|
if (flights) {
|
|
|
|
addFlightsInFile(path.join(baseDir, `${fileName}.py`), flights);
|
|
|
|
}
|
2022-01-07 23:24:24 +03:00
|
|
|
console.log(`Converted notebook ${fileName}\r\n`);
|
2021-10-30 10:37:32 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
/**
|
|
|
|
* @typedef {Object} Host
|
|
|
|
* @property {string} file
|
|
|
|
* @property {string} host
|
|
|
|
*/
|
|
|
|
/**
|
|
|
|
* @returns {Host[]}
|
|
|
|
*/
|
2022-09-20 08:36:03 +03:00
|
|
|
async function runNotebooks(selectedNotebook, host) {
|
2022-06-16 04:04:15 +03:00
|
|
|
let files = fs
|
2021-10-30 10:37:32 +03:00
|
|
|
.readdirSync(baseDir)
|
|
|
|
.filter((f) => f.startsWith(filePrefix) && f.endsWith(".py"));
|
2022-06-16 04:04:15 +03:00
|
|
|
console.log("Available notebooks:");
|
|
|
|
files.forEach((file) => {
|
|
|
|
console.log(` ${file}`);
|
|
|
|
});
|
|
|
|
if (selectedNotebook) {
|
|
|
|
const nbFileName = `${selectedNotebook}.py`;
|
2022-09-23 00:38:57 +03:00
|
|
|
if (host) {
|
|
|
|
files = [nbFileName];
|
|
|
|
} else {
|
|
|
|
console.log(`Should only run ${nbFileName}`);
|
|
|
|
files = files.filter((f) => f === nbFileName);
|
|
|
|
if (files.length === 0) {
|
|
|
|
console.log(`Could not find any matching notebook for ${nbFileName}.`);
|
|
|
|
exit(1);
|
|
|
|
}
|
2022-06-16 04:04:15 +03:00
|
|
|
}
|
|
|
|
}
|
2021-10-30 10:37:32 +03:00
|
|
|
const hosts = [];
|
|
|
|
for (const f of files) {
|
2022-09-20 08:36:03 +03:00
|
|
|
host = host || (await runNotebook(f));
|
2022-06-16 04:04:15 +03:00
|
|
|
hosts.push({ file: f, host: host });
|
|
|
|
console.log(`file: ${f}, host: ${host}`);
|
2021-10-30 10:37:32 +03:00
|
|
|
}
|
|
|
|
return hosts;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
*
|
|
|
|
* @param {Host[]} hosts
|
|
|
|
*/
|
|
|
|
function writeCypressSettings(hosts) {
|
|
|
|
fs.writeFileSync(
|
|
|
|
path.join(__dirname, "../apps/widget-e2e/cypress.env.json"),
|
|
|
|
JSON.stringify({
|
|
|
|
hosts
|
|
|
|
})
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2022-09-20 08:36:03 +03:00
|
|
|
function e2e(watch, selectedNotebook, flights, host) {
|
2022-07-22 20:39:37 +03:00
|
|
|
console.log(`Running e2e for notebook ${selectedNotebook}`);
|
2022-06-16 04:04:15 +03:00
|
|
|
let notebookArgs = [];
|
|
|
|
if (selectedNotebook) {
|
|
|
|
// remove prefix "responsibleaidashboard"
|
|
|
|
// remove dashes and make camel case
|
|
|
|
let notebookKey = selectedNotebook.substring(
|
|
|
|
"responsibleaidashboard".length,
|
|
|
|
selectedNotebook.length
|
|
|
|
);
|
|
|
|
notebookKey = notebookKey
|
|
|
|
.split("-")
|
|
|
|
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
|
|
|
.join("");
|
2022-06-24 07:51:07 +03:00
|
|
|
if (flights) {
|
|
|
|
// append flights (if any) with first letter capitalized
|
|
|
|
notebookKey =
|
|
|
|
notebookKey +
|
|
|
|
flights
|
|
|
|
.split(",")
|
|
|
|
.map((flight) => flight.charAt(0).toUpperCase() + flight.slice(1))
|
|
|
|
.join("");
|
|
|
|
}
|
2022-06-16 04:04:15 +03:00
|
|
|
console.log(
|
|
|
|
`Determined notebook key ${notebookKey} for notebook ${selectedNotebook}.`
|
|
|
|
);
|
|
|
|
notebookArgs = ["--spec", `**/responsibleaitoolbox${notebookKey}/**`];
|
|
|
|
}
|
2021-10-30 10:37:32 +03:00
|
|
|
const { status, stderr } = spawnSync(
|
|
|
|
"node",
|
2022-06-16 04:04:15 +03:00
|
|
|
[
|
|
|
|
nxPath,
|
|
|
|
"e2e",
|
|
|
|
"widget-e2e",
|
|
|
|
...notebookArgs,
|
|
|
|
watch ? "--watch" : undefined
|
|
|
|
],
|
2021-10-30 10:37:32 +03:00
|
|
|
{
|
|
|
|
stdio: "inherit",
|
|
|
|
cwd: path.join(__dirname, "..")
|
|
|
|
}
|
|
|
|
);
|
|
|
|
if (status) {
|
|
|
|
throw new Error(`Failed to run e2e:\r\n\r\n${stderr}`);
|
|
|
|
}
|
2022-06-16 04:04:15 +03:00
|
|
|
console.log("e2e finished\r\n");
|
2021-10-30 10:37:32 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async function main() {
|
|
|
|
commander
|
|
|
|
.option("-w, --watch", "Watch mode")
|
2022-06-16 04:04:15 +03:00
|
|
|
.option("--skipgen", "Skip notebook generation")
|
2022-09-20 08:36:03 +03:00
|
|
|
.option(
|
|
|
|
"--host [host]",
|
2023-02-12 08:14:10 +03:00
|
|
|
"Skip notebook running and use host provided to run e2e: use full url 'http://localhost:5000' or port number"
|
2022-09-20 08:36:03 +03:00
|
|
|
)
|
2022-06-16 04:04:15 +03:00
|
|
|
.option("-n, --notebook [notebook]", "Run specific notebook")
|
2022-06-24 07:51:07 +03:00
|
|
|
.option(
|
|
|
|
"-f, --flights [flights]",
|
|
|
|
"Use flights separated by comma (no whitespace). Not specifying flights means that no flights are used."
|
|
|
|
)
|
2021-10-30 10:37:32 +03:00
|
|
|
.parse(process.argv)
|
|
|
|
.outputHelp();
|
2022-06-24 07:51:07 +03:00
|
|
|
const skipgen = commander.opts().skipgen;
|
2022-09-20 08:36:03 +03:00
|
|
|
const watch = commander.opts().watch;
|
|
|
|
let host = commander.opts().host;
|
|
|
|
if (host && !isNaN(parseInt(host))) {
|
|
|
|
host = `http://localhost:${host}`;
|
|
|
|
}
|
2022-06-24 07:51:07 +03:00
|
|
|
const notebook = commander.opts().notebook;
|
2022-09-20 08:36:03 +03:00
|
|
|
if (host && !notebook) {
|
|
|
|
throw new Error("Notebook is required when host is specified.");
|
|
|
|
}
|
2022-06-24 07:51:07 +03:00
|
|
|
let flights = commander.opts().flights;
|
2022-07-22 20:39:37 +03:00
|
|
|
console.log("Checking flights: " + flights);
|
2022-07-01 09:00:16 +03:00
|
|
|
if (flights?.toString() === "true") {
|
2022-06-24 07:51:07 +03:00
|
|
|
// -f passed without arguments
|
|
|
|
flights = undefined;
|
2022-07-22 20:39:37 +03:00
|
|
|
console.log("setting flights to undefined!!!");
|
2022-06-24 07:51:07 +03:00
|
|
|
}
|
2022-02-10 02:00:48 +03:00
|
|
|
checkIfAllNotebooksHaveTests();
|
2022-09-20 08:36:03 +03:00
|
|
|
if (!skipgen && !host) {
|
2022-06-24 07:51:07 +03:00
|
|
|
convertNotebooks(notebook, flights);
|
2022-09-20 08:36:03 +03:00
|
|
|
} else {
|
|
|
|
console.log("Skipping converting notebooks", skipgen, host);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (host && watch) {
|
|
|
|
host = await serve(host);
|
2022-06-16 04:04:15 +03:00
|
|
|
}
|
2022-09-20 08:36:03 +03:00
|
|
|
console.log(`Running watch mode on ${host}`);
|
|
|
|
|
|
|
|
const hosts = await runNotebooks(notebook, host);
|
2021-10-30 10:37:32 +03:00
|
|
|
writeCypressSettings(hosts);
|
2022-07-22 20:39:37 +03:00
|
|
|
for (var fileName of fileNames) {
|
|
|
|
if (notebook && fileName !== notebook) {
|
|
|
|
console.log(
|
|
|
|
`Skipping e2e for ${fileName}. Looking for ${notebook} only.`
|
|
|
|
);
|
|
|
|
continue;
|
|
|
|
}
|
2022-09-20 08:36:03 +03:00
|
|
|
e2e(watch, fileName, flights, host);
|
2022-07-22 20:39:37 +03:00
|
|
|
}
|
2021-10-30 10:37:32 +03:00
|
|
|
process.exit(0);
|
|
|
|
}
|
2022-01-07 23:24:24 +03:00
|
|
|
|
2021-10-30 10:37:32 +03:00
|
|
|
function onExit() {
|
|
|
|
console.log("Existing e2e");
|
|
|
|
}
|
|
|
|
async function onExitRequested() {
|
|
|
|
process.exit();
|
|
|
|
}
|
|
|
|
|
|
|
|
main();
|
|
|
|
|
|
|
|
process.stdin.resume();
|
|
|
|
process.on("SIGINT", onExitRequested);
|
|
|
|
|
|
|
|
// catches "kill pid" (for example: nodemon restart)
|
|
|
|
process.on("SIGUSR1", onExitRequested);
|
|
|
|
process.on("SIGUSR2", onExitRequested);
|
|
|
|
|
|
|
|
process.on("exit", onExit);
|