Bump deps version and migrate to npm (#5443)

dependencies/required.txt

@@ -19,6 +19,6 @@ scikit-learn >= 0.24.1
 scipy < 1.8 ; python_version < "3.8"
 scipy ; python_version >= "3.8"
 tqdm
-typeguard
+typeguard >= 3.0.0
 typing_extensions >= 4.0.0
 websockets >= 10.1

docs/source/conf.py

@@ -116,6 +116,8 @@ linkcheck_ignore = [
     r'https://docs\.nvidia\.com/deeplearning/',
     r'https://cla\.opensource\.microsoft\.com',
     r'https://www\.docker\.com/',
+
+    r'https://pytorch-lightning\.readthedocs\.io/en/stable/guides/data\.html'  # FIXME
 ]
 
 # Ignore all links located in release.rst

nni/experiment/config/utils/internal.py

@@ -89,8 +89,8 @@ def fields(config: ConfigBase) -> list[dataclasses.Field]:
 
 def is_instance(value, type_hint) -> bool:
     try:
-        typeguard.check_type('_', value, type_hint)
+        typeguard.check_type(value, type_hint)
-    except TypeError:
+    except typeguard.TypeCheckError:
         return False
     return True
 

pipelines/fast-test.yml

@@ -110,8 +110,8 @@ stages:
 
     - script: |
         set -e
-        yarn --cwd ts/webui
+        npm --prefix ts/webui install
-        yarn --cwd ts/webui sanity-check
+        npm --prefix ts/webui run sanity-check
       displayName: Sanity check (WebUI)
 
 - stage: filter

pipelines/full-test-hpo.yml

@@ -43,7 +43,6 @@ stages:
     - script: |
         set -e
         (cd test && python -m pytest ut)
-        export PATH=$PATH:$PWD/toolchain/yarn/bin
         export CI=true
         (cd ts/nni_manager && npm run test)
       displayName: SDK unit test

pipelines/templates/cache-dependencies-template.yml

@@ -82,7 +82,8 @@ steps:
   condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
 
 - script: |
-    yarn --cwd ts/webui install
+    cd ts/webui
+    npm install
   displayName: Install Web UI dependencies
   condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'legacy')))
 

setup.py

@@ -69,25 +69,25 @@ import setup_ts
 
 release = os.environ.get('NNI_RELEASE')
 
-def _get_jupyter_lab_version():
+#def _get_jupyter_lab_version():
-    try:
+#    try:
-        import jupyterlab
+#        import jupyterlab
-        return jupyterlab.__version__
+#        return jupyterlab.__version__
-    except ImportError:
+#    except ImportError:
-        return '3.x'
+#        return '3.x'
 
-jupyter_lab_major_version = _get_jupyter_lab_version().split('.')[0]
+#jupyter_lab_major_version = _get_jupyter_lab_version().split('.')[0]
 
-def check_jupyter_lab_version():
+#def check_jupyter_lab_version():
-    environ_version = os.environ.get('JUPYTER_LAB_VERSION')
+#    environ_version = os.environ.get('JUPYTER_LAB_VERSION')
-
+#
-    jupyter_lab_version = _get_jupyter_lab_version()
+#    jupyter_lab_version = _get_jupyter_lab_version()
-
+#
-    if environ_version:
+#    if environ_version:
-        if jupyter_lab_version.split('.')[0] != environ_version.split('.')[0]:
+#        if jupyter_lab_version.split('.')[0] != environ_version.split('.')[0]:
-            sys.exit(f'ERROR: To build a jupyter lab extension, run "JUPYTER_LAB_VERSION={jupyter_lab_version}", current: {environ_version} ')
+#            sys.exit(f'ERROR: To build a jupyter lab extension, run "JUPYTER_LAB_VERSION={jupyter_lab_version}", current: {environ_version} ')
-    elif jupyter_lab_version.split('.')[0] != '3':
+#    elif jupyter_lab_version.split('.')[0] != '3':
-        sys.exit(f'ERROR: To build a jupyter lab extension, run "JUPYTER_LAB_VERSION={jupyter_lab_version}" first for nondefault version(3.x)')
+#        sys.exit(f'ERROR: To build a jupyter lab extension, run "JUPYTER_LAB_VERSION={jupyter_lab_version}" first for nondefault version(3.x)')
 
 def _setup():
     setuptools.setup(
@@ -146,9 +146,9 @@ def _setup():
 
 def _get_data_files():
     data_files = []
-    if jupyter_lab_major_version == '2':
+#    if jupyter_lab_major_version == '2':
-        extension_file = glob.glob("nni_node/jupyter-extension/extensions/nni-jupyter-extension*.tgz")
+#        extension_file = glob.glob("nni_node/jupyter-extension/extensions/nni-jupyter-extension*.tgz")
-        data_files = [('share/jupyter/lab/extensions', extension_file)]
+#        data_files = [('share/jupyter/lab/extensions', extension_file)]
     return data_files
 
 def _find_python_packages():
@@ -221,7 +221,7 @@ class BuildTs(Command):
         pass
 
     def run(self):
-        check_jupyter_lab_version()
+        #check_jupyter_lab_version()
         setup_ts.build(release)
 
 class Build(build):
@@ -229,7 +229,7 @@ class Build(build):
         if not release:
             sys.exit('Please set environment variable "NNI_RELEASE=<release_version>"')
 
-        check_jupyter_lab_version()
+        #check_jupyter_lab_version()
 
         if os.path.islink('nni_node/main.js'):
             sys.exit('A development build already exists. Please uninstall NNI and run "python3 setup.py clean".')

setup_ts.py

@@ -8,7 +8,7 @@ This script is called by `setup.py` and common users should avoid using this dir
 It compiles TypeScript source files in `ts` directory,
 and copies (or links) JavaScript output as well as dependencies to `nni_node`.
 
-You can set environment `GLOBAL_TOOLCHAIN=1` to use global node and yarn, if you know what you are doing.
+You can set environment `GLOBAL_TOOLCHAIN=1` to use global node and npm, if you know what you are doing.
 """
 
 from io import BytesIO
@@ -25,7 +25,6 @@ from zipfile import ZipFile
 
 
 node_version = 'v18.15.0'
-yarn_version = 'v1.22.19'
 
 def _print(*args, color='cyan'):
     color_code = {'yellow': 33, 'cyan': 36}[color]
@@ -34,12 +33,12 @@ def _print(*args, color='cyan'):
     else:
         print(f'\033[1;{color_code}m#', *args, '\033[0m', flush=True)
 
-def _get_jupyter_lab_version():
+#def _get_jupyter_lab_version():
-    try:
+#    try:
-        import jupyterlab
+#        import jupyterlab
-        return jupyterlab.__version__
+#        return jupyterlab.__version__
-    except ImportError:
+#    except ImportError:
-        return '3.x'
+#        return '3.x'
 
 def _get_glibc_minor_version():  # type: () -> int | None
     try:
@@ -86,7 +85,7 @@ def _get_node_downloader():
         node_extractor = lambda data: tarfile.open(fileobj=BytesIO(data), mode='r:xz')
     return node_download_url, node_spec, node_extractor
 
-jupyter_lab_major_version = _get_jupyter_lab_version().split('.')[0]
+#jupyter_lab_major_version = _get_jupyter_lab_version().split('.')[0]
 
 def build(release):
     """
@@ -101,13 +100,13 @@ def build(release):
     if release or not os.environ.get('GLOBAL_TOOLCHAIN'):
         download_toolchain()
     prepare_nni_node()
-    update_package()
+    #update_package()
     compile_ts(release)
     if release or sys.platform == 'win32':
         copy_nni_node(release)
     else:
         symlink_nni_node()
-    restore_package()
+    #restore_package()
 
 def clean():
     """
@@ -132,9 +131,6 @@ if sys.platform == 'linux' or sys.platform == 'darwin':
 
     npm_executable = 'bin/npm'
 
-    yarn_executable = 'yarn'
-    yarn_download_url = f'https://github.com/yarnpkg/yarn/releases/download/{yarn_version}/yarn-{yarn_version}.tar.gz'
-
     path_env_separator = ':'
 
 elif sys.platform == 'win32':
@@ -146,9 +142,6 @@ elif sys.platform == 'win32':
 
     npm_executable = 'npm.cmd'
 
-    yarn_executable = 'yarn.cmd'
-    yarn_download_url = f'https://github.com/yarnpkg/yarn/releases/download/{yarn_version}/yarn-{yarn_version}.tar.gz'
-
     path_env_separator = ';'
 
 else:
@@ -157,7 +150,7 @@ else:
 
 def download_toolchain():
     """
-    Download and extract node and yarn.
+    Download and extract node.
     """
     if Path('toolchain/node', node_executable_in_tarball).is_file():
         return
@@ -174,34 +167,25 @@ def download_toolchain():
     shutil.rmtree('toolchain/node', ignore_errors=True)
     Path('toolchain', node_spec).rename('toolchain/node')
 
-    _print(f'Downloading yarn from {yarn_download_url}')
+#def update_package():
-    resp = requests.get(yarn_download_url)
+#    if jupyter_lab_major_version == '2':
-    resp.raise_for_status()
+#        package_json = json.load(open('ts/jupyter_extension/package.json'))
-    _print('Extracting yarn')
+#        json.dump(package_json, open('ts/jupyter_extension/.package_default.json', 'w'), indent=2)
-    tarball = tarfile.open(fileobj=BytesIO(resp.content), mode='r:gz')
+#
-    tarball.extractall('toolchain')
+#        package_json['scripts']['build'] = 'tsc && jupyter labextension link .'
-    shutil.rmtree('toolchain/yarn', ignore_errors=True)
+#        package_json['dependencies']['@jupyterlab/application'] = '^2.3.0'
-    Path(f'toolchain/yarn-{yarn_version}').rename('toolchain/yarn')
+#        package_json['dependencies']['@jupyterlab/launcher'] = '^2.3.0'
+#
+#        package_json['jupyterlab']['outputDir'] = 'build'
+#        json.dump(package_json, open('ts/jupyter_extension/package.json', 'w'), indent=2)
+#        print(f'updated package.json with {json.dumps(package_json, indent=2)}')
 
-def update_package():
+#def restore_package():
-    if jupyter_lab_major_version == '2':
+#    if jupyter_lab_major_version == '2':
-        package_json = json.load(open('ts/jupyter_extension/package.json'))
+#        package_json = json.load(open('ts/jupyter_extension/.package_default.json'))
-        json.dump(package_json, open('ts/jupyter_extension/.package_default.json', 'w'), indent=2)
+#        print(f'stored package.json with {json.dumps(package_json, indent=2)}')
-
+#        json.dump(package_json, open('ts/jupyter_extension/package.json', 'w'), indent=2)
-        package_json['scripts']['build'] = 'tsc && jupyter labextension link .'
+#        os.remove('ts/jupyter_extension/.package_default.json')
-        package_json['dependencies']['@jupyterlab/application'] = '^2.3.0'
-        package_json['dependencies']['@jupyterlab/launcher'] = '^2.3.0'
-
-        package_json['jupyterlab']['outputDir'] = 'build'
-        json.dump(package_json, open('ts/jupyter_extension/package.json', 'w'), indent=2)
-        print(f'updated package.json with {json.dumps(package_json, indent=2)}')
-
-def restore_package():
-    if jupyter_lab_major_version == '2':
-        package_json = json.load(open('ts/jupyter_extension/.package_default.json'))
-        print(f'stored package.json with {json.dumps(package_json, indent=2)}')
-        json.dump(package_json, open('ts/jupyter_extension/package.json', 'w'), indent=2)
-        os.remove('ts/jupyter_extension/.package_default.json')
 
 def prepare_nni_node():
     """
@@ -219,7 +203,7 @@ def prepare_nni_node():
 
 def compile_ts(release):
     """
-    Use yarn to download dependencies and compile TypeScript code.
+    Use npm to download dependencies and compile TypeScript code.
     """
     _print('Building NNI manager')
     _npm('ts/nni_manager', 'install')
@@ -229,21 +213,21 @@ def compile_ts(release):
     shutil.copytree('ts/nni_manager/config', 'ts/nni_manager/dist/config')
 
     _print('Building web UI')
-    _yarn('ts/webui')
+    _npm('ts/webui', 'install')
     if release:
-        _yarn('ts/webui', 'release')
+        _npm('ts/webui', 'run', 'release')
     else:
-        _yarn('ts/webui', 'build')
+        _npm('ts/webui', 'run', 'build')
 
-    _print('Building JupyterLab extension')
+    #_print('Building JupyterLab extension')
-    try:
+    #try:
-        _yarn('ts/jupyter_extension')
+    #    _yarn('ts/jupyter_extension')
-        _yarn('ts/jupyter_extension', 'build')
+    #    _yarn('ts/jupyter_extension', 'build')
-    except Exception:
+    #except Exception:
-        if release:
+    #    if release:
-            raise
+    #        raise
-        _print('Failed to build JupyterLab extension, skip for develop mode', color='yellow')
+    #    _print('Failed to build JupyterLab extension, skip for develop mode', color='yellow')
-        _print(traceback.format_exc(), color='yellow')
+    #    _print(traceback.format_exc(), color='yellow')
 
 
 def symlink_nni_node():
@@ -260,11 +244,11 @@ def symlink_nni_node():
 
     _symlink('ts/webui/build', 'nni_node/static')
 
-    if jupyter_lab_major_version == '2':
+    #if jupyter_lab_major_version == '2':
-        _symlink('ts/jupyter_extension/build', 'nni_node/jupyter-extension')
+    #    _symlink('ts/jupyter_extension/build', 'nni_node/jupyter-extension')
-        _symlink(os.path.join(sys.exec_prefix, 'share/jupyter/lab/extensions'), 'nni_node/jupyter-extension/extensions')
+    #    _symlink(os.path.join(sys.exec_prefix, 'share/jupyter/lab/extensions'), 'nni_node/jupyter-extension/extensions')
-    elif Path('ts/jupyter_extension/dist').exists():
+    #elif Path('ts/jupyter_extension/dist').exists():
-        _symlink('ts/jupyter_extension/dist', 'nni_node/jupyter-extension')
+    #    _symlink('ts/jupyter_extension/dist', 'nni_node/jupyter-extension')
 
 
 def copy_nni_node(version):
@@ -302,36 +286,28 @@ def copy_nni_node(version):
 
     # reinstall without development dependencies
     prod_path = Path('nni_node').resolve()
-    _yarn(str(prod_path), 'install', '--production')
+    _npm(str(prod_path), 'install', '--omit', 'dev')
 
     shutil.copytree('ts/webui/build', 'nni_node/static')
 
-    if jupyter_lab_major_version == '2':
+    #if jupyter_lab_major_version == '2':
-        shutil.copytree('ts/jupyter_extension/build', 'nni_node/jupyter-extension/build')
+    #    shutil.copytree('ts/jupyter_extension/build', 'nni_node/jupyter-extension/build')
-        shutil.copytree(os.path.join(sys.exec_prefix, 'share/jupyter/lab/extensions'), 'nni_node/jupyter-extension/extensions')
+    #    shutil.copytree(os.path.join(sys.exec_prefix, 'share/jupyter/lab/extensions'), 'nni_node/jupyter-extension/extensions')
-    elif version or Path('ts/jupyter_extension/dist').exists():
+    #elif version or Path('ts/jupyter_extension/dist').exists():
-        shutil.copytree('ts/jupyter_extension/dist', 'nni_node/jupyter-extension')
+    #    shutil.copytree('ts/jupyter_extension/dist', 'nni_node/jupyter-extension')
 
 
-_yarn_env = dict(os.environ)
+_npm_env = dict(os.environ)
 # `Path('nni_node').resolve()` does not work on Windows if the directory not exists
-_yarn_env['PATH'] = str(Path().resolve() / 'nni_node') + path_env_separator + os.environ['PATH']
+_npm_env['PATH'] = str(Path().resolve() / 'nni_node') + path_env_separator + os.environ['PATH']
-_yarn_path = Path().resolve() / 'toolchain/yarn/bin' / yarn_executable
 _npm_path = Path().resolve() / 'toolchain/node' / npm_executable
 
-def _yarn(path, *args):
-    _print('yarn ' + ' '.join(args) + f' (path: {path})')
-    if os.environ.get('GLOBAL_TOOLCHAIN'):
-        subprocess.run(['yarn', *args], cwd=path, check=True)
-    else:
-        subprocess.run([str(_yarn_path), *args], cwd=path, check=True, env=_yarn_env)
-
 def _npm(path, *args):
     _print('npm ' + ' '.join(args) + f' (path: {path})')
     if os.environ.get('GLOBAL_TOOLCHAIN'):
         subprocess.run(['npm', *args], cwd=path, check=True)
     else:
-        subprocess.run([str(_npm_path), *args], cwd=path, check=True, env=_yarn_env)
+        subprocess.run([str(_npm_path), *args], cwd=path, check=True, env=_npm_env)
 
 
 def _symlink(target_file, link_location):

ts/nni_manager/common/trial_keeper/main.ts

@@ -100,7 +100,7 @@ async function main(): Promise<void> {
     }, 1000);
 }
 
-if (!process.argv[1].endsWith('mocha')) {  // the unit test imports all scripts and will reach here
+if (!process.argv[1].includes('mocha')) {  // the unit test imports all scripts and will reach here
     main().catch(error => {
         logger.critical(error);
         console.error(util.inspect(error));

ts/nni_manager/package.json

@@ -16,26 +16,26 @@
     "express": "^4.18.2",
     "express-joi-validator": "^2.0.1",
     "express-ws": "^5.0.2",
-    "glob": "^8.0.3",
+    "glob": "^8.1.0",
     "http-proxy": "^1.18.1",
-    "ignore": "^5.1.8",
+    "ignore": "^5.2.4",
-    "js-base64": "^3.7.2",
+    "js-base64": "^3.7.5",
     "js-yaml": "^4.1.0",
     "kubernetes-client": "^6.12.1",
     "lockfile": "^1.0.4",
     "python-shell": "^3.0.0",
     "rx": "^4.1.0",
-    "sqlite3": "^5.1.2",
+    "sqlite3": "^5.1.6",
-    "ssh2": "^1.4.0",
+    "ssh2": "^1.11.0",
     "stream-buffers": "^3.0.2",
     "tail-stream": "^0.3.4",
-    "tar": "^6.1.12",
+    "tar": "^6.1.13",
     "tree-kill": "^1.2.2",
     "ts-deferred": "^1.0.4",
     "typescript-ioc": "^1.2.6",
     "typescript-string-operations": "^1.4.1",
-    "ws": "^8.10.0",
+    "ws": "^8.13.0",
-    "yargs": "^17.6.2"
+    "yargs": "^17.7.1"
   },
   "devDependencies": {
     "@types/app-module-path": ">=2.2.0",
@@ -65,25 +65,19 @@
     "eslint": ">=8.26.0",
     "mocha": ">=10.1.0",
     "node-gyp": ">=9.3.0",
-    "npm": ">=8.11.0",
+    "npm": ">=9.6.1",
     "nyc": ">=15.1.0",
     "request": ">=2.88.2",
     "tmp": ">=0.2.1",
     "ts-node": ">=10.9.1",
     "typescript": ">=4.8.4"
   },
-  "resolutions": {
+  "overrides": {
-    "acorn": ">=8.3.0",
+    "cacheable-request": ">=10.2.8",
+    "got": ">=12.6.0",
     "hoek": ">=6.1.3",
-    "node.extend": ">=1.1.8",
+    "node-jose": ">=2.2.0",
-    "y18n": ">=5.0.8",
+    "require-glob": ">=4.0.1"
-    "yargs-parser": ">=20.2.7",
-    "joi": ">=17.4.0",
-    "node-forge": ">=0.10.0",
-    "glob-parent": ">=6.0.0",
-    "node-gyp": ">=8.4.1",
-    "strip-ansi": "=6.0.1",
-    "http-signature": ">=1.3.6"
   },
   "engines": {
     "node": ">=18.15.0"
@@ -110,4 +104,4 @@
     "sourceMap": true,
     "instrument": true
   }
-}
+}

ts/nni_manager/test/common/command_channel/websocket.test.ts

@@ -1,161 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT license.
-
-import assert from 'assert/strict';
-import { setTimeout } from 'timers/promises';
-
-import WebSocket from 'ws';
-
-import { WebSocketChannelServer, UnitTestHelpers } from 'common/command_channel/websocket';
-import { DefaultMap } from 'common/default_map';
-import { Deferred } from 'common/deferred';
-import globals from 'common/globals/unittest';
-import type { Command } from 'common/command_channel/interface';
-import { RestServer, UnitTestHelpers as RestServerHelpers } from 'rest_server';
-
-describe('## websocket command channel ##', () => {
-    before(beforeHook);
-
-    it('start', () => testStart());
-
-    /* send and receive messages on normal connections */
-    it('connect', () => testConnect());
-    it('message', () => testMessage());
-
-    /* client 1 proactively reconnect */
-    it('reconnect', () => testReconnect());
-    it('message', () => testMessage());
-
-    /* client 2 loses responsive */
-    it('no response', () => testNoResponse());
-    it('message', () => testMessage());
-
-    it('shutdown', () => testShutdown());
-
-    after(afterHook);
-});
-
-/* global states */
-
-const heartbeatInterval: number = 10;  // NOTE: increase this if the pipeline fails randomly
-UnitTestHelpers.setHeartbeatInterval(heartbeatInterval);
-
-let server!: WebSocketChannelServer;
-let client1!: Client;
-let client2!: Client;
-const serverReceivedCommands: DefaultMap<string, Command[]> = new DefaultMap(Array) as any;
-
-let restServer!: RestServer;
-
-/* test cases */
-
-async function testStart(): Promise<void> {
-    server = new WebSocketChannelServer('ut', 'ut', 2);
-    server.onReceive((channelId, command) => {
-        serverReceivedCommands.get(channelId).push(unpackCommand(command));
-    });
-    await server.start();
-    assert.equal(server.getChannelUrl('1'), `ws://localhost:${globals.args.port}/ut/1`);
-}
-
-async function testConnect(): Promise<void> {
-    client1 = new Client(server.getChannelUrl('1'));
-    client2 = new Client(server.getChannelUrl('2'));
-    await Promise.all([ client1.opened.promise, client2.opened.promise ]);
-}
-
-async function testReconnect(): Promise<void> {
-    const oldClient = client1;
-    client1 = new Client(server.getChannelUrl('1'));
-    await Promise.all([ oldClient.closed.promise, client1.opened.promise ]);
-}
-
-async function testNoResponse(): Promise<void> {
-    await client2.mockNoResponse(heartbeatInterval * 5);
-    client2 = new Client(server.getChannelUrl('2'));
-    await client2.opened.promise;
-}
-
-async function testMessage(): Promise<void> {
-    serverReceivedCommands.forEach(commands => { commands.length = 0; });
-    client1.received.length = 0;
-    client2.received.length = 0;
-
-    server.send('1', packCommand(1));       // server -> 1
-    client2.send(packCommand('二'));        // server <- 2
-    client2.send(packCommand(3));           // server <- 2
-    server.send('2', packCommand('四'));    // server -> 2
-    client1.send(packCommand(5));           // server <- 1
-    server.send('1', packCommand(6));       // server -> 1
-
-    await setTimeout(heartbeatInterval);
-
-    assert.deepEqual(client1.received, [ 1, 6 ]);
-    assert.deepEqual(client2.received, [ '四' ]);
-    assert.deepEqual(serverReceivedCommands.get('1'), [ 5 ]);
-    assert.deepEqual(serverReceivedCommands.get('2'), [ '二', 3 ]);
-}
-
-async function testShutdown(): Promise<void> {
-    await server.shutdown();
-    await Promise.all([ client1.closed.promise, client2.closed.promise ]);
-}
-
-/* helpers */
-
-async function beforeHook(): Promise<void> {
-    globals.reset();
-    //globals.showLog();
-    restServer = new RestServer(0, '');
-    await restServer.start();
-    globals.args.port = RestServerHelpers.getPort(restServer);
-}
-
-async function afterHook() {
-    if (restServer) {
-        await restServer.shutdown();
-    }
-    globals.reset();
-    RestServerHelpers.reset();
-}
-
-function packCommand(value: any): Command {
-    return { type: 'ut', value } as Command;
-}
-
-function unpackCommand(command: Command): any {
-    assert.equal(command.type, 'ut');
-    return (command as any).value;
-}
-
-class Client {
-    ws: WebSocket;
-    received: any[] = [];
-    opened: Deferred<void> = new Deferred();
-    closed: Deferred<void> = new Deferred();
-
-    constructor(url: string) {
-        this.ws = new WebSocket(url);
-        this.ws.on('message', (data, _isBinary) => {
-            const command = JSON.parse(data.toString());
-            this.received.push(unpackCommand(command));
-        });
-        this.ws.on('open', () => {
-            this.opened.resolve();
-        });
-        this.ws.on('close', () => {
-            this.closed.resolve();
-        });
-    }
-
-    send(command: Command): void {
-        this.ws.send(JSON.stringify(command));
-    }
-
-    async mockNoResponse(time: number): Promise<void> {
-        this.ws.pause();
-        await setTimeout(time);
-        this.ws.terminate();
-        this.ws.resume();
-    }
-}

ts/webui/.gitignore

@@ -19,5 +19,3 @@
 .env.production.local
 
 npm-debug.log*
-yarn-debug.log*
-yarn-error.log*

ts/webui/package.json

@@ -3,104 +3,94 @@
   "version": "0.1.0",
   "license": "MIT",
   "dependencies": {
-    "axios": "^0.26.0",
+    "@fluentui/react": "^8.106.6",
-    "babel-jest": "^27.0.2",
+    "@uifabric/react-hooks": "^7.16.4",
-    "babel-loader": "^8.2.2",
+    "axios": "^1.3.4",
-    "babel-plugin-named-asset-import": "^0.3.2",
+    "copy-to-clipboard": "^3.3.3",
-    "babel-preset-react-app": "^10.0.0",
+    "d3": "^7.8.2",
-    "case-sensitive-paths-webpack-plugin": "^2.4.0",
+    "echarts": "^5.4.1",
-    "copy-to-clipboard": "^3.0.8",
+    "echarts-for-react": "^3.0.2",
-    "css-loader": "^5.2.6",
+    "json5": "^2.2.3",
-    "css-minimizer-webpack-plugin": "^3.0.1",
-    "d3": "^6.7.0",
-    "dotenv": "^10.0.0",
-    "dotenv-expand": "^5.1.0",
-    "echarts": "^5.2.2",
-    "echarts-for-react": "^3.0.0",
-    "file-loader": "^6.2.0",
-    "fs-extra": "^10.0.0",
-    "html-webpack-plugin": "^5.3.1",
-    "jest": "^27.0.4",
-    "jest-environment-jsdom-fourteen": "^1.0.1",
-    "jest-resolve": "^27.0.4",
-    "jest-watch-typeahead": "^0.6.4",
-    "json5": "^2.1.1",
-    "mini-css-extract-plugin": "^1.6.0",
-    "monaco-editor": "^0.25.0",
-    "monaco-editor-webpack-plugin": "^4.0.0",
-    "node-sass": "8.0.0",
     "parcoord-es": "^2.2.10",
-    "pnp-webpack-plugin": "^1.6.4",
-    "postcss-flexbugs-fixes": "^5.0.2",
-    "postcss-loader": "^6.1.0",
-    "postcss-normalize": "^10.0.0",
-    "postcss-preset-env": "^6.6.0",
     "rc-progress": "^3.4.1",
-    "react": "^16.8.6",
+    "react": "^17.0.2",
-    "react-app-polyfill": "^1.0.0",
+    "react-dom": "^17.0.2",
-    "react-dev-utils": "^11.0.4",
+    "react-json-tree": "^0.18.0",
-    "react-dom": "^16.8.6",
+    "react-monaco-editor": "^0.52.0",
-    "react-json-tree": "^0.11.2",
+    "react-paginate": "^8.1.4",
-    "react-monaco-editor": "^0.32.1",
+    "react-router-dom": "^6.9.0"
-    "react-paginate": "^6.3.2",
-    "react-responsive": "^8.1.1",
-    "react-router-dom": "^6.3.0",
-    "react-table": "^7.0.0-rc.15",
-    "resolve": "^1.10.0",
-    "sass-loader": "^12.1.0",
-    "semver": "^7.3.5",
-    "style-loader": "^2.0.0",
-    "url-loader": "^4.1.1",
-    "webpack": "^5.39.0",
-    "webpack-manifest-plugin": "^3.1.1",
-    "workbox-webpack-plugin": "^6.5.3"
   },
   "devDependencies": {
-    "@babel/core": "^7.14.6",
+    "@babel/core": ">=7.21.3",
-    "@babel/eslint-parser": "^7.14.5",
+    "@babel/eslint-parser": ">=7.21.3",
-    "@babel/preset-react": "^7.14.5",
+    "@babel/preset-react": ">=7.18.6",
-    "@babel/preset-typescript": "^7.14.5",
+    "@babel/preset-typescript": ">=7.21.0",
-    "@babel/runtime": "^7.14.6",
+    "@babel/runtime": ">=7.21.0",
-    "@fluentui/react": "^7.135.0",
+    "@svgr/webpack": ">=6.5.1",
-    "@svgr/webpack": "^6.1.2",
+    "@types/d3": ">=7.4.0",
-    "@types/d3": "^6.7.0",
+    "@types/node": ">=18.15.3",
-    "@types/node": "^15.12.2",
+    "@types/react": ">=17.0.53",
-    "@types/react": "^16.8.15",
+    "@types/react-dom": ">=17.0.19",
-    "@types/react-dom": "^16.8.4",
+    "@typescript-eslint/eslint-plugin": ">=5.55.0",
-    "@typescript-eslint/eslint-plugin": "^5.8.0",
+    "@typescript-eslint/parser": ">=5.55.0",
-    "@typescript-eslint/parser": "^5.8.0",
+    "babel-jest": ">=29.5.0",
-    "concurrently": "^6.2.0",
+    "babel-loader": ">=9.1.2",
-    "eslint": "^7.28.0",
+    "babel-plugin-named-asset-import": ">=0.3.8",
-    "eslint-config-prettier": "^8.3.0",
+    "babel-preset-react-app": ">=10.0.1",
-    "eslint-config-react-app": "^6.0.0",
+    "case-sensitive-paths-webpack-plugin": ">=2.4.0",
-    "eslint-loader": "^4.0.2",
+    "concurrently": ">=7.6.0",
-    "eslint-plugin-flowtype": "^5.7.2",
+    "css-loader": ">=5.2.7",
-    "eslint-plugin-import": "^2.23.4",
+    "css-minimizer-webpack-plugin": ">=4.2.2",
-    "eslint-plugin-jsx-a11y": "^6.4.1",
+    "dotenv": ">=16.0.3",
-    "eslint-plugin-prettier": "^3.1.0",
+    "dotenv-expand": ">=10.0.0",
-    "eslint-plugin-react": "^7.24.0",
+    "eslint": ">=8.36.0",
-    "eslint-plugin-react-hooks": "^4.2.0",
+    "eslint-config-prettier": ">=8.7.0",
-    "express": "^4.17.1",
+    "eslint-config-react-app": ">=7.0.1",
-    "npm": ">=8.11.0",
+    "eslint-plugin-flowtype": ">=8.0.3",
-    "prettier": "^2.3.1",
+    "eslint-plugin-import": ">=2.27.5",
-    "stylelint": "^13.7.0",
+    "eslint-plugin-jsx-a11y": ">=6.7.1",
-    "stylelint-config-standard": "^22.0.0",
+    "eslint-plugin-prettier": ">=4.2.1",
-    "trim-newlines": "^3.0.0",
+    "eslint-plugin-react": ">=7.32.2",
-    "typescript": "^4.3.2",
+    "eslint-plugin-react-hooks": ">=4.6.0",
-    "webpack-dev-server": ">=4.9.0"
+    "eslint-webpack-plugin": ">=4.0.0",
+    "express": ">=4.18.2",
+    "file-loader": ">=6.2.0",
+    "fs-extra": ">=11.1.0",
+    "html-webpack-plugin": ">=5.5.0",
+    "jest": ">=29.5.0",
+    "mini-css-extract-plugin": ">=2.7.3",
+    "monaco-editor-webpack-plugin": ">=7.0.1",
+    "npm": ">=9.6.1",
+    "pnp-webpack-plugin": ">=1.7.0",
+    "postcss-flexbugs-fixes": ">=5.0.2",
+    "postcss-loader": ">=7.0.2",
+    "postcss-normalize": ">=10.0.1",
+    "postcss-preset-env": ">=8.0.1",
+    "react-dev-utils": ">=12.0.1",
+    "resolve": ">=1.22.1",
+    "sass-loader": ">=13.2.0",
+    "style-loader": ">=3.3.2",
+    "typescript": ">=4.9.5",
+    "url-loader": ">=4.1.1",
+    "webpack": ">=5.76.1",
+    "webpack-dev-server": ">=4.12.0",
+    "webpack-manifest-plugin": ">=5.0.0",
+    "workbox-webpack-plugin": ">=6.5.4"
   },
   "proxy": "http://localhost:12138",
   "scripts": {
     "start": "node --max-old-space-size=3072 scripts/start.js",
     "build": "node --max-old-space-size=3072 scripts/developmentBuild.js",
     "release": "node --max-old-space-size=3072 scripts/productionBuild.js",
-    "sanity-check": "yarn sanity-check:tsc && yarn sanity-check:eslint",
+    "sanity-check": "npm run sanity-check:tsc && npm run sanity-check:eslint",
     "sanity-check:tsc": "tsc",
     "sanity-check:eslint": "eslint ./ --ext .tsx,.ts",
     "test": "node --max-old-space-size=3072 scripts/test.js",
     "stylelint": "npx stylelint **/*{.css,.scss}",
     "mock": "node scripts/server.js",
-    "dev": "concurrently \"yarn mock\" \"yarn start\""
+    "dev": "concurrently \"npm run mock\" \"npm run start\"",
+    "tsc": "tsc",
+    "eslint": "eslint"
   },
   "eslintConfig": {
     "extends": "react-app"
@@ -114,17 +104,9 @@
       "@babel/preset-typescript"
     ]
   },
-  "resolutions": {
+  "overrides": {
-    "yargs": ">=17.0.1",
+    "d3-color": ">=3.1.0",
-    "acorn": ">=8.4.0",
+    "minimatch": ">=7.4.2"
-    "y18n": ">=5.0.8",
-    "serialize-javascript": ">=5.0.1",
-    "css-what": ">=5.0.1",
-    "browserslist": ">=4.16.6",
-    "trim-newlines": ">=4.0.2",
-    "glob-parent": ">=6.0.0",
-    "strip-ansi": "=6.0.1",
-    "immer": ">=9.0.7"
   },
   "jest": {
     "verbose": true

ts/webui/scripts/config/webpack.config.js

@@ -15,7 +15,6 @@ const CssMinimizerPlugin = require('css-minimizer-webpack-plugin');
 const { WebpackManifestPlugin } = require('webpack-manifest-plugin');
 const InterpolateHtmlPlugin = require('react-dev-utils/InterpolateHtmlPlugin');
 const WorkboxWebpackPlugin = require('workbox-webpack-plugin');
-const WatchMissingNodeModulesPlugin = require('react-dev-utils/WatchMissingNodeModulesPlugin');
 const ModuleScopePlugin = require('react-dev-utils/ModuleScopePlugin');
 const getCSSModuleLocalIdent = require('react-dev-utils/getCSSModuleLocalIdent');
 const paths = require('./paths');
@@ -27,13 +26,14 @@ const paths = require('./paths');
 const getClientEnvironment = require('./env');
 const ModuleNotFoundPlugin = require('react-dev-utils/ModuleNotFoundPlugin');
 const ForkTsCheckerWebpackPlugin = require('react-dev-utils/ForkTsCheckerWebpackPlugin');
-const typescriptFormatter = require('react-dev-utils/typescriptFormatter');
 
 const postcssNormalize = require('postcss-normalize');
 
 // monaco-editor highlight
 const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin');
 
+const ESLintPlugin = require('eslint-webpack-plugin');
+
 // Source maps are resource heavy and can cause out of memory issue for large source files.
 const shouldUseSourceMap = process.env.GENERATE_SOURCEMAP !== 'false';
 // Some apps do not need the benefits of saving a web request, so not inlining the chunk
@@ -302,23 +302,6 @@ module.exports = function(webpackEnv) {
           test: /\.[cm]?(js|tsx?)$/,
           parser: { requireEnsure: false },
         },
-        // First, run the linter.
-        // It's important to do this before Babel processes the JS.
-        {
-          test: /\.(js|mjs|jsx|ts|tsx)$/,
-          enforce: 'pre',
-          use: [
-            {
-              options: {
-                formatter: require.resolve('react-dev-utils/eslintFormatter'),
-                eslintPath: require.resolve('eslint'),
-                
-              },
-              loader: require.resolve('eslint-loader'),
-            },
-          ],
-          include: paths.appSrc,
-        },
         {
           // "oneOf" will traverse all following loaders until one will
           // match the requirements. When no loader matches it will fall
@@ -479,6 +462,11 @@ module.exports = function(webpackEnv) {
       ],
     },
     plugins: [
+      new ESLintPlugin({
+        extensions: ['js', 'mjs', 'jsx', 'ts', 'tsx'],
+        formatter: require.resolve('react-dev-utils/eslintFormatter'),
+        eslintPath: require.resolve('eslint'),
+      }),
       // Generates an `index.html` file with the <script> injected.
       new HtmlWebpackPlugin(
         Object.assign(
@@ -536,12 +524,6 @@ module.exports = function(webpackEnv) {
       // a plugin that prints an error when you attempt to do this.
       // See https://github.com/facebook/create-react-app/issues/240
       isEnvDevelopment && new CaseSensitivePathsPlugin(),
-      // If you require a missing module and then `npm install` it, you still have
-      // to restart the development server for Webpack to discover it. This plugin
-      // makes the discovery automatic so you don't have to restart.
-      // See https://github.com/facebook/create-react-app/issues/186
-      isEnvDevelopment &&
-        new WatchMissingNodeModulesPlugin(paths.appNodeModules),
       isEnvProduction &&
         new MiniCssExtractPlugin({
           // Options similar to the same options in webpackOptions.output
@@ -591,25 +573,20 @@ module.exports = function(webpackEnv) {
       // TypeScript type checking
       useTypeScript &&
         new ForkTsCheckerWebpackPlugin({
-          typescript: resolve.sync('typescript', {
+          //typescript: resolve.sync('typescript', { basedir: paths.appNodeModules }),
-            basedir: paths.appNodeModules,
-          }),
           async: isEnvDevelopment,
-          useTypescriptIncrementalApi: true,
+          //useTypescriptIncrementalApi: true,
-          checkSyntacticErrors: true,
+          //checkSyntacticErrors: true,
-          tsconfig: paths.appTsConfig,
+          //tsconfig: paths.appTsConfig,
-          eslint: false,
+          //reportFiles: [
-          reportFiles: [
+          //  '**',
-            '**',
+          //  '!**/__tests__/**',
-            '!**/__tests__/**',
+          //  '!**/?(*.)(spec|test).*',
-            '!**/?(*.)(spec|test).*',
+          //  '!**/src/setupProxy.*',
-            '!**/src/setupProxy.*',
+          //  '!**/src/setupTests.*',
-            '!**/src/setupTests.*',
+          //],
-          ],
+          //watch: paths.appSrc,
-          watch: paths.appSrc,
+          //formatter: isEnvProduction ? 'basic' : undefined,
-          silent: true,
-          // The formatter is invoked directly in WebpackDevServerUtils during development
-          formatter: isEnvProduction ? typescriptFormatter : undefined,
         }),
     ].filter(Boolean),
     // Some libraries import Node modules but don't use them in the browser.

ts/webui/src/components/common/ExpandableDetails/OpenRow.tsx

@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import { Stack, PrimaryButton, Pivot, PivotItem, DefaultButton } from '@fluentui/react';
 import * as copy from 'copy-to-clipboard';
-import JSONTree from 'react-json-tree';
+import { JSONTree } from 'react-json-tree';
 import { Trial } from '@model/trial';
 import { MANAGER_IP, RETIARIIPARAMETERS } from '@static/const';
 import { EXPERIMENT, TRIALS } from '@static/datamodel';
@@ -85,7 +85,7 @@ const OpenRow = (props: OpenRowProps): any => {
                                 <Stack className='bgHyper'>
                                     <JSONTree
                                         hideRoot={true}
-                                        shouldExpandNode={() => true} // default expandNode
+                                        shouldExpandNodeInitially={() => true} // default expandNode
                                         getItemString={() => null} // remove the {} items
                                         data={reformatRetiariiParameter(originParameters as any)}
                                     />

ts/webui/src/components/common/PaginationTable.tsx

@@ -86,7 +86,7 @@ const PaginationTable = (props: IDetailsListProps): any => {
                     pageRangeDisplayed={2}
                     onPageChange={_onPageSelect.bind(this)}
                     containerClassName={itemsCount === 0 ? 'pagination hidden' : 'pagination'}
-                    subContainerClassName={'pages pagination'}
+                    //subContainerClassName={'pages pagination'}
                     disableInitialCallback={false}
                     activeClassName={'active'}
                 />

ts/webui/src/components/nav/slideNav/TrialConfigPanel.tsx

@@ -45,7 +45,7 @@ const TrialConfigPanel = (props: LogPanelProps): any => {
         return blacklist.includes(key) ? undefined : val;
     };
     const profile = lodash.cloneDeep(EXPERIMENT.profile);
-    profile.execDuration = convertDuration(profile.execDuration);
+    profile.execDuration = convertDuration(profile.execDuration) as any; // FIXME
     const prettyWidth = innerWidth > 1400 ? 100 : 60;
     const showProfile = JSON.stringify(profile, filter, 2);