diff --git a/docs/cli.md b/docs/cli.md index 173e9545..06e9c234 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -103,17 +103,17 @@ sb deploy [--docker-image] #### Optional arguments -| Name | Default | Description | -|-----------------------|-------------------------|-------------------------------------------------------------------------------| +| Name | Default | Description | +|-----------------------|-------------------------|-----------------------------------------------------------------------------------| | `--docker-image` `-i` | `superbench/superbench` | Docker image URI, [here](./user-tutorial/container-images.mdx) listed all images. | -| `--docker-password` | `None` | Docker registry password if authentication is needed. | -| `--docker-username` | `None` | Docker registry username if authentication is needed. | -| `--host-file` `-f` | `None` | Path to Ansible inventory host file. | -| `--host-list` `-l` | `None` | Comma separated host list. | -| `--host-password` | `None` | Host password or key passphase if needed. | -| `--host-username` | `None` | Host username if needed. | -| `--output-dir` | `None` | Path to output directory, outputs/{datetime} will be used if not specified. | -| `--private-key` | `None` | Path to private key if needed. | +| `--docker-password` | `None` | Docker registry password if authentication is needed. | +| `--docker-username` | `None` | Docker registry username if authentication is needed. | +| `--host-file` `-f` | `None` | Path to Ansible inventory host file. | +| `--host-list` `-l` | `None` | Comma separated host list. | +| `--host-password` | `None` | Host password or key passphase if needed. | +| `--host-username` | `None` | Host username if needed. | +| `--output-dir` | `None` | Path to output directory, outputs/{datetime} will be used if not specified. | +| `--private-key` | `None` | Path to private key if needed. | #### Global arguments @@ -281,6 +281,7 @@ sb run [--config-file] [--host-list] [--host-password] [--host-username] + [--no-docker] [--output-dir] [--private-key] ``` @@ -298,6 +299,7 @@ sb run [--config-file] | `--host-list` `-l` | `None` | Comma separated host list. | | `--host-password` | `None` | Host password or key passphase if needed. | | `--host-username` | `None` | Host username if needed. | +| `--no-docker` | `False` | Run on host directly without Docker. | | `--output-dir` | `None` | Path to output directory, outputs/{datetime} will be used if not specified. | | `--private-key` | `None` | Path to private key if needed. | @@ -320,6 +322,12 @@ and default benchmarking configuration: sb run --docker-image superbench/cuda:11.1 --host-file ./host.ini ``` +Run kernel launch benchmarks on host directly without using Docker: +```bash title="SB CLI" +sb run --no-docker --host-list localhost --config-override \ + superbench.enable=kernel-launch superbench.env.SB_MICRO_PATH=/path/to/superbenchmark +``` + ### `sb version` Print the current SuperBench CLI version. diff --git a/docs/getting-started/run-superbench.md b/docs/getting-started/run-superbench.md index 649759fe..f0086db7 100644 --- a/docs/getting-started/run-superbench.md +++ b/docs/getting-started/run-superbench.md @@ -40,3 +40,10 @@ After deployment, you can start to run the SuperBench benchmarks on all managed ```bash sb run -f local.ini -c resnet.yaml ``` + +:::tip TIP +For environments that cannot start containers through `sb deploy`, e.g., a Kubernetes cluster. +You can create a privileged container with `superbench/superbench` image, skip `sb deploy`, and run `sb run` directly inside the container with `--no-docker` argument: +`sb run --no-docker -l localhost -c resnet.yaml`. + +::: diff --git a/setup.py b/setup.py index 4ef72915..76beba0a 100644 --- a/setup.py +++ b/setup.py @@ -12,10 +12,17 @@ import sys import pathlib from typing import List, Tuple +import pkg_resources from setuptools import setup, find_packages, Command import superbench +try: + pkg_resources.require(['pip>=18']) +except (pkg_resources.VersionConflict, pkg_resources.DistributionNotFound): + print('Try upgrade pip to latest version, for example, python3 -m pip install --upgrade pip') + raise + here = pathlib.Path(__file__).parent.resolve() long_description = (here / 'README.md').read_text(encoding='utf-8') @@ -150,7 +157,6 @@ setup( 'pyyaml>=5.3', 'seaborn>=0.11.2', 'tcping>=0.1.1rc1', - 'types-Markdown>=3.3.0' 'xlrd>=2.0.1', 'xlsxwriter>=1.3.8', 'xmltodict>=0.12.0', @@ -166,6 +172,8 @@ setup( 'pytest-cov>=2.11.1', 'pytest-subtests>=0.4.0', 'pytest>=6.2.2', + 'types-markdown', + 'types-pkg_resources', 'types-pyyaml', 'vcrpy>=4.1.1', 'yapf==0.31.0', diff --git a/superbench/benchmarks/build.sh b/superbench/benchmarks/build.sh index 3f659b31..bc9b4a31 100755 --- a/superbench/benchmarks/build.sh +++ b/superbench/benchmarks/build.sh @@ -5,6 +5,7 @@ set -e +MPI_HOME="${MPI_HOME:-/usr/local/mpi}" SB_MICRO_PATH="${SB_MICRO_PATH:-/usr/local}" for dir in micro_benchmarks/*/ ; do @@ -12,7 +13,7 @@ for dir in micro_benchmarks/*/ ; do SOURCE_DIR=$dir BUILD_ROOT=$dir/build mkdir -p $BUILD_ROOT - cmake -DCMAKE_INSTALL_PREFIX=$SB_MICRO_PATH -DCMAKE_BUILD_TYPE=Release -S $SOURCE_DIR -B $BUILD_ROOT + cmake -DCMAKE_PREFIX_PATH=$MPI_HOME -DCMAKE_INSTALL_PREFIX=$SB_MICRO_PATH -DCMAKE_BUILD_TYPE=Release -S $SOURCE_DIR -B $BUILD_ROOT cmake --build $BUILD_ROOT cmake --install $BUILD_ROOT fi diff --git a/superbench/cli/_commands.py b/superbench/cli/_commands.py index c666f82d..ce0d9d99 100644 --- a/superbench/cli/_commands.py +++ b/superbench/cli/_commands.py @@ -43,6 +43,7 @@ class SuperBenchCommandsLoader(CLICommandsLoader): ac.argument('docker_image', options_list=('--docker-image', '-i'), type=str, help='Docker image URI.') ac.argument('docker_username', type=str, help='Docker registry username if authentication is needed.') ac.argument('docker_password', type=str, help='Docker registry password if authentication is needed.') + ac.argument('no_docker', action='store_true', help='Run on host directly without Docker.') ac.argument( 'host_file', options_list=('--host-file', '-f'), type=str, help='Path to Ansible inventory host file.' ) diff --git a/superbench/cli/_handler.py b/superbench/cli/_handler.py index e8b1435d..22ed82d6 100644 --- a/superbench/cli/_handler.py +++ b/superbench/cli/_handler.py @@ -96,6 +96,7 @@ def process_runner_arguments( docker_image='superbench/superbench', docker_username=None, docker_password=None, + no_docker=False, host_file=None, host_list=None, host_username=None, @@ -111,6 +112,7 @@ def process_runner_arguments( docker_image (str, optional): Docker image URI. Defaults to superbench/superbench:latest. docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None. docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None. + no_docker (bool, optional): Run on host directly without Docker. Defaults to False. host_file (str, optional): Path to Ansible inventory host file. Defaults to None. host_list (str, optional): Comma separated host list. Defaults to None. host_username (str, optional): Host username if needed. Defaults to None. @@ -144,6 +146,7 @@ def process_runner_arguments( 'username': docker_username, 'password': docker_password, 'registry': split_docker_domain(docker_image)[0], + 'skip': no_docker, } ) # Ansible config @@ -234,6 +237,7 @@ def deploy_command_handler( docker_image=docker_image, docker_username=docker_username, docker_password=docker_password, + no_docker=False, host_file=host_file, host_list=host_list, host_username=host_username, @@ -250,6 +254,7 @@ def run_command_handler( docker_image='superbench/superbench', docker_username=None, docker_password=None, + no_docker=False, host_file=None, host_list=None, host_username=None, @@ -267,6 +272,7 @@ def run_command_handler( docker_image (str, optional): Docker image URI. Defaults to superbench/superbench:latest. docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None. docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None. + no_docker (bool, optional): Run on host directly without Docker. Defaults to False. host_file (str, optional): Path to Ansible inventory host file. Defaults to None. host_list (str, optional): Comma separated host list. Defaults to None. host_username (str, optional): Host username if needed. Defaults to None. @@ -284,6 +290,7 @@ def run_command_handler( docker_image=docker_image, docker_username=docker_username, docker_password=docker_password, + no_docker=no_docker, host_file=host_file, host_list=host_list, host_username=host_username, diff --git a/superbench/cli/_help.py b/superbench/cli/_help.py index 8c4179a5..3e349613 100644 --- a/superbench/cli/_help.py +++ b/superbench/cli/_help.py @@ -59,6 +59,10 @@ helps['run'] = """ - name: run all benchmarks on all nodes in ./host.ini using image "superbench/cuda:11.1" and default benchmarking configuration text: {cli_name} run --docker-image superbench/cuda:11.1 --host-file ./host.ini + - name: run kernel launch benchmarks on host directly without using Docker + text: > + {cli_name} run --no-docker --host-list localhost + --config-override superbench.enable=kernel-launch superbench.env.SB_MICRO_PATH=/path/to/superbenchmark """.format(cli_name=CLI_NAME) helps['benchmark'] = """ diff --git a/superbench/runner/playbooks/check_env.yaml b/superbench/runner/playbooks/check_env.yaml index 15e69060..53ed7d42 100644 --- a/superbench/runner/playbooks/check_env.yaml +++ b/superbench/runner/playbooks/check_env.yaml @@ -4,15 +4,17 @@ max_fail_percentage: 0 vars: container: sb-workspace + skip: '{{ no_docker | default(false) }}' tasks: - name: Checking container status shell: docker inspect --format={{ '{{.State.Running}}' }} {{ container }} register: result ignore_errors: true become: yes + when: not skip - fail: msg: Container {{ container }} is not running. - when: result is failed or result.stdout != "true" + when: (not skip) and (result is failed or result.stdout != "true") - name: Runtime Environment Update hosts: all @@ -22,6 +24,8 @@ container: sb-workspace sb_nodes: '{{ hostvars.values() | map(attribute="ansible_hostname") | sort }}' sb_env: | + # sb env + HOST_WS={{ ansible_user_dir }}/sb-workspace # pytorch env NNODES={{ sb_nodes | length }} NODE_RANK={{ lookup('ansible.utils.index_of', sb_nodes, 'eq', ansible_hostname) }} @@ -31,6 +35,11 @@ # config env {{ env | default('') }} tasks: + - name: Ensure Workspace + file: + path: '{{ workspace }}' + state: directory + mode: 0755 - name: Updating Config copy: src: '{{ output_dir }}/sb.config.yaml' diff --git a/superbench/runner/runner.py b/superbench/runner/runner.py index 3d6a718a..1f0f70c5 100644 --- a/superbench/runner/runner.py +++ b/superbench/runner/runner.py @@ -186,6 +186,7 @@ class SuperBenchRunner(): self._ansible_client.get_playbook_config( 'check_env.yaml', extravars={ + 'no_docker': bool(self._docker_config.skip), 'output_dir': str(self._output_path), 'env': '\n'.join(f'{k}={v}' for k, v in self._sb_config.superbench.env.items()), } @@ -388,13 +389,18 @@ class SuperBenchRunner(): timeout = self._sb_benchmarks[benchmark_name].timeout env_list = '--env-file /tmp/sb.env' + if self._docker_config.skip: + env_list = 'set -o allexport && source /tmp/sb.env && set +o allexport' for k, v in mode.env.items(): if isinstance(v, str): - env_list += f' -e {k}={str(v).format(proc_rank=mode.proc_rank, proc_num=mode.proc_num)}' + envvar = f'{k}={str(v).format(proc_rank=mode.proc_rank, proc_num=mode.proc_num)}' + env_list += f' -e {envvar}' if not self._docker_config.skip else f' && export {envvar}' + + fcmd = "docker exec {env_list} sb-workspace bash -c '{command}'" + if self._docker_config.skip: + fcmd = "bash -c '{env_list} && cd $HOST_WS && {command}'" ansible_runner_config = self._ansible_client.get_shell_config( - "docker exec {env_list} sb-workspace bash -c '{command}'".format( - env_list=env_list, command=self.__get_mode_command(benchmark_name, mode, timeout) - ) + fcmd.format(env_list=env_list, command=self.__get_mode_command(benchmark_name, mode, timeout)) ) if mode.name == 'mpi': ansible_runner_config = self._ansible_client.update_mpi_config(ansible_runner_config) diff --git a/tests/cli/test_sb.py b/tests/cli/test_sb.py index af9b3752..b239ca20 100644 --- a/tests/cli/test_sb.py +++ b/tests/cli/test_sb.py @@ -69,6 +69,10 @@ class SuperBenchCLIScenarioTest(ScenarioTest): """Test sb run.""" self.cmd('sb run --host-list localhost --config-override superbench.enable=none', checks=[NoneCheck()]) + def test_sb_run_skipdocker(self): + """Test sb run without docker.""" + self.cmd('sb run -l localhost -C superbench.enable=none --no-docker', checks=[NoneCheck()]) + def test_sb_run_no_docker_auth(self): """Test sb run, only --docker-username argument, should fail.""" result = self.cmd('sb run --docker-username test-user', expect_failure=True) diff --git a/tests/runner/test_runner.py b/tests/runner/test_runner.py index 1011de47..965090c4 100644 --- a/tests/runner/test_runner.py +++ b/tests/runner/test_runner.py @@ -25,7 +25,12 @@ class RunnerTestCase(unittest.TestCase): self.default_config = OmegaConf.create(yaml.load(fp, Loader=yaml.SafeLoader)) self.sb_output_dir = tempfile.mkdtemp() - self.runner = SuperBenchRunner(self.default_config, None, None, self.sb_output_dir) + self.runner = SuperBenchRunner( + self.default_config, + OmegaConf.create({}), + OmegaConf.create({}), + self.sb_output_dir, + ) def tearDown(self): """Hook method for deconstructing the test fixture after testing it."""