microsoft
/
superbenchmark
зеркало из https://github.com/microsoft/superbenchmark.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

CLI - Add command sb [version,deploy,exec,run] (#10) 

- Add CLI commands
  * sb version
  * sb deploy
  * sb exec
  * sb run
- Add interface with executor and runner
- Add cli test cases
This commit is contained in:
Yifan Xiong 2021-03-12 13:16:43 +08:00 коммит произвёл GitHub
Родитель ebea2d5053
Коммит 5d11579a10
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
15 изменённых файлов: 596 добавлений и 9 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
MANIFEST.in
Просмотреть файл

@ -1 +1,5 @@
include LICENSE README.md
recursive-include superbench *.py
recursive-include superbench *.yaml
global-exclude *.pyc
global-exclude __pycache__

25
setup.py
Просмотреть файл

@ -132,18 +132,23 @@ setup(
keywords='benchmark, AI systems',
packages=find_packages(exclude=['tests']),
python_requires='>=3.6, <4',
install_requires=[],
install_requires=[
'hydra-colorlog>=1.0.0',
'hydra-core>=1.0.4',
'knack>=0.7.2',
],
extras_require={
'dev': ['pre-commit>=2.10.0'],
'test': [
'yapf>=0.30.0',
'mypy>=0.800',
'flake8>=3.8.4',
'flake8-quotes>=3.2.0',
'flake8-docstrings>=1.5.0',
'flake8-quotes>=3.2.0',
'flake8>=3.8.4',
'mypy>=0.800',
'pydocstyle>=5.1.1',
'pytest>=6.2.2',
'pytest-cov>=2.11.1',
'pytest>=6.2.2',
'vcrpy>=4.1.1',
'yapf>=0.30.0',
],
'torch': [
'torch==1.7.0',
@ -151,9 +156,13 @@ setup(
'transformers==4.3.3',
],
},
package_data={},
include_package_data=True,
entry_points={
'console_scripts': [],
'console_scripts': [
'sb = superbench.cli.sb:main',
'sb-exec = superbench.cli.sb_exec:main',
'sb-run = superbench.cli.sb_run:main',
],
},
cmdclass={
'format': Formatter,

8
superbench/cli/__init__.py Normal file
Просмотреть файл

@ -0,0 +1,8 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench cli module."""
from .sb import SuperBenchCLI
__all__ = ['SuperBenchCLI']

54
superbench/cli/_commands.py Normal file
Просмотреть файл

@ -0,0 +1,54 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench CLI commands."""
from knack.arguments import ArgumentsContext
from knack.commands import CLICommandsLoader, CommandGroup
class SuperBenchCommandsLoader(CLICommandsLoader):
"""SuperBench CLI commands loader."""
def load_command_table(self, args):
"""Load commands into the command table.
Args:
args (list): List of arguments from the command line.
Returns:
collections.OrderedDict: Load commands into the command table.
"""
with CommandGroup(self, '', 'superbench.cli._handler#{}') as g:
g.command('version', 'version_command_handler')
g.command('deploy', 'deploy_command_handler')
g.command('exec', 'exec_command_handler')
g.command('run', 'run_command_handler')
return super().load_command_table(args)
def load_arguments(self, command):
"""Load arguments for commands.
Args:
command: The command to load arguments for.
"""
with ArgumentsContext(self, '') as ac:
ac.argument('docker_image', options_list=('--docker-image', '-i'), type=str, help='Docker image URI.')
ac.argument('docker_username', type=str, help='Docker registry username if authentication is needed.')
ac.argument('docker_password', type=str, help='Docker registry password if authentication is needed.')
ac.argument(
'host_file', options_list=('--host-file', '-f'), type=str, help='Path to Ansible inventory host file.'
)
ac.argument('host_list', options_list=('--host-list', '-l'), type=str, help='Comma separated host list.')
ac.argument('host_username', type=str, help='Host username if needed.')
ac.argument('host_password', type=str, help='Host password or key passphase if needed.')
ac.argument('private_key', type=str, help='Path to private key if needed.')
ac.argument(
'config_file', options_list=('--config-file', '-c'), type=str, help='Path to SuperBench config file.'
)
ac.argument(
'config_override',
options_list=('--config-override', '-C'),
type=str,
help='Extra arguments to override config_file, following Hydra syntax.'
)
super().load_arguments(command)

160
superbench/cli/_handler.py Normal file
Просмотреть файл

@ -0,0 +1,160 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench CLI command handler."""
import os
import yaml
from pathlib import Path
from knack.util import CLIError
import superbench
from superbench.common.utils import get_sb_command, get_config, new_output_dir
def check_argument_file(name, file):
"""Check file path in CLI arguments.
Args:
name (str): argument name.
file (str): file path.
Raises:
CLIError: If file does not exist.
"""
if file and not Path(file).exists():
raise CLIError('{} {} does not exist.'.format(name, file))
def version_command_handler():
"""Print the current SuperBench tool version.
Returns:
str: current SuperBench tool version.
"""
return superbench.__version__
def deploy_command_handler(
docker_image,
docker_username=None,
docker_password=None,
host_file=None,
host_list=None,
host_username=None,
host_password=None,
private_key=None
):
"""Deploy the SuperBench environments to all given nodes.
Deploy SuperBench environments on all nodes, including:
1. check drivers
2. install required system dependencies
3. install Docker and container runtime
4. pull Docker image
Args:
docker_image (str): Docker image URI.
docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
host_file (str, optional): Path to Ansible inventory host file. Defaults to None.
host_list (str, optional): Comma separated host list. Defaults to None.
host_username (str, optional): Host username if needed. Defaults to None.
host_password (str, optional): Host password or key passphase if needed. Defaults to None.
private_key (str, optional): Path to private key if needed. Defaults to None.
Raises:
CLIError: If input arguments are invalid.
"""
if not (host_file or host_list):
raise CLIError('Must specify one of host_file or host_list.')
check_argument_file('host_file', host_file)
check_argument_file('private_key', private_key)
raise NotImplementedError
def exec_command_handler(
docker_image, docker_username=None, docker_password=None, config_file=None, config_override=None
):
"""Run the SuperBench benchmarks locally.
Args:
docker_image (str): Docker image URI.
docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
config_file (str, optional): Path to SuperBench config file. Defaults to None.
config_override (str, optional): Extra arguments to override config_file,
following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None.
Raises:
CLIError: If input arguments are invalid.
"""
if bool(docker_username) != bool(docker_password):
raise CLIError('Must specify both docker_username and docker_password if authentication is needed.')
check_argument_file('config_file', config_file)
# Dump configs into outputs/date/config.merge.yaml
config = get_config(config_file)
config['docker'] = {}
for key in ['image', 'username', 'password']:
config['docker'][key] = eval('docker_{}'.format(key))
output_dir = new_output_dir()
with (Path(output_dir) / 'config.merge.yaml').open(mode='w') as f:
yaml.safe_dump(config, f)
os.system(get_sb_command('sb-exec', output_dir, config_override or ''))
def run_command_handler(
docker_image,
docker_username=None,
docker_password=None,
host_file=None,
host_list=None,
host_username=None,
host_password=None,
private_key=None,
config_file=None,
config_override=None
):
"""Run the SuperBench benchmarks distributedly.
Run all benchmarks on given nodes.
Args:
docker_image (str): Docker image URI.
docker_username (str, optional): Docker registry username if authentication is needed. Defaults to None.
docker_password (str, optional): Docker registry password if authentication is needed. Defaults to None.
host_file (str, optional): Path to Ansible inventory host file. Defaults to None.
host_list (str, optional): Comma separated host list. Defaults to None.
host_username (str, optional): Host username if needed. Defaults to None.
host_password (str, optional): Host password or key passphase if needed. Defaults to None.
private_key (str, optional): Path to private key if needed. Defaults to None.
config_file (str, optional): Path to SuperBench config file. Defaults to None.
config_override (str, optional): Extra arguments to override config_file,
following [Hydra syntax](https://hydra.cc/docs/advanced/override_grammar/basic). Defaults to None.
Raises:
CLIError: If input arguments are invalid.
"""
if bool(docker_username) != bool(docker_password):
raise CLIError('Must specify both docker_username and docker_password if authentication is needed.')
if not (host_file or host_list):
raise CLIError('Must specify one of host_file or host_list.')
check_argument_file('host_file', host_file)
check_argument_file('private_key', private_key)
check_argument_file('config_file', config_file)
# Dump configs into outputs/date/config.merge.yaml
config = get_config(config_file)
config['docker'] = {}
for key in ['image', 'username', 'password']:
config['docker'][key] = eval('docker_{}'.format(key))
config['ansible'] = {}
for key in ['file', 'list', 'username', 'password']:
config['ansible']['host_{}'.format(key)] = eval('host_{}'.format(key))
output_dir = new_output_dir()
with (Path(output_dir) / 'config.merge.yaml').open(mode='w') as f:
yaml.safe_dump(config, f)
os.system(get_sb_command('sb-run', output_dir, config_override or ''))

69
superbench/cli/_help.py Normal file
Просмотреть файл

@ -0,0 +1,69 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench CLI help."""
from knack.help import CLIHelp
from knack.help_files import helps
CLI_NAME = 'sb'
WELCOME_MESSAGE = r"""
_____ ____ _
/ ____| | _ \ | |
| (___ _ _ _ __ ___ _ __| |_) | ___ _ __ ___| |__
\___ \| | | | '_ \ / _ \ '__| _ < / _ \ '_ \ / __| '_ \
____) | |_| | |_) | __/ | | |_) | __/ | | | (__| | | |
|_____/ \__,_| .__/ \___|_| |____/ \___|_| |_|\___|_| |_|
| |
|_|
Welcome to the SB CLI!
"""
helps['version'] = """
type: command
short-summary: Print the current SuperBench CLI version.
examples:
- name: print version
text: {cli_name} version
""".format(cli_name=CLI_NAME)
helps['deploy'] = """
type: command
short-summary: Deploy the SuperBench environments to all given nodes.
examples:
- name: deploy image "superbench/cuda:11.1" to all nodes in ./host.yaml
text: {cli_name} deploy --docker-image superbench/cuda:11.1 --host-file ./host.yaml
- name: deploy image "superbench/rocm:4.0" to node-0 and node-2, using key file id_rsa for ssh
text: {cli_name} deploy --docker-image superbench/rocm:4.0 --host-list node-0,node-2 --private-key id_rsa
""".format(cli_name=CLI_NAME)
helps['exec'] = """
type: command
short-summary: Execute the SuperBench benchmarks locally.
examples:
- name: execute all benchmarks using image "superbench/cuda:11.1" and default benchmarking configuration
text: {cli_name} exec --docker-image superbench/cuda:11.1
- name: execute all benchmarks using image "superbench/rocm:4.0" and custom config file ./config.yaml
text: {cli_name} exec --docker-image superbench/rocm:4.0 --config-file ./config.yaml
""".format(cli_name=CLI_NAME)
helps['run'] = """
type: command
short-summary: Run the SuperBench benchmarks distributedly.
examples:
- name: run all benchmarks on all nodes in ./host.yaml using image "superbench/cuda:11.1"
and default benchmarking configuration
text: {cli_name} run --docker-image superbench/cuda:11.1 --host-file ./host.yaml
""".format(cli_name=CLI_NAME)
class SuperBenchCLIHelp(CLIHelp):
"""SuperBench CLI help loader."""
def __init__(self, cli_ctx=None):
"""Init CLI help loader.
Args:
cli_ctx (knack.cli.CLI, optional): CLI Context. Defaults to None.
"""
super().__init__(cli_ctx=cli_ctx, welcome_message=WELCOME_MESSAGE)

50
superbench/cli/sb.py Normal file
Просмотреть файл

@ -0,0 +1,50 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench command line interface."""
import sys
from knack import CLI
import superbench
from superbench.cli._help import CLI_NAME, SuperBenchCLIHelp
from superbench.cli._commands import SuperBenchCommandsLoader
class SuperBenchCLI(CLI):
"""The main driver for SuperBench CLI."""
def get_cli_version(self):
"""Get the CLI version.
Returns:
str: CLI semantic version.
"""
return superbench.__version__
@classmethod
def get_cli(cls):
"""Get CLI instance.
Returns:
SuperBenchCLI: An instance for SuperBench CLI.
"""
return cls(
cli_name=CLI_NAME,
config_env_var_prefix=CLI_NAME,
commands_loader_cls=SuperBenchCommandsLoader,
help_cls=SuperBenchCLIHelp,
)
def main():
"""The main function for CLI."""
sb_cli = SuperBenchCLI.get_cli()
exit_code = sb_cli.invoke(sys.argv[1:])
sys.exit(exit_code)
if __name__ == '__main__':
main()

22
superbench/cli/sb_exec.py Normal file
Просмотреть файл

@ -0,0 +1,22 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench sb exec command."""
import hydra
from superbench.common.utils import logger
@hydra.main(config_path='../config', config_name='default')
def main(config):
"""The main entrypoint for sb-exec."""
logger.info(config)
# executor = SuperBenchExecutor(config)
# executor.exec()
if __name__ == '__main__':
main()

22
superbench/cli/sb_run.py Normal file
Просмотреть файл

@ -0,0 +1,22 @@
#!/usr/bin/env python3
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench sb run command."""
import hydra
from superbench.common.utils import logger
@hydra.main(config_path='../config', config_name='default')
def main(config):
"""The main entrypoint for sb-run."""
logger.info(config)
# runner = SuperBenchRunner(config)
# runner.run()
if __name__ == '__main__':
main()

4
superbench/common/utils/__init__.py
Просмотреть файл

@ -4,5 +4,7 @@
"""Exposes the interface of SuperBench common utilities."""
from .logging import logger
from .file_handler import new_output_dir, get_config
from .command import get_sb_command
__all__ = ['logger']
__all__ = ['logger', 'new_output_dir', 'get_config', 'get_sb_command']

24
superbench/common/utils/command.py Normal file
Просмотреть файл

@ -0,0 +1,24 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""Utilities for command."""
def get_sb_command(cli, output_path, config_override):
"""Get sb command for sb-run or sb-exec.
Args:
cli (str): CLI name.
output_path (str): Output directory path.
config_override (str): Extra arguments to override config.
Returns:
str: Command to run.
"""
sb_cmd = '{cli} ' \
'--config-name=config.merge ' \
'--config-dir={path} ' \
'hydra.run.dir={path} ' \
'hydra.sweep.dir={path} ' \
'{args}'.format(cli=cli, path=output_path, args=config_override)
return sb_cmd.strip()

42
superbench/common/utils/file_handler.py Normal file
Просмотреть файл

@ -0,0 +1,42 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""Utilities for file."""
import yaml
from pathlib import Path
from datetime import datetime
def new_output_dir():
"""Generate a new output directory.
Generate a new output directory name based on current time and create it on filesystem.
Returns:
str: Output directory name.
"""
output_name = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
output_path = Path('.', 'outputs', output_name)
output_path.mkdir(mode=0o755, parents=True, exist_ok=True)
return str(output_path)
def get_config(config_file):
"""Read SuperBench config yaml.
Read config file, use default config if None is provided.
Args:
config_file (str): config file path.
Returns:
dict: Config object, None if file does not exist.
"""
here = Path(__file__).parent.resolve()
p = Path(config_file) if config_file else here / '../../config/default.yaml'
if not p.is_file():
return None
with p.open() as f:
config = yaml.safe_load(f)
return config

4
superbench/config/__init__.py Normal file
Просмотреть файл

@ -0,0 +1,4 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench config module."""

25
superbench/config/default.yaml Normal file
Просмотреть файл

@ -0,0 +1,25 @@
# @package _global_
# Hydra config
hydra:
run:
dir: ./outputs/${now:%Y-%m-%d_%H-%M-%S}
sweep:
dir: ./outputs/${now:%Y-%m-%d_%H-%M-%S}
job_logging:
formatters:
colorlog:
format: >-
[%(cyan)s%(asctime)s %(hostname)s%(reset)s][%(blue)s%(filename)s:%(lineno)s%(reset)s][%(log_color)s%(levelname)s%(reset)s] %(message)s
defaults:
- hydra/job_logging: colorlog
- hydra/hydra_logging: colorlog
# SuperBench config
superbench:
use: []
benchmarks:
pytorch_models:
enable: true
parameters:
batch_size: 32

92
tests/cli/test_sb.py Normal file
Просмотреть файл

@ -0,0 +1,92 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
"""SuperBench CLI command and scenario tests."""
import io
import contextlib
from functools import wraps
from knack.testsdk import ScenarioTest, StringCheck, NoneCheck
import superbench
from superbench.cli import SuperBenchCLI
def capture_system_exit(func):
"""Decorator to capture SystemExit in testing.
Args:
func (Callable): Decorated function.
Returns:
Callable: Decorator.
"""
@wraps(func)
def wrapper(self, *args, **kwargs):
f = io.StringIO()
with self.assertRaises(SystemExit) as cm, contextlib.redirect_stderr(f):
func(self, *args, **kwargs)
self.assertEqual(cm.exception.code, 2)
self.stderr = f.getvalue()
return wrapper
class SuperBenchCLIScenarioTest(ScenarioTest):
"""A class whose instances are CLI single test cases.
Args:
ScenarioTest (knack.testsdk.ScenarioTest): Test class for knack.
"""
def __init__(self, method_name):
"""Override __init__ method for ScenarioTest.
Args:
method_name (str): ScenarioTest method_name.
"""
sb_cli = SuperBenchCLI.get_cli()
super().__init__(sb_cli, method_name)
def test_sb_version(self):
"""Test sb version."""
self.cmd('sb version', checks=[StringCheck(superbench.__version__)])
def test_sb_deploy(self):
"""Test sb deploy."""
self.cmd('sb deploy --docker-image test:cuda11.1 --host-list localhost', expect_failure=True)
@capture_system_exit
def test_sb_deploy_no_docker_image(self):
"""Test sb deploy, no --docker-image argument, should fail."""
self.cmd('sb deploy', expect_failure=True)
self.assertIn('sb deploy: error: the following arguments are required: --docker-image', self.stderr)
def test_sb_exec(self):
"""Test sb exec."""
self.cmd('sb exec --docker-image test:cuda11.1', checks=[NoneCheck()])
@capture_system_exit
def test_sb_exec_no_docker_image(self):
"""Test sb exec, no --docker-image argument, should fail."""
self.cmd('sb exec', expect_failure=True)
self.assertIn('sb exec: error: the following arguments are required: --docker-image', self.stderr)
def test_sb_run(self):
"""Test sb run."""
self.cmd('sb run --docker-image test:cuda11.1 --host-list localhost', checks=[NoneCheck()])
@capture_system_exit
def test_sb_run_no_docker_image(self):
"""Test sb run, no --docker-image argument, should fail."""
self.cmd('sb run', expect_failure=True)
self.assertIn('sb run: error: the following arguments are required: --docker-image', self.stderr)
def test_sb_run_no_host(self):
"""Test sb run, no --host-file or --host-list, should fail."""
result = self.cmd('sb run --docker-image test:cuda11.1', expect_failure=True)
self.assertEqual(result.exit_code, 1)
def test_sb_run_nonexist_host_file(self):
"""Test sb run, --host-file does not exist, should fail."""
result = self.cmd('sb run --docker-image test:cuda11.1 --host-file ./nonexist.yaml', expect_failure=True)
self.assertEqual(result.exit_code, 1)