diff --git a/Makefile b/Makefile index fd3dbc4fbe..a75a14989f 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ doc: conda-env doc-prereqs clean-doc conda run -n ${CONDA_ENV_NAME} make -j -C doc/ html test -s doc/build/html/index.html test -s doc/build/html/generated/mlos_core.optimizers.BaseOptimizer.html - test -s doc/build/html/generated/mlos_bench.main.optimize.html + test -s doc/build/html/generated/mlos_bench.run_opt.optimize.html test -s doc/build/html/api/mlos_core/mlos_core.html test -s doc/build/html/api/mlos_bench/mlos_bench.html cp doc/staticwebapp.config.json doc/build/html/ diff --git a/doc/source/overview.rst b/doc/source/overview.rst index 2ed5275666..70771848c8 100644 --- a/doc/source/overview.rst +++ b/doc/source/overview.rst @@ -39,13 +39,27 @@ This is a list of major functions and classes provided by `mlos_bench`. Main ==== -.. currentmodule:: mlos_bench.main + +:doc:`run_opt.py ` + + The main optimization loop script. + +.. currentmodule:: mlos_bench.run_opt .. autosummary:: :toctree: generated/ :template: functions.rst optimize +:doc:`run_bench.py ` + + A helper script for testing a single application/workload run. + +.. currentmodule:: mlos_bench.run_bench +.. autosummary:: + :toctree: generated/ + :template: function.rst + Optimizer ========= .. currentmodule:: mlos_bench.opt diff --git a/mlos_bench/README.md b/mlos_bench/README.md index 857999a5e2..7af3a55cc7 100644 --- a/mlos_bench/README.md +++ b/mlos_bench/README.md @@ -30,11 +30,11 @@ Installation instructions for `az` (Azure CLI) [can be found here](https://docs. az account get-access-token ``` -3. Make a copy of `services-example.json` so that we can adapt it with our own details. +3. Make a copy of `services.json` so that we can adapt it with our own details. For example, ```sh - cp config/azure/services-example.json config/azure/services-mine.json + cp config/azure/services.json ./services-mine.json ``` 4. Modify our new service configuration file with our target resource group information, as well as desired VM name, denoted in `{{ ... }}` below. @@ -63,11 +63,11 @@ For example, ] ``` -5. Make a copy of `config-example.json` so that we can adapt it with our own details. +5. Make a copy of `env-azure-ubuntu-redis.json` so that we can adapt it with our own details. For example, ```sh - cp config/config-example.json config/config-mine.json + cp config/env-azure-ubuntu-redis.json ./config-mine.json ``` 6. Modify our new configuration file with desired resource names, denoted in `{{ ... }}` below. @@ -82,7 +82,7 @@ For example, ], "include_services": [ - "{{ Path to your new service config, e.g. ./mlos_bench/config/azure/services-mine.json }}" + "{{ Path to your new service config, e.g. ./services-mine.json }}" ], "config": { @@ -123,11 +123,19 @@ Create and activate the environment with: conda activate mlos_core ``` -8. Run our configuration through `mlos_bench`. +8. Store the Azure security credentials in the `global.json` config file. + + ```sh + az account get-access-token > ./global.json + ``` + > You have to repeat that operation every hour or so to update the file with the new access token. + + +9. Run our configuration through `mlos_bench`. We can do so and pipe the output into log file `osat.log` as follows: ```sh - python mlos_bench/mlos_bench/main.py --config mlos_bench/config/config-mine.json --accessToken "$(az account get-access-token --query accessToken --output tsv)" 2>&1 > ./osat.log + python mlos_bench/mlos_bench/run_opt.py --config ./config-mine.json --global ./global.json 2>&1 | tee ./osat.log ``` -9. Check `osat.log` to verify we get output corresponding to the `ls -l /` command we remotely executed in the VM. +10. Check `osat.log` to verify we get output corresponding to the `ls -l /` command we remotely executed in the VM. diff --git a/mlos_bench/config/azure/azuredeploy-ubuntu-vm.json b/mlos_bench/config/azure/azuredeploy-ubuntu-vm.json index 787e6850fd..a4701b8062 100644 --- a/mlos_bench/config/azure/azuredeploy-ubuntu-vm.json +++ b/mlos_bench/config/azure/azuredeploy-ubuntu-vm.json @@ -4,7 +4,7 @@ "parameters": { "vmName": { "type": "string", - "defaultValue": "osat-linux-vm", + "defaultValue": "os-autotune-linux-vm", "metadata": { "description": "OS Autotune Linux VM" } @@ -68,21 +68,21 @@ }, "virtualNetworkName": { "type": "string", - "defaultValue": "osat-vnet", + "defaultValue": "os-autotune-vnet", "metadata": { "description": "Name of the VNET" } }, "subnetName": { "type": "string", - "defaultValue": "osat-subnet", + "defaultValue": "os-autotune-subnet", "metadata": { "description": "Name of the subnet in the virtual network" } }, "networkSecurityGroupName": { "type": "string", - "defaultValue": "osat-sg", + "defaultValue": "os-autotune-nsg", "metadata": { "description": "Name of the Network Security Group" } @@ -259,4 +259,4 @@ "value": "[format('ssh {0}@{1}', parameters('adminUsername'), reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))).dnsSettings.fqdn)]" } } -} \ No newline at end of file +} diff --git a/mlos_bench/config/azure/services.json b/mlos_bench/config/azure/services.json index dd6fe77720..1e64b9d4d7 100644 --- a/mlos_bench/config/azure/services.json +++ b/mlos_bench/config/azure/services.json @@ -5,9 +5,12 @@ "config": { "deployTemplatePath": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json", - "resourceGroup": "sergiym-os-autotune", - "deploymentName": "sergiym-os-autotune-001", - "vmName": "osat-linux-vm" + "subscription": "AZURE SUBSCRIPTION ID", + "accessToken": "AZURE ACCESS TOKEN, E.G. FROM `az account get-access-token`", + + "resourceGroup": "os-autotune", + "deploymentName": "os-autotune-001", + "vmName": "os-autotune-linux-vm" } } ] diff --git a/mlos_bench/config/config.json b/mlos_bench/config/env-azure-ubuntu-redis.json similarity index 64% rename from mlos_bench/config/config.json rename to mlos_bench/config/env-azure-ubuntu-redis.json index fd3ad1db51..37601b455b 100644 --- a/mlos_bench/config/config.json +++ b/mlos_bench/config/env-azure-ubuntu-redis.json @@ -23,13 +23,13 @@ "const_args": { - "adminUsername": "sergiym", + "adminUsername": "os-autotune", "authenticationType": "sshPublicKey", - "adminPasswordOrKey": "NOT USED", + "adminPasswordOrKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZnRBZZWYoPaAf4ZFP3rj1IKpdjW3QySgfOfrw6gUcqWN8OnUdxXzJJ35DxDhtZY3Qq038Y7IHfocrxrKL2ID/rrYvo77/uuqaEpELOXU9YA1ervTqlUagARSfeoUQDOZkDsbm603D4/3t3l0hxNHPGDyqjLtP2U+RelhLEngaSGSvwAiFNHJef8ldDfOxgBSVprBvU+48g4mZp5QZ2O50gzhcoFI2+Ho/Am4el2SS6ptxS1BjeVR1/GoLpB/POYfWqgs5/0rrG2gUyRQZdvY97LhMVd3jUK0DeiRnQpv44fnKTh8px3sl6N5sTg9a4tL8PBZC31MqP7QTj9KqMh8HfMp0wEIXPUWinBpTjXtknZiOWodLuAtZ82GzO8sW5HhXW5ZdU6aaNZ1JUakQX+eWTAzF3qQCp2BDIrXccxa5HMjneLczHug5VH1Y4wo3LyI4QBmxoQzYUsvSDBHAGqSu3u6XJVfJbgA5Di1ykGpVaMda0iLc7wG9ADevEfOmSQIMI2PEW6ZLv/+1w+uHYJi4c+3cIr9nSPpBu1HmefY4Hrz9XtYhFwTAk8RCla9v4WPn0hrVQnTBcuW3a79d87v6O2NIv1gosnsEWpWwIHycIe3G/Y6btsoDHcYMWbWewSOVsTru5UAfiZYZk8psjMwc4wcGkIHxcgzFlWb9r8ZOzw== os-autotune-fake-key", - "virtualNetworkName": "sergiym-osat-vnet", - "subnetName": "sergiym-osat-subnet", - "networkSecurityGroupName": "sergiym-osat-sg", + "virtualNetworkName": "os-autotune-vnet", + "subnetName": "os-autotune-subnet", + "networkSecurityGroupName": "os-autotune-nsg", "ubuntuOSVersion": "18.04-LTS" } diff --git a/mlos_bench/config/tunable-values-example.json b/mlos_bench/config/tunable-values-example.json new file mode 100644 index 0000000000..06b23ae321 --- /dev/null +++ b/mlos_bench/config/tunable-values-example.json @@ -0,0 +1,5 @@ +{ + "vmSize": "Standard_B2s", + "rootfs": "ext2", + "kernel.sched_migration_cost_ns": 40000 +} diff --git a/mlos_bench/mlos_bench/environment/base_environment.py b/mlos_bench/mlos_bench/environment/base_environment.py index 9a7d2d6551..c4fbcaa99a 100644 --- a/mlos_bench/mlos_bench/environment/base_environment.py +++ b/mlos_bench/mlos_bench/environment/base_environment.py @@ -179,7 +179,7 @@ class Environment(metaclass=abc.ABCMeta): True if operation is successful, false otherwise. """ - def submit(self, tunables): + def submit(self, tunables: TunableGroups): """ Submit a new experiment to the environment. Set up the environment, if necessary. @@ -195,6 +195,7 @@ class Environment(metaclass=abc.ABCMeta): True if operation is successful, false otherwise. """ _LOG.info("Submit: %s", tunables) + assert isinstance(tunables, TunableGroups) if self.setup(): return self.run(tunables) return False diff --git a/mlos_bench/mlos_bench/environment/composite.py b/mlos_bench/mlos_bench/environment/composite.py index 499a7ebc1c..6d0cfe6a59 100644 --- a/mlos_bench/mlos_bench/environment/composite.py +++ b/mlos_bench/mlos_bench/environment/composite.py @@ -40,11 +40,6 @@ class CompositeEnv(Environment): self._children = [] for child_config in config["children"]: - - local_config = shared_config.copy() - local_config.update(child_config.get("config", {})) - child_config["config"] = local_config - env = build_environment(child_config, shared_config, tunables, self._service) self._children.append(env) self._tunable_params.update(env.tunable_params()) diff --git a/mlos_bench/mlos_bench/environment/persistence.py b/mlos_bench/mlos_bench/environment/persistence.py index c7c5a9eeef..fe7afdc68a 100644 --- a/mlos_bench/mlos_bench/environment/persistence.py +++ b/mlos_bench/mlos_bench/environment/persistence.py @@ -45,12 +45,9 @@ def build_environment(config, global_config=None, tunables=None, service=None): env_name = config["name"] env_class = config["class"] - env_config = config.get("config", {}) - if global_config: - local_config = global_config.copy() - local_config.update(env_config) - env_config = local_config + env_config = config.setdefault("config", {}) + env_config.update(global_config or {}) env_services_path = config.get("include_services") if env_services_path is not None: @@ -86,12 +83,9 @@ def _build_standalone_service(config, global_config=None): An instance of the `Service` class initialized with `config`. """ svc_class = config["class"] - svc_config = config.get("config", {}) - if global_config: - local_config = global_config.copy() - local_config.update(svc_config) - svc_config = local_config + svc_config = config.setdefault("config", {}) + svc_config.update(global_config or {}) _LOG.debug("Creating service: %s", svc_class) service = Service.new(svc_class, svc_config) diff --git a/mlos_bench/mlos_bench/launcher.py b/mlos_bench/mlos_bench/launcher.py new file mode 100644 index 0000000000..1531e75369 --- /dev/null +++ b/mlos_bench/mlos_bench/launcher.py @@ -0,0 +1,105 @@ +""" +Helper functions to launch the benchmark and the optimizer from the command line. +""" + +import json +import logging +import argparse + +from mlos_bench.environment.persistence import load_environment + + +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s', + datefmt='%H:%M:%S' +) + +_LOG = logging.getLogger(__name__) + + +class Launcher: + """ + Common parts of the OS Autotune command line launcher. + """ + + def __init__(self, description='OS Autotune launcher'): + + _LOG.info("Launch: %s", description) + + self._env_config_file = None + self._global_config = {} + self._parser = argparse.ArgumentParser(description=description) + + self._parser.add_argument( + '--config', required=True, + help='Path to JSON file with the configuration' + ' of the benchmarking environment') + + self._parser.add_argument( + '--global', required=False, dest='global_config', + help='Path to JSON file that contains additional (sensitive)' + ' parameters of the benchmarking environment') + + @property + def parser(self): + """ + Get the command line parser (so we can add more arguments to it). + """ + return self._parser + + def parse_args(self): + """ + Parse command line arguments and load global config parameters. + """ + (args, args_rest) = self._parser.parse_known_args() + self._env_config_file = args.config + if args.global_config is not None: + self._global_config = Launcher.load_config(args.global_config) + self._global_config.update(Launcher._try_parse_extra_args(args_rest)) + return args + + @staticmethod + def load_config(json_file_name): + """ + Load JSON config file. + """ + with open(json_file_name, mode='r', encoding='utf-8') as fh_json: + return json.load(fh_json) + + @staticmethod + def _try_parse_extra_args(cmdline): + """ + Helper function to parse global key/value pairs from the command line. + """ + _LOG.debug("Extra args: %s", cmdline) + + config = {} + key = None + for elem in cmdline: + if elem.startswith("--"): + if key is not None: + raise ValueError("Command line argument has no value: " + key) + key = elem[2:] + kv_split = key.split("=", 1) + if len(kv_split) == 2: + config[kv_split[0].strip()] = kv_split[1] + key = None + else: + if key is None: + raise ValueError("Command line argument has no key: " + elem) + config[key.strip()] = elem + key = None + + if key is not None: + raise ValueError("Command line argument has no value: " + key) + + _LOG.debug("Parsed config: %s", config) + return config + + def load_env(self): + """ + Create a new benchmarking environment + from the configs and command line parameters. + """ + return load_environment(self._env_config_file, self._global_config) diff --git a/mlos_bench/mlos_bench/main.py b/mlos_bench/mlos_bench/main.py deleted file mode 100755 index a186917f44..0000000000 --- a/mlos_bench/mlos_bench/main.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 - -""" -OS Autotune main optimization loop. -""" - -import json -import logging -import argparse - -from mlos_bench.opt import Optimizer -from mlos_bench.environment.persistence import load_environment - - -def optimize(env_config_file, global_config): - """ - Main optimization loop. - """ - env = load_environment(env_config_file, global_config) - - opt = Optimizer(env.tunable_params()) - _LOG.info("Env: %s Optimizer: %s", env, opt) - - while opt.not_converged(): - - tunable_values = opt.suggest() - _LOG.info("Suggestion: %s", tunable_values) - env.submit(tunable_values) - - bench_result = env.result() # Block and wait for the final result - _LOG.info("Result: %s = %s", tunable_values, bench_result) - opt.register(tunable_values, bench_result) - - env.teardown() - - best = opt.get_best_observation() - _LOG.info("Env: %s best result: %s", env, best) - return best - -############################################################### - - -def _try_parse_extra_args(cmdline): - """ - Helper function to parse global key/value pairs from the command line. - """ - _LOG.debug("Extra args: %s", cmdline) - - config = {} - key = None - for elem in cmdline: - if elem.startswith("--"): - if key is not None: - raise ValueError("Command line argument has no value: " + key) - key = elem[2:] - kv_split = key.split("=", 1) - if len(kv_split) == 2: - config[kv_split[0].strip()] = kv_split[1] - key = None - else: - if key is None: - raise ValueError("Command line argument has no key: " + elem) - config[key.strip()] = elem - key = None - - if key is not None: - raise ValueError("Command line argument has no value: " + key) - - _LOG.debug("Parsed config: %s", config) - return config - - -def _main(): - - parser = argparse.ArgumentParser( - description='OS Autotune optimizer') - - parser.add_argument( - '--config', required=True, - help='Path to JSON file with the configuration' - ' of the benchmarking environment') - - parser.add_argument( - '--global', required=False, dest='global_config', - help='Path to JSON file that contains additional (sensitive)' - ' parameters of the benchmarking environment') - - (args, args_rest) = parser.parse_known_args() - - global_config = {} - if args.global_config is not None: - with open(args.global_config, encoding='utf-8') as fh_json: - global_config = json.load(fh_json) - - global_config.update(_try_parse_extra_args(args_rest)) - - result = optimize(args.config, global_config) - _LOG.info("Final result: %s", result) - -############################################################### - - -logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s %(pathname)s:%(lineno)d %(levelname)s %(message)s', - datefmt='%H:%M:%S' -) - -_LOG = logging.getLogger(__name__) - -if __name__ == "__main__": - _main() diff --git a/mlos_bench/mlos_bench/run_bench.py b/mlos_bench/mlos_bench/run_bench.py new file mode 100755 index 0000000000..05c1278303 --- /dev/null +++ b/mlos_bench/mlos_bench/run_bench.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +""" +OS Autotune benchmark launcher without involving the optimizer. +Used mostly for development/testing purposes. + +See `--help` output for details. +""" + +import logging + +from mlos_bench.launcher import Launcher + +_LOG = logging.getLogger(__name__) + + +def _main(): + + launcher = Launcher("OS Autotune benchmark") + + launcher.parser.add_argument( + '--tunables', required=True, + help='Path to JSON file that contains values of the tunable parameters') + + args = launcher.parse_args() + + tunable_values = Launcher.load_config(args.tunables) + env = launcher.load_env() + + tunables = env.tunable_params() + for (key, val) in tunable_values.items(): + tunables[key] = val + + _LOG.info("Benchmark: %s with tunables:\n%s", env, tunables) + env.submit(tunables) + + bench_result = env.result() # Block and wait for the final result + _LOG.info("Result: %s", bench_result) + + +if __name__ == "__main__": + _main() diff --git a/mlos_bench/mlos_bench/run_opt.py b/mlos_bench/mlos_bench/run_opt.py new file mode 100755 index 0000000000..cf311f0367 --- /dev/null +++ b/mlos_bench/mlos_bench/run_opt.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +OS Autotune main optimization loop. + +See `--help` output for details. +""" + +import logging + +from mlos_bench.opt import Optimizer +from mlos_bench.launcher import Launcher + +_LOG = logging.getLogger(__name__) + + +def _main(): + launcher = Launcher("OS Autotune optimizer") + launcher.parse_args() + env = launcher.load_env() + result = optimize(env) + _LOG.info("Final result: %s", result) + + +def optimize(env): + """ + Main optimization loop. + """ + opt = Optimizer(env.tunable_params()) + _LOG.info("Env: %s Optimizer: %s", env, opt) + + while opt.not_converged(): + + tunable_values = opt.suggest() + _LOG.info("Suggestion: %s", tunable_values) + env.submit(tunable_values) + + bench_result = env.result() # Block and wait for the final result + _LOG.info("Result: %s = %s", tunable_values, bench_result) + opt.register(tunable_values, bench_result) + + best = opt.get_best_observation() + _LOG.info("Env: %s best result: %s", env, best) + return best + + +if __name__ == "__main__": + _main()