From 552cd4966608a80547b189153498bc3fa6684c65 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Fri, 19 Aug 2022 22:59:37 +0000 Subject: [PATCH] Merged PR 571: Read tunables from a separate config * Store tunables' definitions in a separate config * Implement the `Tunable` wrapper class * Implement tunable groups and collections of covariant parameters Related work items: #457, #458 --- mlos_bench/config/azure/.gitignore | 1 + mlos_bench/config/azure/services-example.json | 16 ++ mlos_bench/config/config-example.json | 114 -------- mlos_bench/config/config.json | 78 ++++++ mlos_bench/config/tunables.json | 44 +++ mlos_bench/mlos_bench/environment/__init__.py | 3 + mlos_bench/mlos_bench/environment/app.py | 13 +- .../mlos_bench/environment/azure/azure_os.py | 5 +- .../mlos_bench/environment/azure/azure_vm.py | 9 +- .../environment/base_environment.py | 88 ++---- .../mlos_bench/environment/base_service.py | 54 +--- .../mlos_bench/environment/composite.py | 29 +- .../mlos_bench/environment/persistence.py | 265 ++++++++++++++++++ mlos_bench/mlos_bench/environment/status.py | 9 +- mlos_bench/mlos_bench/environment/tunable.py | 261 +++++++++++++++++ mlos_bench/mlos_bench/main.py | 27 +- mlos_bench/mlos_bench/opt.py | 5 +- 17 files changed, 755 insertions(+), 266 deletions(-) create mode 100644 mlos_bench/config/azure/.gitignore create mode 100644 mlos_bench/config/azure/services-example.json delete mode 100644 mlos_bench/config/config-example.json create mode 100644 mlos_bench/config/config.json create mode 100644 mlos_bench/config/tunables.json create mode 100644 mlos_bench/mlos_bench/environment/persistence.py create mode 100644 mlos_bench/mlos_bench/environment/tunable.py mode change 100644 => 100755 mlos_bench/mlos_bench/main.py diff --git a/mlos_bench/config/azure/.gitignore b/mlos_bench/config/azure/.gitignore new file mode 100644 index 0000000000..a373fb4b28 --- /dev/null +++ b/mlos_bench/config/azure/.gitignore @@ -0,0 +1 @@ +services.json diff --git a/mlos_bench/config/azure/services-example.json b/mlos_bench/config/azure/services-example.json new file mode 100644 index 0000000000..cbf5c3f153 --- /dev/null +++ b/mlos_bench/config/azure/services-example.json @@ -0,0 +1,16 @@ +[ + { + "class": "mlos_bench.environment.azure.AzureVMService", + + "config": { + "deploy_template_path": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json", + + "subscription": "...", + "resource_group": "sergiym-os-autotune", + "deployment_name": "sergiym-os-autotune-001", + "vmName": "osat-linux-vm", + + "accessToken": "AZURE ACCESS TOKEN (e.g., from `az account get-access-token`)" + } + } +] diff --git a/mlos_bench/config/config-example.json b/mlos_bench/config/config-example.json deleted file mode 100644 index 32c6b4c8aa..0000000000 --- a/mlos_bench/config/config-example.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "name": "Azure VM Ubuntu Redis", - "class": "mlos_bench.environment.CompositeEnv", - - "config": { - - "services": [ - { - "class": "mlos_bench.environment.azure.AzureVMService", - - "config": { - "deploy_template_path": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json", - - "subscription": "...", - "resource_group": "sergiym-os-autotune", - "deployment_name": "sergiym-os-autotune-001", - "vmName": "osat-linux-vm", - - "accessToken": "AZURE ACCESS TOKEN (e.g., from `az account get-access-token`)" - } - } - ], - - "children": [ - { - "name": "Deploy Ubuntu VM on Azure", - "class": "mlos_bench.environment.azure.VMEnv", - - "config": { - - "cost": 1000, - - "tunable_params": { - "vmSize": { - "type": "categorical", - "default": "Standard_B4ms", - "values": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"] - } - }, - - "const_args": { - - "adminUsername": "sergiym", - "authenticationType": "sshPublicKey", - "adminPasswordOrKey": "SSH PUBLIC KEY (e.g., from id_rsa.pub)", - - "virtualNetworkName": "sergiym-osat-vnet", - "subnetName": "sergiym-osat-subnet", - "networkSecurityGroupName": "sergiym-osat-sg", - - "ubuntuOSVersion": "18.04-LTS" - } - } - }, - { - "name": "Boot Ubuntu VM on Azure", - "class": "mlos_bench.environment.azure.OSEnv", - - "config": { - - "cost": 300, - - "tunable_params": { - "rootfs": { - "type": "categorical", - "default": "xfs", - "values": ["xfs", "ext4", "ext2"] - } - }, - - "const_args": { - } - } - }, - { - "name": "Redis on Linux", - "class": "mlos_bench.environment.AppEnv", - - "config": { - - "pollDelay": 10, - - "cost": 1, - - "tunable_params": { - "kernel.sched_migration_cost_ns": { - "type": "int", - "default": -1, - "range": [0, 500000], - "special": [-1] - } - }, - - "const_args": { - "commandId": "RunShellScript", - "script": [ - "ls -l /" - ], - "parameters": [ - { - "name": "param1", - "value": "111" - }, - { - "name": "param2", - "value": "222" - } - ] - } - } - } - ] - } -} diff --git a/mlos_bench/config/config.json b/mlos_bench/config/config.json new file mode 100644 index 0000000000..fd3ad1db51 --- /dev/null +++ b/mlos_bench/config/config.json @@ -0,0 +1,78 @@ +{ + "name": "Azure VM Ubuntu Redis", + "class": "mlos_bench.environment.CompositeEnv", + + "include_tunables": [ + "./mlos_bench/config/tunables.json" + ], + + "include_services": [ + "./mlos_bench/config/azure/services.json" + ], + + "config": { + + "children": [ + { + "name": "Deploy Ubuntu VM on Azure", + "class": "mlos_bench.environment.azure.VMEnv", + + "config": { + + "tunable_params": ["provision"], + + "const_args": { + + "adminUsername": "sergiym", + "authenticationType": "sshPublicKey", + "adminPasswordOrKey": "NOT USED", + + "virtualNetworkName": "sergiym-osat-vnet", + "subnetName": "sergiym-osat-subnet", + "networkSecurityGroupName": "sergiym-osat-sg", + + "ubuntuOSVersion": "18.04-LTS" + } + } + }, + { + "name": "Boot Ubuntu VM on Azure", + "class": "mlos_bench.environment.azure.OSEnv", + + "config": { + "tunable_params": ["boot"], + "const_args": {} + } + }, + { + "name": "Redis on Linux", + "class": "mlos_bench.environment.AppEnv", + + "config": { + + "pollDelay": 10, + + "tunable_params": ["kernel", "redis"], + + "const_args": { + "commandId": "RunShellScript", + "script": [ + "echo $param1 $param2", + "sudo ls -l /" + ], + "parameters": [ + { + "name": "param1", + "value": "111" + }, + { + "name": "param2", + "value": "222" + } + ] + } + } + } + ] + } +} diff --git a/mlos_bench/config/tunables.json b/mlos_bench/config/tunables.json new file mode 100644 index 0000000000..5beab05178 --- /dev/null +++ b/mlos_bench/config/tunables.json @@ -0,0 +1,44 @@ +{ + "provision": { + "cost": 1000, + "params": { + "vmSize": { + "description": "Azure VM size", + "type": "categorical", + "default": "Standard_B4ms", + "values": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"] + } + } + }, + + "boot": { + "cost": 300, + "params": { + "rootfs": { + "description": "Root file system", + "type": "categorical", + "default": "xfs", + "values": ["xfs", "ext4", "ext2"] + } + } + }, + + "kernel": { + "cost": 1, + "params": { + "kernel.sched_migration_cost_ns": { + "description": "Cost of migrating the thread to another core", + "type": "int", + "default": -1, + "range": [0, 500000], + "special": [-1] + } + } + }, + + "redis": { + "cost": 1, + "params": { + } + } +} diff --git a/mlos_bench/mlos_bench/environment/__init__.py b/mlos_bench/mlos_bench/environment/__init__.py index 6bdcd9b01a..b6d0e7858f 100644 --- a/mlos_bench/mlos_bench/environment/__init__.py +++ b/mlos_bench/mlos_bench/environment/__init__.py @@ -3,6 +3,7 @@ Benchmarking environments for OS Autotune. """ from mlos_bench.environment.status import Status +from mlos_bench.environment.tunable import Tunable, TunableGroups from mlos_bench.environment.base_service import Service from mlos_bench.environment.base_environment import Environment @@ -12,6 +13,8 @@ from mlos_bench.environment.composite import CompositeEnv __all__ = [ 'Status', + 'Tunable', + 'TunableGroups', 'Service', 'Environment', 'AppEnv', diff --git a/mlos_bench/mlos_bench/environment/app.py b/mlos_bench/mlos_bench/environment/app.py index 31826c397f..a0b2de8521 100644 --- a/mlos_bench/mlos_bench/environment/app.py +++ b/mlos_bench/mlos_bench/environment/app.py @@ -19,7 +19,7 @@ class AppEnv(Environment): _POLL_DELAY = 5 # Default polling interval in seconds. - def __init__(self, name, config, service=None): + def __init__(self, name, config, tunables, service=None): """ Create a new application environment with a given config. @@ -32,11 +32,13 @@ class AppEnv(Environment): configuration. Each config must have at least the "tunable_params" and the "const_args" sections; the "cost" field can be omitted and is 0 by default. + tunables : TunableGroups + A collection of tunable parameters for *all* environments. service: Service An optional service object (e.g., providing methods to deploy or reboot a VM, etc.). """ - super().__init__(name, config, service) + super().__init__(name, config, tunables, service) self._poll_delay = self.config.get("pollDelay", AppEnv._POLL_DELAY) def setup(self): @@ -59,10 +61,9 @@ class AppEnv(Environment): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) of the OS and application - parameters. Setting these parameters should not require an - OS reboot. + tunables : TunableGroups + A collection of tunable OS and application parameters along with their + values. Setting these parameters should not require an OS reboot. Returns ------- diff --git a/mlos_bench/mlos_bench/environment/azure/azure_os.py b/mlos_bench/mlos_bench/environment/azure/azure_os.py index 40d9db54fe..0eb4ea347d 100644 --- a/mlos_bench/mlos_bench/environment/azure/azure_os.py +++ b/mlos_bench/mlos_bench/environment/azure/azure_os.py @@ -45,8 +45,9 @@ class OSEnv(Environment): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) of the OS boot-time parameters. + tunables : TunableGroups + A collection of groups of tunable parameters + along with the parameters' values. Returns ------- diff --git a/mlos_bench/mlos_bench/environment/azure/azure_vm.py b/mlos_bench/mlos_bench/environment/azure/azure_vm.py index 97864e1a18..16ca52e5c6 100644 --- a/mlos_bench/mlos_bench/environment/azure/azure_vm.py +++ b/mlos_bench/mlos_bench/environment/azure/azure_vm.py @@ -45,10 +45,11 @@ class VMEnv(Environment): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) pairs of tunable parameters. - VMEnv tunables are variable parameters that, together with the - VMEnv configuration, are sufficient to provision and start a VM. + tunables : TunableGroups + A collection of groups of tunable parameters along with the + parameters' values. VMEnv tunables are variable parameters that, + together with the VMEnv configuration, are sufficient to provision + and start a VM. Returns ------- diff --git a/mlos_bench/mlos_bench/environment/base_environment.py b/mlos_bench/mlos_bench/environment/base_environment.py index bf8ee6a552..37a2de5b13 100644 --- a/mlos_bench/mlos_bench/environment/base_environment.py +++ b/mlos_bench/mlos_bench/environment/base_environment.py @@ -8,6 +8,7 @@ import logging import importlib from mlos_bench.environment.status import Status +from mlos_bench.environment.tunable import TunableGroups _LOG = logging.getLogger(__name__) @@ -17,37 +18,8 @@ class Environment(metaclass=abc.ABCMeta): An abstract base of all benchmark environments. """ - @staticmethod - def from_config(config, service=None): - """ - Factory method for a new environment with a given config. - - Parameters - ---------- - config : dict - A dictionary with three mandatory fields: - "name": Human-readable string describing the environment; - "class": FQN of a Python class to instantiate; - "config": Free-format dictionary to pass to the constructor. - service: Service - An optional service object (e.g., providing methods to - deploy or reboot a VM, etc.). - - Returns - ------- - env : Environment - An instance of the `Environment` class initialized with `config`. - """ - env_name = config["name"] - env_class = config["class"] - env_config = config["config"] - _LOG.debug("Creating env: %s :: %s", env_name, env_class) - env = Environment.new(env_name, env_class, env_config, service) - _LOG.info("Created env: %s :: %s", env_name, env) - return env - @classmethod - def new(cls, env_name, class_name, config, service=None): + def new(cls, env_name, class_name, config, tunables=None, service=None): """ Factory method for a new environment with a given config. @@ -63,6 +35,8 @@ class Environment(metaclass=abc.ABCMeta): Free-format dictionary that contains the benchmark environment configuration. It will be passed as a constructor parameter of the class specified by `name`. + tunables : TunableGroups + A collection of groups of tunable parameters for all environments. service: Service An optional service object (e.g., providing methods to deploy or reboot a VM, etc.). @@ -85,9 +59,9 @@ class Environment(metaclass=abc.ABCMeta): env_name, class_name, env_class) assert issubclass(env_class, cls) - return env_class(env_name, config, service) + return env_class(env_name, config, tunables, service) - def __init__(self, name, config, service=None): + def __init__(self, name, config, tunables=None, service=None): """ Create a new environment with a given config. @@ -100,6 +74,8 @@ class Environment(metaclass=abc.ABCMeta): configuration. Each config must have at least the "tunable_params" and the "const_args" sections; the "cost" field can be omitted and is 0 by default. + tunables : TunableGroups + A collection of groups of tunable parameters for all environments. service: Service An optional service object (e.g., providing methods to deploy or reboot a VM, etc.). @@ -110,8 +86,14 @@ class Environment(metaclass=abc.ABCMeta): self._result = (Status.PENDING, None) self._const_args = config.get("const_args", {}) - self._tunable_params = self._parse_tunables( - config.get("tunable_params", {}), config.get("cost", 0)) + + if tunables is None: + tunables = TunableGroups() + + tunable_groups = config.get("tunable_params") + self._tunable_params = ( + tunables.subgroup(tunable_groups) if tunable_groups else tunables + ) if _LOG.isEnabledFor(logging.DEBUG): _LOG.debug("Config for: %s\n%s", @@ -126,36 +108,26 @@ class Environment(metaclass=abc.ABCMeta): env_name=self.name ) - def _parse_tunables(self, tunables, cost=0): - "Augment tunables with the cost." - tunables_cost = {} - for (key, val) in tunables.items(): - tunables_cost[key] = val.copy() - tunables_cost[key]["cost"] = cost - return tunables_cost - def _combine_tunables(self, tunables): """ - Plug tunable values into the base config. If the tunable is unknown, + Plug tunable values into the base config. If the tunable group is unknown, ignore it (it might belong to another environment). This method should never mutate the original config or the tunables. Parameters ---------- - tunables : dict - Flat dictionary of (key, value) pairs of tunable parameters. + tunables : TunableGroups + A collection of groups of tunable parameters + along with the parameters' values. Returns ------- config : dict - Free-format dictionary that contains the new environment - configuration. + Free-format dictionary that contains the new environment configuration. """ - new_config = self._const_args.copy() - for (key, val) in tunables.items(): - if key in self._tunable_params: - new_config[key] = val - return new_config + return tunables.get_param_values( + group_names=self._tunable_params.get_names(), + into_params=self._const_args.copy()) def tunable_params(self): """ @@ -163,8 +135,8 @@ class Environment(metaclass=abc.ABCMeta): Returns ------- - tunables : dict - Flat dictionary of (key, value) pairs of tunable parameters. + tunables : TunableGroups + A collection of covariant groups of tunable parameters. """ return self._tunable_params @@ -201,8 +173,8 @@ class Environment(metaclass=abc.ABCMeta): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) pairs of tunable parameters. + tunables : TunableGroups + A collection of tunable parameters along with their values. Returns ------- @@ -217,8 +189,8 @@ class Environment(metaclass=abc.ABCMeta): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) pairs of tunable parameters. + tunables : TunableGroups + A collection of tunable parameters along with their values. Returns ------- diff --git a/mlos_bench/mlos_bench/environment/base_service.py b/mlos_bench/mlos_bench/environment/base_service.py index 3a2804a744..bc1a4376e4 100644 --- a/mlos_bench/mlos_bench/environment/base_service.py +++ b/mlos_bench/mlos_bench/environment/base_service.py @@ -14,58 +14,6 @@ class Service: An abstract base of all environment services. """ - @staticmethod - def from_config(config): - """ - Factory method for a new service with a given config. - - Parameters - ---------- - config : dict - A dictionary with two mandatory fields: - "class": FQN of a Python class to instantiate; - "config": Free-format dictionary to pass to the constructor. - - Returns - ------- - svc : Service - An instance of the `Service` class initialized with `config`. - """ - svc_class = config["class"] - svc_config = config["config"] - _LOG.debug("Creating service: %s", svc_class) - service = Service.new(svc_class, svc_config) - _LOG.info("Created service: %s", service) - return service - - @staticmethod - def from_config_list(config_list, parent=None): - """ - Factory method for a new service with a given config. - - Parameters - ---------- - config_list : a list of dict - A list where each element is a dictionary with 2 mandatory fields: - "class": FQN of a Python class to instantiate; - "config": Free-format dictionary to pass to the constructor. - parent: Service - An optional reference of the parent service to mix in. - - Returns - ------- - svc : Service - An instance of the `Service` class that is a combination of all - services from the list plus the parent mix-in. - """ - service = Service() - if parent: - service.register(parent.export()) - for config in config_list: - service.register(Service.from_config(config).export()) - _LOG.info("Created mix-in service: %s", service.export()) - return service - @classmethod def new(cls, class_name, config): """ @@ -115,7 +63,7 @@ class Service: self._services = {} if _LOG.isEnabledFor(logging.DEBUG): - _LOG.debug("Config:\n%s", json.dumps(self.config, indent=2)) + _LOG.debug("Service config:\n%s", json.dumps(self.config, indent=2)) def register(self, services): """ diff --git a/mlos_bench/mlos_bench/environment/composite.py b/mlos_bench/mlos_bench/environment/composite.py index ba3d8dab25..a4926cc0b9 100644 --- a/mlos_bench/mlos_bench/environment/composite.py +++ b/mlos_bench/mlos_bench/environment/composite.py @@ -4,8 +4,8 @@ Composite benchmark environment. import logging -from mlos_bench.environment.base_service import Service from mlos_bench.environment.base_environment import Environment +from mlos_bench.environment.persistence import build_environment _LOG = logging.getLogger(__name__) @@ -15,7 +15,7 @@ class CompositeEnv(Environment): Composite benchmark environment. """ - def __init__(self, name, config, service=None): + def __init__(self, name, config, tunables, service=None): """ Create a new environment with a given config. @@ -26,25 +26,26 @@ class CompositeEnv(Environment): config : dict Free-format dictionary that contains the environment configuration. Must have a "children" section. + tunables : TunableGroups + A collection of groups of tunable parameters for *all* environments. service: Service An optional service object (e.g., providing methods to deploy or reboot a VM, etc.). """ - super().__init__(name, config, service) + super().__init__(name, config, tunables, service) - # Propagate all config parameters except "children" and "services" - # to every child config. + # Propagate all config parameters except "children" to every child config. shared_config = config.copy() del shared_config["children"] - del shared_config["services"] - - self._service = Service.from_config_list( - config.get("services", []), parent=service) self._children = [] for child_config in config["children"]: - child_config["config"].update(shared_config) - env = Environment.from_config(child_config, self._service) + + local_config = shared_config.copy() + local_config.update(child_config.get("config", {})) + child_config["config"] = local_config + + env = build_environment(child_config, shared_config, tunables, self._service) self._children.append(env) self._tunable_params.update(env.tunable_params()) @@ -80,9 +81,9 @@ class CompositeEnv(Environment): Parameters ---------- - tunables : dict - Flat dictionary of (key, value) of the parameters from all - children environments. + tunables : TunableGroups + A collection of groups of tunable parameters and their values + from all children environments. Returns ------- diff --git a/mlos_bench/mlos_bench/environment/persistence.py b/mlos_bench/mlos_bench/environment/persistence.py new file mode 100644 index 0000000000..89d8f26438 --- /dev/null +++ b/mlos_bench/mlos_bench/environment/persistence.py @@ -0,0 +1,265 @@ +""" +Helper functions to load, instantiate, and serialize Python objects +that encapsulate benchmark environments, tunable parameters, and +service functions. +""" + +import json +import logging + +from mlos_bench.environment.tunable import TunableGroups +from mlos_bench.environment.base_service import Service +from mlos_bench.environment.base_environment import Environment + +_LOG = logging.getLogger(__name__) + + +def build_environment(config, global_config=None, tunables=None, service=None): + """ + Factory method for a new environment with a given config. + + Parameters + ---------- + config : dict + A dictionary with three mandatory fields: + "name": Human-readable string describing the environment; + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + global_config : dict + Global parameters to add to the environment config. + tunables : TunableGroups + A collection of groups of tunable parameters for all environments. + service: Service + An optional service object (e.g., providing methods to + deploy or reboot a VM, etc.). + + Returns + ------- + env : Environment + An instance of the `Environment` class initialized with `config`. + """ + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Build environment from config:\n%s", + json.dumps(config, indent=2)) + + env_name = config["name"] + env_class = config["class"] + env_config = config.get("config", {}) + + if global_config: + local_config = global_config.copy() + local_config.update(env_config) + env_config = local_config + + env_services_path = config.get("include_services") + if env_services_path is not None: + service = load_services(env_services_path, global_config, service) + + env_tunables_path = config.get("include_tunables") + if env_tunables_path is not None: + tunables = load_tunables(env_tunables_path, tunables) + + _LOG.debug("Creating env: %s :: %s", env_name, env_class) + env = Environment.new(env_name, env_class, env_config, tunables, service) + + _LOG.info("Created env: %s :: %s", env_name, env) + return env + + +def _build_standalone_service(config, global_config=None): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + config : dict + A dictionary with two mandatory fields: + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + global_config : dict + Global parameters to add to the service config. + + Returns + ------- + svc : Service + An instance of the `Service` class initialized with `config`. + """ + svc_class = config["class"] + svc_config = config.get("config", {}) + + if global_config: + local_config = global_config.copy() + local_config.update(svc_config) + svc_config = local_config + + _LOG.debug("Creating service: %s", svc_class) + service = Service.new(svc_class, svc_config) + + _LOG.info("Created service: %s", service) + return service + + +def _build_composite_service(config_list, global_config=None, parent=None): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + config_list : a list of dict + A list where each element is a dictionary with 2 mandatory fields: + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + global_config : dict + Global parameters to add to the service config. + parent: Service + An optional reference of the parent service to mix in. + + Returns + ------- + svc : Service + An instance of the `Service` class that is a combination of all + services from the list plus the parent mix-in. + """ + service = Service() + if parent: + service.register(parent.export()) + for config in config_list: + service.register(_build_standalone_service(config, global_config).export()) + _LOG.info("Created mix-in service: %s", service.export()) + return service + + +def build_service(config, global_config=None, parent=None): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + config : [dict] or dict + A list where each element is a dictionary with 2 mandatory fields: + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + global_config : dict + Global parameters to add to the service config. + parent: Service + An optional reference of the parent service to mix in. + + Returns + ------- + svc : Service + An instance of the `Service` class that is a combination of all + services from the list plus the parent mix-in. + """ + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Build service from config:\n%s", + json.dumps(config, indent=2)) + + if isinstance(config, dict): + if parent is None: + return _build_standalone_service(config, global_config) + config = [config] + + return _build_composite_service(config, global_config, parent) + + +def build_tunables(config, parent=None): + """ + Create a new collection of tunable parameters. + + Parameters + ---------- + config : dict + Python dict of serialized representation of the covariant tunable groups. + parent : TunableGroups + An optional collection of tunables to add to the new collection. + + Returns + ------- + tunables : TunableGroup + Create a new collection of tunable parameters. + """ + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Build tunables from config:\n%s", + json.dumps(config, indent=2)) + + if parent is None: + return TunableGroups(config) + groups = TunableGroups() + groups.update(parent) + groups.update(TunableGroups(config)) + return groups + + +def load_environment(json_file_name, global_config=None, + tunables=None, service=None): + """ + Create a new collection of tunable parameters. + + Parameters + ---------- + json_file_name : str + The environment JSON configuration file. + global_config : dict + Global parameters to add to the environment config. + tunables : TunableGroups + An optional collection of tunables to add to the new collection. + service : Service + An optional reference of the parent service to mix in. + """ + _LOG.info("Load environment: %s", json_file_name) + with open(json_file_name) as fh_json: + config = json.load(fh_json) + return build_environment(config, global_config, tunables, service) + + +def load_services(json_file_names, global_config=None, parent=None): + """ + Create a new collection of tunable parameters. + + Parameters + ---------- + json_file_name : str + The service JSON configuration file. + global_config : dict + Global parameters to add to the service config. + parent : Service + An optional reference of the parent service to mix in. + + Returns + ------- + tunables : TunableGroup + Create a new collection of tunable parameters. + """ + _LOG.info("Load services: %s", json_file_names) + service = Service(global_config) + if parent: + service.register(parent.export()) + for fname in json_file_names: + _LOG.debug("Load services: %s", fname) + with open(fname) as fh_json: + config = json.load(fh_json) + service.register(build_service(config, global_config).export()) + return service + + +def load_tunables(json_file_names, parent=None): + """ + Load a collection of tunable parameters from JSON files. + + Parameters + ---------- + json_file_names : [str] + A list of JSON files to load. + parent : TunableGroups + An optional collection of tunables to add to the new collection. + """ + _LOG.info("Load tunables: %s", json_file_names) + groups = TunableGroups() + if parent is not None: + groups.update(parent) + for fname in json_file_names: + _LOG.debug("Load tunables: %s", fname) + with open(fname) as fh_json: + config = json.load(fh_json) + groups.update(TunableGroups(config)) + return groups diff --git a/mlos_bench/mlos_bench/environment/status.py b/mlos_bench/mlos_bench/environment/status.py index f7f2ca2b0c..71f0a18f70 100644 --- a/mlos_bench/mlos_bench/environment/status.py +++ b/mlos_bench/mlos_bench/environment/status.py @@ -21,6 +21,11 @@ class Status(enum.Enum): @staticmethod def is_good(status): """ - Check if the status is not failed or canceled. + Check if the status of the environment is good. """ - return status not in {Status.CANCELED, Status.FAILED, Status.TIMED_OUT} + return status in { + Status.PENDING, + Status.READY, + Status.RUNNING, + Status.SUCCEEDED, + } diff --git a/mlos_bench/mlos_bench/environment/tunable.py b/mlos_bench/mlos_bench/environment/tunable.py new file mode 100644 index 0000000000..be402f7af5 --- /dev/null +++ b/mlos_bench/mlos_bench/environment/tunable.py @@ -0,0 +1,261 @@ +""" +Tunable parameter definition. +""" +import collections + + +class Tunable: + """ + A tunable parameter definition and its current value. + """ + + def __init__(self, name, config): + """ + Create an instance of a new tunable parameter. + + Parameters + ---------- + name : str + Human-readable identifier of the tunable parameter. + config : dict + Python dict that represents a Tunable (e.g., deserialized from JSON) + """ + self._name = name + self._description = config.get("description") + self._type = config["type"] + self._default = config.get("default") + self._values = config.get("values") + self._range = config.get("range") + self._special = config.get("special") + self._current_value = self._default + if self._type == "categorical": + if not (self._values and isinstance(self._values, collections.abc.Iterable)): + raise ValueError("Must specify values for the categorical type") + if self._range is not None: + raise ValueError("Range must be None for the categorical type") + if self._special is not None: + raise ValueError("Special values must be None for the categorical type") + elif self._type in {"int", "float"}: + if not self._range or len(self._range) != 2 or self._range[0] >= self._range[1]: + raise ValueError("Invalid range: " + self._range) + else: + raise ValueError("Invalid parameter type: " + self._type) + + def __repr__(self): + return "{name}={value}".format(name=self._name, value=self._current_value) + + @property + def value(self): + """ + Get the current value of the tunable. + """ + return self._current_value + + @value.setter + def value(self, value): + """ + Set the current value of the tunable. + """ + self._current_value = value + return value + + +class CovariantTunableGroup: + """ + A collection of tunable parameters. + Changing any of the parameters in the group incurs the same cost of the experiment. + """ + + def __init__(self, name, config): + """ + Create a new group of tunable parameters. + + Parameters + ---------- + name : str + Human-readable identifier of the tunable parameters group. + config : dict + Python dict that represents a CovariantTunableGroup + (e.g., deserialized from JSON). + """ + self._is_updated = True + self._name = name + self._cost = config.get("cost", 0) + self._tunables = { + name: Tunable(name, tunable_config) + for (name, tunable_config) in config.get("params", {}).items() + } + + def reset(self): + """ + Clear the update flag. That is, state that running an experiment with the + current values of the tunables in this group has no extra cost. + """ + self._is_updated = False + + def get_cost(self): + """ + Get the cost of the experiment given current tunable values. + + Returns + ------- + cost : int + Cost of the experiment or 0 if parameters have not been updated. + """ + return self._cost if self._is_updated else 0 + + def get_names(self): + """ + Get the names of all tunables in the group. + """ + return self._tunables.keys() + + def get_values(self): + """ + Get current values of all tunables in the group. + """ + return {name: tunable.value for (name, tunable) in self._tunables.items()} + + def __repr__(self): + return "{name}: {value}".format(name=self._name, value=self._tunables) + + def __getitem__(self, name): + return self._tunables[name].value + + def __setitem__(self, name, value): + self._is_updated = True + self._tunables[name].value = value + return value + + +class TunableGroups: + """ + A collection of covariant groups of tunable parameters. + """ + + def __init__(self, config=None): + """ + Create a new group of tunable parameters. + + Parameters + ---------- + config : dict + Python dict of serialized representation of the covariant tunable groups. + """ + self._index = {} # Index (Tunable id -> CovariantTunableGroup) + self._tunable_groups = {} + for (name, group_config) in (config or {}).items(): + self._add_group(CovariantTunableGroup(name, group_config)) + + def _add_group(self, group): + """ + Add a CovariantTunableGroup to the current collection. + + Parameters + ---------- + group : CovariantTunableGroup + """ + # pylint: disable=protected-access + self._tunable_groups[group._name] = group + self._index.update(dict.fromkeys(group.get_names(), group)) + + def update(self, tunables): + """ + Merge the two collections of covariant tunable groups. + + Parameters + ---------- + tunables : TunableGroups + A collection of covariant tunable groups. + """ + # pylint: disable=protected-access + self._index.update(tunables._index) + self._tunable_groups.update(tunables._tunable_groups) + + def __repr__(self): + return "{ " + ", ".join( + "{}::{}".format(group_name, tunable) + for (group_name, group) in self._tunable_groups.items() + for tunable in group._tunables.values()) + " }" + + def __getitem__(self, name): + """ + Get the current value of a single tunable parameter. + """ + return self._index[name][name] + + def __setitem__(self, name, value): + """ + Update the current value of a single tunable parameter. + """ + # Use double index to make sure we set the is_updated flag of the group + self._index[name][name] = value + + def get_names(self): + """ + Get the names of all covariance groups in the collection. + + Returns + ------- + group_names : [str] + IDs of the covariant tunable groups. + """ + return self._tunable_groups.keys() + + def subgroup(self, group_names): + """ + Select the covariance groups from the current set and create a new + TunableGroups object that consists of those covariance groups. + + Parameters + ---------- + group_names : [str] + IDs of the covariant tunable groups. + + Returns + ------- + tunables : TunableGroups + A collection of covariant tunable groups. + """ + # pylint: disable=protected-access + tunables = TunableGroups() + for name in group_names: + tunables._add_group(self._tunable_groups[name]) + return tunables + + def get_param_values(self, group_names=None, into_params=None): + """ + Get the current values of the tunables that belong to the specified covariance groups. + + Parameters + ---------- + group_names : [str] or None + IDs of the covariant tunable groups. + Select parameters from all groups if omitted. + into_params : dict of {str: value} + An optional dict to copy the parameters and their values into. + + Returns + ------- + into_params : dict of {str: value} + Flat dict of all parameters and their values from given covariance groups. + """ + if group_names is None: + group_names = self.get_names() + if into_params is None: + into_params = {} + for name in group_names: + into_params.update(self._tunable_groups[name].get_values()) + return into_params + + def reset(self, group_names=None): + """ + Clear the update flag of given covariant groups. + + Parameters + ---------- + group_names : [str] or None + IDs of the (covariant) tunable groups. Reset all groups if omitted. + """ + for name in (group_names or self.get_names()): + self._tunable_groups[name].reset() diff --git a/mlos_bench/mlos_bench/main.py b/mlos_bench/mlos_bench/main.py old mode 100644 new mode 100755 index a8cf4ae7fb..f98a7403b4 --- a/mlos_bench/mlos_bench/main.py +++ b/mlos_bench/mlos_bench/main.py @@ -1,20 +1,21 @@ +#!/usr/bin/env python3 + """ OS Autotune main optimization loop. """ -import sys -import json import logging +import argparse from mlos_bench.opt import Optimizer -from mlos_bench.environment import Environment +from mlos_bench.environment.persistence import load_environment -def optimize(config): +def optimize(env_config_file): """ Main optimization loop. """ - env = Environment.from_config(config) + env = load_environment(env_config_file) opt = Optimizer(env.tunable_params()) _LOG.info("Env: %s Optimizer: %s", env, opt) @@ -38,15 +39,21 @@ def optimize(config): def _main(): - with open(sys.argv[1]) as fh_json: - config = json.load(fh_json) + parser = argparse.ArgumentParser( + description='OS Autotune optimizer') - if _LOG.isEnabledFor(logging.DEBUG): - _LOG.debug("Config:\n%s", json.dumps(config, indent=2)) + parser.add_argument( + '--config', required=True, + help='Path to JSON file with the configuration' + ' of the benchmarking environment') - result = optimize(config) + args = parser.parse_args() + + result = optimize(args.config) _LOG.info("Final result: %s", result) +############################################################### + logging.basicConfig( level=logging.DEBUG, diff --git a/mlos_bench/mlos_bench/opt.py b/mlos_bench/mlos_bench/opt.py index e73b407c25..3630ed1d4a 100644 --- a/mlos_bench/mlos_bench/opt.py +++ b/mlos_bench/mlos_bench/opt.py @@ -24,9 +24,8 @@ class Optimizer: "Generate the next suggestion." # For now, get just the default values. # FIXME: Need to iterate over the actual values. - tunables = { - key: val.get("default") for (key, val) in self._tunables.items() - } + # TODO: Use self._tunables.copy() here when implemented. + tunables = self._tunables # TODO: Populate the tunables with some random values _LOG.info("Suggest: %s", tunables) return tunables