зеркало из https://github.com/microsoft/MLOS.git
Merged PR 571: Read tunables from a separate config
* Store tunables' definitions in a separate config * Implement the `Tunable` wrapper class * Implement tunable groups and collections of covariant parameters Related work items: #457, #458
This commit is contained in:
Sergiy Matusevych
Родитель
4ad1aa0495
Коммит
552cd49666
|
|
@ -0,0 +1 @@
|
|||
services.json
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
[
|
||||
{
|
||||
"class": "mlos_bench.environment.azure.AzureVMService",
|
||||
|
||||
"config": {
|
||||
"deploy_template_path": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json",
|
||||
|
||||
"subscription": "...",
|
||||
"resource_group": "sergiym-os-autotune",
|
||||
"deployment_name": "sergiym-os-autotune-001",
|
||||
"vmName": "osat-linux-vm",
|
||||
|
||||
"accessToken": "AZURE ACCESS TOKEN (e.g., from `az account get-access-token`)"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
@ -1,114 +0,0 @@
|
|||
{
|
||||
"name": "Azure VM Ubuntu Redis",
|
||||
"class": "mlos_bench.environment.CompositeEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"services": [
|
||||
{
|
||||
"class": "mlos_bench.environment.azure.AzureVMService",
|
||||
|
||||
"config": {
|
||||
"deploy_template_path": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json",
|
||||
|
||||
"subscription": "...",
|
||||
"resource_group": "sergiym-os-autotune",
|
||||
"deployment_name": "sergiym-os-autotune-001",
|
||||
"vmName": "osat-linux-vm",
|
||||
|
||||
"accessToken": "AZURE ACCESS TOKEN (e.g., from `az account get-access-token`)"
|
||||
}
|
||||
}
|
||||
],
|
||||
|
||||
"children": [
|
||||
{
|
||||
"name": "Deploy Ubuntu VM on Azure",
|
||||
"class": "mlos_bench.environment.azure.VMEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"cost": 1000,
|
||||
|
||||
"tunable_params": {
|
||||
"vmSize": {
|
||||
"type": "categorical",
|
||||
"default": "Standard_B4ms",
|
||||
"values": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"]
|
||||
}
|
||||
},
|
||||
|
||||
"const_args": {
|
||||
|
||||
"adminUsername": "sergiym",
|
||||
"authenticationType": "sshPublicKey",
|
||||
"adminPasswordOrKey": "SSH PUBLIC KEY (e.g., from id_rsa.pub)",
|
||||
|
||||
"virtualNetworkName": "sergiym-osat-vnet",
|
||||
"subnetName": "sergiym-osat-subnet",
|
||||
"networkSecurityGroupName": "sergiym-osat-sg",
|
||||
|
||||
"ubuntuOSVersion": "18.04-LTS"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Boot Ubuntu VM on Azure",
|
||||
"class": "mlos_bench.environment.azure.OSEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"cost": 300,
|
||||
|
||||
"tunable_params": {
|
||||
"rootfs": {
|
||||
"type": "categorical",
|
||||
"default": "xfs",
|
||||
"values": ["xfs", "ext4", "ext2"]
|
||||
}
|
||||
},
|
||||
|
||||
"const_args": {
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Redis on Linux",
|
||||
"class": "mlos_bench.environment.AppEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"pollDelay": 10,
|
||||
|
||||
"cost": 1,
|
||||
|
||||
"tunable_params": {
|
||||
"kernel.sched_migration_cost_ns": {
|
||||
"type": "int",
|
||||
"default": -1,
|
||||
"range": [0, 500000],
|
||||
"special": [-1]
|
||||
}
|
||||
},
|
||||
|
||||
"const_args": {
|
||||
"commandId": "RunShellScript",
|
||||
"script": [
|
||||
"ls -l /"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "param1",
|
||||
"value": "111"
|
||||
},
|
||||
{
|
||||
"name": "param2",
|
||||
"value": "222"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
{
|
||||
"name": "Azure VM Ubuntu Redis",
|
||||
"class": "mlos_bench.environment.CompositeEnv",
|
||||
|
||||
"include_tunables": [
|
||||
"./mlos_bench/config/tunables.json"
|
||||
],
|
||||
|
||||
"include_services": [
|
||||
"./mlos_bench/config/azure/services.json"
|
||||
],
|
||||
|
||||
"config": {
|
||||
|
||||
"children": [
|
||||
{
|
||||
"name": "Deploy Ubuntu VM on Azure",
|
||||
"class": "mlos_bench.environment.azure.VMEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"tunable_params": ["provision"],
|
||||
|
||||
"const_args": {
|
||||
|
||||
"adminUsername": "sergiym",
|
||||
"authenticationType": "sshPublicKey",
|
||||
"adminPasswordOrKey": "NOT USED",
|
||||
|
||||
"virtualNetworkName": "sergiym-osat-vnet",
|
||||
"subnetName": "sergiym-osat-subnet",
|
||||
"networkSecurityGroupName": "sergiym-osat-sg",
|
||||
|
||||
"ubuntuOSVersion": "18.04-LTS"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Boot Ubuntu VM on Azure",
|
||||
"class": "mlos_bench.environment.azure.OSEnv",
|
||||
|
||||
"config": {
|
||||
"tunable_params": ["boot"],
|
||||
"const_args": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Redis on Linux",
|
||||
"class": "mlos_bench.environment.AppEnv",
|
||||
|
||||
"config": {
|
||||
|
||||
"pollDelay": 10,
|
||||
|
||||
"tunable_params": ["kernel", "redis"],
|
||||
|
||||
"const_args": {
|
||||
"commandId": "RunShellScript",
|
||||
"script": [
|
||||
"echo $param1 $param2",
|
||||
"sudo ls -l /"
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "param1",
|
||||
"value": "111"
|
||||
},
|
||||
{
|
||||
"name": "param2",
|
||||
"value": "222"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
{
|
||||
"provision": {
|
||||
"cost": 1000,
|
||||
"params": {
|
||||
"vmSize": {
|
||||
"description": "Azure VM size",
|
||||
"type": "categorical",
|
||||
"default": "Standard_B4ms",
|
||||
"values": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"boot": {
|
||||
"cost": 300,
|
||||
"params": {
|
||||
"rootfs": {
|
||||
"description": "Root file system",
|
||||
"type": "categorical",
|
||||
"default": "xfs",
|
||||
"values": ["xfs", "ext4", "ext2"]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"kernel": {
|
||||
"cost": 1,
|
||||
"params": {
|
||||
"kernel.sched_migration_cost_ns": {
|
||||
"description": "Cost of migrating the thread to another core",
|
||||
"type": "int",
|
||||
"default": -1,
|
||||
"range": [0, 500000],
|
||||
"special": [-1]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"redis": {
|
||||
"cost": 1,
|
||||
"params": {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@ Benchmarking environments for OS Autotune.
|
|||
"""
|
||||
|
||||
from mlos_bench.environment.status import Status
|
||||
from mlos_bench.environment.tunable import Tunable, TunableGroups
|
||||
from mlos_bench.environment.base_service import Service
|
||||
from mlos_bench.environment.base_environment import Environment
|
||||
|
||||
|
|
@ -12,6 +13,8 @@ from mlos_bench.environment.composite import CompositeEnv
|
|||
|
||||
__all__ = [
|
||||
'Status',
|
||||
'Tunable',
|
||||
'TunableGroups',
|
||||
'Service',
|
||||
'Environment',
|
||||
'AppEnv',
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class AppEnv(Environment):
|
|||
|
||||
_POLL_DELAY = 5 # Default polling interval in seconds.
|
||||
|
||||
def __init__(self, name, config, service=None):
|
||||
def __init__(self, name, config, tunables, service=None):
|
||||
"""
|
||||
Create a new application environment with a given config.
|
||||
|
||||
|
|
@ -32,11 +32,13 @@ class AppEnv(Environment):
|
|||
configuration. Each config must have at least the "tunable_params"
|
||||
and the "const_args" sections; the "cost" field can be omitted
|
||||
and is 0 by default.
|
||||
tunables : TunableGroups
|
||||
A collection of tunable parameters for *all* environments.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
"""
|
||||
super().__init__(name, config, service)
|
||||
super().__init__(name, config, tunables, service)
|
||||
self._poll_delay = self.config.get("pollDelay", AppEnv._POLL_DELAY)
|
||||
|
||||
def setup(self):
|
||||
|
|
@ -59,10 +61,9 @@ class AppEnv(Environment):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) of the OS and application
|
||||
parameters. Setting these parameters should not require an
|
||||
OS reboot.
|
||||
tunables : TunableGroups
|
||||
A collection of tunable OS and application parameters along with their
|
||||
values. Setting these parameters should not require an OS reboot.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -45,8 +45,9 @@ class OSEnv(Environment):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) of the OS boot-time parameters.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters
|
||||
along with the parameters' values.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -45,10 +45,11 @@ class VMEnv(Environment):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) pairs of tunable parameters.
|
||||
VMEnv tunables are variable parameters that, together with the
|
||||
VMEnv configuration, are sufficient to provision and start a VM.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters along with the
|
||||
parameters' values. VMEnv tunables are variable parameters that,
|
||||
together with the VMEnv configuration, are sufficient to provision
|
||||
and start a VM.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import logging
|
|||
import importlib
|
||||
|
||||
from mlos_bench.environment.status import Status
|
||||
from mlos_bench.environment.tunable import TunableGroups
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -17,37 +18,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
An abstract base of all benchmark environments.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_config(config, service=None):
|
||||
"""
|
||||
Factory method for a new environment with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
A dictionary with three mandatory fields:
|
||||
"name": Human-readable string describing the environment;
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
|
||||
Returns
|
||||
-------
|
||||
env : Environment
|
||||
An instance of the `Environment` class initialized with `config`.
|
||||
"""
|
||||
env_name = config["name"]
|
||||
env_class = config["class"]
|
||||
env_config = config["config"]
|
||||
_LOG.debug("Creating env: %s :: %s", env_name, env_class)
|
||||
env = Environment.new(env_name, env_class, env_config, service)
|
||||
_LOG.info("Created env: %s :: %s", env_name, env)
|
||||
return env
|
||||
|
||||
@classmethod
|
||||
def new(cls, env_name, class_name, config, service=None):
|
||||
def new(cls, env_name, class_name, config, tunables=None, service=None):
|
||||
"""
|
||||
Factory method for a new environment with a given config.
|
||||
|
||||
|
|
@ -63,6 +35,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
Free-format dictionary that contains the benchmark environment
|
||||
configuration. It will be passed as a constructor parameter of
|
||||
the class specified by `name`.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters for all environments.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
|
|
@ -85,9 +59,9 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
env_name, class_name, env_class)
|
||||
|
||||
assert issubclass(env_class, cls)
|
||||
return env_class(env_name, config, service)
|
||||
return env_class(env_name, config, tunables, service)
|
||||
|
||||
def __init__(self, name, config, service=None):
|
||||
def __init__(self, name, config, tunables=None, service=None):
|
||||
"""
|
||||
Create a new environment with a given config.
|
||||
|
||||
|
|
@ -100,6 +74,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
configuration. Each config must have at least the "tunable_params"
|
||||
and the "const_args" sections; the "cost" field can be omitted
|
||||
and is 0 by default.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters for all environments.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
|
|
@ -110,8 +86,14 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
self._result = (Status.PENDING, None)
|
||||
|
||||
self._const_args = config.get("const_args", {})
|
||||
self._tunable_params = self._parse_tunables(
|
||||
config.get("tunable_params", {}), config.get("cost", 0))
|
||||
|
||||
if tunables is None:
|
||||
tunables = TunableGroups()
|
||||
|
||||
tunable_groups = config.get("tunable_params")
|
||||
self._tunable_params = (
|
||||
tunables.subgroup(tunable_groups) if tunable_groups else tunables
|
||||
)
|
||||
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Config for: %s\n%s",
|
||||
|
|
@ -126,36 +108,26 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
env_name=self.name
|
||||
)
|
||||
|
||||
def _parse_tunables(self, tunables, cost=0):
|
||||
"Augment tunables with the cost."
|
||||
tunables_cost = {}
|
||||
for (key, val) in tunables.items():
|
||||
tunables_cost[key] = val.copy()
|
||||
tunables_cost[key]["cost"] = cost
|
||||
return tunables_cost
|
||||
|
||||
def _combine_tunables(self, tunables):
|
||||
"""
|
||||
Plug tunable values into the base config. If the tunable is unknown,
|
||||
Plug tunable values into the base config. If the tunable group is unknown,
|
||||
ignore it (it might belong to another environment). This method should
|
||||
never mutate the original config or the tunables.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) pairs of tunable parameters.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters
|
||||
along with the parameters' values.
|
||||
|
||||
Returns
|
||||
-------
|
||||
config : dict
|
||||
Free-format dictionary that contains the new environment
|
||||
configuration.
|
||||
Free-format dictionary that contains the new environment configuration.
|
||||
"""
|
||||
new_config = self._const_args.copy()
|
||||
for (key, val) in tunables.items():
|
||||
if key in self._tunable_params:
|
||||
new_config[key] = val
|
||||
return new_config
|
||||
return tunables.get_param_values(
|
||||
group_names=self._tunable_params.get_names(),
|
||||
into_params=self._const_args.copy())
|
||||
|
||||
def tunable_params(self):
|
||||
"""
|
||||
|
|
@ -163,8 +135,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
|
||||
Returns
|
||||
-------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) pairs of tunable parameters.
|
||||
tunables : TunableGroups
|
||||
A collection of covariant groups of tunable parameters.
|
||||
"""
|
||||
return self._tunable_params
|
||||
|
||||
|
|
@ -201,8 +173,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) pairs of tunable parameters.
|
||||
tunables : TunableGroups
|
||||
A collection of tunable parameters along with their values.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
@ -217,8 +189,8 @@ class Environment(metaclass=abc.ABCMeta):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) pairs of tunable parameters.
|
||||
tunables : TunableGroups
|
||||
A collection of tunable parameters along with their values.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -14,58 +14,6 @@ class Service:
|
|||
An abstract base of all environment services.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def from_config(config):
|
||||
"""
|
||||
Factory method for a new service with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
A dictionary with two mandatory fields:
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
|
||||
Returns
|
||||
-------
|
||||
svc : Service
|
||||
An instance of the `Service` class initialized with `config`.
|
||||
"""
|
||||
svc_class = config["class"]
|
||||
svc_config = config["config"]
|
||||
_LOG.debug("Creating service: %s", svc_class)
|
||||
service = Service.new(svc_class, svc_config)
|
||||
_LOG.info("Created service: %s", service)
|
||||
return service
|
||||
|
||||
@staticmethod
|
||||
def from_config_list(config_list, parent=None):
|
||||
"""
|
||||
Factory method for a new service with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config_list : a list of dict
|
||||
A list where each element is a dictionary with 2 mandatory fields:
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
parent: Service
|
||||
An optional reference of the parent service to mix in.
|
||||
|
||||
Returns
|
||||
-------
|
||||
svc : Service
|
||||
An instance of the `Service` class that is a combination of all
|
||||
services from the list plus the parent mix-in.
|
||||
"""
|
||||
service = Service()
|
||||
if parent:
|
||||
service.register(parent.export())
|
||||
for config in config_list:
|
||||
service.register(Service.from_config(config).export())
|
||||
_LOG.info("Created mix-in service: %s", service.export())
|
||||
return service
|
||||
|
||||
@classmethod
|
||||
def new(cls, class_name, config):
|
||||
"""
|
||||
|
|
@ -115,7 +63,7 @@ class Service:
|
|||
self._services = {}
|
||||
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Config:\n%s", json.dumps(self.config, indent=2))
|
||||
_LOG.debug("Service config:\n%s", json.dumps(self.config, indent=2))
|
||||
|
||||
def register(self, services):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -4,8 +4,8 @@ Composite benchmark environment.
|
|||
|
||||
import logging
|
||||
|
||||
from mlos_bench.environment.base_service import Service
|
||||
from mlos_bench.environment.base_environment import Environment
|
||||
from mlos_bench.environment.persistence import build_environment
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -15,7 +15,7 @@ class CompositeEnv(Environment):
|
|||
Composite benchmark environment.
|
||||
"""
|
||||
|
||||
def __init__(self, name, config, service=None):
|
||||
def __init__(self, name, config, tunables, service=None):
|
||||
"""
|
||||
Create a new environment with a given config.
|
||||
|
||||
|
|
@ -26,25 +26,26 @@ class CompositeEnv(Environment):
|
|||
config : dict
|
||||
Free-format dictionary that contains the environment
|
||||
configuration. Must have a "children" section.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters for *all* environments.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
"""
|
||||
super().__init__(name, config, service)
|
||||
super().__init__(name, config, tunables, service)
|
||||
|
||||
# Propagate all config parameters except "children" and "services"
|
||||
# to every child config.
|
||||
# Propagate all config parameters except "children" to every child config.
|
||||
shared_config = config.copy()
|
||||
del shared_config["children"]
|
||||
del shared_config["services"]
|
||||
|
||||
self._service = Service.from_config_list(
|
||||
config.get("services", []), parent=service)
|
||||
|
||||
self._children = []
|
||||
for child_config in config["children"]:
|
||||
child_config["config"].update(shared_config)
|
||||
env = Environment.from_config(child_config, self._service)
|
||||
|
||||
local_config = shared_config.copy()
|
||||
local_config.update(child_config.get("config", {}))
|
||||
child_config["config"] = local_config
|
||||
|
||||
env = build_environment(child_config, shared_config, tunables, self._service)
|
||||
self._children.append(env)
|
||||
self._tunable_params.update(env.tunable_params())
|
||||
|
||||
|
|
@ -80,9 +81,9 @@ class CompositeEnv(Environment):
|
|||
|
||||
Parameters
|
||||
----------
|
||||
tunables : dict
|
||||
Flat dictionary of (key, value) of the parameters from all
|
||||
children environments.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters and their values
|
||||
from all children environments.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,265 @@
|
|||
"""
|
||||
Helper functions to load, instantiate, and serialize Python objects
|
||||
that encapsulate benchmark environments, tunable parameters, and
|
||||
service functions.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from mlos_bench.environment.tunable import TunableGroups
|
||||
from mlos_bench.environment.base_service import Service
|
||||
from mlos_bench.environment.base_environment import Environment
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_environment(config, global_config=None, tunables=None, service=None):
|
||||
"""
|
||||
Factory method for a new environment with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
A dictionary with three mandatory fields:
|
||||
"name": Human-readable string describing the environment;
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
global_config : dict
|
||||
Global parameters to add to the environment config.
|
||||
tunables : TunableGroups
|
||||
A collection of groups of tunable parameters for all environments.
|
||||
service: Service
|
||||
An optional service object (e.g., providing methods to
|
||||
deploy or reboot a VM, etc.).
|
||||
|
||||
Returns
|
||||
-------
|
||||
env : Environment
|
||||
An instance of the `Environment` class initialized with `config`.
|
||||
"""
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Build environment from config:\n%s",
|
||||
json.dumps(config, indent=2))
|
||||
|
||||
env_name = config["name"]
|
||||
env_class = config["class"]
|
||||
env_config = config.get("config", {})
|
||||
|
||||
if global_config:
|
||||
local_config = global_config.copy()
|
||||
local_config.update(env_config)
|
||||
env_config = local_config
|
||||
|
||||
env_services_path = config.get("include_services")
|
||||
if env_services_path is not None:
|
||||
service = load_services(env_services_path, global_config, service)
|
||||
|
||||
env_tunables_path = config.get("include_tunables")
|
||||
if env_tunables_path is not None:
|
||||
tunables = load_tunables(env_tunables_path, tunables)
|
||||
|
||||
_LOG.debug("Creating env: %s :: %s", env_name, env_class)
|
||||
env = Environment.new(env_name, env_class, env_config, tunables, service)
|
||||
|
||||
_LOG.info("Created env: %s :: %s", env_name, env)
|
||||
return env
|
||||
|
||||
|
||||
def _build_standalone_service(config, global_config=None):
|
||||
"""
|
||||
Factory method for a new service with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
A dictionary with two mandatory fields:
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
global_config : dict
|
||||
Global parameters to add to the service config.
|
||||
|
||||
Returns
|
||||
-------
|
||||
svc : Service
|
||||
An instance of the `Service` class initialized with `config`.
|
||||
"""
|
||||
svc_class = config["class"]
|
||||
svc_config = config.get("config", {})
|
||||
|
||||
if global_config:
|
||||
local_config = global_config.copy()
|
||||
local_config.update(svc_config)
|
||||
svc_config = local_config
|
||||
|
||||
_LOG.debug("Creating service: %s", svc_class)
|
||||
service = Service.new(svc_class, svc_config)
|
||||
|
||||
_LOG.info("Created service: %s", service)
|
||||
return service
|
||||
|
||||
|
||||
def _build_composite_service(config_list, global_config=None, parent=None):
|
||||
"""
|
||||
Factory method for a new service with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config_list : a list of dict
|
||||
A list where each element is a dictionary with 2 mandatory fields:
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
global_config : dict
|
||||
Global parameters to add to the service config.
|
||||
parent: Service
|
||||
An optional reference of the parent service to mix in.
|
||||
|
||||
Returns
|
||||
-------
|
||||
svc : Service
|
||||
An instance of the `Service` class that is a combination of all
|
||||
services from the list plus the parent mix-in.
|
||||
"""
|
||||
service = Service()
|
||||
if parent:
|
||||
service.register(parent.export())
|
||||
for config in config_list:
|
||||
service.register(_build_standalone_service(config, global_config).export())
|
||||
_LOG.info("Created mix-in service: %s", service.export())
|
||||
return service
|
||||
|
||||
|
||||
def build_service(config, global_config=None, parent=None):
|
||||
"""
|
||||
Factory method for a new service with a given config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : [dict] or dict
|
||||
A list where each element is a dictionary with 2 mandatory fields:
|
||||
"class": FQN of a Python class to instantiate;
|
||||
"config": Free-format dictionary to pass to the constructor.
|
||||
global_config : dict
|
||||
Global parameters to add to the service config.
|
||||
parent: Service
|
||||
An optional reference of the parent service to mix in.
|
||||
|
||||
Returns
|
||||
-------
|
||||
svc : Service
|
||||
An instance of the `Service` class that is a combination of all
|
||||
services from the list plus the parent mix-in.
|
||||
"""
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Build service from config:\n%s",
|
||||
json.dumps(config, indent=2))
|
||||
|
||||
if isinstance(config, dict):
|
||||
if parent is None:
|
||||
return _build_standalone_service(config, global_config)
|
||||
config = [config]
|
||||
|
||||
return _build_composite_service(config, global_config, parent)
|
||||
|
||||
|
||||
def build_tunables(config, parent=None):
|
||||
"""
|
||||
Create a new collection of tunable parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
Python dict of serialized representation of the covariant tunable groups.
|
||||
parent : TunableGroups
|
||||
An optional collection of tunables to add to the new collection.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tunables : TunableGroup
|
||||
Create a new collection of tunable parameters.
|
||||
"""
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Build tunables from config:\n%s",
|
||||
json.dumps(config, indent=2))
|
||||
|
||||
if parent is None:
|
||||
return TunableGroups(config)
|
||||
groups = TunableGroups()
|
||||
groups.update(parent)
|
||||
groups.update(TunableGroups(config))
|
||||
return groups
|
||||
|
||||
|
||||
def load_environment(json_file_name, global_config=None,
|
||||
tunables=None, service=None):
|
||||
"""
|
||||
Create a new collection of tunable parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
json_file_name : str
|
||||
The environment JSON configuration file.
|
||||
global_config : dict
|
||||
Global parameters to add to the environment config.
|
||||
tunables : TunableGroups
|
||||
An optional collection of tunables to add to the new collection.
|
||||
service : Service
|
||||
An optional reference of the parent service to mix in.
|
||||
"""
|
||||
_LOG.info("Load environment: %s", json_file_name)
|
||||
with open(json_file_name) as fh_json:
|
||||
config = json.load(fh_json)
|
||||
return build_environment(config, global_config, tunables, service)
|
||||
|
||||
|
||||
def load_services(json_file_names, global_config=None, parent=None):
|
||||
"""
|
||||
Create a new collection of tunable parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
json_file_name : str
|
||||
The service JSON configuration file.
|
||||
global_config : dict
|
||||
Global parameters to add to the service config.
|
||||
parent : Service
|
||||
An optional reference of the parent service to mix in.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tunables : TunableGroup
|
||||
Create a new collection of tunable parameters.
|
||||
"""
|
||||
_LOG.info("Load services: %s", json_file_names)
|
||||
service = Service(global_config)
|
||||
if parent:
|
||||
service.register(parent.export())
|
||||
for fname in json_file_names:
|
||||
_LOG.debug("Load services: %s", fname)
|
||||
with open(fname) as fh_json:
|
||||
config = json.load(fh_json)
|
||||
service.register(build_service(config, global_config).export())
|
||||
return service
|
||||
|
||||
|
||||
def load_tunables(json_file_names, parent=None):
|
||||
"""
|
||||
Load a collection of tunable parameters from JSON files.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
json_file_names : [str]
|
||||
A list of JSON files to load.
|
||||
parent : TunableGroups
|
||||
An optional collection of tunables to add to the new collection.
|
||||
"""
|
||||
_LOG.info("Load tunables: %s", json_file_names)
|
||||
groups = TunableGroups()
|
||||
if parent is not None:
|
||||
groups.update(parent)
|
||||
for fname in json_file_names:
|
||||
_LOG.debug("Load tunables: %s", fname)
|
||||
with open(fname) as fh_json:
|
||||
config = json.load(fh_json)
|
||||
groups.update(TunableGroups(config))
|
||||
return groups
|
||||
|
|
@ -21,6 +21,11 @@ class Status(enum.Enum):
|
|||
@staticmethod
|
||||
def is_good(status):
|
||||
"""
|
||||
Check if the status is not failed or canceled.
|
||||
Check if the status of the environment is good.
|
||||
"""
|
||||
return status not in {Status.CANCELED, Status.FAILED, Status.TIMED_OUT}
|
||||
return status in {
|
||||
Status.PENDING,
|
||||
Status.READY,
|
||||
Status.RUNNING,
|
||||
Status.SUCCEEDED,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,261 @@
|
|||
"""
|
||||
Tunable parameter definition.
|
||||
"""
|
||||
import collections
|
||||
|
||||
|
||||
class Tunable:
|
||||
"""
|
||||
A tunable parameter definition and its current value.
|
||||
"""
|
||||
|
||||
def __init__(self, name, config):
|
||||
"""
|
||||
Create an instance of a new tunable parameter.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Human-readable identifier of the tunable parameter.
|
||||
config : dict
|
||||
Python dict that represents a Tunable (e.g., deserialized from JSON)
|
||||
"""
|
||||
self._name = name
|
||||
self._description = config.get("description")
|
||||
self._type = config["type"]
|
||||
self._default = config.get("default")
|
||||
self._values = config.get("values")
|
||||
self._range = config.get("range")
|
||||
self._special = config.get("special")
|
||||
self._current_value = self._default
|
||||
if self._type == "categorical":
|
||||
if not (self._values and isinstance(self._values, collections.abc.Iterable)):
|
||||
raise ValueError("Must specify values for the categorical type")
|
||||
if self._range is not None:
|
||||
raise ValueError("Range must be None for the categorical type")
|
||||
if self._special is not None:
|
||||
raise ValueError("Special values must be None for the categorical type")
|
||||
elif self._type in {"int", "float"}:
|
||||
if not self._range or len(self._range) != 2 or self._range[0] >= self._range[1]:
|
||||
raise ValueError("Invalid range: " + self._range)
|
||||
else:
|
||||
raise ValueError("Invalid parameter type: " + self._type)
|
||||
|
||||
def __repr__(self):
|
||||
return "{name}={value}".format(name=self._name, value=self._current_value)
|
||||
|
||||
@property
|
||||
def value(self):
|
||||
"""
|
||||
Get the current value of the tunable.
|
||||
"""
|
||||
return self._current_value
|
||||
|
||||
@value.setter
|
||||
def value(self, value):
|
||||
"""
|
||||
Set the current value of the tunable.
|
||||
"""
|
||||
self._current_value = value
|
||||
return value
|
||||
|
||||
|
||||
class CovariantTunableGroup:
|
||||
"""
|
||||
A collection of tunable parameters.
|
||||
Changing any of the parameters in the group incurs the same cost of the experiment.
|
||||
"""
|
||||
|
||||
def __init__(self, name, config):
|
||||
"""
|
||||
Create a new group of tunable parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Human-readable identifier of the tunable parameters group.
|
||||
config : dict
|
||||
Python dict that represents a CovariantTunableGroup
|
||||
(e.g., deserialized from JSON).
|
||||
"""
|
||||
self._is_updated = True
|
||||
self._name = name
|
||||
self._cost = config.get("cost", 0)
|
||||
self._tunables = {
|
||||
name: Tunable(name, tunable_config)
|
||||
for (name, tunable_config) in config.get("params", {}).items()
|
||||
}
|
||||
|
||||
def reset(self):
|
||||
"""
|
||||
Clear the update flag. That is, state that running an experiment with the
|
||||
current values of the tunables in this group has no extra cost.
|
||||
"""
|
||||
self._is_updated = False
|
||||
|
||||
def get_cost(self):
|
||||
"""
|
||||
Get the cost of the experiment given current tunable values.
|
||||
|
||||
Returns
|
||||
-------
|
||||
cost : int
|
||||
Cost of the experiment or 0 if parameters have not been updated.
|
||||
"""
|
||||
return self._cost if self._is_updated else 0
|
||||
|
||||
def get_names(self):
|
||||
"""
|
||||
Get the names of all tunables in the group.
|
||||
"""
|
||||
return self._tunables.keys()
|
||||
|
||||
def get_values(self):
|
||||
"""
|
||||
Get current values of all tunables in the group.
|
||||
"""
|
||||
return {name: tunable.value for (name, tunable) in self._tunables.items()}
|
||||
|
||||
def __repr__(self):
|
||||
return "{name}: {value}".format(name=self._name, value=self._tunables)
|
||||
|
||||
def __getitem__(self, name):
|
||||
return self._tunables[name].value
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
self._is_updated = True
|
||||
self._tunables[name].value = value
|
||||
return value
|
||||
|
||||
|
||||
class TunableGroups:
|
||||
"""
|
||||
A collection of covariant groups of tunable parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, config=None):
|
||||
"""
|
||||
Create a new group of tunable parameters.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict
|
||||
Python dict of serialized representation of the covariant tunable groups.
|
||||
"""
|
||||
self._index = {} # Index (Tunable id -> CovariantTunableGroup)
|
||||
self._tunable_groups = {}
|
||||
for (name, group_config) in (config or {}).items():
|
||||
self._add_group(CovariantTunableGroup(name, group_config))
|
||||
|
||||
def _add_group(self, group):
|
||||
"""
|
||||
Add a CovariantTunableGroup to the current collection.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
group : CovariantTunableGroup
|
||||
"""
|
||||
# pylint: disable=protected-access
|
||||
self._tunable_groups[group._name] = group
|
||||
self._index.update(dict.fromkeys(group.get_names(), group))
|
||||
|
||||
def update(self, tunables):
|
||||
"""
|
||||
Merge the two collections of covariant tunable groups.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
tunables : TunableGroups
|
||||
A collection of covariant tunable groups.
|
||||
"""
|
||||
# pylint: disable=protected-access
|
||||
self._index.update(tunables._index)
|
||||
self._tunable_groups.update(tunables._tunable_groups)
|
||||
|
||||
def __repr__(self):
|
||||
return "{ " + ", ".join(
|
||||
"{}::{}".format(group_name, tunable)
|
||||
for (group_name, group) in self._tunable_groups.items()
|
||||
for tunable in group._tunables.values()) + " }"
|
||||
|
||||
def __getitem__(self, name):
|
||||
"""
|
||||
Get the current value of a single tunable parameter.
|
||||
"""
|
||||
return self._index[name][name]
|
||||
|
||||
def __setitem__(self, name, value):
|
||||
"""
|
||||
Update the current value of a single tunable parameter.
|
||||
"""
|
||||
# Use double index to make sure we set the is_updated flag of the group
|
||||
self._index[name][name] = value
|
||||
|
||||
def get_names(self):
|
||||
"""
|
||||
Get the names of all covariance groups in the collection.
|
||||
|
||||
Returns
|
||||
-------
|
||||
group_names : [str]
|
||||
IDs of the covariant tunable groups.
|
||||
"""
|
||||
return self._tunable_groups.keys()
|
||||
|
||||
def subgroup(self, group_names):
|
||||
"""
|
||||
Select the covariance groups from the current set and create a new
|
||||
TunableGroups object that consists of those covariance groups.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
group_names : [str]
|
||||
IDs of the covariant tunable groups.
|
||||
|
||||
Returns
|
||||
-------
|
||||
tunables : TunableGroups
|
||||
A collection of covariant tunable groups.
|
||||
"""
|
||||
# pylint: disable=protected-access
|
||||
tunables = TunableGroups()
|
||||
for name in group_names:
|
||||
tunables._add_group(self._tunable_groups[name])
|
||||
return tunables
|
||||
|
||||
def get_param_values(self, group_names=None, into_params=None):
|
||||
"""
|
||||
Get the current values of the tunables that belong to the specified covariance groups.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
group_names : [str] or None
|
||||
IDs of the covariant tunable groups.
|
||||
Select parameters from all groups if omitted.
|
||||
into_params : dict of {str: value}
|
||||
An optional dict to copy the parameters and their values into.
|
||||
|
||||
Returns
|
||||
-------
|
||||
into_params : dict of {str: value}
|
||||
Flat dict of all parameters and their values from given covariance groups.
|
||||
"""
|
||||
if group_names is None:
|
||||
group_names = self.get_names()
|
||||
if into_params is None:
|
||||
into_params = {}
|
||||
for name in group_names:
|
||||
into_params.update(self._tunable_groups[name].get_values())
|
||||
return into_params
|
||||
|
||||
def reset(self, group_names=None):
|
||||
"""
|
||||
Clear the update flag of given covariant groups.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
group_names : [str] or None
|
||||
IDs of the (covariant) tunable groups. Reset all groups if omitted.
|
||||
"""
|
||||
for name in (group_names or self.get_names()):
|
||||
self._tunable_groups[name].reset()
|
||||
|
|
@ -1,20 +1,21 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
OS Autotune main optimization loop.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
from mlos_bench.opt import Optimizer
|
||||
from mlos_bench.environment import Environment
|
||||
from mlos_bench.environment.persistence import load_environment
|
||||
|
||||
|
||||
def optimize(config):
|
||||
def optimize(env_config_file):
|
||||
"""
|
||||
Main optimization loop.
|
||||
"""
|
||||
env = Environment.from_config(config)
|
||||
env = load_environment(env_config_file)
|
||||
|
||||
opt = Optimizer(env.tunable_params())
|
||||
_LOG.info("Env: %s Optimizer: %s", env, opt)
|
||||
|
|
@ -38,15 +39,21 @@ def optimize(config):
|
|||
|
||||
def _main():
|
||||
|
||||
with open(sys.argv[1]) as fh_json:
|
||||
config = json.load(fh_json)
|
||||
parser = argparse.ArgumentParser(
|
||||
description='OS Autotune optimizer')
|
||||
|
||||
if _LOG.isEnabledFor(logging.DEBUG):
|
||||
_LOG.debug("Config:\n%s", json.dumps(config, indent=2))
|
||||
parser.add_argument(
|
||||
'--config', required=True,
|
||||
help='Path to JSON file with the configuration'
|
||||
' of the benchmarking environment')
|
||||
|
||||
result = optimize(config)
|
||||
args = parser.parse_args()
|
||||
|
||||
result = optimize(args.config)
|
||||
_LOG.info("Final result: %s", result)
|
||||
|
||||
###############################################################
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
|
|
|
|||
|
|
@ -24,9 +24,8 @@ class Optimizer:
|
|||
"Generate the next suggestion."
|
||||
# For now, get just the default values.
|
||||
# FIXME: Need to iterate over the actual values.
|
||||
tunables = {
|
||||
key: val.get("default") for (key, val) in self._tunables.items()
|
||||
}
|
||||
# TODO: Use self._tunables.copy() here when implemented.
|
||||
tunables = self._tunables
|
||||
# TODO: Populate the tunables with some random values
|
||||
_LOG.info("Suggest: %s", tunables)
|
||||
return tunables
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче