Merged PR 588: Implement a script to run a single benchmark

* Move all generic command line launching functionality into a separate module.
* Update the README file to use the new script and the parameters.
* Change the config parameters propagation policy: now values from the parent config override the parameters at the lower levels. This way we can keep the default values in the main config and push the actual values from the globals. E.g., see the `"accessToken"` and `"subscription"` in this diff

Related work items: #501

Makefile

@@ -146,7 +146,7 @@ doc: conda-env doc-prereqs clean-doc
 	conda run -n ${CONDA_ENV_NAME} make -j -C doc/ html
 	test -s doc/build/html/index.html
 	test -s doc/build/html/generated/mlos_core.optimizers.BaseOptimizer.html
-	test -s doc/build/html/generated/mlos_bench.main.optimize.html
+	test -s doc/build/html/generated/mlos_bench.run_opt.optimize.html
 	test -s doc/build/html/api/mlos_core/mlos_core.html
 	test -s doc/build/html/api/mlos_bench/mlos_bench.html
 	cp doc/staticwebapp.config.json doc/build/html/

doc/source/overview.rst

@@ -39,13 +39,27 @@ This is a list of major functions and classes provided by `mlos_bench`.
 
 Main
 ====
-.. currentmodule:: mlos_bench.main
+
+:doc:`run_opt.py </api/mlos_bench/mlos_bench.run_opt>`
+
+    The main optimization loop script.
+
+.. currentmodule:: mlos_bench.run_opt
 .. autosummary::
    :toctree: generated/
    :template: functions.rst
 
    optimize
 
+:doc:`run_bench.py </api/mlos_bench/mlos_bench.run_bench>`
+
+    A helper script for testing a single application/workload run.
+
+.. currentmodule:: mlos_bench.run_bench
+.. autosummary::
+   :toctree: generated/
+   :template: function.rst
+
 Optimizer
 =========
 .. currentmodule:: mlos_bench.opt

mlos_bench/README.md

@@ -30,11 +30,11 @@ Installation instructions for `az` (Azure CLI) [can be found here](https://docs.
     az account get-access-token
     ```
 
-3. Make a copy of `services-example.json` so that we can adapt it with our own details.
+3. Make a copy of `services.json` so that we can adapt it with our own details.
 For example,
 
     ```sh
-    cp config/azure/services-example.json config/azure/services-mine.json
+    cp config/azure/services.json ./services-mine.json
     ```
 
 4. Modify our new service configuration file with our target resource group information, as well as desired VM name, denoted in `{{ ... }}` below.
@@ -63,11 +63,11 @@ For example,
     ]
     ```
 
-5. Make a copy of `config-example.json` so that we can adapt it with our own details.
+5. Make a copy of `env-azure-ubuntu-redis.json` so that we can adapt it with our own details.
 For example,
 
     ```sh
-    cp config/config-example.json config/config-mine.json
+    cp config/env-azure-ubuntu-redis.json ./config-mine.json
     ```
 
 6. Modify our new configuration file with desired resource names, denoted in `{{ ... }}` below.
@@ -82,7 +82,7 @@ For example,
         ],
 
         "include_services": [
-            "{{ Path to your new service config, e.g. ./mlos_bench/config/azure/services-mine.json }}"
+            "{{ Path to your new service config, e.g. ./services-mine.json }}"
         ],
 
         "config": {
@@ -123,11 +123,19 @@ Create and activate the environment with:
     conda activate mlos_core
     ```
 
-8. Run our configuration through `mlos_bench`.
+8. Store the Azure security credentials in the `global.json` config file.
+
+    ```sh
+    az account get-access-token > ./global.json
+    ```
+    > You have to repeat that operation every hour or so to update the file with the new access token.
+
+
+9. Run our configuration through `mlos_bench`.
 We can do so and pipe the output into log file `osat.log` as follows:
 
     ```sh
-    python mlos_bench/mlos_bench/main.py --config mlos_bench/config/config-mine.json --accessToken "$(az account get-access-token --query accessToken --output tsv)" 2>&1 > ./osat.log
+    python mlos_bench/mlos_bench/run_opt.py --config ./config-mine.json --global ./global.json 2>&1 | tee ./osat.log
     ```
 
-9. Check `osat.log` to verify we get output corresponding to the `ls -l /` command we remotely executed in the VM.
+10. Check `osat.log` to verify we get output corresponding to the `ls -l /` command we remotely executed in the VM.

mlos_bench/config/azure/azuredeploy-ubuntu-vm.json

@@ -4,7 +4,7 @@
   "parameters": {
     "vmName": {
       "type": "string",
-      "defaultValue": "osat-linux-vm",
+      "defaultValue": "os-autotune-linux-vm",
       "metadata": {
         "description": "OS Autotune Linux VM"
       }
@@ -68,21 +68,21 @@
     },
     "virtualNetworkName": {
       "type": "string",
-      "defaultValue": "osat-vnet",
+      "defaultValue": "os-autotune-vnet",
       "metadata": {
         "description": "Name of the VNET"
       }
     },
     "subnetName": {
       "type": "string",
-      "defaultValue": "osat-subnet",
+      "defaultValue": "os-autotune-subnet",
       "metadata": {
         "description": "Name of the subnet in the virtual network"
       }
     },
     "networkSecurityGroupName": {
       "type": "string",
-      "defaultValue": "osat-sg",
+      "defaultValue": "os-autotune-nsg",
       "metadata": {
         "description": "Name of the Network Security Group"
       }
@@ -259,4 +259,4 @@
       "value": "[format('ssh {0}@{1}', parameters('adminUsername'), reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))).dnsSettings.fqdn)]"
     }
   }
-}
+}

mlos_bench/config/azure/services.json

@@ -5,9 +5,12 @@
         "config": {
             "deployTemplatePath": "./mlos_bench/config/azure/azuredeploy-ubuntu-vm.json",
 
-            "resourceGroup": "sergiym-os-autotune",
-            "deploymentName": "sergiym-os-autotune-001",
-            "vmName": "osat-linux-vm"
+            "subscription": "AZURE SUBSCRIPTION ID",
+            "accessToken": "AZURE ACCESS TOKEN, E.G. FROM `az account get-access-token`",
+
+            "resourceGroup": "os-autotune",
+            "deploymentName": "os-autotune-001",
+            "vmName": "os-autotune-linux-vm"
         }
     }
 ]

mlos_bench/config/env-azure-ubuntu-redis.json

@@ -23,13 +23,13 @@
 
                     "const_args": {
 
-                        "adminUsername": "sergiym",
+                        "adminUsername": "os-autotune",
                         "authenticationType": "sshPublicKey",
-                        "adminPasswordOrKey": "NOT USED",
+                        "adminPasswordOrKey": "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZnRBZZWYoPaAf4ZFP3rj1IKpdjW3QySgfOfrw6gUcqWN8OnUdxXzJJ35DxDhtZY3Qq038Y7IHfocrxrKL2ID/rrYvo77/uuqaEpELOXU9YA1ervTqlUagARSfeoUQDOZkDsbm603D4/3t3l0hxNHPGDyqjLtP2U+RelhLEngaSGSvwAiFNHJef8ldDfOxgBSVprBvU+48g4mZp5QZ2O50gzhcoFI2+Ho/Am4el2SS6ptxS1BjeVR1/GoLpB/POYfWqgs5/0rrG2gUyRQZdvY97LhMVd3jUK0DeiRnQpv44fnKTh8px3sl6N5sTg9a4tL8PBZC31MqP7QTj9KqMh8HfMp0wEIXPUWinBpTjXtknZiOWodLuAtZ82GzO8sW5HhXW5ZdU6aaNZ1JUakQX+eWTAzF3qQCp2BDIrXccxa5HMjneLczHug5VH1Y4wo3LyI4QBmxoQzYUsvSDBHAGqSu3u6XJVfJbgA5Di1ykGpVaMda0iLc7wG9ADevEfOmSQIMI2PEW6ZLv/+1w+uHYJi4c+3cIr9nSPpBu1HmefY4Hrz9XtYhFwTAk8RCla9v4WPn0hrVQnTBcuW3a79d87v6O2NIv1gosnsEWpWwIHycIe3G/Y6btsoDHcYMWbWewSOVsTru5UAfiZYZk8psjMwc4wcGkIHxcgzFlWb9r8ZOzw== os-autotune-fake-key",
 
-                        "virtualNetworkName": "sergiym-osat-vnet",
-                        "subnetName": "sergiym-osat-subnet",
-                        "networkSecurityGroupName": "sergiym-osat-sg",
+                        "virtualNetworkName": "os-autotune-vnet",
+                        "subnetName": "os-autotune-subnet",
+                        "networkSecurityGroupName": "os-autotune-nsg",
 
                         "ubuntuOSVersion": "18.04-LTS"
                     }

mlos_bench/config/tunable-values-example.json

@@ -0,0 +1,5 @@
+{
+    "vmSize": "Standard_B2s",
+    "rootfs": "ext2",
+    "kernel.sched_migration_cost_ns": 40000
+}

mlos_bench/mlos_bench/environment/base_environment.py

@@ -179,7 +179,7 @@ class Environment(metaclass=abc.ABCMeta):
             True if operation is successful, false otherwise.
         """
 
-    def submit(self, tunables):
+    def submit(self, tunables: TunableGroups):
         """
         Submit a new experiment to the environment. Set up the environment,
         if necessary.
@@ -195,6 +195,7 @@ class Environment(metaclass=abc.ABCMeta):
             True if operation is successful, false otherwise.
         """
         _LOG.info("Submit: %s", tunables)
+        assert isinstance(tunables, TunableGroups)
         if self.setup():
             return self.run(tunables)
         return False

mlos_bench/mlos_bench/environment/composite.py

@@ -40,11 +40,6 @@ class CompositeEnv(Environment):
 
         self._children = []
         for child_config in config["children"]:
-
-            local_config = shared_config.copy()
-            local_config.update(child_config.get("config", {}))
-            child_config["config"] = local_config
-
             env = build_environment(child_config, shared_config, tunables, self._service)
             self._children.append(env)
             self._tunable_params.update(env.tunable_params())

mlos_bench/mlos_bench/environment/persistence.py

@@ -45,12 +45,9 @@ def build_environment(config, global_config=None, tunables=None, service=None):
 
     env_name = config["name"]
     env_class = config["class"]
-    env_config = config.get("config", {})
 
-    if global_config:
-        local_config = global_config.copy()
-        local_config.update(env_config)
-        env_config = local_config
+    env_config = config.setdefault("config", {})
+    env_config.update(global_config or {})
 
     env_services_path = config.get("include_services")
     if env_services_path is not None:
@@ -86,12 +83,9 @@ def _build_standalone_service(config, global_config=None):
         An instance of the `Service` class initialized with `config`.
     """
     svc_class = config["class"]
-    svc_config = config.get("config", {})
 
-    if global_config:
-        local_config = global_config.copy()
-        local_config.update(svc_config)
-        svc_config = local_config
+    svc_config = config.setdefault("config", {})
+    svc_config.update(global_config or {})
 
     _LOG.debug("Creating service: %s", svc_class)
     service = Service.new(svc_class, svc_config)

mlos_bench/mlos_bench/launcher.py

@@ -0,0 +1,105 @@
+"""
+Helper functions to launch the benchmark and the optimizer from the command line.
+"""
+
+import json
+import logging
+import argparse
+
+from mlos_bench.environment.persistence import load_environment
+
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s',
+    datefmt='%H:%M:%S'
+)
+
+_LOG = logging.getLogger(__name__)
+
+
+class Launcher:
+    """
+    Common parts of the OS Autotune command line launcher.
+    """
+
+    def __init__(self, description='OS Autotune launcher'):
+
+        _LOG.info("Launch: %s", description)
+
+        self._env_config_file = None
+        self._global_config = {}
+        self._parser = argparse.ArgumentParser(description=description)
+
+        self._parser.add_argument(
+            '--config', required=True,
+            help='Path to JSON file with the configuration'
+                 ' of the benchmarking environment')
+
+        self._parser.add_argument(
+            '--global', required=False, dest='global_config',
+            help='Path to JSON file that contains additional (sensitive)'
+                 ' parameters of the benchmarking environment')
+
+    @property
+    def parser(self):
+        """
+        Get the command line parser (so we can add more arguments to it).
+        """
+        return self._parser
+
+    def parse_args(self):
+        """
+        Parse command line arguments and load global config parameters.
+        """
+        (args, args_rest) = self._parser.parse_known_args()
+        self._env_config_file = args.config
+        if args.global_config is not None:
+            self._global_config = Launcher.load_config(args.global_config)
+        self._global_config.update(Launcher._try_parse_extra_args(args_rest))
+        return args
+
+    @staticmethod
+    def load_config(json_file_name):
+        """
+        Load JSON config file.
+        """
+        with open(json_file_name, mode='r', encoding='utf-8') as fh_json:
+            return json.load(fh_json)
+
+    @staticmethod
+    def _try_parse_extra_args(cmdline):
+        """
+        Helper function to parse global key/value pairs from the command line.
+        """
+        _LOG.debug("Extra args: %s", cmdline)
+
+        config = {}
+        key = None
+        for elem in cmdline:
+            if elem.startswith("--"):
+                if key is not None:
+                    raise ValueError("Command line argument has no value: " + key)
+                key = elem[2:]
+                kv_split = key.split("=", 1)
+                if len(kv_split) == 2:
+                    config[kv_split[0].strip()] = kv_split[1]
+                    key = None
+            else:
+                if key is None:
+                    raise ValueError("Command line argument has no key: " + elem)
+                config[key.strip()] = elem
+                key = None
+
+        if key is not None:
+            raise ValueError("Command line argument has no value: " + key)
+
+        _LOG.debug("Parsed config: %s", config)
+        return config
+
+    def load_env(self):
+        """
+        Create a new benchmarking environment
+        from the configs and command line parameters.
+        """
+        return load_environment(self._env_config_file, self._global_config)

mlos_bench/mlos_bench/main.py

@@ -1,112 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-OS Autotune main optimization loop.
-"""
-
-import json
-import logging
-import argparse
-
-from mlos_bench.opt import Optimizer
-from mlos_bench.environment.persistence import load_environment
-
-
-def optimize(env_config_file, global_config):
-    """
-    Main optimization loop.
-    """
-    env = load_environment(env_config_file, global_config)
-
-    opt = Optimizer(env.tunable_params())
-    _LOG.info("Env: %s Optimizer: %s", env, opt)
-
-    while opt.not_converged():
-
-        tunable_values = opt.suggest()
-        _LOG.info("Suggestion: %s", tunable_values)
-        env.submit(tunable_values)
-
-        bench_result = env.result()  # Block and wait for the final result
-        _LOG.info("Result: %s = %s", tunable_values, bench_result)
-        opt.register(tunable_values, bench_result)
-
-    env.teardown()
-
-    best = opt.get_best_observation()
-    _LOG.info("Env: %s best result: %s", env, best)
-    return best
-
-###############################################################
-
-
-def _try_parse_extra_args(cmdline):
-    """
-    Helper function to parse global key/value pairs from the command line.
-    """
-    _LOG.debug("Extra args: %s", cmdline)
-
-    config = {}
-    key = None
-    for elem in cmdline:
-        if elem.startswith("--"):
-            if key is not None:
-                raise ValueError("Command line argument has no value: " + key)
-            key = elem[2:]
-            kv_split = key.split("=", 1)
-            if len(kv_split) == 2:
-                config[kv_split[0].strip()] = kv_split[1]
-                key = None
-        else:
-            if key is None:
-                raise ValueError("Command line argument has no key: " + elem)
-            config[key.strip()] = elem
-            key = None
-
-    if key is not None:
-        raise ValueError("Command line argument has no value: " + key)
-
-    _LOG.debug("Parsed config: %s", config)
-    return config
-
-
-def _main():
-
-    parser = argparse.ArgumentParser(
-        description='OS Autotune optimizer')
-
-    parser.add_argument(
-        '--config', required=True,
-        help='Path to JSON file with the configuration'
-             ' of the benchmarking environment')
-
-    parser.add_argument(
-        '--global', required=False, dest='global_config',
-        help='Path to JSON file that contains additional (sensitive)'
-             ' parameters of the benchmarking environment')
-
-    (args, args_rest) = parser.parse_known_args()
-
-    global_config = {}
-    if args.global_config is not None:
-        with open(args.global_config, encoding='utf-8') as fh_json:
-            global_config = json.load(fh_json)
-
-    global_config.update(_try_parse_extra_args(args_rest))
-
-    result = optimize(args.config, global_config)
-    _LOG.info("Final result: %s", result)
-
-###############################################################
-
-
-logging.basicConfig(
-    level=logging.DEBUG,
-    format='%(asctime)s %(pathname)s:%(lineno)d %(levelname)s %(message)s',
-    datefmt='%H:%M:%S'
-)
-
-_LOG = logging.getLogger(__name__)
-
-if __name__ == "__main__":
-    _main()

mlos_bench/mlos_bench/run_bench.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+"""
+OS Autotune benchmark launcher without involving the optimizer.
+Used mostly for development/testing purposes.
+
+See `--help` output for details.
+"""
+
+import logging
+
+from mlos_bench.launcher import Launcher
+
+_LOG = logging.getLogger(__name__)
+
+
+def _main():
+
+    launcher = Launcher("OS Autotune benchmark")
+
+    launcher.parser.add_argument(
+        '--tunables', required=True,
+        help='Path to JSON file that contains values of the tunable parameters')
+
+    args = launcher.parse_args()
+
+    tunable_values = Launcher.load_config(args.tunables)
+    env = launcher.load_env()
+
+    tunables = env.tunable_params()
+    for (key, val) in tunable_values.items():
+        tunables[key] = val
+
+    _LOG.info("Benchmark: %s with tunables:\n%s", env, tunables)
+    env.submit(tunables)
+
+    bench_result = env.result()  # Block and wait for the final result
+    _LOG.info("Result: %s", bench_result)
+
+
+if __name__ == "__main__":
+    _main()

mlos_bench/mlos_bench/run_opt.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""
+OS Autotune main optimization loop.
+
+See `--help` output for details.
+"""
+
+import logging
+
+from mlos_bench.opt import Optimizer
+from mlos_bench.launcher import Launcher
+
+_LOG = logging.getLogger(__name__)
+
+
+def _main():
+    launcher = Launcher("OS Autotune optimizer")
+    launcher.parse_args()
+    env = launcher.load_env()
+    result = optimize(env)
+    _LOG.info("Final result: %s", result)
+
+
+def optimize(env):
+    """
+    Main optimization loop.
+    """
+    opt = Optimizer(env.tunable_params())
+    _LOG.info("Env: %s Optimizer: %s", env, opt)
+
+    while opt.not_converged():
+
+        tunable_values = opt.suggest()
+        _LOG.info("Suggestion: %s", tunable_values)
+        env.submit(tunable_values)
+
+        bench_result = env.result()  # Block and wait for the final result
+        _LOG.info("Result: %s = %s", tunable_values, bench_result)
+        opt.register(tunable_values, bench_result)
+
+    best = opt.get_best_observation()
+    _LOG.info("Env: %s best result: %s", env, best)
+    return best
+
+
+if __name__ == "__main__":
+    _main()