diff --git a/.gitignore b/.gitignore index ed9b26b6de..dbff8f0077 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,9 @@ cython_debug/ # vim swap files .*.swp +.*.swo + +junit/test-results.xml junit/test-results.xml @@ -147,3 +150,12 @@ junit/test-results.xml .doc-prereqs.build-stamp .pylint.build-stamp .pytest.build-stamp + +.vs/ +.vscode/ +!.vscode/settings.json +!.vscode/extensions.json + +# Test config. May contain sensitive information +# like Azure credentials etc. +/config.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000000..83f97bc6c7 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,11 @@ +{ + "recommendations": [ + "EditorConfig.EditorConfig", + "streetsidesoftware.code-spell-checker", + "ms-python.vscode-pylance", + "ms-python.python", + "donjayamanne.python-environment-manager", + "lextudio.restructuredtext", + "trond-snekvik.simple-rst" + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 9f88d34589..317c200e1c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { - "python.defaultInterpreterPath": "${env:HOME}/.conda/envs/mlos_core/bin/python", + "makefile.extensionOutputFolder": "./.vscode", + "python.defaultInterpreterPath": "${env:HOME}${env:USERPROFILE}/.conda/envs/mlos_core/bin/python", "python.linting.enabled": true, "python.linting.pylintEnabled": true, "python.testing.pytestEnabled": true, @@ -7,6 +8,7 @@ "Skopt", "conda", "configspace", + "dataframe", "emukit", "gpbo", "ipykernel", @@ -26,5 +28,10 @@ "tolist", "xlabel", "ylabel" - ] + ], + "restructuredtext.linter.doc8.extraArgs": [ + "--ignore D001" + ], + "esbonio.sphinx.confDir": "${workspaceFolder}/doc/source", + "esbonio.sphinx.buildDir": "${workspaceFolder}/doc/build/" } diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000000..9e841e7a26 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE diff --git a/Makefile b/Makefile index 9af4b922af..32141e1159 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ check: pylint pylint: conda-env .pylint.build-stamp .pylint.build-stamp: $(PYTHON_FILES) .pylintrc - conda run -n ${CONDA_DEFAULT_ENV} pylint -j0 mlos_core + conda run -n ${CONDA_DEFAULT_ENV} pylint -j0 mlos_core mlos_bench touch .pylint.build-stamp .PHONY: test @@ -32,17 +32,17 @@ test: pytest pytest: conda-env .pytest.build-stamp .pytest.build-stamp: $(PYTHON_FILES) pytest.ini - #conda run -n ${CONDA_DEFAULT_ENV} pytest -n auto --cov=mlos_core --cov-report=xml mlos_core/ - conda run -n ${CONDA_DEFAULT_ENV} pytest --cov=mlos_core --cov-report=xml mlos_core/ --junitxml=junit/test-results.xml + #conda run -n ${CONDA_DEFAULT_ENV} pytest -n auto --cov=mlos_core --cov-report=xml mlos_core/ mlos_bench/ + conda run -n ${CONDA_DEFAULT_ENV} pytest --cov --cov-report=xml mlos_core/ mlos_bench/ --junitxml=junit/test-results.xml touch .pytest.build-stamp .PHONY: dist dist: bdist_wheel .PHONY: bdist_wheel -bdist_wheel: conda-env dist/mlos_core-*-py3-none-any.whl +bdist_wheel: conda-env dist/mlos_core-*-py3-none-any.whl dist/mlos_bench-*-py3-none-any.whl -dist/mlos_core-*-py3-none-any.whl: setup.py $(PYTHON_FILES) +dist/mlos_bench-*-py3-none-any.whl dist/mlos_core-*-py3-none-any.whl: setup.py $(PYTHON_FILES) conda run -n ${CONDA_DEFAULT_ENV} python3 setup.py bdist_wheel .doc-prereqs.build-stamp: doc/requirements.txt @@ -54,8 +54,9 @@ doc-prereqs: .doc-prereqs.build-stamp .PHONY: doc doc: conda-env doc-prereqs - cd doc/ && conda run -n ${CONDA_DEFAULT_ENV} sphinx-apidoc -f -e -o source/api ../mlos_core - conda run -n ${CONDA_DEFAULT_ENV} make -C doc/ html + rm -f doc/build/html/index.html + cd doc/ && conda run -n ${CONDA_DEFAULT_ENV} sphinx-apidoc -f -e -M -o source/api .. ../setup.py ../pytest_configure.py + conda run -n ${CONDA_DEFAULT_ENV} make -j -C doc/ html test -s doc/build/html/index.html cp doc/staticwebapp.config.json doc/build/html/ @@ -80,3 +81,4 @@ dist-clean: clean: clean-check clean-test dist-clean clean-doc rm -f .conda-env.build-stamp .conda-env.*.build-stamp rm -rf mlos_core.egg-info + rm -rf mlos_bench.egg-info diff --git a/config/azure/azuredeploy.json b/config/azure/azuredeploy.json new file mode 100644 index 0000000000..787e6850fd --- /dev/null +++ b/config/azure/azuredeploy.json @@ -0,0 +1,262 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "vmName": { + "type": "string", + "defaultValue": "osat-linux-vm", + "metadata": { + "description": "OS Autotune Linux VM" + } + }, + "adminUsername": { + "type": "string", + "metadata": { + "description": "Username for the Virtual Machine." + } + }, + "authenticationType": { + "type": "string", + "defaultValue": "sshPublicKey", + "allowedValues": [ + "sshPublicKey", + "password" + ], + "metadata": { + "description": "Type of authentication to use on the Virtual Machine. SSH key is recommended." + } + }, + "adminPasswordOrKey": { + "type": "secureString", + "metadata": { + "description": "SSH Key or password for the Virtual Machine. SSH key is recommended." + } + }, + "dnsLabelPrefix": { + "type": "string", + "defaultValue": "[toLower(format('{0}-{1}', parameters('vmName'), uniqueString(resourceGroup().id)))]", + "metadata": { + "description": "Unique DNS Name for the Public IP used to access the Virtual Machine." + } + }, + "ubuntuOSVersion": { + "type": "string", + "defaultValue": "18.04-LTS", + "allowedValues": [ + "12.04.5-LTS", + "14.04.5-LTS", + "16.04.0-LTS", + "18.04-LTS" + ], + "metadata": { + "description": "The Ubuntu version for the VM. This will pick a fully patched image of this given Ubuntu version." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + }, + "vmSize": { + "type": "string", + "defaultValue": "Standard_B2s", + "metadata": { + "description": "The size of the VM" + } + }, + "virtualNetworkName": { + "type": "string", + "defaultValue": "osat-vnet", + "metadata": { + "description": "Name of the VNET" + } + }, + "subnetName": { + "type": "string", + "defaultValue": "osat-subnet", + "metadata": { + "description": "Name of the subnet in the virtual network" + } + }, + "networkSecurityGroupName": { + "type": "string", + "defaultValue": "osat-sg", + "metadata": { + "description": "Name of the Network Security Group" + } + } + }, + "variables": { + "publicIPAddressName": "[format('{0}PublicIP', parameters('vmName'))]", + "networkInterfaceName": "[format('{0}NetInt', parameters('vmName'))]", + "osDiskType": "Standard_LRS", + "subnetAddressPrefix": "10.1.0.0/24", + "addressPrefix": "10.1.0.0/16", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[format('/home/{0}/.ssh/authorized_keys', parameters('adminUsername'))]", + "keyData": "[parameters('adminPasswordOrKey')]" + } + ] + } + } + }, + "resources": [ + { + "type": "Microsoft.Network/networkInterfaces", + "apiVersion": "2021-05-01", + "name": "[variables('networkInterfaceName')]", + "location": "[parameters('location')]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtualNetworkName'), parameters('subnetName'))]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('networkSecurityGroupName'))]", + "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))]", + "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtualNetworkName'), parameters('subnetName'))]" + ] + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "apiVersion": "2021-05-01", + "name": "[parameters('networkSecurityGroupName')]", + "location": "[parameters('location')]", + "properties": { + "securityRules": [ + { + "name": "SSH", + "properties": { + "priority": 1000, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22" + } + } + ] + } + }, + { + "type": "Microsoft.Network/virtualNetworks", + "apiVersion": "2021-05-01", + "name": "[parameters('virtualNetworkName')]", + "location": "[parameters('location')]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('addressPrefix')]" + ] + } + } + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "apiVersion": "2021-05-01", + "name": "[format('{0}/{1}', parameters('virtualNetworkName'), parameters('subnetName'))]", + "properties": { + "addressPrefix": "[variables('subnetAddressPrefix')]", + "privateEndpointNetworkPolicies": "Enabled", + "privateLinkServiceNetworkPolicies": "Enabled" + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', parameters('virtualNetworkName'))]" + ] + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "apiVersion": "2021-05-01", + "name": "[variables('publicIPAddressName')]", + "location": "[parameters('location')]", + "sku": { + "name": "Basic" + }, + "properties": { + "publicIPAllocationMethod": "Dynamic", + "publicIPAddressVersion": "IPv4", + "dnsSettings": { + "domainNameLabel": "[parameters('dnsLabelPrefix')]" + }, + "idleTimeoutInMinutes": 4 + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "apiVersion": "2021-11-01", + "name": "[parameters('vmName')]", + "location": "[parameters('location')]", + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "storageProfile": { + "osDisk": { + "createOption": "FromImage", + "managedDisk": { + "storageAccountType": "[variables('osDiskType')]" + } + }, + "imageReference": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "[parameters('ubuntuOSVersion')]", + "version": "latest" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + } + ] + }, + "osProfile": { + "computerName": "[parameters('vmName')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPasswordOrKey')]", + "linuxConfiguration": "[if(equals(parameters('authenticationType'), 'password'), null(), variables('linuxConfiguration'))]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]" + ] + } + ], + "outputs": { + "adminUsername": { + "type": "string", + "value": "[parameters('adminUsername')]" + }, + "hostname": { + "type": "string", + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))).dnsSettings.fqdn]" + }, + "sshCommand": { + "type": "string", + "value": "[format('ssh {0}@{1}', parameters('adminUsername'), reference(resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPAddressName'))).dnsSettings.fqdn)]" + } + } +} \ No newline at end of file diff --git a/config/azure/azuredeploy.parameters.json b/config/azure/azuredeploy.parameters.json new file mode 100644 index 0000000000..60c43a42f6 --- /dev/null +++ b/config/azure/azuredeploy.parameters.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "adminUsername": { + "value": "GEN-UNIQUE" + }, + "adminPasswordOrKey": { + "value": "GEN-PASSWORD" + }, + "dnsLabelPrefix": { + "value": "GEN-UNIQUE" + } + } +} diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000000..96c792f803 --- /dev/null +++ b/config/config.json @@ -0,0 +1,104 @@ +{ + "name": "Azure VM Ubuntu Redis", + "class": "mlos_bench.environment.CompositeEnv", + + "config": { + + "services": [ + { + "class": "mlos_bench.environment.azure.AzureVMService", + + "config": { + "template_path": "./config/azure/azuredeploy.json", + + "subscription": "...", + "resource_group": "sergiym-os-autotune", + "deployment_name": "sergiym-os-autotune-001", + "vmName": "osat-linux-vm", + + "accessToken": "AZURE ACCESS TOKEN (e.g., from `az account get-access-token`)" + } + } + ], + + "children": [ + { + "name": "Deploy Ubuntu VM on Azure", + "class": "mlos_bench.environment.azure.VMEnv", + + "config": { + + "cost": 1000, + + "tunable_params": { + "vmSize": { + "type": "categorical", + "default": "Standard_B4ms", + "values": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"] + } + }, + + "const_args": { + + "adminUsername": "sergiym", + "authenticationType": "sshPublicKey", + "adminPasswordOrKey": "SSH PUBLIC KEY (e.g., from id_rsa.pub)", + + "virtualNetworkName": "sergiym-osat-vnet", + "subnetName": "sergiym-osat-subnet", + "networkSecurityGroupName": "sergiym-osat-sg", + + "ubuntuOSVersion": "18.04-LTS" + } + } + }, + { + "name": "Boot Ubuntu VM on Azure", + "class": "mlos_bench.environment.azure.OSEnv", + + "config": { + + "cost": 300, + + "tunable_params": { + "rootfs": { + "type": "categorical", + "default": "xfs", + "values": ["xfs", "ext4", "ext2"] + } + }, + + "const_args": { + } + } + }, + { + "name": "Redis on Linux", + "class": "mlos_bench.environment.AppEnv", + + "config": { + + "cost": 1, + + "tunable_params": { + "kernel.sched_migration_cost_ns": { + "type": "int", + "default": -1, + "range": [0, 500000], + "special": [-1] + } + }, + + "const_args": { + "commandId": "RunBenchmark", + "script": [ + "ls -l /", + "uname -a", + "sysctl kernel.sched_migration_cost_ns=${kernel.sched_migration_cost_ns}" + ] + } + } + } + ] + } +} diff --git a/db/osat.sql b/db/osat.sql new file mode 100644 index 0000000000..a2bcda717b --- /dev/null +++ b/db/osat.sql @@ -0,0 +1,92 @@ + +DROP TABLE IF EXISTS benchmark CASCADE; +DROP TABLE IF EXISTS benchmark_status CASCADE; +DROP TABLE IF EXISTS environment CASCADE; +DROP TABLE IF EXISTS composite_environment CASCADE; +DROP TABLE IF EXISTS experiment CASCADE; +DROP TABLE IF EXISTS tunable_parameter CASCADE; +DROP TABLE IF EXISTS tunable_value CASCADE; + +DROP TYPE IF EXISTS string_id_t CASCADE; +DROP TYPE IF EXISTS benchmark_status_t CASCADE; + +CREATE DOMAIN string_id_t AS varchar(255) NOT NULL; + +CREATE TYPE benchmark_status_t AS ENUM ( + 'pending', 'running', 'failed', 'canceled', 'completed'); + +-- Each environment is a collection of scripts and configuration templates +-- required to run an experiment, along with a (Python) class name that +-- contains the code that actually launches the scripts and runs the benchmarks. +CREATE TABLE environment ( + id string_id_t PRIMARY KEY, + name text NOT NULL, + class string_id_t, -- Python class that implements the experiment + config_path text, -- (relative) git path to scripts and config templates + config_version text, -- git branch or commit id of the scripts + parameters json, -- Static parameters to plug into the config + cost float -- Cost of changing the parameters' values. +); + +-- Composite environments are trees of environment instances. +-- environment.class of the root environment is a (Python) class that +-- implements the composition. +CREATE TABLE composite_environment ( + root_id string_id_t REFERENCES environment(id), + parent_id string_id_t REFERENCES environment(id), + child_id string_id_t REFERENCES environment(id), + + PRIMARY KEY (root_id, parent_id, child_id) +); + +-- An experiment is a series of benchmarks for the given environment. +CREATE TABLE experiment ( + id string_id_t PRIMARY KEY, + environment_id string_id_t REFERENCES environment(id), + ts timestamp NOT NULL DEFAULT now(), + parameters json -- Parameters to plug into the environment config +); + +CREATE TABLE benchmark ( + id serial NOT NULL PRIMARY KEY, + experiment_id string_id_t REFERENCES experiment(id), + ts timestamp NOT NULL DEFAULT now(), + parameters json, -- Benchmark-specific parameters, e.g., VM id (NOT tunables!) + final_status benchmark_status_t, + final_result float +); + +CREATE TABLE benchmark_status ( + benchmark_id integer NOT NULL REFERENCES benchmark(id), + ts timestamp NOT NULL DEFAULT now(), + status benchmark_status_t NOT NULL, + result float, + telemetry json, + + PRIMARY KEY (benchmark_id, ts) +); + +-- Tunable parameters' descriptions. +-- Should be deserializeable into ConfigSpace. +CREATE TABLE tunable_parameter ( + environment_id string_id_t REFERENCES environment(id), + name string_id_t, + type string_id_t, + range json, + default_value json, + + PRIMARY KEY (environment_id, name) +); + +-- Values of the tunables for a given benchmark. +-- (Maybe, store as JSON column in the benchmark table?) +CREATE TABLE tunable_value ( + benchmark_id integer NOT NULL REFERENCES benchmark(id), + environment_id string_id_t REFERENCES environment(id), + name string_id_t, + value json, + + PRIMARY KEY (benchmark_id, name), + FOREIGN KEY(environment_id, name) + REFERENCES tunable_parameter(environment_id, name) +); diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000000..dba662f7f5 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,15 @@ +# Documentation Generation + +Documentation is generated using [`sphinx`](https://www.sphinx-doc.org/). + +```sh +make -C .. doc +``` + +## Testing with Docker + +```sh +./nginx-docker.sh restart +``` + +> Now browse to `http://localhost` diff --git a/doc/nginx-default.conf b/doc/nginx-default.conf new file mode 100644 index 0000000000..a5965a488b --- /dev/null +++ b/doc/nginx-default.conf @@ -0,0 +1,49 @@ +# vim: set ft=nginx: +server { + listen 80; + listen [::]:80; + server_name localhost; + + #access_log /var/log/nginx/host.access.log main; + + # Expects ./doc to be mapped to /doc in the nginx container. + location / { + #root /usr/share/nginx/html; + root /doc/build/html/; + autoindex on; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + # proxy the PHP scripts to Apache listening on 127.0.0.1:80 + # + #location ~ \.php$ { + # proxy_pass http://127.0.0.1; + #} + + # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000 + # + #location ~ \.php$ { + # root html; + # fastcgi_pass 127.0.0.1:9000; + # fastcgi_index index.php; + # fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name; + # include fastcgi_params; + #} + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + # + #location ~ /\.ht { + # deny all; + #} +} + diff --git a/doc/nginx-docker.sh b/doc/nginx-docker.sh new file mode 100755 index 0000000000..a01bf1a3bf --- /dev/null +++ b/doc/nginx-docker.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# A quick script to start a local webserver for testing the sphinx documentation. + +scriptdir=$(dirname "$(readlink -f "$0")") +cd "$scriptdir" + +if [ "$1" == 'start' ]; then + docker run -d --name mlos-doc-nginx -v $PWD/nginx-default.conf:/etc/nginx/conf.d/default.conf -v $PWD:/doc -p 80:80 nginx +elif [ "$1" == 'stop' ]; then + docker stop mlos-doc-nginx || true + docker rm mlos-doc-nginx || true +elif [ "$1" == 'restart' ]; then + "$0" 'stop' + "$0" 'start' +else + echo "ERROR: Invalid argument: $0." >&2 + echo "Usage: $0 [start|stop|restart]" + exit 1 +fi diff --git a/doc/source/api.rst b/doc/source/api.rst deleted file mode 100644 index c8ef10d61c..0000000000 --- a/doc/source/api.rst +++ /dev/null @@ -1,32 +0,0 @@ -############################# -API -############################# - -This is a list of all functions and classes provided by mlos_core. - -.. currentmodule:: mlos_core - -Optimizers -============== -.. currentmodule:: mlos_core.optimizers - -.. autosummary:: - :toctree: generated/ - :template: class.rst - - RandomOptimizer - EmukitOptimizer - SkoptOptimizer - - -Spaces -========= - - -.. currentmodule:: mlos_core.spaces -.. autosummary:: - :toctree: generated/ - :template: function.rst - - configspace_to_emukit_space - configspace_to_skopt_space diff --git a/doc/source/conf.py b/doc/source/conf.py index 75180cc264..c3b74a2f83 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -21,7 +21,7 @@ sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- -project = 'mlos-core' +project = 'MlosCore' copyright = '2022, GSL' author = 'GSL' diff --git a/doc/source/index.rst b/doc/source/index.rst index fde919be11..d8e3d351f7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,7 +1,7 @@ -Welcome to mlos-core's documentation! -===================================== +Welcome to MlosCore's documentation! +==================================== -This repository contains a stripped down implementation of essentially just the core optimizer and config space description APIs from the original `MLOS `_. +This repository contains a stripped down implementation of essentially just the core optimizer and config space description APIs from the original `MLOS `_ as well as the `mlos-bench` module intended to help automate and manage running experiments for autotuning systems with `mlos-core`. It is intended to provide a simplified, easier to consume (e.g. via ``pip``), with lower dependencies abstraction to @@ -9,7 +9,7 @@ It is intended to provide a simplified, easier to consume (e.g. via ``pip``), wi - an "optimizer" service abstraction (e.g. ``register()`` and ``suggest()``) so we can easily swap out different implementations methods of searching (e.g. random, BO, etc.) - provide some helpers for automating optimization experiment runner loops and data collection -For these design requirements we intend to reuse as much from existing OSS libraries as possible. +For these design requirements we intend to reuse as much from existing OSS libraries as possible and layer policies and optimizations specifically geared towards autotuning over top. .. toctree:: :hidden: @@ -17,7 +17,14 @@ For these design requirements we intend to reuse as much from existing OSS libra :caption: Documentation installation - api + overview + +.. toctree:: + :hidden: + :maxdepth: 4 + :caption: API Reference + + api/modules .. toctree:: :maxdepth: 2 diff --git a/doc/source/installation.rst b/doc/source/installation.rst index c32ad96f60..caaf2e82fb 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -6,13 +6,13 @@ Development 0. Create the `mlos_core` Conda environment. -.. code-block:: shell + .. code-block:: shell conda env create -f conda-envs/mlos_core.yml -or + or -.. code-block:: shell + .. code-block:: shell # This will also ensure the environment is update to date using "conda env update -f conda-envs/mlos_core.yml" make conda-env @@ -20,7 +20,7 @@ or 1. Initialize the shell environment. -.. code-block:: shell + .. code-block:: shell conda activate mlos_core @@ -31,13 +31,13 @@ Distributing 1. Build the *wheel* file. -.. code-block:: shell + .. code-block:: shell make dist 2. Install it (e.g. after copying it somewhere else). -.. code-block:: shell + .. code-block:: shell # this will install it with emukit support: pip install dist/mlos_core-0.0.3-py3-none-any.whl[emukit] diff --git a/doc/source/overview.rst b/doc/source/overview.rst new file mode 100644 index 0000000000..2ed5275666 --- /dev/null +++ b/doc/source/overview.rst @@ -0,0 +1,81 @@ +############################# +mlos-core API +############################# + +This is a list of major functions and classes provided by `mlos_core`. + +.. currentmodule:: mlos_core + +Optimizers +============== +.. currentmodule:: mlos_core.optimizers +.. autosummary:: + :toctree: generated/ + :template: class.rst + + BaseOptimizer + RandomOptimizer + EmukitOptimizer + SkoptOptimizer + + +Spaces +========= +.. currentmodule:: mlos_core.spaces +.. autosummary:: + :toctree: generated/ + :template: function.rst + + configspace_to_emukit_space + configspace_to_skopt_space + +############################# +mlos-bench API +############################# + +This is a list of major functions and classes provided by `mlos_bench`. + +.. currentmodule:: mlos_bench + +Main +==== +.. currentmodule:: mlos_bench.main +.. autosummary:: + :toctree: generated/ + :template: functions.rst + + optimize + +Optimizer +========= +.. currentmodule:: mlos_bench.opt +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Optimizer + +Environments +============ +.. currentmodule:: mlos_bench.environment +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Environment + AppEnv + CompositeEnv + Service + Status + +Azure +----- + +.. currentmodule:: mlos_bench.environment.azure +.. autosummary:: + :toctree: generated/ + :template: class.rst + + OSEnv + VMEnv + AzureVMService diff --git a/mlos_bench/__init__.py b/mlos_bench/__init__.py new file mode 100644 index 0000000000..a99203dd0d --- /dev/null +++ b/mlos_bench/__init__.py @@ -0,0 +1,3 @@ +""" +OS Autotune project. +""" diff --git a/mlos_bench/environment/__init__.py b/mlos_bench/environment/__init__.py new file mode 100644 index 0000000000..f0deb31e3c --- /dev/null +++ b/mlos_bench/environment/__init__.py @@ -0,0 +1,21 @@ +""" +Benchmarking environments for OS Autotune. +""" + +from mlos_bench.environment.status import Status +from mlos_bench.environment.base_svc import Service +from mlos_bench.environment.base_env import Environment + +from mlos_bench.environment.app import AppEnv +from mlos_bench.environment.composite import CompositeEnv +from mlos_bench.environment import azure + + +__all__ = [ + 'Status', + 'Service', + 'Environment', + 'AppEnv', + 'CompositeEnv', + 'azure', +] diff --git a/mlos_bench/environment/app.py b/mlos_bench/environment/app.py new file mode 100644 index 0000000000..468e553d82 --- /dev/null +++ b/mlos_bench/environment/app.py @@ -0,0 +1,74 @@ +"Application-specific benchmark environment." + +import json +import logging + +from mlos_bench.environment import Environment, Status + +_LOG = logging.getLogger(__name__) + + +class AppEnv(Environment): + "Application-level benchmark environment." + + def setup(self): + """ + Check if the environment is ready and set up the application + and benchmarks, if necessary. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Set up") + return True + + def run(self, tunables): + """ + Submit a new experiment to the application environment. + (Re)configure an application and launch the benchmark. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) of the OS and application + parameters. Setting these parameters should not require an + OS reboot. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Run: %s", tunables) + + # FIXME: Plug in the tunables into the script for remote execution + # params = self._combine_tunables(tunables) + params = self._const_args + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Benchmark:\n%s", json.dumps(params, indent=2)) + + # TODO: Configure the application and start the benchmark + (status, _output) = self._service.remote_exec(params) + return status in {Status.PENDING, Status.READY} + + def result(self): + """ + Get the results of the benchmark. This is a blocking call that waits + for the completion of the benchmark. It can have PENDING status only if + the environment object has been read from the storage and not updated + with the actual status yet. + + Returns + ------- + (benchmark_status, benchmark_result) : (enum, float) + A pair of (benchmark status, benchmark result) values. + benchmark_status is of type mlos_bench.environment.Status. + benchmark_result is a floating point time of the benchmark in + seconds or None if the status is not COMPLETED. + """ + self._result = (Status.COMPLETED, 123.456) + _LOG.info("Benchmark result: %s", self._result) + return self._result diff --git a/mlos_bench/environment/azure/__init__.py b/mlos_bench/environment/azure/__init__.py new file mode 100644 index 0000000000..2eedd591bb --- /dev/null +++ b/mlos_bench/environment/azure/__init__.py @@ -0,0 +1,14 @@ +""" +Azure-specific benchmark environments for OS Autotune. +""" + +from mlos_bench.environment.azure.azure_vm import VMEnv +from mlos_bench.environment.azure.azure_os import OSEnv +from mlos_bench.environment.azure.azure_services import AzureVMService + + +__all__ = [ + 'VMEnv', + 'OSEnv', + 'AzureVMService' +] diff --git a/mlos_bench/environment/azure/azure_os.py b/mlos_bench/environment/azure/azure_os.py new file mode 100644 index 0000000000..96e0888f58 --- /dev/null +++ b/mlos_bench/environment/azure/azure_os.py @@ -0,0 +1,60 @@ +"OS-level benchmark environment on Azure." + +import json +import logging + +from mlos_bench.environment import Environment, Status + +_LOG = logging.getLogger(__name__) + + +class OSEnv(Environment): + "Boot-time environment for Azure VM." + + def setup(self): + """ + Check if the Azure VM is provisioned and can be booted. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Set up") + return True + + def teardown(self): + """ + Clean up and shut down the VM without deprovisioning it. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Tear down") + return True + + def run(self, tunables): + """ + Check if Azure VM is up and running. (Re)boot it, if necessary. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) of the OS boot-time parameters. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Run: %s", tunables) + params = self._combine_tunables(tunables) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Start VM:\n%s", json.dumps(params, indent=2)) + + # TODO: Reboot the OS when config parameters change + (status, _output) = self._service.vm_start(params) + return status in {Status.PENDING, Status.READY} diff --git a/mlos_bench/environment/azure/azure_services.py b/mlos_bench/environment/azure/azure_services.py new file mode 100644 index 0000000000..c8ad97c124 --- /dev/null +++ b/mlos_bench/environment/azure/azure_services.py @@ -0,0 +1,234 @@ +"OS-level benchmark environment on Azure." + +import json +import logging +import requests + +from mlos_bench.environment.status import Status +from mlos_bench.environment.base_svc import Service + + +_LOG = logging.getLogger(__name__) + + +class AzureVMService(Service): + "Helper methods to manage VMs on Azure." + + # Azure REST API calls as described in + # https://docs.microsoft.com/en-us/rest/api/compute/virtual-machines + + _URL_DEPLOY = "https://management.azure.com" \ + "/subscriptions/%s" \ + "/resourceGroups/%s" \ + "/providers/Microsoft.Resources" \ + "/deployments/%s" \ + "?api-version=2022-05-01" + + _URL_START = "https://management.azure.com" \ + "/subscriptions/%s" \ + "/resourceGroups/%s" \ + "/providers/Microsoft.Compute" \ + "/virtualMachines/%s" \ + "/start?api-version=2022-03-01" + + _URL_RUN = "https://management.azure.com/" \ + "/subscriptions/%s" \ + "/resourceGroups/%s" \ + "/providers/Microsoft.Compute" \ + "/virtualMachines/%s" \ + "/runCommand?api-version=2022-03-01" + + def __init__(self, config): + """ + Create a new instance of Azure services proxy. + + Parameters + ---------- + config : dict + Free-format dictionary that contains the benchmark environment + configuration. + """ + super().__init__(config) + self.register([self.vm_deploy, self.vm_start, self.remote_exec]) + + with open(config['template_path']) as fh_json: + self._template = json.load(fh_json) + + self._url_deploy = AzureVMService._URL_DEPLOY % ( + config["subscription"], + config["resource_group"], + config["deployment_name"] + ) + + self._headers = { + # Access token from `az account get-access-token`: + "Authorization": "Bearer " + config["accessToken"] + } + + self._url_start = AzureVMService._URL_START % ( + config["subscription"], + config["resource_group"], + config["vmName"] + ) + + self._url_run = AzureVMService._URL_RUN % ( + config["subscription"], + config["resource_group"], + config["vmName"] + ) + + self._headers = { + # Access token from `az account get-access-token`: + "Authorization": "Bearer " + config["accessToken"] + } + + @staticmethod + def _build_parameters(tunables): + """ + Merge tunables with other parameters and convert into + ARM Template format. + """ + return {key: {"value": val} for (key, val) in tunables.items()} + + @staticmethod + def _extract_parameters(json_data): + """ + Extract parameters from the ARM Template REST response JSON. + + Returns + ------- + parameters : dict + Flat dictionary of parameters and their values. + """ + return { + key: val.get("value") + for (key, val) in json_data.get( + "properties", {}).get("parameters", {}).items() + } + + def vm_deploy(self, tunables): + """ + Check if Azure VM is ready. (Re)provision it, if necessary. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + VMEnv tunables are variable parameters that, together with the + VMEnv configuration, are sufficient to provision a VM. + + Returns + ------- + result : (Status, dict={}) + A pair of Status and result. The result is always {}. + Status is one of {PENDING, READY, FAILED} + """ + _LOG.info("Deploy VM: %s :: %s", self.config["vmName"], tunables) + + json_req = { + "properties": { + "mode": "Incremental", + "template": self._template, + "parameters": AzureVMService._build_parameters(tunables) + } + } + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Request: PUT %s\n%s", + self._url_deploy, json.dumps(json_req, indent=2)) + + response = requests.put( + self._url_deploy, headers=self._headers, json=json_req) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Response: %s\n%s", response, + json.dumps(response.json(), indent=2)) + else: + _LOG.info("Response: %s", response) + + if response.status_code == 200: + params = AzureVMService._extract_parameters(response.json()) + _LOG.info("Extracted parameters: %s", params) + return (Status.READY, params) + elif response.status_code == 201: + return (Status.PENDING, {}) + else: + _LOG.error("Response: %s :: %s", response, response.text) + # _LOG.error("Bad Request:\n%s", response.request.body) + return (Status.FAILED, {}) + + def vm_start(self, tunables): + """ + Start the VM on Azure. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + + Returns + ------- + result : (Status, dict={}) + A pair of Status and result. The result is always {}. + Status is one of {PENDING, READY, FAILED} + """ + _LOG.info("Start VM: %s :: %s", self.config["vmName"], tunables) + _LOG.debug("Request: POST %s", self._url_start) + + response = requests.post(self._url_start, headers=self._headers) + _LOG.info("Response: %s", response) + + if response.status_code == 200: + return (Status.PENDING, {}) + elif response.status_code == 202: + return (Status.READY, {}) + else: + _LOG.error("Response: %s :: %s", response, response.text) + # _LOG.error("Bad Request:\n%s", response.request.body) + return (Status.FAILED, {}) + + def remote_exec(self, tunables): + """ + Run a command on Azure VM. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + Must have "commandId", "parameters", or "script" keys. + + Returns + ------- + result : (Status, dict) + A pair of Status and result. + Status is one of {PENDING, READY, FAILED} + """ + + _LOG.info("Run a command on VM: %s :: %s %s %s", + self.config["vmName"], tunables["commandId"], + tunables.get("parameters", []), + tunables.get("script", [])) + + json_req = tunables # Pass to REST request as-is. + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Request: POST %s\n%s", + self._url_run, json.dumps(json_req, indent=2)) + + response = requests.post( + self._url_run, headers=self._headers, json=json_req) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Response: %s\n%s", response, + json.dumps(response.json(), indent=2)) + else: + _LOG.info("Response: %s", response) + + if response.status_code == 200: + # TODO: extract the results from JSON response + return (Status.READY, {}) + elif response.status_code == 202: + return (Status.PENDING, {}) + else: + _LOG.error("Response: %s :: %s", response, response.text) + # _LOG.error("Bad Request:\n%s", response.request.body) + return (Status.FAILED, {}) diff --git a/mlos_bench/environment/azure/azure_vm.py b/mlos_bench/environment/azure/azure_vm.py new file mode 100644 index 0000000000..761bd53b99 --- /dev/null +++ b/mlos_bench/environment/azure/azure_vm.py @@ -0,0 +1,61 @@ +"VM-level benchmark environment on Azure." + +import json +import logging + +from mlos_bench.environment import Environment, Status + +_LOG = logging.getLogger(__name__) + + +class VMEnv(Environment): + "Azure VM environment." + + def setup(self): + """ + Check if the Azure VM can be provisioned. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Set up") + return True + + def teardown(self): + """ + Shut down the VM and release it. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Tear down") + return True + + def run(self, tunables): + """ + Check if Azure VM is ready. (Re)provision and start it, if necessary. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + VMEnv tunables are variable parameters that, together with the + VMEnv configuration, are sufficient to provision and start a VM. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Run: %s", tunables) + params = self._combine_tunables(tunables) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Deploy VM:\n%s", json.dumps(params, indent=2)) + + (status, _output) = self._service.vm_deploy(params) + return status in {Status.PENDING, Status.READY} diff --git a/mlos_bench/environment/base_env.py b/mlos_bench/environment/base_env.py new file mode 100644 index 0000000000..515e04572b --- /dev/null +++ b/mlos_bench/environment/base_env.py @@ -0,0 +1,260 @@ +"A hierarchy of benchmark environments." + +import abc +import json +import logging +import importlib + +from mlos_bench.environment import Status + +_LOG = logging.getLogger(__name__) + + +class Environment(metaclass=abc.ABCMeta): + "An abstract base of all benchmark environments." + + @staticmethod + def from_config(config, service=None): + """ + Factory method for a new environment with a given config. + + Parameters + ---------- + config : dict + A dictionary with three mandatory fields: + "name": Human-readable string describing the environment; + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + service: Service + An optional service object (e.g., providing methods to + deploy or reboot a VM, etc.). + + Returns + ------- + env : Environment + An instance of the `Environment` class initialized with `config`. + """ + env_name = config["name"] + env_class = config["class"] + env_config = config["config"] + _LOG.debug("Creating env: %s :: %s", env_name, env_class) + env = Environment.new(env_name, env_class, env_config, service) + _LOG.info("Created env: %s :: %s", env_name, env) + return env + + @classmethod + def new(cls, env_name, class_name, config, service=None): + """ + Factory method for a new environment with a given config. + + Parameters + ---------- + env_name: str + Human-readable name of the environment. + class_name: str + FQN of a Python class to instantiate, e.g., + "mlos_bench.environment.azure.VMEnv". + Must be derived from the `Environment` class. + config : dict + Free-format dictionary that contains the benchmark environment + configuration. It will be passed as a constructor parameter of + the class specified by `name`. + service: Service + An optional service object (e.g., providing methods to + deploy or reboot a VM, etc.). + + Returns + ------- + env : Environment + An instance of the `Environment` class initialized with `config`. + """ + # We need to import mlos_bench to make the factory methods + # like `Environment.new()` work. + class_name_split = class_name.split(".") + module_name = ".".join(class_name_split[:-1]) + class_id = class_name_split[-1] + + env_module = importlib.import_module(module_name) + env_class = getattr(env_module, class_id) + + _LOG.info("Instantiating: %s :: class %s = %s", + env_name, class_name, env_class) + + assert issubclass(env_class, cls) + return env_class(env_name, config, service) + + def __init__(self, name, config, service=None): + """ + Create a new environment with a given config. + + Parameters + ---------- + name: str + Human-readable name of the environment. + config : dict + Free-format dictionary that contains the benchmark environment + configuration. Each config must have at least the "tunable_params" + and the "const_args" sections; the "cost" field can be omitted + and is 0 by default. + service: Service + An optional service object (e.g., providing methods to + deploy or reboot a VM, etc.). + """ + self.name = name + self.config = config + self._service = service + self._result = (Status.PENDING, None) + + self._const_args = config.get("const_args", {}) + self._tunable_params = self._parse_tunables( + config.get("tunable_params", {}), config.get("cost", 0)) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Config for: %s\n%s", + name, json.dumps(self.config, indent=2)) + + def __str__(self): + return self.name + + def __repr__(self): + return "Env: %s :: '%s'" % (self.__class__, self.name) + + def _parse_tunables(self, tunables, cost=0): + "Augment tunables with the cost." + tunables_cost = {} + for (key, val) in tunables.items(): + tunables_cost[key] = val.copy() + tunables_cost[key]["cost"] = cost + return tunables_cost + + def _combine_tunables(self, tunables): + """ + Plug tunable values into the base config. If the tunable is unknown, + ignore it (it might belong to another environment). This method should + never mutate the original config or the tunables. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + + Returns + ------- + config : dict + Free-format dictionary that contains the new environment + configuration. + """ + new_config = self._const_args.copy() + for (key, val) in tunables.items(): + if key in self._tunable_params: + new_config[key] = val + return new_config + + def tunable_params(self): + """ + Get the configuration space of the given environment. + + Returns + ------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + """ + return self._tunable_params + + def setup(self): + """ + Set up a new benchmark environment, if necessary. This method must be + idempotent, i.e., calling it several times in a row should be + equivalent to a single call. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + return True + + def teardown(self): + """ + Tear down the benchmark environment. This method must be idempotent, + i.e., calling it several times in a row should be equivalent to a + single call. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + return True + + @abc.abstractmethod + def run(self, tunables): + """ + Submit a new experiment to the environment. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + + def submit(self, tunables): + """ + Submit a new experiment to the environment. Set up the environment, + if necessary. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) pairs of tunable parameters. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.info("Submit: %s", tunables) + if self.setup(): + return self.run(tunables) + return False + + def status(self): + """ + Get the status of the environment. + + Returns + ------- + status : mlos_bench.environment.Status + Current status of the benchmark environment. + """ + return self._result[0] + + def result(self): + """ + Get the results of the benchmark. This is a blocking call that waits + for the completion of the benchmark. It can have PENDING status only if + the environment object has been read from the storage and not updated + with the actual status yet. + + Base implementation returns the results of the last .update() call. + + Returns + ------- + (benchmark_status, benchmark_result) : (enum, float) + A pair of (benchmark status, benchmark result) values. + benchmark_status is one one of: + PENDING + RUNNING + COMPLETED + CANCELED + FAILED + benchmark_result is a floating point time of the benchmark in + seconds or None if the status is not COMPLETED. + """ + _LOG.info("Result: %s", self._result) + return self._result diff --git a/mlos_bench/environment/base_svc.py b/mlos_bench/environment/base_svc.py new file mode 100644 index 0000000000..6f6d15d75a --- /dev/null +++ b/mlos_bench/environment/base_svc.py @@ -0,0 +1,139 @@ +"Base class for the service mix-ins." + +import json +import logging +import importlib + +_LOG = logging.getLogger(__name__) + + +class Service: + "An abstract base of all environment services." + + @staticmethod + def from_config(config): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + config : dict + A dictionary with two mandatory fields: + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + + Returns + ------- + svc : Service + An instance of the `Service` class initialized with `config`. + """ + svc_class = config["class"] + svc_config = config["config"] + _LOG.debug("Creating service: %s", svc_class) + service = Service.new(svc_class, svc_config) + _LOG.info("Created service: %s", service) + return service + + @staticmethod + def from_config_list(config_list, parent=None): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + config_list : a list of dict + A list where each element is a dictionary with 2 mandatory fields: + "class": FQN of a Python class to instantiate; + "config": Free-format dictionary to pass to the constructor. + parent: Service + An optional reference of the parent service to mix in. + + Returns + ------- + svc : Service + An instance of the `Service` class that is a combination of all + services from the list plus the parent mix-in. + """ + service = Service() + if parent: + service.register(parent.export()) + for config in config_list: + service.register(Service.from_config(config).export()) + _LOG.info("Created mix-in service: %s", service.export()) + return service + + @classmethod + def new(cls, class_name, config): + """ + Factory method for a new service with a given config. + + Parameters + ---------- + class_name: str + FQN of a Python class to instantiate, e.g., + "mlos_bench.environment.azure.AzureVMService". + Must be derived from the `Service` class. + config : dict + Free-format dictionary that contains the service configuration. + It will be passed as a constructor parameter of the class + specified by `class_name`. + + Returns + ------- + svc : Service + An instance of the `Service` class initialized with `config`. + """ + # We need to import mlos_bench to make the factory methods + # like `Service.new()` work. + class_name_split = class_name.split(".") + module_name = ".".join(class_name_split[:-1]) + class_id = class_name_split[-1] + + env_module = importlib.import_module(module_name) + svc_class = getattr(env_module, class_id) + _LOG.info("Instantiating: %s :: %s", class_name, svc_class) + + assert issubclass(svc_class, cls) + return svc_class(config) + + def __init__(self, config=None): + """ + Create a new service with a given config. + + Parameters + ---------- + config : dict + Free-format dictionary that contains the service configuration. + It will be passed as a constructor parameter of the class + specified by `class_name`. + """ + self.config = config or {} + self._services = {} + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Config:\n%s", json.dumps(self.config, indent=2)) + + def register(self, services): + """ + Register new mix-in services. + + Parameters + ---------- + services : dict or list + A dictionary of string -> function pairs. + """ + if not isinstance(services, dict): + services = {svc.__name__: svc for svc in services} + self._services.update(services) + self.__dict__.update(self._services) + + def export(self): + """ + Return a dictionary of functions available in this service. + + Returns + ------- + services : dict + A dictionary of string -> function pairs. + """ + return self._services diff --git a/mlos_bench/environment/composite.py b/mlos_bench/environment/composite.py new file mode 100644 index 0000000000..4cc58189a7 --- /dev/null +++ b/mlos_bench/environment/composite.py @@ -0,0 +1,106 @@ +"Composite benchmark environment." + +import logging + +from mlos_bench.environment import Environment, Service + +_LOG = logging.getLogger(__name__) + + +class CompositeEnv(Environment): + "Composite benchmark environment." + + def __init__(self, name, config, service=None): + """ + Create a new environment with a given config. + + Parameters + ---------- + name: str + Human-readable name of the environment. + config : dict + Free-format dictionary that contains the environment + configuration. Must have a "children" section. + service: Service + An optional service object (e.g., providing methods to + deploy or reboot a VM, etc.). + """ + super().__init__(name, config, service) + + # Propagate all config parameters except "children" and "services" + # to every child config. + shared_config = config.copy() + del shared_config["children"] + del shared_config["services"] + + self._service = Service.from_config_list( + config.get("services", []), parent=service) + + self._children = [] + for child_config in config["children"]: + child_config["config"].update(shared_config) + env = Environment.from_config(child_config, self._service) + self._children.append(env) + self._tunable_params.update(env.tunable_params()) + + def setup(self): + """ + Set up the children environments. + + Returns + ------- + is_success : bool + True if all children setup() operations are successful, + false otherwise. + """ + _LOG.debug("Set up: %s", self._children) + return all(env.setup() for env in self._children) + + def teardown(self): + """ + Tear down the children environments. + + Returns + ------- + is_success : bool + True if all children operations are successful, false otherwise. + """ + reverse_children = self._children.copy().reverse() + _LOG.debug("Tear down: %s", reverse_children) + return all(env.teardown() for env in reverse_children) + + def run(self, tunables): + """ + Submit a new experiment to the environment. + + Parameters + ---------- + tunables : dict + Flat dictionary of (key, value) of the parameters from all + children environments. + + Returns + ------- + is_success : bool + True if operation is successful, false otherwise. + """ + _LOG.debug("Run: %s with %s", self._children, tunables) + return all(env.run(tunables) for env in self._children) + + def result(self): + """ + Get the results of the benchmark. + + Returns + ------- + (benchmark_status, benchmark_result) : (enum, float) + A pair of (benchmark status, benchmark result) values. + benchmark_status is of type mlos_bench.environment.Status. + benchmark_result is a floating point time of the benchmark in + seconds or None if the status is not COMPLETED. + """ + # For now, we just return the result of the last child environment + # in the sequence. TODO: have a way to select the right result from + # the children, or identify which environment actually provides the + # final result that will be used in the optimization. + return self._children[-1].result() diff --git a/mlos_bench/environment/status.py b/mlos_bench/environment/status.py new file mode 100644 index 0000000000..f38c684b72 --- /dev/null +++ b/mlos_bench/environment/status.py @@ -0,0 +1,20 @@ +""" +Enum for the status of the benchmark. +""" + +import enum + + +class Status(enum.Enum): + "Enum for the status of the benchmark." + PENDING = 1 + READY = 2 + RUNNING = 3 + COMPLETED = 4 + CANCELED = 5 + FAILED = 6 + + @staticmethod + def is_good(status): + "Check if the status is not failed or canceled." + return status not in {Status.CANCELED, Status.FAILED} diff --git a/mlos_bench/main.py b/mlos_bench/main.py new file mode 100644 index 0000000000..1d9e58aad5 --- /dev/null +++ b/mlos_bench/main.py @@ -0,0 +1,59 @@ +""" +OS Autotune main optimization loop. +""" + +import sys +import json +import logging + +from mlos_bench.opt import Optimizer +from mlos_bench.environment import Environment + + +def optimize(config): + "Main optimization loop." + + env = Environment.from_config(config) + + opt = Optimizer(env.tunable_params()) + _LOG.info("Env: %s Optimizer: %s", env, opt) + + while opt.not_converged(): + + tunable_values = opt.suggest() + _LOG.info("Suggestion: %s", tunable_values) + env.submit(tunable_values) + + bench_result = env.result() # Block and wait for the final result + _LOG.info("Result: %s = %s", tunable_values, bench_result) + opt.register(tunable_values, bench_result) + + best = opt.get_best_observation() + _LOG.info("Env: %s best result: %s", env, best) + return best + +############################################################### + + +def _main(): + + with open(sys.argv[1]) as fh_json: + config = json.load(fh_json) + + if _LOG.isEnabledFor(logging.DEBUG): + _LOG.debug("Config:\n%s", json.dumps(config, indent=2)) + + result = optimize(config) + _LOG.info("Final result: %s", result) + + +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s %(pathname)s:%(lineno)d %(levelname)s %(message)s', + datefmt='%H:%M:%S' +) + +_LOG = logging.getLogger(__name__) + +if __name__ == "__main__": + _main() diff --git a/mlos_bench/opt.py b/mlos_bench/opt.py new file mode 100644 index 0000000000..ec3d4f6eaa --- /dev/null +++ b/mlos_bench/opt.py @@ -0,0 +1,46 @@ +""" +OS Autotune main optimization loop. +""" + +import logging + +_LOG = logging.getLogger(__name__) + + +class Optimizer: + "Toy random optimizer to test out the Environment API." + + _MAX_ITER = 1 + + def __init__(self, tunables): + _LOG.info("Create: %s", tunables) + self._iter_left = Optimizer._MAX_ITER + self._tunables = tunables + self._last_values = None + + def suggest(self): + "Generate the next suggestion." + # For now, get just the default values. + # FIXME: Need to iterate over the actual values. + tunables = { + key: val.get("default") for (key, val) in self._tunables.items() + } + # TODO: Populate the tunables with some random values + _LOG.info("Suggest: %s", tunables) + return tunables + + def register(self, tunables, bench): + "Register the observation for the given configuration." + (bench_status, bench_result) = bench + _LOG.info("Register: %s = %s %s", tunables, bench_status, bench_result) + self._last_values = tunables + self._iter_left -= 1 + + def not_converged(self): + "Return True if not converged, False otherwise." + return self._iter_left > 0 + + def get_best_observation(self): + "Get the best observation so far." + # FIXME: Use the tunables' values, as passed into .register() + return (self._last_values, 0.0) diff --git a/setup.py b/setup.py index 8d6acdf297..19896dd41b 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,15 @@ """ -Setup instructions for the mlos_core package. +Setup instructions for the mlos_core and mlos_bench packages. """ from setuptools import setup, find_packages +version='0.0.4' + +# TODO: Create separate whl packages for mlos-core and mlos-bench? setup( - name="mlos-core", - version="0.0.3", + name='mlos-core', + version=version, packages=find_packages(), install_requires=[ 'scikit-learn>=0.22.1', @@ -19,10 +22,10 @@ setup( 'emukit': 'emukit', 'skopt': 'scikit-optimize', }, - author="Microsoft", - author_email="amueller@microsoft.com", - description=("MLOS Core Python interface for parameter optimization."), - license="", - keywords="", - #python_requires='>=3.7', + author='Microsoft', + author_email='mlos-maintainers@service.microsoft.com', + description=('MLOS Core Python interface for parameter optimization.'), + license='MIT', + keywords='', + # python_requires='>=3.7', )