зеркало из https://github.com/microsoft/MLOS.git
Various documentation improvements (#912)
# Pull Request ## Title Various documentation improvements --- ## Description - [x] Some spelling fixups - [x] sidebar navigation depth tweaks - [x] Tunables improvements - [x] Split type related things out from tunable.py to tunable_types.py - [x] Associated import fixups - [x] Add Examples to Tunable class - [x] Optimizer documentation improvements - [x] Add more examples - [x] Add more cross references - [x] "See Also" rendering mixups. - [x] mlos_viz dabl doc fixups - [x] LlamaTune doc fixups - [ ] ~Environments~ (future PR) - [ ] ~Services~ (future PR) - [x] Disable Azure intersphinx resolution: https://github.com/Azure/azure-sdk-for-python/issues/39316, https://github.com/Quansight-Labs/intersphinx_registry/issues/57 See Also: #891 --- ## Type of Change - Documentation update - Refactor --- --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Brian Kroth
GitHub
Родитель
6b30ee0177
Коммит
f2b3e814c7
|
|
@ -35,6 +35,7 @@
|
|||
"iloc",
|
||||
"ipykernel",
|
||||
"iterrows",
|
||||
"jsons",
|
||||
"jsonschema",
|
||||
"jupyterlab",
|
||||
"keepalive",
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ scriptpath=$(readlink -f "$0")
|
|||
scriptdir=$(dirname "$scriptpath")
|
||||
cd "$scriptdir"
|
||||
|
||||
SKIP_NGINX_BUILD=${SKIP_NGINX_BUILD:-false}
|
||||
|
||||
if [ -f ../.devcontainer/.env ]; then
|
||||
source ../.devcontainer/.env
|
||||
fi
|
||||
|
|
@ -28,13 +30,15 @@ cmd="${1:-}"
|
|||
if [ "$cmd" == 'start' ]; then
|
||||
set -x
|
||||
tmpdir=$(mktemp -d)
|
||||
docker build --progress=plain -t mlos-doc-nginx \
|
||||
--build-arg http_proxy=${http_proxy:-} \
|
||||
--build-arg https_proxy=${https_proxy:-} \
|
||||
--build-arg no_proxy=${no_proxy:-} \
|
||||
--build-arg NGINX_PORT=$NGINX_PORT \
|
||||
-f Dockerfile "$tmpdir"
|
||||
rmdir "$tmpdir"
|
||||
if ! $SKIP_NGINX_BUILD; then
|
||||
docker build --progress=plain -t mlos-doc-nginx \
|
||||
--build-arg http_proxy=${http_proxy:-} \
|
||||
--build-arg https_proxy=${https_proxy:-} \
|
||||
--build-arg no_proxy=${no_proxy:-} \
|
||||
--build-arg NGINX_PORT=$NGINX_PORT \
|
||||
-f Dockerfile "$tmpdir"
|
||||
rmdir "$tmpdir"
|
||||
fi
|
||||
docker run -d --name mlos-doc-nginx \
|
||||
-v "$repo_root/doc/nginx-default.conf":/etc/nginx/templates/default.conf.template \
|
||||
-v "$repo_root/doc":/doc \
|
||||
|
|
|
|||
|
|
@ -136,8 +136,10 @@ def is_on_github_actions():
|
|||
intersphinx_mapping = get_intersphinx_mapping(
|
||||
packages={
|
||||
"asyncssh",
|
||||
"azure-core",
|
||||
"azure-identity",
|
||||
# Azure SDKs removed their intersphinx publishing.
|
||||
# https://github.com/Azure/azure-sdk-for-python/issues/39316
|
||||
# "azure-core",
|
||||
# "azure-identity",
|
||||
"configspace",
|
||||
"matplotlib",
|
||||
"numpy",
|
||||
|
|
@ -166,12 +168,15 @@ CUSTOM_REF_TYPE_MAP: dict[tuple[str, str], str] = {
|
|||
("ConcreteOptimizer", "class"): "data",
|
||||
("ConcreteSpaceAdapter", "class"): "data",
|
||||
("DistributionName", "class"): "data",
|
||||
("mlos_bench.tunables.tunable_types.DistributionName", "class"): "data",
|
||||
("FlamlDomain", "class"): "data",
|
||||
("mlos_core.spaces.converters.flaml.FlamlDomain", "class"): "data",
|
||||
("TunableValue", "class"): "data",
|
||||
("mlos_bench.tunables.tunable.TunableValue", "class"): "data",
|
||||
("mlos_bench.tunables.tunable_types.TunableValue", "class"): "data",
|
||||
("TunableValueType", "class"): "data",
|
||||
("mlos_bench.tunables.tunable_types.TunableValueType", "class"): "data",
|
||||
("TunableValueTypeName", "class"): "data",
|
||||
("mlos_bench.tunables.tunable_types.TunableValueTypeName", "class"): "data",
|
||||
("T_co", "class"): "data",
|
||||
("CoroReturnType", "class"): "data",
|
||||
("FutureReturnType", "class"): "data",
|
||||
|
|
@ -230,6 +235,9 @@ nitpick_ignore = [
|
|||
nitpick_ignore_regex = [
|
||||
# Ignore some external references that don't use sphinx for their docs.
|
||||
(r"py:.*", r"flaml\..*"),
|
||||
# Azure SDKs removed their intersphinx publishing.
|
||||
# https://github.com/Azure/azure-sdk-for-python/issues/39316
|
||||
(r"py:.*", r"azure\..*"),
|
||||
]
|
||||
|
||||
# Which documents to include in the build.
|
||||
|
|
@ -295,6 +303,11 @@ autoapi_keep_files = not is_on_github_actions() # for local testing
|
|||
#
|
||||
html_theme = "sphinx_rtd_theme"
|
||||
|
||||
html_theme_options = {
|
||||
"canonical_url": "https://microsoft.github.io/MLOS/",
|
||||
"navigation_depth": -1,
|
||||
}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ MLOS Documentation
|
|||
|
||||
Documentation is generated from both the `source tree markdown <source_tree_docs/index.html>`_ and the Python docstrings for each of the packages with navigation links on the side.
|
||||
|
||||
MLOS also has a fairly large test suite, so when in doubt, check the `code <https://github.com/microsoft/MLOS>`_ for additional examples or reach out to the maintainers on `Github <https://github.com/microsoft/MLOS>`_!
|
||||
|
||||
.. toctree::
|
||||
:caption: Source Tree Documentation
|
||||
:maxdepth: 4
|
||||
|
|
|
|||
|
|
@ -15,7 +15,9 @@ Overview
|
|||
``mlos_bench`` can be installed from `pypi <https://pypi.org/project/mlos-bench>`_
|
||||
via ``pip install mlos-bench`` and executed using the ``mlos_bench`` `command
|
||||
<../../mlos_bench.run.usage.html>`_ using a collection of `json` `configs
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_.
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_
|
||||
which are managed and documented in the :py:mod:`~mlos_bench.config` module and
|
||||
elsewhere in this package documentation.
|
||||
|
||||
It is intended to be used with :py:mod:`mlos_core` via
|
||||
:py:class:`~mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer` to help
|
||||
|
|
@ -59,8 +61,9 @@ intended to be modular and composable to allow reuse and faster development of n
|
|||
benchmarking environments or autotuning experiments.
|
||||
|
||||
Where possible, the framework will provide common configs for reuse (e.g., deploying
|
||||
a VM on Azure, or run benchbase against a database system) to allow users to focus
|
||||
on the specifics of their experiments.
|
||||
a VM on Azure, or run `benchbase <https://github.com/cmu-db/benchbase>`_ against
|
||||
a database system) to allow users to focus on the specifics of their
|
||||
experiments.
|
||||
|
||||
Where none are currently available, one can create them external to MLOS, however
|
||||
users are also encouraged to `submit PRs or Issues
|
||||
|
|
@ -125,18 +128,20 @@ critically, non-reusable scripts to setup and later parse results and are hence
|
|||
harder to scale to many users.
|
||||
|
||||
With these features as a part of the MLOS ecosystem, benchmarking can become a
|
||||
*service* that any developer, admin, research, etc. can use and adapt.
|
||||
*service* that any developer, admin, researcher, etc. can use and adapt.
|
||||
|
||||
See below for more information on the classes in this package.
|
||||
|
||||
Notes
|
||||
-----
|
||||
Note that while the docstrings in this package are generated from the source code
|
||||
and hence sometimes more focused on the implementation details, most user
|
||||
interactions with the package will be through the `json configs
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_. Even
|
||||
so it may be useful to look at the source code to understand how those are
|
||||
interpreted.
|
||||
Note that while the docstrings in this package are generated from the source
|
||||
code and hence sometimes more focused on the implementation details, we do try
|
||||
to provide some (testable) examples here, but most user interactions with the
|
||||
package will be through the `json configs
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/config/>`_.
|
||||
Even so it may be useful to look at the documentation here (perhaps especially
|
||||
starting with :py:mod:`mlos_bench.config`) and, if necessary, the `source code
|
||||
<https://github.com/microsoft/MLOS>`_ to understand how those are interpreted.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
|
|
|||
|
|
@ -208,12 +208,14 @@ Well Known Variables
|
|||
Here is a list of some well known variables that are provided or required by the
|
||||
system and may be used in the config files:
|
||||
|
||||
- ``$experiment_id``: A unique identifier for the experiment.
|
||||
- ``$experiment_id``: A unique identifier for the ``Experiment``.
|
||||
Typically provided in globals.
|
||||
- ``$trial_id``: A unique identifier for the trial currently being executed.
|
||||
- ``$trial_id``: A unique identifier for the ``Trial`` currently being executed.
|
||||
This can be useful in the configs for :py:mod:`mlos_bench.environments` for
|
||||
instance (e.g., when writing scripts).
|
||||
- TODO: Document more variables here.
|
||||
- ``$trial_runner_id``: A unique identifier for the ``TrialRunner``.
|
||||
This can be useful when running multiple trials in parallel (e.g., to
|
||||
provision a numbered VM per worker).
|
||||
|
||||
Tunable Configs
|
||||
^^^^^^^^^^^^^^^
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ A module for managing config schemas and their validation.
|
|||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.config.schemas.config_schemas : The module handling the actual schema
|
||||
definitions and validation.
|
||||
mlos_bench.config.schemas.config_schemas :
|
||||
The module handling the actual schema definitions and validation.
|
||||
"""
|
||||
|
||||
from mlos_bench.config.schemas.config_schemas import CONFIG_SCHEMA_DIR, ConfigSchema
|
||||
|
|
|
|||
|
|
@ -116,11 +116,12 @@ Notes
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.config` : Overview of the configuration system.
|
||||
:py:mod:`mlos_bench.services` : Overview of the Services available to the
|
||||
Environments and their configurations.
|
||||
:py:mod:`mlos_bench.tunables` : Overview of the Tunables available to the
|
||||
Environments and their configurations.
|
||||
:py:mod:`mlos_bench.config` :
|
||||
Overview of the configuration system.
|
||||
:py:mod:`mlos_bench.services` :
|
||||
Overview of the Services available to the Environments and their configurations.
|
||||
:py:mod:`mlos_bench.tunables` :
|
||||
Overview of the Tunables available to the Environments and their configurations.
|
||||
"""
|
||||
|
||||
from mlos_bench.environments.base_environment import Environment
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ from mlos_bench.config.schemas import ConfigSchema
|
|||
from mlos_bench.dict_templater import DictTemplater
|
||||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import instantiate_from_config, merge_parameters
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ from typing import Any, Literal
|
|||
from mlos_bench.environments.base_environment import Environment
|
||||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ from mlos_bench.environments.script_env import ScriptEnv
|
|||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.services.types.local_exec_type import SupportsLocalExec
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import datetime_parser, path_join
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ from mlos_bench.environments.status import Status
|
|||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.services.types.fileshare_type import SupportsFileShareOps
|
||||
from mlos_bench.services.types.local_exec_type import SupportsLocalExec
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ import numpy
|
|||
from mlos_bench.environments.base_environment import Environment
|
||||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ from mlos_bench.environments.status import Status
|
|||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.services.types.host_ops_type import SupportsHostOps
|
||||
from mlos_bench.services.types.remote_exec_type import SupportsRemoteExec
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ from collections.abc import Iterable
|
|||
|
||||
from mlos_bench.environments.base_environment import Environment
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import try_parse_val
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ from mlos_bench.services.config_persistence import ConfigPersistenceService
|
|||
from mlos_bench.services.local.local_exec import LocalExecService
|
||||
from mlos_bench.services.types.config_loader_type import SupportsConfigLoading
|
||||
from mlos_bench.storage.base_storage import Storage
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import try_parse_val
|
||||
|
||||
_LOG_LEVEL = logging.INFO
|
||||
|
|
|
|||
|
|
@ -3,10 +3,274 @@
|
|||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Interfaces and wrapper classes for optimizers to be used in mlos_bench for autotuning or
|
||||
benchmarking.
|
||||
Interfaces and wrapper classes for optimizers to be used in :py:mod:`mlos_bench` for
|
||||
autotuning or benchmarking.
|
||||
|
||||
TODO: Improve documentation here.
|
||||
Overview
|
||||
++++++++
|
||||
|
||||
One of the main purposes of the mlos_bench :py:class:`.Optimizer` class is to
|
||||
provide a wrapper for the :py:mod:`mlos_core.optimizers` via the
|
||||
:py:class:`.MlosCoreOptimizer` in order to perform autotuning.
|
||||
|
||||
However, several other *config suggesters* that conform to the Optimizer APIs are
|
||||
also available for use:
|
||||
|
||||
- :py:class:`.GridSearchOptimizer` :
|
||||
Useful for exhaustive search of a *small* parameter space.
|
||||
- :py:class:`.OneShotOptimizer` :
|
||||
Useful for one-off config experimentation and benchmarking.
|
||||
- :py:class:`.ManualOptimizer` :
|
||||
Useful for repeatedly testing a small set of known configs.
|
||||
|
||||
API
|
||||
+++
|
||||
|
||||
Like the mlos_core :py:class:`~mlos_core.optimizers.optimizer.BaseOptimizer`, the
|
||||
core APIs here are :py:meth:`.Optimizer.suggest` and :py:meth:`.Optimizer.register`.
|
||||
|
||||
The :py:meth:`.Optimizer.bulk_register` method is also available to pre-warm a new
|
||||
Optimizer instance using observations from a prior set of
|
||||
:py:class:`~mlos_bench.storage.base_storage.Storage.Trial` runs (e.g., from the
|
||||
:py:mod:`mlos_bench.storage`).
|
||||
|
||||
.. note::
|
||||
We also refer to this as "merging" this only makes sense if the past Trials
|
||||
were run from a set of Experiments *compatible* with this one (e.g., same
|
||||
software, workload, VM size, overlapping parameter spaces, etc.).
|
||||
Automatically determining whether that makes sense to do is challenging and
|
||||
is left to the user to ensure for now.
|
||||
|
||||
Stopping Conditions
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
Currently the :py:meth:`.Optimizer.not_converged` method only checks that the number
|
||||
of suggestions is less than the ``max_suggestions`` property of the Optimizer
|
||||
config.
|
||||
|
||||
However, in the future we intend to implement more sophisticated stopping conditions
|
||||
(e.g., total time, convergence, cost budget, etc.).
|
||||
|
||||
Spaces
|
||||
++++++
|
||||
|
||||
Unlike mlos_core, the :py:mod:`mlos_bench.optimizers` operate on
|
||||
:py:mod:`~mlos_bench.tunables` instead of :py:class:`ConfigSpace.ConfigurationSpace`
|
||||
instances, so mlos_bench handles conversions internally (see
|
||||
:py:mod:`mlos_bench.optimizers.convert_configspace`).
|
||||
|
||||
Config
|
||||
++++++
|
||||
|
||||
Typically these tunables are combined from the individual Environments they are
|
||||
associated with and loaded via JSON config files.
|
||||
|
||||
In the Examples used within this module's documentation we will simply represent
|
||||
them as JSON strings for explanatory purposes.
|
||||
|
||||
Several properties are common to all Optimizers, but some are specific to the
|
||||
Optimizer being used.
|
||||
The JSON schemas control what is considered a valid configuration for an Optimizer.
|
||||
In the case of an :py:class:`.MlosCoreOptimizer`, the valid options can often be
|
||||
inferred from the constructor arguments of the corresponding
|
||||
:py:class:`mlos_core.optimizers` class.
|
||||
|
||||
Similarly for the SpaceAdapterType, the valid options can be inferred from the
|
||||
individual :py:mod:`mlos_core.spaces.adapters` class constructors.
|
||||
|
||||
Generally speaking though the JSON config for an Optimizer will look something
|
||||
like the following:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"class": "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer",
|
||||
"description": "MlosCoreOptimizer",
|
||||
"config": {
|
||||
"max_suggestions": 1000,
|
||||
"optimization_targets": {
|
||||
// Your optimization target(s) mapped to their respective
|
||||
// optimization goals.
|
||||
"throughput": "max",
|
||||
"cost": "min",
|
||||
},
|
||||
"start_with_defaults": true,
|
||||
"seed": 42,
|
||||
|
||||
// Optionally override the default space adapter type.
|
||||
// Must be one of the mlos_core SpaceAdapterType enum values.
|
||||
// e.g., LlamaTune is a method for automatically doing space reduction
|
||||
// from the original space.
|
||||
"space_adapter_type": "LLAMATUNE",
|
||||
"space_adapter_config": {
|
||||
// Optional space adapter configuration.
|
||||
// The JSON schema controls the valid properties here.
|
||||
// In general check the constructor arguments of the specified
|
||||
// SpaceAdapterType.
|
||||
"num_low_dims": 10,
|
||||
"max_unique_values_per_param": 20,
|
||||
}
|
||||
|
||||
// Now starts a collection of key-value pairs that are specific to
|
||||
// the Optimizer class chosen.
|
||||
|
||||
// Override the default optimizer type.
|
||||
// Must be one of the mlos_core OptimizerType enum values.
|
||||
"optimizer_type": "SMAC", // e.g., "RANDOM", "FLAML", "SMAC"
|
||||
|
||||
// Optionally provide some additional configuration options for the optimizer.
|
||||
// Note: these are optimizer-specific and may not be supported by all optimizers.
|
||||
// For instance the following example is only supported by the SMAC optimizer.
|
||||
// In general, for MlosCoreOptimizers you can look at the arguments
|
||||
// to the corresponding OptimizerType in the mlos_core module.
|
||||
"n_random_init": 20,
|
||||
"n_random_probability": 0.25, // increased to prioritize exploration
|
||||
}
|
||||
|
||||
However, it can also be as simple as the following and sane defaults will be
|
||||
used for the rest.
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"class": "mlos_bench.optimizers.MlosCoreOptimizer"
|
||||
}
|
||||
|
||||
Notes
|
||||
-----
|
||||
The full set of supported properties is specified in the `JSON schemas for optimizers
|
||||
<https://github.com/microsoft/MLOS/blob/main/mlos_bench/mlos_bench/config/schemas/optimizers/>`_.
|
||||
and can be seen in some of the `test examples in the source tree
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/tests/config/schemas/optimizers/test-cases/good/>`_.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.config` :
|
||||
For more information about the mlos_bench configuration system.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Note: All of the examples in this module are expressed in Python for testing
|
||||
purposes.
|
||||
|
||||
>>> # Load tunables from a JSON string.
|
||||
>>> # Note: normally these would be automatically loaded from the Environment(s)'s
|
||||
>>> # `include_tunables` config parameter.
|
||||
>>> #
|
||||
>>> import json5 as json
|
||||
>>> import mlos_core.optimizers
|
||||
>>> from mlos_bench.environments.status import Status
|
||||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
... "params": {
|
||||
... "flags": {
|
||||
... "type": "categorical",
|
||||
... "values": ["on", "off", "auto"],
|
||||
... "default": "auto",
|
||||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [1, 100],
|
||||
... "default": 10,
|
||||
... },
|
||||
... "float_param": {
|
||||
... "type": "float",
|
||||
... "range": [0, 100],
|
||||
... "default": 50.0,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> tunables = service.load_tunables(jsons=[json_config])
|
||||
>>> # Here's the defaults:
|
||||
>>> tunables.get_param_values()
|
||||
{'flags': 'auto', 'int_param': 10, 'float_param': 50.0}
|
||||
|
||||
>>> # Load a JSON config string for an MlosCoreOptimizer.
|
||||
>>> # You must specify an mlos_bench Optimizer class in the JSON config.
|
||||
>>> # (e.g., "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer")
|
||||
>>> # All optimizers support the following config properties at a minimum:
|
||||
>>> sorted(Optimizer.BASE_SUPPORTED_CONFIG_PROPS)
|
||||
['max_suggestions', 'optimization_targets', 'seed', 'start_with_defaults']
|
||||
|
||||
>>> # When using the MlosCoreOptimizer, we can also specify some additional
|
||||
>>> # properties, for instance the optimizer_type, which is one of the mlos_core
|
||||
>>> # OptimizerType enum values:
|
||||
>>> print([member.name for member in mlos_core.optimizers.OptimizerType])
|
||||
['RANDOM', 'FLAML', 'SMAC']
|
||||
|
||||
>>> # We can also specify an optional space_adapter_type, which can sometimes
|
||||
>>> # help manipulate the configuration space to something more manageable.
|
||||
>>> print([member.name for member in mlos_core.spaces.adapters.SpaceAdapterType])
|
||||
['IDENTITY', 'LLAMATUNE']
|
||||
|
||||
>>> # Here's an example JSON config for an MlosCoreOptimizer.
|
||||
>>> optimizer_json_config = '''
|
||||
... {
|
||||
... "class": "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer",
|
||||
... "description": "MlosCoreOptimizer",
|
||||
... "config": {
|
||||
... "max_suggestions": 1000,
|
||||
... "optimization_targets": {
|
||||
... "throughput": "max",
|
||||
... "cost": "min",
|
||||
... },
|
||||
... "start_with_defaults": true,
|
||||
... "seed": 42,
|
||||
... // Override the default optimizer type
|
||||
... // Must be one of the mlos_core OptimizerType enum values.
|
||||
... "optimizer_type": "SMAC",
|
||||
... // Optionally provide some additional configuration options for the optimizer.
|
||||
... // Note: these are optimizer-specific and may not be supported by all optimizers.
|
||||
... "n_random_init": 25,
|
||||
... "n_random_probability": 0.01,
|
||||
... // Optionally override the default space adapter type
|
||||
... // Must be one of the mlos_core SpaceAdapterType enum values.
|
||||
... // LlamaTune is a method for automatically doing space reduction
|
||||
... // from the original space.
|
||||
... /*
|
||||
... "space_adapter_type": "LLAMATUNE",
|
||||
... "space_adapter_config": {
|
||||
... // Note: these values are probably too low,
|
||||
... // but it's just for demonstration.
|
||||
... "num_low_dims": 2,
|
||||
... "max_unique_values_per_param": 10,
|
||||
... }
|
||||
... */
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> config = json.loads(optimizer_json_config)
|
||||
>>> optimizer = service.build_optimizer(
|
||||
... tunables=tunables,
|
||||
... service=service,
|
||||
... config=config,
|
||||
... )
|
||||
|
||||
>>> suggested_config_1 = optimizer.suggest()
|
||||
>>> # Default should be suggested first, per json config.
|
||||
>>> suggested_config_1.get_param_values()
|
||||
{'flags': 'auto', 'int_param': 10, 'float_param': 50.0}
|
||||
>>> # Get another suggestion.
|
||||
>>> # Note that multiple suggestions can be pending prior to
|
||||
>>> # registering their scores, supporting parallel trial execution.
|
||||
>>> suggested_config_2 = optimizer.suggest()
|
||||
>>> suggested_config_2.get_param_values()
|
||||
{'flags': 'auto', 'int_param': 99, 'float_param': 5.8570134453475}
|
||||
>>> # Register some scores.
|
||||
>>> # Note: Maximization problems track negative scores to produce a minimization problem.
|
||||
>>> optimizer.register(suggested_config_1, Status.SUCCEEDED, {"throughput": 42, "cost": 19})
|
||||
{'throughput': -42.0, 'cost': 19.0}
|
||||
>>> optimizer.register(suggested_config_2, Status.SUCCEEDED, {"throughput": 7, "cost": 17.2})
|
||||
{'throughput': -7.0, 'cost': 17.2}
|
||||
>>> (best_score, best_config) = optimizer.get_best_observation()
|
||||
>>> best_score
|
||||
{'throughput': 42.0, 'cost': 19.0}
|
||||
>>> assert best_config == suggested_config_1
|
||||
"""
|
||||
|
||||
from mlos_bench.optimizers.base_optimizer import Optimizer
|
||||
|
|
|
|||
|
|
@ -2,8 +2,14 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Base class for an interface between the benchmarking framework and mlos_core
|
||||
optimizers.
|
||||
"""
|
||||
Base class for an interface between the benchmarking framework and :py:mod:`mlos_core`
|
||||
optimizers and other config suggestion methods.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.optimizers :
|
||||
For more information on the available optimizers and their usage.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
|
@ -19,16 +25,16 @@ from mlos_bench.config.schemas import ConfigSchema
|
|||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.optimizers.convert_configspace import tunable_groups_to_configspace
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import strtobool
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Optimizer(ContextManager, metaclass=ABCMeta): # pylint: disable=too-many-instance-attributes
|
||||
"""An abstract interface between the benchmarking framework and mlos_core
|
||||
optimizers.
|
||||
"""An abstract interface between the benchmarking framework and :py:mod:`mlos_core`
|
||||
optimizers and other config suggestion methods.
|
||||
"""
|
||||
|
||||
# See Also: mlos_bench/mlos_bench/config/schemas/optimizers/optimizer-schema.json
|
||||
|
|
@ -312,6 +318,8 @@ class Optimizer(ContextManager, metaclass=ABCMeta): # pylint: disable=too-many-
|
|||
)
|
||||
if status.is_succeeded() == (score is None): # XOR
|
||||
raise ValueError("Status and score must be consistent.")
|
||||
# FIXME: should maximization problems return -score values to the user, or
|
||||
# keep that as an internal nuance.
|
||||
return self._get_scores(status, score)
|
||||
|
||||
def _get_scores(
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Functions to convert TunableGroups to ConfigSpace for use with the mlos_core
|
||||
optimizers.
|
||||
"""Functions to convert :py:class:`.TunableGroups` that :py:mod:`mlos_bench` uses to to
|
||||
:py:class:`ConfigSpace.ConfigurationSpace` for use with the
|
||||
:py:mod:`mlos_core.optimizers`.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import logging
|
||||
from collections.abc import Hashable
|
||||
|
||||
|
|
@ -23,8 +25,9 @@ from ConfigSpace import (
|
|||
from ConfigSpace.hyperparameters import NumericalHyperparameter
|
||||
from ConfigSpace.types import NotSet
|
||||
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import try_parse_val
|
||||
from mlos_core.spaces.converters.util import (
|
||||
QUANTIZATION_BINS_META_KEY,
|
||||
|
|
@ -34,14 +37,9 @@ from mlos_core.spaces.converters.util import (
|
|||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TunableValueKind:
|
||||
"""
|
||||
Enum for the kind of the tunable value (special or not).
|
||||
class TunableValueKind(enum.Enum):
|
||||
"""Enum for the kind of the tunable value (special or not)."""
|
||||
|
||||
It is not a true enum because ConfigSpace wants string values.
|
||||
"""
|
||||
|
||||
# pylint: disable=too-few-public-methods
|
||||
SPECIAL = "special"
|
||||
RANGE = "range"
|
||||
|
||||
|
|
@ -170,19 +168,19 @@ def _tunable_to_configspace(
|
|||
),
|
||||
CategoricalHyperparameter(
|
||||
name=type_name,
|
||||
choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE],
|
||||
choices=[TunableValueKind.SPECIAL.value, TunableValueKind.RANGE.value],
|
||||
weights=switch_weights,
|
||||
default_value=TunableValueKind.SPECIAL,
|
||||
default_value=TunableValueKind.SPECIAL.value,
|
||||
),
|
||||
]
|
||||
)
|
||||
conf_space.add(
|
||||
[
|
||||
EqualsCondition(
|
||||
conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL
|
||||
conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL.value
|
||||
),
|
||||
EqualsCondition(
|
||||
conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE
|
||||
conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE.value
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
@ -242,10 +240,10 @@ def tunable_values_to_configuration(tunables: TunableGroups) -> Configuration:
|
|||
if tunable.special:
|
||||
(special_name, type_name) = special_param_names(tunable.name)
|
||||
if tunable.value in tunable.special:
|
||||
values[type_name] = TunableValueKind.SPECIAL
|
||||
values[type_name] = TunableValueKind.SPECIAL.value
|
||||
values[special_name] = tunable.value
|
||||
else:
|
||||
values[type_name] = TunableValueKind.RANGE
|
||||
values[type_name] = TunableValueKind.RANGE.value
|
||||
values[tunable.name] = tunable.value
|
||||
else:
|
||||
values[tunable.name] = tunable.value
|
||||
|
|
@ -263,7 +261,7 @@ def configspace_data_to_tunable_values(data: dict) -> dict[str, TunableValue]:
|
|||
specials = [special_param_name_strip(k) for k in data.keys() if special_param_name_is_temp(k)]
|
||||
for k in specials:
|
||||
(special_name, type_name) = special_param_names(k)
|
||||
if data[type_name] == TunableValueKind.SPECIAL:
|
||||
if data[type_name] == TunableValueKind.SPECIAL.value:
|
||||
data[k] = data[special_name]
|
||||
if special_name in data:
|
||||
del data[special_name]
|
||||
|
|
@ -278,7 +276,8 @@ def special_param_names(name: str) -> tuple[str, str]:
|
|||
Generate the names of the auxiliary hyperparameters that correspond to a tunable
|
||||
that can have special values.
|
||||
|
||||
NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic.
|
||||
NOTE: ``!`` characters are currently disallowed in :py:class:`.Tunable` names in
|
||||
order handle this logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
|
|
@ -299,7 +298,8 @@ def special_param_name_is_temp(name: str) -> bool:
|
|||
"""
|
||||
Check if name corresponds to a temporary ConfigSpace parameter.
|
||||
|
||||
NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic.
|
||||
NOTE: ``!`` characters are currently disallowed in :py:class:`.Tunable` names in
|
||||
order handle this logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
|
|
@ -318,7 +318,8 @@ def special_param_name_strip(name: str) -> str:
|
|||
"""
|
||||
Remove the temporary suffix from a special parameter name.
|
||||
|
||||
NOTE: `!` characters are currently disallowed in Tunable names in order handle this logic.
|
||||
NOTE: ``!`` characters are currently disallowed in :py:class:`.Tunable` names in
|
||||
order handle this logic.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
|
|
|
|||
|
|
@ -2,7 +2,104 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Grid search optimizer for mlos_bench."""
|
||||
"""
|
||||
Grid search Optimizer for mlos_bench.
|
||||
|
||||
Grid search is a simple optimizer that exhaustively searches the configuration space.
|
||||
|
||||
To do this it generates a grid of configurations to try, and then suggests them one by one.
|
||||
|
||||
Therefore, the number of configurations to try is the product of the
|
||||
:py:attr:`~mlos_bench.tunables.tunable.Tunable.cardinality` of each of the
|
||||
:py:mod:`~mlos_bench.tunables`.
|
||||
(i.e., non :py:attr:`quantized <mlos_bench.tunables.tunable.Tunable.quantization_bins>`
|
||||
tunables are not supported).
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> # Load tunables from a JSON string.
|
||||
>>> # Note: normally these would be automatically loaded from the Environment(s)'s
|
||||
>>> # `include_tunables` config parameter.
|
||||
>>> #
|
||||
>>> import json5 as json
|
||||
>>> from mlos_bench.environments.status import Status
|
||||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
... "params": {
|
||||
... "colors": {
|
||||
... "type": "categorical",
|
||||
... "values": ["red", "blue", "green"],
|
||||
... "default": "green",
|
||||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [1, 3],
|
||||
... "default": 2,
|
||||
... },
|
||||
... "float_param": {
|
||||
... "type": "float",
|
||||
... "range": [0, 1],
|
||||
... "default": 0.5,
|
||||
... // Quantize the range into 3 bins
|
||||
... "quantization_bins": 3,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> tunables = service.load_tunables(jsons=[json_config])
|
||||
>>> # Check the defaults:
|
||||
>>> tunables.get_param_values()
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
|
||||
>>> # Now create a GridSearchOptimizer from a JSON config string.
|
||||
>>> optimizer_json_config = '''
|
||||
... {
|
||||
... "class": "mlos_bench.optimizers.grid_search_optimizer.GridSearchOptimizer",
|
||||
... "description": "GridSearchOptimizer",
|
||||
... "config": {
|
||||
... "max_suggestions": 100,
|
||||
... "optimization_targets": {"score": "max"},
|
||||
... "start_with_defaults": true
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> config = json.loads(optimizer_json_config)
|
||||
>>> grid_search_optimizer = service.build_optimizer(
|
||||
... tunables=tunables,
|
||||
... service=service,
|
||||
... config=config,
|
||||
... )
|
||||
>>> # Should have 3 values for each of the 3 tunables
|
||||
>>> len(list(grid_search_optimizer.pending_configs))
|
||||
27
|
||||
>>> next(grid_search_optimizer.pending_configs)
|
||||
{'colors': 'red', 'float_param': 0, 'int_param': 1}
|
||||
>>> suggested_config_1 = grid_search_optimizer.suggest()
|
||||
>>> # Default should be suggested first, per json config.
|
||||
>>> suggested_config_1.get_param_values()
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
>>> # Get another suggestion.
|
||||
>>> # Note that multiple suggestions can be pending prior to
|
||||
>>> # registering their scores, supporting parallel trial execution.
|
||||
>>> suggested_config_2 = grid_search_optimizer.suggest()
|
||||
>>> suggested_config_2.get_param_values()
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
>>> # Register some scores.
|
||||
>>> # Note: Maximization problems track negative scores to produce a minimization problem.
|
||||
>>> grid_search_optimizer.register(suggested_config_1, Status.SUCCEEDED, {"score": 42})
|
||||
{'score': -42.0}
|
||||
>>> grid_search_optimizer.register(suggested_config_2, Status.SUCCEEDED, {"score": 7})
|
||||
{'score': -7.0}
|
||||
>>> (best_score, best_config) = grid_search_optimizer.get_best_observation()
|
||||
>>> best_score
|
||||
{'score': 42.0}
|
||||
>>> assert best_config == suggested_config_1
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
|
@ -15,14 +112,21 @@ from mlos_bench.environments.status import Status
|
|||
from mlos_bench.optimizers.convert_configspace import configspace_data_to_tunable_values
|
||||
from mlos_bench.optimizers.track_best_optimizer import TrackBestOptimizer
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GridSearchOptimizer(TrackBestOptimizer):
|
||||
"""Grid search optimizer."""
|
||||
"""
|
||||
Grid search optimizer.
|
||||
|
||||
See :py:mod:`above <mlos_bench.optimizers.grid_search_optimizer>` for more details.
|
||||
"""
|
||||
|
||||
MAX_CONFIGS = 10000
|
||||
"""Maximum number of configurations to enumerate."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -52,7 +156,7 @@ class GridSearchOptimizer(TrackBestOptimizer):
|
|||
raise ValueError(
|
||||
f"Unquantized tunables are not supported for grid search: {self._tunables}"
|
||||
)
|
||||
if size > 10000:
|
||||
if size > self.MAX_CONFIGS:
|
||||
_LOG.warning(
|
||||
"Large number %d of config points requested for grid search: %s",
|
||||
size,
|
||||
|
|
|
|||
|
|
@ -3,21 +3,99 @@
|
|||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Manual config suggestor (Optimizer) for mlos_bench that proposes an explicit sequence of
|
||||
Manual config suggester (Optimizer) for mlos_bench that proposes an explicit sequence of
|
||||
configurations.
|
||||
|
||||
This is useful for testing and validation, as it allows you to run a sequence of
|
||||
configurations in a cyclic fashion.
|
||||
|
||||
TODO: Add an example configuration.
|
||||
Examples
|
||||
--------
|
||||
>>> # Load tunables from a JSON string.
|
||||
>>> # Note: normally these would be automatically loaded from the Environment(s)'s
|
||||
>>> # `include_tunables` config parameter.
|
||||
>>> #
|
||||
>>> import json5 as json
|
||||
>>> from mlos_bench.environments.status import Status
|
||||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
... "params": {
|
||||
... "colors": {
|
||||
... "type": "categorical",
|
||||
... "values": ["red", "blue", "green"],
|
||||
... "default": "green",
|
||||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [1, 3],
|
||||
... "default": 2,
|
||||
... },
|
||||
... "float_param": {
|
||||
... "type": "float",
|
||||
... "range": [0, 1],
|
||||
... "default": 0.5,
|
||||
... // Quantize the range into 3 bins
|
||||
... "quantization_bins": 3,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> tunables = service.load_tunables(jsons=[json_config])
|
||||
>>> # Check the defaults:
|
||||
>>> tunables.get_param_values()
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
|
||||
>>> # Now create a ManualOptimizer from a JSON config string.
|
||||
>>> optimizer_json_config = '''
|
||||
... {
|
||||
... "class": "mlos_bench.optimizers.manual_optimizer.ManualOptimizer",
|
||||
... "description": "ManualOptimizer",
|
||||
... "config": {
|
||||
... "max_cycles": 3,
|
||||
... "tunable_values_cycle": [
|
||||
... {"colors": "red", "int_param": 1, "float_param": 0.0},
|
||||
... {"colors": "blue", "int_param": 3, "float_param": 1.0},
|
||||
... // special case: {} - represents the defaults, without
|
||||
... // having to copy them from the tunables JSON
|
||||
... // (which is presumably specified elsewhere)
|
||||
... {},
|
||||
... ],
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> config = json.loads(optimizer_json_config)
|
||||
>>> optimizer = service.build_optimizer(
|
||||
... tunables=tunables,
|
||||
... service=service,
|
||||
... config=config,
|
||||
... )
|
||||
>>> # Run the optimizer.
|
||||
>>> # Note that the cycle will repeat 3 times, as specified in the config.
|
||||
>>> while optimizer.not_converged():
|
||||
... suggestion = optimizer.suggest()
|
||||
... print(suggestion.get_param_values())
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
{'colors': 'blue', 'int_param': 3, 'float_param': 1.0}
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
{'colors': 'blue', 'int_param': 3, 'float_param': 1.0}
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
{'colors': 'blue', 'int_param': 3, 'float_param': 1.0}
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from mlos_bench.optimizers.mock_optimizer import MockOptimizer
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,139 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""A wrapper for mlos_core optimizers for mlos_bench."""
|
||||
"""
|
||||
A wrapper for :py:mod:`mlos_core.optimizers` for :py:mod:`mlos_bench`.
|
||||
|
||||
Config
|
||||
------
|
||||
The JSON config for an :py:class:`.MlosCoreOptimizer` generally takes the
|
||||
following basic structure:
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.optimizers` :
|
||||
Another working example of an :py:class:`.MlosCoreOptimizer`.
|
||||
:py:mod:`mlos_core.optimizers` :
|
||||
Documentation on the underlying mlos_core Optimizers.
|
||||
:py:mod:`mlos_core.spaces.adapters` :
|
||||
Documentation on the underlying mlos_core SpaceAdapters.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> # Load tunables from a JSON string.
|
||||
>>> # Note: normally these would be automatically loaded from the Environment(s)'s
|
||||
>>> # `include_tunables` config parameter.
|
||||
>>> #
|
||||
>>> import json5 as json
|
||||
>>> import mlos_core.optimizers
|
||||
>>> from mlos_bench.environments.status import Status
|
||||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
... "params": {
|
||||
... "flags": {
|
||||
... "type": "categorical",
|
||||
... "values": ["on", "off", "auto"],
|
||||
... "default": "auto",
|
||||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [1, 100],
|
||||
... "default": 10,
|
||||
... },
|
||||
... "float_param": {
|
||||
... "type": "float",
|
||||
... "range": [0, 100],
|
||||
... "default": 50.0,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> tunables = service.load_tunables(jsons=[json_config])
|
||||
>>> # Here's the defaults:
|
||||
>>> tunables.get_param_values()
|
||||
{'flags': 'auto', 'int_param': 10, 'float_param': 50.0}
|
||||
|
||||
>>> # When using the MlosCoreOptimizer, we can also specify some additional
|
||||
>>> # properties, for instance the optimizer_type, which is one of the mlos_core
|
||||
>>> # OptimizerType enum values:
|
||||
>>> print([member.name for member in mlos_core.optimizers.OptimizerType])
|
||||
['RANDOM', 'FLAML', 'SMAC']
|
||||
|
||||
>>> # We can also specify
|
||||
>>> # properties, for instance the optimizer_type, which is one of the mlos_core
|
||||
>>> # OptimizerType enum values:
|
||||
>>> print([member.name for member in mlos_core.optimizers.OptimizerType])
|
||||
['RANDOM', 'FLAML', 'SMAC']
|
||||
|
||||
>>> # Here's an example JSON config for an MlosCoreOptimizer.
|
||||
>>> optimizer_json_config = '''
|
||||
... {
|
||||
... "class": "mlos_bench.optimizers.mlos_core_optimizer.MlosCoreOptimizer",
|
||||
... "description": "MlosCoreOptimizer",
|
||||
... "config": {
|
||||
... "max_suggestions": 1000,
|
||||
... "optimization_targets": {
|
||||
... "throughput": "max",
|
||||
... "cost": "min",
|
||||
... },
|
||||
... "start_with_defaults": true,
|
||||
... "seed": 42,
|
||||
... // Override the default optimizer type
|
||||
... // Must be one of the mlos_core OptimizerType enum values.
|
||||
... "optimizer_type": "SMAC",
|
||||
... // Optionally provide some additional configuration options for the optimizer.
|
||||
... // Note: these are optimizer-specific and may not be supported by all optimizers.
|
||||
... "n_random_init": 25,
|
||||
... "n_random_probability": 0.01,
|
||||
... // Optionally override the default space adapter type
|
||||
... // Must be one of the mlos_core SpaceAdapterType enum values.
|
||||
... // LlamaTune is a method for automatically doing space reduction
|
||||
... // from the original space.
|
||||
... "space_adapter_type": "LLAMATUNE",
|
||||
... "space_adapter_config": {
|
||||
... // Note: these values are probably too low,
|
||||
... // but it's just for demonstration.
|
||||
... "num_low_dims": 2,
|
||||
... "max_unique_values_per_param": 10,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> config = json.loads(optimizer_json_config)
|
||||
>>> optimizer = service.build_optimizer(
|
||||
... tunables=tunables,
|
||||
... service=service,
|
||||
... config=config,
|
||||
... )
|
||||
|
||||
>>> suggested_config_1 = optimizer.suggest()
|
||||
>>> # Normally default values should be suggested first, per json config.
|
||||
>>> # However, since LlamaTune is being employed here, the first suggestion may
|
||||
>>> # be projected to a slightly different space.
|
||||
>>> suggested_config_1.get_param_values()
|
||||
{'flags': 'auto', 'int_param': 1, 'float_param': 55.5555555555556}
|
||||
>>> # Get another suggestion.
|
||||
>>> # Note that multiple suggestions can be pending prior to
|
||||
>>> # registering their scores, supporting parallel trial execution.
|
||||
>>> suggested_config_2 = optimizer.suggest()
|
||||
>>> suggested_config_2.get_param_values()
|
||||
{'flags': 'on', 'int_param': 78, 'float_param': 88.8888888888889}
|
||||
>>> # Register some scores.
|
||||
>>> # Note: Maximization problems track negative scores to produce a minimization problem.
|
||||
>>> optimizer.register(suggested_config_1, Status.SUCCEEDED, {"throughput": 42, "cost": 19})
|
||||
{'throughput': -42.0, 'cost': 19.0}
|
||||
>>> optimizer.register(suggested_config_2, Status.SUCCEEDED, {"throughput": 7, "cost": 17.2})
|
||||
{'throughput': -7.0, 'cost': 17.2}
|
||||
>>> (best_score, best_config) = optimizer.get_best_observation()
|
||||
>>> best_score
|
||||
{'throughput': 42.0, 'cost': 19.0}
|
||||
>>> assert best_config == suggested_config_1
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -20,8 +152,8 @@ from mlos_bench.optimizers.convert_configspace import (
|
|||
special_param_names,
|
||||
)
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_core.data_classes import Observations
|
||||
from mlos_core.optimizers import (
|
||||
DEFAULT_OPTIMIZER_TYPE,
|
||||
|
|
@ -77,6 +209,12 @@ class MlosCoreOptimizer(Optimizer):
|
|||
|
||||
if space_adapter_type is not None:
|
||||
space_adapter_type = getattr(SpaceAdapterType, space_adapter_type)
|
||||
assert isinstance(space_adapter_type, SpaceAdapterType)
|
||||
if space_adapter_type == SpaceAdapterType.LLAMATUNE:
|
||||
# This is probably a sane default, especially when
|
||||
# bulk_registering old configs (e.g., Experiment resume), but is
|
||||
# not currently exposed in the config schema.
|
||||
space_adapter_config["use_approximate_reverse_mapping"] = True
|
||||
|
||||
self._opt: BaseOptimizer = OptimizerFactory.create(
|
||||
parameter_space=self.config_space,
|
||||
|
|
@ -181,8 +319,8 @@ class MlosCoreOptimizer(Optimizer):
|
|||
(special_name, type_name) = special_param_names(tunable.name)
|
||||
tunables_names += [special_name, type_name]
|
||||
is_special = df_configs[tunable.name].apply(tunable.special.__contains__)
|
||||
df_configs[type_name] = TunableValueKind.RANGE
|
||||
df_configs.loc[is_special, type_name] = TunableValueKind.SPECIAL
|
||||
df_configs[type_name] = TunableValueKind.RANGE.value
|
||||
df_configs.loc[is_special, type_name] = TunableValueKind.SPECIAL.value
|
||||
if tunable.type == "int":
|
||||
# Make int column NULLABLE:
|
||||
df_configs[tunable.name] = df_configs[tunable.name].astype("Int64")
|
||||
|
|
|
|||
|
|
@ -2,7 +2,14 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Mock optimizer for mlos_bench."""
|
||||
"""
|
||||
Mock optimizer for mlos_bench.
|
||||
|
||||
Mostly intended for testing and validation. This optimizer produces random suggestions.
|
||||
The range of the suggestions can be controlled by a config.
|
||||
|
||||
See the test cases or example json configs for more details.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import random
|
||||
|
|
@ -11,8 +18,9 @@ from collections.abc import Callable, Sequence
|
|||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.optimizers.track_best_optimizer import TrackBestOptimizer
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,87 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""No-op optimizer for mlos_bench that proposes a single configuration."""
|
||||
"""
|
||||
No-op optimizer for mlos_bench that proposes a single configuration.
|
||||
|
||||
Explicit configs (partial or full) are possible using configuration files.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> # Load tunables from a JSON string.
|
||||
>>> # Note: normally these would be automatically loaded from the Environment(s)'s
|
||||
>>> # `include_tunables` config parameter.
|
||||
>>> #
|
||||
>>> import json5 as json
|
||||
>>> from mlos_bench.environments.status import Status
|
||||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
... "params": {
|
||||
... "colors": {
|
||||
... "type": "categorical",
|
||||
... "values": ["red", "blue", "green"],
|
||||
... "default": "green",
|
||||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [1, 3],
|
||||
... "default": 2,
|
||||
... },
|
||||
... "float_param": {
|
||||
... "type": "float",
|
||||
... "range": [0, 1],
|
||||
... "default": 0.5,
|
||||
... // Quantize the range into 3 bins
|
||||
... "quantization_bins": 3,
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... }
|
||||
... '''
|
||||
>>> tunables = service.load_tunables(jsons=[json_config])
|
||||
>>> # Check the defaults:
|
||||
>>> tunables.get_param_values()
|
||||
{'colors': 'green', 'int_param': 2, 'float_param': 0.5}
|
||||
|
||||
>>> # Load a JSON config of some tunable values to explicitly test.
|
||||
>>> # Normally these would be provided by the
|
||||
>>> # `mlos_bench --tunable-values`
|
||||
>>> # CLI option.
|
||||
>>> tunable_values_json = '''
|
||||
... {
|
||||
... "colors": "red",
|
||||
... "int_param": 1,
|
||||
... "float_param": 0.0
|
||||
... }
|
||||
... '''
|
||||
>>> tunable_values = json.loads(tunable_values_json)
|
||||
>>> tunables.assign(tunable_values).get_param_values()
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
>>> assert not tunables.is_defaults()
|
||||
|
||||
>>> # Now create a OneShotOptimizer from a JSON config string.
|
||||
>>> optimizer_json_config = '''
|
||||
... {
|
||||
... "class": "mlos_bench.optimizers.one_shot_optimizer.OneShotOptimizer",
|
||||
... }
|
||||
... '''
|
||||
>>> config = json.loads(optimizer_json_config)
|
||||
>>> optimizer = service.build_optimizer(
|
||||
... tunables=tunables,
|
||||
... service=service,
|
||||
... config=config,
|
||||
... )
|
||||
>>> # Run the optimizer.
|
||||
>>> # Note that it will only run for a single iteration and return the values we set.
|
||||
>>> while optimizer.not_converged():
|
||||
... suggestion = optimizer.suggest()
|
||||
... print(suggestion.get_param_values())
|
||||
{'colors': 'red', 'int_param': 1, 'float_param': 0.0}
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Mock optimizer for mlos_bench."""
|
||||
"""TrackBestOptimizer base class for mlos_bench."""
|
||||
|
||||
import logging
|
||||
from abc import ABCMeta
|
||||
|
|
@ -10,8 +10,8 @@ from abc import ABCMeta
|
|||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.optimizers.base_optimizer import Optimizer
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,14 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Helper functions to load, instantiate, and serialize Python objects that encapsulate
|
||||
a benchmark :py:class:`.Environment`, :py:mod:`~mlos_bench.tunables`,
|
||||
:py:class:`.Service` functions, etc from JSON configuration files and strings.
|
||||
"""
|
||||
Helper functions to load, instantiate, and serialize Python objects that encapsulate a
|
||||
benchmark :py:class:`.Environment`, :py:mod:`~mlos_bench.tunables`, :py:class:`.Service`
|
||||
functions, etc from JSON configuration files and strings.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.config : Overview of the configuration system.
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
|
@ -21,8 +26,8 @@ from mlos_bench.environments.base_environment import Environment
|
|||
from mlos_bench.optimizers.base_optimizer import Optimizer
|
||||
from mlos_bench.services.base_service import Service
|
||||
from mlos_bench.services.types.config_loader_type import SupportsConfigLoading
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import (
|
||||
instantiate_from_config,
|
||||
merge_parameters,
|
||||
|
|
@ -417,7 +422,7 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
parent_args: dict[str, TunableValue] | None = None,
|
||||
service: Service | None = None,
|
||||
) -> Environment:
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
# pylint: disable=too-many-arguments
|
||||
"""
|
||||
Factory method for a new :py:class:`.Environment` with a given config.
|
||||
|
||||
|
|
@ -589,7 +594,7 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
parent_args: dict[str, TunableValue] | None = None,
|
||||
service: Service | None = None,
|
||||
) -> Environment:
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
# pylint: disable=too-many-arguments
|
||||
"""
|
||||
Load and build new :py:class:`.Environment` from the config file or JSON string.
|
||||
|
||||
|
|
@ -611,6 +616,10 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
-------
|
||||
env : Environment
|
||||
A new benchmarking environment.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.environments : Examples of environment configurations.
|
||||
"""
|
||||
config = self.load_config(json, ConfigSchema.ENVIRONMENT)
|
||||
assert isinstance(config, dict)
|
||||
|
|
@ -624,7 +633,7 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
parent_args: dict[str, TunableValue] | None = None,
|
||||
service: Service | None = None,
|
||||
) -> list[Environment]:
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
# pylint: disable=too-many-arguments
|
||||
"""
|
||||
Load and build a list of Environments from the config file or JSON string.
|
||||
|
||||
|
|
@ -647,6 +656,10 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
-------
|
||||
env : list[Environment]
|
||||
A list of new benchmarking environments.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.environments : Examples of environment configurations.
|
||||
"""
|
||||
config = self.load_config(json, ConfigSchema.ENVIRONMENT)
|
||||
return [self.build_environment(config, tunables, global_config, parent_args, service)]
|
||||
|
|
@ -679,6 +692,10 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
-------
|
||||
service : Service
|
||||
A collection of service methods.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.services : Examples of service configurations.
|
||||
"""
|
||||
_LOG.info("Load services: %s parent: %s", jsons, parent.__class__.__name__)
|
||||
service = Service({}, global_config, parent)
|
||||
|
|
@ -711,6 +728,10 @@ class ConfigPersistenceService(Service, SupportsConfigLoading):
|
|||
-------
|
||||
tunables : TunableGroups
|
||||
The larger collection of tunable parameters.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.tunables : Examples of tunable parameter configurations.
|
||||
"""
|
||||
_LOG.info("Load tunables: '%s'", jsons)
|
||||
if parent is None:
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from mlos_bench.services.local.temp_dir_context import TempDirContextService
|
|||
from mlos_bench.services.types.local_exec_type import SupportsLocalExec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from collections.abc import Iterable
|
|||
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
||||
|
||||
from mlos_bench.config.schemas.config_schemas import ConfigSchema
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
# Avoid's circular import issues.
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import tempfile
|
|||
from collections.abc import Iterable, Mapping
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ tunable parameters).
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which
|
||||
includes some basic examples in the documentation.
|
||||
:py:mod:`mlos_bench.storage` :
|
||||
The base storage module for mlos_bench, which includes some basic examples in
|
||||
the documentation.
|
||||
"""
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from datetime import datetime
|
||||
|
|
@ -23,7 +24,7 @@ from pytz import UTC
|
|||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.storage.base_tunable_config_data import TunableConfigData
|
||||
from mlos_bench.storage.util import kv_df_to_dict
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mlos_bench.storage.base_tunable_config_trial_group_data import (
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ be used by one or more trials.
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which
|
||||
includes some basic examples in the documentation.
|
||||
:py:mod:`mlos_bench.storage` :
|
||||
The base storage module for mlos_bench, which includes some basic examples in
|
||||
the documentation.
|
||||
"""
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from typing import Any
|
||||
|
|
@ -19,7 +20,7 @@ from typing import Any
|
|||
import pandas
|
||||
|
||||
from mlos_bench.storage.util import kv_df_to_dict
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
class TunableConfigData(metaclass=ABCMeta):
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ easier analysis.
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which
|
||||
includes some basic examples in the documentation.
|
||||
:py:mod:`mlos_bench.storage` :
|
||||
The base storage module for mlos_bench, which includes some basic examples in
|
||||
the documentation.
|
||||
"""
|
||||
|
||||
from abc import ABCMeta, abstractmethod
|
||||
|
|
|
|||
|
|
@ -24,8 +24,9 @@ tree for some configuration examples.
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.storage` : The base storage module for mlos_bench, which
|
||||
includes some basic examples in the documentation.
|
||||
:py:mod:`mlos_bench.storage` :
|
||||
The base storage module for mlos_bench, which includes some basic examples in
|
||||
the documentation.
|
||||
"""
|
||||
from mlos_bench.storage.sql.storage import SqlStorage
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
import pandas
|
||||
|
||||
from mlos_bench.tunables.tunable import TunableValue, TunableValueTypeTuple
|
||||
from mlos_bench.tunables.tunable_types import TunableValue, TunableValueTypeTuple
|
||||
from mlos_bench.util import try_parse_val
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ from typing import Any
|
|||
import pytest
|
||||
|
||||
from mlos_bench.environments.base_environment import Environment
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
def check_env_success(
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
import pytest
|
||||
|
||||
from mlos_bench.environments.base_environment import Environment
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_GROUPS = {
|
||||
"group": ["a", "b"],
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ from mlos_bench.services.config_persistence import ConfigPersistenceService
|
|||
from mlos_bench.tests import requires_docker
|
||||
from mlos_bench.tests.environments import check_env_success
|
||||
from mlos_bench.tests.services.remote.ssh import SshTestServerInfo
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
@requires_docker
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ import pytest
|
|||
|
||||
from mlos_bench.environments.status import Status
|
||||
from mlos_bench.optimizers.grid_search_optimizer import GridSearchOptimizer
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from mlos_bench.environments.status import Status
|
|||
from mlos_bench.optimizers.base_optimizer import Optimizer
|
||||
from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer
|
||||
from mlos_bench.optimizers.mock_optimizer import MockOptimizer
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import pytest
|
|||
|
||||
from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
from mlos_bench.services.local.local_exec import LocalExecService
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
from mlos_bench.util import path_join
|
||||
|
||||
# pylint: disable=redefined-outer-name
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from mlos_bench.services.local.temp_dir_context import TempDirContextService
|
|||
from mlos_bench.services.types.local_exec_type import SupportsLocalExec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mlos_bench.tunables.tunable import TunableValue
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
import json5 as json
|
||||
import pytest
|
||||
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValueTypeName
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_types import TunableValueTypeName
|
||||
|
||||
|
||||
def test_tunable_name() -> None:
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
import json5 as json
|
||||
import pytest
|
||||
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValueTypeName
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_types import TunableValueTypeName
|
||||
|
||||
|
||||
def test_categorical_distribution() -> None:
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ from mlos_bench.optimizers.convert_configspace import (
|
|||
special_param_names,
|
||||
tunable_groups_to_configspace,
|
||||
)
|
||||
from mlos_bench.tunables.tunable import DistributionName
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import DistributionName
|
||||
|
||||
_CS_HYPERPARAMETER = {
|
||||
("float", "beta"): BetaFloatHyperparameter,
|
||||
|
|
|
|||
|
|
@ -87,9 +87,9 @@ def configuration_space() -> ConfigurationSpace:
|
|||
),
|
||||
CategoricalHyperparameter(
|
||||
name=kernel_sched_migration_cost_ns_type,
|
||||
choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE],
|
||||
choices=[TunableValueKind.SPECIAL.value, TunableValueKind.RANGE.value],
|
||||
weights=[0.5, 0.5],
|
||||
default_value=TunableValueKind.SPECIAL,
|
||||
default_value=TunableValueKind.SPECIAL.value,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
@ -98,12 +98,12 @@ def configuration_space() -> ConfigurationSpace:
|
|||
EqualsCondition(
|
||||
spaces[kernel_sched_migration_cost_ns_special],
|
||||
spaces[kernel_sched_migration_cost_ns_type],
|
||||
TunableValueKind.SPECIAL,
|
||||
TunableValueKind.SPECIAL.value,
|
||||
),
|
||||
EqualsCondition(
|
||||
spaces["kernel_sched_migration_cost_ns"],
|
||||
spaces[kernel_sched_migration_cost_ns_type],
|
||||
TunableValueKind.RANGE,
|
||||
TunableValueKind.RANGE.value,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@
|
|||
"""Unit tests for deep copy of tunable objects and groups."""
|
||||
|
||||
from mlos_bench.tunables.covariant_group import CovariantTunableGroup
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
def test_copy_tunable_int(tunable_int: Tunable) -> None:
|
||||
|
|
|
|||
|
|
@ -30,28 +30,36 @@ Tunable
|
|||
|
||||
The :py:class:`~mlos_bench.tunables.tunable.Tunable` class is used to define a
|
||||
single tunable parameter.
|
||||
A ``Tunable`` can be a ``categorical`` or numeric (``int`` or ``float``) and always has
|
||||
at least a domain (``range`` or set of ``values``) and default.
|
||||
A ``Tunable`` has a :py:attr:`~.Tunable.type` and can be a ``categorical`` or
|
||||
numeric (``int`` or ``float``) and always has at least a domain
|
||||
(:py:attr:`~.Tunable.range` or set of :py:attr:`~.Tunable.values`) and a
|
||||
:py:attr:`~.Tunable.default`.
|
||||
Each type can also have a number of additional properties that can optionally be set
|
||||
to help control the sampling of the tunable.
|
||||
|
||||
For instance:
|
||||
|
||||
- Numeric tunables can have a ``distribution`` property to specify the sampling
|
||||
distribution. ``log`` sampling can also be enabled for numeric tunables.
|
||||
- Categorical tunables can have a ``values_weights`` property to specify biased
|
||||
sampling of the values
|
||||
- ``special`` values can be marked to indicate that they need more explicit testing
|
||||
This can be useful for values that indicate "automatic" or "disabled" behavior.
|
||||
- Numeric tunables can have a :py:attr:`~.Tunable.distribution` property to specify the sampling
|
||||
distribution. :py:attr:`log <.Tunable.is_log>` sampling can also be enabled for
|
||||
numeric tunables.
|
||||
- Categorical tunables can have a :py:attr:`values_weights <.Tunable.weights>`
|
||||
property to specify biased sampling of the values
|
||||
- :py:attr:`~.Tunable.special` values can be marked to indicate that they need more
|
||||
explicit testing. This can be useful for values that indicate "automatic" or
|
||||
"disabled" behavior.
|
||||
|
||||
The full set of supported properties can be found in the `JSON schema for tunable
|
||||
The :py:class:`~mlos_bench.tunables.tunable.Tunable` class attributes documentation
|
||||
and :py:class:`~mlos_bench.tunables.tunable_types.TunableDict` class documentation
|
||||
provides some more information and example on the available properties.
|
||||
|
||||
The full set of supported properties is specified in the `JSON schema for tunable
|
||||
parameters
|
||||
<https://github.com/microsoft/MLOS/blob/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-params-schema.json>`_
|
||||
and seen in some of the `test examples in the source tree
|
||||
and can be seen in some of the `test examples in the source tree
|
||||
<https://github.com/microsoft/MLOS/tree/main/mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/>`_.
|
||||
|
||||
CovariantGroup
|
||||
++++++++++++++
|
||||
CovariantTunableGroup
|
||||
+++++++++++++++++++++
|
||||
|
||||
The :py:class:`~mlos_bench.tunables.covariant_group.CovariantTunableGroup` class is
|
||||
used to define a group of related tunable parameters that are all configured
|
||||
|
|
@ -65,6 +73,15 @@ TunableGroups
|
|||
The :py:class:`~mlos_bench.tunables.tunable_groups.TunableGroups` class is used to
|
||||
define an entire set of tunable parameters (e.g., combined set of covariant groups).
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
Currently we lack a config language for expressing constraints between tunables
|
||||
|
||||
(e.g., ``a < b`` or ``a + b < c``)
|
||||
|
||||
This is supported in the underlying :py:mod:`mlos_core` library, but is not yet
|
||||
exposed in the ``mlos_bench`` config API.
|
||||
|
||||
Usage
|
||||
^^^^^
|
||||
|
||||
|
|
@ -81,27 +98,34 @@ Then individual covariant groups can be enabled via the ``tunable_params`` and
|
|||
See the :py:mod:`mlos_bench.config` and :py:mod:`mlos_bench.environments` module
|
||||
documentation for more information.
|
||||
|
||||
In benchmarking-only mode (e.g., without an ``Optimizer`` specified), ``mlos_bench``
|
||||
can still run with a particular set of ``--tunable-values`` (e.g., a simple
|
||||
key-value file declaring a set of values to assign to the set of configured tunable
|
||||
parameters) in order to manually explore a configuration space.
|
||||
In benchmarking-only mode (e.g., without an ``Optimizer`` specified),
|
||||
`mlos_bench <../../../mlos_bench.run.usage.html>`_ can still run with a
|
||||
particular set of ``--tunable-values`` (e.g., a simple key-value file declaring
|
||||
a set of values to assign to the set of configured tunable parameters) in order
|
||||
to manually explore a configuration space.
|
||||
|
||||
See the :py:mod:`mlos_bench.run` module documentation for more information.
|
||||
See the :py:class:`~mlos_bench.optimizers.one_shot_optimizer.OneShotOptimizer`
|
||||
and :py:mod:`mlos_bench.run` module documentation and the for more information.
|
||||
|
||||
During an Environment's ``setup`` and ``run`` phases the tunables can be exported to
|
||||
a JSON file using the ``dump_params_file`` property of the Environment config for
|
||||
the user scripts to use when configuring the target system.
|
||||
The ``meta`` property of the tunable config can be used to add additional
|
||||
information for this step (e.g., a unit suffix to append to the value).
|
||||
During an Environment's
|
||||
:py:meth:`~mlos_bench.environments.base_environment.Environment.setup` and
|
||||
:py:meth:`~mlos_bench.environments.base_environment.Environment.run` phases the
|
||||
tunables can be exported to a JSON file using the ``dump_params_file`` property
|
||||
of the Environment config for the user scripts to use when configuring the
|
||||
target system.
|
||||
The :py:attr:`~.Tunable.meta` property of the tunable config can be used to add
|
||||
additional information for this step (e.g., a unit suffix to append to the
|
||||
value).
|
||||
|
||||
See the :py:mod:`mlos_bench.environments` module documentation for more information.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Here's a short (incomplete) example of some of the TunableGroups JSON configuration
|
||||
options, expressed in Python (for testing purposes).
|
||||
However, most of the time you will be loading these from a JSON config file stored
|
||||
along with the associated Environment config.
|
||||
Here's a short (incomplete) example of some of the :py:class:`.TunableGroups`
|
||||
JSON configuration options, expressed in Python (for testing purposes).
|
||||
However, most of the time you will be loading these from a JSON config file
|
||||
stored along with the associated
|
||||
:py:class:`~mlos_bench.environments.base_environment.Environment` config.
|
||||
|
||||
For more tunable parameters examples refer to the `JSON schema
|
||||
<https://github.com/microsoft/MLOS/blob/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-params-schema.json>`_
|
||||
|
|
@ -116,6 +140,7 @@ There are also examples of `tunable values in the source tree
|
|||
>>> from mlos_bench.services.config_persistence import ConfigPersistenceService
|
||||
>>> service = ConfigPersistenceService()
|
||||
>>> json_config = '''
|
||||
... // Use json5 (or jsonc) syntax to allow comments and other more flexible syntax.
|
||||
... {
|
||||
... "group_1": {
|
||||
... "cost": 1,
|
||||
|
|
@ -131,18 +156,21 @@ There are also examples of `tunable values in the source tree
|
|||
... },
|
||||
... "int_param": {
|
||||
... "type": "int",
|
||||
... "range": [-1, 10],
|
||||
... "range": [1, 10],
|
||||
... "default": 5,
|
||||
... // Mark some values as "special", that need more explicit testing.
|
||||
... // e.g., maybe these indicate "automatic" or "disabled" behavior for
|
||||
... // the system being tested instead of an explicit size
|
||||
... "special": [-1, 0],
|
||||
... // Optionally specify a sampling distribution.
|
||||
... // Optionally specify a sampling distribution
|
||||
... // to influence which values to prioritize.
|
||||
... "distribution": {
|
||||
... "type": "uniform" // alternatively, "beta" or "normal"
|
||||
... },
|
||||
... // Free form key-value pairs that can be used with the
|
||||
... // tunable upon sampling for composing configs.
|
||||
... // These can be retrieved later to help generate
|
||||
... // config files from the sampled tunables.
|
||||
... "meta": {
|
||||
... "suffix": "MB"
|
||||
... }
|
||||
|
|
@ -197,10 +225,13 @@ Notes
|
|||
-----
|
||||
Internally, :py:class:`.TunableGroups` are converted to
|
||||
:external:py:class:`ConfigSpace.ConfigurationSpace` objects for use with
|
||||
:py:mod:`mlos_core`.
|
||||
:py:mod:`mlos_core` using the :py:mod:`mlos_bench.optimizers.convert_configspace`.
|
||||
See the "Spaces" section in the :py:mod:`mlos_core` module documentation for more
|
||||
information.
|
||||
|
||||
In order to handle sampling of :py:attr:`.Tunable.special` values, the ``!``
|
||||
character is prohibited from being used in the name of a :py:class:`.Tunable`.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.config` : Overview of the configuration system.
|
||||
|
|
@ -210,8 +241,9 @@ See Also
|
|||
:py:meth:`.TunableGroups.assign` : Notes on special cases for assigning tunable values.
|
||||
"""
|
||||
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_groups import TunableGroups
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
__all__ = [
|
||||
"Tunable",
|
||||
|
|
|
|||
|
|
@ -2,18 +2,33 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""Tunable parameter definition."""
|
||||
"""
|
||||
CovariantTunableGroup class definition.
|
||||
|
||||
A collection of :py:class:`.Tunable` parameters that are updated together (e.g.,
|
||||
with the same cost).
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.tunables.tunable_groups : TunableGroups class definition.
|
||||
"""
|
||||
|
||||
import copy
|
||||
from collections.abc import Iterable
|
||||
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
|
||||
class CovariantTunableGroup:
|
||||
"""
|
||||
A collection of tunable parameters.
|
||||
A collection of :py:class:`.Tunable` parameters.
|
||||
|
||||
Changing any of the parameters in the group incurs the same cost of the experiment.
|
||||
|
||||
See Also
|
||||
--------
|
||||
mlos_bench.tunables.tunable_groups : TunableGroups class definition.
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, config: dict):
|
||||
|
|
|
|||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -2,14 +2,102 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""TunableGroups definition."""
|
||||
"""
|
||||
TunableGroups definition.
|
||||
|
||||
A collection of :py:class:`.CovariantTunableGroup` s of :py:class:`.Tunable`
|
||||
parameters.
|
||||
|
||||
Used to define the configuration space for an
|
||||
:py:class:`~mlos_bench.environments.base_environment.Environment` for an
|
||||
:py:class:`~mlos_bench.optimizers.base_optimizer.Optimizer` to explore.
|
||||
|
||||
Config
|
||||
++++++
|
||||
|
||||
The configuration of the tunable parameters is generally given via a JSON config file.
|
||||
The syntax looks something like this:
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{ // starts a TunableGroups config (e.g., for one Environment)
|
||||
"group1": { // starts a CovariantTunableGroup config
|
||||
"cost": 7,
|
||||
"params": {
|
||||
"param1": { // starts a Tunable config, named "param1"
|
||||
"type": "int",
|
||||
"range": [0, 100],
|
||||
"default": 50
|
||||
},
|
||||
"param2": { // starts a new Tunable config, named "param2", within that same group
|
||||
"type": "float",
|
||||
"range": [0.0, 100.0],
|
||||
"default": 50.0
|
||||
},
|
||||
"param3": {
|
||||
"type": "categorical",
|
||||
"values": ["on", "off", "auto"],
|
||||
"default": "auto"
|
||||
}
|
||||
}
|
||||
},
|
||||
"group2": { // starts a new CovariantTunableGroup config
|
||||
"cost": 7,
|
||||
"params": {
|
||||
"some_param1": {
|
||||
"type": "int",
|
||||
"range": [0, 10],
|
||||
"default": 5
|
||||
},
|
||||
"some_param2": {
|
||||
"type": "float",
|
||||
"range": [0.0, 100.0],
|
||||
"default": 50.0
|
||||
},
|
||||
"some_param3": {
|
||||
"type": "categorical",
|
||||
"values": ["red", "green", "blue"],
|
||||
"default": "green"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
The JSON config is expected to be a dictionary of covariant tunable groups.
|
||||
|
||||
Each covariant group has a name and a cost associated with changing any/all of the
|
||||
parameters in that covariant group.
|
||||
|
||||
Each group has a dictionary of :py:class:`.Tunable` parameters, where the key is
|
||||
the name of the parameter and the value is a dictionary of the parameter's
|
||||
configuration (see the :py:class:`.Tunable` class for more information on the
|
||||
different ways they can be configured).
|
||||
|
||||
Generally tunables are associated with an
|
||||
:py:class:`~mlos_bench.environments.base_environment.Environment` and included along
|
||||
with the Environment's config directory (e.g., ``env-name-tunables.mlos.jsonc``) and
|
||||
referenced in the Environment config using the ``include_tunables`` property.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.tunables` :
|
||||
For more information on tunable parameters and their configuration.
|
||||
:py:mod:`mlos_bench.tunables.tunable` :
|
||||
Tunable parameter definition.
|
||||
:py:mod:`mlos_bench.config` :
|
||||
Configuration system for mlos_bench.
|
||||
:py:mod:`mlos_bench.environments` :
|
||||
Environment configuration and setup.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import logging
|
||||
from collections.abc import Generator, Iterable, Mapping
|
||||
|
||||
from mlos_bench.config.schemas import ConfigSchema
|
||||
from mlos_bench.tunables.covariant_group import CovariantTunableGroup
|
||||
from mlos_bench.tunables.tunable import Tunable, TunableValue
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
from mlos_bench.tunables.tunable_types import TunableValue
|
||||
|
||||
_LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -30,8 +118,8 @@ class TunableGroups:
|
|||
|
||||
See Also
|
||||
--------
|
||||
:py:mod:`mlos_bench.tunables` : for more information on tunable parameters and
|
||||
their configuration.
|
||||
:py:mod:`mlos_bench.tunables` :
|
||||
For more information on tunable parameters and their configuration.
|
||||
"""
|
||||
if config is None:
|
||||
config = {}
|
||||
|
|
@ -356,8 +444,8 @@ class TunableGroups:
|
|||
param_values : Mapping[str, TunableValue]
|
||||
Dictionary mapping Tunable parameter names to new values.
|
||||
|
||||
As a special behavior when the mapping is empty the method will restore
|
||||
the default values rather than no-op.
|
||||
As a special behavior when the mapping is empty (``{}``) the method will
|
||||
restore the default values rather than no-op.
|
||||
This allows an empty dictionary in json configs to be used to reset the
|
||||
tunables to defaults without having to copy the original values from the
|
||||
tunable_params definition.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,386 @@
|
|||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
#
|
||||
"""
|
||||
Helper types for :py:class:`~mlos_bench.tunables.tunable.Tunable`.
|
||||
|
||||
The main class of interest to most users in this module is :py:class:`.TunableDict`,
|
||||
which provides the typed conversions from a JSON config to a config used for
|
||||
creating a :py:class:`~mlos_bench.tunables.tunable.Tunable`.
|
||||
|
||||
The other types are mostly used for type checking and documentation purposes.
|
||||
"""
|
||||
|
||||
# NOTE: pydoctest doesn't scan variable docstrings so we put the examples in the
|
||||
# Tunable class docstrings.
|
||||
# These type aliases are moved here mostly to allow easier documentation reading of
|
||||
# the Tunable class itself.
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Literal, TypeAlias, TypedDict
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# Used to allow for shorter docstring references.
|
||||
from mlos_bench.tunables.tunable import Tunable
|
||||
|
||||
TunableValue: TypeAlias = int | float | str | None
|
||||
"""A :py:class:`TypeAlias` for a :py:class:`~.Tunable` parameter value."""
|
||||
|
||||
TunableValueType: TypeAlias = type[int] | type[float] | type[str]
|
||||
"""A :py:class:`TypeAlias` for :py:class:`~.Tunable` value
|
||||
:py:attr:`data type <.Tunable.dtype>`.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.dtype <.Tunable.dtype>` : Example of accepted types.
|
||||
"""
|
||||
|
||||
TunableValueTypeTuple = (int, float, str, type(None))
|
||||
"""
|
||||
Tunable value ``type`` tuple.
|
||||
|
||||
Notes
|
||||
-----
|
||||
For checking whether a param is a :py:data:`.TunableValue` with
|
||||
:py:func:`isinstance`.
|
||||
"""
|
||||
|
||||
TunableValueTypeName = Literal["int", "float", "categorical"]
|
||||
"""
|
||||
The accepted string names of a :py:class:`~.Tunable` value :py:attr:`~.Tunable.type`.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.type <.Tunable.type>` : Example of accepted type names.
|
||||
"""
|
||||
|
||||
TUNABLE_DTYPE: dict[TunableValueTypeName, TunableValueType] = {
|
||||
"int": int,
|
||||
"float": float,
|
||||
"categorical": str,
|
||||
}
|
||||
"""
|
||||
Maps :py:class:`~.Tunable` types to their corresponding Python data types by name.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.dtype <.Tunable.dtype>` : Example of type mappings.
|
||||
"""
|
||||
|
||||
TunableValuesDict = dict[str, TunableValue]
|
||||
"""Tunable values dictionary type."""
|
||||
|
||||
DistributionName = Literal["uniform", "normal", "beta"]
|
||||
"""
|
||||
The :py:attr:`~.Tunable.distribution` type names for a :py:class:`~.Tunable` value.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution <.Tunable.distribution>` :
|
||||
Example of accepted distribution names.
|
||||
"""
|
||||
|
||||
|
||||
class DistributionDictOpt(TypedDict, total=False): # total=False allows for optional fields
|
||||
"""
|
||||
A :py:class:`TypedDict` for a :py:class:`~.Tunable` parameter's optional
|
||||
:py:attr:`~.Tunable.distribution_params` config.
|
||||
|
||||
Mostly used for type checking. These are the types expected to be received from
|
||||
the json config.
|
||||
|
||||
Notes
|
||||
-----
|
||||
:py:class:`.DistributionDict` contains the required fields for the
|
||||
:py:attr:`Tunable.distribution <.Tunable.distribution>` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution_params <.Tunable.distribution_params>` :
|
||||
Examples of distribution parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore # pylint: disable=useless-super-delegation
|
||||
""".. comment: don't inherit the docstring"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
params: dict[str, float] | None
|
||||
"""
|
||||
The parameters for the distribution.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution_params <.Tunable.distribution_params>` :
|
||||
Examples of distribution parameters.
|
||||
"""
|
||||
|
||||
|
||||
class DistributionDict(DistributionDictOpt):
|
||||
"""
|
||||
A :py:class:`TypedDict` for a :py:class:`~.Tunable` parameter's required
|
||||
:py:attr:`~.Tunable.distribution` config parameters.
|
||||
|
||||
Mostly used for type checking. These are the types expected to be received from the
|
||||
json config.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution <.Tunable.distribution>` :
|
||||
Examples of Tunables with distributions.
|
||||
:py:attr:`Tunable.distribution_params <.Tunable.distribution_params>` :
|
||||
Examples of distribution parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore # pylint: disable=useless-super-delegation
|
||||
""".. comment: don't inherit the docstring"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
type: DistributionName
|
||||
"""
|
||||
The name of the distribution.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution <.Tunable.distribution>` :
|
||||
Examples of distribution names.
|
||||
"""
|
||||
|
||||
|
||||
class TunableDictOpt(TypedDict, total=False): # total=False allows for optional fields
|
||||
"""
|
||||
A :py:class:`TypedDict` for a :py:class:`~.Tunable` parameter's optional config
|
||||
parameters.
|
||||
|
||||
Mostly used for type checking. These are the types expected to be received from
|
||||
the json config.
|
||||
|
||||
Notes
|
||||
-----
|
||||
:py:class:`TunableDict` contains the required fields for the
|
||||
:py:class:`~.Tunable` parameter.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore # pylint: disable=useless-super-delegation
|
||||
""".. comment: don't inherit the docstring"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# Optional fields:
|
||||
|
||||
description: str | None
|
||||
"""
|
||||
Description of the :py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.description <.Tunable.description>`
|
||||
"""
|
||||
|
||||
values: list[str | None] | None
|
||||
"""
|
||||
List of values (or categories) for a "categorical" type :py:class:`~.Tunable`
|
||||
parameter.
|
||||
|
||||
A list of values is required for "categorical" type Tunables.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.categories <.Tunable.categories>`
|
||||
:py:attr:`Tunable.values <.Tunable.values>`
|
||||
"""
|
||||
|
||||
range: Sequence[int] | Sequence[float] | None
|
||||
"""
|
||||
The range of values for an "int" or "float" type :py:class:`~.Tunable` parameter.
|
||||
|
||||
Must be a sequence of two values: ``[min, max]``.
|
||||
|
||||
A range is required for "int" and "float" type Tunables.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.range <.Tunable.range>` : Examples of ranges.
|
||||
:py:attr:`Tunable.values <.Tunable.values>`
|
||||
"""
|
||||
|
||||
special: list[int] | list[float] | None
|
||||
"""
|
||||
List of special values for an "int" or "float" type :py:class:`~.Tunable` parameter.
|
||||
|
||||
These are values that are considered special by the target system (e.g.,
|
||||
``null``, ``0``, ``-1``, ``auto``, etc.) and should be sampled with higher
|
||||
weights.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.special <.Tunable.special>` : Examples of special values.
|
||||
"""
|
||||
|
||||
quantization_bins: int | None
|
||||
"""
|
||||
The number of quantization bins for an "int" or "float" type :py:class:`~.Tunable`
|
||||
parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.quantization_bins <.Tunable.quantization_bins>` :
|
||||
Examples of quantized Tunables.
|
||||
"""
|
||||
|
||||
log: bool | None
|
||||
"""
|
||||
Whether to use log sampling for an "int" or "float" type :py:class:`~.Tunable`
|
||||
parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.is_log <.Tunable.is_log>`
|
||||
"""
|
||||
|
||||
distribution: DistributionDict | None
|
||||
"""
|
||||
Optional sampling distribution configuration for an "int" or "float" type
|
||||
:py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.distribution <.Tunable.distribution>` :
|
||||
Examples of distributions.
|
||||
:py:attr:`Tunable.distribution_params <.Tunable.distribution_params>` :
|
||||
Examples of distribution parameters.
|
||||
"""
|
||||
|
||||
values_weights: list[float] | None
|
||||
"""
|
||||
Optional sampling weights for the values of a "categorical" type
|
||||
:py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.weights <.Tunable.weights>` : Examples of weighted sampling Tunables.
|
||||
"""
|
||||
|
||||
special_weights: list[float] | None
|
||||
"""
|
||||
Optional sampling weights for the special values of an "int" or "float" type
|
||||
:py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.weights <.Tunable.weights>` : Examples of weighted sampling Tunables.
|
||||
"""
|
||||
|
||||
range_weight: float | None
|
||||
"""
|
||||
Optional sampling weight for the main ranges of an "int" or "float" type
|
||||
:py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.range_weight <.Tunable.range_weight>` :
|
||||
Examples of weighted sampling Tunables.
|
||||
"""
|
||||
|
||||
meta: dict[str, Any]
|
||||
"""
|
||||
Free form dict to store additional metadata for the :py:class:`~.Tunable` parameter
|
||||
(e.g., unit suffix, etc.)
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.meta <.Tunable.meta>` : Examples of Tunables with metadata.
|
||||
"""
|
||||
|
||||
|
||||
class TunableDict(TunableDictOpt):
|
||||
"""
|
||||
A :py:class:`TypedDict` for a :py:class:`~.Tunable` parameter's required config
|
||||
parameters.
|
||||
|
||||
Mostly used for type checking. These are the types expected to be received from
|
||||
the json config.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> # Example values of the TunableDict
|
||||
>>> TunableDict({'type': 'int', 'default': 0, 'range': [0, 10]})
|
||||
{'type': 'int', 'default': 0, 'range': [0, 10]}
|
||||
|
||||
>>> # Example values of the TunableDict with optional fields
|
||||
>>> TunableDict({'type': 'categorical', 'default': 'a', 'values': ['a', 'b']})
|
||||
{'type': 'categorical', 'default': 'a', 'values': ['a', 'b']}
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore # pylint: disable=useless-super-delegation
|
||||
""".. comment: don't inherit the docstring"""
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# Required fields
|
||||
|
||||
type: TunableValueTypeName
|
||||
"""
|
||||
The name of the type of the :py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.type <.Tunable.type>` : Examples of type names.
|
||||
"""
|
||||
|
||||
default: TunableValue
|
||||
"""
|
||||
The default value of the :py:class:`~.Tunable` parameter.
|
||||
|
||||
See Also
|
||||
--------
|
||||
:py:attr:`Tunable.default <.Tunable.default>`
|
||||
"""
|
||||
|
||||
|
||||
def tunable_dict_from_dict(config: dict[str, Any]) -> TunableDict:
|
||||
"""
|
||||
Creates a TunableDict from a regular dict.
|
||||
|
||||
Notes
|
||||
-----
|
||||
Mostly used for type checking while instantiating a
|
||||
:py:class:`~.Tunable` from a json config.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict[str, Any]
|
||||
A regular dict that represents a :py:class:`.TunableDict`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
TunableDict
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> # Example values of the TunableDict
|
||||
>>> import json5 as json
|
||||
>>> config = json.loads("{'type': 'int', 'default': 0, 'range': [0, 10]}")
|
||||
>>> config
|
||||
{'type': 'int', 'default': 0, 'range': [0, 10]}
|
||||
>>> typed_dict = tunable_dict_from_dict(config)
|
||||
>>> typed_dict
|
||||
{'type': 'int', 'description': None, 'default': 0, 'values': None, 'range': [0, 10], 'quantization_bins': None, 'log': None, 'distribution': None, 'special': None, 'values_weights': None, 'special_weights': None, 'range_weight': None, 'meta': {}}
|
||||
""" # pylint: disable=line-too-long # noqa: E501
|
||||
_type = config.get("type")
|
||||
if _type not in TUNABLE_DTYPE:
|
||||
raise ValueError(f"Invalid parameter type: {_type}")
|
||||
_meta = config.get("meta", {})
|
||||
return TunableDict(
|
||||
type=_type,
|
||||
description=config.get("description"),
|
||||
default=config.get("default"),
|
||||
values=config.get("values"),
|
||||
range=config.get("range"),
|
||||
quantization_bins=config.get("quantization_bins"),
|
||||
log=config.get("log"),
|
||||
distribution=config.get("distribution"),
|
||||
special=config.get("special"),
|
||||
values_weights=config.get("values_weights"),
|
||||
special_weights=config.get("special_weights"),
|
||||
range_weight=config.get("range_weight"),
|
||||
meta=_meta,
|
||||
)
|
||||
|
|
@ -17,9 +17,15 @@ for all Optimizers and provides the core
|
|||
This module also provides a simple :py:class:`~.OptimizerFactory` class to
|
||||
:py:meth:`~.OptimizerFactory.create` an Optimizer.
|
||||
|
||||
Examples
|
||||
Generally speaking, the :py:mod:`mlos_core` package is intended to be used in
|
||||
conjunction with the :py:mod:`mlos_bench` package, which provides a higher-level
|
||||
interface for running benchmarks and autotuning experiments.
|
||||
|
||||
See Also
|
||||
--------
|
||||
TODO: Add example usage here.
|
||||
mlos_bench.optimizers.mlos_core_optimizer :
|
||||
The mlos_bench Optimizer class that uses the mlos_core Optimizers to run
|
||||
autotuning experiments.
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
|
|
|||
|
|
@ -20,9 +20,15 @@ information on how to do this with :py:mod:`mlos_bench`.
|
|||
This module provides a simple :py:class:`.SpaceAdapterFactory` class to
|
||||
:py:meth:`~.SpaceAdapterFactory.create` space adapters.
|
||||
|
||||
Examples
|
||||
Generally speaking, the :py:mod:`mlos_core` package is intended to be used in
|
||||
conjunction with the :py:mod:`mlos_bench` package, which provides a higher-level
|
||||
interface for running benchmarks and autotuning experiments.
|
||||
|
||||
See Also
|
||||
--------
|
||||
TODO: Add example usage here.
|
||||
mlos_bench.optimizers.mlos_core_optimizer :
|
||||
The mlos_bench Optimizer class that uses the mlos_core Optimizers and
|
||||
SpaceAdapters to run autotuning experiments.
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ class BaseSpaceAdapter(metaclass=ABCMeta):
|
|||
|
||||
def __init__(self, *, orig_parameter_space: ConfigSpace.ConfigurationSpace):
|
||||
self._orig_parameter_space: ConfigSpace.ConfigurationSpace = orig_parameter_space
|
||||
# Use the same random state as the original parameter space.
|
||||
self._random_state = orig_parameter_space.random
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
|
|
|||
|
|
@ -65,11 +65,17 @@ class LlamaTuneAdapter(BaseSpaceAdapter): # pylint: disable=too-many-instance-a
|
|||
The original (user-provided) parameter space to optimize.
|
||||
num_low_dims : int
|
||||
Number of dimensions used in the low-dimensional parameter search space.
|
||||
special_param_values_dict : dict | None
|
||||
Dictionary of special
|
||||
special_param_values : dict | None
|
||||
Dictionary of special parameter values.
|
||||
Each key is the name of a parameter, and the value is either:
|
||||
- an integer (special value), or
|
||||
- a tuple of an integer and a float (special integer value and biasing percentage)
|
||||
max_unique_values_per_param : int | None
|
||||
Number of unique values per parameter. Used to discretize the parameter space.
|
||||
If `None` space discretization is disabled.
|
||||
use_approximate_reverse_mapping : bool
|
||||
Whether to use an approximate reverse mapping to help register
|
||||
configurations during resume.
|
||||
"""
|
||||
super().__init__(orig_parameter_space=orig_parameter_space)
|
||||
|
||||
|
|
|
|||
|
|
@ -32,10 +32,15 @@ __version__ = VERSION
|
|||
|
||||
|
||||
class MlosVizMethod(Enum):
|
||||
"""What method to use for visualizing the experiment results."""
|
||||
"""What method to use for visualizing the Experiment results."""
|
||||
|
||||
DABL = "dabl"
|
||||
"""Use DABL for automatic data correlation and visualization of
|
||||
:py:class:`~.ExperimentData`.
|
||||
"""
|
||||
|
||||
AUTO = DABL # use dabl as the current default
|
||||
"""The default automatic :py:class:`~.ExperimentData` visualization method."""
|
||||
|
||||
|
||||
def ignore_plotter_warnings(plotter_method: MlosVizMethod = MlosVizMethod.AUTO) -> None:
|
||||
|
|
@ -46,7 +51,7 @@ def ignore_plotter_warnings(plotter_method: MlosVizMethod = MlosVizMethod.AUTO)
|
|||
Parameters
|
||||
----------
|
||||
plotter_method: MlosVizMethod
|
||||
The method to use for visualizing the experiment results.
|
||||
The method to use for visualizing the Experiment results.
|
||||
"""
|
||||
base.ignore_plotter_warnings()
|
||||
if plotter_method == MlosVizMethod.DABL:
|
||||
|
|
@ -67,14 +72,14 @@ def plot(
|
|||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Plots the results of the experiment.
|
||||
Plots the results of the given :py:class:`~.ExperimentData`.
|
||||
|
||||
Intended to be used from a Jupyter notebook.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exp_data: ExperimentData
|
||||
The experiment data to plot.
|
||||
The Experiment data to plot.
|
||||
results_df : pandas.DataFrame | None
|
||||
Optional `results_df` to plot.
|
||||
If not provided, defaults to :py:attr:`.ExperimentData.results_df` property.
|
||||
|
|
@ -82,7 +87,7 @@ def plot(
|
|||
Optional objectives to plot.
|
||||
If not provided, defaults to :py:attr:`.ExperimentData.objectives` property.
|
||||
plotter_method: MlosVizMethod
|
||||
The method to use for visualizing the experiment results.
|
||||
The method to use for visualizing the Experiment results.
|
||||
filter_warnings: bool
|
||||
Whether or not to filter some warnings from the plotter.
|
||||
kwargs : dict
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче