зеркало из https://github.com/microsoft/nni.git
Mutable shortcut and enhancements (#5336)
This commit is contained in:
Yuge Zhang
GitHub
Родитель
9b59d6d797
Коммит
19cb631537
|
|
@ -12,6 +12,7 @@ _init_logger()
|
|||
from .common.framework import *
|
||||
from .common.serializer import trace, dump, load
|
||||
from .experiment import Experiment
|
||||
from .mutable.shortcut import *
|
||||
from .runtime.env_vars import dispatcher_env_vars
|
||||
from .runtime.log import enable_global_logging, silence_stdout
|
||||
from .utils import ClassArgsValidator
|
||||
|
|
|
|||
|
|
@ -9,6 +9,53 @@ from .annotation import MutableAnnotation
|
|||
from .mutable import LabeledMutable, MutableSymbol, Categorical, Numerical
|
||||
|
||||
|
||||
def randint(label: str, lower: int, upper: int) -> Categorical[int]:
|
||||
"""Choosing a random integer between lower (inclusive) and upper (exclusive).
|
||||
|
||||
Currently it is translated to a :func:`choice`.
|
||||
This behavior might change in future releases.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.randint('x', 1, 5)
|
||||
Categorical([1, 2, 3, 4], label='x')
|
||||
"""
|
||||
return RandomInteger(lower, upper, label=label)
|
||||
|
||||
|
||||
def lognormal(label: str, mu: float, sigma: float) -> Numerical:
|
||||
"""Log-normal (in the context of NNI) is defined as the exponential transformation of a normal random variable,
|
||||
with mean ``mu`` and deviation ``sigma``. That is::
|
||||
|
||||
exp(normal(mu, sigma))
|
||||
|
||||
In another word, the logarithm of the return value is normally distributed.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.lognormal('x', 4., 2.)
|
||||
Numerical(-inf, inf, mu=4.0, sigma=2.0, log_distributed=True, label='x')
|
||||
>>> nni.lognormal('x', 0., 1.).random()
|
||||
2.3308575497749584
|
||||
>>> np.log(x) for x in nni.lognormal('x', 4., 2.).grid(granularity=2)]
|
||||
[2.6510204996078364, 4.0, 5.348979500392163]
|
||||
"""
|
||||
return Numerical(mu=mu, sigma=sigma, log_distributed=True, label=label)
|
||||
|
||||
|
||||
def qlognormal(label: str, mu: float, sigma: float, quantize: float) -> Numerical:
|
||||
"""A combination of :func:`qnormal` and :func:`lognormal`.
|
||||
|
||||
Similar to :func:`qloguniform`, the quantize is done **after** the sample is drawn from the log-normal distribution.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.qlognormal('x', 4., 2., 1.)
|
||||
Numerical(-inf, inf, mu=4.0, sigma=2.0, q=1.0, log_distributed=True, label='x')
|
||||
"""
|
||||
return Numerical(mu=mu, sigma=sigma, log_distributed=True, quantize=quantize, label=label)
|
||||
|
||||
|
||||
class Permutation(MutableSymbol):
|
||||
"""Get a permutation of several values.
|
||||
Not implemented. Kept as a placeholder.
|
||||
|
|
@ -23,7 +70,12 @@ class RandomInteger(Categorical[int]):
|
|||
but this class gives better semantics,
|
||||
and is consistent with the old ``randint``.
|
||||
"""
|
||||
pass
|
||||
def __init__(self, lower: int, upper: int, label: str | None = None) -> None:
|
||||
if not isinstance(lower, int) or not isinstance(upper, int):
|
||||
raise TypeError('lower and upper must be integers.')
|
||||
if lower >= upper:
|
||||
raise ValueError('lower must be strictly smaller than upper.')
|
||||
super().__init__(list(range(lower, upper)), label=label)
|
||||
|
||||
|
||||
class NonNegativeRandomInteger(RandomInteger):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class SampleValidationError(ValueError):
|
|||
|
||||
def __str__(self) -> str:
|
||||
if self.paths:
|
||||
return self.msg + ' (path:' + ' -> '.join(self.paths) + ')'
|
||||
return self.msg + ' (path:' + ' -> '.join(map(str, self.paths)) + ')'
|
||||
else:
|
||||
return self.msg
|
||||
|
||||
|
|
|
|||
|
|
@ -18,13 +18,13 @@ from typing import Any, Callable
|
|||
from .mutable import Mutable, Sample
|
||||
from .utils import NoContextError, ContextStack
|
||||
|
||||
|
||||
_ENSURE_FROZEN_STRICT = True
|
||||
_FROZEN_CONTEXT_KEY = '_frozen'
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def ensure_frozen(mutable: Mutable | Any, sample: Sample | None = None, retries: int = 1000) -> Any:
|
||||
def ensure_frozen(mutable: Mutable | Any, *, strict: bool = True, sample: Sample | None = None, retries: int = 1000) -> Any:
|
||||
"""Ensure a mutable is frozen. Used when passing the mutable to a function which doesn't accept a mutable.
|
||||
|
||||
If the argument is not a mutable, nothing happens.
|
||||
|
|
@ -37,6 +37,8 @@ def ensure_frozen(mutable: Mutable | Any, sample: Sample | None = None, retries:
|
|||
----------
|
||||
mutable : nni.mutable.Mutable or any
|
||||
The mutable to freeze.
|
||||
strict
|
||||
Whether to raise an error if sample context is not provided and not found.
|
||||
sample
|
||||
The context to freeze the mutable with.
|
||||
retries
|
||||
|
|
@ -44,12 +46,14 @@ def ensure_frozen(mutable: Mutable | Any, sample: Sample | None = None, retries:
|
|||
|
||||
Examples
|
||||
--------
|
||||
>>> from nni.mutable import Mutable, ensure_frozen
|
||||
>>> ensure_frozen(Categorical([1, 2, 3]))
|
||||
1
|
||||
>>> ensure_frozen(Categorical([1, 2, 3], label='a'), sample={'a': 2})
|
||||
>>> with frozen_context({'a': 2}):
|
||||
... ensure_frozen(Categorical([1, 2, 3], label='a'))
|
||||
2
|
||||
>>> ensure_frozen('anything')
|
||||
>>> ensure_frozen(Categorical([1, 2, 3]), strict=False)
|
||||
1
|
||||
>>> ensure_frozen(Categorical([1, 2, 3], label='a'), sample={'a': 2}, strict=False)
|
||||
2
|
||||
>>> ensure_frozen('anything', strict=False)
|
||||
'anything'
|
||||
"""
|
||||
if not isinstance(mutable, Mutable):
|
||||
|
|
@ -74,10 +78,16 @@ def ensure_frozen(mutable: Mutable | Any, sample: Sample | None = None, retries:
|
|||
mutable, sample)
|
||||
raise
|
||||
else:
|
||||
if retries < 0:
|
||||
raise RuntimeError('Cannot freeze mutable. Please provide a context.')
|
||||
if retries < 0 or (_ENSURE_FROZEN_STRICT and strict):
|
||||
raise RuntimeError(
|
||||
f'No frozen context is found for {mutable!r}. Assuming no context. '
|
||||
'If you are using NAS, you are probably using `ensure_frozen` in forward, or outside the init of ModelSpace. '
|
||||
'Please avoid doing this as they will lead to erroneous results.'
|
||||
)
|
||||
|
||||
_logger.warning('No frozen context is found for %s. Assuming no context.', repr(mutable))
|
||||
# TODO: Currently only mutable parameters in NAS evaluator end up here.
|
||||
# It might cause consistency issues between multiple parameters without context.
|
||||
# I don't want to throw a warning here, but there should be a smarter way to do this.
|
||||
return mutable.robust_default(retries=retries)
|
||||
|
||||
|
||||
|
|
@ -197,7 +207,7 @@ class frozen_factory:
|
|||
|
||||
Parameters
|
||||
----------
|
||||
function
|
||||
callable
|
||||
The function to be invoked.
|
||||
sample
|
||||
The sample to be used as the frozen context.
|
||||
|
|
@ -210,8 +220,8 @@ class frozen_factory:
|
|||
|
||||
# NOTE: mutations on ``init_args`` and ``init_kwargs`` themselves are not supported.
|
||||
|
||||
def __init__(self, function: Callable[..., Any], sample: Sample | frozen_context):
|
||||
self.function = function
|
||||
def __init__(self, callable: Callable[..., Any], sample: Sample | frozen_context): # pylint: disable=redefined-builtin
|
||||
self.callable = callable
|
||||
if not isinstance(sample, frozen_context):
|
||||
self.sample = frozen_context(sample)
|
||||
else:
|
||||
|
|
@ -219,7 +229,7 @@ class frozen_factory:
|
|||
|
||||
def __call__(self, *init_args, **init_kwargs):
|
||||
with self.sample:
|
||||
return self.function(*init_args, **init_kwargs)
|
||||
return self.callable(*init_args, **init_kwargs)
|
||||
|
||||
def __repr__(self):
|
||||
return f'frozen_factory(function={self.function}, sample={self.sample.value})'
|
||||
return f'frozen_factory(callable={self.callable}, arch={self.sample.value})'
|
||||
|
|
|
|||
|
|
@ -101,6 +101,11 @@ def _mutable_equal(mutable1: Any, mutable2: Any) -> bool:
|
|||
else:
|
||||
return False
|
||||
return True
|
||||
if isinstance(mutable1, np.ndarray):
|
||||
if not isinstance(mutable2, np.ndarray):
|
||||
return False
|
||||
return np.array_equal(mutable1, mutable2)
|
||||
|
||||
return mutable1 == mutable2
|
||||
|
||||
|
||||
|
|
@ -342,6 +347,14 @@ class Mutable:
|
|||
"""Return a string representation of the extra information."""
|
||||
return ''
|
||||
|
||||
def as_legacy_dict(self) -> dict:
|
||||
"""Convert the mutable into the legacy dict representation.
|
||||
|
||||
For example, ``{"_type": "choice", "_value": [1, 2, 3]}`` is the legacy dict representation of
|
||||
``nni.mutable.Categorical([1, 2, 3])``.
|
||||
"""
|
||||
raise NotImplementedError(f'as_legacy_dict is not implemented for this type of mutable: {type(self)}.')
|
||||
|
||||
def equals(self, other: Any) -> bool:
|
||||
"""Compare two mutables.
|
||||
|
||||
|
|
@ -482,7 +495,7 @@ class Mutable:
|
|||
# Used in ``nni.trace``.
|
||||
# Calling ``ensure_frozen()`` by default.
|
||||
from .frozen import ensure_frozen
|
||||
return ensure_frozen(self)
|
||||
return ensure_frozen(self, strict=False)
|
||||
|
||||
|
||||
class LabeledMutable(Mutable):
|
||||
|
|
@ -601,6 +614,14 @@ class MutableSymbol(LabeledMutable, Symbol, MutableExpression):
|
|||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}({self.extra_repr()})'
|
||||
|
||||
def int(self) -> MutableExpression[int]:
|
||||
"""Cast the mutable to an integer."""
|
||||
return MutableExpression.to_int(self)
|
||||
|
||||
def float(self) -> MutableExpression[float]:
|
||||
"""Cast the mutable to a float."""
|
||||
return MutableExpression.to_float(self)
|
||||
|
||||
|
||||
class Categorical(MutableSymbol, Generic[Choice]):
|
||||
"""Choosing one from a list of categorical values.
|
||||
|
|
@ -639,8 +660,8 @@ class Categorical(MutableSymbol, Generic[Choice]):
|
|||
) -> None:
|
||||
values = list(values)
|
||||
assert values, 'Categorical values must not be empty.'
|
||||
self.label = auto_label(label)
|
||||
self.values = values
|
||||
self.label: str = auto_label(label)
|
||||
self.values: list[Choice] = values
|
||||
self.weights = weights if weights is not None else [1 / len(values)] * len(values)
|
||||
|
||||
if default is not MISSING:
|
||||
|
|
@ -678,6 +699,12 @@ class Categorical(MutableSymbol, Generic[Choice]):
|
|||
def __len__(self):
|
||||
return len(self.values)
|
||||
|
||||
def as_legacy_dict(self) -> dict:
|
||||
return {
|
||||
'_type': 'choice',
|
||||
'_value': self.values,
|
||||
}
|
||||
|
||||
def default(self, memo: Sample | None = None) -> Choice:
|
||||
"""The default() of :class:`Categorical` is the first value unless default value is set.
|
||||
|
||||
|
|
@ -1061,6 +1088,9 @@ class Numerical(MutableSymbol):
|
|||
self.quantize = quantize
|
||||
self.low = low
|
||||
self.high = high
|
||||
self.mu = mu
|
||||
self.sigma = sigma
|
||||
self.log_distributed = log_distributed
|
||||
|
||||
self.label = auto_label(label)
|
||||
|
||||
|
|
@ -1104,7 +1134,15 @@ class Numerical(MutableSymbol):
|
|||
self.label == other.label
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f'{self.low}, {self.high}, label={self.label!r}'
|
||||
rv = f'{self.low}, {self.high}, '
|
||||
if self.mu is not None and self.sigma is not None:
|
||||
rv += f'mu={self.mu}, sigma={self.sigma}, '
|
||||
if self.quantize is not None:
|
||||
rv += f'q={self.quantize}, '
|
||||
if self.log_distributed:
|
||||
rv += 'log_distributed=True, '
|
||||
rv += f'label={self.label!r}'
|
||||
return rv
|
||||
|
||||
def check_contains(self, sample: Sample) -> SampleValidationError | None:
|
||||
if self.label not in sample:
|
||||
|
|
@ -1118,8 +1156,12 @@ class Numerical(MutableSymbol):
|
|||
return SampleValidationError(f'{sample_val} is higher than upper bound {self.high}')
|
||||
if self.distribution.pdf(sample_val) == 0:
|
||||
return SampleValidationError(f'{sample_val} is not in the distribution {self.distribution}')
|
||||
if self.quantize is not None and abs(sample_val % self.quantize) > 1e-6:
|
||||
return SampleValidationError(f'{sample_val} is not a multiple of {self.quantize}')
|
||||
if self.quantize is not None and (
|
||||
abs(sample_val - self.low) > 1e-6 and
|
||||
abs(self.high - sample_val) > 1e-6 and
|
||||
abs(sample_val - round(sample_val / self.quantize) * self.quantize) > 1e-6
|
||||
):
|
||||
return SampleValidationError(f'{sample_val} is not on the boundary and not a multiple of {self.quantize}')
|
||||
return None
|
||||
|
||||
def qclip(self, x: float) -> float:
|
||||
|
|
@ -1199,6 +1241,7 @@ class Numerical(MutableSymbol):
|
|||
|
||||
if granularity is None:
|
||||
granularity = 1
|
||||
assert granularity > 0
|
||||
|
||||
err = self.check_contains(memo)
|
||||
if isinstance(err, SampleMissingError):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,184 @@
|
|||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
|
||||
"""High-level API for mutables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
__all__ = [
|
||||
'choice', 'uniform', 'quniform', 'loguniform', 'qloguniform',
|
||||
'normal', 'qnormal',
|
||||
]
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from .mutable import Categorical, Numerical
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from nni.nas.nn.pytorch import LayerChoice
|
||||
|
||||
T = TypeVar('T')
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def choice(label: str, choices: list[T]) -> Categorical[T] | LayerChoice:
|
||||
"""Choose from a list of options.
|
||||
|
||||
By default, it will create a :class:`~nni.mutable.Categorical` object.
|
||||
``choices`` should be a list of numbers or a list of strings.
|
||||
Using arbitrary objects as members of this list (like sublists, a mixture of numbers and strings, or null values)
|
||||
should work in most cases, but may trigger undefined behaviors.
|
||||
If PyTorch modules are presented in the choices, it will create a :class:`~nni.nas.nn.pytorch.LayerChoice`.
|
||||
|
||||
For most search algorithms, choice are non-ordinal.
|
||||
Even if the choices are numbers, they will still be treated as individual options,
|
||||
and their numeric values will be neglected.
|
||||
|
||||
Nested choices (i.e., choice inside one of the options) is not currently supported by this API.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.choice('x', [1, 2, 3])
|
||||
Categorical([1, 2, 3], label='x')
|
||||
>>> nni.choice('conv', [nn.Conv2d(3, 3, 3), nn.Conv2d(3, 3, 5)])
|
||||
LayerChoice(
|
||||
label='conv'
|
||||
(0): Conv2d(3, 3, kernel_size=(3, 3), stride=(1, 1))
|
||||
(1): Conv2d(3, 3, kernel_size=(5, 5), stride=(1, 1))
|
||||
)
|
||||
"""
|
||||
# Comment out before nas.nn is merged.
|
||||
# try:
|
||||
# from torch.nn import Module
|
||||
# if all(isinstance(c, Module) for c in choices):
|
||||
# from nni.nas.nn.pytorch import LayerChoice
|
||||
# return LayerChoice(choices, label=auto_label(label))
|
||||
|
||||
# from torch import Tensor
|
||||
# if any(isinstance(c, Tensor) for c in choices):
|
||||
# raise TypeError(
|
||||
# 'Please do not use choice to choose from tensors. '
|
||||
# 'If you are using this in forward, please use `InputChoice` explicitly in `__init__` instead.')
|
||||
# except ImportError:
|
||||
# # In case PyTorch is not installed.
|
||||
# pass
|
||||
|
||||
return Categorical(choices, label=label)
|
||||
|
||||
|
||||
def uniform(label: str, low: float, high: float) -> Numerical:
|
||||
"""Uniformly sampled between low and high.
|
||||
When optimizing, this variable is constrained to a two-sided interval.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.uniform('x', 0, 1)
|
||||
Numerical(0, 1, label='x')
|
||||
"""
|
||||
if low >= high:
|
||||
raise ValueError('low must be strictly smaller than high.')
|
||||
return Numerical(low, high, label=label)
|
||||
|
||||
|
||||
def quniform(label: str, low: float, high: float, quantize: float) -> Numerical:
|
||||
"""Sampling from ``uniform(low, high)`` but the final value is
|
||||
determined using ``clip(round(uniform(low, high) / q) * q, low, high)``,
|
||||
where the clip operation is used to constrain the generated value within the bounds.
|
||||
|
||||
For example, for low, high, quantize being specified as 0, 10, 2.5 respectively,
|
||||
possible values are [0, 2.5, 5.0, 7.5, 10.0].
|
||||
For 2, 10, 5, possible values are [2., 5., 10.].
|
||||
|
||||
Suitable for a discrete value with respect to which the objective is still somewhat “smooth”,
|
||||
but which should be bounded both above and below.
|
||||
Note that the return values will always be float.
|
||||
If you want to uniformly choose an **integer** from a range [low, high],
|
||||
you can use::
|
||||
|
||||
nni.quniform(low - 0.5, high + 0.5, 1).int()
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.quniform('x', 2.5, 5.5, 2.)
|
||||
Numerical(2.5, 5.5, q=2.0, label='x')
|
||||
"""
|
||||
|
||||
if isinstance(quantize, int):
|
||||
_logger.warning('Though quantize is an integer (%d) in quniform, the returned value will always be float. '
|
||||
'Use `.int()` to convert to integer.', quantize)
|
||||
if low >= high:
|
||||
raise ValueError('low must be strictly smaller than high.')
|
||||
|
||||
return Numerical(low, high, quantize=quantize, label=label)
|
||||
|
||||
|
||||
def loguniform(label: str, low: float, high: float) -> Numerical:
|
||||
"""Draw from a range [low, high] according to a loguniform distribution::
|
||||
|
||||
exp(uniform(log(low), log(high))),
|
||||
|
||||
so that the logarithm of the return value is uniformly distributed.
|
||||
|
||||
Since logarithm is taken here, low and high must be strictly greater than 0.
|
||||
|
||||
This is often used in variables which are log-distributed in experience,
|
||||
such as learning rate (which we often choose from 1e-1, 1e-3, 1e-6...).
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.loguniform('x', 1e-5, 1e-3)
|
||||
Numerical(1e-05, 0.001, log_distributed=True, label='x')
|
||||
>>> list(nni.loguniform('x', 1e-5, 1e-3).grid(granularity=2))
|
||||
[3.1622776601683795e-05, 0.0001, 0.00031622776601683794]
|
||||
"""
|
||||
if low >= high:
|
||||
raise ValueError('low must be strictly smaller than high.')
|
||||
if low <= 0 or high <= 0:
|
||||
raise ValueError('low and high must be strictly greater than 0.')
|
||||
return Numerical(low, high, log_distributed=True, label=label)
|
||||
|
||||
|
||||
def qloguniform(label: str, low: float, high: float, quantize: float) -> Numerical:
|
||||
"""A combination of :func:`quniform` and :func:`loguniform`.
|
||||
|
||||
Note that the quantize is done **after** the sample is drawn from the log-uniform distribution.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.qloguniform('x', 1e-5, 1e-3, 1e-4)
|
||||
Numerical(1e-05, 0.001, q=0.0001, log_distributed=True, label='x')
|
||||
"""
|
||||
return Numerical(low, high, log_distributed=True, quantize=quantize, label=label)
|
||||
|
||||
|
||||
def normal(label: str, mu: float, sigma: float) -> Numerical:
|
||||
"""Declare a normal distribution with mean ``mu`` and standard deviation ``sigma``.
|
||||
|
||||
The variable is unbounded, meaning that any real number from ``-inf`` to ``+inf`` can be possibly sampled.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.normal('x', 0, 1)
|
||||
Numerical(-inf, inf, mu=0, sigma=1, label='x')
|
||||
>>> nni.normal('x', 0, 1).random()
|
||||
-0.30621273862239057
|
||||
"""
|
||||
if sigma <= 0:
|
||||
raise ValueError('Standard deviation must be strictly greater than 0.')
|
||||
|
||||
return Numerical(mu=mu, sigma=sigma, label=label)
|
||||
|
||||
|
||||
def qnormal(label: str, mu: float, sigma: float, quantize: float) -> Numerical:
|
||||
"""Similar to :func:`quniform`, except the uniform distribution is replaced with a normal distribution.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> nni.qnormal('x', 0., 1., 0.1)
|
||||
Numerical(-inf, inf, mu=0.0, sigma=1.0, q=0.1, label='x')
|
||||
>>> nni.qnormal('x', 0., 1., 0.1).random()
|
||||
-0.1
|
||||
"""
|
||||
return Numerical(mu=mu, sigma=sigma, quantize=quantize, label=label)
|
||||
|
|
@ -30,6 +30,7 @@ from __future__ import annotations
|
|||
|
||||
__all__ = ['Symbol', 'SymbolicExpression']
|
||||
|
||||
import itertools
|
||||
import math
|
||||
import operator
|
||||
from typing import Any, Iterable, Type, NoReturn, Callable, Iterator, overload
|
||||
|
|
@ -228,6 +229,76 @@ class SymbolicExpression:
|
|||
return symbol_obj.expr_cls(lambda t, c, f: t if c else f, '{} if {} else {}', [true, pred, false])
|
||||
return true if pred else false
|
||||
|
||||
@symbolic_staticmethod
|
||||
def case(pred_expr_pairs: list[tuple[Any, Any]]) -> SymbolicExpression | Any: # type: ignore
|
||||
"""Return the first expression with predicate that is true.
|
||||
|
||||
For example::
|
||||
|
||||
if (x < y) return 17;
|
||||
else if (x > z) return 23;
|
||||
else (y > z) return 31;
|
||||
|
||||
Equivalent to::
|
||||
|
||||
SymbolicExpression.case([(x < y, 17), (x > z, 23), (y > z, 31)])
|
||||
"""
|
||||
|
||||
def _case_fn(*pred_expr_pairs):
|
||||
assert len(pred_expr_pairs) % 2 == 0
|
||||
for pred, expr in zip(pred_expr_pairs[::2], pred_expr_pairs[1::2]):
|
||||
if pred:
|
||||
return expr
|
||||
raise RuntimeError('No matching case')
|
||||
|
||||
chained_pairs = list(itertools.chain(*pred_expr_pairs))
|
||||
symbol_obj = first_symbolic_object(*chained_pairs)
|
||||
if symbol_obj is not None:
|
||||
return symbol_obj.expr_cls(
|
||||
_case_fn,
|
||||
'case([' + ', '.join(['({}, {})'] * len(pred_expr_pairs)) + '])',
|
||||
chained_pairs
|
||||
)
|
||||
return _case_fn(*chained_pairs)
|
||||
|
||||
@symbolic_staticmethod
|
||||
def switch_case(branch: Any, expressions: dict[Any, Any]) -> SymbolicExpression | Any:
|
||||
"""Select the expression that matches the branch.
|
||||
|
||||
C-style switch:
|
||||
|
||||
.. code-block:: cpp
|
||||
|
||||
switch (branch) { // c-style switch
|
||||
case 0: return 17;
|
||||
case 1: return 31;
|
||||
}
|
||||
|
||||
Equivalent to::
|
||||
|
||||
SymbolicExpression.switch_case(branch, {0: 17, 1: 31})
|
||||
"""
|
||||
|
||||
def _switch_fn(branch, *expressions):
|
||||
# TODO: support lazy evaluation.
|
||||
assert len(expressions) % 2 == 0
|
||||
keys = expressions[::2]
|
||||
values = expressions[1::2]
|
||||
for key, value in zip(keys, values):
|
||||
if key == branch:
|
||||
return value
|
||||
raise RuntimeError(f'No matching case for {branch}')
|
||||
|
||||
expanded_expression = list(itertools.chain(*expressions.items()))
|
||||
symbol_obj = first_symbolic_object(branch, *expanded_expression)
|
||||
if symbol_obj is not None:
|
||||
return symbol_obj.expr_cls(
|
||||
_switch_fn,
|
||||
'switch_case({}, {{' + ', '.join(['{}: {}'] * len(expressions)) + '}})',
|
||||
[branch, *expanded_expression]
|
||||
)
|
||||
return expressions[branch]
|
||||
|
||||
@symbolic_staticmethod
|
||||
def max(arg0: Iterable[Any] | Any, *args: Any) -> Any:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def reset_uid(namespace: str = 'default') -> None:
|
|||
_last_uid[namespace] = 0
|
||||
|
||||
|
||||
class NoContextError(Exception):
|
||||
class NoContextError(IndexError):
|
||||
"""Exception raised when context is missing."""
|
||||
pass
|
||||
|
||||
|
|
@ -163,9 +163,9 @@ class label_scope:
|
|||
# The full "path" of current scope.
|
||||
# It should also contain the part after the last ``/``.
|
||||
# No validation here, because it's not considered as public API.
|
||||
self.path = _path
|
||||
if self.path is not None:
|
||||
assert self.path, 'path should not be empty'
|
||||
self._path = _path
|
||||
if self._path is not None:
|
||||
assert self._path, 'path should not be empty'
|
||||
|
||||
if _path:
|
||||
self.basename = _path[-1]
|
||||
|
|
@ -178,7 +178,7 @@ class label_scope:
|
|||
# Its path should not change.
|
||||
# Otherwise, we compute the path based on its parent.
|
||||
|
||||
if self.path is None:
|
||||
if self._path is None:
|
||||
parent_scope = label_scope.current()
|
||||
if self.basename is None:
|
||||
if parent_scope is None:
|
||||
|
|
@ -194,9 +194,10 @@ class label_scope:
|
|||
self.basename = parent_scope.next_label()
|
||||
|
||||
if parent_scope is not None:
|
||||
self.path = parent_scope.path + [self.basename]
|
||||
assert parent_scope.path is not None, 'Parent scope is not entered.'
|
||||
self._path = parent_scope.path + [self.basename]
|
||||
else:
|
||||
self.path = [self.basename]
|
||||
self._path = [self.basename]
|
||||
|
||||
# Since path is sometimes already set (e.g., when re-enter),
|
||||
# parent_scope is not necessarily the real parent of current scope.
|
||||
|
|
@ -221,6 +222,10 @@ class label_scope:
|
|||
return False
|
||||
return self.path == other.path
|
||||
|
||||
@property
|
||||
def path(self) -> list[str] | None:
|
||||
return self._path
|
||||
|
||||
@property
|
||||
def absolute_scope(self) -> str:
|
||||
"""Alias of name."""
|
||||
|
|
|
|||
|
|
@ -240,15 +240,15 @@ stages:
|
|||
|
||||
- template: templates/install-nni.yml
|
||||
|
||||
- script: |
|
||||
CI=true npm --prefix ts/nni_manager run test --exclude test/core/nnimanager.test.ts
|
||||
displayName: TypeScript unit test
|
||||
|
||||
- script: |
|
||||
cd test
|
||||
python -m pytest ut
|
||||
displayName: Python unit test
|
||||
|
||||
- script: |
|
||||
CI=true npm --prefix ts/nni_manager run test --exclude test/core/nnimanager.test.ts
|
||||
displayName: TypeScript unit test
|
||||
|
||||
- script: |
|
||||
cd test
|
||||
python training_service/nnitest/run_tests.py --config training_service/config/pr_tests.yml
|
||||
|
|
|
|||
|
|
@ -50,6 +50,13 @@ def test_frozen_context_complex():
|
|||
|
||||
|
||||
def test_ensure_frozen(caplog):
|
||||
assert ensure_frozen(Categorical([1, 2, 3]), strict=False) == 1
|
||||
assert ensure_frozen(Categorical([1, 2, 3], label='a'), sample={'a': 2}, strict=False) == 2
|
||||
assert ensure_frozen('anything', strict=False) == 'anything'
|
||||
|
||||
with pytest.raises(RuntimeError, match='context'):
|
||||
ensure_frozen(Categorical([1, 2, 3], label='a'))
|
||||
|
||||
with frozen_context({'a': 1, 'b': 2}):
|
||||
assert ensure_frozen(Categorical([1, 2], label='a')) == 1
|
||||
assert ensure_frozen(Categorical([1, 2], label='b')) == 2
|
||||
|
|
@ -59,7 +66,7 @@ def test_ensure_frozen(caplog):
|
|||
assert 'add_mutable' in caplog.text
|
||||
|
||||
with frozen_context.bypass():
|
||||
assert ensure_frozen(Categorical([1, 2], label='a', default=2)) == 2
|
||||
assert ensure_frozen(Categorical([1, 2], label='a', default=2), strict=False) == 2
|
||||
with pytest.raises(RuntimeError, match='context'):
|
||||
ensure_frozen(Categorical([1, 2], label='a'), retries=-1)
|
||||
|
||||
|
|
|
|||
|
|
@ -239,6 +239,11 @@ def test_numerical():
|
|||
assert len(list(a.grid(granularity=10))) == 51
|
||||
assert a.random() % 2 == 0
|
||||
|
||||
a = Numerical(low=1, high=3, quantize=0.75)
|
||||
assert len(list(a.grid(granularity=10))) == 4
|
||||
for x in a.grid(granularity=10):
|
||||
assert a.contains({a.label: x})
|
||||
|
||||
a = Numerical(low=2, high=6, log_distributed=True, label='x')
|
||||
for _ in range(10):
|
||||
assert 2 < a.random() < 6
|
||||
|
|
@ -559,6 +564,9 @@ def test_grid():
|
|||
{'c': None, 'a': 3, 'b': 4}
|
||||
]
|
||||
|
||||
lst = MutableList([1, 2, 3])
|
||||
assert list(lst.grid()) == [[1, 2, 3]]
|
||||
|
||||
|
||||
def test_equals():
|
||||
assert _mutable_equal(Categorical([1, 2, 3], label='x'), Categorical([1, 2, 3], label='x'))
|
||||
|
|
@ -657,3 +665,6 @@ def test_equals():
|
|||
MutableDict({'a': Categorical([1, 2], label='a'), 'x': Categorical([3, 4], label='b')}),
|
||||
MutableDict({'a': Categorical([1, 2], label='a'), 'x': Categorical([3, 4], label='x')}),
|
||||
)
|
||||
|
||||
assert _mutable_equal(np.zeros_like((2, 2)), np.zeros_like((2, 2)))
|
||||
assert not _mutable_equal(np.zeros_like((2, 2)), np.ones_like((2, 2)))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
from collections import Counter
|
||||
|
||||
import nni
|
||||
from nni.mutable._notimplemented import randint, lognormal, qlognormal
|
||||
|
||||
def test_choice():
|
||||
t = nni.choice('t', ['a', 'b', 'c'])
|
||||
assert repr(t) == "Categorical(['a', 'b', 'c'], label='t')"
|
||||
|
||||
def test_randint():
|
||||
t = randint('x', 1, 5)
|
||||
assert repr(t) == "RandomInteger([1, 2, 3, 4], label='x')"
|
||||
|
||||
def test_uniform():
|
||||
t = nni.uniform('x', 0, 1)
|
||||
assert repr(t) == "Numerical(0, 1, label='x')"
|
||||
|
||||
def test_quniform():
|
||||
t = nni.quniform('x', 2.5, 5.5, 2.)
|
||||
assert repr(t) == "Numerical(2.5, 5.5, q=2.0, label='x')"
|
||||
t = nni.quniform('x', 0.5, 3.5, 1).int()
|
||||
counter = Counter()
|
||||
for _ in range(900):
|
||||
counter[t.random()] += 1
|
||||
for key, value in counter.items():
|
||||
assert 250 <= value <= 350
|
||||
assert isinstance(key, int)
|
||||
assert key in [1, 2, 3]
|
||||
|
||||
def test_loguniform():
|
||||
t = nni.loguniform('x', 1e-5, 1e-3)
|
||||
assert repr(t) == "Numerical(1e-05, 0.001, log_distributed=True, label='x')"
|
||||
for _ in range(100):
|
||||
assert 1e-5 < t.random() < 1e-3
|
||||
|
||||
def test_qloguniform():
|
||||
t = nni.qloguniform('x', 1e-5, 1e-3, 1e-4)
|
||||
assert repr(t) == "Numerical(1e-05, 0.001, q=0.0001, log_distributed=True, label='x')"
|
||||
for x in t.grid(granularity=8):
|
||||
assert (x == 1e-5 or abs(x - round(x / 1e-4) * 1e-4) < 1e-12) and 1e-5 <= x <= 1e-3
|
||||
|
||||
def test_normal():
|
||||
t = nni.normal('x', 0, 1)
|
||||
assert repr(t) == "Numerical(-inf, inf, mu=0, sigma=1, label='x')"
|
||||
assert -4 < t.random() < 4
|
||||
|
||||
def test_qnormal():
|
||||
t = nni.qnormal('x', 0., 1., 0.1)
|
||||
assert repr(t) == "Numerical(-inf, inf, mu=0.0, sigma=1.0, q=0.1, label='x')"
|
||||
|
||||
def test_lognormal():
|
||||
t = lognormal('x', 4., 2.)
|
||||
assert repr(t) == "Numerical(-inf, inf, mu=4.0, sigma=2.0, log_distributed=True, label='x')"
|
||||
assert 54 < list(t.grid(granularity=1))[0] < 55
|
||||
|
||||
def test_qlognormal():
|
||||
t = qlognormal('x', 4., 2., 1.)
|
||||
assert repr(t) == "Numerical(-inf, inf, mu=4.0, sigma=2.0, q=1.0, log_distributed=True, label='x')"
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
from nni.mutable.symbol import Symbol
|
||||
import pytest
|
||||
from nni.mutable.symbol import Symbol, SymbolicExpression
|
||||
|
||||
|
||||
def test_symbol_repr():
|
||||
|
|
@ -9,3 +10,24 @@ def test_symbol_repr():
|
|||
assert expr.evaluate({'x': 2, 'y': 3}) == 10
|
||||
expr = x * x
|
||||
assert repr(expr) == f"Symbol('x') * Symbol('x')"
|
||||
|
||||
|
||||
def test_switch_case():
|
||||
x, y = Symbol('x'), Symbol('y')
|
||||
expr = SymbolicExpression.switch_case(x, {0: y, 1: x * 2})
|
||||
assert str(expr) == 'switch_case(x, {0: y, 1: (x * 2)})'
|
||||
assert expr.evaluate({'x': 0, 'y': 3}) == 3
|
||||
assert expr.evaluate({'x': 1, 'y': 3}) == 2
|
||||
with pytest.raises(RuntimeError, match='No matching case'):
|
||||
expr.evaluate({'x': 2, 'y': 3})
|
||||
|
||||
|
||||
def test_case():
|
||||
x, y, z = Symbol('x'), Symbol('y'), Symbol('z')
|
||||
expr = SymbolicExpression.case([(x < y, 17), (x > z, 23), (y > z, 31)])
|
||||
assert str(expr) == 'case([((x < y), 17), ((x > z), 23), ((y > z), 31)])'
|
||||
assert expr.evaluate({'x': 1, 'y': 2, 'z': 3}) == 17
|
||||
assert expr.evaluate({'x': 2, 'y': 1, 'z': 0}) == 23
|
||||
assert expr.evaluate({'x': 1, 'y': 2, 'z': 0}) == 17
|
||||
with pytest.raises(RuntimeError, match='No matching case'):
|
||||
assert expr.evaluate({'x': 2, 'y': 1, 'z': 3})
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ def test_kill_process():
|
|||
assert end_time - start_time < 2
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.skip(reason='The test has too many failures.')
|
||||
def test_kill_process_slow_no_patience():
|
||||
process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
|
||||
time.sleep(1) # wait 1 second for the process to launch and register hooks
|
||||
|
|
@ -69,7 +69,7 @@ def test_kill_process_slow_no_patience():
|
|||
return
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.skip(reason='The test has too many failures.')
|
||||
def test_kill_process_slow_patiently():
|
||||
process = subprocess.Popen([sys.executable, __file__, '--mode', 'kill_slow'])
|
||||
time.sleep(1) # wait 1 second for the process to launch and register hooks
|
||||
|
|
@ -80,8 +80,7 @@ def test_kill_process_slow_patiently():
|
|||
# assert end_time - start_time > 1 # This check is disabled because it's not stable
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != 'linux', reason='Signal issues on non-linux.')
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.skip(reason='The test has too many failures.')
|
||||
def test_kill_process_interrupted():
|
||||
# Launch a subprocess that launches and kills another subprocess
|
||||
process = multiprocessing.Process(target=process_patiently_kill)
|
||||
|
|
|
|||
|
|
@ -217,7 +217,8 @@ nni.report_final_result(param['x'])
|
|||
// wait for it to request parameter
|
||||
await paramSent.get(trial).promise;
|
||||
// wait a while for it to report first intermediate result
|
||||
await setTimeout(100); // TODO: use an env var to distinguish pipeline so we can reduce the delay
|
||||
// might be longer on macOS
|
||||
await setTimeout(process.platform == 'darwin' ? 1000 : 100); // TODO: use an env var to distinguish pipeline so we can reduce the delay
|
||||
await ts.stopTrial(trial);
|
||||
|
||||
// the callbacks should be invoked
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче