зеркало из https://github.com/microsoft/archai.git
Remove DatasetProvider from ModelEvaluator api's. (#184)
* Remove DatasetProvider from ModelEvaluator api's. Co-authored-by: Gustavo Rosa <gth.rosa@uol.com.br> Co-authored-by: piero2c <pierokauffmann@gmail.com>
This commit is contained in:
Chris Lovett
GitHub
Родитель
6eef2a17cd
Коммит
324397f586
|
|
@ -0,0 +1,4 @@
|
|||
[flake8]
|
||||
ignore = E111,E402,E722,W503,W504,F405,F403
|
||||
max-line-length = 127
|
||||
max-complexity = 10
|
||||
|
|
@ -36,7 +36,7 @@ _atexit_reg = False # is hook for atexit registered?
|
|||
def get_conf(conf:Optional[Config]=None)->Config:
|
||||
if conf is not None:
|
||||
return conf
|
||||
return Config.get_global_instance()
|
||||
return Config.get_inst()
|
||||
|
||||
def get_conf_common(conf:Optional[Config]=None)->Config:
|
||||
return get_conf(conf)['common']
|
||||
|
|
@ -103,7 +103,7 @@ def get_state()->CommonState:
|
|||
def init_from(state:CommonState)->None:
|
||||
global _tb_writer
|
||||
|
||||
Config.set_global_instance(state.conf)
|
||||
Config.set_inst(state.conf)
|
||||
|
||||
_tb_writer = state.tb_writer
|
||||
|
||||
|
|
@ -140,7 +140,7 @@ def common_init(config_filepath: Optional[str]=None,
|
|||
|
||||
# setup global instance
|
||||
conf = create_conf(config_filepath, param_args, use_args)
|
||||
Config.set_global_instance(conf)
|
||||
Config.set_inst(conf)
|
||||
|
||||
# setup env vars which might be used in paths
|
||||
update_envvars(conf)
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@ class MoBananasSearch(Searcher):
|
|||
self,
|
||||
search_space: BayesOptSearchSpace,
|
||||
search_objectives: SearchObjectives,
|
||||
dataset_provider: DatasetProvider,
|
||||
output_dir: str,
|
||||
surrogate_model: Optional[Predictor] = None,
|
||||
num_iters: Optional[int] = 10,
|
||||
|
|
@ -58,7 +57,6 @@ class MoBananasSearch(Searcher):
|
|||
search_objectives: Search objectives. Expensive objectives (registered with
|
||||
`compute_intensive=True`) will be estimated using a surrogate model during
|
||||
certain parts of the search. Cheap objectives will be always evaluated directly.
|
||||
dataset_provider: Dataset provider.
|
||||
output_dir: Output directory.
|
||||
surrogate_model: Surrogate model. If `None`, a `PredictiveDNNEnsemble` will be used.
|
||||
num_iters: Number of iterations.
|
||||
|
|
@ -85,7 +83,6 @@ class MoBananasSearch(Searcher):
|
|||
self.output_dir.mkdir(exist_ok=True)
|
||||
|
||||
self.search_space = search_space
|
||||
self.dataset_provider = dataset_provider
|
||||
self.surrogate_model = surrogate_model
|
||||
|
||||
# Objectives
|
||||
|
|
@ -122,7 +119,7 @@ class MoBananasSearch(Searcher):
|
|||
"""
|
||||
|
||||
encoded_archs = np.vstack([self.search_space.encode(m) for m in all_pop])
|
||||
target = np.array([self.search_state.all_evaluated_objs[obj] for obj in self.so.exp_objs]).T
|
||||
target = np.array([self.search_state.all_evaluated_objs[obj] for obj in self.so.expensive_objectives]).T
|
||||
|
||||
return encoded_archs, target
|
||||
|
||||
|
|
@ -143,7 +140,7 @@ class MoBananasSearch(Searcher):
|
|||
while len(valid_sample) < num_models and nb_tries < patience:
|
||||
sample = [self.search_space.random_sample() for _ in range(num_models)]
|
||||
|
||||
_, valid_indices = self.so.validate_constraints(sample, self.dataset_provider)
|
||||
_, valid_indices = self.so.validate_constraints(sample)
|
||||
valid_sample += [sample[i] for i in valid_indices]
|
||||
|
||||
return valid_sample[:num_models]
|
||||
|
|
@ -173,7 +170,7 @@ class MoBananasSearch(Searcher):
|
|||
mutated_model = self.search_space.mutate(p)
|
||||
mutated_model.metadata["parent"] = p.archid
|
||||
|
||||
if not self.so.is_model_valid(mutated_model, self.dataset_provider):
|
||||
if not self.so.is_model_valid(mutated_model):
|
||||
continue
|
||||
|
||||
if mutated_model.archid not in self.seen_archs:
|
||||
|
|
@ -204,7 +201,7 @@ class MoBananasSearch(Searcher):
|
|||
|
||||
return {
|
||||
obj_name: MeanVar(pred_results.mean[:, i], pred_results.var[:, i])
|
||||
for i, obj_name in enumerate(self.so.exp_objs)
|
||||
for i, obj_name in enumerate(self.so.expensive_objectives)
|
||||
}
|
||||
|
||||
def thompson_sampling(
|
||||
|
|
@ -257,7 +254,7 @@ class MoBananasSearch(Searcher):
|
|||
all_pop.extend(unseen_pop)
|
||||
|
||||
logger.info(f"Evaluating objectives for {len(unseen_pop)} architectures ...")
|
||||
iter_results = self.so.eval_all_objs(unseen_pop, self.dataset_provider)
|
||||
iter_results = self.so.eval_all_objs(unseen_pop)
|
||||
|
||||
self.seen_archs.update([m.archid for m in unseen_pop])
|
||||
|
||||
|
|
@ -295,11 +292,11 @@ class MoBananasSearch(Searcher):
|
|||
|
||||
# Predicts expensive objectives using surrogate model
|
||||
# and calculates cheap objectives for mutated architectures
|
||||
logger.info(f"Predicting {list(self.so.exp_objs.keys())} for new architectures using surrogate model ...")
|
||||
logger.info(f"Predicting {self.so.expensive_objective_names} for new architectures using surrogate model ...")
|
||||
pred_expensive_objs = self.predict_expensive_objectives(mutated)
|
||||
|
||||
logger.info(f"Calculating cheap objectives {list(self.so.cheap_objs.keys())} for new architectures ...")
|
||||
cheap_objs = self.so.eval_cheap_objs(mutated, self.dataset_provider)
|
||||
logger.info(f"Calculating cheap objectives {self.so.cheap_objective_names} for new architectures ...")
|
||||
cheap_objs = self.so.eval_cheap_objs(mutated)
|
||||
|
||||
# Selects `num_candidates`-archtiectures for next iteration using Thompson Sampling
|
||||
selected_indices = self.thompson_sampling(mutated, self.num_candidates, pred_expensive_objs, cheap_objs)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from typing import List, Optional
|
|||
from overrides import overrides
|
||||
from tqdm import tqdm
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.common.ordered_dict_logger import OrderedDictLogger
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.search_objectives import SearchObjectives
|
||||
|
|
@ -34,7 +33,6 @@ class EvolutionParetoSearch(Searcher):
|
|||
self,
|
||||
search_space: EvolutionarySearchSpace,
|
||||
search_objectives: SearchObjectives,
|
||||
dataset_provider: DatasetProvider,
|
||||
output_dir: str,
|
||||
num_iters: Optional[int] = 10,
|
||||
init_num_models: Optional[int] = 10,
|
||||
|
|
@ -52,7 +50,6 @@ class EvolutionParetoSearch(Searcher):
|
|||
Args:
|
||||
search_space: Discrete search space compatible with evolutionary algorithms.
|
||||
search_objectives: Search objectives.
|
||||
dataset_provider: Dataset provider.
|
||||
output_dir: Output directory.
|
||||
num_iters: Number of iterations.
|
||||
init_num_models: Number of initial models to evaluate.
|
||||
|
|
@ -76,7 +73,6 @@ class EvolutionParetoSearch(Searcher):
|
|||
self.iter_num = 0
|
||||
self.search_space = search_space
|
||||
self.so = search_objectives
|
||||
self.dataset_provider = dataset_provider
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
|
|
@ -120,7 +116,7 @@ class EvolutionParetoSearch(Searcher):
|
|||
while len(valid_sample) < num_models and nb_tries < patience:
|
||||
sample = [self.search_space.random_sample() for _ in range(num_models)]
|
||||
|
||||
_, valid_indices = self.so.validate_constraints(sample, self.dataset_provider)
|
||||
_, valid_indices = self.so.validate_constraints(sample)
|
||||
valid_sample += [sample[i] for i in valid_indices]
|
||||
|
||||
return valid_sample[:num_models]
|
||||
|
|
@ -150,7 +146,7 @@ class EvolutionParetoSearch(Searcher):
|
|||
mutated_model = self.search_space.mutate(p)
|
||||
mutated_model.metadata["parent"] = p.archid
|
||||
|
||||
if not self.so.is_model_valid(mutated_model, self.dataset_provider):
|
||||
if not self.so.is_model_valid(mutated_model):
|
||||
continue
|
||||
|
||||
if mutated_model.archid not in self.seen_archs:
|
||||
|
|
@ -185,11 +181,11 @@ class EvolutionParetoSearch(Searcher):
|
|||
child = self.search_space.crossover([p1, p2])
|
||||
nb_tries = 0
|
||||
|
||||
while not self.so.is_model_valid(child, self.dataset_provider) and nb_tries < patience:
|
||||
while not self.so.is_model_valid(child) and nb_tries < patience:
|
||||
child = self.search_space.crossover([p1, p2])
|
||||
nb_tries += 1
|
||||
|
||||
if child and self.so.is_model_valid(child, self.dataset_provider):
|
||||
if child and self.so.is_model_valid(child):
|
||||
if child.archid not in children_ids and child.archid not in self.seen_archs:
|
||||
child.metadata["generation"] = self.iter_num
|
||||
child.metadata["parents"] = f"{p1.archid},{p2.archid}"
|
||||
|
|
@ -242,9 +238,9 @@ class EvolutionParetoSearch(Searcher):
|
|||
self.on_search_iteration_start(unseen_pop)
|
||||
|
||||
# Calculates objectives
|
||||
logger.info(f"Calculating search objectives {list(self.so.objs.keys())} for {len(unseen_pop)} models ...")
|
||||
logger.info(f"Calculating search objectives {list(self.so.objective_names)} for {len(unseen_pop)} models ...")
|
||||
|
||||
results = self.so.eval_all_objs(unseen_pop, self.dataset_provider)
|
||||
results = self.so.eval_all_objs(unseen_pop)
|
||||
self.search_state.add_iteration_results(
|
||||
unseen_pop,
|
||||
results,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from typing import List, Optional
|
|||
from overrides import overrides
|
||||
from tqdm import tqdm
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.common.ordered_dict_logger import OrderedDictLogger
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.search_objectives import SearchObjectives
|
||||
|
|
@ -24,7 +23,6 @@ class LocalSearch(Searcher):
|
|||
self,
|
||||
search_space: EvolutionarySearchSpace,
|
||||
search_objectives: SearchObjectives,
|
||||
dataset_provider: DatasetProvider,
|
||||
output_dir: str,
|
||||
num_iters: Optional[int] = 10,
|
||||
init_num_models: Optional[int] = 10,
|
||||
|
|
@ -40,7 +38,6 @@ class LocalSearch(Searcher):
|
|||
Args:
|
||||
search_space (EvolutionarySearchSpace): Discrete search space compatible with evolutionary algorithms
|
||||
search_objectives (SearchObjectives): Search objectives
|
||||
dataset_provider (DatasetProvider): Dataset provider used to evaluate models
|
||||
output_dir (str): Output directory
|
||||
num_iters (int, optional): Number of search iterations. Defaults to 10.
|
||||
init_num_models (int, optional): Number of initial models. Defaults to 10.
|
||||
|
|
@ -59,7 +56,6 @@ class LocalSearch(Searcher):
|
|||
self.iter_num = 0
|
||||
self.search_space = search_space
|
||||
self.so = search_objectives
|
||||
self.dataset_provider = dataset_provider
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
|
|
@ -98,7 +94,7 @@ class LocalSearch(Searcher):
|
|||
while len(valid_sample) < num_models and nb_tries < patience:
|
||||
sample = [self.search_space.random_sample() for _ in range(num_models)]
|
||||
|
||||
_, valid_indices = self.so.validate_constraints(sample, self.dataset_provider)
|
||||
_, valid_indices = self.so.validate_constraints(sample)
|
||||
valid_sample += [sample[i] for i in valid_indices]
|
||||
|
||||
return valid_sample[:num_models]
|
||||
|
|
@ -128,7 +124,7 @@ class LocalSearch(Searcher):
|
|||
mutated_model = self.search_space.mutate(p)
|
||||
mutated_model.metadata["parent"] = p.archid
|
||||
|
||||
if not self.so.is_model_valid(mutated_model, self.dataset_provider):
|
||||
if not self.so.is_model_valid(mutated_model):
|
||||
continue
|
||||
|
||||
if mutated_model.archid not in self.seen_archs:
|
||||
|
|
@ -161,9 +157,9 @@ class LocalSearch(Searcher):
|
|||
break
|
||||
|
||||
# Calculates objectives
|
||||
logger.info(f"Calculating search objectives {list(self.so.objs.keys())} for {len(unseen_pop)} models ...")
|
||||
logger.info(f"Calculating search objectives {list(self.so.objective_names)} for {len(unseen_pop)} models ...")
|
||||
|
||||
results = self.so.eval_all_objs(unseen_pop, self.dataset_provider)
|
||||
results = self.so.eval_all_objs(unseen_pop)
|
||||
self.search_state.add_iteration_results(
|
||||
unseen_pop,
|
||||
results,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import List, Optional
|
|||
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.common.ordered_dict_logger import OrderedDictLogger
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.search_objectives import SearchObjectives
|
||||
|
|
@ -30,7 +29,6 @@ class RandomSearch(Searcher):
|
|||
self,
|
||||
search_space: DiscreteSearchSpace,
|
||||
search_objectives: SearchObjectives,
|
||||
dataset_provider: DatasetProvider,
|
||||
output_dir: str,
|
||||
num_iters: Optional[int] = 10,
|
||||
samples_per_iter: Optional[int] = 10,
|
||||
|
|
@ -43,7 +41,6 @@ class RandomSearch(Searcher):
|
|||
Args:
|
||||
search_space: Discrete search space.
|
||||
search_objectives: Search objectives.
|
||||
dataset_provider: Dataset provider.
|
||||
output_dir: Output directory.
|
||||
num_iters: Number of iterations.
|
||||
samples_per_iter: Number of samples per iteration.
|
||||
|
|
@ -60,7 +57,6 @@ class RandomSearch(Searcher):
|
|||
self.iter_num = 0
|
||||
self.search_space = search_space
|
||||
self.so = search_objectives
|
||||
self.dataset_provider = dataset_provider
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
|
|
@ -97,7 +93,7 @@ class RandomSearch(Searcher):
|
|||
while len(valid_sample) < num_models and nb_tries < patience:
|
||||
sample = [self.search_space.random_sample() for _ in range(num_models)]
|
||||
|
||||
_, valid_indices = self.so.validate_constraints(sample, self.dataset_provider)
|
||||
_, valid_indices = self.so.validate_constraints(sample)
|
||||
valid_sample += [sample[i] for i in valid_indices if sample[i].archid not in self.seen_archs]
|
||||
|
||||
return valid_sample[:num_models]
|
||||
|
|
@ -112,9 +108,9 @@ class RandomSearch(Searcher):
|
|||
unseen_pop = self.sample_models(self.samples_per_iter)
|
||||
|
||||
# Calculates objectives
|
||||
logger.info(f"Calculating search objectives {list(self.so.objs.keys())} for {len(unseen_pop)} models ...")
|
||||
logger.info(f"Calculating search objectives {list(self.so.objective_names)} for {len(unseen_pop)} models ...")
|
||||
|
||||
results = self.so.eval_all_objs(unseen_pop, self.dataset_provider)
|
||||
results = self.so.eval_all_objs(unseen_pop)
|
||||
self.search_state.add_iteration_results(unseen_pop, results)
|
||||
|
||||
# Records evaluated archs to avoid computing the same architecture twice
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from typing import List, Optional
|
|||
from overrides import overrides
|
||||
from tqdm import tqdm
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.common.ordered_dict_logger import OrderedDictLogger
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.search_objectives import SearchObjectives
|
||||
|
|
@ -34,7 +33,6 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
self,
|
||||
search_space: EvolutionarySearchSpace,
|
||||
search_objectives: SearchObjectives,
|
||||
dataset_provider: DatasetProvider,
|
||||
output_dir: str,
|
||||
num_iters: Optional[int] = 10,
|
||||
init_num_models: Optional[int] = 10,
|
||||
|
|
@ -50,7 +48,6 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
Args:
|
||||
search_space: Discrete search space compatible with evolutionary algorithms.
|
||||
search_objectives: Search objectives.
|
||||
dataset_provider: Dataset provider.
|
||||
output_dir: Output directory.
|
||||
num_iters: Number of iterations.
|
||||
init_num_models: Number of initial models to evaluate.
|
||||
|
|
@ -71,7 +68,6 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
self.iter_num = 0
|
||||
self.search_space = search_space
|
||||
self.so = search_objectives
|
||||
self.dataset_provider = dataset_provider
|
||||
self.output_dir = Path(output_dir)
|
||||
self.output_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
|
|
@ -111,7 +107,7 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
while len(valid_sample) < num_models and nb_tries < patience:
|
||||
sample = [self.search_space.random_sample() for _ in range(num_models)]
|
||||
|
||||
_, valid_indices = self.so.validate_constraints(sample, self.dataset_provider)
|
||||
_, valid_indices = self.so.validate_constraints(sample)
|
||||
valid_sample += [sample[i] for i in valid_indices]
|
||||
|
||||
return valid_sample[:num_models]
|
||||
|
|
@ -141,7 +137,7 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
mutated_model = self.search_space.mutate(p)
|
||||
mutated_model.metadata["parent"] = p.archid
|
||||
|
||||
if not self.so.is_model_valid(mutated_model, self.dataset_provider):
|
||||
if not self.so.is_model_valid(mutated_model):
|
||||
continue
|
||||
|
||||
if mutated_model.archid not in self.seen_archs:
|
||||
|
|
@ -174,9 +170,9 @@ class RegularizedEvolutionSearch(Searcher):
|
|||
break
|
||||
|
||||
# Calculates objectives
|
||||
logger.info(f"Calculating search objectives {list(self.so.objs.keys())} for {len(iter_members)} models ...")
|
||||
logger.info(f"Calculating search objectives {list(self.so.objective_names)} for {len(iter_members)} models ...")
|
||||
|
||||
results = self.so.eval_all_objs(iter_members, self.dataset_provider)
|
||||
results = self.so.eval_all_objs(iter_members)
|
||||
|
||||
self.search_state.add_iteration_results(
|
||||
iter_members,
|
||||
|
|
|
|||
|
|
@ -89,8 +89,7 @@ class SuccessiveHalvingSearch(Searcher):
|
|||
|
||||
results = self.objectives.eval_all_objs(
|
||||
selected_models,
|
||||
self.dataset_provider,
|
||||
budgets={obj_name: current_budget for obj_name in self.objectives.objs},
|
||||
budgets={obj_name: current_budget for obj_name in self.objectives.objectives},
|
||||
)
|
||||
|
||||
# Logs results and saves iteration models
|
||||
|
|
|
|||
|
|
@ -16,21 +16,26 @@ class ModelEvaluator(EnforceOverrides):
|
|||
Evaluators are general-use classes used to evaluate architectures in
|
||||
given criteria (task performance, speed, size, etc.).
|
||||
|
||||
Subclasses of `ModelEvaluator` are expected to implement `ModelEvaluator.evaluate`.
|
||||
Subclasses of `ModelEvaluator` are expected to implement `ModelEvaluator.evaluate`
|
||||
|
||||
Synchronous evaluators are computed by search algorithms sequentially.
|
||||
For parallel / async. execution, please refer to `archai.api.AsyncModelEvaluator`.
|
||||
|
||||
For a list of bult-in evaluators, please check `archai.discrete_search.evaluators`.
|
||||
For a list of built-in evaluators, please check `archai.discrete_search.evaluators`.
|
||||
|
||||
Examples:
|
||||
>>> class MyValTaskAccuracy(ModelEvaluator):
|
||||
>>> def __init__(self, batch_size: int = 32):
|
||||
>>> def __init__(self, dataset: DatasetProvider, batch_size: int = 32):
|
||||
>>> self.dataset = dataset
|
||||
>>> self.batch_size = batch_size
|
||||
>>>
|
||||
>>> @overrides
|
||||
>>> def evaluate(self, model: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None):
|
||||
>>> _, val_data = dataset.get_train_val_datasets()
|
||||
>>> def get_name(self) -> str:
|
||||
>>> return f'MyValTaskAccuracy_on_{self.dataset.get_name()}}'
|
||||
>>>
|
||||
>>> @overrides
|
||||
>>> def evaluate(self, model: ArchaiModel, budget: Optional[float] = None):
|
||||
>>> _, val_data = self.dataset.get_train_val_datasets()
|
||||
>>> val_dl = torch.utils.data.DataLoader(val_data, batch_size=self.batch_size)
|
||||
>>>
|
||||
>>> with torch.no_grad():
|
||||
|
|
@ -44,14 +49,14 @@ class ModelEvaluator(EnforceOverrides):
|
|||
>>>
|
||||
>>> class NumberOfModules(ModelEvaluator):
|
||||
>>> @overrides
|
||||
>>> def evaluate(self, model: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None):
|
||||
>>> def evaluate(self, model: ArchaiModel, budget: Optional[float] = None):
|
||||
>>> return len(list(model.arch.modules()))
|
||||
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def evaluate(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
"""Evaluate an `ArchaiModel` instance, optionally using a `DatasetProvider` and budget value.
|
||||
def evaluate(self, arch: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
"""Evaluate an `ArchaiModel` instance, optionally using a budget value.
|
||||
|
||||
Args:
|
||||
arch: Model to be evaluated.
|
||||
|
|
@ -79,35 +84,35 @@ class AsyncModelEvaluator(EnforceOverrides):
|
|||
fashion, by sending evaluation jobs to a queue and fetching the results later.
|
||||
|
||||
Subclasses of `AsyncModelEvaluator` are expected to implement
|
||||
`AsyncModelEvaluator.send(arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float])`
|
||||
`AsyncModelEvaluator.send(arch: ArchaiModel, budget: Optional[float])`
|
||||
and `AsyncModelEvaluator.fetch_all()`.
|
||||
|
||||
`AsyncModelEvaluator.send` is a non-blocking call that schedules an evaluation job for a
|
||||
given (model, dataset, budget) triplet. `AsyncModelEvaluator.fetch_all` is a blocking call
|
||||
given (model, budget) triplet. `AsyncModelEvaluator.fetch_all` is a blocking call
|
||||
that waits and gathers the results from current evaluation jobs and cleans the job queue.
|
||||
|
||||
For a list of bult-in evaluators, please check `archai.discrete_search.evaluators`.
|
||||
For a list of built-in evaluators, please check `archai.discrete_search.evaluators`.
|
||||
|
||||
>>> my_obj = MyAsyncObj() # My AsyncModelEvaluator subclass
|
||||
>>> my_obj = MyAsyncObj(dataset) # My AsyncModelEvaluator subclass
|
||||
>>>
|
||||
>>> # Non blocking calls
|
||||
>>> my_obj.send(model_1, dataset_provider, budget=None)
|
||||
>>> my_obj.send(model_2, dataset_provider, budget=None)
|
||||
>>> my_obj.send(model_3, dataset_provider, budget=None)
|
||||
>>> my_obj.send(model_1, budget=None)
|
||||
>>> my_obj.send(model_2, budget=None)
|
||||
>>> my_obj.send(model_3, budget=None)
|
||||
>>>
|
||||
>>> # Blocking call
|
||||
>>> eval_results = my_obj.fetch_all()
|
||||
>>> assert len(eval_resuls) == 3
|
||||
>>> assert len(eval_results) == 3
|
||||
>>>
|
||||
>>> # Job queue is reset after `fetch_call` method
|
||||
>>> my_obj.send(model_4, dataset_provider, budget=None)
|
||||
>>> my_obj.send(model_4, budget=None)
|
||||
>>> assert len(my_obj.fetch_all()) == 1
|
||||
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def send(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
"""Send an evaluation job for a given (model, dataset, budget) triplet.
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
"""Send an evaluation job for a given (model, budget) triplet.
|
||||
|
||||
Args:
|
||||
arch: Model to be evaluated.
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import numpy as np
|
|||
import yaml
|
||||
from tqdm import tqdm
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import (
|
||||
AsyncModelEvaluator,
|
||||
|
|
@ -15,6 +14,22 @@ from archai.discrete_search.api.model_evaluator import (
|
|||
)
|
||||
|
||||
|
||||
class SearchConstraint:
|
||||
def __init__(self, name, evaluator, constraint):
|
||||
self.name = name
|
||||
self.evaluator = evaluator
|
||||
self.constraint = constraint
|
||||
|
||||
|
||||
class SearchObjective:
|
||||
def __init__(self, name, model_evaluator, higher_is_better, compute_intensive, constraint):
|
||||
self.name = name
|
||||
self.evaluator = model_evaluator
|
||||
self.higher_is_better = higher_is_better
|
||||
self.compute_intensive = compute_intensive
|
||||
self.constraint = constraint
|
||||
|
||||
|
||||
class SearchObjectives:
|
||||
"""Search objectives and constraints."""
|
||||
|
||||
|
|
@ -29,30 +44,53 @@ class SearchObjectives:
|
|||
|
||||
Args:
|
||||
cache_objective_evaluation: If `True`, objective evaluations are cached using the
|
||||
tuple `(obj_name, archid, dataset_provider_name, budget)` as key.
|
||||
tuple `(obj_name, archid, budget)` as key.
|
||||
|
||||
"""
|
||||
|
||||
self.cache_objective_evaluation = cache_objective_evaluation
|
||||
self._cache_objective_evaluation = cache_objective_evaluation
|
||||
|
||||
self.cheap_objs = {}
|
||||
self.exp_objs = {}
|
||||
self.extra_constraints = {}
|
||||
self._objs = {}
|
||||
self._extra_constraints = {}
|
||||
|
||||
# Cache key: (obj_name, archid, dataset provider name, budget)
|
||||
self.cache: Dict[Tuple[str, str, str, Optional[float]], Optional[float]] = {}
|
||||
# Cache key: (obj_name, archid, budget)
|
||||
self._cache: Dict[Tuple[str, str, Optional[float]], Optional[float]] = {}
|
||||
|
||||
@property
|
||||
def objs(self) -> Dict[str, Dict]:
|
||||
def objective_names(self) -> List[str]:
|
||||
"""Return a list of all objective names."""
|
||||
return list(self._objs.keys())
|
||||
|
||||
@property
|
||||
def cheap_objective_names(self) -> List[str]:
|
||||
"""Return a list of cheap objective names."""
|
||||
return list(self.cheap_objectives.keys())
|
||||
|
||||
@property
|
||||
def expensive_objective_names(self) -> List[str]:
|
||||
"""Return a list of expensive objective names."""
|
||||
return list(self.expensive_objectives.keys())
|
||||
|
||||
@property
|
||||
def objectives(self) -> Dict[str, SearchObjective]:
|
||||
"""Return a dictionary of all objectives."""
|
||||
|
||||
return dict(self.cheap_objs, **self.exp_objs)
|
||||
return self._objs
|
||||
|
||||
@property
|
||||
def objs_and_constraints(self) -> Dict[str, Dict]:
|
||||
"""Return a dictionary of all objectives and constraints."""
|
||||
def cheap_objectives(self) -> Dict[str, SearchObjective]:
|
||||
"""Return a dictionary of cheap objectives."""
|
||||
return self._filter_objs(self._objs, lambda x: not x.compute_intensive)
|
||||
|
||||
return dict(self.objs, **self.extra_constraints)
|
||||
@property
|
||||
def expensive_objectives(self) -> Dict[str, SearchObjective]:
|
||||
"""Return a dictionary of expensive objectives."""
|
||||
return self._filter_objs(self._objs, lambda x: x.compute_intensive)
|
||||
|
||||
@property
|
||||
def constraints(self) -> Dict[str, SearchConstraint]:
|
||||
"""Return a dictionary of all the constraints."""
|
||||
|
||||
return self._extra_constraints
|
||||
|
||||
def add_objective(
|
||||
self,
|
||||
|
|
@ -77,17 +115,15 @@ class SearchObjectives:
|
|||
"""
|
||||
|
||||
assert isinstance(model_evaluator, (ModelEvaluator, AsyncModelEvaluator))
|
||||
assert name not in dict(
|
||||
self.objs, **self.extra_constraints
|
||||
), f"There is already an objective or constraint named {name}."
|
||||
assert name not in self._objs, f"There is already an objective {name}."
|
||||
assert name not in self._extra_constraints, f"There is already an constraint named {name}."
|
||||
|
||||
obj = {"evaluator": model_evaluator, "higher_is_better": higher_is_better, "constraint": constraint}
|
||||
obj = SearchObjective(name, model_evaluator, higher_is_better, compute_intensive, constraint)
|
||||
|
||||
if compute_intensive:
|
||||
assert constraint is None, "Constraints can only be set for cheap objectives (compute_intensive=False)."
|
||||
self.exp_objs[name] = obj
|
||||
else:
|
||||
self.cheap_objs[name] = obj
|
||||
|
||||
self._objs[name] = obj
|
||||
|
||||
def add_constraint(
|
||||
self, name: str, model_evaluator: Union[ModelEvaluator, AsyncModelEvaluator], constraint: Tuple[float, float]
|
||||
|
|
@ -106,23 +142,18 @@ class SearchObjectives:
|
|||
"""
|
||||
|
||||
assert isinstance(model_evaluator, (ModelEvaluator, AsyncModelEvaluator))
|
||||
assert name not in dict(
|
||||
self.extra_constraints, **self.objs
|
||||
), f"There is already an objective or constraint named {name}."
|
||||
assert name not in self._objs, f"There is already an objective {name}."
|
||||
assert name not in self._extra_constraints, f"There is already an constraint named {name}."
|
||||
|
||||
self.extra_constraints[name] = {
|
||||
"evaluator": model_evaluator,
|
||||
"constraint": constraint,
|
||||
}
|
||||
self._extra_constraints[name] = SearchConstraint(name, model_evaluator, constraint)
|
||||
|
||||
def _filter_objs(self, objs: Dict[str, Dict], field_name: str, query_fn: Callable) -> Dict[str, Dict]:
|
||||
return {obj_name: obj_dict for obj_name, obj_dict in objs.items() if query_fn(obj_dict[field_name])}
|
||||
def _filter_objs(self, objs: Dict[str, Dict], query_fn: Callable) -> Dict[str, Dict]:
|
||||
return {obj_name: obj_dict for obj_name, obj_dict in objs.items() if query_fn(obj_dict)}
|
||||
|
||||
def _eval_objs(
|
||||
self,
|
||||
objs: Dict[str, Dict],
|
||||
models: List[ArchaiModel],
|
||||
dataset_providers: Union[DatasetProvider, List[DatasetProvider]],
|
||||
budgets: Optional[Dict[str, List[Any]]] = None,
|
||||
progress_bar: Optional[bool] = False,
|
||||
) -> Dict[str, np.ndarray]:
|
||||
|
|
@ -133,21 +164,15 @@ class SearchObjectives:
|
|||
budgets = budgets or {}
|
||||
budgets = {obj_name: budgets.get(obj_name, [None] * len(models)) for obj_name in objs}
|
||||
|
||||
# Casts dataset_providers to a list if necessary
|
||||
if not isinstance(dataset_providers, list):
|
||||
dataset_providers = [dataset_providers] * len(models)
|
||||
|
||||
# Splits `objs` in sync and async
|
||||
sync_objs = self._filter_objs(objs, "evaluator", lambda x: isinstance(x, ModelEvaluator))
|
||||
async_objs = self._filter_objs(objs, "evaluator", lambda x: isinstance(x, AsyncModelEvaluator))
|
||||
|
||||
assert all(len(dataset_providers) == len(models) == len(b) for b in budgets.values())
|
||||
sync_objs = self._filter_objs(objs, lambda x: isinstance(x.evaluator, ModelEvaluator))
|
||||
async_objs = self._filter_objs(objs, lambda x: isinstance(x.evaluator, AsyncModelEvaluator))
|
||||
|
||||
# Initializes evaluation results with cached results
|
||||
eval_results = {
|
||||
obj_name: [
|
||||
self.cache.get((obj_name, model.archid, data.__class__.__name__, budget))
|
||||
for model, data, budget in zip(models, dataset_providers, budgets[obj_name])
|
||||
self._cache.get((obj_name, model.archid, budget))
|
||||
for model, budget in zip(models, budgets[obj_name])
|
||||
]
|
||||
for obj_name in objs
|
||||
}
|
||||
|
|
@ -167,7 +192,7 @@ class SearchObjectives:
|
|||
)
|
||||
|
||||
for i in pbar:
|
||||
obj_d["evaluator"].send(models[i], dataset_providers[i], budgets[obj_name][i])
|
||||
obj_d.evaluator.send(models[i], budgets[obj_name][i])
|
||||
|
||||
# Calculates synchronous objectives in order
|
||||
for obj_name, obj_d in sync_objs.items():
|
||||
|
|
@ -178,8 +203,8 @@ class SearchObjectives:
|
|||
)
|
||||
|
||||
for i in pbar:
|
||||
eval_results[obj_name][i] = obj_d["evaluator"].evaluate(
|
||||
models[i], dataset_providers[i], budgets[obj_name][i]
|
||||
eval_results[obj_name][i] = obj_d.evaluator.evaluate(
|
||||
models[i], budgets[obj_name][i]
|
||||
)
|
||||
|
||||
# Gets results from async objectives
|
||||
|
|
@ -190,7 +215,7 @@ class SearchObjectives:
|
|||
)
|
||||
|
||||
for obj_name, obj_d in pbar:
|
||||
results = obj_d["evaluator"].fetch_all()
|
||||
results = obj_d.evaluator.fetch_all()
|
||||
|
||||
assert len(eval_indices[obj_name]) == len(results), "Received a different amount of results than expected."
|
||||
|
||||
|
|
@ -198,17 +223,16 @@ class SearchObjectives:
|
|||
eval_results[obj_name][eval_i] = results[result_i]
|
||||
|
||||
# Updates cache
|
||||
if self.cache_objective_evaluation:
|
||||
if self._cache_objective_evaluation:
|
||||
for obj_name in objs:
|
||||
for i in eval_indices[obj_name]:
|
||||
cache_tuple = (
|
||||
obj_name,
|
||||
models[i].archid,
|
||||
dataset_providers[i].__class__.__name__,
|
||||
budgets[obj_name][i],
|
||||
)
|
||||
|
||||
self.cache[cache_tuple] = eval_results[obj_name][i]
|
||||
self._cache[cache_tuple] = eval_results[obj_name][i]
|
||||
|
||||
assert len(set(len(r) for r in eval_results.values())) == 1
|
||||
|
||||
|
|
@ -225,8 +249,8 @@ class SearchObjectives:
|
|||
|
||||
valid_mask = np.logical_and.reduce(
|
||||
[
|
||||
(obj_r >= objs_or_constraints[obj_name]["constraint"][0])
|
||||
& (obj_r <= objs_or_constraints[obj_name]["constraint"][1])
|
||||
(obj_r >= objs_or_constraints[obj_name].constraint[0])
|
||||
& (obj_r <= objs_or_constraints[obj_name].constraint[1])
|
||||
for obj_name, obj_r in results.items()
|
||||
]
|
||||
)
|
||||
|
|
@ -236,7 +260,6 @@ class SearchObjectives:
|
|||
def validate_constraints(
|
||||
self,
|
||||
models: List[ArchaiModel],
|
||||
dataset_providers: Union[DatasetProvider, List[DatasetProvider]],
|
||||
progress_bar: Optional[bool] = False,
|
||||
) -> Tuple[Dict[str, np.ndarray], np.ndarray]:
|
||||
"""Evaluate constraints for a list of models and returns the indices of models that
|
||||
|
|
@ -244,7 +267,6 @@ class SearchObjectives:
|
|||
|
||||
Args:
|
||||
models: List of models to evaluate.
|
||||
dataset_providers: Dataset provider or list of dataset providers.
|
||||
progress_bar: Whether to show progress bar.
|
||||
|
||||
Returns:
|
||||
|
|
@ -252,45 +274,42 @@ class SearchObjectives:
|
|||
|
||||
"""
|
||||
|
||||
# Gets all constraints from cheap_objectives and extra_constraints
|
||||
# Gets all constraints from cheap_objectives that have constraints and extra_constraints
|
||||
constraints = dict(
|
||||
self.extra_constraints, **self._filter_objs(self.cheap_objs, "constraint", lambda x: x is not None)
|
||||
self._extra_constraints, **self._filter_objs(self.cheap_objectives, lambda x: x.constraint is not None)
|
||||
)
|
||||
|
||||
if not constraints:
|
||||
return {}, np.arange(len(models))
|
||||
|
||||
eval_results = self._eval_objs(constraints, models, dataset_providers, budgets=None, progress_bar=progress_bar)
|
||||
eval_results = self._eval_objs(constraints, models, budgets=None, progress_bar=progress_bar)
|
||||
|
||||
return eval_results, self._get_valid_arch_indices(constraints, eval_results)
|
||||
|
||||
def is_model_valid(self, model: ArchaiModel, dataset_provider: DatasetProvider) -> bool:
|
||||
def is_model_valid(self, model: ArchaiModel) -> bool:
|
||||
"""Check if a model satisfies all constraints.
|
||||
|
||||
Args:
|
||||
model: Model to check.
|
||||
dataset_provider: Dataset provider.
|
||||
|
||||
Returns:
|
||||
`True` if model is valid, `False` otherwise.
|
||||
|
||||
"""
|
||||
|
||||
_, idx = self.validate_constraints([model], dataset_provider, progress_bar=False)
|
||||
_, idx = self.validate_constraints([model], progress_bar=False)
|
||||
return len(idx) > 0
|
||||
|
||||
def eval_cheap_objs(
|
||||
self,
|
||||
models: List[ArchaiModel],
|
||||
dataset_providers: Union[DatasetProvider, List[DatasetProvider]],
|
||||
budgets: Optional[Dict[str, List]] = None,
|
||||
progress_bar: Optional[bool] = False,
|
||||
) -> Dict[str, np.ndarray]:
|
||||
"""Evaluate all cheap objectives for a list of models and dataset provider(s).
|
||||
"""Evaluate all cheap objectives for a list of models.
|
||||
|
||||
Args:
|
||||
models: List of models to evaluate.
|
||||
dataset_providers: Dataset provider or list of dataset providers.
|
||||
budgets: Budgets for each objective.
|
||||
progress_bar: Whether to show progress bar.
|
||||
|
||||
|
|
@ -298,22 +317,18 @@ class SearchObjectives:
|
|||
Dictionary with evaluation results.
|
||||
|
||||
"""
|
||||
|
||||
return self._eval_objs(self.cheap_objs, models, dataset_providers, budgets, progress_bar)
|
||||
return self._eval_objs(self.cheap_objectives, models, budgets, progress_bar)
|
||||
|
||||
def eval_expensive_objs(
|
||||
self,
|
||||
models: List[ArchaiModel],
|
||||
dataset_providers: Union[DatasetProvider, List[DatasetProvider]],
|
||||
budgets: Optional[Dict[str, List]] = None,
|
||||
progress_bar: Optional[bool] = False,
|
||||
) -> Dict[str, np.ndarray]:
|
||||
"""Evaluate all expensive objective functions for a list of models and
|
||||
dataset provider(s).
|
||||
"""Evaluate all expensive objective functions for a list of models.
|
||||
|
||||
Args:
|
||||
models: List of models to evaluate.
|
||||
dataset_providers: Dataset provider or list of dataset providers.
|
||||
budgets: Budgets for each objective.
|
||||
progress_bar: Whether to show progress bar. Defaults to False.
|
||||
|
||||
|
|
@ -322,20 +337,18 @@ class SearchObjectives:
|
|||
|
||||
"""
|
||||
|
||||
return self._eval_objs(self.exp_objs, models, dataset_providers, budgets, progress_bar)
|
||||
return self._eval_objs(self.expensive_objectives, models, budgets, progress_bar)
|
||||
|
||||
def eval_all_objs(
|
||||
self,
|
||||
models: List[ArchaiModel],
|
||||
dataset_providers: Union[DatasetProvider, List[DatasetProvider]],
|
||||
budgets: Optional[Dict[str, List]] = None,
|
||||
progress_bar: Optional[bool] = False,
|
||||
) -> Dict[str, np.ndarray]:
|
||||
"""Evaluate all objective functions for a list of models and dataset provider(s).
|
||||
"""Evaluate all objective functions for a list of models.
|
||||
|
||||
Args:
|
||||
models: List of models to evaluate.
|
||||
dataset_providers: Dataset provider or list of dataset providers.
|
||||
budgets: Budgets for each objective.
|
||||
progress_bar: Whether to show progress bar.
|
||||
|
||||
|
|
@ -344,7 +357,7 @@ class SearchObjectives:
|
|||
|
||||
"""
|
||||
|
||||
return self._eval_objs(dict(self.exp_objs, **self.cheap_objs), models, dataset_providers, budgets, progress_bar)
|
||||
return self._eval_objs(self._objs, models, budgets, progress_bar)
|
||||
|
||||
def save_cache(self, file_path: str) -> None:
|
||||
"""Save the state of the `SearchObjectives` object to a YAML file.
|
||||
|
|
@ -355,7 +368,7 @@ class SearchObjectives:
|
|||
"""
|
||||
|
||||
with open(file_path, "w", encoding="utf-8") as f:
|
||||
yaml.dump(self.cache, f)
|
||||
yaml.dump(self._cache, f)
|
||||
|
||||
def load_cache(self, file_path: str) -> None:
|
||||
"""Load the state of the `SearchObjectives` object from a YAML file.
|
||||
|
|
@ -366,4 +379,19 @@ class SearchObjectives:
|
|||
"""
|
||||
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
self.cache = yaml.load(f)
|
||||
self._cache = yaml.load(f)
|
||||
|
||||
def lookup_cache(self, obj_name: str, arch_id: str, budget: Optional[int]) -> Optional[float]:
|
||||
"""Look up the cache for a specific objective, architecture and budget.
|
||||
|
||||
Args:
|
||||
obj_name: Name of objective.
|
||||
arch_id: Architecture ID.
|
||||
budget: Budget.
|
||||
|
||||
Returns:
|
||||
Evaluation result if found in cache, `None` otherwise.
|
||||
|
||||
"""
|
||||
|
||||
return self._cache.get((obj_name, arch_id, budget), None)
|
||||
|
|
@ -51,7 +51,7 @@ class SearchResults:
|
|||
|
||||
return {
|
||||
obj_name: np.array([r for iter_results in self.results for r in iter_results[obj_name]], dtype=np.float32)
|
||||
for obj_name in self.objectives.objs
|
||||
for obj_name in self.objectives.objectives
|
||||
}
|
||||
|
||||
def add_iteration_results(
|
||||
|
|
@ -70,7 +70,7 @@ class SearchResults:
|
|||
|
||||
"""
|
||||
|
||||
assert all(obj_name in evaluation_results for obj_name in self.objectives.objs)
|
||||
assert all(obj_name in evaluation_results for obj_name in self.objectives.objectives)
|
||||
assert all(len(r) == len(models) for r in evaluation_results.values())
|
||||
|
||||
extra_model_data = copy.deepcopy(extra_model_data) or dict()
|
||||
|
|
@ -117,7 +117,7 @@ class SearchResults:
|
|||
obj_name: np.concatenate(
|
||||
[self.results[it][obj_name] for it in range(start_iteration, end_iteration)], axis=0
|
||||
)
|
||||
for obj_name in self.objectives.objs.keys()
|
||||
for obj_name in self.objectives.objective_names
|
||||
}
|
||||
|
||||
all_iteration_nums = np.array(
|
||||
|
|
@ -203,10 +203,10 @@ class SearchResults:
|
|||
max_x, max_y = status_df[obj_x].max(), status_df[obj_y].max()
|
||||
status_df["x"], status_df["y"] = status_df[obj_x], status_df[obj_y]
|
||||
|
||||
if self.objectives.objs[obj_x]["higher_is_better"]:
|
||||
if self.objectives.objectives[obj_x].higher_is_better:
|
||||
status_df["x"] = max_x - status_df["x"]
|
||||
|
||||
if self.objectives.objs[obj_y]["higher_is_better"]:
|
||||
if self.objectives.objectives[obj_y].higher_is_better:
|
||||
status_df["y"] = max_y - status_df["y"]
|
||||
|
||||
colors = plt.cm.plasma(np.linspace(0, 1, self.iteration_num + 1))
|
||||
|
|
@ -254,7 +254,7 @@ class SearchResults:
|
|||
path = Path(directory)
|
||||
path.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
objective_names = list(self.objectives.objs.keys())
|
||||
objective_names = list(self.objectives.objective_names)
|
||||
plots = []
|
||||
|
||||
for i, obj_x in enumerate(objective_names):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from typing import Any, Dict, Optional
|
|||
import nats_bench
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
from archai.discrete_search.search_spaces.benchmark.natsbench_tss import (
|
||||
|
|
@ -56,7 +55,7 @@ class NatsbenchMetric(ModelEvaluator):
|
|||
self.total_time_spent = 0
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> Optional[float]:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> Optional[float]:
|
||||
natsbench_id = self.archid_pattern.match(model.archid)
|
||||
budget = int(budget) if budget else budget
|
||||
|
||||
|
|
|
|||
|
|
@ -29,5 +29,5 @@ class EvaluationFunction(ModelEvaluator):
|
|||
self.evaluation_fn = evaluation_fn
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
return self.evaluation_fn(model, dataset_provider, budget)
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return self.evaluation_fn(model, budget)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from typing import List, Optional
|
|||
from overrides import overrides
|
||||
from torch import nn
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
|
||||
|
|
@ -34,7 +33,7 @@ class NonEmbeddingParamsProxy(ModelEvaluator):
|
|||
self.trainable_only = trainable_only
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
total_params = sum(
|
||||
param.numel() for param in model.arch.parameters() if not self.trainable_only or param.requires_grad
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import torch
|
|||
from onnxruntime import InferenceSession
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
from archai.discrete_search.search_spaces.nlp.transformer_flex.search_space import (
|
||||
|
|
@ -119,7 +118,7 @@ class TransformerFlexOnnxLatency(ModelEvaluator):
|
|||
return float(np.median(runner) if self.use_median else np.mean(runner))
|
||||
|
||||
@overrides
|
||||
def evaluate(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, arch: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
model = self._load_and_prepare(arch.metadata["config"])
|
||||
|
||||
# There is a bug for Python < 3.10 when using TemporaryFile with Windows,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ from typing import Any, Dict, Optional
|
|||
import torch
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
from archai.discrete_search.search_spaces.nlp.transformer_flex.search_space import (
|
||||
|
|
@ -69,7 +68,7 @@ class TransformerFlexOnnxMemory(ModelEvaluator):
|
|||
return prepare_model_for_onnx(model, self.search_space.arch_type)
|
||||
|
||||
@overrides
|
||||
def evaluate(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, arch: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
model = self._load_and_prepare(arch.metadata["config"])
|
||||
|
||||
# There is a bug for Python < 3.10 when using TemporaryFile with Windows,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ import onnxruntime as rt
|
|||
import torch
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.common.timing import MeasureBlockTime
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
|
|
@ -63,7 +62,7 @@ class AvgOnnxLatency(ModelEvaluator):
|
|||
self.device = device
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
model.arch.to("cpu")
|
||||
|
||||
# Exports model to ONNX
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ def _ray_wrap_training_fn(training_fn) -> Callable:
|
|||
class ProgressiveTraining(ModelEvaluator):
|
||||
"""Progressive training evaluator."""
|
||||
|
||||
def __init__(self, search_space: DiscreteSearchSpace, training_fn: Callable) -> None:
|
||||
def __init__(self, search_space: DiscreteSearchSpace, dataset: DatasetProvider, training_fn: Callable) -> None:
|
||||
"""Initialize the evaluator.
|
||||
|
||||
Args:
|
||||
|
|
@ -40,17 +40,18 @@ class ProgressiveTraining(ModelEvaluator):
|
|||
|
||||
self.search_space = search_space
|
||||
self.training_fn = training_fn
|
||||
self.dataset = dataset
|
||||
|
||||
# Training state buffer (e.g optimizer state) for each architecture id
|
||||
self.training_states = {}
|
||||
|
||||
@overrides
|
||||
def evaluate(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, arch: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
# Tries to retrieve previous training state
|
||||
tr_state = self.training_states.get(arch.archid, None)
|
||||
|
||||
# Computes metric and updates training state
|
||||
metric_result, updated_tr_state = self.training_fn(arch, dataset, budget, tr_state)
|
||||
metric_result, updated_tr_state = self.training_fn(arch, self.dataset, budget, tr_state)
|
||||
self.training_states[arch.archid] = updated_tr_state
|
||||
|
||||
return metric_result
|
||||
|
|
@ -62,6 +63,7 @@ class RayProgressiveTraining(AsyncModelEvaluator):
|
|||
def __init__(
|
||||
self,
|
||||
search_space: DiscreteSearchSpace,
|
||||
dataset: DatasetProvider,
|
||||
training_fn: Callable,
|
||||
timeout: Optional[float] = None,
|
||||
force_stop: Optional[bool] = False,
|
||||
|
|
@ -78,6 +80,7 @@ class RayProgressiveTraining(AsyncModelEvaluator):
|
|||
"""
|
||||
|
||||
self.search_space = search_space
|
||||
self.dataset = dataset
|
||||
|
||||
if ray_kwargs:
|
||||
self.compute_fn = ray.remote(**ray_kwargs)(_ray_wrap_training_fn(training_fn))
|
||||
|
|
@ -98,12 +101,12 @@ class RayProgressiveTraining(AsyncModelEvaluator):
|
|||
self.training_states = {}
|
||||
|
||||
@overrides
|
||||
def send(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
# Stores original model reference
|
||||
self.models.append(arch)
|
||||
|
||||
current_tr_state = self.training_states.get(arch.archid, None)
|
||||
self.results_ref.append(self.compute_fn.remote(arch, dataset, budget, current_tr_state))
|
||||
self.results_ref.append(self.compute_fn.remote(arch, self.dataset, budget, current_tr_state))
|
||||
|
||||
@overrides
|
||||
def fetch_all(self) -> List[Union[float, None]]:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from typing import Dict, List, Optional, Union
|
|||
import torch
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import ModelEvaluator
|
||||
from archai.discrete_search.evaluators.pt_profiler_utils.pt_profiler_eval import profile
|
||||
|
|
@ -30,7 +29,7 @@ class TorchNumParameters(ModelEvaluator):
|
|||
self.trainable_only = trainable_only
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
total_params = sum(
|
||||
param.numel() for param in model.arch.parameters() if not self.trainable_only or param.requires_grad
|
||||
)
|
||||
|
|
@ -71,7 +70,7 @@ class TorchFlops(ModelEvaluator):
|
|||
self.ignore_layers = ignore_layers
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return profile(
|
||||
model.arch,
|
||||
self.forward_args,
|
||||
|
|
@ -105,7 +104,7 @@ class TorchMacs(ModelEvaluator):
|
|||
self.ignore_layers = ignore_layers
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return profile(
|
||||
model.arch,
|
||||
self.forward_args,
|
||||
|
|
@ -152,7 +151,7 @@ class TorchLatency(ModelEvaluator):
|
|||
self.ignore_layers = ignore_layers
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return profile(
|
||||
model.arch,
|
||||
self.forward_args,
|
||||
|
|
@ -198,7 +197,7 @@ class TorchPeakCudaMemory(ModelEvaluator):
|
|||
self.ignore_layers = ignore_layers
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return profile(
|
||||
model.arch,
|
||||
self.forward_args,
|
||||
|
|
@ -232,7 +231,7 @@ class TorchPeakCpuMemory(ModelEvaluator):
|
|||
self.forward_kwargs = forward_kwargs or {}
|
||||
|
||||
@overrides
|
||||
def evaluate(self, model: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None):
|
||||
def evaluate(self, model: ArchaiModel, budget: Optional[float] = None):
|
||||
model.arch.to("cpu")
|
||||
forward_args = tuple([arg.to("cpu") for arg in self.forward_args])
|
||||
forward_kwargs = {key: value.to("cpu") for key, value in self.forward_kwargs.items()}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from typing import Callable, List, Optional, Union
|
|||
import ray
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import (
|
||||
AsyncModelEvaluator,
|
||||
|
|
@ -15,8 +14,8 @@ from archai.discrete_search.api.model_evaluator import (
|
|||
|
||||
|
||||
def _wrap_metric_calculate(class_method) -> Callable:
|
||||
def _calculate(arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> Callable:
|
||||
return class_method(arch, dataset, budget)
|
||||
def _calculate(arch: ArchaiModel, budget: Optional[float] = None) -> Callable:
|
||||
return class_method(arch, budget)
|
||||
|
||||
return _calculate
|
||||
|
||||
|
|
@ -58,8 +57,8 @@ class RayParallelEvaluator(AsyncModelEvaluator):
|
|||
self.object_refs = []
|
||||
|
||||
@overrides
|
||||
def send(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
self.object_refs.append(self.compute_fn.remote(arch, dataset, budget))
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
self.object_refs.append(self.compute_fn.remote(arch, budget))
|
||||
|
||||
@overrides
|
||||
def fetch_all(self) -> List[Union[float, None]]:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ from azure.data.tables import TableServiceClient, UpdateMode
|
|||
from azure.storage.blob import BlobServiceClient
|
||||
from overrides import overrides
|
||||
|
||||
from archai.api.dataset_provider import DatasetProvider
|
||||
from archai.discrete_search.api.archai_model import ArchaiModel
|
||||
from archai.discrete_search.api.model_evaluator import AsyncModelEvaluator
|
||||
|
||||
|
|
@ -151,7 +150,7 @@ class RemoteAzureBenchmarkEvaluator(AsyncModelEvaluator):
|
|||
return self.table_client.upsert_entity(entity, mode=UpdateMode.REPLACE)
|
||||
|
||||
@overrides
|
||||
def send(self, arch: ArchaiModel, dataset_provider: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
archid = str(arch.archid)
|
||||
|
||||
# Checks if architecture was already benchmarked
|
||||
|
|
|
|||
|
|
@ -24,12 +24,12 @@ def get_pareto_frontier(
|
|||
|
||||
"""
|
||||
|
||||
assert all(obj_name in objectives.objs for obj_name in evaluation_results)
|
||||
assert all(obj_name in objectives.objectives for obj_name in evaluation_results)
|
||||
assert all(len(r) == len(models) for r in evaluation_results.values())
|
||||
|
||||
# Inverts maximization objectives
|
||||
inverted_results = {
|
||||
obj_name: (-obj_r if objectives.objs[obj_name]["higher_is_better"] else obj_r)
|
||||
obj_name: (-obj_r if objectives.objectives[obj_name].higher_is_better else obj_r)
|
||||
for obj_name, obj_r in evaluation_results.items()
|
||||
}
|
||||
|
||||
|
|
@ -62,12 +62,12 @@ def get_non_dominated_sorting(
|
|||
|
||||
"""
|
||||
|
||||
assert all(obj_name in objectives.objs for obj_name in evaluation_results)
|
||||
assert all(obj_name in objectives.objectives for obj_name in evaluation_results)
|
||||
assert all(len(r) == len(models) for r in evaluation_results.values())
|
||||
|
||||
# Inverts maximization objectives
|
||||
inverted_results = {
|
||||
obj_name: (-obj_r if objectives.objs[obj_name]["higher_is_better"] else obj_r)
|
||||
obj_name: (-obj_r if objectives.objectives[obj_name].higher_is_better else obj_r)
|
||||
for obj_name, obj_r in evaluation_results.items()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,6 @@ When the search is finished the next cell in the notebook will download the resu
|
|||
|
||||
|
||||
```json
|
||||
Top model results:
|
||||
{
|
||||
"init_num_models": 10,
|
||||
"partial_training_epochs": 1,
|
||||
|
|
@ -59,7 +58,8 @@ Top model results:
|
|||
"id_1250cb3f_0fc9_4cad_a4f2_e627db5c66e8": {
|
||||
"archid": "(1, 1, 32)",
|
||||
"val_acc": 0.2237
|
||||
},
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You can run the last cell of the notebook to compare the inference accuracy of the top
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class AmlTrainingValAccuracy(AsyncModelEvaluator):
|
|||
self.store = ArchaiStore(storage_account_name, storage_account_key)
|
||||
|
||||
@overrides
|
||||
def send(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
self.models += [arch.arch.get_archid()]
|
||||
|
||||
@overrides
|
||||
|
|
|
|||
|
|
@ -36,9 +36,10 @@ exclude_patterns = [
|
|||
"tests/**",
|
||||
]
|
||||
extlinks = {"github": ("https://github.com/microsoft/archai/tree/main/%s", "%s")}
|
||||
source_suffix = ".rst"
|
||||
source_suffix = [".rst", ".md"]
|
||||
master_doc = "index"
|
||||
language = "en"
|
||||
html_baseurl = "https://microsoft.github.io/archai/"
|
||||
|
||||
# Options for HTML output
|
||||
html_title = project
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,136 @@
|
|||
from typing import List, Optional
|
||||
|
||||
from overrides import overrides
|
||||
import numpy as np
|
||||
import torch
|
||||
import re
|
||||
from torch import nn
|
||||
from archai.discrete_search.api import ArchaiModel
|
||||
|
||||
import json
|
||||
from random import Random
|
||||
from archai.discrete_search.api import DiscreteSearchSpace
|
||||
from model import MyModel
|
||||
|
||||
|
||||
class CNNSearchSpace(DiscreteSearchSpace):
|
||||
def __init__(self, min_layers: int = 1, max_layers: int = 12,
|
||||
kernel_list=(1, 3, 5, 7), hidden_list=(16, 32, 64, 128),
|
||||
seed: int = 1):
|
||||
|
||||
self.min_layers = min_layers
|
||||
self.max_layers = max_layers
|
||||
self.kernel_list = kernel_list
|
||||
self.hidden_list = hidden_list
|
||||
|
||||
self.rng = Random(seed)
|
||||
|
||||
def get_archid(self, model: MyModel) -> str:
|
||||
return f'L={model.nb_layers}, K={model.kernel_size}, H={model.hidden_dim}'
|
||||
|
||||
@overrides
|
||||
def random_sample(self) -> ArchaiModel:
|
||||
# Randomly chooses architecture parameters
|
||||
nb_layers = self.rng.randint(self.min_layers, self.max_layers)
|
||||
kernel_size = self.rng.choice(self.kernel_list)
|
||||
hidden_dim = self.rng.choice(self.hidden_list)
|
||||
|
||||
model = MyModel(nb_layers, kernel_size, hidden_dim)
|
||||
|
||||
# Wraps model into ArchaiModel
|
||||
return ArchaiModel(arch=model, archid=self.get_archid(model))
|
||||
|
||||
@overrides
|
||||
def save_arch(self, model: ArchaiModel, file: str):
|
||||
with open(file, 'w') as fp:
|
||||
json.dump({
|
||||
'nb_layers': model.arch.nb_layers,
|
||||
'kernel_size': model.arch.kernel_size,
|
||||
'hidden_dim': model.arch.hidden_dim
|
||||
}, fp)
|
||||
|
||||
@overrides
|
||||
def load_arch(self, file: str):
|
||||
config = json.load(open(file))
|
||||
model = MyModel(**config)
|
||||
|
||||
return ArchaiModel(arch=model, archid=self.get_archid(model))
|
||||
|
||||
@overrides
|
||||
def save_model_weights(self, model: ArchaiModel, file: str):
|
||||
state_dict = model.arch.get_state_dict()
|
||||
torch.save(state_dict, file)
|
||||
|
||||
@overrides
|
||||
def load_model_weights(self, model: ArchaiModel, file: str):
|
||||
model.arch.load_state_dict(torch.load(file))
|
||||
|
||||
def from_archid(self, archid: str) -> MyModel:
|
||||
# parse the format 'L=1, K=5, H=64'
|
||||
regex = re.compile(r'L=(\d+), K=(\d+), H=(\d+)')
|
||||
m = regex.match(archid).groups()
|
||||
if len(m) == 3:
|
||||
config = {
|
||||
'nb_layers': int(m[0]),
|
||||
'kernel_size': int(m[1]),
|
||||
'hidden_dim': int(m[2])
|
||||
}
|
||||
return MyModel(**config)
|
||||
else:
|
||||
raise Exception(f"Archid '{archid}' is not in the correct format")
|
||||
|
||||
def ensure_model(self, model: ArchaiModel):
|
||||
if not isinstance(model.arch, MyModel):
|
||||
model.arch = self.from_archid(model.archid)
|
||||
|
||||
|
||||
from archai.discrete_search.api.search_space import EvolutionarySearchSpace, BayesOptSearchSpace
|
||||
|
||||
|
||||
class CNNSearchSpaceExt(CNNSearchSpace, EvolutionarySearchSpace, BayesOptSearchSpace):
|
||||
''' We are subclassing CNNSearchSpace just to save up space'''
|
||||
|
||||
@overrides
|
||||
def mutate(self, model_1: ArchaiModel) -> ArchaiModel:
|
||||
self.ensure_model(model_1)
|
||||
config = {
|
||||
'nb_layers': model_1.arch.nb_layers,
|
||||
'kernel_size': model_1.arch.kernel_size,
|
||||
'hidden_dim': model_1.arch.hidden_dim
|
||||
}
|
||||
|
||||
if self.rng.random() < 0.2:
|
||||
config['nb_layers'] = self.rng.randint(self.min_layers, self.max_layers)
|
||||
|
||||
if self.rng.random() < 0.2:
|
||||
config['kernel_size'] = self.rng.choice(self.kernel_list)
|
||||
|
||||
if self.rng.random() < 0.2:
|
||||
config['hidden_dim'] = self.rng.choice(self.hidden_list)
|
||||
|
||||
mutated_model = MyModel(**config)
|
||||
|
||||
return ArchaiModel(
|
||||
arch=mutated_model, archid=self.get_archid(mutated_model)
|
||||
)
|
||||
|
||||
@overrides
|
||||
def crossover(self, model_list: List[ArchaiModel]) -> ArchaiModel:
|
||||
for m in model_list:
|
||||
self.ensure_model(m)
|
||||
new_config = {
|
||||
'nb_layers': self.rng.choice([m.arch.nb_layers for m in model_list]),
|
||||
'kernel_size': self.rng.choice([m.arch.kernel_size for m in model_list]),
|
||||
'hidden_dim': self.rng.choice([m.arch.hidden_dim for m in model_list]),
|
||||
}
|
||||
|
||||
crossover_model = MyModel(**new_config)
|
||||
|
||||
return ArchaiModel(
|
||||
arch=crossover_model, archid=self.get_archid(crossover_model)
|
||||
)
|
||||
|
||||
@overrides
|
||||
def encode(self, model: ArchaiModel) -> np.ndarray:
|
||||
self.ensure_model(model)
|
||||
return np.array([model.arch.nb_layers, model.arch.kernel_size, model.arch.hidden_dim])
|
||||
|
|
@ -149,7 +149,7 @@
|
|||
"id": "c8237506",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"`ArchParamTree` are used to generate `ArchConfig` objects, that specify the choosen architecture configuration. We can sample a configuration using `arch_param_tree.sample_config()`"
|
||||
"`ArchParamTree` are used to generate `ArchConfig` objects, that specify the chosen architecture configuration. We can sample a configuration using `arch_param_tree.sample_config()`"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -195,7 +195,7 @@
|
|||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"7"
|
||||
"3"
|
||||
]
|
||||
},
|
||||
"execution_count": 7,
|
||||
|
|
@ -216,7 +216,7 @@
|
|||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"16"
|
||||
"8"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
|
|
@ -300,36 +300,36 @@
|
|||
"text/plain": [
|
||||
"MyModel(\n",
|
||||
" (stem_conv): Sequential(\n",
|
||||
" (0): Conv2d(3, 16, kernel_size=(3, 3), stride=(4, 4), padding=(1, 1))\n",
|
||||
" (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (0): Conv2d(3, 8, kernel_size=(3, 3), stride=(4, 4), padding=(1, 1))\n",
|
||||
" (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (2): ReLU()\n",
|
||||
" )\n",
|
||||
" (layers): Sequential(\n",
|
||||
" (0): MyConvBlock(\n",
|
||||
" (op): Sequential(\n",
|
||||
" (0): Conv2d(16, 16, kernel_size=(7, 7), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (2): ReLU()\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (1): MyConvBlock(\n",
|
||||
" (op): Sequential(\n",
|
||||
" (0): Conv2d(16, 16, kernel_size=(7, 7), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (2): ReLU()\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (2): MyConvBlock(\n",
|
||||
" (op): Sequential(\n",
|
||||
" (0): Conv2d(16, 16, kernel_size=(7, 7), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (2): ReLU()\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (3): MyConvBlock(\n",
|
||||
" (op): Sequential(\n",
|
||||
" (0): Conv2d(16, 16, kernel_size=(7, 7), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (0): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=same)\n",
|
||||
" (1): BatchNorm2d(8, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
|
||||
" (2): ReLU()\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
|
|
@ -506,7 +506,7 @@
|
|||
"text/plain": [
|
||||
"ArchConfig({\n",
|
||||
" \"stem_config\": {\n",
|
||||
" \"kernel_size\": 5\n",
|
||||
" \"kernel_size\": 3\n",
|
||||
" },\n",
|
||||
" \"conv_kernel_size\": 3,\n",
|
||||
" \"num_ch\": 8\n",
|
||||
|
|
@ -541,7 +541,7 @@
|
|||
"data": {
|
||||
"text/plain": [
|
||||
"ArchConfig({\n",
|
||||
" \"kernel_size\": 5\n",
|
||||
" \"kernel_size\": 3\n",
|
||||
"})"
|
||||
]
|
||||
},
|
||||
|
|
@ -563,7 +563,7 @@
|
|||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"5"
|
||||
"3"
|
||||
]
|
||||
},
|
||||
"execution_count": 19,
|
||||
|
|
@ -632,7 +632,7 @@
|
|||
"text/plain": [
|
||||
"ArchConfig({\n",
|
||||
" \"stem_config\": {\n",
|
||||
" \"kernel_size\": 5\n",
|
||||
" \"kernel_size\": 3\n",
|
||||
" },\n",
|
||||
" \"conv_kernel_size\": 5,\n",
|
||||
" \"num_ch\": 32\n",
|
||||
|
|
@ -659,7 +659,7 @@
|
|||
"text/plain": [
|
||||
"ArchConfig({\n",
|
||||
" \"stem_config\": {\n",
|
||||
" \"kernel_size\": 5\n",
|
||||
" \"kernel_size\": 3\n",
|
||||
" },\n",
|
||||
" \"conv_kernel_size\": 5,\n",
|
||||
" \"num_ch\": 32\n",
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,34 @@
|
|||
from torch import nn
|
||||
|
||||
|
||||
class MyModel(nn.Module):
|
||||
def __init__(self, nb_layers: int = 5, kernel_size: int = 3, hidden_dim: int = 32):
|
||||
super().__init__()
|
||||
|
||||
self.nb_layers = nb_layers
|
||||
self.kernel_size = kernel_size
|
||||
self.hidden_dim = hidden_dim
|
||||
|
||||
layer_list = []
|
||||
|
||||
for i in range(nb_layers):
|
||||
in_ch = (1 if i == 0 else hidden_dim)
|
||||
|
||||
layer_list += [
|
||||
nn.Conv2d(in_ch, hidden_dim, kernel_size=kernel_size, padding=(kernel_size-1)//2),
|
||||
nn.BatchNorm2d(hidden_dim),
|
||||
nn.ReLU(),
|
||||
]
|
||||
|
||||
layer_list += [
|
||||
nn.AdaptiveAvgPool2d(output_size=(1, 1)),
|
||||
nn.Conv2d(hidden_dim, 10, kernel_size=1)
|
||||
]
|
||||
|
||||
self.model = nn.Sequential(*layer_list)
|
||||
|
||||
def forward(self, x):
|
||||
return self.model(x).squeeze()
|
||||
|
||||
def get_archid(self):
|
||||
return f'({self.nb_layers}, {self.kernel_size}, {self.hidden_dim})'
|
||||
|
|
@ -2,10 +2,10 @@
|
|||
# Licensed under the MIT license.
|
||||
|
||||
import torch
|
||||
from torch_testbed import cifar10_models, utils
|
||||
from torch_testbed import cifar10_models
|
||||
from torch_testbed.dataloader_dali import cifar10_dataloaders
|
||||
from torch_testbed.timing import MeasureTime, clear_timings, print_all_timings
|
||||
|
||||
from archai.common import utils
|
||||
|
||||
utils.setup_cuda(42, local_rank=0)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
# Licensed under the MIT license.
|
||||
|
||||
import torch
|
||||
from torch_testbed import cifar10_models, utils
|
||||
from torch_testbed import cifar10_models
|
||||
from torch_testbed.timing import MeasureTime, print_all_timings
|
||||
|
||||
from archai.common import utils
|
||||
|
||||
utils.setup_cuda(42, local_rank=0)
|
||||
|
||||
|
|
|
|||
7
setup.py
7
setup.py
|
|
@ -34,7 +34,9 @@ dependencies = [
|
|||
"opencv-python",
|
||||
"opt_einsum",
|
||||
"overrides==3.1.0",
|
||||
"pandas",
|
||||
"psutil",
|
||||
"pydata-sphinx-theme==0.13.1",
|
||||
"pytest",
|
||||
"pytorch-lightning",
|
||||
"pyyaml",
|
||||
|
|
@ -60,6 +62,9 @@ dependencies_dict = {y: x for x, y in (re.findall(r"^(([^!=<>~ ]+)(?:[!=<>~ ].*)
|
|||
|
||||
|
||||
def filter_dependencies(*pkgs):
|
||||
for pkg in pkgs:
|
||||
if pkg not in dependencies_dict:
|
||||
raise ValueError(f"Package {pkg} not found in dependencies")
|
||||
return [dependencies_dict[pkg] for pkg in pkgs]
|
||||
|
||||
|
||||
|
|
@ -81,6 +86,8 @@ extras_require["docs"] = filter_dependencies(
|
|||
"nbimporter",
|
||||
"nbsphinx",
|
||||
"nbval",
|
||||
"pandas",
|
||||
"pydata-sphinx-theme",
|
||||
"sphinx",
|
||||
"sphinx-book-theme",
|
||||
"sphinx-git",
|
||||
|
|
|
|||
|
|
@ -15,9 +15,9 @@ def search_objectives():
|
|||
rng2 = Random(2)
|
||||
rng3 = Random(3)
|
||||
|
||||
o1 = EvaluationFunction(lambda m, d, b: rng1.random())
|
||||
o2 = EvaluationFunction(lambda m, d, b: rng2.random())
|
||||
r = EvaluationFunction(lambda m, d, b: rng3.random())
|
||||
o1 = EvaluationFunction(lambda m, b: rng1.random())
|
||||
o2 = EvaluationFunction(lambda m, b: rng2.random())
|
||||
r = EvaluationFunction(lambda m, b: rng3.random())
|
||||
|
||||
so = SearchObjectives()
|
||||
so.add_objective("Random1", o1, higher_is_better=False, compute_intensive=False, constraint=(0.0, 0.4))
|
||||
|
|
|
|||
|
|
@ -26,4 +26,4 @@ def surrogate_model(search_objectives):
|
|||
|
||||
return MeanVar(self.mean_rng.random(size=(n, self.n_objs)), self.var_rng.random(size=(n, self.n_objs)))
|
||||
|
||||
return DummyPredictor(len(search_objectives.exp_objs))
|
||||
return DummyPredictor(len(search_objectives.expensive_objectives))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ def test_bananas(output_dir, search_space, search_objectives, surrogate_model):
|
|||
algo = MoBananasSearch(
|
||||
search_space=search_space,
|
||||
search_objectives=search_objectives,
|
||||
dataset_provider=None,
|
||||
output_dir=output_dir,
|
||||
surrogate_model=surrogate_model,
|
||||
num_iters=2,
|
||||
|
|
@ -36,5 +35,5 @@ def test_bananas(output_dir, search_space, search_objectives, surrogate_model):
|
|||
all_models = [m for iter_r in search_results.results for m in iter_r["models"]]
|
||||
|
||||
# Checks if all registered models satisfy constraints
|
||||
_, valid_models = search_objectives.validate_constraints(all_models, None)
|
||||
_, valid_models = search_objectives.validate_constraints(all_models)
|
||||
assert len(valid_models) == len(all_models)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ def output_dir(tmp_path_factory):
|
|||
|
||||
|
||||
def test_evolution_pareto(output_dir, search_space, search_objectives):
|
||||
algo = EvolutionParetoSearch(search_space, search_objectives, None, output_dir, num_iters=3, init_num_models=5)
|
||||
algo = EvolutionParetoSearch(search_space, search_objectives, output_dir, num_iters=3, init_num_models=5)
|
||||
|
||||
search_results = algo.search()
|
||||
assert len(os.listdir(output_dir)) > 0
|
||||
|
|
@ -25,5 +25,5 @@ def test_evolution_pareto(output_dir, search_space, search_objectives):
|
|||
all_models = [m for iter_r in search_results.results for m in iter_r["models"]]
|
||||
|
||||
# Checks if all registered models satisfy constraints
|
||||
_, valid_models = search_objectives.validate_constraints(all_models, None)
|
||||
_, valid_models = search_objectives.validate_constraints(all_models)
|
||||
assert len(valid_models) == len(all_models)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ def test_local_search(output_dir, search_space, search_objectives):
|
|||
algo = LocalSearch(
|
||||
search_space,
|
||||
search_objectives,
|
||||
None,
|
||||
output_dir,
|
||||
num_iters=2,
|
||||
init_num_models=5,
|
||||
|
|
@ -34,5 +33,5 @@ def test_local_search(output_dir, search_space, search_objectives):
|
|||
all_models = [m for iter_r in search_results.results for m in iter_r["models"]]
|
||||
|
||||
# Checks if all registered models satisfy constraints
|
||||
_, valid_models = search_objectives.validate_constraints(all_models, None)
|
||||
_, valid_models = search_objectives.validate_constraints(all_models)
|
||||
assert len(valid_models) == len(all_models)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ def output_dir(tmp_path_factory):
|
|||
|
||||
|
||||
def test_random_search(output_dir, search_space, search_objectives):
|
||||
algo = RandomSearch(search_space, search_objectives, None, output_dir, num_iters=2, samples_per_iter=5)
|
||||
algo = RandomSearch(search_space, search_objectives, output_dir, num_iters=2, samples_per_iter=5)
|
||||
|
||||
search_results = algo.search()
|
||||
assert len(os.listdir(output_dir)) > 0
|
||||
|
|
@ -25,5 +25,5 @@ def test_random_search(output_dir, search_space, search_objectives):
|
|||
all_models = [m for iter_r in search_results.results for m in iter_r["models"]]
|
||||
|
||||
# Checks if all registered models satisfy constraints
|
||||
_, valid_models = search_objectives.validate_constraints(all_models, None)
|
||||
_, valid_models = search_objectives.validate_constraints(all_models)
|
||||
assert len(valid_models) == len(all_models)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ def test_regularized_evolution(output_dir, search_space, search_objectives, surr
|
|||
algo = RegularizedEvolutionSearch(
|
||||
search_space=search_space,
|
||||
search_objectives=search_objectives,
|
||||
dataset_provider=None,
|
||||
output_dir=output_dir,
|
||||
num_iters=5,
|
||||
init_num_models=4,
|
||||
|
|
@ -37,5 +36,5 @@ def test_regularized_evolution(output_dir, search_space, search_objectives, surr
|
|||
all_models = [m for iter_r in search_results.results for m in iter_r["models"]]
|
||||
|
||||
# Checks if all registered models satisfy constraints
|
||||
_, valid_models = search_objectives.validate_constraints(all_models, None)
|
||||
_, valid_models = search_objectives.validate_constraints(all_models)
|
||||
assert len(valid_models) == len(all_models)
|
||||
|
|
|
|||
|
|
@ -15,20 +15,20 @@ from archai.discrete_search.api.model_evaluator import (
|
|||
|
||||
|
||||
class MyModelEvaluator(ModelEvaluator):
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, dataset) -> None:
|
||||
super().__init__()
|
||||
|
||||
@overrides
|
||||
def evaluate(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> float:
|
||||
def evaluate(self, arch: ArchaiModel, budget: Optional[float] = None) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
class MyAsyncModelEvaluator(AsyncModelEvaluator):
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, dataset) -> None:
|
||||
super().__init__()
|
||||
|
||||
@overrides
|
||||
def send(self, arch: ArchaiModel, dataset: DatasetProvider, budget: Optional[float] = None) -> None:
|
||||
def send(self, arch: ArchaiModel, budget: Optional[float] = None) -> None:
|
||||
return MagicMock()
|
||||
|
||||
@overrides
|
||||
|
|
@ -41,8 +41,8 @@ def test_model_evaluator():
|
|||
dataset = MagicMock()
|
||||
|
||||
# Assert that mocked value is returned
|
||||
model_evaluator = MyModelEvaluator()
|
||||
value = model_evaluator.evaluate(arch, dataset, budget=None)
|
||||
model_evaluator = MyModelEvaluator(dataset)
|
||||
value = model_evaluator.evaluate(arch, budget=None)
|
||||
assert value == 0.0
|
||||
|
||||
|
||||
|
|
@ -51,8 +51,8 @@ def test_async_model_evaluator():
|
|||
dataset = MagicMock()
|
||||
|
||||
# Assert that mocked method runs
|
||||
async_model_evaluator = MyAsyncModelEvaluator()
|
||||
assert async_model_evaluator.send(arch, dataset, budget=None)
|
||||
async_model_evaluator = MyAsyncModelEvaluator(dataset)
|
||||
assert async_model_evaluator.send(arch, budget=None)
|
||||
|
||||
# Assert that mocked value is returned
|
||||
values = async_model_evaluator.fetch_all()
|
||||
|
|
|
|||
|
|
@ -45,18 +45,18 @@ def test_eval_all_objs(models):
|
|||
AvgOnnxLatency(input_shape=(1, 1, 192), num_trials=1, input_dtype="torch.LongTensor"),
|
||||
higher_is_better=False,
|
||||
)
|
||||
search_objectives.add_objective("Budget Value", EvaluationFunction(lambda m, d, b: b), higher_is_better=True)
|
||||
search_objectives.add_objective("Budget Value", EvaluationFunction(lambda m, b: b), higher_is_better=True)
|
||||
|
||||
# Assert that objectives are evaluated and return a dictionary
|
||||
result = search_objectives.eval_all_objs(
|
||||
models, None, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
models, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
)
|
||||
assert all(len(r) == len(models) for r in result.values())
|
||||
|
||||
|
||||
def test_eval_subsets(sample_input, models):
|
||||
num_params_obj = TorchNumParameters()
|
||||
num_params = [num_params_obj.evaluate(m, None, None) for m in models]
|
||||
num_params = [num_params_obj.evaluate(m) for m in models]
|
||||
max_params = max(num_params)
|
||||
|
||||
search_objectives = SearchObjectives(cache_objective_evaluation=False)
|
||||
|
|
@ -73,22 +73,22 @@ def test_eval_subsets(sample_input, models):
|
|||
higher_is_better=False,
|
||||
)
|
||||
search_objectives.add_constraint("NumParameters", TorchNumParameters(), (max_params - 0.5, max_params + 0.5))
|
||||
search_objectives.add_objective("Budget Value", EvaluationFunction(lambda m, d, b: b), higher_is_better=True)
|
||||
search_objectives.add_objective("Budget Value", EvaluationFunction(lambda m, b: b), higher_is_better=True)
|
||||
|
||||
# Assert that cheap objectives are evaluated and return a dictionary
|
||||
result = search_objectives.eval_cheap_objs(
|
||||
models, None, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
models, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
)
|
||||
assert set(result.keys()) == {"Flops"}
|
||||
|
||||
# Assert that constraints are valid
|
||||
c_values, c_indices = search_objectives.validate_constraints(models, None)
|
||||
c_values, c_indices = search_objectives.validate_constraints(models)
|
||||
assert len(c_values) == 2
|
||||
assert len(c_indices) == 1
|
||||
|
||||
# Assert that expensive objectives are evaluated and return a dictionary
|
||||
result = search_objectives.eval_expensive_objs(
|
||||
models, None, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
models, budgets={"Budget Value": list(range(len(models)))}, progress_bar=True
|
||||
)
|
||||
assert set(result.keys()) == {"OnnxLatency", "Budget Value"}
|
||||
|
||||
|
|
@ -108,19 +108,19 @@ def test_eval_cache(sample_input, models):
|
|||
higher_is_better=False,
|
||||
)
|
||||
search_objectives.add_constraint("NumberOfParameters", TorchNumParameters(), (0, float("inf")))
|
||||
search_objectives.add_constraint("Random number", EvaluationFunction(lambda m, d, b: random.random()), (0.0, 1.0))
|
||||
search_objectives.add_constraint("Random number", EvaluationFunction(lambda m, b: random.random()), (0.0, 1.0))
|
||||
|
||||
# Assert that cheap objectives are evaluated and cached
|
||||
result = search_objectives.eval_cheap_objs(models, None, progress_bar=True)
|
||||
result = search_objectives.eval_cheap_objs(models, progress_bar=True)
|
||||
assert len(result) == 1
|
||||
assert ("Flops", models[0].archid, "NoneType", None) in search_objectives.cache
|
||||
assert search_objectives.lookup_cache("Flops", models[0].archid, None)
|
||||
|
||||
assert search_objectives.is_model_valid(models[0], None)
|
||||
assert ("NumberOfParameters", models[0].archid, "NoneType", None) in search_objectives.cache
|
||||
assert ("Random number", models[0].archid, "NoneType", None) in search_objectives.cache
|
||||
assert search_objectives.is_model_valid(models[0])
|
||||
assert search_objectives.lookup_cache("NumberOfParameters", models[0].archid, None)
|
||||
assert search_objectives.lookup_cache("Random number", models[0].archid, None)
|
||||
|
||||
# Assert that cached value is correct and constraints are valid
|
||||
cached_val = search_objectives.cache[("Random number", models[0].archid, "NoneType", None)]
|
||||
cons_vals, cons_filtered = search_objectives.validate_constraints(models, None, False)
|
||||
cached_val = search_objectives.lookup_cache("Random number", models[0].archid, None)
|
||||
cons_vals, cons_filtered = search_objectives.validate_constraints(models, False)
|
||||
assert cons_vals["Random number"][0] == cached_val
|
||||
assert len(cons_filtered) == len(models)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ def test_add_iteration_results():
|
|||
search_results = SearchResults(search_space, objectives)
|
||||
models = [ArchaiModel(torch.nn.Linear(10, 1), "archid")]
|
||||
|
||||
obj_name = list(objectives.objs.keys())[0]
|
||||
obj_name = objectives.objective_names[0]
|
||||
evaluation_results = {obj_name: np.array([0.5], dtype=np.float32)}
|
||||
search_results.add_iteration_results(models, evaluation_results)
|
||||
|
||||
|
|
|
|||
|
|
@ -38,19 +38,19 @@ def model():
|
|||
def test_total_params_proxy(model):
|
||||
# Assert that the number of trainable parameters is correct
|
||||
proxy = TorchNumParameters(trainable_only=True)
|
||||
num_params = proxy.evaluate(model, None)
|
||||
num_params = proxy.evaluate(model)
|
||||
assert num_params == sum(param.numel() for param in model.arch.parameters() if param.requires_grad)
|
||||
|
||||
# Assert that the number of all parameters is correct
|
||||
proxy = TorchNumParameters(trainable_only=False)
|
||||
num_params = proxy.evaluate(model, None)
|
||||
num_params = proxy.evaluate(model)
|
||||
assert num_params == sum(param.numel() for param in model.arch.parameters())
|
||||
|
||||
|
||||
def test_non_embedding_params_proxy(model):
|
||||
# Assert that the number of non-embedding trainable parameters is correct
|
||||
proxy = NonEmbeddingParamsProxy(trainable_only=True)
|
||||
non_embedding_params = proxy.evaluate(model, None)
|
||||
non_embedding_params = proxy.evaluate(model)
|
||||
embedding_params = sum(param.numel() for param in model.arch.embd.parameters() if param.requires_grad)
|
||||
assert non_embedding_params + embedding_params == sum(
|
||||
param.numel() for param in model.arch.parameters() if param.requires_grad
|
||||
|
|
@ -58,6 +58,6 @@ def test_non_embedding_params_proxy(model):
|
|||
|
||||
# Assert that the number of non-embedding parameters is correct
|
||||
proxy = NonEmbeddingParamsProxy(trainable_only=False)
|
||||
non_embedding_params = proxy.evaluate(model, None)
|
||||
non_embedding_params = proxy.evaluate(model)
|
||||
embedding_params = sum(param.numel() for param in model.arch.embd.parameters())
|
||||
assert non_embedding_params + embedding_params == sum(param.numel() for param in model.arch.parameters())
|
||||
|
|
|
|||
|
|
@ -21,5 +21,5 @@ def test_transformer_flex_onnx_latency(search_space):
|
|||
objective = TransformerFlexOnnxLatency(search_space)
|
||||
|
||||
# Assert that the returned latency is valid
|
||||
latency = objective.evaluate(arch, None)
|
||||
latency = objective.evaluate(arch)
|
||||
assert latency > 0.0
|
||||
|
|
|
|||
|
|
@ -21,5 +21,5 @@ def test_transformer_flex_onnx_memory(search_space):
|
|||
objective = TransformerFlexOnnxMemory(search_space)
|
||||
|
||||
# Assert that the returned memory is valid
|
||||
memory = objective.evaluate(arch, None)
|
||||
memory = objective.evaluate(arch)
|
||||
assert memory > 0.0
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ from archai.discrete_search.evaluators.functional import EvaluationFunction
|
|||
|
||||
|
||||
def test_evaluation_function():
|
||||
evaluator = EvaluationFunction(lambda a, d, b: 1)
|
||||
evaluator = EvaluationFunction(lambda a, b: 1)
|
||||
|
||||
# Assert that evaluator can evaluate the argument function
|
||||
value = evaluator.evaluate(None, None, None)
|
||||
value = evaluator.evaluate(None, None)
|
||||
assert isinstance(evaluator.evaluation_fn, Callable)
|
||||
assert value == 1
|
||||
|
|
|
|||
|
|
@ -34,48 +34,48 @@ def sample_input():
|
|||
|
||||
def test_torch_num_params(models):
|
||||
torch_num_params = TorchNumParameters()
|
||||
num_params = [torch_num_params.evaluate(model, None, None) for model in models]
|
||||
num_params = [torch_num_params.evaluate(model) for model in models]
|
||||
assert all(p > 0 for p in num_params)
|
||||
|
||||
torch_num_params2 = TorchNumParameters(exclude_cls=[torch.nn.BatchNorm2d])
|
||||
num_params2 = [torch_num_params2.evaluate(model, None, None) for model in models]
|
||||
num_params2 = [torch_num_params2.evaluate(model) for model in models]
|
||||
assert all(p > 0 for p in num_params2)
|
||||
|
||||
torch_num_params3 = TorchNumParameters(exclude_cls=[torch.nn.BatchNorm2d], trainable_only=False)
|
||||
num_params3 = [torch_num_params3.evaluate(model, None, None) for model in models]
|
||||
num_params3 = [torch_num_params3.evaluate(model) for model in models]
|
||||
assert all(p > 0 for p in num_params3)
|
||||
|
||||
|
||||
def test_torch_flops(models, sample_input):
|
||||
torch_flops = TorchFlops(forward_args=sample_input)
|
||||
flops = [torch_flops.evaluate(model, None, None) for model in models]
|
||||
flops = [torch_flops.evaluate(model) for model in models]
|
||||
assert all(f > 0 for f in flops)
|
||||
|
||||
|
||||
def test_torch_macs(models, sample_input):
|
||||
torch_macs = TorchMacs(forward_args=sample_input)
|
||||
macs = [torch_macs.evaluate(model, None, None) for model in models]
|
||||
macs = [torch_macs.evaluate(model) for model in models]
|
||||
assert all(m > 0 for m in macs)
|
||||
|
||||
|
||||
def test_torch_latency(models, sample_input):
|
||||
torch_latency = TorchLatency(forward_args=sample_input, num_warmups=2, num_samples=2)
|
||||
latency = [torch_latency.evaluate(model, None, None) for model in models]
|
||||
latency = [torch_latency.evaluate(model) for model in models]
|
||||
assert all(lt > 0 for lt in latency)
|
||||
|
||||
torch_latency2 = TorchLatency(forward_args=sample_input, num_warmups=0, num_samples=3, use_median=True)
|
||||
latency2 = [torch_latency2.evaluate(model, None, None) for model in models]
|
||||
latency2 = [torch_latency2.evaluate(model) for model in models]
|
||||
assert all(lt > 0 for lt in latency2)
|
||||
|
||||
|
||||
def test_torch_peak_cuda_memory(models, sample_input):
|
||||
if torch.cuda.is_available():
|
||||
torch_peak_memory = TorchPeakCudaMemory(forward_args=sample_input)
|
||||
peak_memory = [torch_peak_memory.evaluate(model, None, None) for model in models]
|
||||
peak_memory = [torch_peak_memory.evaluate(model) for model in models]
|
||||
assert all(m > 0 for m in peak_memory)
|
||||
|
||||
|
||||
def test_torch_peak_cpu_memory(models, sample_input):
|
||||
torch_peak_memory = TorchPeakCpuMemory(forward_args=sample_input)
|
||||
peak_memory = [torch_peak_memory.evaluate(model, None, None) for model in models]
|
||||
peak_memory = [torch_peak_memory.evaluate(model) for model in models]
|
||||
assert all(m > 0 for m in peak_memory)
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@ def test_get_pareto_frontier():
|
|||
}
|
||||
|
||||
objectives = SearchObjectives()
|
||||
objectives.add_objective("obj1", EvaluationFunction(lambda m, d, b: b), higher_is_better=True)
|
||||
objectives.add_objective("obj2", EvaluationFunction(lambda m, d, b: b), higher_is_better=False)
|
||||
objectives.add_objective("obj1", EvaluationFunction(lambda m, b: b), higher_is_better=True)
|
||||
objectives.add_objective("obj2", EvaluationFunction(lambda m, b: b), higher_is_better=False)
|
||||
|
||||
result = get_pareto_frontier(models, evaluation_results, objectives)
|
||||
|
||||
|
|
@ -48,8 +48,8 @@ def test_get_non_dominated_sorting():
|
|||
}
|
||||
|
||||
objectives = SearchObjectives()
|
||||
objectives.add_objective("obj1", EvaluationFunction(lambda m, d, b: b), higher_is_better=True)
|
||||
objectives.add_objective("obj2", EvaluationFunction(lambda m, d, b: b), higher_is_better=False)
|
||||
objectives.add_objective("obj1", EvaluationFunction(lambda m, b: b), higher_is_better=True)
|
||||
objectives.add_objective("obj2", EvaluationFunction(lambda m, b: b), higher_is_better=False)
|
||||
|
||||
result = get_non_dominated_sorting(models, evaluation_results, objectives)
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче