From 0562f2c09df7a558f76f5f84417b054d1ced431a Mon Sep 17 00:00:00 2001 From: Debadeepta Dey Date: Sun, 11 Apr 2021 20:14:10 -0700 Subject: [PATCH] cifar10 results after finding speed diff on clusters. --- .vscode/launch.json | 8 +-- archai/algos/natsbench/natsbench_utils.py | 52 ++++++++++++++++ archai/algos/random_natsbench/__init__.py | 0 .../random_natsbench_tss_exp_runner.py | 40 +++++++++++++ .../random_natsbench_tss_searcher.py | 44 ++++++++++++++ confs/algos/natsbench_regular_eval.yaml | 4 +- confs/algos/proxynas_natsbench_space.yaml | 4 +- run_all_ft_analysis.bat | 59 ++++++++++++++----- .../proxynas_plots/cross_exp_conf.yaml | 36 ++++++++++- 9 files changed, 221 insertions(+), 26 deletions(-) create mode 100644 archai/algos/natsbench/natsbench_utils.py create mode 100644 archai/algos/random_natsbench/__init__.py create mode 100644 archai/algos/random_natsbench/random_natsbench_tss_exp_runner.py create mode 100644 archai/algos/random_natsbench/random_natsbench_tss_searcher.py diff --git a/.vscode/launch.json b/.vscode/launch.json index d4dfaecf..5d1274a5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -228,7 +228,7 @@ "request": "launch", "program": "${cwd}/scripts/main.py", "console": "integratedTerminal", - "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "flower102"] + "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "ImageNet16-120"] }, { "name": "Proxynas-Natsbench-Space-Toy", @@ -655,7 +655,7 @@ "request": "launch", "program": "${cwd}/scripts/reports/proxynas_plots/cross_exp_plots.py", "console": "integratedTerminal", - "args": ["--dataset", "natsbench_flower102", + "args": ["--dataset", "natsbench_cifar10", "--conf-location", "scripts/reports/proxynas_plots/cross_exp_conf.yaml"] }, { @@ -664,8 +664,8 @@ "request": "launch", "program": "${cwd}/scripts/reports/analysis_natsbench_zerocost.py", "console": "integratedTerminal", - "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\zc_synthetic_cifar10", - "--out-dir", "D:\\archai_experiment_reports"] + "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\zc_flower102", + "--out-dir", "D:\\archai_experiment_reports", "--reg-evals-file", "D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml"] }, { "name": "Analysis Zero Cost Conditional Natsbench Experiments", diff --git a/archai/algos/natsbench/natsbench_utils.py b/archai/algos/natsbench/natsbench_utils.py new file mode 100644 index 00000000..e7ba438b --- /dev/null +++ b/archai/algos/natsbench/natsbench_utils.py @@ -0,0 +1,52 @@ +from copy import deepcopy +from typing import Optional +import importlib +import sys +import string +import os + +from overrides import overrides + +import torch +from torch import nn + +from overrides import overrides, EnforceOverrides + +from archai.common.trainer import Trainer +from archai.common.config import Config +from archai.common.common import logger +from archai.datasets import data +from archai.nas.model_desc import ModelDesc +from archai.nas.model_desc_builder import ModelDescBuilder +from archai.nas import nas_utils +from archai.common import ml_utils, utils +from archai.common.metrics import EpochMetrics, Metrics +from archai.nas.model import Model +from archai.common.checkpoint import CheckPoint +from archai.nas.evaluater import Evaluater +from archai.algos.proxynas.freeze_trainer import FreezeTrainer +from archai.algos.proxynas.conditional_trainer import ConditionalTrainer + +from nats_bench import create +from archai.algos.natsbench.lib.models import get_cell_based_tiny_net + + + +def model_from_natsbench_tss(arch_index:int, dataset_name:str, natsbench_location:str)->Model: + + # create natsbench api + api = create(natsbench_location, 'tss', fast_mode=True, verbose=True) + + if arch_index > 15625 or arch_index < 0: + logger.warn(f'architecture id {arch_index} is invalid ') + + supported_datasets = {'cifar10', 'cifar100', 'ImageNet16-120'} + + if dataset_name not in supported_datasets: + raise NotImplementedError + + config = api.get_net_config(arch_index, dataset_name) + + model = get_cell_based_tiny_net(config) + + return model \ No newline at end of file diff --git a/archai/algos/random_natsbench/__init__.py b/archai/algos/random_natsbench/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/archai/algos/random_natsbench/random_natsbench_tss_exp_runner.py b/archai/algos/random_natsbench/random_natsbench_tss_exp_runner.py new file mode 100644 index 00000000..bb836287 --- /dev/null +++ b/archai/algos/random_natsbench/random_natsbench_tss_exp_runner.py @@ -0,0 +1,40 @@ + + +import overrides +from typing import Optional, Type, Tuple + +from archai.nas.exp_runner import ExperimentRunner +from archai.nas.model_desc_builder import ModelDescBuilder +from archai.nas.arch_trainer import TArchTrainer +from archai.common import common +from archai.common import utils +from archai.common.config import Config +from archai.nas.evaluater import Evaluater, EvalResult +from archai.nas.searcher import Searcher, SearchResult +from archai.nas.finalizers import Finalizers +from archai.nas.random_finalizers import RandomFinalizers +from archai.nas.model_desc_builder import ModelDescBuilder + + +class RandomNatsbenchTssExpRunner(ExperimentRunner): + + @overrides + def run_search(self, conf_search:Config)->SearchResult: + search = self.searcher() + return search.search(conf_search) + + @overrides + def run_eval(self, conf_eval:Config)->EvalResult: + evaler = self.evaluater() + return evaler.evaluate(conf_eval) + + @overrides + def searcher(self)->Searcher: + return RandomNatsbenchSearcher() + + @overrides + def evaluater(self)->Evaluater: + return RandomNatsbenchEvaluater() + + + diff --git a/archai/algos/random_natsbench/random_natsbench_tss_searcher.py b/archai/algos/random_natsbench/random_natsbench_tss_searcher.py new file mode 100644 index 00000000..284a6d6e --- /dev/null +++ b/archai/algos/random_natsbench/random_natsbench_tss_searcher.py @@ -0,0 +1,44 @@ +import math as ma +from typing import Set + +from archai.nas.searcher import Searcher, SearchResult +from archai.common.config import Config +from archai.nas.model_desc_builder import ModelDescBuilder +from archai.nas.arch_trainer import TArchTrainer +from archai.common.trainer import Trainer +from archai.nas.model_desc import CellType, ModelDesc +from archai.datasets import data +from archai.nas.model import Model +from archai.common.metrics import EpochMetrics, Metrics +from archai.common import utils +from archai.nas.finalizers import Finalizers +from archai.algos.proxynas.conditional_trainer import ConditionalTrainer +from archai.algos.proxynas.freeze_trainer import FreezeTrainer +from archai.algos.natsbench.natsbench_utils import model_from_natsbench_tss + +class RandomNatsbenchTssSearcher(Searcher): + def search(conf_search:Config)->SearchResult: + + # region config vars + max_num_models = conf_search['max_num_models'] + ratio_fastest_duration = conf_search['ratio_fastest_duration'] + top1_acc_threshold = conf_search['top1_acc_threshold'] + # endregion + + counter = 0 + best_trains = [-ma.Inf] + fastest_cond_train = ma.Inf + archids_sampled = set() + while counter < max_num_models: + # sample a random model from tss + + # if during conditional training it + # starts exceeding fastest time to + # reach threshold by a ratio then early + # terminate it + + # + + + + diff --git a/confs/algos/natsbench_regular_eval.yaml b/confs/algos/natsbench_regular_eval.yaml index 9e9f797c..b3faa262 100644 --- a/confs/algos/natsbench_regular_eval.yaml +++ b/confs/algos/natsbench_regular_eval.yaml @@ -12,7 +12,7 @@ nas: model_desc: num_edges_to_sample: 2 loader: - train_batch: 256 # natsbench uses 256 + train_batch: 1024 # natsbench uses 256 aug: '' # random flip and crop are already there in default params trainer: # matching natsbench paper closely plotsdir: '' @@ -24,7 +24,7 @@ nas: l1_alphas: 0.0 # weight to be applied to sum(abs(alphas)) to loss term logger_freq: 1000 # after every N updates dump loss and other metrics in logger title: 'eval_train' - epochs: 18 + epochs: 30 batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM lossfn: type: 'CrossEntropyLoss' diff --git a/confs/algos/proxynas_natsbench_space.yaml b/confs/algos/proxynas_natsbench_space.yaml index bc77768d..1d3ca716 100644 --- a/confs/algos/proxynas_natsbench_space.yaml +++ b/confs/algos/proxynas_natsbench_space.yaml @@ -23,7 +23,7 @@ nas: trainer: plotsdir: '' use_val: False - top1_acc_threshold: 0.2 # after some accuracy we will shift into training only the last 'n' layers + top1_acc_threshold: 0.1 # after some accuracy we will shift into training only the last 'n' layers apex: _copy: '/common/apex' aux_weight: '_copy: /nas/eval/model_desc/aux_weight' @@ -61,7 +61,7 @@ nas: l1_alphas: 0.0 # weight to be applied to sum(abs(alphas)) to loss term logger_freq: 1000 # after every N updates dump loss and other metrics in logger title: 'eval_train' - epochs: 15 + epochs: 10 batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM lossfn: type: 'CrossEntropyLoss' diff --git a/run_all_ft_analysis.bat b/run_all_ft_analysis.bat index 8b5ec983..5fbb9ede 100644 --- a/run_all_ft_analysis.bat +++ b/run_all_ft_analysis.bat @@ -53,6 +53,16 @@ @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb256_ftlr0.1_fte15_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports + +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports + +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports +python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports + + @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e01 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e02 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e04 --out-dir D:\\archai_experiment_reports @@ -74,6 +84,25 @@ @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e08 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e10 --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e01_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e02_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e04_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e06_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e08_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e10_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e20_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e30_scu --out-dir D:\\archai_experiment_reports + +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e01_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e02_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e04_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e06_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e08_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e10_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e20_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e30_scu --out-dir D:\\archai_experiment_reports + + @REM cifar100 @REM python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\ft_c100_fb1024_ftlr0.1_fte5_ct256_ftt0.3 --out-dir D:\\archai_experiment_reports @@ -190,24 +219,22 @@ python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\arch @REM Flower 102 -@REM ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2 -@REM ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2 -@REM ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2 -@REM ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2 +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -@REM ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2 -@REM ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2 -@REM ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2 -@REM ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2 +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml @REM Nasbench 101 diff --git a/scripts/reports/proxynas_plots/cross_exp_conf.yaml b/scripts/reports/proxynas_plots/cross_exp_conf.yaml index f14c5613..7ef7cd90 100644 --- a/scripts/reports/proxynas_plots/cross_exp_conf.yaml +++ b/scripts/reports/proxynas_plots/cross_exp_conf.yaml @@ -17,15 +17,21 @@ natsbench_cifar10: # ft_fb2048_ftlr1.5_fte10_ct256_ftt0.3: 'fastarchrank stage 2: batch 2048, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.3' # ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 1.5,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.6' + # ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 1.5,
epochs 5, stage 1: batch 256, thresh 0.6 scu' + # ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.6 scu' # ft_fb512_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 1.5,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb512_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.6' # ft_fb256_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 1.5,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb256_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 1.5,
epochs 10, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' + ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6 scu' + ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6 scu' # ft_fb512_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' - ft_fb256_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' + # ft_fb512_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6 scu' + # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6 scu' + # ft_fb256_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6' # ft_fb256_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.6' # ft_fb256_ftlr0.1_fte15_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.6' # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c9: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.6, frozen till: cell 9' @@ -49,6 +55,14 @@ natsbench_cifar10: nb_reg_b1024_e10: 'regular train: batch 1024 epochs 10' nb_reg_b1024_e20: 'regular train: batch 1024 epochs 20' nb_reg_b1024_e30: 'regular train: batch 1024 epochs 30' + nb_reg_b1024_e01_scu: 'regular train: batch 1024 epochs 01 scu' + nb_reg_b1024_e02_scu: 'regular train: batch 1024 epochs 02 scu' + nb_reg_b1024_e04_scu: 'regular train: batch 1024 epochs 04 scu' + nb_reg_b1024_e06_scu: 'regular train: batch 1024 epochs 06 scu' + nb_reg_b1024_e08_scu: 'regular train: batch 1024 epochs 08 scu' + nb_reg_b1024_e10_scu: 'regular train: batch 1024 epochs 10 scu' + nb_reg_b1024_e20_scu: 'regular train: batch 1024 epochs 20 scu' + nb_reg_b1024_e30_scu: 'regular train: batch 1024 epochs 30 scu' nb_reg_b512_e01: 'regular train: batch 512 epochs 01' nb_reg_b512_e02: 'regular train: batch 512 epochs 02' nb_reg_b512_e04: 'regular train: batch 512 epochs 04' @@ -61,6 +75,15 @@ natsbench_cifar10: nb_reg_b256_e06: 'regular train: batch 256 epochs 06' nb_reg_b256_e08: 'regular train: batch 256 epochs 08' nb_reg_b256_e10: 'regular train: batch 256 epochs 10' + nb_reg_b256_e01_scu: 'regular train: batch 256 epochs 01 scu' + nb_reg_b256_e02_scu: 'regular train: batch 256 epochs 02 scu' + nb_reg_b256_e04_scu: 'regular train: batch 256 epochs 04 scu' + nb_reg_b256_e06_scu: 'regular train: batch 256 epochs 06 scu' + nb_reg_b256_e08_scu: 'regular train: batch 256 epochs 08 scu' + nb_reg_b256_e10_scu: 'regular train: batch 256 epochs 10 scu' + nb_reg_b256_e20_scu: 'regular train: batch 256 epochs 20 scu' + nb_reg_b256_e30_scu: 'regular train: batch 256 epochs 30 scu' + colors: @@ -175,12 +198,21 @@ natsbench_flower102: ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.2' ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1,
epochs 30, stage 1: batch 256, thresh 0.2' - + ft_f102_fb512_ftlr0.1_fte5_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.2' + ft_f102_fb512_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + ft_f102_fb512_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.2' + ft_f102_fb256_ftlr0.1_fte5_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 5, stage 1: batch 256, thresh 0.2' + ft_f102_fb256_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 10, stage 1: batch 256, thresh 0.2' + ft_f102_fb256_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1,
epochs 15, stage 1: batch 256, thresh 0.2' + shortreg: nb_f102_reg_b256_e10: 'regular train: batch 2048 epochs 10' nb_f102_reg_b256_e20: 'regular train: batch 2048 epochs 20' nb_f102_reg_b256_e30: 'regular train: batch 2048 epochs 30' + zero_cost: + zc_flower102: 'zero cost score at initialization' + colors: freezetrain: 'red' zero_cost: 'green'