diff --git a/.amltignore b/.amltignore index 33f6a648..7351cc6a 100644 --- a/.amltignore +++ b/.amltignore @@ -1,10 +1,5 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -# Folders to be ignored when using AMLT -devices/ -dockers/ -docs/ -scripts/ tests/ -tools/ \ No newline at end of file +tools/ +archived/ +dockers/ +docs/ \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 18687c66..4fd2e451 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -803,8 +803,9 @@ "request": "launch", "program": "${cwd}/scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py", "console": "integratedTerminal", - "args": ["--results-dir", "F:\\archaiphilly\\phillytools\\ls_far_ftt0.6_max300_ratio2.0_fixedseeds", - "--out-dir", "F:\\archai_experiment_reports"] + "args": ["--results-dir", "F:\\archaiphilly\\phillytools\\ls_far_c100_ftt0.3_max300_ratio4.0_fixedseeds", + "--out-dir", "F:\\archai_experiment_reports", "--natsbench_loc", "C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple", + "--dataset", "cifar100"] }, { "name": "CurrentFile", diff --git a/confs/algos/local_natsbench_tss_far.yaml b/confs/algos/local_natsbench_tss_far.yaml index 1d63a0dc..014c8c57 100644 --- a/confs/algos/local_natsbench_tss_far.yaml +++ b/confs/algos/local_natsbench_tss_far.yaml @@ -44,7 +44,7 @@ dataset: {} # default dataset settings comes from __include__ on the top nas: search: - use_fear: False + use_fear: True max_num_models: 300 ratio_fastest_duration: 1.2 natsbench: @@ -62,9 +62,9 @@ nas: aug: '' # additional augmentations to use cutout: 0 # cutout length, use cutout augmentation when > 0 load_train: True # load train split of dataset - train_batch: 1024 + train_batch: 256 freeze_loader: - train_batch: 1024 # batch size for freeze training. + train_batch: 512 # batch size for freeze training. train_workers: 4 # if null then gpu_count*4 test_workers: '_copy: ../train_workers' # if null then 4 load_test: False # load test split of dataset @@ -84,7 +84,7 @@ nas: grad_clip: 5.0 # grads above this value is clipped logger_freq: 1000 # after every N updates dump loss and other metrics in logger title: 'arch_train' - epochs: 2 + epochs: 200 batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM # additional vals for the derived class plotsdir: '' #empty string means no plots, other wise plots are generated for each epoch in this dir diff --git a/run_all_ft_analysis.bat b/run_all_ft_analysis.bat index 241f460f..ca51bfc4 100644 --- a/run_all_ft_analysis.bat +++ b/run_all_ft_analysis.bat @@ -54,7 +54,7 @@ @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb256_ftlr0.1_fte15_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports -python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports +@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports @@ -354,3 +354,34 @@ python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\arch @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb101_reg_b256_e30 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb101_reg_b256_e108 --out-dir D:\\archai_experiment_reports @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb101_reg_b256_e108_rms --out-dir D:\\archai_experiment_reports + + +@REM Local Search + +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_ftt0.6_max300_ratio2.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_ftt0.6_max300_ratio2.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_ftt0.6_max300_ratio2.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 + +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_max300_b1024_e02_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_max300_b1024_e20_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_max300_b1024_e30_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_max300_b1024_e40_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_max300_b1024_e50_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar10 + +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_c100_ftt0.3_max300_ratio2.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_c100_ftt0.3_max300_ratio4.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_c100_ftt0.3_max300_ratio8.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_c100_max300_b1024_e02_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_c100_max300_b1024_e20_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_c100_max300_b1024_e30_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_c100_max300_b1024_e40_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_c100_max300_b1024_e50_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset cifar100 + +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_i16_ftt0.2_max300_ratio2.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_i16_ftt0.2_max300_ratio4.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_far_i16_ftt0.2_max300_ratio8.0_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_i16_max300_b1024_e02_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_i16_max300_b1024_e20_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_i16_max300_b1024_e30_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_i16_max300_b1024_e40_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 +@REM python scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py --results-dir F:\\archaiphilly\\phillytools\\ls_reg_i16_max300_b1024_e50_fixedseeds --out-dir F:\\archai_experiment_reports --natsbench_loc C:\\Users\\dedey\\dataroot\\natsbench\\NATS-tss-v1_0-3ffb9-simple --dataset ImageNet16-120 diff --git a/scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py b/scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py index 018282e0..ad65e548 100644 --- a/scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py +++ b/scripts/reports/fear_analysis/analysis_local_search_natsbench_tss.py @@ -30,7 +30,7 @@ import multiprocessing from multiprocessing import Pool from archai.common import utils -from archai.common.ordereddict_logger import OrderedDictLogger +from archai.algos.natsbench.natsbench_utils import create_natsbench_tss_api from archai.common.analysis_utils import epoch_nodes, parse_a_job, fix_yaml, remove_seed_part, group_multi_runs, collect_epoch_nodes, EpochStats, FoldStats, stat2str, get_epoch_stats, get_summary_text, get_details_text, plot_epochs, write_report import re @@ -43,6 +43,10 @@ def main(): help='folder with experiment results from pt') parser.add_argument('--out-dir', '-o', type=str, default=r'~/logdir/reports', help='folder to output reports') + parser.add_argument('--natsbench_loc', '-n', type=str, + help='location of natsbench') + parser.add_argument('--dataset', type=str, + help='dataset name') args, extra_args = parser.parse_known_args() # root dir where all results are stored @@ -77,6 +81,11 @@ def main(): logs[key] = val[0] confs[key] = val[1] + # create the natsbench api + api = create_natsbench_tss_api(args.natsbench_loc) + + ds_name = args.dataset + raw_data = {} @@ -84,10 +93,15 @@ def main(): # Get total duration of the run # which is the sum of all conditional and freeze # trainings over all architectures + # or all regular runs if it is shortened regular evaluation duration = 0.0 arch_ids_touched = [] + # best test accuracy as function of time + best_train_till_now = -ma.inf + best_test_duration_storage = [] + archs_touched_counter = 0 for skey in logs[key].keys(): if 'conditional' in skey or 'freeze' in skey: @@ -98,11 +112,28 @@ def main(): for ekey in logs[key][skey][train_key]['epochs'].keys(): eduration = logs[key][skey][train_key]['epochs'][ekey]['train']['duration'] duration += eduration + + train_acc = logs[key][skey][train_key]['best_train']['top1'] + if train_acc >= best_train_till_now: + best_train_till_now = train_acc + arch_id = int(skey.split('_')[-1]) + info = api.get_more_info(arch_id, ds_name, hp=200, is_random=False) + test_acc = info['test-accuracy'] + best_test_duration_storage.append((duration, test_acc, train_acc)) + elif 'regular_training' in skey: for ekey in logs[key][skey]['arch_train']['epochs'].keys(): eduration = logs[key][skey]['arch_train']['epochs'][ekey]['train']['duration'] duration += eduration - + + train_acc = logs[key][skey]['arch_train']['best_train']['top1'] + if train_acc >= best_train_till_now: + best_train_till_now = train_acc + arch_id = int(skey.split('_')[-1]) + info = api.get_more_info(arch_id, ds_name, hp=200, is_random=False) + test_acc = info['test-accuracy'] + best_test_duration_storage.append((duration, test_acc, train_acc)) + if 'conditional' in skey or 'freeze' in skey or 'regular_training' in skey: arch_id = int(skey.split('_')[-1]) if arch_id not in arch_ids_touched: @@ -111,9 +142,9 @@ def main(): # find the test error of the best local minima (by train error) - best_test = logs[key]['best_minimum'][2] - - raw_data[key] = (duration, best_test) + best_test_local = logs[key]['best_minimum'][2] + best_test = best_test_duration_storage[-1][1] + raw_data[key] = (duration, best_test, best_test_duration_storage) run_durations = [raw_data[key][0] for key in raw_data.keys()] max_accs = [raw_data[key][1] for key in raw_data.keys()] @@ -124,11 +155,15 @@ def main(): avg_max_acc = statistics.mean(max_accs) stderr_max_acc = sem(np.array(max_accs)) + trajs = [raw_data[x][2] for x in raw_data.keys()] + data_to_save = {} data_to_save['avg_duration'] = avg_duration data_to_save['stderr_duration'] = stderr_duration data_to_save['avg_max_acc'] = avg_max_acc data_to_save['stderr_max_acc'] = stderr_max_acc + data_to_save['trajs'] = trajs + savename = os.path.join(out_dir, 'raw_data.yaml') with open(savename, 'w') as f: diff --git a/scripts/reports/fear_plots/cross_local_search.yaml b/scripts/reports/fear_plots/cross_local_search.yaml index 7c895604..3f71bd76 100644 --- a/scripts/reports/fear_plots/cross_local_search.yaml +++ b/scripts/reports/fear_plots/cross_local_search.yaml @@ -6,11 +6,54 @@ natsbench_cifar10: fastarchrank: ls_far_ftt0.6_max300_ratio2.0_fixedseeds: 'ls_far_ftt0.6_max300_ratio2.0_fixedseeds' - + ls_far_ftt0.6_max300_ratio4.0_fixedseeds: 'ls_far_ftt0.6_max300_ratio4.0_fixedseeds' + ls_far_ftt0.6_max300_ratio8.0_fixedseeds: 'ls_far_ftt0.6_max300_ratio8.0_fixedseeds' + regular: ls_reg_max300_b1024_e02_fixedseeds: 'ls_reg_max300_b1024_e02_fixedseeds' + ls_reg_max300_b1024_e20_fixedseeds: 'ls_reg_max300_b1024_e20_fixedseeds' + ls_reg_max300_b1024_e30_fixedseeds: 'ls_reg_max300_b1024_e30_fixedseeds' + ls_reg_max300_b1024_e40_fixedseeds: 'ls_reg_max300_b1024_e40_fixedseeds' + ls_reg_max300_b1024_e50_fixedseeds: 'ls_reg_max300_b1024_e50_fixedseeds' + + colors: + fastarchrank: 'green' + regular: 'red' + + +natsbench_cifar100: + + fastarchrank: + ls_far_c100_ftt0.3_max300_ratio2.0_fixedseeds: 'ls_far_c100_ftt0.3_max300_ratio2.0_fixedseeds' + ls_far_c100_ftt0.3_max300_ratio4.0_fixedseeds: 'ls_far_c100_ftt0.3_max300_ratio4.0_fixedseeds' + ls_far_c100_ftt0.3_max300_ratio8.0_fixedseeds: 'ls_far_c100_ftt0.3_max300_ratio8.0_fixedseeds' + + regular: + ls_reg_c100_max300_b1024_e02_fixedseeds: 'ls_reg_c100_max300_b1024_e02_fixedseeds' + ls_reg_c100_max300_b1024_e20_fixedseeds: 'ls_reg_c100_max300_b1024_e20_fixedseeds' + ls_reg_c100_max300_b1024_e30_fixedseeds: 'ls_reg_c100_max300_b1024_e30_fixedseeds' + ls_reg_c100_max300_b1024_e40_fixedseeds: 'ls_reg_c100_max300_b1024_e40_fixedseeds' + ls_reg_c100_max300_b1024_e50_fixedseeds: 'ls_reg_c100_max300_b1024_e50_fixedseeds' colors: fastarchrank: 'green' regular: 'red' + + +natsbench_imagenet16-120: + + fastarchrank: + ls_far_i16_ftt0.2_max300_ratio2.0_fixedseeds: ls_far_i16_ftt0.2_max300_ratio2.0_fixedseeds + ls_far_i16_ftt0.2_max300_ratio4.0_fixedseeds: ls_far_i16_ftt0.2_max300_ratio4.0_fixedseeds + ls_far_i16_ftt0.2_max300_ratio8.0_fixedseeds: ls_far_i16_ftt0.2_max300_ratio8.0_fixedseeds + regular: + ls_reg_i16_max300_b1024_e02_fixedseeds: ls_reg_i16_max300_b1024_e02_fixedseeds + ls_reg_i16_max300_b1024_e20_fixedseeds: ls_reg_i16_max300_b1024_e20_fixedseeds + ls_reg_i16_max300_b1024_e30_fixedseeds: ls_reg_i16_max300_b1024_e30_fixedseeds + ls_reg_i16_max300_b1024_e40_fixedseeds: ls_reg_i16_max300_b1024_e40_fixedseeds + ls_reg_i16_max300_b1024_e50_fixedseeds: ls_reg_i16_max300_b1024_e50_fixedseeds + + colors: + fastarchrank: 'green' + regular: 'red'