Added params and flops based spearman correlation computation.

.vscode/launch.json

@@ -745,7 +745,7 @@
             "request": "launch",
             "program": "${cwd}/scripts/reports/fear_analysis/analysis_create_darts_space_benchmark.py",
             "console": "integratedTerminal",
-            "args": ["--results-dir", "/home/dedey/archaiphilly/amlt/darts_constant_random_ninapro",
+            "args": ["--results-dir", "/home/dedey/archaiphilly/amlt/darts_constant_random_scifar100",
             "--out-dir", "/home/dedey/archai_experiment_reports"]
         },
         {
@@ -820,8 +820,8 @@
             "request": "launch",
             "program": "${cwd}/scripts/reports/fear_analysis/analysis_natsbench_nonstandard_generate_benchmark.py",
             "console": "integratedTerminal",
-            "args": ["--results-dir", "F:\\archaiphilly\\phillytools\\nb_reg_b256_e200_sc10",
-                "--out-dir", "F:\\archai_experiment_reports"]
+            "args": ["--results-dir", "/home/dedey/archaiphilly/amlt/natsbench_constant_random_cifar100",
+                "--out-dir", "/home/dedey/archai_experiment_reports"]
         },
         {
             "name": "Analysis Regular Natsbench TSS, SSS and Nb101 Space",
@@ -868,7 +868,9 @@
             "args": ["--results-dir", "/home/dedey/archaiphilly/amlt/darts_constant_random_zerocost_ninapro",
                 "--out-dir", "/home/dedey/archai_experiment_reports", 
                 "--reg-evals-file", 
-                "/home/dedey/archai_experiment_reports/darts_constant_random_ninapro/darts_benchmark.yaml"]
+                "/home/dedey/archai_experiment_reports/natsbench_constant_random_cifar100/arch_id_test_accuracy.yaml",
+                "--params-flops-file",
+                "/home/dedey/archai_experiment_reports/natsbench_constant_random_cifar100/arch_id_params_flops.yaml"]
         },
         {
             "name": "Analysis Zero Cost Natsbench Epochs Experiments",

scripts/reports/fear_analysis/analysis_create_darts_space_benchmark.py

@@ -70,7 +70,7 @@ def main():
     #     a = parse_a_job(job_dir)
 
     # parallel parsing of yaml logs
-    num_workers = 8
+    num_workers = 48
     with Pool(num_workers) as p:
         a = p.map(parse_a_job, job_dirs)
 
@@ -91,18 +91,20 @@ def main():
 
     archid_testacc = {}
     archid_params = {}
+    archid_flops = {}
     for key in logs.keys():
         if 'eval' in key:
             try:
                 test_acc = logs[key]['regular_evaluate']['eval_arch']['eval_train']['best_test']['top1']
                 arch_id = confs[key]['nas']['eval']['dartsspace']['arch_index']
-                print(f'arch_index {arch_id}, test_acc {test_acc}')
                 archid_testacc[arch_id] = test_acc
-
+                
                 # get the number of params if in logs (most have it unless the early part is missing)
                 if 'num_params' in logs[key]['regular_evaluate']['eval_arch']['eval_train']:
                     num_params = logs[key]['regular_evaluate']['eval_arch']['eval_train']['num_params']
                     archid_params[arch_id] = num_params
+                    mega_flops_per_batch = logs[key]['regular_evaluate']['eval_arch']['eval_train']['mega_flops_per_batch']
+                    archid_flops[arch_id] = mega_flops_per_batch
 
             except KeyError as err:
                 print(f'KeyError {err} not in {key}!')
@@ -116,11 +118,14 @@ def main():
     # to see how the distribution looks
     testaccs = []
     params = []
+    flops = []
     for archid in archid_params.keys():
         num_params = archid_params[archid]
         test_acc = archid_testacc[archid]
+        num_flops = archid_flops[archid]
         testaccs.append(test_acc)
         params.append(num_params)
+        flops.append(num_flops)
 
     fig = go.Figure()
     fig.add_trace(go.Scatter(x=params, y=testaccs, mode='markers'))
@@ -134,8 +139,13 @@ def main():
 
     # compute spearman correlation of #params vs. test accuracy
     param_spe, param_sp_value = spearmanr(testaccs, params)
+    flop_spe, flop_sp_value = spearmanr(testaccs, flops)
     print(f'Spearman correlation of #params vs. test accuracy is {param_spe}')
-
+    print(f'Spearman correlation of #flops vs. test accuracy is {flop_spe}')
+    savename = os.path.join(out_dir, 'darts_space_params_flops_spe.txt')
+    with open(savename, 'w') as f:
+        f.write(f'Spe #params vs. test accuracy: {param_spe}')
+        f.write(f'Spe #flops vs. test accuracy: {flop_spe}')
     
 
 

scripts/reports/fear_analysis/analysis_natsbench_nonstandard_generate_benchmark.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Type, Iterator, Tuple
 import glob
 import os
 import pathlib
-from collections import OrderedDict, defaultdict
+from collections import OrderedDict, defaultdict, namedtuple
 from scipy.stats.stats import _two_sample_transform
 import yaml
 from inspect import getsourcefile
@@ -69,7 +69,11 @@ def main():
     #     a = parse_a_job(job_dir)
 
     # parallel parsing of yaml logs
+<<<<<<< HEAD
     num_workers = 12
+=======
+    num_workers = 48
+>>>>>>> 88356eb6 (Added params and flops based spearman correlation computation.)
     with Pool(num_workers) as p:
         a = p.map(parse_a_job, job_dirs)
 
@@ -104,17 +108,28 @@ def main():
 
     
     # create a dict with arch_id: regular eval score as entries
-    # and save since synthetic cifar10 is not part of the benchmark
+    # and save since synthetic cifar10 or other new datasets 
+    # are not part of the benchmark
     arch_id_reg_eval = {}
+    arch_id_params_flops = {}
+
     for key in logs.keys():
         arch_id = confs[key]['nas']['eval']['natsbench']['arch_index']
         reg_eval = logs[key]['eval_arch']['eval_train']['best_test']['top1']
+        num_params = logs[key]['eval_arch']['eval_train']['num_params']
+        mega_flops_per_batch = logs[key]['eval_arch']['eval_train']['mega_flops_per_batch']
+        # store
         arch_id_reg_eval[arch_id] = reg_eval
+        arch_id_params_flops[arch_id] = {'params': num_params, 'flops': mega_flops_per_batch}
 
     savename = os.path.join(out_dir, 'arch_id_test_accuracy_synthetic_cifar10.yaml')
     with open(savename, 'w') as f:
         yaml.dump(arch_id_reg_eval, f)
 
+    savename = os.path.join(out_dir, 'arch_id_params_flops.yaml')
+    with open(savename, 'w') as f:
+        yaml.dump(arch_id_params_flops, f)
+
     # now create a list of regular evaluation and corresponding synflow scores
     # to compute spearman's correlation
     all_reg_evals = []

scripts/reports/fear_analysis/analysis_natsbench_zerocost.py

@@ -48,6 +48,9 @@ def main():
     parser.add_argument('--reg-evals-file', '-r', type=str, default=None,
                         help='optional yaml file which contains full evaluation \
                             of architectures on new datasets not part of natsbench')
+    parser.add_argument('--params-flops-file', '-p', type=str, default=None,
+                        help='optional yaml file which contains params flops information \
+                            of architectures on new datasets not part of natsbench')
     args, extra_args = parser.parse_known_args()
 
     # root dir where all results are stored
@@ -63,9 +66,16 @@ def main():
     os.makedirs(out_dir, exist_ok=True)
 
     # if optional regular evaluation lookup file is provided
+    reg_evals_data = None
     if args.reg_evals_file:
         with open(args.reg_evals_file, 'r') as f:
             reg_evals_data = yaml.load(f, Loader=yaml.Loader)
+    
+    # if optional params flops lookup file is provided
+    params_flops_data = None
+    if args.params_flops_file:
+        with open(args.params_flops_file, 'r') as f:
+            params_flops_data = yaml.load(f, Loader=yaml.Loader)
 
     # get list of all structured logs for each job
     logs = {}
@@ -78,7 +88,7 @@ def main():
     #     a = parse_a_job(job_dir)
 
     # parallel parsing of yaml logs
-    num_workers = 12
+    num_workers = 20
     with Pool(num_workers) as p:
         a = p.map(parse_a_job, job_dirs)
 
@@ -106,15 +116,16 @@ def main():
             logs.pop(key)
             continue
 
-    
+
     all_arch_ids = []
     all_reg_evals = []
+    all_params_evals = []
+    all_flops_evals = []
     all_zerocost_init_evals = defaultdict(list)
 
     for key in logs.keys():
         if 'eval' in key:
-            try:
-                
+            try:                
                 # regular evaluation 
                 # important to get this first since if it is not 
                 # available for non-benchmark datasets we need to 
@@ -137,7 +148,14 @@ def main():
                         # may not have full evaluation accuracies available. Remove them from consideration.
                         continue
                     reg_eval_top1 = reg_evals_data[arch_id_in_bench]
+
+                    if params_flops_data:
+                        params = params_flops_data[arch_id_in_bench]['params']
+                        flops = params_flops_data[arch_id_in_bench]['flops']
+    
                 all_reg_evals.append(reg_eval_top1)
+                all_params_evals.append(params)
+                all_flops_evals.append(flops)
 
                 # zerocost initial scores
                 #-------------------------------
@@ -158,17 +176,28 @@ def main():
                 print(f'KeyError {err} not in {key}')
                 sys.exit()
 
-    # Store some key numbers in results.txt
-    results_savename = os.path.join(out_dir, 'results.txt')
-    with open(results_savename, 'w') as f:
-        f.write(f'Total valid archs processed: {len(all_reg_evals)} \n')
-
-    print(f'Total valid archs processed: {len(all_reg_evals)}')
-
     # Sanity check
     for measure in ZEROCOST_MEASURES:
         assert len(all_reg_evals) == len(all_zerocost_init_evals[measure])
     assert len(all_reg_evals) == len(all_arch_ids)
+    if params_flops_data:
+        assert len(all_reg_evals) == len(all_params_evals)
+        assert len(all_reg_evals) == len(all_flops_evals)
+
+    spe_params, _ = spearmanr(all_reg_evals, all_params_evals)
+    spe_flops, _ = spearmanr(all_reg_evals, all_flops_evals)
+
+    # Store some key numbers in results.txt
+    results_savename = os.path.join(out_dir, 'results.txt')
+    with open(results_savename, 'w') as f:
+        f.write(f'Total valid archs processed: {len(all_reg_evals)} \n')
+        f.write(f'Spearman wrt params: {spe_params}')
+        f.write(f'Spearman wrt flops: {spe_flops}')
+
+    print(f'Total valid archs processed: {len(all_reg_evals)}')
+    print(f'Spearman wrt params: {spe_params}')
+    print(f'Spearman wrt flops: {spe_flops}')
+
 
     top_percent_range = range(2, 101, 2)
     # Rank correlations at top n percent of architectures