Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.

.vscode/launch.json

@@ -740,12 +740,8 @@
             "request": "launch",
             "program": "${cwd}/archai/nlp/nas/characterize_search_space.py",
             "console": "integratedTerminal",
-<<<<<<< HEAD
-            "args": ["--list-of-npys", "/home/dedey/archai_experiment_reports/transnasbench_heatmaps/list.txt",
-                    "--out-dir", "/home/dedey/archai_experiment_reports/transnasbench_heatmaps"]
-=======
-            "args": []
->>>>>>> c97f7e8b (refactor(ds): restores main branch for everything except `discrete_search` folder)
+            "args": ["--results-dir", "C:\\Users\\dedey\\Documents\\archaiphilly\\phillytools\\proxynas_regular_and_parameterless",
+            "--out-dir", "C:\\Users\\dedey\\archai_experiment_reports"]
         },
         {
             "name": "CurrentFile",

confs/algos/proxynas.yaml

@@ -0,0 +1,28 @@
+__include__: 'darts.yaml' # just use darts defaults
+
+#primitives: ['max_pool_3x3', 'avg_pool_3x3', 'skip_connect', 'sep_conv_3x3', 'sep_conv_5x5', 'dil_conv_3x3', 'dil_conv_5x5']
+
+primitives: ['max_pool_3x3', 'avg_pool_3x3', 'skip_connect']
+
+nas:
+  search:
+    model_desc:
+      num_edges_to_sample: 2 # number of edges each node will take input from
+
+  eval:
+    model_desc:
+      num_edges_to_sample: 2
+    trainer:
+      plotsdir: ''
+      epochs: 600
+      aux_weight: 0.0
+      drop_path_prob: 0.0
+      proxynas:
+        val_top1_acc_threshold: 0.85 # after some accuracy we will shift into training only the last layer
+        freeze_epochs: 200
+        freeze_lr: 0.001
+        freeze_decay: 0.0
+        freeze_momentum: 0.0
+        train_regular: True
+        identifiers_to_unfreeze: ['logits_op._op']
+        aux_weight: 0.0 # disable auxiliary loss part during finetuning

scripts/reports/analysis_aggregate.py

@@ -0,0 +1,95 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import argparse
+from typing import Dict, List, Type, Iterator, Tuple
+import glob
+import os
+import pathlib
+from collections import OrderedDict
+import yaml
+from inspect import getsourcefile
+import re
+
+from runstats import Statistics
+
+import matplotlib
+matplotlib.use('Agg')
+import seaborn as sns
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+
+from archai.common import utils
+from archai.common.ordereddict_logger import OrderedDictLogger
+from analysis_utils import epoch_nodes, fix_yaml, remove_seed_part, group_multi_runs, collect_epoch_nodes, EpochStats, FoldStats, stat2str, get_epoch_stats, get_summary_text, get_details_text, plot_epochs, write_report
+
+import re
+
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Report creator')
+    parser.add_argument('--results-dir', '-d', type=str,
+                        default=r'~/logdir/proxynas_test_0001',
+                        help='folder with experiment results from pt')
+    parser.add_argument('--out-dir', '-o', type=str, default=r'~/logdir/reports',
+                        help='folder to output reports')
+    args, extra_args = parser.parse_known_args()
+
+    # root dir where all results are stored
+    results_dir = pathlib.Path(utils.full_path(args.results_dir))
+    print(f'results_dir: {results_dir}')
+
+    # extract experiment name which is top level directory
+    exp_name = results_dir.parts[-1]
+
+    # create results dir for experiment
+    out_dir = utils.full_path(os.path.join(args.out_dir, exp_name))
+    print(f'out_dir: {out_dir}')
+    os.makedirs(out_dir, exist_ok=True)
+
+    # get list of all structured logs for each job
+    logs = {}
+    job_count = 0
+    for job_dir in results_dir.iterdir():
+        job_count += 1
+        for subdir in job_dir.iterdir():
+            if not subdir.is_dir():
+                continue
+            # currently we expect that each job was ExperimentRunner job which should have
+            # _search or _eval folders
+            if subdir.stem.endswith('_search'):
+                sub_job = 'search'
+            elif subdir.stem.endswith('_eval'):
+                sub_job = 'eval'
+            else:
+                raise RuntimeError(f'Sub directory "{subdir}" in job "{job_dir}" must '
+                                  'end with either _search or _eval which '
+                                  'should be the case if ExperimentRunner was used.')
+
+            logs_filepath = os.path.join(str(subdir), 'log.yaml')
+            if os.path.isfile(logs_filepath):
+                fix_yaml(logs_filepath)
+                with open(logs_filepath, 'r') as f:
+                    key = job_dir.name + ':' + sub_job
+                    logs[key] = yaml.load(f, Loader=yaml.Loader)
+
+    # create list of epoch nodes having same path in the logs
+    grouped_logs = group_multi_runs(logs)
+    collated_grouped_logs = collect_epoch_nodes(grouped_logs)
+    summary_text, details_text = '', ''
+
+    for log_key, grouped_logs in collated_grouped_logs.items():
+        # for each path for epochs nodes, compute stats
+        for node_path, logs_epochs_nodes in grouped_logs.items():
+            collated_epoch_stats = get_epoch_stats(node_path, logs_epochs_nodes)
+            summary_text += get_summary_text(log_key, out_dir, node_path, collated_epoch_stats, len(logs_epochs_nodes))
+            details_text += get_details_text(log_key, out_dir, node_path, collated_epoch_stats, len(logs_epochs_nodes))
+
+    write_report('summary.md', **vars())
+    write_report('details.md', **vars())
+
+
+if __name__ == '__main__':
+    main()

scripts/reports/analysis_proxynas.py

@@ -10,7 +10,10 @@ from collections import OrderedDict
 import yaml
 from inspect import getsourcefile
 import re
+<<<<<<< HEAD
 from tqdm import tqdm
+=======
+>>>>>>> d6e6e107 (Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.)
 
 from scipy.stats import kendalltau, spearmanr
 
@@ -53,6 +56,7 @@ def main():
     # get list of all structured logs for each job
     logs = {}
     job_count = 0
+<<<<<<< HEAD
     for job_dir in tqdm(results_dir.iterdir()):
         if job_dir.is_dir():
             job_count += 1
@@ -85,6 +89,31 @@ def main():
                             all_good = False
                         if all_good:
                             print(f'{key} is all good')
+=======
+    for job_dir in results_dir.iterdir():
+        job_count += 1
+        for subdir in job_dir.iterdir():
+            if not subdir.is_dir():
+                continue
+            # currently we expect that each job was ExperimentRunner job which should have
+            # _search or _eval folders
+            if subdir.stem.endswith('_search'):
+                sub_job = 'search'
+            elif subdir.stem.endswith('_eval'):
+                sub_job = 'eval'
+            else:
+                raise RuntimeError(f'Sub directory "{subdir}" in job "{job_dir}" must '
+                                  'end with either _search or _eval which '
+                                  'should be the case if ExperimentRunner was used.')
+
+            logs_filepath = os.path.join(str(subdir), 'log.yaml')
+            if os.path.isfile(logs_filepath):
+                fix_yaml(logs_filepath)
+                with open(logs_filepath, 'r') as f:
+                    key = job_dir.name + ':' + sub_job
+                    logs[key] = yaml.load(f, Loader=yaml.Loader)
+
+>>>>>>> d6e6e107 (Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.)
 
 
     # logs['proxynas_parameterless_seed_198980.3100969506_freeze_lr_0.001:eval']['eval_arch']['eval_train']['best_val']['top1']
@@ -93,6 +122,7 @@ def main():
     all_freeze_evals = []
     for key in logs.keys():
         if 'eval' in key:
+<<<<<<< HEAD
             try:
                 reg_eval_top1 = logs[key]['eval_arch']['eval_train']['best_val']['top1']
                 freeze_eval_top1 = logs[key]['freeze_evaluate']['eval_arch']['eval_train']['best_val']['top1']
@@ -101,25 +131,41 @@ def main():
                 all_freeze_evals.append(freeze_eval_top1)
             except KeyError as err:
                 print(f'KeyError {err} in {key}')
+=======
+            reg_eval_top1 = logs[key]['eval_arch']['eval_train']['best_val']['top1']
+            freeze_eval_top1 = logs[key]['freeze_evaluate']['eval_arch']['eval_train']['best_val']['top1']
+
+            all_reg_evals.append(reg_eval_top1)
+            all_freeze_evals.append(freeze_eval_top1)
+>>>>>>> d6e6e107 (Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.)
         
     tau, p_value = kendalltau(all_reg_evals, all_freeze_evals)
     spe, sp_value = spearmanr(all_reg_evals, all_freeze_evals)
     print(f'Kendall Tau score: {tau}, p_value {p_value}')
     print(f'Spearman corr: {spe}, p_value {sp_value}')
+<<<<<<< HEAD
     results_savename = os.path.join(results_dir, 'results.txt')
     with open(results_savename, 'w') as f:
         f.write(f'Kendall Tau score: {tau}, p_value {p_value}')
         f.write(f'Spearman corr: {spe}, p_value {sp_value}')
 
+=======
+>>>>>>> d6e6e107 (Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.)
     
     plt.scatter(all_reg_evals, all_freeze_evals)
     plt.xlabel('Val top1 at 600 epochs')
     plt.ylabel('Freeze training')
+<<<<<<< HEAD
     plt.title('Freeze training at 0.75 val top1 followed by 200 epochs')
     savename = os.path.join(results_dir, 'proxynas_0.75_freeze_training_200_epochs.png')
     plt.savefig(savename, dpi=plt.gcf().dpi, bbox_inches='tight')
 
 
+=======
+    plt.title('Freeze training at 0.6 val top1 followed by 200 epochs')
+    plt.savefig('proxynas_0.6_freeze_training_200_epochs.png',
+        dpi=plt.gcf().dpi, bbox_inches='tight')
+>>>>>>> d6e6e107 (Added analysis scripts to compute kendall tau and spearman's correlation for proxynas experiments.)
 
 if __name__ == '__main__':
     main()