Added plotting code to display distribution of architectures.

.vscode/launch.json

@@ -560,7 +560,7 @@
             "request": "launch",
             "program": "${cwd}/scripts/reports/analysis_freeze_natsbench_space.py",
             "console": "integratedTerminal",
-            "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\fa_nb1_se_fb256_ftlr0.001_fte5_ct256_ftt0.6",
+            "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\ft_fb256_ftlr0.1_fte10_ct256_ftt0.6_c9",
             "--out-dir", "D:\\archai_experiment_reports"]
         },
         {

archai/algos/nasbench101/model.py

@@ -90,7 +90,7 @@ class Network(nn.Module):
 
         # flatten all activations 
         flattened = [torch.flatten(x, start_dim=1)  for x in activations]
-        feat = torch.cat(flattened, 0)
+        feat = torch.cat(flattened, 1)
         
         return feat
 

archai/algos/nasbench101/nasbench101_evaluater.py

@@ -49,7 +49,6 @@ class Nb101RegularEvaluater(Evaluater):
         nasbench101_location = os.path.join(dataroot, 'nb101', 'nasbench_full.pkl')
         # endregion
 
-        assert arch_index
         assert nasbench101_location
 
         return self._model_from_nasbench101(arch_index, dataset_name, nasbench101_location)

confs/algos/nb101_regular_eval.yaml

@@ -12,7 +12,7 @@ nas:
   eval:
     model_filename: False # disable model saving to speedup download
     nasbench101:
-      arch_index: 6738
+      arch_index: 0
     model_desc:
       num_edges_to_sample: 2
     loader:
@@ -28,7 +28,7 @@ nas:
       l1_alphas: 0.0   # weight to be applied to sum(abs(alphas)) to loss term
       logger_freq: 1000 # after every N updates dump loss and other metrics in logger
       title: 'eval_train'
-      epochs: 8
+      epochs: 108
       batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM
       lossfn:
         type: 'CrossEntropyLoss'

run_all_ft_analysis.bat

@@ -178,6 +178,9 @@
 @REM python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_se_fb256_ftlr0.025_fte5_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports
 @REM python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_se_fb256_ftlr0.025_fte10_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports
 
+python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_s123_fb256_ftlr0.001_fte5_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports
+python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_s123_fb256_ftlr0.001_fte10_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports
+
 @REM python scripts/reports/temp_analysis.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_s3_fb256_ftlr0.001_fte5_nocond --out-dir D:\\archai_experiment_reports
 @REM python scripts/reports/temp_analysis.py --results-dir D:\\archaiphilly\\phillytools\\fa_nb1_s3_fb256_ftlr0.001_fte10_nocond --out-dir D:\\archai_experiment_reports
 

scripts/nasbench101/distribution_of_archs.py

@@ -0,0 +1,27 @@
+import logging
+import random
+from archai.algos.nasbench101.nasbench101_dataset import Nasbench101Dataset
+from archai.algos.nasbench101 import model_builder
+
+import plotly.express as px
+
+def main():
+
+    # create dataset
+    nsds= Nasbench101Dataset('~/dataroot/nasbench_ds/nasbench_full.pkl')
+
+    test_accs = []
+
+    for arch_id in range(len(nsds)):
+        all_trials = nsds.get_test_acc(arch_id)
+        test_accuracy = sum(all_trials) / len(all_trials)
+        test_accs.append(test_accuracy)
+
+    fig = px.histogram(test_accs, labels={'x': 'Test Accuracy', 'y': 'Counts'})
+    fig.show()
+
+
+
+
+if __name__ == '__main__':
+    main()

scripts/reports/analysis_freeze_natsbench_space.py

@@ -239,6 +239,11 @@ def main():
     fig.write_html(savename)
     fig.show()
 
+    fig = px.histogram(all_reg_evals, labels={'x': 'Test Accuracy', 'y': 'Counts'})
+    savename = os.path.join(out_dir, 'distribution_of_reg_evals.html')
+    fig.write_html(savename)
+    fig.show()
+
     # Freeze training results at last epoch        
     freeze_tau, freeze_p_value = kendalltau(all_reg_evals, all_freeze_evals_last)
     freeze_spe, freeze_sp_value = spearmanr(all_reg_evals, all_freeze_evals_last)

scripts/reports/proxynas_plots/cross_exp_conf.yaml

@@ -124,6 +124,8 @@ nasbench101:
                 fa_nb1_se_fb256_ftlr0.01_fte10_ct256_ftt0.6,
                 fa_nb1_se_fb256_ftlr0.025_fte5_ct256_ftt0.6,
                 fa_nb1_se_fb256_ftlr0.025_fte10_ct256_ftt0.6,
+                fa_nb1_s123_fb256_ftlr0.001_fte5_ct256_ftt0.6,
+                fa_nb1_s123_fb256_ftlr0.001_fte10_ct256_ftt0.6,
                 fa_nb1_s3_fb256_ftlr0.001_fte5_nocond,
                 fa_nb1_s3_fb256_ftlr0.001_fte10_nocond,
                 fa_nb1_s3_fb256_ftlr0.01_fte5_nocond,