cifar10 results after finding speed diff on clusters.

.vscode/launch.json

@@ -228,7 +228,7 @@
             "request": "launch",
             "program": "${cwd}/scripts/main.py",
             "console": "integratedTerminal",
-            "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "flower102"]
+            "args": ["--full", "--algos", "proxynas_natsbench_space", "--datasets", "ImageNet16-120"]
         },
         {
             "name": "Proxynas-Natsbench-Space-Toy",
@@ -655,7 +655,7 @@
             "request": "launch",
             "program": "${cwd}/scripts/reports/proxynas_plots/cross_exp_plots.py",
             "console": "integratedTerminal",
-            "args": ["--dataset", "natsbench_flower102",
+            "args": ["--dataset", "natsbench_cifar10",
             "--conf-location", "scripts/reports/proxynas_plots/cross_exp_conf.yaml"]
         },
         {
@@ -664,8 +664,8 @@
             "request": "launch",
             "program": "${cwd}/scripts/reports/analysis_natsbench_zerocost.py",
             "console": "integratedTerminal",
-            "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\zc_synthetic_cifar10",
+            "args": ["--results-dir", "D:\\archaiphilly\\phillytools\\zc_flower102",
-                "--out-dir", "D:\\archai_experiment_reports"]
+                "--out-dir", "D:\\archai_experiment_reports", "--reg-evals-file", "D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml"]
         },
         {
             "name": "Analysis Zero Cost Conditional Natsbench Experiments",

archai/algos/natsbench/natsbench_utils.py

@@ -0,0 +1,52 @@
+from copy import deepcopy
+from typing import Optional
+import importlib
+import sys
+import string
+import os
+
+from overrides import overrides
+
+import torch
+from torch import nn
+
+from overrides import overrides, EnforceOverrides
+
+from archai.common.trainer import Trainer
+from archai.common.config import Config
+from archai.common.common import logger
+from archai.datasets import data
+from archai.nas.model_desc import ModelDesc
+from archai.nas.model_desc_builder import ModelDescBuilder
+from archai.nas import nas_utils
+from archai.common import ml_utils, utils
+from archai.common.metrics import EpochMetrics, Metrics
+from archai.nas.model import Model
+from archai.common.checkpoint import CheckPoint
+from archai.nas.evaluater import Evaluater
+from archai.algos.proxynas.freeze_trainer import FreezeTrainer
+from archai.algos.proxynas.conditional_trainer import ConditionalTrainer
+
+from nats_bench import create
+from archai.algos.natsbench.lib.models import get_cell_based_tiny_net
+
+
+
+def model_from_natsbench_tss(arch_index:int, dataset_name:str, natsbench_location:str)->Model:
+
+        # create natsbench api
+        api = create(natsbench_location, 'tss', fast_mode=True, verbose=True)
+
+        if arch_index > 15625 or arch_index < 0:
+            logger.warn(f'architecture id {arch_index} is invalid ')
+
+        supported_datasets = {'cifar10', 'cifar100', 'ImageNet16-120'}
+        
+        if dataset_name not in supported_datasets:
+            raise NotImplementedError
+
+        config = api.get_net_config(arch_index, dataset_name)
+        
+        model = get_cell_based_tiny_net(config)
+
+        return model

archai/algos/random_natsbench/random_natsbench_tss_exp_runner.py

@@ -0,0 +1,40 @@
+
+
+import overrides
+from typing import Optional, Type, Tuple
+
+from archai.nas.exp_runner import ExperimentRunner
+from archai.nas.model_desc_builder import ModelDescBuilder
+from archai.nas.arch_trainer import TArchTrainer
+from archai.common import common
+from archai.common import utils
+from archai.common.config import Config
+from archai.nas.evaluater import Evaluater, EvalResult
+from archai.nas.searcher import Searcher, SearchResult
+from archai.nas.finalizers import Finalizers
+from archai.nas.random_finalizers import RandomFinalizers
+from archai.nas.model_desc_builder import ModelDescBuilder
+
+
+class RandomNatsbenchTssExpRunner(ExperimentRunner):
+
+    @overrides
+    def run_search(self, conf_search:Config)->SearchResult:
+        search = self.searcher()
+        return search.search(conf_search)
+
+    @overrides
+    def run_eval(self, conf_eval:Config)->EvalResult:
+        evaler = self.evaluater()
+        return evaler.evaluate(conf_eval)
+
+    @overrides
+    def searcher(self)->Searcher:
+        return RandomNatsbenchSearcher()
+
+    @overrides
+    def evaluater(self)->Evaluater:
+        return RandomNatsbenchEvaluater()
+
+    
+

archai/algos/random_natsbench/random_natsbench_tss_searcher.py

@@ -0,0 +1,44 @@
+import math as ma
+from typing import Set
+
+from archai.nas.searcher import Searcher, SearchResult
+from archai.common.config import Config
+from archai.nas.model_desc_builder import ModelDescBuilder
+from archai.nas.arch_trainer import TArchTrainer
+from archai.common.trainer import Trainer
+from archai.nas.model_desc import CellType, ModelDesc
+from archai.datasets import data
+from archai.nas.model import Model
+from archai.common.metrics import EpochMetrics, Metrics
+from archai.common import utils
+from archai.nas.finalizers import Finalizers
+from archai.algos.proxynas.conditional_trainer import ConditionalTrainer
+from archai.algos.proxynas.freeze_trainer import FreezeTrainer
+from archai.algos.natsbench.natsbench_utils import model_from_natsbench_tss
+
+class RandomNatsbenchTssSearcher(Searcher):
+    def search(conf_search:Config)->SearchResult:
+
+        # region config vars
+        max_num_models = conf_search['max_num_models']
+        ratio_fastest_duration = conf_search['ratio_fastest_duration']
+        top1_acc_threshold = conf_search['top1_acc_threshold']
+        # endregion
+
+        counter = 0
+        best_trains = [-ma.Inf]
+        fastest_cond_train = ma.Inf
+        archids_sampled = set()
+        while counter < max_num_models:
+            # sample a random model from tss
+            
+            # if during conditional training it
+            # starts exceeding fastest time to
+            # reach threshold by a ratio then early
+            # terminate it
+
+            #  
+
+        
+
+        

confs/algos/natsbench_regular_eval.yaml

@@ -12,7 +12,7 @@ nas:
     model_desc:
       num_edges_to_sample: 2
     loader:
-      train_batch: 256 # natsbench uses 256
+      train_batch: 1024 # natsbench uses 256
       aug: '' # random flip and crop are already there in default params
     trainer: # matching natsbench paper closely
       plotsdir: ''
@@ -24,7 +24,7 @@ nas:
       l1_alphas: 0.0   # weight to be applied to sum(abs(alphas)) to loss term
       logger_freq: 1000 # after every N updates dump loss and other metrics in logger
       title: 'eval_train'
-      epochs: 18
+      epochs: 30
       batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM
       lossfn:
         type: 'CrossEntropyLoss'

confs/algos/proxynas_natsbench_space.yaml

@@ -23,7 +23,7 @@ nas:
     trainer:
       plotsdir: ''
       use_val: False
-      top1_acc_threshold: 0.2 # after some accuracy we will shift into training only the last 'n' layers
+      top1_acc_threshold: 0.1 # after some accuracy we will shift into training only the last 'n' layers
       apex:
         _copy: '/common/apex'
       aux_weight: '_copy: /nas/eval/model_desc/aux_weight'
@@ -61,7 +61,7 @@ nas:
       l1_alphas: 0.0  # weight to be applied to sum(abs(alphas)) to loss term
       logger_freq: 1000 # after every N updates dump loss and other metrics in logger
       title: 'eval_train'
-      epochs: 15
+      epochs: 10
       batch_chunks: 1 # split batch into these many chunks and accumulate gradients so we can support GPUs with lower RAM
       lossfn:
         type: 'CrossEntropyLoss'

run_all_ft_analysis.bat

@@ -53,6 +53,16 @@
 
 @REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb256_ftlr0.1_fte15_ct256_ftt0.6 --out-dir D:\\archai_experiment_reports
 
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte5_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_scu --out-dir D:\\archai_experiment_reports
+
+
 @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e01 --out-dir D:\\archai_experiment_reports
 @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e02 --out-dir D:\\archai_experiment_reports
 @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e04 --out-dir D:\\archai_experiment_reports
@@ -74,6 +84,25 @@
 @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e08 --out-dir D:\\archai_experiment_reports
 @REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e10 --out-dir D:\\archai_experiment_reports
 
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e01_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e02_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e04_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e06_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e08_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e10_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e20_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b1024_e30_scu --out-dir D:\\archai_experiment_reports
+
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e01_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e02_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e04_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e06_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e08_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e10_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e20_scu --out-dir D:\\archai_experiment_reports
+@REM python scripts/reports/analysis_regular_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\nb_reg_b256_e30_scu --out-dir D:\\archai_experiment_reports
+
+
 @REM cifar100
 
 @REM python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\archaiphilly\\phillytools\\ft_c100_fb1024_ftlr0.1_fte5_ct256_ftt0.3 --out-dir D:\\archai_experiment_reports
@@ -190,24 +219,22 @@ python scripts/reports/analysis_freeze_natsbench_space.py --results-dir D:\\arch
 
 @REM Flower 102
 
-@REM ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
 
-@REM ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb512_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-@REM ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2
 
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
+@REM python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb256_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb2048_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte5_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
-python scripts/reports/analysis_freeze_natsbench_space_new.py --results-dir D:\\archaiphilly\\phillytools\\ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2 --out-dir D:\\archai_experiment_reports --reg-evals-file D:\\archai_experiment_reports\\nb_f102_b256_reg200\\archid_testacc.yaml
 
 @REM Nasbench 101
 

scripts/reports/proxynas_plots/cross_exp_conf.yaml

@@ -17,15 +17,21 @@ natsbench_cifar10:
     # ft_fb2048_ftlr1.5_fte10_ct256_ftt0.3: 'fastarchrank stage 2: batch 2048, lr 1.5, <br> epochs 10, stage 1: batch 256, thresh 0.3'
     # ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 1.5, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 1.5, <br> epochs 10, stage 1: batch 256, thresh 0.6'
+    # ft_fb1024_ftlr1.5_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 1.5, <br> epochs 5, stage 1: batch 256, thresh 0.6 scu'
+    # ft_fb1024_ftlr1.5_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 1.5, <br> epochs 10, stage 1: batch 256, thresh 0.6 scu'
     # ft_fb512_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 1.5, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb512_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 1.5, <br> epochs 10, stage 1: batch 256, thresh 0.6'                 
     # ft_fb256_ftlr1.5_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 1.5, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb256_ftlr1.5_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 1.5, <br> epochs 10, stage 1: batch 256, thresh 0.6' 
     # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.6'
+    ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6 scu'
+    ft_fb1024_ftlr0.1_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.6 scu'
     # ft_fb512_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.6'
-    ft_fb256_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6'
+    # ft_fb512_ftlr0.1_fte5_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6 scu'
+    # ft_fb512_ftlr0.1_fte10_ct256_ftt0.6_scu: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.6 scu'
+    # ft_fb256_ftlr0.1_fte5_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6'
     # ft_fb256_ftlr0.1_fte10_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.6'
     # ft_fb256_ftlr0.1_fte15_ct256_ftt0.6: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 15, stage 1: batch 256, thresh 0.6'
     # ft_fb1024_ftlr0.1_fte5_ct256_ftt0.6_c9: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.6, frozen till: cell 9' 
@@ -49,6 +55,14 @@ natsbench_cifar10:
     nb_reg_b1024_e10: 'regular train: batch 1024 epochs 10'
     nb_reg_b1024_e20: 'regular train: batch 1024 epochs 20'
     nb_reg_b1024_e30: 'regular train: batch 1024 epochs 30'
+    nb_reg_b1024_e01_scu: 'regular train: batch 1024 epochs 01 scu'
+    nb_reg_b1024_e02_scu: 'regular train: batch 1024 epochs 02 scu'
+    nb_reg_b1024_e04_scu: 'regular train: batch 1024 epochs 04 scu'
+    nb_reg_b1024_e06_scu: 'regular train: batch 1024 epochs 06 scu'
+    nb_reg_b1024_e08_scu: 'regular train: batch 1024 epochs 08 scu'
+    nb_reg_b1024_e10_scu: 'regular train: batch 1024 epochs 10 scu'
+    nb_reg_b1024_e20_scu: 'regular train: batch 1024 epochs 20 scu'
+    nb_reg_b1024_e30_scu: 'regular train: batch 1024 epochs 30 scu'
     nb_reg_b512_e01: 'regular train: batch 512 epochs 01'
     nb_reg_b512_e02: 'regular train: batch 512 epochs 02'
     nb_reg_b512_e04: 'regular train: batch 512 epochs 04'
@@ -61,6 +75,15 @@ natsbench_cifar10:
     nb_reg_b256_e06: 'regular train: batch 256 epochs 06'
     nb_reg_b256_e08: 'regular train: batch 256 epochs 08'
     nb_reg_b256_e10: 'regular train: batch 256 epochs 10'
+    nb_reg_b256_e01_scu: 'regular train: batch 256 epochs 01 scu'
+    nb_reg_b256_e02_scu: 'regular train: batch 256 epochs 02 scu'
+    nb_reg_b256_e04_scu: 'regular train: batch 256 epochs 04 scu'
+    nb_reg_b256_e06_scu: 'regular train: batch 256 epochs 06 scu'
+    nb_reg_b256_e08_scu: 'regular train: batch 256 epochs 08 scu'
+    nb_reg_b256_e10_scu: 'regular train: batch 256 epochs 10 scu'
+    nb_reg_b256_e20_scu: 'regular train: batch 256 epochs 20 scu'
+    nb_reg_b256_e30_scu: 'regular train: batch 256 epochs 30 scu'
+
 
 
   colors:
@@ -175,12 +198,21 @@ natsbench_flower102:
     ft_f102_fb1024_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.2'
     ft_f102_fb1024_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 15, stage 1: batch 256, thresh 0.2'
     ft_f102_fb1024_ftlr0.1_fte30_ct256_ftt0.2: 'fastarchrank stage 2: batch 1024, lr 0.1, <br> epochs 30, stage 1: batch 256, thresh 0.2'
-  
+    ft_f102_fb512_ftlr0.1_fte5_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.2'
+    ft_f102_fb512_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.2'
+    ft_f102_fb512_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 512, lr 0.1, <br> epochs 15, stage 1: batch 256, thresh 0.2'
+    ft_f102_fb256_ftlr0.1_fte5_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 5, stage 1: batch 256, thresh 0.2'
+    ft_f102_fb256_ftlr0.1_fte10_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 10, stage 1: batch 256, thresh 0.2'
+    ft_f102_fb256_ftlr0.1_fte15_ct256_ftt0.2: 'fastarchrank stage 2: batch 256, lr 0.1, <br> epochs 15, stage 1: batch 256, thresh 0.2'
+
   shortreg: 
     nb_f102_reg_b256_e10: 'regular train: batch 2048 epochs 10'
     nb_f102_reg_b256_e20: 'regular train: batch 2048 epochs 20'
     nb_f102_reg_b256_e30: 'regular train: batch 2048 epochs 30'
 
+  zero_cost:
+    zc_flower102: 'zero cost score at initialization'
+
   colors:
     freezetrain: 'red'
     zero_cost: 'green'