Feature/secrets.cfg (#43)

* update configuration.cfg to secrets.cfg

* fix node count pretty print

* add more unit tests

* configure pytest to only run tests from the 'tests' directory

* add missing space

* remove static call to loading secrets file

.gitignore

@@ -1,5 +1,5 @@
 # custom
-configuration.cfg
+secrets.cfg
 my-custom-scripts/
 
 # Virtual environments

README.md

@@ -13,7 +13,7 @@ A suite of distributed tools to help engineers scale their work into Azure.
 ```bash
     pip install -e .
 ```
-4. Rename 'configuration.cfg.template' to 'configuration.cfg' and fill in the fields for your Batch account and Storage account. These fields can be found in the Azure portal.
+4. Rename 'secrets.cfg.template' to 'secrets.cfg' and fill in the fields for your Batch account and Storage account. These fields can be found in the Azure portal.
 
    To complete this step, you will need an Azure account that has a Batch account and Storage account:
     - To create an Azure account: https://azure.microsoft.com/free/
@@ -62,7 +62,7 @@ You can also add a user directly in this command using the same inputs as the `a
 When your cluster is ready, create a user for your cluster (if you didn't already do so when creating your cluster):
 ```bash
 # **Recommended usage**
-# Add a user with a ssh public key. It will use the value specified in the configuration.cfg (Either path to the file or the actual key)
+# Add a user with a ssh public key. It will use the value specified in the secrets.cfg (Either path to the file or the actual key)
 azb spark cluster add-user \
     --id <my-cluster-id> \
     --username <username>

dtde/azure_api.py

@@ -5,7 +5,7 @@ from . import config
 from .version import __version__
 
 
-global_config = config.get()
+global_config = None
 
 batch_client = None
 batch_config = None
@@ -95,6 +95,8 @@ def get_batch_config() -> BatchConfig:
 
 
 def __load_batch_config():
+    global_config = config.get()
+
     global batch_config
 
     if not global_config.has_option('Batch', 'batchaccountkey'):
@@ -128,6 +130,7 @@ def __load_batch_client():
 
 
 def __load_blob_client():
+    global_config = config.get()
     global blob_client
 
     if not global_config.has_option('Storage', 'storageaccountkey'):

dtde/clusterlib.py

@@ -252,9 +252,11 @@ class Cluster:
             pool.target_low_priority_nodes
         self.dedicated_nodes = pool.current_dedicated_nodes
         self.low_pri_nodes = pool.current_low_priority_nodes
+        self.target_dedicated_nodes = pool.target_dedicated_nodes
+        self.target_low_pri_nodes = pool.target_low_priority_nodes
 
 
-def pretty_node_count(cluster: Cluster)-> str:
+def pretty_node_count(cluster: Cluster) -> str:
     if cluster.pool.allocation_state is batch_models.AllocationState.resizing:
         return '{} -> {}'.format(
             cluster.total_current_nodes,
@@ -263,6 +265,28 @@ def pretty_node_count(cluster: Cluster)-> str:
         return '{}'.format(cluster.total_current_nodes)
 
 
+def pretty_dedicated_node_count(cluster: Cluster)-> str:
+    if (cluster.pool.allocation_state is batch_models.AllocationState.resizing\
+        or cluster.pool.state is batch_models.PoolState.deleting)\
+        and cluster.dedicated_nodes != cluster.target_dedicated_nodes:
+        return '{} -> {}'.format(
+            cluster.dedicated_nodes,
+            cluster.target_dedicated_nodes)
+    else:
+        return '{}'.format(cluster.dedicated_nodes)
+
+
+def pretty_low_pri_node_count(cluster: Cluster)-> str:
+    if (cluster.pool.allocation_state is batch_models.AllocationState.resizing\
+        or cluster.pool.state is batch_models.PoolState.deleting)\
+        and cluster.low_pri_nodes != cluster.target_low_pri_nodes:
+        return '{} -> {}'.format(
+            cluster.low_pri_nodes,
+            cluster.target_low_pri_nodes)
+    else:
+        return '{}'.format(cluster.low_pri_nodes)
+
+
 def print_cluster(cluster: Cluster):
     node_count = pretty_node_count(cluster)
 
@@ -272,8 +296,8 @@ def print_cluster(cluster: Cluster):
     log.info("State:          %s", cluster.visible_state)
     log.info("Node Size:      %s", cluster.vm_size)
     log.info("Nodes:          %s", node_count)
-    log.info("| Dedicated:    %s", cluster.dedicated_nodes)
-    log.info("| Low priority: %s", cluster.low_pri_nodes)
+    log.info("| Dedicated:    %s", pretty_dedicated_node_count(cluster))
+    log.info("| Low priority: %s", pretty_low_pri_node_count(cluster))
     log.info("")
 
     print_format = '{:<36}| {:<15} | {:<21}| {:<8}'

dtde/config.py

@@ -11,7 +11,7 @@ global_config = None
 
 def load_config(path: str=constants.DEFAULT_CONFIG_PATH):
     """
-        Loads the config file at the root of the repository(configuration.cfg)
+        Loads the config file at the root of the repository(secrets.cfg)
     """
     global global_config
     if not os.path.isfile(path):
@@ -25,5 +25,4 @@ def load_config(path: str=constants.DEFAULT_CONFIG_PATH):
 def get() -> configparser.ConfigParser:
     if not global_config:
         load_config()
-
     return global_config

dtde/constants.py

@@ -23,7 +23,7 @@ ROOT_PATH = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
 """
     Path to the configuration file
 """
-DEFAULT_CONFIG_PATH = os.path.join(os.getcwd(), 'configuration.cfg')
+DEFAULT_CONFIG_PATH = os.path.join(os.getcwd(), 'secrets.cfg')
 
 """
     Key of the metadata entry for the pool that is used to store the master node id

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+testpaths = tests

tests/test_clusterlib.py

@@ -0,0 +1,78 @@
+from dtde import clusterlib
+import azure.batch.models as batch_models
+
+
+def create_mock_cluster(
+        pool_id='test',
+        vm_size='standard_f2',
+        target_dedicated_nodes=3,
+        current_dedcated_nodes=0,
+        target_low_priority_nodes=2,
+        current_low_priority_nodes=0,
+        state=batch_models.PoolState.active,
+        allocation_state=batch_models.AllocationState.resizing
+) -> batch_models.CloudPool:
+    pool = batch_models.CloudPool(
+        id=pool_id,
+        current_low_priority_nodes=current_low_priority_nodes,
+        target_low_priority_nodes=target_low_priority_nodes,
+        current_dedicated_nodes=current_dedcated_nodes,
+        target_dedicated_nodes=target_dedicated_nodes,
+        vm_size=vm_size,
+        state=state,
+        allocation_state=allocation_state
+    )
+    cluster = clusterlib.Cluster(pool)
+    return cluster
+
+
+def test_node_count_steady():
+    cluster = create_mock_cluster(
+        current_dedcated_nodes=3,
+        current_low_priority_nodes=2,
+        allocation_state=batch_models.AllocationState.steady
+    )
+    expected_value = '5'
+    value = clusterlib.pretty_node_count(cluster)
+    assert expected_value == value
+
+
+def test_node_count_resizing():
+    cluster = create_mock_cluster()
+    expected_value = '0 -> 5'
+    value = clusterlib.pretty_node_count(cluster)
+    assert expected_value == value
+
+
+def test_pretty_print_dedicated_nodes_steady():
+    cluster = create_mock_cluster(
+        current_dedcated_nodes=3,
+        allocation_state=batch_models.AllocationState.steady
+    )
+    expected_value = '3'
+    value = clusterlib.pretty_dedicated_node_count(cluster)
+    assert expected_value == value
+
+
+def test_pretty_print_dedicated_nodes_resizing():
+    cluster = create_mock_cluster()
+    expected_value = '0 -> 3'
+    value = clusterlib.pretty_dedicated_node_count(cluster)
+    assert expected_value == value
+
+
+def test_pretty_print_low_priority_nodes_steady():
+    cluster = create_mock_cluster(
+        current_low_priority_nodes=2,
+        allocation_state=batch_models.AllocationState.steady
+    )
+    expected_value = '2'
+    value = clusterlib.pretty_low_pri_node_count(cluster)
+    assert expected_value == value
+
+
+def test_pretty_print_low_priority_nodes_resizing():
+    cluster = create_mock_cluster()
+    expected_value = '0 -> 2'
+    value = clusterlib.pretty_low_pri_node_count(cluster)
+    assert expected_value == value