Code cleanup (in progress)

test/arm_template_deploy.py

@@ -98,11 +98,12 @@ class ArmTemplateDeploy:
             },
         )
 
-    def serialize(self, to_file=None, *args, **kwargs):
+    def serialize(self, store_template=False, to_file=None, *args, **kwargs):
         """
-        Serialize this object into a JSON string. The Resource/NetworkManager
-        members are not serialized since deserialized instances should still
-        authenticate.
+        Serialize this object into a JSON string. The Resource/Network/Storage
+        Mgmt Client members are not serialized since deserialized instances
+        should re-authenticate. Similarly, the deploy_params members can
+        contain secrets (depending on the template), so it is not saved.
 
         If to_file is passed with a non-empty string value, the JSON string
         will be saved to a file whose name (including path) is to_file's value.
@@ -113,9 +114,11 @@ class ArmTemplateDeploy:
         _this.pop("rm_client", None)  # don't want to save these for security
         _this.pop("nm_client", None)
         _this.pop("st_client", None)
-        _this.pop("template", None)
         _this.pop("deploy_params", None)
 
+        if not store_template:
+            _this.pop("template", None)
+
         if to_file:
             with open(to_file, "w") as tf:
                 json.dump(_this, tf)

test/lib/helpers.py

@@ -70,37 +70,33 @@ def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False, timeout=None):
 def run_ssh_commands(ssh_client, commands, **kwargs):
     """
     Runs a list of commands on the server connected via ssh_client.
-
-    If ex_on_nonzero_rc is True, an Exception is raised if any command fails
-    (i.e., non-zero exit code).
     """
     log = logging.getLogger("run_ssh_commands")
     results = []
-    for cmd in commands:
-        cmd = cmd.strip()
-        log.debug("command to run: {}".format(cmd))
-        if cmd:  # only run non-empty commands
-            results.append(run_ssh_command(ssh_client, cmd, **kwargs))
+    for command in commands:
+        command = command.strip()
+        if command:  # only run non-empty commands
+            log.debug("command to run: {}".format(command))
+            results.append(run_ssh_command(ssh_client, command, **kwargs))
     return results
 
 
 def split_ip_range(ip_range):
     """
-    split_ip_range will take in an IP address range split by a hyphen
+    This function will split ip_range into a list of all IPs in that range.
+
+    ip_range is in an IP address range split by a hyphen
     (e.g., "10.0.0.1-10.0.0.9").
-
-    It will split it to a list of all IPs in that range.
     """
+    from ipaddress import ip_address
+
     ip_list = ip_range.split("-")
-    ip1 = ip_list[0]
-    ip2 = ip_list[1]
+    ip_0 = ip_list[0]
+    ip_1 = ip_list[1]
 
-    ip1_split = ip1.split(".")
-    ip_low = ip1_split[-1]
-    ip_hi = ip2.split(".")[-1]
-
-    ip_prefix = ".".join(ip1_split[:-1]) + "."
-    return [ip_prefix + str(n) for n in range(int(ip_low), int(ip_hi) + 1)]
+    ip_start = int(ip_address(ip_0).packed.hex(), 16)
+    ip_end = int(ip_address(ip_1).packed.hex(), 16)
+    return [ip_address(ip).exploded for ip in range(ip_start, ip_end + 1)]
 
 
 def upload_gsi(averecmd_params):

test/lib/pytest_fixtures.py

@@ -46,7 +46,7 @@ def mnt_nodes(ssh_con, test_vars):
 def resource_group(test_vars):
     log = logging.getLogger("resource_group")
     rg = test_vars["atd_obj"].create_resource_group()
-    log.info("Created Resource Group: {}".format(rg))
+    log.info("Resource Group: {}".format(rg))
     return rg
 
 
@@ -54,12 +54,13 @@ def resource_group(test_vars):
 def storage_account(test_vars):
     log = logging.getLogger("storage_account")
     atd = test_vars["atd_obj"]
-    storage_account = atd.st_client.storage_accounts.get_properties(
+    sa = atd.st_client.storage_accounts.get_properties(
         atd.resource_group,
-        atd.deploy_id + "sa"
+        atd.storage_account
     )
-    log.info("Linked Storage Account: {}".format(storage_account))
-    return storage_account
+    log.info("Storage Account: {}".format(sa))
+    return sa
+
 
 @pytest.fixture()
 def scp_cli(ssh_con):
@@ -103,4 +104,4 @@ def test_vars():
         log.debug("Saving vars to {} (VFXT_TEST_VARS_FILE)".format(
                   os.environ["VFXT_TEST_VARS_FILE"]))
         with open(os.environ["VFXT_TEST_VARS_FILE"], "w") as vtvf:
-            json.dump(vars, vtvf)
+            json.dump(vars, vtvf, sort_keys=True, indent=4)

test/test_avere_template_deploy.py

@@ -44,7 +44,6 @@ class TestVfxtTemplateDeploy:
         }
         test_vars["controller_name"] = atd.deploy_params["controllerName"]
         test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"]
-        test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"]
 
         log.debug("Generated deploy parameters: \n{}".format(
                   json.dumps(atd.deploy_params, indent=4)))

test/test_edasim.py

@@ -1,47 +1,46 @@
 #!/usr/bin/python3
 
 """
-Driver for testing edasim
+Driver for testing EDASIM
 """
 
+# standard imports
 import json
-import os
-from time import sleep
 import logging
+import os
+import sys
+
+# from requirements.txt
 import pytest
 import requests
-from scp import SCPClient
-
-from sshtunnel import SSHTunnelForwarder
 
+# local libraries
 from lib import helpers
-from lib.pytest_fixtures import (averecmd_params, mnt_nodes,  # noqa: F401
-                                 resource_group, storage_account,
-                                 scp_cli, ssh_con, test_vars, vs_ips)
-
-# logging.basicConfig(level=logging.DEBUG)
+from lib.pytest_fixtures import (mnt_nodes, resource_group,  # noqa: F401
+                                 scp_cli, ssh_con, storage_account, test_vars)
 
 
 class TestEdasim:
-    def test_download_go(self, ssh_con):
+    def test_download_go(self, ssh_con):  # noqa: F811
         commands = """
             sudo apt -y install golang-go
-            mkdir ~/gopath
+            mkdir -p ~/gopath
             echo "export GOPATH=$HOME/gopath" >> ~/.profile
             echo "export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin" >> ~/.profile
             source ~/.profile && cd $GOPATH && go get -v github.com/Azure/Avere/src/go/...
             """.split("\n")
         helpers.run_ssh_commands(ssh_con, commands)
 
-    def test_storage_account(self, test_vars, resource_group, storage_account, ssh_con):
-        td = test_vars["atd_obj"]
-        storage_keys = td.st_client.storage_accounts.list_keys(
+    def test_storage_account(self, resource_group, ssh_con, storage_account, test_vars):  # noqa: F811, E501
+        log = logging.getLogger("test_storage_account")
+        atd = test_vars["atd_obj"]
+        storage_keys = atd.st_client.storage_accounts.list_keys(
             resource_group.name,
             storage_account.name)
         storage_keys = {v.key_name: v.value for v in storage_keys.keys}
         key = storage_keys['key1']
-        print("storage_account = {}".format(storage_account.name))
-        print("key = {}".format(key))
+        log.debug("storage_account = {}".format(storage_account.name))
+        log.debug("key = {}".format(key))
         commands = """
             export AZURE_STORAGE_ACCOUNT= {0}
             export AZURE_STORAGE_ACCOUNT_KEY={1}
@@ -49,13 +48,12 @@ class TestEdasim:
         helpers.run_ssh_commands(ssh_con, commands)
         test_vars["cmd1"] = "AZURE_STORAGE_ACCOUNT=\"{}\" AZURE_STORAGE_ACCOUNT_KEY=\"{}\" ".format(storage_account.name, key)
 
-    def test_event_hub(self, test_vars, ssh_con):
-        td = test_vars["atd_obj"]
-        td.template = requests.get(
+    def test_event_hub(self, ssh_con, test_vars):  # noqa: F811
+        log = logging.getLogger("test_event_hub")
+        atd = test_vars["atd_obj"]
+        atd.template = requests.get(
                 url='https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/301-eventHub-create-authrule-namespace-and-eventHub/azuredeploy.json').json()
-        with open(os.path.expanduser(r'~/.ssh/id_rsa.pub'), 'r') as ssh_pub_f:
-            ssh_pub_key = ssh_pub_f.read()
-        td.deploy_params = {
+        atd.deploy_params = {
             'namespaceName': "edasimeventhub2",
             'namespaceAuthorizationRuleName': 'edasimeventhub',
             'eventHubName': 'edasimeventhub2',
@@ -63,14 +61,14 @@ class TestEdasim:
             'eventhubAuthorizationRuleName1': 'edasimeventhub1',
             'consumerGroupName': 'edasimtest',
         }
-
-        logging.debug('> Generated deploy parameters: \n{}'.format(
-            json.dumps(td.deploy_params, indent=4)))
-        deploy_result = helpers.wait_for_op(td.deploy())
+        atd.deploy_name = "test_event_hub"
+        log.debug('Generated deploy parameters: \n{}'.format(
+            json.dumps(atd.deploy_params, indent=4)))
+        deploy_result = helpers.wait_for_op(atd.deploy())
         test_vars["deploy_eh_outputs"] = deploy_result.properties.outputs
-        print(test_vars["deploy_eh_outputs"])
+        log.debug(test_vars["deploy_eh_outputs"])
         policy_primary_key = test_vars["deploy_eh_outputs"]["eventHubSharedAccessPolicyPrimaryKey"]["value"]
-        print(policy_primary_key)
+        log.debug("policy_primary_key = {}".format(policy_primary_key))
         commands = """
             export AZURE_EVENTHUB_SENDERKEYNAME="RootManageSharedAccessKey"
             export AZURE_EVENTHUB_SENDERKEY={0}
@@ -79,7 +77,6 @@ class TestEdasim:
         helpers.run_ssh_commands(ssh_con, commands)
         test_vars["cmd2"] = "AZURE_EVENTHUB_SENDERKEYNAME=\"RootManageSharedAccessKey\" AZURE_EVENTHUB_SENDERKEY=\"{}\" AZURE_EVENTHUB_NAMESPACENAME=\"edasimeventhub2\"".format(policy_primary_key)
 
-
     def test_edasim_setup(self, mnt_nodes, ssh_con):  # noqa: F811
         commands = """
             sudo mkdir -p /nfs/node0/bootstrap
@@ -105,29 +102,27 @@ class TestEdasim:
             """.split("\n")
         helpers.run_ssh_commands(ssh_con, commands)
 
-    def test_edasim_deploy(self, test_vars, vs_ips, ssh_con):  # noqa: F811
-        td = test_vars["atd_obj"]
-        td.template = requests.get(
-                url='https://raw.githubusercontent.com/Azure/Avere/master/src/go/cmd/edasim/deploymentartifacts/template/azuredeploy.json').json()
+    def test_edasim_deploy(self, test_vars):  # noqa: F811
+        atd = test_vars["atd_obj"]
         with open(os.path.expanduser(r'~/.ssh/id_rsa.pub'), 'r') as ssh_pub_f:
             ssh_pub_key = ssh_pub_f.read()
-        # print(">>>>> " + test_vars["cmd1"] + test_vars["cmd2"] + " <<<<<")
-        # orig_params = td.deploy_params.copy()
-        td.deploy_params = {
+        with open("{}/src/go/cmd/edasim/deploymentartifacts/template/azuredeploy.json".format(
+                  os.environ["BUILD_SOURCESDIRECTORY"])) as tfile:
+            atd.template = json.load(tfile)
+        atd.deploy_params = {
             "secureAppEnvironmentVariables": test_vars["cmd1"] + test_vars["cmd2"],
-            "uniquename": td.deploy_id,
+            "uniquename": atd.deploy_id,
             "sshKeyData": ssh_pub_key,
-            "virtualNetworkResourceGroup": td.resource_group,
-            "virtualNetworkName": td.deploy_id + "-vnet",
-            "virtualNetworkSubnetName": td.deploy_id + "-subnet",
-            "nfsCommaSeparatedAddresses": ",".join(vs_ips),
+            "virtualNetworkResourceGroup": atd.resource_group,
+            "virtualNetworkName": atd.deploy_id + "-vnet",
+            "virtualNetworkSubnetName": atd.deploy_id + "-subnet",
+            "nfsCommaSeparatedAddresses": ",".join(test_vars["cluster_vs_ips"]),
             "nfsExportPath": "/msazure",
         }
-
-        td.deploy_name = "test_edasim"
-        deploy_result = helpers.wait_for_op(td.deploy())
+        atd.deploy_name = "test_edasim_deploy"
+        deploy_result = helpers.wait_for_op(atd.deploy())
         test_vars["deploy_edasim_outputs"] = deploy_result.properties.outputs
 
 
 if __name__ == "__main__":
-    pytest.main()
+    pytest.main(sys.argv)

test/test_vdbench.py

@@ -37,15 +37,15 @@ class TestVDBench:
 
     def test_vdbench_deploy(self, test_vars):  # noqa: F811
         log = logging.getLogger("test_vdbench_deploy")
-        td = test_vars["atd_obj"]
+        atd = test_vars["atd_obj"]
         with open(os.path.expanduser(r"~/.ssh/id_rsa.pub"), "r") as ssh_pub_f:
             ssh_pub_key = ssh_pub_f.read()
         with open("{}/src/client/vmas/azuredeploy.json".format(
                   os.environ["BUILD_SOURCESDIRECTORY"])) as tfile:
-            td.template = json.load(tfile)
-        orig_params = td.deploy_params.copy()
-        td.deploy_params = {
-            "uniquename": td.deploy_id,
+            atd.template = json.load(tfile)
+        orig_params = atd.deploy_params.copy()
+        atd.deploy_params = {
+            "uniquename": atd.deploy_id,
             "sshKeyData": ssh_pub_key,
             "virtualNetworkResourceGroup": orig_params["virtualNetworkResourceGroup"],
             "virtualNetworkName": orig_params["virtualNetworkName"],
@@ -55,8 +55,8 @@ class TestVDBench:
             "nfsExportPath": "/msazure",
             "bootstrapScriptPath": "/bootstrap/bootstrap.vdbench.sh",
         }
-        td.deploy_name = "test_vdbench"
-        deploy_result = helpers.wait_for_op(td.deploy())
+        atd.deploy_name = "test_vdbench"
+        deploy_result = helpers.wait_for_op(atd.deploy())
         test_vars["deploy_vd_outputs"] = deploy_result.properties.outputs
 
     def test_vdbench_run(self, test_vars):  # noqa: F811

test/test_vfxt_cluster_status.py

@@ -91,7 +91,6 @@ class TestVfxtSupport:
         Artifacts are stored to local directories.
         """
         log = logging.getLogger("test_collect_artifacts")
-
         artifacts_dir = "vfxt_artifacts_" + test_vars["atd_obj"].deploy_id
         nodes = run_averecmd(**averecmd_params, method="node.list")
         log.debug("nodes found: {}".format(nodes))