Added test_artifacts_collect

test/arm_template_deploy.py

@@ -98,8 +98,11 @@ class ArmTemplateDeploy:
         This method returns the JSON string.
         """
         _this = self.__dict__
-        _this.pop('rm_client', None)  # don't want to save these
+        _this.pop('rm_client', None)  # don't want to save these for security
         _this.pop('nm_client', None)
+        _this.pop('st_client', None)
+        _this.pop('template', None)
+        _this.pop('deploy_params', None)
 
         if to_file:
             with open(to_file, 'w') as tf:

test/lib/helpers.py

@@ -16,19 +16,20 @@ def create_ssh_client(username, hostname, port=22, password=None):
     return ssh_client
 
 
-def run_averecmd(ssh_client, node_ip, password, method, user='admin', args=''):
+def run_averecmd(ssh_client, node_ip, password, method, user='admin', args='',
+                 timeout=60):
     """Run averecmd on the vFXT controller connected via ssh_client."""
     cmd = "averecmd --raw --no-check-certificate " + \
           "--user {0} --password {1} --server {2} {3} {4}".format(
             user, password, node_ip, method, args)
-    result = run_ssh_command(ssh_client, cmd)['stdout']
+    result = run_ssh_command(ssh_client, cmd, timeout=timeout)['stdout']
     try:
         return ast.literal_eval(result)
     except (ValueError, SyntaxError):
         return str(result).strip()  # could not eval, return as a string
 
 
-def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False):
+def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False, timeout=None):
     """
     Run a command on the server connected via ssh_client.
 
@@ -38,33 +39,33 @@ def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False):
     log = logging.getLogger("run_ssh_command")
 
     log.debug("command to run: {}".format(command))
-    cmd_stdin, cmd_stdout, cmd_stderr = ssh_client.exec_command(command)
+    cmd_in, cmd_out, cmd_err = ssh_client.exec_command(command, timeout=timeout)
 
-    cmd_rc = cmd_stdout.channel.recv_exit_status()
+    cmd_rc = cmd_out.channel.recv_exit_status()
     log.debug("command exit code: {}".format(cmd_rc))
 
-    cmd_stdout = "".join(cmd_stdout.readlines())
-    log.debug("command output (stdout): {}".format(cmd_stdout))
+    cmd_out = "".join(cmd_out.readlines())
+    log.debug("command output (stdout): {}".format(cmd_out))
 
-    cmd_stderr = "".join(cmd_stderr.readlines())
-    log.debug("command output (stderr): {}".format(cmd_stderr))
+    cmd_err = "".join(cmd_err.readlines())
+    log.debug("command output (stderr): {}".format(cmd_err))
 
     if cmd_rc and not ignore_nonzero_rc:
         log.error(
             '"{}" failed with exit code {}.\n\tSTDOUT: {}\n\tSTDERR: {}'.format(
-                command, cmd_rc, cmd_stdout, cmd_stderr)
+                command, cmd_rc, cmd_out, cmd_err)
         )
         assert(0 == cmd_rc)
 
     return {
         "command": command,
-        "stdout": cmd_stdout,
-        "stderr": cmd_stderr,
+        "stdout": cmd_out,
+        "stderr": cmd_err,
         "rc": cmd_rc
     }
 
 
-def run_ssh_commands(ssh_client, commands, ignore_nonzero_rc=False):
+def run_ssh_commands(ssh_client, commands, **kwargs):
     """
     Runs a list of commands on the server connected via ssh_client.
 
@@ -77,7 +78,7 @@ def run_ssh_commands(ssh_client, commands, ignore_nonzero_rc=False):
         cmd = cmd.strip()
         log.debug("command to run: {}".format(cmd))
         if cmd:  # only run non-empty commands
-            results.append(run_ssh_command(ssh_client, cmd, ignore_nonzero_rc))
+            results.append(run_ssh_command(ssh_client, cmd, **kwargs))
     return results
 
 

test/lib/pytest_fixtures.py

@@ -13,7 +13,7 @@ from lib import helpers
 def averecmd_params(ssh_con, test_vars):
     return {
         "ssh_client": ssh_con,
-        "password": test_vars["atd_obj"].deploy_params["adminPassword"],
+        "password": os.environ["AVERE_ADMIN_PW"],
         "node_ip": test_vars["deploy_outputs"]["mgmt_ip"]["value"]
     }
 

test/test_avere_template_deploy.py

@@ -35,7 +35,8 @@ class TestVfxtTemplateDeploy:
             "controllerSSHKeyData": ssh_pub_key,
             "adminPassword": os.environ["AVERE_ADMIN_PW"],
             "controllerPassword": os.environ["AVERE_CONTROLLER_PW"],
-            "enableCloudTraceDebugging": True
+            "enableCloudTraceDebugging": True,
+            "avereInstanceType": "Standard_D16s_v3"
         }
         test_vars["controller_name"] = td.deploy_params["controllerName"]
         test_vars["controller_user"] = td.deploy_params["controllerAdminUsername"]

test/test_vfxt_cluster_status.py

@@ -6,13 +6,18 @@ Driver for testing Azure ARM template-based deployment of the Avere vFXT.
 
 import logging
 import os
+import tarfile
+from time import sleep
 
 import pytest
+from scp import SCPClient
 
-from lib.helpers import run_averecmd, run_ssh_commands, upload_gsi
+from lib.helpers import (create_ssh_client, run_averecmd, run_ssh_commands,
+                         upload_gsi)
 from lib.pytest_fixtures import (averecmd_params, mnt_nodes,  # noqa: F401
                                  resource_group, scp_cli, ssh_con, test_vars,
                                  vs_ips)
+from sshtunnel import SSHTunnelForwarder
 
 
 class TestVfxtClusterStatus:
@@ -60,6 +65,61 @@ class TestVfxtClusterStatus:
 
         assert(not cores_found)
 
+    def test_artifacts_collect(self, averecmd_params, scp_cli, test_vars):  # noqa: F811
+        log = logging.getLogger("test_collect_artifacts")
+
+        artifacts_dir = "vfxt_artifacts_" + test_vars["atd_obj"].deploy_id
+        nodes = run_averecmd(**averecmd_params, method="node.list")
+        for node in nodes:
+            node_dir = artifacts_dir + "/" + node
+            node_dir_log = node_dir + "/log"
+            node_dir_trace = node_dir + "/trace"
+
+            os.makedirs(node_dir_trace, exist_ok=True)
+            os.makedirs(node_dir_log, exist_ok=True)
+
+            node_ip = run_averecmd(**averecmd_params,
+                                   method="node.get",
+                                   args=node)[node]["primaryClusterIP"]["IP"]
+            log.debug("node {}, using IP {}".format(node, node_ip))
+            with SSHTunnelForwarder(
+                test_vars["controller_ip"],
+                ssh_username=test_vars["controller_user"],
+                ssh_pkey=os.path.expanduser(r"~/.ssh/id_rsa"),
+                remote_bind_address=(node_ip, 22),
+            ) as ssh_tunnel:
+                sleep(1)
+                try:
+                    ssh_client = create_ssh_client(
+                        "admin",
+                        "127.0.0.1",
+                        ssh_tunnel.local_bind_port,
+                        password=os.environ["AVERE_ADMIN_PW"]
+                    )
+                    scp_client = SCPClient(ssh_client.get_transport())
+                    try:
+                        var_log_files = ["messages", "xmlrpc.log"]
+                        for f in var_log_files:
+                            if not f.strip():
+                                continue
+                            scp_client.get("/var/log/" + f.strip(),
+                                           node_dir_log, recursive=True)
+                        scp_client.get("/support/trace/rolling",
+                                       node_dir_trace, recursive=True)
+
+                        # TODO: 2019-0125: Turned off until for now.
+                        # scp_client.get("/support/gsi",
+                        #                node_dir, recursive=True)
+                        # scp_client.get("/support/cores",
+                        #                node_dir, recursive=True)
+                    finally:
+                        scp_client.close()
+                finally:
+                    ssh_client.close()
+
+        with tarfile.open(artifacts_dir + ".tar.gz", "w:gz") as tar:
+            tar.add(artifacts_dir, arcname=os.path.basename(artifacts_dir))
+
 
 if __name__ == "__main__":
     pytest.main()