diff --git a/test/arm_template_deploy.py b/test/arm_template_deploy.py index 45131750..12cb42be 100644 --- a/test/arm_template_deploy.py +++ b/test/arm_template_deploy.py @@ -98,8 +98,11 @@ class ArmTemplateDeploy: This method returns the JSON string. """ _this = self.__dict__ - _this.pop('rm_client', None) # don't want to save these + _this.pop('rm_client', None) # don't want to save these for security _this.pop('nm_client', None) + _this.pop('st_client', None) + _this.pop('template', None) + _this.pop('deploy_params', None) if to_file: with open(to_file, 'w') as tf: diff --git a/test/lib/helpers.py b/test/lib/helpers.py index 7f8b07dd..78ea26e5 100644 --- a/test/lib/helpers.py +++ b/test/lib/helpers.py @@ -16,19 +16,20 @@ def create_ssh_client(username, hostname, port=22, password=None): return ssh_client -def run_averecmd(ssh_client, node_ip, password, method, user='admin', args=''): +def run_averecmd(ssh_client, node_ip, password, method, user='admin', args='', + timeout=60): """Run averecmd on the vFXT controller connected via ssh_client.""" cmd = "averecmd --raw --no-check-certificate " + \ "--user {0} --password {1} --server {2} {3} {4}".format( user, password, node_ip, method, args) - result = run_ssh_command(ssh_client, cmd)['stdout'] + result = run_ssh_command(ssh_client, cmd, timeout=timeout)['stdout'] try: return ast.literal_eval(result) except (ValueError, SyntaxError): return str(result).strip() # could not eval, return as a string -def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False): +def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False, timeout=None): """ Run a command on the server connected via ssh_client. @@ -38,33 +39,33 @@ def run_ssh_command(ssh_client, command, ignore_nonzero_rc=False): log = logging.getLogger("run_ssh_command") log.debug("command to run: {}".format(command)) - cmd_stdin, cmd_stdout, cmd_stderr = ssh_client.exec_command(command) + cmd_in, cmd_out, cmd_err = ssh_client.exec_command(command, timeout=timeout) - cmd_rc = cmd_stdout.channel.recv_exit_status() + cmd_rc = cmd_out.channel.recv_exit_status() log.debug("command exit code: {}".format(cmd_rc)) - cmd_stdout = "".join(cmd_stdout.readlines()) - log.debug("command output (stdout): {}".format(cmd_stdout)) + cmd_out = "".join(cmd_out.readlines()) + log.debug("command output (stdout): {}".format(cmd_out)) - cmd_stderr = "".join(cmd_stderr.readlines()) - log.debug("command output (stderr): {}".format(cmd_stderr)) + cmd_err = "".join(cmd_err.readlines()) + log.debug("command output (stderr): {}".format(cmd_err)) if cmd_rc and not ignore_nonzero_rc: log.error( '"{}" failed with exit code {}.\n\tSTDOUT: {}\n\tSTDERR: {}'.format( - command, cmd_rc, cmd_stdout, cmd_stderr) + command, cmd_rc, cmd_out, cmd_err) ) assert(0 == cmd_rc) return { "command": command, - "stdout": cmd_stdout, - "stderr": cmd_stderr, + "stdout": cmd_out, + "stderr": cmd_err, "rc": cmd_rc } -def run_ssh_commands(ssh_client, commands, ignore_nonzero_rc=False): +def run_ssh_commands(ssh_client, commands, **kwargs): """ Runs a list of commands on the server connected via ssh_client. @@ -77,7 +78,7 @@ def run_ssh_commands(ssh_client, commands, ignore_nonzero_rc=False): cmd = cmd.strip() log.debug("command to run: {}".format(cmd)) if cmd: # only run non-empty commands - results.append(run_ssh_command(ssh_client, cmd, ignore_nonzero_rc)) + results.append(run_ssh_command(ssh_client, cmd, **kwargs)) return results diff --git a/test/lib/pytest_fixtures.py b/test/lib/pytest_fixtures.py index 76d4b553..b8b27987 100644 --- a/test/lib/pytest_fixtures.py +++ b/test/lib/pytest_fixtures.py @@ -13,7 +13,7 @@ from lib import helpers def averecmd_params(ssh_con, test_vars): return { "ssh_client": ssh_con, - "password": test_vars["atd_obj"].deploy_params["adminPassword"], + "password": os.environ["AVERE_ADMIN_PW"], "node_ip": test_vars["deploy_outputs"]["mgmt_ip"]["value"] } diff --git a/test/test_avere_template_deploy.py b/test/test_avere_template_deploy.py index b537490f..8e7a5328 100755 --- a/test/test_avere_template_deploy.py +++ b/test/test_avere_template_deploy.py @@ -35,7 +35,8 @@ class TestVfxtTemplateDeploy: "controllerSSHKeyData": ssh_pub_key, "adminPassword": os.environ["AVERE_ADMIN_PW"], "controllerPassword": os.environ["AVERE_CONTROLLER_PW"], - "enableCloudTraceDebugging": True + "enableCloudTraceDebugging": True, + "avereInstanceType": "Standard_D16s_v3" } test_vars["controller_name"] = td.deploy_params["controllerName"] test_vars["controller_user"] = td.deploy_params["controllerAdminUsername"] diff --git a/test/test_vfxt_cluster_status.py b/test/test_vfxt_cluster_status.py index f414beaa..9d03a0e4 100644 --- a/test/test_vfxt_cluster_status.py +++ b/test/test_vfxt_cluster_status.py @@ -6,13 +6,18 @@ Driver for testing Azure ARM template-based deployment of the Avere vFXT. import logging import os +import tarfile +from time import sleep import pytest +from scp import SCPClient -from lib.helpers import run_averecmd, run_ssh_commands, upload_gsi +from lib.helpers import (create_ssh_client, run_averecmd, run_ssh_commands, + upload_gsi) from lib.pytest_fixtures import (averecmd_params, mnt_nodes, # noqa: F401 resource_group, scp_cli, ssh_con, test_vars, vs_ips) +from sshtunnel import SSHTunnelForwarder class TestVfxtClusterStatus: @@ -60,6 +65,61 @@ class TestVfxtClusterStatus: assert(not cores_found) + def test_artifacts_collect(self, averecmd_params, scp_cli, test_vars): # noqa: F811 + log = logging.getLogger("test_collect_artifacts") + + artifacts_dir = "vfxt_artifacts_" + test_vars["atd_obj"].deploy_id + nodes = run_averecmd(**averecmd_params, method="node.list") + for node in nodes: + node_dir = artifacts_dir + "/" + node + node_dir_log = node_dir + "/log" + node_dir_trace = node_dir + "/trace" + + os.makedirs(node_dir_trace, exist_ok=True) + os.makedirs(node_dir_log, exist_ok=True) + + node_ip = run_averecmd(**averecmd_params, + method="node.get", + args=node)[node]["primaryClusterIP"]["IP"] + log.debug("node {}, using IP {}".format(node, node_ip)) + with SSHTunnelForwarder( + test_vars["controller_ip"], + ssh_username=test_vars["controller_user"], + ssh_pkey=os.path.expanduser(r"~/.ssh/id_rsa"), + remote_bind_address=(node_ip, 22), + ) as ssh_tunnel: + sleep(1) + try: + ssh_client = create_ssh_client( + "admin", + "127.0.0.1", + ssh_tunnel.local_bind_port, + password=os.environ["AVERE_ADMIN_PW"] + ) + scp_client = SCPClient(ssh_client.get_transport()) + try: + var_log_files = ["messages", "xmlrpc.log"] + for f in var_log_files: + if not f.strip(): + continue + scp_client.get("/var/log/" + f.strip(), + node_dir_log, recursive=True) + scp_client.get("/support/trace/rolling", + node_dir_trace, recursive=True) + + # TODO: 2019-0125: Turned off until for now. + # scp_client.get("/support/gsi", + # node_dir, recursive=True) + # scp_client.get("/support/cores", + # node_dir, recursive=True) + finally: + scp_client.close() + finally: + ssh_client.close() + + with tarfile.open(artifacts_dir + ".tar.gz", "w:gz") as tar: + tar.add(artifacts_dir, arcname=os.path.basename(artifacts_dir)) + if __name__ == "__main__": pytest.main()