Azure
/
Avere
зеркало из https://github.com/Azure/Avere.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Some formatting changes

This commit is contained in:
Omar Zevallos 2019-01-15 17:57:31 -05:00
Родитель 2eb9fb425c
Коммит 49a804d053
4 изменённых файлов: 117 добавлений и 116 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

36
test/lib/helpers.py
Просмотреть файл

@ -12,16 +12,16 @@ def wait_for_op(op, timeout_sec=60):
op is an AzureOperationPoller object. op is an AzureOperationPoller object.
""" """
log = logging.getLogger('wait_for_op') log = logging.getLogger("wait_for_op")
time_start = time() time_start = time()
while not op.done(): while not op.done():
op.wait(timeout=timeout_sec) op.wait(timeout=timeout_sec)
log.info('>> operation status: {0} ({1} sec)'.format( log.info(">> operation status: {0} ({1} sec)".format(
op.status(), int(time() - time_start))) op.status(), int(time() - time_start)))
result = op.result() result = op.result()
if result: if result:
log.info('>> operation result: {}'.format(result)) log.info(">> operation result: {}".format(result))
log.info('>> result.properties: {}'.format(result.properties)) log.info(">> result.properties: {}".format(result.properties))
return result return result
@ -30,8 +30,9 @@ def create_ssh_client(username, hostname, port=22, password=None):
ssh_client = paramiko.SSHClient() ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys() ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(username=username, hostname=hostname, port=port, ssh_client.connect(
password=password) username=username, hostname=hostname, port=port, password=password
)
return ssh_client return ssh_client
@ -43,28 +44,29 @@ def run_ssh_commands(ssh_client, commands):
Raises an Exception if any command fails (i.e., non-zero exit code). Raises an Exception if any command fails (i.e., non-zero exit code).
""" """
log = logging.getLogger('run_ssh_commands') log = logging.getLogger("run_ssh_commands")
for cmd in commands: for cmd in commands:
cmd = cmd.strip() cmd = cmd.strip()
if not cmd: # do not run empty "commands" if not cmd: # do not run empty "commands"
continue continue
log.debug('command to run: {}'.format(cmd)) log.debug("command to run: {}".format(cmd))
cmd_stdin, cmd_stdout, cmd_stderr = ssh_client.exec_command(cmd) cmd_stdin, cmd_stdout, cmd_stderr = ssh_client.exec_command(cmd)
cmd_rc = cmd_stdout.channel.recv_exit_status() cmd_rc = cmd_stdout.channel.recv_exit_status()
log.debug('command exit code: {}'.format(cmd_rc)) log.debug("command exit code: {}".format(cmd_rc))
cmd_stdout = ''.join(cmd_stdout.readlines()) cmd_stdout = "".join(cmd_stdout.readlines())
log.debug('command output (stdout): {}'.format(cmd_stdout)) log.debug("command output (stdout): {}".format(cmd_stdout))
cmd_stderr = ''.join(cmd_stderr.readlines()) cmd_stderr = "".join(cmd_stderr.readlines())
log.debug('command output (stderr): {}'.format(cmd_stderr)) log.debug("command output (stderr): {}".format(cmd_stderr))
if cmd_rc: if cmd_rc:
raise Exception( raise Exception(
'"{}" failed with exit code {}.\n\tSTDOUT: {}\n\tSTDERR: {}' '"{}" failed with exit code {}.\n\tSTDOUT: {}\n\tSTDERR: {}'.format(
.format(cmd, cmd_rc, cmd_stdout, cmd_stderr) cmd, cmd_rc, cmd_stdout, cmd_stderr
)
) )
@ -83,9 +85,9 @@ def split_ip_range(vserver_ips):
ip_hi = ip2.split(".")[-1] ip_hi = ip2.split(".")[-1]
ip_prefix = ".".join(ip1_split[:-1]) + "." ip_prefix = ".".join(ip1_split[:-1]) + "."
vserver_list = [ip_prefix + str(n) for n in range(int(ip_low), int(ip_hi)+1)] vserver_list = [ip_prefix + str(n) for n in range(int(ip_low), int(ip_hi) + 1)]
return vserver_list return vserver_list
if __name__ == '__main__': if __name__ == "__main__":
pass pass

51
test/lib/pytest_fixtures.py
Просмотреть файл

@ -12,43 +12,42 @@ from lib import helpers
# FIXTURES #################################################################### # FIXTURES ####################################################################
@pytest.fixture(scope='module') @pytest.fixture(scope="module")
def group_vars(): def group_vars():
""" """
Instantiates an ArmTemplateDeploy object, creates the resource group as Instantiates an ArmTemplateDeploy object, creates the resource group as
test-group setup, and deletes the resource group as test-group teardown. test-group setup, and deletes the resource group as test-group teardown.
""" """
log = logging.getLogger('group_vars') log = logging.getLogger("group_vars")
vars = {} vars = {}
if 'VFXT_TEST_VARS_FILE' in os.environ and \ if "VFXT_TEST_VARS_FILE" in os.environ and \
os.path.isfile(os.environ['VFXT_TEST_VARS_FILE']): os.path.isfile(os.environ["VFXT_TEST_VARS_FILE"]):
log.debug('Loading into vars from {} (VFXT_TEST_VARS_FILE)'.format( log.debug("Loading into vars from {} (VFXT_TEST_VARS_FILE)".format(
os.environ['VFXT_TEST_VARS_FILE'])) os.environ["VFXT_TEST_VARS_FILE"]))
with open(os.environ['VFXT_TEST_VARS_FILE'], 'r') as vtvf: with open(os.environ["VFXT_TEST_VARS_FILE"], "r") as vtvf:
vars = {**vars, **json.load(vtvf)} vars = {**vars, **json.load(vtvf)}
log.debug('Loaded the following JSON into vars: {}'.format( log.debug("Loaded the following JSON into vars: {}".format(
json.dumps(vars, sort_keys=True, indent=4))) json.dumps(vars, sort_keys=True, indent=4)))
vars['atd_obj'] = ArmTemplateDeploy(_fields=vars.pop('atd_obj', {})) vars["atd_obj"] = ArmTemplateDeploy(_fields=vars.pop("atd_obj", {}))
rg = vars['atd_obj'].create_resource_group() rg = vars["atd_obj"].create_resource_group()
log.info('Created Resource Group: {}'.format(rg)) log.info("Created Resource Group: {}".format(rg))
yield vars yield vars
vars['atd_obj'] = json.loads(vars['atd_obj'].serialize()) vars["atd_obj"] = json.loads(vars["atd_obj"].serialize())
if 'VFXT_TEST_VARS_FILE' in os.environ: if "VFXT_TEST_VARS_FILE" in os.environ:
log.debug('vars: {}'.format( log.debug("vars: {}".format(json.dumps(vars, sort_keys=True, indent=4)))
json.dumps(vars, sort_keys=True, indent=4))) log.debug("Saving vars to {} (VFXT_TEST_VARS_FILE)".format(
log.debug('Saving vars to {} (VFXT_TEST_VARS_FILE)'.format( os.environ["VFXT_TEST_VARS_FILE"]))
os.environ['VFXT_TEST_VARS_FILE'])) with open(os.environ["VFXT_TEST_VARS_FILE"], "w") as vtvf:
with open(os.environ['VFXT_TEST_VARS_FILE'], 'w') as vtvf:
json.dump(vars, vtvf) json.dump(vars, vtvf)
@pytest.fixture() @pytest.fixture()
def ssh_client(group_vars): def ssh_client(group_vars):
client = helpers.create_ssh_client(group_vars['controller_user'], client = helpers.create_ssh_client(group_vars["controller_user"],
group_vars['controller_ip']) group_vars["controller_ip"])
yield client yield client
client.close() client.close()
@ -62,11 +61,11 @@ def scp_client(ssh_client):
@pytest.fixture() @pytest.fixture()
def vserver_ip_list(group_vars): def vserver_ip_list(group_vars):
if 'vserver_ip_list' not in group_vars: if "vserver_ip_list" not in group_vars:
vserver_ips = group_vars['deploy_outputs']["vserveR_IPS"]["value"] vserver_ips = group_vars["deploy_outputs"]["vserveR_IPS"]["value"]
group_vars['vserver_ip_list'] = helpers.split_ip_range(vserver_ips) group_vars["vserver_ip_list"] = helpers.split_ip_range(vserver_ips)
return group_vars['vserver_ip_list'] return group_vars["vserver_ip_list"]
if __name__ == '__main__': if __name__ == "__main__":
pytest.main() pytest.main()

85
test/test_avere_template_deploy.py
Просмотреть файл

@ -17,82 +17,81 @@ from lib.pytest_fixtures import (group_vars, scp_client, ssh_client,
# TEST CASES ################################################################## # TEST CASES ##################################################################
class TestDeployment: class TestDeployment:
def test_deploy_template(self, group_vars): def test_deploy_template(self, group_vars):
log = logging.getLogger('test_deploy_template') log = logging.getLogger("test_deploy_template")
td = group_vars['atd_obj'] td = group_vars["atd_obj"]
with open('{}/src/vfxt/azuredeploy-auto.json'.format( with open("{}/src/vfxt/azuredeploy-auto.json".format(
os.environ['BUILD_SOURCESDIRECTORY'])) as tfile: os.environ["BUILD_SOURCESDIRECTORY"])) as tfile:
td.template = json.load(tfile) td.template = json.load(tfile)
with open(os.path.expanduser(r'~/.ssh/id_rsa.pub'), 'r') as ssh_pub_f: with open(os.path.expanduser(r"~/.ssh/id_rsa.pub"), "r") as ssh_pub_f:
ssh_pub_key = ssh_pub_f.read() ssh_pub_key = ssh_pub_f.read()
td.deploy_params = { td.deploy_params = {
'virtualNetworkResourceGroup': td.resource_group, "virtualNetworkResourceGroup": td.resource_group,
'virtualNetworkName': td.deploy_id + '-vnet', "virtualNetworkName": td.deploy_id + "-vnet",
'virtualNetworkSubnetName': td.deploy_id + '-subnet', "virtualNetworkSubnetName": td.deploy_id + "-subnet",
'avereBackedStorageAccountName': td.deploy_id + 'sa', "avereBackedStorageAccountName": td.deploy_id + "sa",
'controllerName': td.deploy_id + '-con', "controllerName": td.deploy_id + "-con",
'controllerAdminUsername': 'azureuser', "controllerAdminUsername": "azureuser",
'controllerAuthenticationType': 'sshPublicKey', "controllerAuthenticationType": "sshPublicKey",
'controllerSSHKeyData': ssh_pub_key, "controllerSSHKeyData": ssh_pub_key,
'adminPassword': os.environ['AVERE_ADMIN_PW'], "adminPassword": os.environ["AVERE_ADMIN_PW"],
'controllerPassword': os.environ['AVERE_CONTROLLER_PW'] "controllerPassword": os.environ["AVERE_CONTROLLER_PW"],
} }
group_vars['controller_name'] = td.deploy_params['controllerName'] group_vars["controller_name"] = td.deploy_params["controllerName"]
group_vars['controller_user'] = td.deploy_params['controllerAdminUsername'] group_vars["controller_user"] = td.deploy_params["controllerAdminUsername"]
log.debug('Generated deploy parameters: \n{}'.format( log.debug("Generated deploy parameters: \n{}".format(
json.dumps(td.deploy_params, indent=4))) json.dumps(td.deploy_params, indent=4)))
td.deploy_name = 'test_deploy_template' td.deploy_name = "test_deploy_template"
try: try:
deploy_result = helpers.wait_for_op(td.deploy()) deploy_result = helpers.wait_for_op(td.deploy())
group_vars['deploy_outputs'] = deploy_result.properties.outputs group_vars["deploy_outputs"] = deploy_result.properties.outputs
finally: finally:
group_vars['controller_ip'] = td.nm_client.public_ip_addresses.get( group_vars["controller_ip"] = td.nm_client.public_ip_addresses.get(
td.resource_group, td.resource_group, "publicip-" + group_vars["controller_name"]
'publicip-' + group_vars['controller_name']).ip_address ).ip_address
def test_get_vfxt_log(self, group_vars, scp_client): def test_get_vfxt_log(self, group_vars, scp_client):
log = logging.getLogger('test_get_vfxt_log') log = logging.getLogger("test_get_vfxt_log")
log.info('Getting vfxt.log from controller: {}'.format( log.info("Getting vfxt.log from controller: {}".format(
group_vars['controller_name'])) group_vars["controller_name"]))
scp_client.get(r'~/vfxt.log', scp_client.get(r"~/vfxt.log",
r'./vfxt.' + group_vars['controller_name'] + '.log') r"./vfxt." + group_vars["controller_name"] + ".log")
def test_mount_nodes_on_controller(self, vserver_ip_list, ssh_client): def test_mount_nodes_on_controller(self, vserver_ip_list, ssh_client):
commands = """ commands = """
sudo apt-get update sudo apt-get update
sudo apt-get install nfs-common sudo apt-get install nfs-common
""".split('\n') """.split("\n")
for i, vs_ip in enumerate(vserver_ip_list): for i, vs_ip in enumerate(vserver_ip_list):
commands.append('sudo mkdir -p /nfs/node{}'.format(i)) commands.append("sudo mkdir -p /nfs/node{}".format(i))
commands.append('sudo chown nobody:nogroup /nfs/node{}'.format(i)) commands.append("sudo chown nobody:nogroup /nfs/node{}".format(i))
fstab_line = "{}:/msazure /nfs/node{} nfs ".format(vs_ip, i) + \ fstab_line = "{}:/msazure /nfs/node{} nfs ".format(vs_ip, i) + \
"hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0" "hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0"
commands.append('sudo sh -c \'echo "{}" >> /etc/fstab\''.format( commands.append("sudo sh -c 'echo \"{}\" >> /etc/fstab'".format(
fstab_line)) fstab_line))
commands.append('sudo mount -a') commands.append("sudo mount -a")
helpers.run_ssh_commands(ssh_client, commands) helpers.run_ssh_commands(ssh_client, commands)
def test_ping_nodes(self, vserver_ip_list, ssh_client): def test_ping_nodes(self, vserver_ip_list, ssh_client):
commands = [] commands = []
for vs_ip in vserver_ip_list: for vs_ip in vserver_ip_list:
commands.append('ping -c 3 {}'.format(vs_ip)) commands.append("ping -c 3 {}".format(vs_ip))
helpers.run_ssh_commands(ssh_client, commands) helpers.run_ssh_commands(ssh_client, commands)
def test_node_basic_fileops(self, group_vars, ssh_client, scp_client): def test_node_basic_fileops(self, group_vars, ssh_client, scp_client):
script_name = 'check_node_basic_fileops.sh' script_name = "check_node_basic_fileops.sh"
scp_client.put('{0}/test/{1}'.format( scp_client.put("{0}/test/{1}".format(
os.environ['BUILD_SOURCESDIRECTORY'], script_name), os.environ["BUILD_SOURCESDIRECTORY"], script_name),
r'~/.') r"~/.")
commands = """ commands = """
chmod +x {0} chmod +x {0}
./{0} ./{0}
""".format(script_name).split('\n') """.format(script_name).split("\n")
helpers.run_ssh_commands(ssh_client, commands) helpers.run_ssh_commands(ssh_client, commands)
if __name__ == '__main__': if __name__ == "__main__":
pytest.main() pytest.main()

61
test/test_vdbench.py
Просмотреть файл

@ -18,7 +18,6 @@ from sshtunnel import SSHTunnelForwarder
class VDBench: class VDBench:
def test_vdbench_setup(self, group_vars, ssh_client): def test_vdbench_setup(self, group_vars, ssh_client):
# TODO: Ensure nodes are mounted on controller. (fixture?) # TODO: Ensure nodes are mounted on controller. (fixture?)
commands = """ commands = """
@ -29,60 +28,62 @@ class VDBench:
sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbenchVerify.sh https://raw.githubusercontent.com/Azure/Avere/master/src/clientapps/vdbench/vdbenchVerify.sh sudo curl --retry 5 --retry-delay 5 -o /nfs/node0/bootstrap/vdbenchVerify.sh https://raw.githubusercontent.com/Azure/Avere/master/src/clientapps/vdbench/vdbenchVerify.sh
sudo chmod +x /nfs/node0/bootstrap/vdbenchVerify.sh sudo chmod +x /nfs/node0/bootstrap/vdbenchVerify.sh
/nfs/node0/bootstrap/vdbenchVerify.sh /nfs/node0/bootstrap/vdbenchVerify.sh
""".split('\n') """.split("\n")
helpers.run_ssh_commands(ssh_client, commands) helpers.run_ssh_commands(ssh_client, commands)
def test_vdbench_deploy(self, group_vars, vserver_ip_list): def test_vdbench_deploy(self, group_vars, vserver_ip_list):
td = group_vars['atd_obj'] td = group_vars["atd_obj"]
with open(os.path.expanduser(r'~/.ssh/id_rsa.pub'), 'r') as ssh_pub_f: with open(os.path.expanduser(r"~/.ssh/id_rsa.pub"), "r") as ssh_pub_f:
ssh_pub_key = ssh_pub_f.read() ssh_pub_key = ssh_pub_f.read()
with open('{}/src/client/vmas/azuredeploy.json'.format( with open("{}/src/client/vmas/azuredeploy.json".format(
os.environ['BUILD_SOURCESDIRECTORY'])) as tfile: os.environ["BUILD_SOURCESDIRECTORY"])) as tfile:
td.template = json.load(tfile) td.template = json.load(tfile)
orig_params = td.deploy_params.copy() orig_params = td.deploy_params.copy()
td.deploy_params = { td.deploy_params = {
'uniquename': td.deploy_id, "uniquename": td.deploy_id,
'sshKeyData': ssh_pub_key, "sshKeyData": ssh_pub_key,
'virtualNetworkResourceGroup': orig_params['virtualNetworkResourceGroup'], "virtualNetworkResourceGroup": orig_params["virtualNetworkResourceGroup"],
'virtualNetworkName': orig_params['virtualNetworkName'], "virtualNetworkName": orig_params["virtualNetworkName"],
'virtualNetworkSubnetName': orig_params['virtualNetworkSubnetName'], "virtualNetworkSubnetName": orig_params["virtualNetworkSubnetName"],
'nfsCommaSeparatedAddresses': ','.join(vserver_ip_list), "nfsCommaSeparatedAddresses": ",".join(vserver_ip_list),
'vmCount': 12, "vmCount": 12,
'nfsExportPath': '/msazure', "nfsExportPath": "/msazure",
'bootstrapScriptPath': '/bootstrap/bootstrap.vdbench.sh' "bootstrapScriptPath": "/bootstrap/bootstrap.vdbench.sh",
} }
td.deploy_name = 'test_vdbench' td.deploy_name = "test_vdbench"
deploy_result = helpers.wait_for_op(td.deploy()) deploy_result = helpers.wait_for_op(td.deploy())
group_vars['deploy_vd_outputs'] = deploy_result.properties.outputs group_vars["deploy_vd_outputs"] = deploy_result.properties.outputs
def test_vdbench_template_run(self, group_vars): def test_vdbench_template_run(self, group_vars):
node_ip = group_vars['deploy_vd_outputs']["nodE_0_IP_ADDRESS"]["value"] node_ip = group_vars["deploy_vd_outputs"]["nodE_0_IP_ADDRESS"]["value"]
with SSHTunnelForwarder( with SSHTunnelForwarder(
group_vars['controller_ip'], group_vars["controller_ip"],
ssh_username=group_vars['controller_user'], ssh_username=group_vars["controller_user"],
ssh_pkey=os.path.expanduser(r'~/.ssh/id_rsa'), ssh_pkey=os.path.expanduser(r"~/.ssh/id_rsa"),
remote_bind_address=(node_ip, 22) remote_bind_address=(node_ip, 22),
) as ssh_tunnel: ) as ssh_tunnel:
sleep(1) sleep(1)
try: try:
ssh_client = helpers.create_ssh_client( ssh_client = helpers.create_ssh_client(
group_vars['controller_user'], group_vars["controller_user"],
'127.0.0.1', "127.0.0.1",
ssh_tunnel.local_bind_port) ssh_tunnel.local_bind_port,
)
scp_client = SCPClient(ssh_client.get_transport()) scp_client = SCPClient(ssh_client.get_transport())
try: try:
scp_client.put(os.path.expanduser(r'~/.ssh/id_rsa'), scp_client.put(os.path.expanduser(r"~/.ssh/id_rsa"),
r'~/.ssh/id_rsa') r"~/.ssh/id_rsa")
finally: finally:
scp_client.close() scp_client.close()
commands = """~/copy_idrsa.sh commands = """
~/copy_idrsa.sh
cd cd
""".split('\n') """.split("\n")
# ./run_vdbench.sh inmem.conf uniquestring1 # TODO: reenable # ./run_vdbench.sh inmem.conf uniquestring1 # TODO: reenable
helpers.run_ssh_commands(ssh_client, commands) helpers.run_ssh_commands(ssh_client, commands)
finally: finally:
ssh_client.close() ssh_client.close()
if __name__ == '__main__': if __name__ == "__main__":
pytest.main() pytest.main()