From 9b429e0f19eb6cc7737beded0f1a760e3938c4a1 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Wed, 13 Feb 2019 17:19:24 -0500 Subject: [PATCH 01/13] BYO VNET vFXT deployment support --- src/vfxt/azuredeploy.vnet.json | 88 +++++++++++++++++++++++++++++++ test/conftest.py | 27 +++++++++- test/test_vfxt_template_deploy.py | 64 ++++++++++++++++++---- 3 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 src/vfxt/azuredeploy.vnet.json diff --git a/src/vfxt/azuredeploy.vnet.json b/src/vfxt/azuredeploy.vnet.json new file mode 100644 index 00000000..452509a6 --- /dev/null +++ b/src/vfxt/azuredeploy.vnet.json @@ -0,0 +1,88 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "virtualNetworkName": { + "type": "string", + "metadata": { + "description": "The name of the virtual network (VNET)." + } + }, + "virtualNetworkSubnetName": { + "type": "string", + "metadata": { + "description": "The name of the subnet in the VNET." + } + }, + "vnetAddressSpacePrefix": { + "type": "string", + "defaultValue": "10.0.0.0/16", + "metadata": { + "description": "The IP address prefix of the virtual network (VNET)." + } + }, + "subnetAddressRangePrefix": { + "type": "string", + "defaultValue": "10.0.0.0/24", + "metadata": { + "description": "The IP address range prefix of the subnet in the VNET." + } + } + }, + "variables": { + "virtualNetworkName": "[parameters('virtualNetworkName')]", + "subnetName": "[parameters('virtualNetworkSubnetName')]", + "addressPrefix": "[parameters('vnetAddressSpacePrefix')]", + "subnetPrefix": "[parameters('subnetAddressRangePrefix')]" + }, + "resources": [ + { + "apiVersion": "2017-10-01", + "type": "Microsoft.Network/virtualNetworks", + "name": "[variables('virtualNetworkName')]", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "[variables('addressPrefix')]" + ] + }, + "subnets": [ + { + "name": "[variables('subnetName')]", + "properties": { + "addressPrefix": "[variables('subnetPrefix')]", + "serviceEndpoints": [ + { + "service": "Microsoft.Storage" + } + ] + } + } + ] + } + } + ], + "outputs": { + "resource_group": { + "type": "string", + "value": "[resourceGroup().name]" + }, + "location": { + "type": "string", + "value": "[resourceGroup().location]" + }, + "virtual_network_name": { + "type": "string", + "value": "[variables('virtualNetworkName')]" + }, + "subnet_name": { + "type": "string", + "value": "[variables('subnetName')]" + }, + "subnet_id": { + "type": "string", + "value": "[concat(resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName')),'/subnets/',variables('subnetName'))]" + } + } +} \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index a8bc1087..17fcac4f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -5,11 +5,12 @@ import os # from requirements.txt import pytest -from arm_template_deploy import ArmTemplateDeploy from scp import SCPClient # local libraries -from lib.helpers import (create_ssh_client, run_ssh_command, run_ssh_commands) +from arm_template_deploy import ArmTemplateDeploy +from lib.helpers import (create_ssh_client, run_ssh_command, run_ssh_commands, + wait_for_op) # COMMAND-LINE OPTIONS ######################################################## @@ -188,3 +189,25 @@ def test_vars(request): log.debug("Saving vars to {} (test_vars_file)".format(test_vars_file)) with open(test_vars_file, "w") as vtvf: json.dump(vars, vtvf, **cja) + + +@pytest.fixture() +def ext_vnet(test_vars): + log = logging.getLogger("ext_vnet") + vnet_atd = ArmTemplateDeploy( + resource_group=test_vars["atd_obj"].deploy_id + "-vnet-rg" + ) + rg = vnet_atd.create_resource_group() + log.info("Resource Group: {}".format(rg)) + + vnet_atd.deploy_name = "ext_vnet" + with open("{}/src/vfxt/azuredeploy.vnet.json".format( + test_vars["build_root"])) as tfile: + vnet_atd.template = json.load(tfile) + vnet_atd.deploy_params = { + "virtualNetworkName": test_vars["atd_obj"].deploy_id + "-vnet", + "virtualNetworkSubnetName": test_vars["atd_obj"].deploy_id + "-subnet", + } + test_vars["ext_vnet"] = wait_for_op(vnet_atd.deploy()).properties.outputs + log.debug(test_vars["ext_vnet"]) + return test_vars["ext_vnet"] diff --git a/test/test_vfxt_template_deploy.py b/test/test_vfxt_template_deploy.py index 493ec2a3..84b00e23 100755 --- a/test/test_vfxt_template_deploy.py +++ b/test/test_vfxt_template_deploy.py @@ -28,21 +28,23 @@ class TestVfxtTemplateDeploy: with open(test_vars["ssh_pub_key"], "r") as ssh_pub_f: ssh_pub_key = ssh_pub_f.read() atd.deploy_params = { - "avereInstanceType": "Standard_D16s_v3", - "avereClusterName": atd.deploy_id + "-cluster", - "virtualNetworkResourceGroup": atd.resource_group, - "virtualNetworkName": atd.deploy_id + "-vnet", - "virtualNetworkSubnetName": atd.deploy_id + "-subnet", + "adminPassword": os.environ["AVERE_ADMIN_PW"], "avereBackedStorageAccountName": atd.deploy_id + "sa", - "controllerName": atd.deploy_id + "-con", + "avereClusterName": atd.deploy_id + "-cluster", + "avereInstanceType": "Standard_E32s_v3", + "avereNodeCount": 3, "controllerAdminUsername": "azureuser", "controllerAuthenticationType": "sshPublicKey", - "controllerSSHKeyData": ssh_pub_key, + "controllerName": atd.deploy_id + "-con", "controllerPassword": os.environ["AVERE_CONTROLLER_PW"], - "avereNodeCount": 3, - "adminPassword": os.environ["AVERE_ADMIN_PW"], - "rbacRoleAssignmentUniqueId": str(uuid4()), + "controllerSSHKeyData": ssh_pub_key, "enableCloudTraceDebugging": True, + "rbacRoleAssignmentUniqueId": str(uuid4()), + + "createVirtualNetwork": True, + "virtualNetworkName": atd.deploy_id + "-vnet", + "virtualNetworkResourceGroup": atd.resource_group, + "virtualNetworkSubnetName": atd.deploy_id + "-subnet", } test_vars["controller_name"] = atd.deploy_params["controllerName"] test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] @@ -59,6 +61,48 @@ class TestVfxtTemplateDeploy: atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address + def test_deploy_template_byovnet(self, resource_group, test_vars, ext_vnet): # noqa: E501, F811 + log = logging.getLogger("test_deploy_template_byovnet") + atd = test_vars["atd_obj"] + with open("{}/src/vfxt/azuredeploy-auto.json".format( + test_vars["build_root"])) as tfile: + atd.template = json.load(tfile) + with open(test_vars["ssh_pub_key"], "r") as ssh_pub_f: + ssh_pub_key = ssh_pub_f.read() + atd.deploy_params = { + "adminPassword": os.environ["AVERE_ADMIN_PW"], + "avereBackedStorageAccountName": atd.deploy_id + "sa", + "avereClusterName": atd.deploy_id + "-cluster", + "avereInstanceType": "Standard_E32s_v3", + "avereNodeCount": 3, + "controllerAdminUsername": "azureuser", + "controllerAuthenticationType": "sshPublicKey", + "controllerName": atd.deploy_id + "-con", + "controllerPassword": os.environ["AVERE_CONTROLLER_PW"], + "controllerSSHKeyData": ssh_pub_key, + "enableCloudTraceDebugging": True, + "rbacRoleAssignmentUniqueId": str(uuid4()), + + "createVirtualNetwork": False, + "virtualNetworkResourceGroup": ext_vnet["resource_group"]["value"], + "virtualNetworkName": ext_vnet["virtual_network_name"]["value"], + "virtualNetworkSubnetName": ext_vnet["subnet_name"]["value"], + } + test_vars["controller_name"] = atd.deploy_params["controllerName"] + test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] + + log.debug("Generated deploy parameters: \n{}".format( + json.dumps(atd.deploy_params, indent=4))) + atd.deploy_name = "test_deploy_template_byovnet" + try: + deploy_outputs = wait_for_op(atd.deploy()).properties.outputs + test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] + test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) + finally: + test_vars["controller_ip"] = atd.nm_client.public_ip_addresses.get( + atd.resource_group, "publicip-" + test_vars["controller_name"] + ).ip_address + if __name__ == "__main__": pytest.main(sys.argv) From b27ee7516c26b4206c44b6992353a7f0ce47fc25 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Thu, 14 Feb 2019 17:06:10 -0500 Subject: [PATCH 02/13] Merge "no stor acct" changes --- azure-pipelines.yml | 8 +- src/vfxt/azuredeploy.vnet.json | 156 ++++++++++++++++++++++++++++-- test/conftest.py | 16 ++- test/test_edasim.py | 2 +- test/test_vdbench.py | 2 +- test/test_vfxt_cluster_status.py | 2 +- test/test_vfxt_template_deploy.py | 96 +++++++++--------- 7 files changed, 217 insertions(+), 65 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c3b0cda3..66d85bbc 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -86,19 +86,19 @@ jobs: -k TestVfxtSupport \ --doctest-modules --junitxml=junit/test-results03.xml - CONTROLLER_IP=$(jq -r .controller_ip $VFXT_TEST_VARS_FILE) + PUBLIC_IP=$(jq -r .public_ip $VFXT_TEST_VARS_FILE) CONTROLLER_NAME=$(jq -r .controller_name $VFXT_TEST_VARS_FILE) CONTROLLER_USER=$(jq -r .controller_user $VFXT_TEST_VARS_FILE) - echo "CONTROLLER_IP : $CONTROLLER_IP" + echo "PUBLIC_IP : $PUBLIC_IP" echo "CONTROLLER_NAME: $CONTROLLER_NAME" echo "CONTROLLER_USER: $CONTROLLER_USER" ARTIFACTS_DIR="$BUILD_SOURCESDIRECTORY/test_artifacts" mkdir -p $ARTIFACTS_DIR tar -zcvf ${ARTIFACTS_DIR}/vfxt_artifacts_${CONTROLLER_NAME}.tar.gz vfxt_artifacts_* - scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r $CONTROLLER_USER@$CONTROLLER_IP:~/*.log $ARTIFACTS_DIR/. - scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ~/.ssh/* $CONTROLLER_USER@$CONTROLLER_IP:~/.ssh/. + scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r $CONTROLLER_USER@$PUBLIC_IP:~/*.log $ARTIFACTS_DIR/. + scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ~/.ssh/* $CONTROLLER_USER@$PUBLIC_IP:~/.ssh/. echo "vfxt.log from $CONTROLLER_NAME:" cat $ARTIFACTS_DIR/vfxt.log diff --git a/src/vfxt/azuredeploy.vnet.json b/src/vfxt/azuredeploy.vnet.json index 452509a6..dac41941 100644 --- a/src/vfxt/azuredeploy.vnet.json +++ b/src/vfxt/azuredeploy.vnet.json @@ -2,14 +2,22 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { + "uniqueName": { + "type": "string", + "metadata": { + "description": "The unique name used as a basis for resource names." + } + }, "virtualNetworkName": { "type": "string", + "defaultValue": "[concat(parameters('uniqueName'), '-vnet')]", "metadata": { "description": "The name of the virtual network (VNET)." } }, "virtualNetworkSubnetName": { "type": "string", + "defaultValue": "[concat(parameters('uniqueName'), '-subnet')]", "metadata": { "description": "The name of the subnet in the VNET." } @@ -27,13 +35,44 @@ "metadata": { "description": "The IP address range prefix of the subnet in the VNET." } + }, + "jumpboxAdminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "The administrative username for the jumpbox." + } + }, + "jumpboxSSHKeyData": { + "type": "string", + "metadata": { + "description": "The SSH public key used to connect to the jumpbox." + } } }, "variables": { + "vmSku": "Standard_A1", + "uniqueName": "[parameters('uniqueName')]", "virtualNetworkName": "[parameters('virtualNetworkName')]", "subnetName": "[parameters('virtualNetworkSubnetName')]", "addressPrefix": "[parameters('vnetAddressSpacePrefix')]", - "subnetPrefix": "[parameters('subnetAddressRangePrefix')]" + "subnetPrefix": "[parameters('subnetAddressRangePrefix')]", + "subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', variables('virtualNetworkName'), variables('subnetName'))]", + "publicIPAddressName": "[concat(variables('uniqueName'), '-publicip')]", + "storageAccountType": "Standard_LRS", + "jumpboxName": "[concat('jbox-', variables('uniqueName'))]", + "jumpboxSAName": "[concat(variables('uniqueName'), 'jbsa')]", + "jumpboxOSDiskName": "[concat(variables('jumpboxName'), '-osdisk')]", + "jumpboxIPConfigName": "[concat(variables('jumpboxName'), '-ipconfig')]", + "jumpboxNicName": "[concat(variables('jumpboxName'), '-nic')]", + "jumpboxSSHKeyPath": "[concat('/home/',parameters('jumpboxAdminUsername'),'/.ssh/authorized_keys')]", + "osType": { + "publisher": "Canonical", + "offer": "UbuntuServer", + "sku": "16.04-LTS", + "version": "latest" + }, + "imageReference": "[variables('osType')]" }, "resources": [ { @@ -61,28 +100,127 @@ } ] } + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('publicIPAddressName')]", + "location": "[resourceGroup().location]", + "apiVersion": "2017-10-01", + "properties": { + "publicIPAllocationMethod": "Static" + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "name": "[variables('jumpboxNicName')]", + "location": "[resourceGroup().location]", + "apiVersion": "2017-10-01", + "dependsOn": [ + "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]", + "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "[variables('jumpboxIPConfigName')]", + "properties": { + "privateIPAllocationMethod": "Dynamic", + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]" + }, + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[variables('jumpboxSAName')]", + "location": "[resourceGroup().location]", + "apiVersion": "2015-06-15", + "properties": { + "accountType": "[variables('storageAccountType')]" + } + }, + { + "type": "Microsoft.Compute/virtualMachines", + "name": "[variables('jumpboxName')]", + "location": "[resourceGroup().location]", + "apiVersion": "2017-03-30", + "dependsOn": [ + "[concat('Microsoft.Storage/storageAccounts/', variables('jumpboxSAName'))]", + "[concat('Microsoft.Network/networkInterfaces/', variables('jumpboxNicName'))]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[variables('vmSku')]" + }, + "osProfile": { + "computerName": "[variables('jumpboxName')]", + "adminUsername": "[parameters('jumpboxAdminUsername')]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[variables('jumpboxSSHKeyPath')]", + "keyData": "[parameters('jumpboxSSHKeyData')]" + } + ] + } + } + }, + "storageProfile": { + "imageReference": "[variables('imageReference')]", + "osDisk": { + "name": "[variables('jumpboxOSDiskName')]", + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',variables('jumpboxNicName'))]" + } + ] + }, + "diagnosticsProfile": { + "bootDiagnostics": { + "enabled": true, + "storageUri": "[concat('http://',variables('jumpboxSAName'),'.blob.core.windows.net')]" + } + } + } } ], "outputs": { - "resource_group": { - "type": "string", - "value": "[resourceGroup().name]" - }, "location": { "type": "string", "value": "[resourceGroup().location]" }, - "virtual_network_name": { + "public_ip_address": { "type": "string", - "value": "[variables('virtualNetworkName')]" + "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))).ipAddress]" + }, + "resource_group": { + "type": "string", + "value": "[resourceGroup().name]" + }, + "subnet_id": { + "type": "string", + "value": "[concat(resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName')),'/subnets/',variables('subnetName'))]" }, "subnet_name": { "type": "string", "value": "[variables('subnetName')]" }, - "subnet_id": { + "virtual_network_name": { "type": "string", - "value": "[concat(resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName')),'/subnets/',variables('subnetName'))]" + "value": "[variables('virtualNetworkName')]" } } } \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index 7c645690..5aa51257 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -123,7 +123,7 @@ def scp_cli(ssh_con): @pytest.fixture() def ssh_con(test_vars): client = create_ssh_client(test_vars["controller_user"], - test_vars["controller_ip"], + test_vars["public_ip"], key_filename=test_vars["ssh_priv_key"]) yield client client.close() @@ -196,8 +196,13 @@ def test_vars(request): @pytest.fixture() def ext_vnet(test_vars): + """ + Creates a resource group containing a new VNET, subnet, public IP, and + jumpbox for use in other tests. + """ log = logging.getLogger("ext_vnet") vnet_atd = ArmTemplateDeploy( + location=test_vars["location"], resource_group=test_vars["atd_obj"].deploy_id + "-vnet-rg" ) rg = vnet_atd.create_resource_group() @@ -207,9 +212,14 @@ def ext_vnet(test_vars): with open("{}/src/vfxt/azuredeploy.vnet.json".format( test_vars["build_root"])) as tfile: vnet_atd.template = json.load(tfile) + + with open(test_vars["ssh_pub_key"], "r") as ssh_pub_f: + ssh_pub_key = ssh_pub_f.read() + vnet_atd.deploy_params = { - "virtualNetworkName": test_vars["atd_obj"].deploy_id + "-vnet", - "virtualNetworkSubnetName": test_vars["atd_obj"].deploy_id + "-subnet", + "uniqueName": test_vars["atd_obj"].deploy_id, + "jumpboxAdminUsername": "azureuser", + "jumpboxSSHKeyData": ssh_pub_key } test_vars["ext_vnet"] = wait_for_op(vnet_atd.deploy()).properties.outputs log.debug(test_vars["ext_vnet"]) diff --git a/test/test_edasim.py b/test/test_edasim.py index 5eee639b..ae5301bf 100644 --- a/test/test_edasim.py +++ b/test/test_edasim.py @@ -126,7 +126,7 @@ class TestEdasim: log = logging.getLogger("test_edasim_run") node_ip = test_vars["deploy_edasim_outputs"]["jobsubmitter_0_ip_address"]["value"] with SSHTunnelForwarder( - test_vars["controller_ip"], + test_vars["public_ip"], ssh_username=test_vars["controller_user"], ssh_pkey=test_vars["ssh_priv_key"], remote_bind_address=(node_ip, 22), diff --git a/test/test_vdbench.py b/test/test_vdbench.py index 3d58959b..cc950c96 100644 --- a/test/test_vdbench.py +++ b/test/test_vdbench.py @@ -60,7 +60,7 @@ class TestVDBench: log = logging.getLogger("test_vdbench_run") node_ip = test_vars["deploy_vd_outputs"]["node_0_ip_address"]["value"] with SSHTunnelForwarder( - test_vars["controller_ip"], + test_vars["public_ip"], ssh_username=test_vars["controller_user"], ssh_pkey=test_vars["ssh_priv_key"], remote_bind_address=(node_ip, 22), diff --git a/test/test_vfxt_cluster_status.py b/test/test_vfxt_cluster_status.py index 1643bfa1..86fc4b4f 100644 --- a/test/test_vfxt_cluster_status.py +++ b/test/test_vfxt_cluster_status.py @@ -112,7 +112,7 @@ class TestVfxtSupport: args=node)[node]["primaryClusterIP"]["IP"] log.debug("tunneling to node {} using IP {}".format(node, node_ip)) with SSHTunnelForwarder( - test_vars["controller_ip"], + test_vars["public_ip"], ssh_username=test_vars["controller_user"], ssh_pkey=test_vars["ssh_priv_key"], remote_bind_address=(node_ip, 22), diff --git a/test/test_vfxt_template_deploy.py b/test/test_vfxt_template_deploy.py index e03eda3c..1166d068 100755 --- a/test/test_vfxt_template_deploy.py +++ b/test/test_vfxt_template_deploy.py @@ -49,6 +49,7 @@ class TestVfxtTemplateDeploy: } test_vars["controller_name"] = atd.deploy_params["controllerName"] test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] + test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] log.debug("Generated deploy parameters: \n{}".format( json.dumps(atd.deploy_params, indent=4))) @@ -58,7 +59,51 @@ class TestVfxtTemplateDeploy: test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) finally: - test_vars["controller_ip"] = atd.nm_client.public_ip_addresses.get( + test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( + atd.resource_group, "publicip-" + test_vars["controller_name"] + ).ip_address + + def test_no_storage_account_deploy(self, resource_group, test_vars): # noqa: E501, F811 + log = logging.getLogger("test_deploy_template") + atd = test_vars["atd_obj"] + with open("{}/src/vfxt/azuredeploy-auto.json".format( + test_vars["build_root"])) as tfile: + atd.template = json.load(tfile) + with open(test_vars["ssh_pub_key"], "r") as ssh_pub_f: + ssh_pub_key = ssh_pub_f.read() + atd.deploy_params = { + "adminPassword": os.environ["AVERE_ADMIN_PW"], + "avereClusterName": atd.deploy_id + "-cluster", + "avereInstanceType": "Standard_E32s_v3", + "avereNodeCount": 3, + "controllerAdminUsername": "azureuser", + "controllerAuthenticationType": "sshPublicKey", + "controllerName": atd.deploy_id + "-con", + "controllerPassword": os.environ["AVERE_CONTROLLER_PW"], + "controllerSSHKeyData": ssh_pub_key, + "enableCloudTraceDebugging": True, + "rbacRoleAssignmentUniqueId": str(uuid4()), + + "createVirtualNetwork": True, + "virtualNetworkName": atd.deploy_id + "-vnet", + "virtualNetworkResourceGroup": atd.resource_group, + "virtualNetworkSubnetName": atd.deploy_id + "-subnet", + + "useAvereBackedStorageAccount": False, + "avereBackedStorageAccountName": atd.deploy_id + "sa", # BUG + } + test_vars["controller_name"] = atd.deploy_params["controllerName"] + test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] + log.debug("Generated deploy parameters: \n{}".format( + json.dumps(atd.deploy_params, indent=4))) + atd.deploy_name = "test_deploy_template" + try: + deploy_outputs = wait_for_op(atd.deploy()).properties.outputs + test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] + test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) + time.sleep(60) + finally: + test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address @@ -91,47 +136,6 @@ class TestVfxtTemplateDeploy: } test_vars["controller_name"] = atd.deploy_params["controllerName"] test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] - test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] - log.debug("Generated deploy parameters: \n{}".format( - json.dumps(atd.deploy_params, indent=4))) - atd.deploy_name = "test_deploy_template" - try: - deploy_outputs = wait_for_op(atd.deploy()).properties.outputs - test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] - test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) - finally: - test_vars["controller_ip"] = atd.nm_client.public_ip_addresses.get( - atd.resource_group, "publicip-" + test_vars["controller_name"] - ).ip_address - - def test_no_storage_account_deploy(self, resource_group, test_vars): # noqa: F811 - log = logging.getLogger("test_deploy_template") - atd = test_vars["atd_obj"] - with open("{}/src/vfxt/azuredeploy-auto.json".format( - test_vars["build_root"])) as tfile: - atd.template = json.load(tfile) - with open(test_vars["ssh_pub_key"], "r") as ssh_pub_f: - ssh_pub_key = ssh_pub_f.read() - atd.deploy_params = { - "avereInstanceType": "Standard_E32s_v3", - "avereClusterName": atd.deploy_id + "-cluster", - "virtualNetworkResourceGroup": atd.resource_group, - "virtualNetworkName": atd.deploy_id + "-vnet", - "virtualNetworkSubnetName": atd.deploy_id + "-subnet", - "avereBackedStorageAccountName": atd.deploy_id + "sa", - "controllerName": atd.deploy_id + "-con", - "controllerAdminUsername": "azureuser", - "controllerAuthenticationType": "sshPublicKey", - "controllerSSHKeyData": ssh_pub_key, - "controllerPassword": os.environ["AVERE_CONTROLLER_PW"], - "avereNodeCount": 3, - "adminPassword": os.environ["AVERE_ADMIN_PW"], - "rbacRoleAssignmentUniqueId": str(uuid4()), - "enableCloudTraceDebugging": True, - "useAvereBackedStorageAccount": False, - } - test_vars["controller_name"] = atd.deploy_params["controllerName"] - test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] log.debug("Generated deploy parameters: \n{}".format( json.dumps(atd.deploy_params, indent=4))) atd.deploy_name = "test_deploy_template_byovnet" @@ -140,11 +144,11 @@ class TestVfxtTemplateDeploy: test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) finally: - test_vars["controller_ip"] = atd.nm_client.public_ip_addresses.get( - atd.resource_group, "publicip-" + test_vars["controller_name"] - ).ip_address + # test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( + # atd.resource_group, "publicip-" + test_vars["controller_name"] + # ).ip_address + pass - time.sleep(60) if __name__ == "__main__": pytest.main(sys.argv) From 7677b4db16c0a31e1bf07383d7958eda47db1b7a Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Thu, 14 Feb 2019 17:27:25 -0500 Subject: [PATCH 03/13] BYO VNET testcase, pipeline changes --- azure-pipelines.yml | 21 +++++++++++++--- src/vfxt/azuredeploy-auto.json | 8 +++--- src/vfxt/src/base-template.json | 8 +++--- src/vfxt/src/marketplace.zip | Bin 12302 -> 12292 bytes test/test_vfxt_template_deploy.py | 40 +++++++++++++++++++++--------- 5 files changed, 54 insertions(+), 23 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 66d85bbc..ef42ea66 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ # https://aka.ms/yaml -trigger: -- master -- releases/* +# trigger: +# - master +# - releases/* variables: VFXT_TEST_VARS_FILE: 'pipelines.json' @@ -63,6 +63,21 @@ jobs: AZURE_CLIENT_SECRET: $(AZURE-CLIENT-SECRET) AZURE_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID) + - script: | + pytest --disable-pytest-warnings test/test_vfxt_template_deploy.py \ + -k test_byovnet_deploy \ + --location $VFXT_DEPLOY_LOCATION \ + --doctest-modules --junitxml=junit/test-results01b.xml + displayName: 'Test deploy of vFXT with a VNET in a different resource group' + condition: and(succeeded(), eq(variables['RUN_BYOVNET'], 'true')) + env: + AVERE_ADMIN_PW: $(AVERE-ADMIN-PW) + AVERE_CONTROLLER_PW: $(AVERE-CONTROLLER-PW) + AZURE_TENANT_ID: $(AZURE-TENANT-ID) + AZURE_CLIENT_ID: $(AZURE-CLIENT-ID) + AZURE_CLIENT_SECRET: $(AZURE-CLIENT-SECRET) + AZURE_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID) + - script: | if [ -f $VFXT_TEST_VARS_FILE ]; then cat $VFXT_TEST_VARS_FILE diff --git a/src/vfxt/azuredeploy-auto.json b/src/vfxt/azuredeploy-auto.json index a2b1c2a1..958782ab 100644 --- a/src/vfxt/azuredeploy-auto.json +++ b/src/vfxt/azuredeploy-auto.json @@ -24,19 +24,19 @@ "type": "string", "defaultValue": "[resourceGroup().name]", "metadata": { - "description": "The resource group name for the VNET. If createVirtualNetwork is set to true, the current resource group must be specified, otherwise the value should be blank." + "description": "The resource group name for the VNET. If createVirtualNetwork is set to true, this field should be blank. Otherwise, provide the name of the resource group containing an existing VNET." } }, "virtualNetworkName": { "type": "string", "metadata": { - "description": "The name used for the virtual network. If createVirtualNetwork is set to true, you may reuse the unique name above." + "description": "The unique name used for the virtual network. If createVirtualNetwork is set to true, you may reuse the unique name above." } }, "virtualNetworkSubnetName": { "type": "string", "metadata": { - "description": "The unique name used for the virtual network subnet. If createVirtualNetwork is set to true, you may reuse the unique name above." + "description": "The unique name used for the virtual network subnet. If createVirtualNetwork is set to true, you may reuse the unique name above." } }, "vnetAddressSpacePrefix":{ @@ -189,7 +189,7 @@ "virtualNetworkSubnetName": "[parameters('virtualNetworkSubnetName')]", "addressPrefix": "[parameters('vnetAddressSpacePrefix')]", "subnetPrefix": "[parameters('subnetAddressRangePrefix')]", - "useAvereBackedStorageAccount": "[parameters('useAvereBackedStorageAccount')]", + "useAvereBackedStorageAccount": "[parameters('useAvereBackedStorageAccount')]", "avereBackedStorageAccountName": "[parameters('avereBackedStorageAccountName')]", "controllerName": "[parameters('controllerName')]", "controllerAdminUsername": "[parameters('controllerAdminUsername')]", diff --git a/src/vfxt/src/base-template.json b/src/vfxt/src/base-template.json index 455cb7b1..e2bf5341 100644 --- a/src/vfxt/src/base-template.json +++ b/src/vfxt/src/base-template.json @@ -24,19 +24,19 @@ "type": "string", "defaultValue": "[resourceGroup().name]", "metadata": { - "description": "The resource group name for the VNET. If createVirtualNetwork is set to true, the current resource group must be specified, otherwise the value should be blank." + "description": "The resource group name for the VNET. If createVirtualNetwork is set to true, this field should be blank. Otherwise, provide the name of the resource group containing an existing VNET." } }, "virtualNetworkName": { "type": "string", "metadata": { - "description": "The name used for the virtual network. If createVirtualNetwork is set to true, you may reuse the unique name above." + "description": "The unique name used for the virtual network. If createVirtualNetwork is set to true, you may reuse the unique name above." } }, "virtualNetworkSubnetName": { "type": "string", "metadata": { - "description": "The unique name used for the virtual network subnet. If createVirtualNetwork is set to true, you may reuse the unique name above." + "description": "The unique name used for the virtual network subnet. If createVirtualNetwork is set to true, you may reuse the unique name above." } }, "vnetAddressSpacePrefix":{ @@ -189,7 +189,7 @@ "virtualNetworkSubnetName": "[parameters('virtualNetworkSubnetName')]", "addressPrefix": "[parameters('vnetAddressSpacePrefix')]", "subnetPrefix": "[parameters('subnetAddressRangePrefix')]", - "useAvereBackedStorageAccount": "[parameters('useAvereBackedStorageAccount')]", + "useAvereBackedStorageAccount": "[parameters('useAvereBackedStorageAccount')]", "avereBackedStorageAccountName": "[parameters('avereBackedStorageAccountName')]", "controllerName": "[parameters('controllerName')]", "controllerAdminUsername": "[parameters('controllerAdminUsername')]", diff --git a/src/vfxt/src/marketplace.zip b/src/vfxt/src/marketplace.zip index 4ebc44c9427d0dba30e2008f73e3a3d1675d1d1f..f7b9feb8b7b3e7510eca3834eb53df682a69c448 100644 GIT binary patch delta 11484 zcmV<2EF;s7V1!^7P)h>@6aWAK2mnZnPEJi~^Q`qE001(P7bZ9FL}c}AWg;Rd711#R zh>9Buf=hQt1v|(pyWY9~eGaIZu9;cY)qU^GWOj9x3FkcLJo^IY@gI?NM}PCvmvRBc z_hN8Q)WXQ}y<#2ssaE`VxF1NNRml7uyB_}Bd{1$4FpM;9m4gDne{L9l3+#e^Cz?zaO#nR~#C9FsZt>Ccurd@IgX!cYI(Du_irzlpAD>3bjgXd zA_W0ouj>fkm+iuIQHL0J55X0{)S;z4$&=*ds;19LZcRP_E|d zXG(C`NTKeMuix=2Q z`^73zu^N!%V!1s%fHC*>7?%=zMVHKq&Ao=axVfha{bW2$`{1J7)0c9b3v8_m zm{gLgl6Dreidy0%5s0S%qa3HO<@=>y2{oj9Wd7MEi8xw`OA-LP^0LvA21oo09A9wT zqyO-lXgr+Qs%JVqd$=O(^4RI*VuFIT;6adRB6en{aDU-H0I8(-(pouLnIwT zn~zdU*B=`VM0u2&sPb6XwU*M56&XX&6(39LiFO4r0Wj$V*M0B<(!)b;bB2?R1SWq; zlH$7{P*Tgl`49_$k732n_7i~7uk5!egUtXa$7+lLOh+P@#h0=W0hP>v;!x8dS1NV} zMWF(vdLIAhzGS05f;p~YhXJp%h7^dcxI*E-?}Qf9fx)=De@KJpgO;^=jC zzc8j{D7dQ>+XukD7N1KHfDRz?D2adNfE=&#_z2uuxUD?=_FsQI{i7ICc+aoL4_`{x z_OLv&fKi2FSaJgV`2*PS$WlT-^7Xh<)Rp|Lrj<35%7fID zVHw7y$C}nc9vixbKH>-v3SE*kUDj2&r63P)N$87g{+MKzP#zD$@#jRals|t!`!bzQ zyfS(I9%b8)`YWQ9V<=g2>7_bw2x1Dab`YNPD!>~XX1pktJVpC)_s@g*suZf!YvCaj^03j?5i4eH;b{r^}C28XP)cURVhDSh3h|pl$W?NU^p;< zB{Ck^Yg9Qf`PvP^+L{&=o~c~_KB^(e)(T5XM3H59ML~Z+C@sW~di|dj zpuJS@4=(n&y`u%Q&a3sQTN=8%wfW+oJ<^|2DYOdy-g#>A>9?2X8lNAIHy(a@F81E5 z@1IY+HRu$`>lNyASg(w|5AxttUV~H&xews1w)q;Emw~6i4%Mn-G3*k;%|hbo{C&;p z&h&dlw^IObRH4oRzFL2mIK}FXg3CG7S4W?T)Me%3LYU5O-=26`Ik*SmY}_4Cw^IGP z@U8~m26~bM-2-(TavQ)|?syNx%b;^vJWs+dxO95=irH5QQ73Vn?G%W=rCXZH?BRhO z00Mk|C`zr*hAe_W-gtW9^Dj@QzcT8V$^&$GJQGIHbJ+Y;5Y>MVtzAh|s4#);=cv7e zdL?By^0tRxu5dXEoo~qM$$)D@oCT}*Ctn$Y8-e4E*{36}rk@60H|9RyzTWg%AaiTe zF9S{|zZL?VL%BQh6x^$B_S0{c{@Ez`7k+k1m-+d0h5M5Mhrd*6znqK1Yr}z~JbFK1 zgWWx=_WL~S)U$t-XAU2Q8aP)=d3^YCGWXIVUAf2e9USm95=;GjfO-P#^&LMv?jeSy zP@Wt3y*2x{Qck#~Hq};3eN+)t11YYIK1!&9JT{w6M5))LW?jV}Zej5=DOLb2&Dssl z6c2QNK6eU-e#N2E%a_x*Lp7mT==|zwz)+GLQFka4n};(|zO({g zDnO-9?!Tg>UkaF!Z3T~tqe)&80MHL5X5hM7jJxfQ#8F{2&Qe+e{+8CChwdwc%;XTI zTxr$*K3Mm?igkFJ#~1ZNE8dpz-xX9ZGTsevA4MIwf2;#A>M7w~4Lt#kUbNyc1UjD) z9iBZ#=lp-V6^Fhu;55QK6)*ROsd_i|S{`d&@n+K%5NI#bPeof97TWYa;z{ZX54zwr!WRXmLdwkzn)9-OM+Mo z0;YA&a)WQzV827Bj!<5mcF>s;X)ylPv8U=&z$ZkSqrXu@fuGUbH;x_BzyAK|hCw}x z2El*C-#=9-jwWWsTvlj*BME9iw~v4>w(y3)gKh9LmLt0KH-e&AVSI_g8@W>ffFh_K z{f%c<^sTvPkzG0bfbf_Y-JWn7zZMgiF+U^5fQz%`nCrNKW&Q8 zb$;WF9MYFGFKGrjw8iR+#9B*MxR!r*h8PU%2HDe>+s?AKbw(Q@p_nGAkSzmmE1jJ& zYr7-fHW+D*w3Z@c$Ia9oO{$LAU6|Xzf+Fpc|^$(D6kW$2eWb>vIR*NfC*7GQ76BxDKVMuX>xdzrvDwja`2$s58%7 zk}5424oxP_+GImoy}@9;kWwjeHK~C>Yf9SYuC!YkA|1~)EueQ%vYX_iUUNVgeH&?a zL|aY6d@|Y4u?MfG>&|FU({kOd!%}m>tR=V~Gqtt3ZLZo@e@*j|FW^co!^V*^_XKGq z4#JEfdev$>9vES;Z5e?JhcJJ-!h6&9K%%R?g@cm<9V1gEi&?L=tS?9Uaw!amWYC`z zIz!I{lF?vMH#BBILnz-HH=tEkQ>VJE&vd4Dtf?r964=8-CUDn$LZ@! zUs2}Lc7XP5+3SswL3e-CQ{cedp>e>g%^@}6sXDsbkc;kY!a%OAGVu_PaC(PHiN$+_ zH;nAvaFxw+fN|fsN1p41Ju_e;95rWf zw--&j)}&uwm=v#hIYcPo5GOGr0pzU<)^a(V4?9!2UuTh6NP$?^TziGo$)IWsRczCM z=d3YdZ8}Z*tL*?^+b9`NN9hd6M2%=?z1D8q6kL`PWrgF$jwKIWKFVQksV+?+n(r*7 z)`>-5PEV6w(k>ki(G zCgfyIX{vv(wHz2lJWRvJX)s#W=&7)(J59hnbldL5_`2iFzP{xT8+)!j&+O<*u}aZ!7U^wnT+X*%CwCeM1(=@5x?BB8Iw>UtV>9i%22-NIFcu!E)RUOeXuRCb$tWj?f%+M%h(mcg-J8X;k&>_Y+ApA(oJU0_4 zYb>`oAxkj}k*TDMAX!dlgQ|$n1>0;idl-Ld%qDa#T2B+Dkqy=`*X=Xz7BlO*jcVRf z)16^BTVXA2xU4ROT4qe5>S(@Op@YpPiCe*F($4a}CgT{PrC|%s)n2~|5zEDHM| zqm|VjCGp6X=2OO}gF$OZ*tHqcNZTtmz~NXQhAVp$@8%t5jjqXX)=Bv&Q&yX1hY$^J z)WGNCw83xzwi|T2JuC%YH;?QUxg38}{S}a^jwNb5(iOG?CtD`uI%~DeuD(q*ee z+6^4|jb3#}_~F#fG{x43czs>V&?@FyM%6Oqs$=B6)k4}hNjB|=lr+uTOL0EMH$Wao z+Mwl_8lBZUCcM;pz|zrHx8}$lcH4)lA%W@Q+A^yXl8aYL6Qg1%*hz~yk9dC(J;-Hv zlP9Yt+MZYr5~|}g7hn<^jK;vsscJ0LL?qnaT&a5sZp~In054m`*(<~j)$x09j zV}(bQDNdcqXlu8-RHIt&C~cNz6{#iWyV+cvc9sI3w^ne^r&(@;_syv& zmZ$}_{=~#5i&+gaBpGXpU1=Sq##`$hjZNzwwWR31J>WlJR=`I#xyE}#;Vv3N}v`uD8 zufE1M3F$`tT@VjAr)qyB(bl(CTGm#CnU2rWiJaxlZC#)(W71gV%&d#d`(#HR=d`wN zWmU&ATnrgbS1{5DJBcDJRAC+aEe3JYbvB_%evPW*5S0U-ZCc5~O}rfnEtIqx#?x^p z*)_?k(H%HqEbcbQGSMt!%%cmjkv2`KUyI4CM8^PRJAb7|P^6s`i8k5p8YOkc&>v6F%90JqW%HUN%)Nq)i z7F|n$`wMD`ds=@-_2Hyj?V!+>9xa3*h^x-FDyr&y#i=c^oeN4&kPHgK{Puk8^_sL_ z#cS)ZRUK$Lki)H^vE;C2gTQ6lr)YVd&$+}{2vdDyH{^I>4dj;LxV;WyRQYKO<=Y9a zq>Q==r(=&*W(L+ly~)n$&(J|NuXn~P)n}?*s7I|c5d427AZr?W3nK1^t(LB1^lV*4 z=q|mHtzM8KB3*M)qGu4qJ}U}9&ib_i`9Tboc{*ejiMrzy1-+-pJPgu8*NTebRrIy; z`=?s7(fIw-Pf5Yx1yqM5&YVHW0_tq4(LiAP1f0}+eSr}vjO!RCLCUfQdyB<#wOe=6 zblgHwifMnzP>QJba*B~^Qme~@-J+gAI^1eDl?LSQP=A+${~2>G;1DGDG7ri;%Y+DB zH;vt9n3EaPZjjrNANkdOk0Bf)Xz}h;7}EmG3@nb$8m`JN6=Q+V>jW<(!A!NQZDBi7 zM?GiRqeYGo3~04IGH;l_NnGf{F7(6+g5)z*QqG>M(s z&OqE9I#Y&JElRX0420(MaGp_u724U7>QFJ<<-^ftD9!rQ4%Z#Pb!?8?ZFTOA$!uADiZ)P_Axg@T!k_uGG*3iqq^dAM4_Y1rIpGfnY2!{H9lXdJ4E zj*fUMad$&%JxNf=ob@bnl1}N6g4*0HXtmsqHeZhFwm90<6{OM6Clumotw_O56wXs5 z^kp@+kz}B*vL4e81DEeZ$i%D~=>}=q)fp!Y-6f7r5yss}P0H={T0CP3JW6*P?2Laz zkfjSkw6(%%6Pu{PhQmrRGES$dLBs*uTncP+%|Wu!MH*yZ!;;W6VFQ9YSwHG=kg)2@ zzSnPY2&VO2kR3ZI$*iVzf#|B@+zI7C=64iF0beaCgTC3>B=uHgU@O&;(7c+YBW^x4 znE7Nmz&C+!_wYX2ji-qpb68b|=URW=@&v)`n6y&$T)Vo%2G(><)?KtiW`WH~i~_Qq z#96EO1aeV!FhXM=OBb!C!lCfoS_-+Z^0f}PK%q%CYmM@WYc_^D6^c622R@FiBS;n!H}I9sMnk);{h3T<~V=Jn0ewQ z%{VgW=)B8l_?U0oxwaKFiaAElvN~JFQyhFJ#)!~QLx=5m zW_~h~9Vb>;WiU_|D`m~6FbW4Ngfv&4k5*R*2y^04ffr6~j+mp6t&@XfkOxe*65E;3 zW&0-8+O9p-Na}MCQpGVwj9P!hPFJivt1dJ~7$yi~jR*FqGZPKO5JH5Y&`g;3xh9K7 zTXnh1#r|-H^|)<)-d`IMlydNn7#gzOQ_0?mnx zfdIwp^46c}3z8Z!IMGG~p&5*4^Vxh%>KKl&q-jWjjwj2sA8*_2Fq&(Ly=Fp~M|$;i z6Vz-CSLXwt%vV?)3Y2L&?^;?P!>|=*w>oBK^_PM*ok7F7k=K8P)*6|QF-&Z%5W7y* ztMJGV;c&=|r}ITF4_Es$<)9Yjsg@y=~c#0*Qtw510P zEwyV@sA*#|YnVbS3l`-F8f;)oS@nA6jCOm`+Aqpv91fv`?rN|mYHu=jsKc5^OMmVG?VTd2!H>wO7TlE*0H=Krw;9(t{T}E z=hUsJSmF56@A>K1qJUeRf;ekZ6-DMbjOVjK18sZJ5uy0`$N8_t=;JHcC#B_=m$T0h zc%EaQ9F@IUkv=~udp`f4r@#L3^2_t}`yHO1YwO2@joK!WoapoIs`!nw!(U#yPyKTH z9rLUA#FK*@J3%2$arg10S={Ns^rZ)L(WW_y?|0~otg?ZC$kUuibKjT=|LZf-?*87W zJDPzTzkWLDityYK`9&aK=pY5wNf+po{~R$&bBUMpOUE-Y(MgNy=wwC^`aqv!>$Hc% z9zkC%K>;@zmJP5a=Zuc=q0jY|LjKjww4WDxqp9}uxHEpdLj_NtV0rr^9#^{mdP|df z9U6aAj+;lW{&pkDL{9@Iz*-))^?fv$;LtM5X`L7#}5-b(_itm(N zXO!T^2vsXOVV6{y_+ebO8!35NzcE-+fX-@`=MoEf%Wi?QLQQ#YMCi$W;;A==?NK;e z@YeQEHg^NvSKRu^FMs^o%Kyb-;x{D!)$TSwPv)hQzBN~l)m+IF8pHyign&woP)ulxI)MyDsudy{@19RZ({oF1+eeHUL-d20U_+ELm^ zl44rknZh3%lU^U?e}}TvKi;k?{Gce^?HKkikfv8bUJ(4A3jAaldrw=q|6e5RJVEjN z`0c>$^?arESVP~I(CfZ$Z;9qx72m(vz=c=+c;DN<>0K3s|3}NzZ+Z0le(_dM!rOI& zAGT`G5Z<=B&qH5Nss1W|ce4Dyr&W61;EQ!SsNqGsHLL8Qe|e@%SKVu0)u!)E>3+_0 zlC!>cZyBz8G=J_cc(Y1#-DUI0mGcf?r!5NKGR@Z+=})Tl+!nT1)$Xbl+`WM1?c&T$ z&u}TyH~h`pkPj&hbl!mQ680N?=)N;}IZJU*h*_F!`i9J$2pv@(6yr<%x*E`vJhjLb zPekI}Sbvg|f7llSBa~EaWO@F{(2MWQD<6qyN(>Oz;^G?xudLsv!ZBtCDS zS=o2Kn-DxdXSw!eWb`7yKQE_THZF zH?7GXk^H_2^8T%YTP2vYve{LK&Qk)OR8s6iN6!0`e~X@-e{+@jhswa;qo{|%0--q9 z(l2j|S5ykz#=fUrL_`<YBHnGd_PcWBHu%HLrNS!Nb#6;u%0&^R};P zPVvtk#wS0Cfww#T=8PyZza736 zs0j{NfBw?ly|=Jc?C1PJ!f&-do?gWI>80*}*!A_v=;Kbl8Snps?DUOq#=cnM`@Rm5DC#WL5=MgG((=TCocuJz4&=hpz;T<-e8%*k77 z<-?F~RuFyf0!4~Z-u?WdO5alixD|6hoP_IVeRP{6Phg#0+|E4=Ug&$c(mi6WEtrORGe))35<%8~jE8W4;zk9dZvAc!~^zyrn z+Xwr8BKzEX+;cya=3htkE0rn!0e^u%{|8V@0|XQR000O8X@^cuwI=}a5extT&Nu)7 zlRhIIe`{0QNb>vsiYbyRL&`GPkc3cox8*>Rr7k=Q2z7TXw>6f=Sc|OjYDNY^@!zkz zXKd>+vSo1g?zSq4EcJBv^t*dzAAe%--`Z>C55$nK?Xk6i)+4pIxoO_ChCJpsVklzW z;@{Imw0t?V-_+(-tKI5&qeP5Daksgj2(HDqf5QXO593gWGHxBEkq$41t>2Z5U#&F_ zu>p^RNF+$?DD)GlWM8-T<#3d0F7vU0}q$2l8j!+6& zf7T{602R{sDEcxH%ty(E@eSkb@~FukWXguA(&oK@x3RcmD(xvh2?-^GWlRqQ>w;u~(5J8hbkl08wz%(>OQiVG% zYy#zd$HB~UG6so7VSHPXBfjT75uqs01P@pvg9>SM(PWv&0x^o@16nbY0D%_NAb>z< z9!4SnwiT$)H~MN8siaAm^^HLKsx_S22f73~$0kSe-nf6SGmoC3?oBAK<*+~ff8=_8 zYR>jYHE;~)t5)0V-T&D zQ-cFLFaf2gcWY#Mwc{>9D?E&~!-vr3!bAiP$D^+|;vU)j_Z9cPU$0+z@Xwa_;rh*$ zeb{*1YHqimUKcHG%%de~fh*Tku6ujz5xAY6!<0;4x7Wrhk7Z!Op_m&aO!p=ursXLjV zT-Ht}&jlFDD9P|N1G;Zsu~Eemr3R;IFA9CwJi>z**oa=RTX<7x?=R4y5B4LcOrA|O zy8iQDTW_wq-oH?@8}#efe_w-*)?>TVEW5wKa_I^J7ht86{n`W#Y6e~z*DQjhr&&Ae z+f8;C>Otls{HxK#F*39n#E%{td8I5W%ZEW22&(|%5g__tEk%JKSVm_AQccX^UA4^} z>zQu%z@`vW2%bSX@!zYgP7^tX?Nem=0yQa=Uy zJc-#5YQKfG#toQI+M^nSs+moqzo`=;HlYRw@|!C0H?h!oWpZoag%m&Zg9nD@QCYW& znu98ul8O(5l~f-_s#7f(ag%lq5AsU_(dW=4Tw+$mmAhh__usvJ@A2J!-`m;Q_C5ao zt?zaEV%Oj4`Mr1Be{U<(61w!+$L{`wRIy^A{GC-b!xfeV7AgX0oM36)ARJBBOEp#S zJ`}?i3;ay5{x(#gW)1|_)>9>#4y! z$utL#gjJNf8~;4{cE-+5zJ6lI-J?&9A5y+S8s=WV>wUg{+d_Yx ztTJ(+H!bW6y_ef$pHb1-r|Svu8lWB3Xkv+tOR$`M+xMB2Fwf15MOTAE-}u#CWuulkjT*icyxJ*Q`FW*VxR;YaFQy9wQYiS^rD&QbxaimrnW2vm5E>LJ)!S}h}pI& zIyxI=g!9~#0IcB?Yh~fOMO>6d)pji0*lv5<@%hQMe`XQAoqF_kRz>e^J$i3fMekib zdhb?6?|nUb?^i`{w;sLSRnhxUkKTt>(Q9wjBiP9j$r;UGQ0we(tZh%Mw2ui6i~%@EBc3pE z55Yy@$@CtpX6EPDf1cqSf^>Q8i;MDzf9_$!dH7wNL*wjgnar^t1Y2kMS@TBcN~y#(yUIpbbz}Mb(LCHELk4X-spjN4Dm2F4^xJ zSWLYx0Ck&Ajn(Am*{FT7o0y>qtiPhE*m}b~^=fRi@*|z=MU~PELN)gX0`26>f5+7h zjtDVRfSHY~i!2@&wb_M>jl6?$tr2&?0x&M-&*)f3g;qkq%ba(biMM^zz}b?<3_4qD zhu{iwC?rqtpGi4krdl(w?=G-$#JbQp*<0b~r<&#tN!&AnC0ILwnucn!CwqhlT zOcQc}zr~U&L>-5?CRTGf-)hjm4fm&s=!f@IU%;}>_PaXCU_s~@hH=-4f4#G_82e0I z4$S6bdtqm5P0!iYE)*g`O$2E$dF3&Ig>#Fsyiw<73Bw|B5>T~BcdIy?E!nr(yWOb( zwP6h}6hSVRsj>?x@H+w>ZRQ8{?2rh_nLl#>779VAi=kTT`W44EFG}>YTXC3udy(uh zW#}zV_$ZaR34ebeC17nJe;|$Gc&a7NxO~X053?9#DPjb{Et<=k6q$%2d&LEZjw&0a zAiVWZLfBRt3>4nXJ(SrMR#S4!i1eptrIXvAFBk-_qC{}Y(E^Qg!d_L8Kq6X5(m?$` z6$-q{YgRtZSU(nb8|*x9xO1IhdRet-?bP?@Z>XX#PG{PaoSb#ee->LmmfGHK#(U0< zIiB3W0$D-Q1tO%=)^mKe>eTgbCKKT%l_MT5QEpY zIXK6Y9n*~vseYI!y{vk2?UPV>%@{5$xicQ$WXZj7XF0+-s&gBVACVV}4pmvh8R9$= z$`=cX&qalfKp-a-60tDjA%VUE<58Dp9(beg8`W+$df%jBus+M?CS&8@Pe1(!P)h*< z6aW+e000O8NQ+KRO=|P3^&$WOGLwxjG7f2nPENHa0PztF007Q7lg%$X25K(=0001P C;=Q&2 delta 11497 zcmZviV{;u0pl)Ncjcwbu?Z)SP&2p2oMG)$|`c2<+l$M5D)~3xHP<;6b-M<9GIBNQO9YJQp2dQ{l%Sy zLBy6RMpx@!BJlG{OA}S6rvX*z&luFZJsa8I9lhETY~CW*Tr*Mrn9?3-}B>G!|C^e^c1-*z~Q zpESG_BamjIA#TjS`U=gXl0wM;%_`P8eH+HstCR{X(d}?o*-z?&wNDwl|1Qf62R8pQ zj*9Cq*^&j0JtEdMkf4)_Tz}{Xz;+ zZM`ShgwK9i`v(uetU%hQ--VW+p~){Xcqy$FMfJRa61MBb!_clHU_%;q^$mguzPMnA zJJTv+9I(`JfVzixWvw|cmk`BQ#m1`@?boez`GEn9=_d0v8Qxjl?CeGX()`JRgfDPO zvU&F_I3GSw$1$rPd-?|@fVb$GRJ8fbk|cx0$ebb zMb%(I5AFDNVr1`cqmXVTg~{e}T1vWji`;+}r_NG)42HZ72JJu-0t4BUeL+s8M6F%< zhxqk~HA?|~HQgig64NAc%OdB@3&y%R@-8^>U41_2YO|NJ&tCyA{;|SKQwOLv5-H%U)%XKHS&Pf_S7x1NE~#1erWd+qw~zW6v1Hlu zNQQWKBLF?BULrdT?nnJlw(Spy?+_prmkoGPGT8eE|1`fSPy!4M5I=pllT1xxyzd1A z`M1enRPF^WM4`!wvnE_7q*@#{E_7O)6bmwwRXrp_I^=MEiFl3HIZBiStiprP=V%X8 zp-50NuAF(7KTQ6x1*9<48~Hs29o>?^E+PjHKe)<&1ob+x^*4@eZRJ}iJF{fJQSq_} z#AEYwHw@%41VwxjP&W?tTmPu+_Tqr!!A|Sf``$_JaKeCZfbi#Sv74hwbnoYbnm3GH z2o*G4e0(vb-!d|uHeDDsZC|eFZ`61u>U&C)ct+`qu`PRjJ=}63Ic0pXnW0aW9rJ#D zKH!)#4kG^_wfvlwQK$jZ?KL>0XY4Ru-Wu`XhKvZP0;Bi>gl=LzlZiaS=K3PBT8n>W zEJ(4S({nwc28qJ3hyD;mnD%zujz#4(#!}2F_-K1O>i5$|T5#sli0>&L6O{a2X$Snf zcL%)#4#kpn&J%sqj_t@C*4=(J8cO7xc0a2~CaoQKAgxHGszJd+Lotnvfqx{>MpNqX zfObYGDM}*%I$t!gynK`mH2drs8445XmUG$jJsv@|n=hpxICS=PD&j;+nRy`T0Jb-M zy{o-LBMXT{)ot>NY#o(VPzn>yu2hYmA}2g4I9YFBD1Nb?%i{-LRxbE`g9*R`uGq*+ zJ2lnL3*&kk|MhAqfep1`Sn7?pM`&msNcyI3iEpe9#K2HMz9}^RvVpZ_#eNNr7IwO_ z?XS9v@;e2GT{?YqKikCLmuewZ2laa9DEYl!9IOl zekEFDw73cu;!9Z~!v_JdmI`+?H5L{*}MGRk#e~e>dF1g^A99+ zOj5R&um&-8JL*ej43|B==S|_!Up&`oGv`DjojvrkMKcfTVDGy%I{G;T#`c}0x|2VT zH71Pf=p1q`&|BbRtjvS~{ojt=4?OYd{5wr?pq6Np2eoqk07^?P<F!8+Bxp8e70R(;WdSK65tx6V=MSD16!P`gksA z2bW`-F0~!2s=a!zHii2p&nxsBhevbRm#T0?pBm*Kt}JM^sPI#_$B==<9#2)PS9f)q zz-f&fem^YxZByoho1O7L>x$8C2NctuE`4)yOb>UEr-*e+p6}Of;do#PZ>RmI#q%u% z{vGhDmI3pFy0jW$Me-|$ND*lNib@Kg zH8@j-!2ta%`&?h;q^@*1aygSoj zdj0!*j_yvYYn`yQA&YjJ75H{w zzIi52lZHDy65w+)ej)rugFZ#<`boP%zvOz~rlx*6+&O&b3~=-E+~nVu%? zE^bV5=C>Cq;jeBfe?rmje_v zW8n)rm0a3txDQ6sY+0ZHp_&11lA zs7;2&u@3N!oCp}_3SO&3$qj*aniq!M>6HB!)|x)B!Sev%xxRR|_hyooUB8PhhutC# z;p{0Y+*xR{LS&^pDvZA7ERSdL#w^B(SB)KHcXZMQ73o6;^uY0&TqotKJ;5R9PgtC= zm#~qiOmGJH%!wGBfo#p(|B+T+J?Z#sq2(2j5$^~2!yca}F1ejnj`AmlHOfq-&e}~M zLa^gFf*~I;WCnXPvjzT}(FHS67S=IdI)mPXPqmdl|7@zFnHW{@XVyP_ADV<1xaf|2 zc4z{Lm(4cF>V^z~1xNo9&g^FKlB)ANLVx<=XolDqT^r%X)a>@%yN2K-f@}tWHL9L4 z6&FsnJc#4GrIFAnn9nVA5y8OPEzR+qxw#=s+=&qQP;U7>I8H2TH;ng2OPNxe(mgke zyazAt?UP5&M*?7&+UnT9n7rYNgxZn-^{zNq-Kn|!6Hy4CWz`TSQU+NaGa%2+A{Ngf z!cjo>A*Axh==iBv75a1iKj)6@6Q>PQmkX^K=*7J;JI2$)3?e}l6O-2{j?@_k zjmahMFq}V|6RR&_N*Po<1>r@TgYb*?LI*8)&Tt!M0Gf(! zS!USUDH~(fv4W4$C7ibf?=E*kXW@LG*o6*oYlZ*TkoG(s^7~F1e{g5F?P@d4$Lm0m z*W@_41Y$T}Bqk!6vr!#*e8+D&{qKiB^pFJ7P{ZLWW04A+wndqFCY+E>EFxgeZq~#R z{&`jAkC%XgjTnu+l;Byj$6CpDLoyvB5w8c6_wlVd+Al?SJER#u8~SDyCIWZKGEPAt z(^^O{9AT4vRVa!-v+K7uFW^#?9KhEbl8xkNG_LQw(Yj_HPdiKXh8MG6)Lg@~GNOAyH3kNk%%^%eA$@$8Q z$lr*W<+wbpzXEdN3u2Gi!3Sx&8ozkB=Z8YGzm2vm>x|gPfExN#VVhQT-3kL|Blua1 z%CI*yVE#tS`R%<-5vlw@cCNH|xG`U^2!FU_2`TnrRGx{S13JJU8RN`== zhzGp!^e>;5P>&W0+f>Yk zC#}|_gve`9Gc5=k$ty+gaRpJcgOW3Ohq^c?;LTaDxYSt3J$-2G6?64mI4M+1nbW26 ztXT(qgBog1Bf#Dp;ubcsKGCLOGW4KBdhnW=!F)Ix`v~NYtodPkx|p6t`!J$jlwO1q zwGqr8R{=sTX+M@_>%tY_pqvFyK48WtgD%#gF09=t_h)OG-jbu%9XR6>WpD)vU^i&x zPCn0J@l9LyZUc$ffk%EFNe1Kre&WU-MPNoXaH)BGRgTbQ@eo`EsCJ`*SVJFY37s9* zR+I;*>CZWG@wrC=r50_fBp=dPSkd78J*xN0@K*NiXYE+S9qoY(1{)=pvyIh4npj@* z`rQADmzy^1w-BF|>+DocwQmkubU)c;-GQD`O*Z^+IVT~$(Wz!p0xdcOOT?*kuNJ~+}kn1+` z)-KHGuRFXgYsY{GX{WWnwcMH1_Y!OBi1}7H$d~jgtvnuz1$0$|tQVcO`<(A91P9Pd zJxb_XM261J^uu)6mpCfG^goCxuc~ooN8}sqM2m{-HjSh1=zAb<`DWpmN9T{LHraQI zH=)6CEgjslmvw6*TBwLhxJzKY>;%IujSM_&$$l0632)#~^szY0V5X2B_srvvYl0ea zX*&eypD7odH4SdKcq@R`*pv26&E49a9FZ8!-Dhn0SGM&@go;j(93G=kfzr=YB^iUx zjbefYA6dIbu1Yy(_N~do-KIvY>#pV?Uy(^y!Y*eOX?*y&@M)fN9qv*Z?lk_I+KYE$ z`ii*-V@IH_heMnYezvDeXrM4KVn3(gjnPABU&9SYWdzK#dsn@_)!ZdO#VBQX@z~MD zCeG)l?0ZY(B-q0(Baq#Ovj|?Nx&~W2JvL@YOmn)WpHhv48EfX6<@AKyv2Zw%bt+R{ zz=E)+IO{hFGb;0QDkuG#FQ@m$%Kny#rjBBXf`MTZ)_*Z+a(DC@TBMlE9%yMq^d{^R z$p7d8=A32y?G6+9d4G z=ZW+*8Ojb5c&9uw?I}lmsQ_-N>L$rY3ZC^CYa8q^n3^F~C5{}LJ`db2@^ZYVS3zEV z{6KBW$d;bw*@hO12{%8m%082`(;1W_K4^r&#@fholUDGSue8~nu~GoMV?O565tJyi zERSWDk94-|%pL)%X+%`M1!xh6ml0h%%qieF?Ov$a<`*rn<-`g8sgc)-z3juZ_}MBB zj zioZbsDYYe*O%Ca^Mb0LSgx~4GU-AM7tusgXL_%*IvX z>TqU47?wE^>G3naDe4i4v{S-P@=;^}Su4uug*!&!UsVITEfp7Pt9fNUV&5tb&N{SH zWe*i#7+QsQH>YvAh~|ZauzhwQaGMoUZ|lZ@zZ z!og&Zdg>6ai?!He=;mb;bL5qB^Tefe$HLNwW(hbQI$Bt@1+^(#q6-+{F4^bp=)NC|k;h@%Nw~IEZ4ad~l^rzdRi?&BO9b%l{5@>-(6DgQVLvulW{SY@$d zBBPH$R7Q(gl~4hv{SAjEu^#Wxx zGOpb4yO?iP|E9oN;ZZv*q;Wiwc8j?lZI=)=7Qfb=Dt|dEc|Z?j0+W=QkSsMH5!0ww z*^SQRS{l>Q5*>K1Q|UPI3jW#-|Cz|Ra=@~)x_ZaKiJ>#e9Jt6BGOgnv9q54PJ!npK zebfzRx72VZ%f9H#wmX$^jGuR>X-@Pau84{AtW;C-xK57MMP16ki&~NMjF=zfvSZzK zJD9jB;yJftz?!h^0&X2@wV6~;4=#LD89Y?Uw2NB8NAvtGv!|x)<~C#y*-5BzCLbhm zSY?w4a#IM$nm@D_uRQQ*dU4fi9iAzvA&U(V$;VIyNFzBFxa3gevsTu}QbD_?JfwNx zwfT`s&{!|X@ygBBO!1i=v;1_6V!M>F*Kn#`+QnDgR_|Cd1=>&rF&SYLRAZ}TE=!^E z5a~oE{uEF~r9fLg)G*Bipe4n!%#ytkfI~_U2{G3HysF0shiIETb_YaDvQ=LNoiEGF zTXR<|wMiTjDkBnpnNB!2O znyd5ls<_5d$+}AKCX`nVx)dg~oriA|7gjiOcDDuDWkLc71RQ!f??8#?eDwni!GRsR z^Z=1ibv|A4qV`+ArVa_QwC~|aT`u;&hKNEPvWcjIg-jb#mZw5k)_BVoJub>+G;)Iz z7~@bX&483<{V~>w&ImZauEmig6>B$8gT7QOC2p z&((yyM; zz!1k0qtFBJUE50FDF@h-P(k*(nW<_qM9tP=-{qXwmPZxWR>cwphR!&(^;5|tVLqZi zl7|9e(o`sO`p`mS)Fp&kSrW-jNU!9>Yt1@*#!&H``}=^>|S9UDjR?hE@>^4%_<>D zVC9<9s7#A5K%CXrs@AzmFo{W-7X9lNVJp4%e6?AH>b?$P#fmm^#zI*atQ81zoo6>i zrrlA)04|oC$CGGdT`ni36p7O@&<$`#W*-#i=>(v#)@3MR7Phr5H!74+CIsn84jSs! z(aHWZg*NzGvBYW9w5G}P%GSbL*N_FV^>Rl1)LrYAT0GZQ=GJ>K2xQ$_>GX?pH?rOA zWp-f;Wm`Q2?H-52Uw_$r;Ghml581Ey7babVYX zv4->tE9f|;&4hAM32fJ*YY>K~MW;n^3gc@{bCba)##17a+um{hu3C@Q(y#AD=P&QT z?11;ZgTcFpH=~>!z}aYU>?%!S*74m%#dB=qpoy`7+K1~K@m=N9&=x3+NI;xk`RCJ3 zk=go5-+l3t>r5Pf)RpH+OjSeV@3FZoxnEmph+k)%vzMQnWr7)jguMX*Co$LSFVE6s zCdXv3i}I~UAV^|%h2Y54JH=5>_8ED`(_>2+j90N-i?gQqK@xEjW^qVfd~j?H^-)aR zO@8h}ip1xFa$_#8PoekYT^naWmn5ueuBrDMCCQJDFCPPtA29*YA}&+5=F=-)NvL*@ zh&HZi`5wJ=q7U{UhvQ>u?tU?Gy7k!SD(%=rV*R)dDU~mkIG{#o<<1E`)`DdGQ7k?b z*mZ5O zYk$9n&31XIz827UMO6=rL7Ut^;`XZ^CYu$srhbSJ`s0UZ>wh$snE?x4yH`DOC3rKR zgrLPm<#(w_8@tD)%^3^N1>>afGaHc!bKBDjoaXnbgY zURu5YRV(ki`f<T}-LfBrVtuHfAqj7F}Nd(?D(Oo0j^x{=gZckVn|Z~H{XoQR?J`cEoqU_C303|;x^ z*zG$gA9pMJhsd{?WIVE};o(*!W4&J0a8_{oi(3u7yx*osW5bmnFS%F1lh_Ic?5!2aDaCZ^ zuk{aN0aZjV4f%R>V*V>tAd8R3=cM)_DFZYxzJLS?Ib_#RvA2uaSe@R5@GZv!6VvFq z(O_EIq}*e@9I)v#_5A)2bMR|qscZKI4qeqG(gwnDa;>L?e^}HPFZZ1aZu9bfjk93v z-Q(Fh!PDPP?)820&s8ObiSOUHNg77#ncpx;fq+$T#!ItGCxD+-*;XQP_Lv`koJdsm_tdgAxp$XN6;-cNVY z4F0N^+d2~;F5hZ$fw65UeSs~+`kzr?uyCM>1qO36!x7X56w6I?QAKW`=)9}W7w^Un z{reb0LltYdb9P(L-FPY`3#r?e!3i4pbE9)1ZiCw}FLldtU1a&``^wSgUd~%S*O$9i zuSWVV={4^29T4LG*9G+d)dd@_f3FEqARuT3AV9*DB)|{nwYG;R$T!l6rr)3eS9@B= zIrFuChHc#_f|*S*aasA|U(|C<|skpJ^t zF5df!SItKv-d(JxgGDmc_YWuEHKCIj@1fdZT`Bo+prXah&BAie+#0LkhRSs2kU7Kf z#bW@I7m%Wdgd`$A#*#vNo?l^5$~Z=_*A82F&DOw)k_0wCB5^J8-lBMc?c}`jhDvz| zg&I01N*9_QhNW&Cf=1}jA`t?wPbe+?+N-bm{$}Rj95b_*Is)Ne0ux5e@Rg6uzaHgF zT66IM_No)Zb?Ph0gRNwY5{+^32tN6&s9iyBaiBXEOjbRl{n%kQUWG0jsXJXEU}R-j zJ#(WnIaV_}?FcGIW*2bRj~O-N!5VD! zc}JL4fsC}E_I$;2s8)$J4p=bAthxgB~?fu)jQ#%Z^gG7L^!T= zBp^nbtddMbH7rOH!vKcD_B#ScCKU=@EGkHAFG?|5EM?1;t*szK+$ulD35Pq9^;s;W zY(ldidhsu&B#S?n$zi>il7xFa&Loyt9&D-Xh&IS?r~Q zbE*pFBUs2PlM?7G(52{VB0RX6S-(k_1O!feO?-Gcur``^dT#$1eYV85(D9_cR!V&H z7(E)&+s~`f@?xQl_-A8B)enWh)Hq4>I*jSSd(5Yvei102MyJUG32>%H?n@c?bit=) z373FQu17EokgKg7CRTX8QoQuoN7zB>DUM9QcWl z1-mcJVN!6N6MCvMB$6S3PK&WITOU!3J_VWoXLM|+FJXsxK##Vs7)tyf*2|MvT}|uU zcwq=b$0gRpUOmgUUvBk%aD7aVJ!WUif)y85Atq$CL;Ax+t8lexVk1$q)b0q4D}(97 z5(k6cOb|PmS@HywKno!%6P0U~3ggpjqB@>KX zz`7SpJo=8+tb}^fS*TSJ0aU32nMEYBrk)d6waIW#+4S#4#OB)?8W$J_%NIj4D3>9v z(qOiLbR!(g_QZo6iQm?AbA>p|`L3y?!TX!ggLh?Qh6d#l3;zZ@NZ|ks`Y{2GA8b*j z{sKaDJ22*B`CDy$QZZ$|=Hcijy*_(4-8=u*Ha2dAKMzx`9>UD0fm_7S!ApX*6`FXj zuP1MxpD}vvv-w%MTOMI}I&d`HkWD?N?wb?@jy!VT?V8 zmH5zvj`Sn(6Wpyv;Nn4a5kZkTYt5y%_FI;J z_up;a-;YJ=21>JrzcDY?SZ4QQ^h-GxjiAx}kB_1%U{cWK&k(qOdJ-$tT4X{u-#6ibrCd#$g|Ba48Tac5sXl581myI9>MrWZ2MG zhUqfJcCtQjX_efrA>qYch%4XkyH7pJ|Mk=H7g>@4IYo;_Me-&%*u47fTZlV+(p3IF zl6I=QMmr5MAFw%k>t{F6&7Aq5eJ$bWlEAyie(Y}HKRAuuJ^Tl|vAanE?@R9e_Ewo3 zXTn>*PMMsIRMd12lNu19m44|`p#3q!^@6{hMWoABjbVx0DnfdP5!8y@MKiPgTvS0_ zJI=|O6&jTBk3k0Eo<$TT4He1TDGFNKOuJY0Ss8c;gL?mip|8<8GwP};AsaPMH(JU; z{k4LQjbW`p#?HYVxy(W4#8G?cGdvx~V`xKLsiJ_~Goq=2#c;A+IJOUGxam-fJvBUR z|1A!+`lNYsoaTc^T;xKxk3>S4lR5>yz*ol&UHWNawG7k51xeP;=lzl|>1+}=oXz^X z&M9y_TA!(fZ$&(5rRh}wD_aoQW_rYa7nV6yd24l@^{y|pLVRiCZ*`QvcKP^-lPMAO zizn>uFo8))T!-^u&QWkDnAK;%Y<$u#FbBcpmW`M8ypA$}cJySKLd{=9R49SqcGCB@D7~sY$`Yr0z_&7$n+f zdODTc(al?lr(u@ZcvsY$2V2)$Un6X$=r<8!ibp0vaGMq>h#)|KzO+?uZx`V62~Rjf zZksOK!{tPonjN8Q|9uZkxQR6H61L$N;`O|m0L_x#*M1f~=7DhU1{>qo%t%R63P z%P*i znN)CgOC^2^iQKoGCVFIPgh%il5*;}JV14r_k6dbic(}%2Lp%)_YfG^2FSmw`NN*$G z_5|afrTeyPTjU@5`#EQGmTk}od7AXNPX~XF3Z9n7fRe*UDFQua7ydG>_5wpM46)RR zW$!{vmJCwUpm7yJ(%QFaCjwoI$@|A=_8-4E#U~<(gSpD+S9B~Tuk-EH_t6B2G6t*O zty<{29G7zPcI6S1>6b9QxNX8mCke9AV8@Smk}6%GwGNXN0)S9>8)MAlevZGk)gs_)v?uDORUq0JoK^@z^K zc5L4iZ;0-)LeImOya7J4_}KE;=33o!je9H~)+?M-yL6##R@_|SfL93`xY+V9F8sEB z-u@lLBk~kFZ}RP|j4`eQ;3;6j9Y)1u#niei?VpFiSG|nz3P9M^i>q(ib5O6xO?j*ENe&$(ICnu8A zN~2sLz?F)iYFrcS{b0Cq;OkYfs!LaV;7Ol@f4?JDQ-p-Vg7`mjWl%6H2ipHu4A3B} NAe Date: Thu, 14 Feb 2019 20:16:04 -0500 Subject: [PATCH 04/13] Changed VNET's RG naming pattern --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 5aa51257..c7258e5b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -203,7 +203,7 @@ def ext_vnet(test_vars): log = logging.getLogger("ext_vnet") vnet_atd = ArmTemplateDeploy( location=test_vars["location"], - resource_group=test_vars["atd_obj"].deploy_id + "-vnet-rg" + resource_group=test_vars["atd_obj"].deploy_id + "-rg-vnet" ) rg = vnet_atd.create_resource_group() log.info("Resource Group: {}".format(rg)) From 9722841e4293846a43d14f10ff8d826dae161077 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 15:34:12 -0500 Subject: [PATCH 05/13] Added get_vm_ips to helpers; using it for controller IPs --- src/vfxt/azuredeploy.vnet.json | 4 ++++ test/lib/helpers.py | 18 ++++++++++++++++++ test/test_vfxt_template_deploy.py | 28 ++++++++++++++++++++-------- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/vfxt/azuredeploy.vnet.json b/src/vfxt/azuredeploy.vnet.json index dac41941..4b28991b 100644 --- a/src/vfxt/azuredeploy.vnet.json +++ b/src/vfxt/azuredeploy.vnet.json @@ -202,6 +202,10 @@ "type": "string", "value": "[resourceGroup().location]" }, + "public_host": { + "type": "string", + "value": "[variables('jumpboxName')]" + }, "public_ip_address": { "type": "string", "value": "[reference(resourceId('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))).ipAddress]" diff --git a/test/lib/helpers.py b/test/lib/helpers.py index 1b954e81..9afeb735 100644 --- a/test/lib/helpers.py +++ b/test/lib/helpers.py @@ -19,6 +19,24 @@ def create_ssh_client(username, hostname, port=22, password=None, key_filename=N return ssh_client +def get_vm_ips(nm_client, resource_group, vm_name): + """ + Get the private and public IP addresses for a given virtual machine. + If a virtual machine has the more than one IP address of each type, then + only the first one (as determined by the Azure SDK) is returned. + + This function returns the following tuple: (private IP, public IP) + + If a given VM does not have a private or public IP address, its tuple + entry will be None. + """ + for nif in nm_client.network_interfaces.list(resource_group): + if vm_name in nif.name: + ipc = nif.ip_configurations[0] + return (ipc.private_ip_address, ipc.public_ip_address) + return (None, None) # (private IP, public IP) + + def run_averecmd(ssh_client, node_ip, password, method, user='admin', args='', timeout=60): """Run averecmd on the vFXT controller connected via ssh_client.""" diff --git a/test/test_vfxt_template_deploy.py b/test/test_vfxt_template_deploy.py index 65e645f0..34f217db 100755 --- a/test/test_vfxt_template_deploy.py +++ b/test/test_vfxt_template_deploy.py @@ -16,10 +16,11 @@ from uuid import uuid4 import pytest # local libraries -from lib.helpers import split_ip_range, wait_for_op +from lib.helpers import get_vm_ips, split_ip_range, wait_for_op class TestVfxtTemplateDeploy: + # TODO: modularize common code def test_deploy_template(self, resource_group, test_vars): # noqa: F811 """ Deploy a vFXT cluster. @@ -53,9 +54,9 @@ class TestVfxtTemplateDeploy: "virtualNetworkSubnetName": atd.deploy_id + "-subnet", } + test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] test_vars["controller_name"] = atd.deploy_params["controllerName"] test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] - test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] log.debug("Generated deploy parameters: \n{}".format( json.dumps(atd.deploy_params, indent=4))) @@ -65,6 +66,9 @@ class TestVfxtTemplateDeploy: test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) finally: + (c_priv_ip, c_pub_ip) = get_vm_ips( + atd.nm_client, atd.resource_group, test_vars["controller_name"]) + test_vars["controller_ip"] = c_pub_ip or c_priv_ip test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address @@ -116,11 +120,14 @@ class TestVfxtTemplateDeploy: test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) time.sleep(60) finally: + (c_priv_ip, c_pub_ip) = get_vm_ips( + atd.nm_client, atd.resource_group, test_vars["controller_name"]) + test_vars["controller_ip"] = c_pub_ip or c_priv_ip test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address - def test_byovnet_deploy(self, resource_group, test_vars, ext_vnet): # noqa: E501, F811 + def test_byovnet_deploy(self, ext_vnet, resource_group, test_vars): # noqa: E501, F811 """ Deploy a vFXT cluster. - do NOT create a new VNET @@ -153,17 +160,22 @@ class TestVfxtTemplateDeploy: "virtualNetworkSubnetName": ext_vnet["subnet_name"]["value"], } + test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] test_vars["controller_name"] = atd.deploy_params["controllerName"] test_vars["controller_user"] = atd.deploy_params["controllerAdminUsername"] - test_vars["storage_account"] = atd.deploy_params["avereBackedStorageAccountName"] - test_vars["public_ip"] = ext_vnet["public_ip_address"]["value"] log.debug("Generated deploy parameters: \n{}".format( json.dumps(atd.deploy_params, indent=4))) atd.deploy_name = "test_deploy_template_byovnet" - deploy_outputs = wait_for_op(atd.deploy()).properties.outputs - test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] - test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) + try: + deploy_outputs = wait_for_op(atd.deploy()).properties.outputs + test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] + test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) + finally: + test_vars["controller_ip"] = get_vm_ips( + atd.nm_client, atd.resource_group, test_vars["controller_name"] + )[0] + test_vars["public_ip"] = ext_vnet["public_ip_address"]["value"] if __name__ == "__main__": From cfac6076226f1d70b61e077326b28ada5ccb6253 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 15:58:42 -0500 Subject: [PATCH 06/13] Added SSH tunnel capability to ssh_con fixture --- test/conftest.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index c7258e5b..1c386205 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,6 +6,7 @@ import os # from requirements.txt import pytest from scp import SCPClient +from sshtunnel import SSHTunnelForwarder # local libraries from arm_template_deploy import ArmTemplateDeploy @@ -122,12 +123,32 @@ def scp_cli(ssh_con): @pytest.fixture() def ssh_con(test_vars): - client = create_ssh_client(test_vars["controller_user"], - test_vars["public_ip"], - key_filename=test_vars["ssh_priv_key"]) + """ SSH to the controller. """ + ssh_params = { + "username": test_vars["controller_user"], + "hostname": test_vars["public_ip"], + "key_filename": test_vars["ssh_priv_key"] + } + + ssh_tunnel = None + if test_vars["public_ip"] != test_vars["controller_ip"]: + ssh_tunnel = SSHTunnelForwarder( + ssh_params["hostname"], + ssh_username=ssh_params["username"], + ssh_pkey=ssh_params["key_filename"], + remote_bind_address=(test_vars["controller_ip"], 22), + ) + ssh_tunnel.start() + ssh_params["hostname"] = "127.0.0.1" + ssh_params["port"] = ssh_tunnel.local_bind_port + + client = create_ssh_client(**ssh_params) yield client client.close() + if ssh_tunnel: + ssh_tunnel.stop() + @pytest.fixture(scope="module") def test_vars(request): From ecdd6f9b7e6c2bcadd9524a3df187366044e211c Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 16:32:27 -0500 Subject: [PATCH 07/13] Added debug output to ssh_con; fixed get pub_ip in get_vm_ips --- test/conftest.py | 19 +++++++++++++++++-- test/lib/helpers.py | 5 ++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 1c386205..3d33b72c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -116,6 +116,7 @@ def storage_account(test_vars): @pytest.fixture() def scp_cli(ssh_con): + """Create an SCP client based on an SSH connection to the controller.""" client = SCPClient(ssh_con.get_transport()) yield client client.close() @@ -123,15 +124,20 @@ def scp_cli(ssh_con): @pytest.fixture() def ssh_con(test_vars): - """ SSH to the controller. """ - ssh_params = { + """Create an SSH connection to the controller.""" + log = logging.getLogger("ssh_con") + ssh_params = { # common parameters for SSH tunnel, connection "username": test_vars["controller_user"], "hostname": test_vars["public_ip"], "key_filename": test_vars["ssh_priv_key"] } ssh_tunnel = None + # If the controller's IP is not the same as the public IP, then we are + # using a jumpbox to get into the VNET containing the controller. In that + # case, create an SSH tunnel before connecting to the controller. if test_vars["public_ip"] != test_vars["controller_ip"]: + log.debug("Creating an SSH tunnel to the jumpbox.") ssh_tunnel = SSHTunnelForwarder( ssh_params["hostname"], ssh_username=ssh_params["username"], @@ -139,14 +145,23 @@ def ssh_con(test_vars): remote_bind_address=(test_vars["controller_ip"], 22), ) ssh_tunnel.start() + log.debug("SSH tunnel connected: {}".format(ssh_params)) + log.debug("Local bind port: {}".format(ssh_tunnel.local_bind_port)) + + # When SSH'ing to the controller below, we'll instead connect to + # localhost through the local bind port connected to the SSH tunnel. ssh_params["hostname"] = "127.0.0.1" ssh_params["port"] = ssh_tunnel.local_bind_port + log.debug("Creating SSH client connection: {}".format(ssh_params)) client = create_ssh_client(**ssh_params) yield client + + log.debug("Closing SSH client connection.") client.close() if ssh_tunnel: + log.debug("Closing SSH tunnel.") ssh_tunnel.stop() diff --git a/test/lib/helpers.py b/test/lib/helpers.py index 9afeb735..34abd866 100644 --- a/test/lib/helpers.py +++ b/test/lib/helpers.py @@ -33,7 +33,10 @@ def get_vm_ips(nm_client, resource_group, vm_name): for nif in nm_client.network_interfaces.list(resource_group): if vm_name in nif.name: ipc = nif.ip_configurations[0] - return (ipc.private_ip_address, ipc.public_ip_address) + pub_ip = ipc.public_ip_address + if pub_ip: + pub_ip = pub_ip.ip_address + return (ipc.private_ip_address, pub_ip) return (None, None) # (private IP, public IP) From 8cf9abbdd949967b170167a1007450092db69659 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 16:53:27 -0500 Subject: [PATCH 08/13] Clean up VNET RG for BYOVNET variant --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ef42ea66..9bc28683 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -166,6 +166,9 @@ jobs: RESOURCE_GROUP=$(jq -r .resource_group $VFXT_TEST_VARS_FILE) echo "RESOURCE_GROUP: $RESOURCE_GROUP" az group delete --yes -n $RESOURCE_GROUP + if [ "true" = "$RUN_BYOVNET" ]; then + az group delete --yes -n "${RESOURCE_GROUP}-vnet" + fi displayName: 'Clean up resource group' condition: and(always(), ne(variables['SKIP_RG_CLEANUP'], 'true')) env: From 80b0807be78341c96c82aef4539d0e7ffab660ed Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 17:37:00 -0500 Subject: [PATCH 09/13] Patch controller_ip = patch_ip for non-BYOVNET variants --- azure-pipelines.yml | 4 ++-- test/test_vfxt_template_deploy.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bc28683..b113e970 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: pytest --disable-pytest-warnings test/test_vfxt_template_deploy.py \ -k test_deploy_template \ --location $VFXT_DEPLOY_LOCATION \ - --doctest-modules --junitxml=junit/test-results01.xml + --doctest-modules --junitxml=junit/test-results01a.xml displayName: 'Test template-based deployment of Avere vFXT' condition: and(succeeded(), eq(variables['RUN_DEPLOY'], 'true')) env: @@ -67,7 +67,7 @@ jobs: pytest --disable-pytest-warnings test/test_vfxt_template_deploy.py \ -k test_byovnet_deploy \ --location $VFXT_DEPLOY_LOCATION \ - --doctest-modules --junitxml=junit/test-results01b.xml + --doctest-modules --junitxml=junit/test-results01c.xml displayName: 'Test deploy of vFXT with a VNET in a different resource group' condition: and(succeeded(), eq(variables['RUN_BYOVNET'], 'true')) env: diff --git a/test/test_vfxt_template_deploy.py b/test/test_vfxt_template_deploy.py index 34f217db..02e28ac8 100755 --- a/test/test_vfxt_template_deploy.py +++ b/test/test_vfxt_template_deploy.py @@ -66,12 +66,13 @@ class TestVfxtTemplateDeploy: test_vars["cluster_mgmt_ip"] = deploy_outputs["mgmt_ip"]["value"] test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) finally: - (c_priv_ip, c_pub_ip) = get_vm_ips( - atd.nm_client, atd.resource_group, test_vars["controller_name"]) - test_vars["controller_ip"] = c_pub_ip or c_priv_ip + # (c_priv_ip, c_pub_ip) = get_vm_ips( + # atd.nm_client, atd.resource_group, test_vars["controller_name"]) + # test_vars["controller_ip"] = c_pub_ip or c_priv_ip test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address + test_vars["controller_ip"] = test_vars["public_ip"] def test_no_storage_account_deploy(self, resource_group, test_vars): # noqa: E501, F811 """ @@ -120,12 +121,13 @@ class TestVfxtTemplateDeploy: test_vars["cluster_vs_ips"] = split_ip_range(deploy_outputs["vserver_ips"]["value"]) time.sleep(60) finally: - (c_priv_ip, c_pub_ip) = get_vm_ips( - atd.nm_client, atd.resource_group, test_vars["controller_name"]) - test_vars["controller_ip"] = c_pub_ip or c_priv_ip + # (c_priv_ip, c_pub_ip) = get_vm_ips( + # atd.nm_client, atd.resource_group, test_vars["controller_name"]) + # test_vars["controller_ip"] = c_pub_ip or c_priv_ip test_vars["public_ip"] = atd.nm_client.public_ip_addresses.get( atd.resource_group, "publicip-" + test_vars["controller_name"] ).ip_address + test_vars["controller_ip"] = test_vars["public_ip"] def test_byovnet_deploy(self, ext_vnet, resource_group, test_vars): # noqa: E501, F811 """ From 87025bee071322f72a841cd496c65c828bbf1ec1 Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Fri, 15 Feb 2019 17:44:17 -0500 Subject: [PATCH 10/13] Added sleep between SSH tunnel create and using it --- test/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 3d33b72c..61b32f8f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ import json import logging import os +from time import sleep # from requirements.txt import pytest @@ -145,6 +146,7 @@ def ssh_con(test_vars): remote_bind_address=(test_vars["controller_ip"], 22), ) ssh_tunnel.start() + sleep(5) log.debug("SSH tunnel connected: {}".format(ssh_params)) log.debug("Local bind port: {}".format(ssh_tunnel.local_bind_port)) From cdec12c7791863f3a4766f8eae11aee07d9ba44b Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Sat, 16 Feb 2019 10:22:07 -0500 Subject: [PATCH 11/13] Moved artifact collection into test_vfxt_cluster_status --- azure-pipelines.yml | 36 ++++++++++++++++---------------- test/conftest.py | 2 +- test/test_vfxt_cluster_status.py | 15 ++++++++++--- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b113e970..6507afeb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,24 +100,7 @@ jobs: pytest --disable-pytest-warnings test/test_vfxt_cluster_status.py \ -k TestVfxtSupport \ --doctest-modules --junitxml=junit/test-results03.xml - - PUBLIC_IP=$(jq -r .public_ip $VFXT_TEST_VARS_FILE) - CONTROLLER_NAME=$(jq -r .controller_name $VFXT_TEST_VARS_FILE) - CONTROLLER_USER=$(jq -r .controller_user $VFXT_TEST_VARS_FILE) - - echo "PUBLIC_IP : $PUBLIC_IP" - echo "CONTROLLER_NAME: $CONTROLLER_NAME" - echo "CONTROLLER_USER: $CONTROLLER_USER" - - ARTIFACTS_DIR="$BUILD_SOURCESDIRECTORY/test_artifacts" - mkdir -p $ARTIFACTS_DIR - tar -zcvf ${ARTIFACTS_DIR}/vfxt_artifacts_${CONTROLLER_NAME}.tar.gz vfxt_artifacts_* - scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -r $CONTROLLER_USER@$PUBLIC_IP:~/*.log $ARTIFACTS_DIR/. - scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ~/.ssh/* $CONTROLLER_USER@$PUBLIC_IP:~/.ssh/. - - echo "vfxt.log from $CONTROLLER_NAME:" - cat $ARTIFACTS_DIR/vfxt.log - displayName: 'Collect vFXT deployment artifacts and dump vfxt.log' + displayName: 'Collect vFXT deployment artifacts' condition: always() env: AVERE_ADMIN_PW: $(AVERE-ADMIN-PW) @@ -127,6 +110,22 @@ jobs: AZURE_CLIENT_SECRET: $(AZURE-CLIENT-SECRET) AZURE_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID) + - script: | + RESOURCE_GROUP=$(jq -r .resource_group $VFXT_TEST_VARS_FILE) + CONTROLLER_NAME=$(jq -r .controller_name $VFXT_TEST_VARS_FILE) + + echo "RESOURCE_GROUP: $RESOURCE_GROUP" + echo "CONTROLLER_NAME: $CONTROLLER_NAME" + + ARTIFACTS_DIR="$BUILD_SOURCESDIRECTORY/test_artifacts" + mkdir -p $ARTIFACTS_DIR + tar -zcvf ${ARTIFACTS_DIR}/vfxt_artifacts_${RESOURCE_GROUP}.tar.gz vfxt_artifacts_* + + echo "vfxt.log from $CONTROLLER_NAME:" + cat vfxt_artifacts_*/vfxt.log + displayName: 'Archive deployment artifacts and dump vfxt.log' + condition: always() + - script: | grep -i -C 5 -e vfxt:ERROR -e exception $BUILD_SOURCESDIRECTORY/test_artifacts/vfxt.log displayName: 'Grep errors from vfxt.log (+/- 5 lines)' @@ -167,6 +166,7 @@ jobs: echo "RESOURCE_GROUP: $RESOURCE_GROUP" az group delete --yes -n $RESOURCE_GROUP if [ "true" = "$RUN_BYOVNET" ]; then + echo "RESOURCE_GROUP (vnet): ${RESOURCE_GROUP}-vnet" az group delete --yes -n "${RESOURCE_GROUP}-vnet" fi displayName: 'Clean up resource group' diff --git a/test/conftest.py b/test/conftest.py index 61b32f8f..ab2c28ef 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -116,7 +116,7 @@ def storage_account(test_vars): @pytest.fixture() -def scp_cli(ssh_con): +def scp_con(ssh_con): """Create an SCP client based on an SSH connection to the controller.""" client = SCPClient(ssh_con.get_transport()) yield client diff --git a/test/test_vfxt_cluster_status.py b/test/test_vfxt_cluster_status.py index 86fc4b4f..e3bf27e7 100644 --- a/test/test_vfxt_cluster_status.py +++ b/test/test_vfxt_cluster_status.py @@ -21,7 +21,7 @@ from lib.helpers import (create_ssh_client, run_averecmd, run_ssh_commands, class TestVfxtClusterStatus: """Basic vFXT cluster health tests.""" - def test_basic_fileops(self, mnt_nodes, scp_cli, ssh_con, test_vars): # noqa: E501, F811 + def test_basic_fileops(self, mnt_nodes, scp_con, ssh_con, test_vars): # noqa: E501, F811 """ Quick check of file operations. See check_node_basic_fileops.sh for more information. @@ -30,7 +30,7 @@ class TestVfxtClusterStatus: pytest.skip("no storage account") script_name = "check_node_basic_fileops.sh" - scp_cli.put( + scp_con.put( "{0}/test/{1}".format(test_vars["build_root"], script_name), r"~/.", ) @@ -86,13 +86,22 @@ class TestVfxtSupport: assert(not cores_found) - def test_artifacts_collect(self, averecmd_params, scp_cli, test_vars): # noqa: F811, E501 + def test_artifacts_collect(self, averecmd_params, scp_con, test_vars): # noqa: F811, E501 """ Collect test artifacts (node logs, rolling trace) from each node. Artifacts are stored to local directories. """ log = logging.getLogger("test_collect_artifacts") artifacts_dir = "vfxt_artifacts_" + test_vars["atd_obj"].deploy_id + os.makedirs(artifacts_dir, exist_ok=True) + + log.debug("Copying logs from controller to {}".format(artifacts_dir)) + scp_con.get("~/*.log", artifacts_dir) + + log.debug("Copying SSH keys to the controller") + scp_con.put(test_vars["ssh_priv_key"], "~/.ssh/.") + scp_con.put(test_vars["ssh_pub_key"], "~/.ssh/.") + nodes = run_averecmd(**averecmd_params, method="node.list") log.debug("nodes found: {}".format(nodes)) for node in nodes: From aed99356ec2cf2366c40953fde5036d9788a36fe Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Sat, 16 Feb 2019 11:10:48 -0500 Subject: [PATCH 12/13] Download controller logs in text_vfxt_cluster_status --- azure-pipelines.yml | 16 +++++++++------- test/test_vfxt_cluster_status.py | 3 ++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6507afeb..23c99318 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,18 +111,20 @@ jobs: AZURE_SUBSCRIPTION_ID: $(AZURE-SUBSCRIPTION-ID) - script: | - RESOURCE_GROUP=$(jq -r .resource_group $VFXT_TEST_VARS_FILE) + DEPLOY_ID=$(jq -r .deploy_id $VFXT_TEST_VARS_FILE) CONTROLLER_NAME=$(jq -r .controller_name $VFXT_TEST_VARS_FILE) + T_ARTIFACTS_DIR="vfxt_artifacts_${DEPLOY_ID}" - echo "RESOURCE_GROUP: $RESOURCE_GROUP" + echo "DEPLOY_ID: $DEPLOY_ID" echo "CONTROLLER_NAME: $CONTROLLER_NAME" + echo "T_ARTIFACTS_DIR: $T_ARTIFACTS_DIR" - ARTIFACTS_DIR="$BUILD_SOURCESDIRECTORY/test_artifacts" - mkdir -p $ARTIFACTS_DIR - tar -zcvf ${ARTIFACTS_DIR}/vfxt_artifacts_${RESOURCE_GROUP}.tar.gz vfxt_artifacts_* + P_ARTIFACTS_DIR="$BUILD_SOURCESDIRECTORY/test_artifacts" + mkdir -p $P_ARTIFACTS_DIR + tar -zcvf ${P_ARTIFACTS_DIR}/vfxt_artifacts_${DEPLOY_ID}.tar.gz ${T_ARTIFACTS_DIR} - echo "vfxt.log from $CONTROLLER_NAME:" - cat vfxt_artifacts_*/vfxt.log + echo "vfxt.log from ${T_ARTIFACTS_DIR}:" + cat ${T_ARTIFACTS_DIR}/vfxt.log displayName: 'Archive deployment artifacts and dump vfxt.log' condition: always() diff --git a/test/test_vfxt_cluster_status.py b/test/test_vfxt_cluster_status.py index e3bf27e7..7e4683a6 100644 --- a/test/test_vfxt_cluster_status.py +++ b/test/test_vfxt_cluster_status.py @@ -96,7 +96,8 @@ class TestVfxtSupport: os.makedirs(artifacts_dir, exist_ok=True) log.debug("Copying logs from controller to {}".format(artifacts_dir)) - scp_con.get("~/*.log", artifacts_dir) + for lf in ["vfxt.log", "enablecloudtrace.log", "create_cluster_command.log"]: + scp_con.get("~/" + lf, artifacts_dir) log.debug("Copying SSH keys to the controller") scp_con.put(test_vars["ssh_priv_key"], "~/.ssh/.") From 7a9861124adfecf2e17e10c97fbc7fe28cfa303f Mon Sep 17 00:00:00 2001 From: Omar Zevallos <45883639+omzevall@users.noreply.github.com> Date: Sat, 16 Feb 2019 16:36:56 -0500 Subject: [PATCH 13/13] Reenable Pipelines triggers --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 23c99318..cd2669b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ # https://aka.ms/yaml -# trigger: -# - master -# - releases/* +trigger: +- master +- releases/* variables: VFXT_TEST_VARS_FILE: 'pipelines.json'