Azure
/
Avere
зеркало из https://github.com/Azure/Avere.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

security fixes: added sshkey and NSG (#110)

This commit is contained in:
Anthony Howe 2018-10-17 17:07:50 -04:00 коммит произвёл GitHub
Родитель 9ec4a9a3b2
Коммит d153155fff
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 213 добавлений и 618 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

39
src/clients/loosevms/azuredeploy.json
Просмотреть файл

@ -15,18 +15,12 @@
"description": "The admin username for the virtual machine clients."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password for the virtual machine clients."
}
},
"sshKeyData": {
"type": "string",
"defaultValue": "",
"minLength": 0,
"metadata": {
"description": "The optional public SSH key used for access to the VM."
"description": "The public SSH key used for access to the VM."
}
},
"vmCount": {
@ -63,13 +57,13 @@
"nfsCommaSeparatedAddresses": {
"type": "string",
"metadata": {
"description": "A comma separated list of Avere vFXT vServer IP Addresses."
"description": "A comma separated list of Avere vFXT vServer IP Addresses. (ex. '10.0.0.12,10.0.0.13,10.0.0.14')"
}
},
"nfsExportPath": {
"type": "string",
"metadata": {
"description": "The path exported from the NFS server that will be mounted."
"description": "The path exported from the NFS server that will be mounted. (ex. '/msazure')"
}
},
"nfsMountBaseDir": {
@ -81,16 +75,14 @@
},
"bootstrapScriptPath": {
"type": "string",
"defaultValue": "/bootstrap/bootstrap.sh",
"metadata": {
"description": "The path on the mounted NFS share to the bootstrap script used for further installation"
"description": "The path on the mounted NFS share to the bootstrap script used for further installation. (ex. '/bootstrap/bootstrap.sh')"
}
}
},
"variables": {
"uniquename": "[concat(parameters('uniquename'))]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"sshKeyData": "[parameters('sshKeyData')]",
"vmCount": "[parameters('vmCount')]",
"vmSize": "[parameters('vmSize')]",
@ -107,16 +99,6 @@
"vmName": "[concat('vm-',variables('uniquename'))]",
"bootstrapNFSIP": "[split(variables('nfsCommaSeparatedAddresses'),',')[0]]",
"sshKeyPath": "[concat('/home/',variables('adminUsername'),'/.ssh/authorized_keys')]",
"linuxConfiguration": {
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
},
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
@ -185,8 +167,17 @@
"osProfile": {
"computername": "[concat(variables('vmName'), '-', copyIndex())]",
"adminUsername": "[variables('adminUsername')]",
"adminPassword": "[variables('adminPassword')]",
"linuxConfiguration": "[if(greater(length(variables('sshKeyData')), 0), variables('linuxConfiguration'), json('null'))]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
}
},
"storageProfile": {
"osDisk": {

39
src/clients/vmas/azuredeploy.json
Просмотреть файл

@ -15,18 +15,12 @@
"description": "The admin username for the virtual machine clients."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password for the virtual machine clients."
}
},
"sshKeyData": {
"type": "string",
"defaultValue": "",
"minLength": 0,
"metadata": {
"description": "The optional public SSH key used for access to the VM."
"description": "The public SSH key used for access to the VM."
}
},
"vmCount": {
@ -63,13 +57,13 @@
"nfsCommaSeparatedAddresses": {
"type": "string",
"metadata": {
"description": "A comma separated list of Avere vFXT vServer IP Addresses."
"description": "A comma separated list of Avere vFXT vServer IP Addresses. (ex. '10.0.0.12,10.0.0.13,10.0.0.14')"
}
},
"nfsExportPath": {
"type": "string",
"metadata": {
"description": "The path exported from the NFS server that will be mounted."
"description": "The path exported from the NFS server that will be mounted. (ex. '/msazure')"
}
},
"nfsMountBaseDir": {
@ -81,16 +75,14 @@
},
"bootstrapScriptPath": {
"type": "string",
"defaultValue": "/bootstrap/bootstrap.sh",
"metadata": {
"description": "The path on the mounted NFS share to the bootstrap script used for further installation"
"description": "The path on the mounted NFS share to the bootstrap script used for further installation. (ex. '/bootstrap/bootstrap.sh')"
}
}
},
"variables": {
"uniquename": "[concat(parameters('uniquename'))]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"sshKeyData": "[parameters('sshKeyData')]",
"vmCount": "[parameters('vmCount')]",
"vmSize": "[parameters('vmSize')]",
@ -108,16 +100,6 @@
"asName": "[concat('as-',variables('uniquename'))]",
"bootstrapNFSIP": "[split(variables('nfsCommaSeparatedAddresses'),',')[0]]",
"sshKeyPath": "[concat('/home/',variables('adminUsername'),'/.ssh/authorized_keys')]",
"linuxConfiguration": {
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
},
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
@ -203,8 +185,17 @@
"osProfile": {
"computername": "[concat(variables('vmName'), '-', copyIndex())]",
"adminUsername": "[variables('adminUsername')]",
"adminPassword": "[variables('adminPassword')]",
"linuxConfiguration": "[if(greater(length(variables('sshKeyData')), 0), variables('linuxConfiguration'), json('null'))]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
}
},
"storageProfile": {
"osDisk": {

39
src/clients/vmss/azuredeploy.json
Просмотреть файл

@ -15,18 +15,12 @@
"description": "The admin username for the virtual machine clients."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password for the virtual machine clients."
}
},
"sshKeyData": {
"type": "string",
"defaultValue": "",
"minLength": 0,
"metadata": {
"description": "The optional public SSH key used for access to the VM."
"description": "The public SSH key used for access to the VM."
}
},
"vmCount": {
@ -63,13 +57,13 @@
"nfsCommaSeparatedAddresses": {
"type": "string",
"metadata": {
"description": "A comma separated list of Avere vFXT vServer IP Addresses."
"description": "A comma separated list of Avere vFXT vServer IP Addresses. (ex. '10.0.0.12,10.0.0.13,10.0.0.14')"
}
},
"nfsExportPath": {
"type": "string",
"metadata": {
"description": "The path exported from the NFS server that will be mounted."
"description": "The path exported from the NFS server that will be mounted. (ex. '/msazure')"
}
},
"nfsMountBaseDir": {
@ -81,16 +75,14 @@
},
"bootstrapScriptPath": {
"type": "string",
"defaultValue": "/bootstrap/bootstrap.sh",
"metadata": {
"description": "The path on the mounted NFS share to the bootstrap script used for further installation"
"description": "The path on the mounted NFS share to the bootstrap script used for further installation. (ex. '/bootstrap/bootstrap.sh')"
}
}
},
"variables": {
"uniquename": "[concat(parameters('uniquename'))]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"sshKeyData": "[parameters('sshKeyData')]",
"vmCount": "[parameters('vmCount')]",
"vmSize": "[parameters('vmSize')]",
@ -107,16 +99,6 @@
"vmName": "[concat('vm-',variables('uniquename'))]",
"bootstrapNFSIP": "[split(variables('nfsCommaSeparatedAddresses'),',')[0]]",
"sshKeyPath": "[concat('/home/',variables('adminUsername'),'/.ssh/authorized_keys')]",
"linuxConfiguration": {
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
},
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
@ -171,8 +153,17 @@
"osProfile": {
"computerNamePrefix": "[variables('vmName')]",
"adminUsername": "[variables('adminUsername')]",
"adminPassword": "[variables('adminPassword')]",
"linuxConfiguration": "[if(greater(length(variables('sshKeyData')), 0), variables('linuxConfiguration'), json('null'))]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('sshKeyData')]"
}
]
}
}
},
"networkProfile": {
"networkInterfaceConfigurations": [

24
src/dataingestor/parts/installdataingestor.sh → src/dataingestor/bootstrap.dataingestor.sh
Просмотреть файл

@ -1,9 +1,7 @@
#!/bin/bash -ex
# variables that must be set beforehand
# AVERE_NAMESPACE_PATH=/msazure
# AVEREVFXT_NODE_IPS="172.16.1.8,172.16.1.9,172.16.1.10"
#
set -x
function retrycmd_if_failure() {
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
for i in $(seq 1 $retries); do
@ -60,16 +58,16 @@ function config_linux() {
}
function mount_avere() {
COUNTER=1
for VFXT in $(echo $AVEREVFXT_NODE_IPS | sed "s/,/ /g")
COUNTER=0
for VFXT in $(echo $NFS_IP_CSV | sed "s/,/ /g")
do
MOUNT_POINT="/nfs/node${COUNTER}"
echo "Mounting to $VFXT:$AVERE_NAMESPACE_PATH to ${MOUNT_POINT}"
sudo mkdir -p $MOUNT_POINT
MOUNT_POINT="${BASE_DIR}${NODE_MOUNT_PREFIX}${COUNTER}"
echo "Mounting to ${VFXT}:${NFS_PATH} to ${MOUNT_POINT}"
mkdir -p $MOUNT_POINT
# no need to write again if it is already there
if grep -v --quiet $VFXT /etc/fstab; then
echo "$VFXT:$AVERE_NAMESPACE_PATH ${MOUNT_POINT} nfs hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0" >> /etc/fstab
sudo mount ${MOUNT_POINT}
echo "${VFXT}:${NFS_PATH} ${MOUNT_POINT} nfs hard,nointr,proto=tcp,mountproto=tcp,retry=30 0 0" >> /etc/fstab
mount ${MOUNT_POINT}
fi
COUNTER=$(($COUNTER + 1))
done
@ -133,11 +131,11 @@ function write_msrsync() {
FILENAME=/usr/bin/msrsync
sudo touch $FILENAME
sudo chmod 755 $FILENAME
sudo wget -O $FILENAME https://raw.githubusercontent.com/jbd/msrsync/master/msrsync
sudo curl --retry 5 --retry-delay 5 -o $FILENAME https://raw.githubusercontent.com/jbd/msrsync/master/msrsync
sudo chmod +x $FILENAME
PRIMEFILE=/usr/bin/prime.py
sudo wget -O $PRIMEFILE https://raw.githubusercontent.com/Azure/Avere/master/src/dataingestor/prime.py
sudo curl --retry 5 --retry-delay 5 -o $PRIMEFILE https://raw.githubusercontent.com/Azure/Avere/master/src/dataingestor/prime.py
sudo chmod +x $PRIMEFILE
}

170
src/dataingestor/dataingestor-azuredeploy.json
Просмотреть файл

@ -1,170 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"uniqueName": {
"type": "string",
"metadata": {
"description": "The unique name used for the data ingestor VM for resource names associated with the VM."
}
},
"vmSize": {
"type": "string",
"defaultValue": "Standard_D4_v3",
"metadata": {
"description": "Size of the VM."
}
},
"adminUsername": {
"type": "string",
"defaultValue": "azureuser",
"metadata": {
"description": "The admin username on the data ingestor VM."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password on the data ingestor VM."
}
},
"subnetId": {
"type": "string",
"metadata": {
"description": "The fully qualified reference to the subnet of the Avere vFXT cluster. Example /subscriptions/SUBSCRIPTION/resourceGroups/RESOURCEGROUP/providers/Microsoft.Network/virtualNetworks/NETWORK_NAME/subnets/SUBNET_NAME."
}
},
"avereVServerCommaSeparatedAddresses": {
"type": "string",
"metadata": {
"description": "A comma separated list of Avere vFXT vServer IP Addresses."
}
},
"avereNamespacePath": {
"type": "string",
"metadata": {
"description": "The Avere vFXT namespace path."
}
}
},
"variables": {
"uniqueName": "[parameters('uniqueName')]",
"vmSize": "[parameters('vmSize')]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"subnetId": "[parameters('subnetId')]",
"avereVServerCommaSeparatedAddresses": "[parameters('avereVServerCommaSeparatedAddresses')]",
"avereNamespacePath": "[parameters('avereNamespacePath')]",
"nicName": "[concat('vmnic-',variables('uniqueName'))]",
"publicIPAddressName": "[concat('publicip-',variables('uniqueName'))]",
"vmName": "[concat('vm-',variables('uniqueName'))]",
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
"sku": "18.04-LTS",
"version": "latest"
},
"environmentVariables": "[concat(' AVERE_NAMESPACE_PATH', variables('avereNamespacePath'), ' AVEREVFXT_NODE_IPS=', variables('avereVServerCommaSeparatedAddresses'))]"
},
"resources": [
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/publicIPAddresses",
"name": "[variables('publicIPAddressName')]",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "Static"
}
},
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/networkInterfaces",
"name": "[variables('nicName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]"
],
"properties": {
"ipConfigurations": [
{
"name": "ipconfig",
"properties": {
"privateIPAllocationMethod": "Dynamic",
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]"
},
"subnet": {
"id": "[variables('subnetId')]"
}
}
}
]
}
},
{
"apiVersion": "2017-12-01",
"tags": {
"averevfxt": "client"
},
"type": "Microsoft.Compute/virtualMachines",
"name": "[variables('vmName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/networkInterfaces/', variables('nicName'))]"
],
"properties": {
"hardwareProfile": {
"vmSize": "[variables('vmSize')]"
},
"osProfile": {
"computername": "[variables('vmName')]",
"adminUsername": "[variables('adminUsername')]",
"adminPassword": "[variables('adminPassword')]",
"customData": "[base64('#cloud-config\n\nwrite_files:\n - encoding: gzip\n content: !!binary |\n H4sIAAAAAAAC/8VXfVObSBj/Wz7FI2E6yVVCiNNrT4vXtKZTb6px1PZuxngMgSVsGxa6u6ip9rvfs0CAxNSzLzfHOOPC8/bb5z2tTWtCmTXxRAQmuda0Flx6nHqTGREgI09CnAkJEwKCqH9hwknksQD5Bu+HJ0P3aHA4PD0evBq6x4OzN44VC+9zxsmC/v71X2fu0Wh/6B4cnzq6/bTftX/t2t1nW9Xxt/po93StpYUZ8yVNGHAi+dyPA5eGbujRGeptd+BGA3wUjRLhGPYuXHlUumJGSOoY/V2QNCZJJh1jexdEREMJjx6tHHIVeBegQBkYbUE+gQ1GqbSzC0GSs6inVAdGdbh58aWinoPxO3ruE/QALpT+CSfeR7i9hXHFQ0PFRnO2hQ24QKARYRWTeogfJTC8Jn4mSQBj3Xgx1pWgsix2l1hRTcYZ2NVHMhNkiSN3CBi1cypqSPNjkLBC4l/MftHqkHipdKdEulkaePJuNOxe/qq4Cg4XHZZiLCwZpxZ+NlHYLEhdJH1PHOx+D0pNUGiC/t4jG25BEgLGHdtImHL0hDkE/e/2+Z/Di53uL53b9jkZXnCOR0NfiSYjVTR9rIA1KtfHeT3zt6fB2tiW8dzuPTCMKy5ahHNtNCkT0pvN/ofialVhTXzfEwgKccwh5IR8RrAVW343/U4pLi5ZXgBMkyVm+WJy4idxTFggwJznVVsHuvWjqipN3ytX3AjJ6YxgvPT/pqP8SJtYVHJd79+Yeg9wyb3Nxk9YSKfujLLsOs/NDXKdJlzC/vDlweDIfX0yOjobHu07LGGUScI9lLskANrGCu6NlUwH7CE2/j3rAQuFqRChvdTjSCOzJRBxkjHpepekMXxejd6h4RPHrlJdzbki23NXGHeHH7Yhgb7RhbVlgTXVO6UHK58eKqXu8ejg6MzRLYRlsSQgxk1prJG8ReocKmCUTUGiPWVqx1g3knPyTUN5Q5HIggTijwHlYGIONJgaBcoSYASRo54rTrGVeFMPb4opSDHEArwZ5mcwV7nHSTNDi657ieH/lFFMhxwkWET6VohRmHxtAOr33GZj+Sob6CaIPB5ssQQzgG+lPJGJI/10K49b/ZrvEs52D6uqp8PeXgPHcjnkLlGyK15bzf1mHhjttlGe4THYnU5dHc1Uyt3nLrLMT6t8en3wdqju6ViZ4PlCVjNpFSqZZH4ExoK5JvhRnATw9MmTdcRcnZpMz58PR4ewV7O06t1P0wIq0pk3dzPhTfNEh7qwTQL6mL1TlB0YGz04Hb07wYDsH5xgKZ6eqcOY6aB9AU3LWxNytcCcEez7F1DEudC3ZKawcI1pZGshRYZc+BzaubjjgG6aEZmlekd1v/pjpMPFg9T2crVaDdfRx4atawvQ6rWvl5g3wcTyHBs1t16Br32hnyYZ9wlaxEYmE44trCGBMcdGzBKp7Au5BWoKFh+8WkS/D3MfMa8gWsDVi3mwWZdsTbsLdJ8IbA9ennlNtJXEz8Z6tYr12yGV9vNCUb9BvgfU9gLUJlxFFCtmUUuARR+QS4tlOAAa2PIyWVmX1C5Tfq7H2LyeEMoEAv5j9PLU6fefaNUhpCxYTgkzxk8klREWgynnKeYOho5ju+qVM8H8rMZCU2hsWZaO1Aq6aV5RHJ6+6r/mB/RYYRvMHuhrssG6wS5/P5TNnwemQuJjvz9eMolAVmFhF1rTFWPBxZz5VUtsVW6XESZAyJMYIilTsWNZUyqjbNLFqW19mARWKfqVTtqk/kAbvVLJYY5qUoWGe1fdAlEmCMedRRImV8FZsSdwQbmLpjD6+LphMycenxwcDtWnxkzguCJ10/kaUBX3A1AN1O9za6AWmgUqwX0L0x7H+hQrNOFrTNU4K1vLaxIKV8Fb7LZqfYO3an0rKra50DUYi3Gbr1gFX2PnarAV20c9GQve1aHaEFhkUOn0Jn8zDkvMXrl5Fmu5ru6orvYPIEiT3ygRAAA=\n path: /opt/avere/parts/installdataingestor.sh\n permissions: \"0744\"\n\n')]"
},
"storageProfile": {
"imageReference": "[variables('imageReference')]"
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces',variables('nicName'))]"
}
]
}
}
},
{
"type": "Microsoft.Compute/virtualMachines/extensions",
"name": "[concat(variables('vmName'), '/configureagent')]",
"apiVersion": "2017-12-01",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Compute/virtualMachines/', variables('vmName'))]"
],
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.0",
"autoUpgradeMinorVersion": true,
"settings": {
"commandToExecute": "[concat(variables('environmentVariables'), ' /bin/bash -c \"/bin/bash /opt/avere/installdataingestor.sh >> /opt/avere/dataingestor-install.log 2>&1\"')]"
}
}
}
],
"outputs": {
"RESOURCE_GROUP": {
"type": "string",
"value": "[resourceGroup().name]"
},
"LOCATION": {
"type": "string",
"value": "[resourceGroup().location]"
},
"SSH_STRING": {
"type": "string",
"value": "[concat(variables('adminUsername'), '@', reference(resourceId('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))).ipAddress)]"
}
}
}

27
src/dataingestor/dataingestor-azuredeploy.parameters.json
Просмотреть файл

@ -1,27 +0,0 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"uniqueName": {
"value": "anhowe0821b"
},
"vmSize": {
"value": "Standard_D4_v3"
},
"adminUsername": {
"value": "azureuser"
},
"adminPassword": {
"value": "AvereNumber1$"
},
"subnetId": {
"value": "/subscriptions/b52fce95-de5f-4b37-afca-db203a5d0b6a/resourceGroups/avereeastusd16/providers/Microsoft.Network/virtualNetworks/vnet-avereeastusd16/subnets/subnet-avereeastusd16"
},
"avereVServerCommaSeparatedAddresses": {
"value": "10.0.0.12,10.0.0.13,10.0.0.14"
},
"avereNamespacePath": {
"value": "msazure"
}
}
}

24
src/dataingestor/dataingestorVerify.sh Normal file
Просмотреть файл

@ -0,0 +1,24 @@
#!/bin/bash
TARGETDIR=/nfs/node0/bootstrap
VDBENCHINSTALL=$TARGETDIR/bootstrap.vdbench.sh
VDBENCHSRC="$TARGETDIR/vdbench*.zip"
if [ ! -d "$TARGETDIR" ]; then
echo "ERROR: directory $TARGETDIR does not exist"
exit 1
else
echo "SUCCESS: $TARGETDIR found"
fi
if ! ls $VDBENCHINSTALL > /dev/null 2>&1; then
echo "MISSING: $VDBENCHINSTALL. Please download the install script per instructions."
else
echo "SUCCESS: $VDBENCHINSTALL found"
fi
if ! ls $VDBENCHSRC > /dev/null 2>&1; then
echo "MISSING: $VDBENCHSRC. Please download the vdbench zip file from Oracle."
else
echo "SUCCESS: $VDBENCHSRC found"
fi

124
src/dataingestor/gen-arm-templates.py
Просмотреть файл

@ -1,124 +0,0 @@
#!/usr/bin/python
import base64
import os
import gzip
import StringIO
import sys
import shutil
import json
import argparse
def buildb64GzipStringFromFile(file):
# read the script file
with open(file) as f:
content = f.read()
compressedbuffer=StringIO.StringIO()
# gzip the script file
# mtime=0 sets a fixed timestamp in GZip header to the Epoch which is January 1st, 1970
# Make sure it doens't change unless the stream changes
with gzip.GzipFile(fileobj=compressedbuffer, mode='wb', mtime=0) as f:
f.write(content)
b64GzipStream=base64.b64encode(compressedbuffer.getvalue())
return b64GzipStream
# Function reads the files from disk,
# and embeds it in a Yaml file as a base-64 enconded string to be
# executed later by template
def buildYamlFileWithWriteFiles(files):
gzipBuffer=StringIO.StringIO()
clusterYamlFile="""#cloud-config
write_files:
%s
"""
writeFileBlock=""" - encoding: gzip
content: !!binary |
%s
path: /opt/avere/%s
permissions: "0744"
"""
filelines=""
for encodeFile in files:
b64GzipString = buildb64GzipStringFromFile(encodeFile)
filelines=filelines+(writeFileBlock % (b64GzipString,encodeFile))
return clusterYamlFile % (filelines)
# processes a Yaml file to be included properly in ARM template
def convertToOneArmTemplateLine(clusterYamlFile):
# remove the \r\n and include \n in body and escape " to \"
return clusterYamlFile.replace("\n", "\\n").replace('"', '\\"')
# Loads the base ARM template file and injects the Yaml for the shell scripts into it.
def processBaseTemplate(baseTemplatePath,
clusterInstallScript,
jumpboxTemplatePath = None,
linuxJumpboxInstallScript = None,
swarmWindowsAgentInstallScript = None,
additionalFiles = [],
windowsAgentDiagnosticsExtensionTemplatePath = None):
#String to replace in JSON file
CLUSTER_YAML_REPLACE_STRING = "#clusterCustomDataInstallYaml"
# Load Base Template
armTemplate = []
with open(baseTemplatePath) as f:
armTemplate = f.read()
# Generate cluster Yaml file for ARM
clusterYamlFile = convertToOneArmTemplateLine(buildYamlFileWithWriteFiles([clusterInstallScript]+additionalFiles))
armTemplate = armTemplate.replace(CLUSTER_YAML_REPLACE_STRING, clusterYamlFile)
# Make sure the final string is valid JSON
try:
json_object = json.loads(armTemplate)
except ValueError, e:
print e
errorFileName = baseTemplatePath + ".err"
with open(errorFileName, "w") as f:
f.write(armTemplate)
print "Invalid armTemplate saved to: " + errorFileName
raise
return armTemplate;
if __name__ == "__main__":
# Parse Arguments
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output_directory", help="Directory to write templates files to. Default is current directory.")
args = parser.parse_args()
if (args.output_directory == None) :
args.output_directory = os.getcwd()
args.output_directory = os.path.expandvars(os.path.normpath(args.output_directory))
if ( os.path.exists(args.output_directory) == False ):
os.mkdir(args.output_directory)
# Input Arm Template Artifacts to be processed in
# Note: These files are not useable ARM templates on thier own or valid JSON
# They require processing by this script.
ARM_INPUT_TEMPLATE_TEMPLATE = "parts/base-template.json"
ARM_INPUT_PARAMETER_TEMPLATE = "parts/base-template.parameters.json"
# Shell Scripts to load into YAML
VDBENCH_INSTALL_SCRIPT = "parts/installdataingestor.sh"
# Output ARM Template Files. WIll Also Output name.parameters.json for each
ARM_OUTPUT_TEMPLATE = "dataingestor-azuredeploy.json"
# build the ARM template for jumpboxless
with open(os.path.join(args.output_directory, ARM_OUTPUT_TEMPLATE), "w") as armTemplate:
clusterTemplate = processBaseTemplate(
baseTemplatePath=ARM_INPUT_TEMPLATE_TEMPLATE,
clusterInstallScript=VDBENCH_INSTALL_SCRIPT)
armTemplate.write(clusterTemplate)
# Write parameter files
shutil.copyfile(ARM_INPUT_PARAMETER_TEMPLATE, os.path.join(args.output_directory, ARM_OUTPUT_TEMPLATE).replace(".json", ".parameters.json") )

167
src/dataingestor/parts/base-template.json
Просмотреть файл

@ -1,167 +0,0 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"uniqueName": {
"type": "string",
"metadata": {
"description": "The unique name used for the data ingestor VM for resource names associated with the VM."
}
},
"vmSize": {
"type": "string",
"defaultValue": "Standard_D4_v3",
"metadata": {
"description": "Size of the VM."
}
},
"adminUsername": {
"type": "string",
"defaultValue": "azureuser",
"metadata": {
"description": "The admin username on the data ingestor VM."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password on the data ingestor VM."
}
},
"subnetId": {
"type": "string",
"metadata": {
"description": "The fully qualified reference to the subnet of the Avere vFXT cluster. Example /subscriptions/SUBSCRIPTION/resourceGroups/RESOURCEGROUP/providers/Microsoft.Network/virtualNetworks/NETWORK_NAME/subnets/SUBNET_NAME."
}
},
"avereVServerCommaSeparatedAddresses": {
"type": "string",
"metadata": {
"description": "A comma separated list of Avere vFXT vServer IP Addresses."
}
},
"avereNamespacePath": {
"type": "string",
"metadata": {
"description": "The Avere vFXT namespace path."
}
}
},
"variables": {
"uniqueName": "[parameters('uniqueName')]",
"vmSize": "[parameters('vmSize')]",
"adminUsername": "[parameters('adminUsername')]",
"adminPassword": "[parameters('adminPassword')]",
"subnetId": "[parameters('subnetId')]",
"avereVServerCommaSeparatedAddresses": "[parameters('avereVServerCommaSeparatedAddresses')]",
"avereNamespacePath": "[parameters('avereNamespacePath')]",
"nicName": "[concat('vmnic-',variables('uniqueName'))]",
"publicIPAddressName": "[concat('publicip-',variables('uniqueName'))]",
"vmName": "[concat('vm-',variables('uniqueName'))]",
"imageReference": {
"publisher": "Canonical",
"offer": "UbuntuServer",
"sku": "18.04-LTS",
"version": "latest"
},
"environmentVariables": "[concat(' AVERE_NAMESPACE_PATH', variables('avereNamespacePath'), ' AVEREVFXT_NODE_IPS=', variables('avereVServerCommaSeparatedAddresses'))]"
},
"resources": [
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/publicIPAddresses",
"name": "[variables('publicIPAddressName')]",
"location": "[resourceGroup().location]",
"properties": {
"publicIPAllocationMethod": "Static"
}
},
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/networkInterfaces",
"name": "[variables('nicName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]"
],
"properties": {
"ipConfigurations": [
{
"name": "ipconfig",
"properties": {
"privateIPAllocationMethod": "Dynamic",
"publicIPAddress": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]"
},
"subnet": {
"id": "[variables('subnetId')]"
}
}
}
]
}
},
{
"apiVersion": "2017-12-01",
"type": "Microsoft.Compute/virtualMachines",
"name": "[variables('vmName')]",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/networkInterfaces/', variables('nicName'))]"
],
"properties": {
"hardwareProfile": {
"vmSize": "[variables('vmSize')]"
},
"osProfile": {
"computername": "[variables('vmName')]",
"adminUsername": "[variables('adminUsername')]",
"adminPassword": "[variables('adminPassword')]",
"customData": "[base64('#clusterCustomDataInstallYaml')]"
},
"storageProfile": {
"imageReference": "[variables('imageReference')]"
},
"networkProfile": {
"networkInterfaces": [
{
"id": "[resourceId('Microsoft.Network/networkInterfaces',variables('nicName'))]"
}
]
}
}
},
{
"type": "Microsoft.Compute/virtualMachines/extensions",
"name": "[concat(variables('vmName'), '/configureagent')]",
"apiVersion": "2017-12-01",
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Compute/virtualMachines/', variables('vmName'))]"
],
"properties": {
"publisher": "Microsoft.Azure.Extensions",
"type": "CustomScript",
"typeHandlerVersion": "2.0",
"autoUpgradeMinorVersion": true,
"settings": {
"commandToExecute": "[concat(variables('environmentVariables'), ' /bin/bash -c \"/bin/bash /opt/avere/installdataingestor.sh >> /opt/avere/dataingestor-install.log 2>&1\"')]"
}
}
}
],
"outputs": {
"RESOURCE_GROUP": {
"type": "string",
"value": "[resourceGroup().name]"
},
"LOCATION": {
"type": "string",
"value": "[resourceGroup().location]"
},
"SSH_STRING": {
"type": "string",
"value": "[concat(variables('adminUsername'), '@', reference(resourceId('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))).ipAddress)]"
}
}
}

27
src/dataingestor/parts/base-template.parameters.json
Просмотреть файл

@ -1,27 +0,0 @@
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"uniqueName": {
"value": "anhowe0821b"
},
"vmSize": {
"value": "Standard_D4_v3"
},
"adminUsername": {
"value": "azureuser"
},
"adminPassword": {
"value": "AvereNumber1$"
},
"subnetId": {
"value": "/subscriptions/b52fce95-de5f-4b37-afca-db203a5d0b6a/resourceGroups/avereeastusd16/providers/Microsoft.Network/virtualNetworks/vnet-avereeastusd16/subnets/subnet-avereeastusd16"
},
"avereVServerCommaSeparatedAddresses": {
"value": "10.0.0.12,10.0.0.13,10.0.0.14"
},
"avereNamespacePath": {
"value": "msazure"
}
}
}

52
src/vfxt-auto/azuredeploy.json
Просмотреть файл

@ -83,10 +83,16 @@
"description": "The controller admin username."
}
},
"controllerSSHKeyData": {
"type": "securestring",
"metadata": {
"description": "The controller public SSH key."
}
},
"adminPassword": {
"type": "securestring",
"metadata": {
"description": "The admin password used for the controller and the Avere vFXT cluster."
"description": "The admin password used for the Avere vFXT cluster."
}
},
"avereClusterRole": {
@ -129,6 +135,7 @@
"servicePrincipalPassword": "[parameters('servicePrincipalPassword')]",
"controllerName": "[parameters('controllerName')]",
"controllerAdminUsername": "[parameters('controllerAdminUsername')]",
"controllerSSHKeyData": "[parameters('controllerSSHKeyData')]",
"adminPassword": "[parameters('adminPassword')]",
"nicName": "[concat('vmnic-',variables('controllerName'))]",
"createVirtualNetwork?": "[parameters('createVirtualNetwork?')]",
@ -146,6 +153,7 @@
"addressPrefix": "10.0.0.0/16",
"subnetPrefix": "10.0.0.0/20",
"publicIPAddressName": "[concat('publicip-',variables('controllerName'))]",
"networkSecurityGroupName": "[concat('nsg-',variables('controllerName'))]",
"vnetID": "[resourceId(variables('virtualNetworkResourceGroup'),'Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
"subnetId": "[concat(variables('vnetID'),'/subnets/',variables('virtualNetworkSubnetName'))]",
"vmName": "[concat('vm-',variables('controllerName'))]",
@ -155,7 +163,8 @@
"createNic?": true,
"storageAccountType": "Standard_LRS",
"storageAccountKind": "StorageV2",
"environmentVariables": "[concat(' AZURE_TENANT_ID=', variables('servicePrincipalTenant'),' AZURE_CLIENT_ID=', variables('servicePrincipalAppId'),' RESOURCE_GROUP=', resourceGroup().name,' LOCATION=', resourceGroup().location,' NETWORK_RESOURCE_GROUP=', variables('virtualNetworkResourceGroup'),' NETWORK=', variables('virtualNetworkName'),' SUBNET=', variables('virtualNetworkSubnetName'),' AVERE_CLUSTER_ROLE=', variables('avereClusterRole'),' CREATE_CLOUD_BACKED_CLUSTER=', variables('useAvereBackedStorageAccount?'),' STORAGE_ACCOUNT=', variables('avereBackedStorageAccountName'),' CACHE_SIZE=', variables('avereCacheSizeGB'),' CLUSTER_NAME=', variables('avereClusterName'),' INSTANCE_TYPE=', variables('avereInstanceType'),' CONTROLLER_ADMIN_USER_NAME=',variables('controllerAdminUsername'),' DEBUG=\"',variables('additionalVFXTParameters'),'\"')]"
"environmentVariables": "[concat(' AZURE_TENANT_ID=', variables('servicePrincipalTenant'),' AZURE_CLIENT_ID=', variables('servicePrincipalAppId'),' RESOURCE_GROUP=', resourceGroup().name,' LOCATION=', resourceGroup().location,' NETWORK_RESOURCE_GROUP=', variables('virtualNetworkResourceGroup'),' NETWORK=', variables('virtualNetworkName'),' SUBNET=', variables('virtualNetworkSubnetName'),' AVERE_CLUSTER_ROLE=', variables('avereClusterRole'),' CREATE_CLOUD_BACKED_CLUSTER=', variables('useAvereBackedStorageAccount?'),' STORAGE_ACCOUNT=', variables('avereBackedStorageAccountName'),' CACHE_SIZE=', variables('avereCacheSizeGB'),' CLUSTER_NAME=', variables('avereClusterName'),' INSTANCE_TYPE=', variables('avereInstanceType'),' CONTROLLER_ADMIN_USER_NAME=',variables('controllerAdminUsername'),' DEBUG=\"',variables('additionalVFXTParameters'),'\"')]",
"sshKeyPath": "[concat('/home/',variables('controllerAdminUsername'),'/.ssh/authorized_keys')]"
},
"resources": [
{
@ -181,6 +190,30 @@
"publicIPAllocationMethod": "Static"
}
},
{
"condition": "[variables('createVirtualNetwork?')]",
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/networkSecurityGroups",
"name": "[variables('networkSecurityGroupName')]",
"location": "[parameters('location')]",
"properties": {
"securityRules": [
{
"name": "default-allow-ssh",
"properties": {
"priority": 1000,
"sourceAddressPrefix": "*",
"protocol": "Tcp",
"destinationPortRange": "22",
"access": "Allow",
"direction": "Inbound",
"sourcePortRange": "*",
"destinationAddressPrefix": "*"
}
}
]
}
},
{
"condition": "[variables('createVirtualNetwork?')]",
"apiVersion": "2017-10-01",
@ -211,7 +244,8 @@
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]",
"[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]"
"[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]",
"[concat('Microsoft.Network/networkSecurityGroups/', variables('networkSecurityGroupName'))]"
],
"properties": {
"ipConfigurations": [
@ -251,7 +285,17 @@
"osProfile": {
"computername": "[variables('vmName')]",
"adminUsername": "[variables('controllerAdminUsername')]",
"adminPassword": "[variables('adminPassword')]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('controllerSSHKeyData')]"
}
]
}
}
},
"storageProfile": {
"imageReference": {

47
src/vfxt-vnet/azuredeploy.json
Просмотреть файл

@ -44,10 +44,10 @@
"description": "The controller admin username."
}
},
"controllerAdminPassword": {
"type": "securestring",
"controllerSSHKeyData": {
"type": "string",
"metadata": {
"description": "The controller admin password."
"description": "The controller public SSH key."
}
}
},
@ -58,15 +58,17 @@
"virtualNetworkSubnetName": "[parameters('virtualNetworkSubnetName')]",
"controllerName": "[parameters('controllerName')]",
"controllerAdminUsername": "[parameters('controllerAdminUsername')]",
"controllerAdminPassword": "[parameters('controllerAdminPassword')]",
"controllerSSHKeyData": "[parameters('controllerSSHKeyData')]",
"controllerVMSize": "Standard_A1_v2",
"nicName": "[concat('vmnic-',variables('controllerName'))]",
"addressPrefix": "10.0.0.0/16",
"subnetPrefix": "10.0.0.0/20",
"publicIPAddressName": "[concat('publicip-',variables('controllerName'))]",
"networkSecurityGroupName": "[concat('nsg-',variables('controllerName'))]",
"vnetID": "[resourceId(variables('virtualNetworkResourceGroup'),'Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
"subnetId": "[concat(variables('vnetID'),'/subnets/',variables('virtualNetworkSubnetName'))]",
"subnetId": "[concat(variables('vnetID'),'/subnets/',variables('virtualNetworkSubnetName'))]",
"vmName": "[concat('vm-',variables('controllerName'))]",
"sshKeyPath": "[concat('/home/',variables('controllerAdminUsername'),'/.ssh/authorized_keys')]",
"pipObject": {
"id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]"
},
@ -96,6 +98,29 @@
"publicIPAllocationMethod": "Static"
}
},
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/networkSecurityGroups",
"name": "[variables('networkSecurityGroupName')]",
"location": "[parameters('location')]",
"properties": {
"securityRules": [
{
"name": "default-allow-ssh",
"properties": {
"priority": 1000,
"sourceAddressPrefix": "*",
"protocol": "Tcp",
"destinationPortRange": "22",
"access": "Allow",
"direction": "Inbound",
"sourcePortRange": "*",
"destinationAddressPrefix": "*"
}
}
]
}
},
{
"condition": "[variables('createVirtualNetwork?')]",
"apiVersion": "2017-10-01",
@ -166,7 +191,17 @@
"osProfile": {
"computername": "[variables('vmName')]",
"adminUsername": "[variables('controllerAdminUsername')]",
"adminPassword": "[variables('controllerAdminPassword')]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('controllerSSHKeyData')]"
}
]
}
}
},
"storageProfile": {
"imageReference": {

48
src/vfxt/azuredeploy.json
Просмотреть файл

@ -15,27 +15,29 @@
"description": "The controller admin username."
}
},
"controllerAdminPassword": {
"controllerSSHKeyData": {
"type": "securestring",
"metadata": {
"description": "The controller admin password."
"description": "The controller public SSH key."
}
}
},
"variables": {
"controllerName": "[parameters('controllerName')]",
"controllerAdminUsername": "[parameters('controllerAdminUsername')]",
"controllerAdminPassword": "[parameters('controllerAdminPassword')]",
"controllerSSHKeyData": "[parameters('controllerSSHKeyData')]",
"nicName": "[concat('vmnic-',variables('controllerName'))]",
"addressPrefix": "10.0.0.0/16",
"subnetName": "[concat('subnet-',variables('controllerName'))]",
"subnetPrefix": "10.0.0.0/24",
"publicIPAddressName": "[concat('publicip-',variables('controllerName'))]",
"networkSecurityGroupName": "[concat('nsg-',variables('controllerName'))]",
"vmName": "[concat('vm-',variables('controllerName'))]",
"virtualNetworkName": "[concat('vnet-',variables('controllerName'))]",
"vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
"subnetId": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]",
"vmSize": "Standard_A1_v2"
"vmSize": "Standard_A1_v2",
"sshKeyPath": "[concat('/home/',variables('controllerAdminUsername'),'/.ssh/authorized_keys')]"
},
"resources": [
{
@ -60,6 +62,29 @@
"publicIPAllocationMethod": "Static"
}
},
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/networkSecurityGroups",
"name": "[variables('networkSecurityGroupName')]",
"location": "[parameters('location')]",
"properties": {
"securityRules": [
{
"name": "default-allow-ssh",
"properties": {
"priority": 1000,
"sourceAddressPrefix": "*",
"protocol": "Tcp",
"destinationPortRange": "22",
"access": "Allow",
"direction": "Inbound",
"sourcePortRange": "*",
"destinationAddressPrefix": "*"
}
}
]
}
},
{
"apiVersion": "2017-10-01",
"type": "Microsoft.Network/virtualNetworks",
@ -88,7 +113,8 @@
"location": "[resourceGroup().location]",
"dependsOn": [
"[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]",
"[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]"
"[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]",
"[concat('Microsoft.Network/networkSecurityGroups/', variables('networkSecurityGroupName'))]"
],
"properties": {
"ipConfigurations": [
@ -130,7 +156,17 @@
"osProfile": {
"computername": "[variables('vmName')]",
"adminUsername": "[variables('controllerAdminUsername')]",
"adminPassword": "[variables('controllerAdminPassword')]"
"linuxConfiguration": {
"disablePasswordAuthentication": "true",
"ssh": {
"publicKeys": [
{
"path": "[variables('sshKeyPath')]",
"keyData": "[variables('controllerSSHKeyData')]"
}
]
}
}
},
"storageProfile": {
"imageReference": {

4
src/vfxt/azuredeploy.parameters.json
Просмотреть файл

@ -8,8 +8,8 @@
"controllerAdminUsername": {
"value": "azureuser"
},
"controllerAdminPassword": {
"value": "GEN-PASSWORD"
"controllerSSHKeyData": {
"value": "GEN-SSHKEY"
}
}
}