Updated Template
This commit is contained in:
CognosysTech
Родитель
87839bd774
Коммит
7501f1d812
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"storageAccountName": {
|
||||
"value": "armdeployspark1"
|
||||
"value": "spkldeploysparke23u1"
|
||||
},
|
||||
"adminUsername": {
|
||||
"value": "adminuser"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,27 @@
|
|||
"Description": "Administrator password used when provisioning virtual machines"
|
||||
}
|
||||
},
|
||||
"imagePublisher": {
|
||||
"type": "string",
|
||||
"defaultValue": "Canonical",
|
||||
"metadata": {
|
||||
"Description": "Image Publisher"
|
||||
}
|
||||
},
|
||||
"imageOffer": {
|
||||
"type": "string",
|
||||
"defaultValue": "UbuntuServer",
|
||||
"metadata": {
|
||||
"Description": "Image Offer"
|
||||
}
|
||||
},
|
||||
"imageSKU": {
|
||||
"type": "string",
|
||||
"defaultValue": "14.04.2-LTS",
|
||||
"metadata": {
|
||||
"Description": "Image SKU"
|
||||
}
|
||||
},
|
||||
"storageAccountName": {
|
||||
"type": "string",
|
||||
"defaultValue": "uniquesparkstoragev12",
|
||||
|
|
@ -216,7 +237,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/publicIPAddresses",
|
||||
"name": "publicIPmaster",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -225,7 +246,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/networkInterfaces",
|
||||
"name": "[concat('nic', copyindex())]",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -256,7 +277,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/networkInterfaces",
|
||||
"name": "[concat('nicsl', copyindex())]",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -283,7 +304,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Compute/virtualMachines",
|
||||
"name": "[concat('myvm', copyindex())]",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -300,33 +321,42 @@
|
|||
"id": "[resourceId('Microsoft.Compute/availabilitySets', 'sparkCluserAS')]"
|
||||
},
|
||||
"hardwareProfile": {
|
||||
"vmSize": "[variables('vmSize')]",
|
||||
"platformGuestAgent": "true"
|
||||
"vmSize": "[variables('vmSize')]"
|
||||
},
|
||||
"osProfile": {
|
||||
"computername": "[concat(variables('computerNamePrefix'), copyIndex())]",
|
||||
"adminUsername": "[parameters('adminUsername')]",
|
||||
"adminPassword": "[parameters('adminPassword')]",
|
||||
"linuxOperatingSystemProfile": {
|
||||
"disableSshPasswordAuthentication": "false"
|
||||
"linuxConfiguration": {
|
||||
"disablePasswordAuthentication": "false"
|
||||
}
|
||||
},
|
||||
"storageProfile": {
|
||||
"sourceImage": {
|
||||
"id": "[variables('sourceImageName')]"
|
||||
},
|
||||
"dataDisks": [
|
||||
"storageProfile": {
|
||||
"imageReference": {
|
||||
"publisher": "[parameters('imagePublisher')]",
|
||||
"offer": "[parameters('imageOffer')]",
|
||||
"sku" : "[parameters('imageSKU')]",
|
||||
"version":"latest"
|
||||
},
|
||||
"osDisk" : {
|
||||
"name": "osdisk",
|
||||
"vhd": {
|
||||
"uri": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/','osdisk', copyindex() ,'.vhd')]"
|
||||
},
|
||||
"caching": "ReadWrite",
|
||||
"createOption": "FromImage"
|
||||
},
|
||||
"dataDisks": [
|
||||
{
|
||||
"name": "datadisk1",
|
||||
"diskSizeGB": "[parameters('dataDiskSize')]",
|
||||
"lun": 0,
|
||||
"vhd": {
|
||||
"Uri": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/','myvm','dataDisk', copyindex() ,'.vhd')]"
|
||||
}
|
||||
},
|
||||
"createOption": "Empty"
|
||||
}
|
||||
],
|
||||
|
||||
"destinationVhdsContainer": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/')]"
|
||||
},
|
||||
"networkProfile": {
|
||||
"networkInterfaces": [
|
||||
|
|
@ -347,7 +377,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Compute/virtualMachines",
|
||||
"name": "[concat('myvmsl', copyindex())]",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -365,21 +395,28 @@
|
|||
"id": "[resourceId('Microsoft.Compute/availabilitySets', 'sparkCluserAS')]"
|
||||
},
|
||||
"hardwareProfile": {
|
||||
"vmSize": "[variables('vmSize')]",
|
||||
"platformGuestAgent": "true"
|
||||
"vmSize": "[variables('vmSize')]"
|
||||
},
|
||||
"osProfile": {
|
||||
"computername": "[concat(variables('computerNamePrefix'),'sl', copyIndex())]",
|
||||
"adminUsername": "[parameters('adminUsername')]",
|
||||
"adminPassword": "[parameters('adminPassword')]",
|
||||
"linuxOperatingSystemProfile": {
|
||||
"disableSshPasswordAuthentication": "false"
|
||||
"linuxConfiguration": {
|
||||
"disablePasswordAuthentication": "false"
|
||||
}
|
||||
},
|
||||
"storageProfile": {
|
||||
"sourceImage": {
|
||||
"id": "[variables('sourceImageName')]"
|
||||
},
|
||||
"osDisk" : {
|
||||
"name": "osdisksl",
|
||||
"vhd": {
|
||||
"uri": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/','osdisksl', copyindex() ,'.vhd')]"
|
||||
},
|
||||
"caching": "ReadWrite",
|
||||
"createOption": "FromImage"
|
||||
},
|
||||
"dataDisks": [
|
||||
{
|
||||
"name": "datadisksl1",
|
||||
|
|
@ -387,11 +424,10 @@
|
|||
"lun": 0,
|
||||
"vhd": {
|
||||
"Uri": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/','myvmsl','dataDisk', copyindex() ,'.vhd')]"
|
||||
}
|
||||
},
|
||||
"createOption": "Empty"
|
||||
}
|
||||
],
|
||||
|
||||
"destinationVhdsContainer": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/')]"
|
||||
]
|
||||
},
|
||||
"networkProfile": {
|
||||
"networkInterfaces": [
|
||||
|
|
@ -405,7 +441,7 @@
|
|||
{
|
||||
"type": "Microsoft.Compute/virtualMachines/extensions",
|
||||
"name": "[concat('myvm', copyindex(), '/installsparkmaster')]",
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"location": "[parameters('region')]",
|
||||
"copy": {
|
||||
"name": "virtualMachineExtensionsLoopMaster",
|
||||
|
|
@ -430,7 +466,7 @@
|
|||
{
|
||||
"type": "Microsoft.Compute/virtualMachines/extensions",
|
||||
"name": "[concat('myvmsl', copyindex(), '/installsparkslave')]",
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"location": "[parameters('region')]",
|
||||
"copy": {
|
||||
"name": "virtualMachineExtensionsLoopSlave",
|
||||
|
|
|
|||
|
|
@ -42,6 +42,27 @@
|
|||
"Description": "Administrator password used when provisioning virtual machines"
|
||||
}
|
||||
},
|
||||
"imagePublisher": {
|
||||
"type": "string",
|
||||
"defaultValue": "Canonical",
|
||||
"metadata": {
|
||||
"Description": "Image Publisher"
|
||||
}
|
||||
},
|
||||
"imageOffer": {
|
||||
"type": "string",
|
||||
"defaultValue": "UbuntuServer",
|
||||
"metadata": {
|
||||
"Description": "Image Offer"
|
||||
}
|
||||
},
|
||||
"imageSKU": {
|
||||
"type": "string",
|
||||
"defaultValue": "14.04.2-LTS",
|
||||
"metadata": {
|
||||
"Description": "Image SKU"
|
||||
}
|
||||
},
|
||||
"computerNamePrefix": {
|
||||
"type": "string",
|
||||
"metadata": {
|
||||
|
|
@ -59,7 +80,7 @@
|
|||
},
|
||||
"resources": [
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/publicIPAddresses",
|
||||
"name": "publicIP",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -68,7 +89,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/networkInterfaces",
|
||||
"name": "nicJumpbox",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -93,7 +114,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Compute/virtualMachines",
|
||||
"name": "jumpbox",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -102,22 +123,31 @@
|
|||
],
|
||||
"properties": {
|
||||
"hardwareProfile": {
|
||||
"vmSize": "[variables('vmSize')]",
|
||||
"platformGuestAgent": "true"
|
||||
"vmSize": "[variables('vmSize')]"
|
||||
},
|
||||
"osProfile": {
|
||||
"computername": "[concat(parameters('computerNamePrefix'), 'jumpbox')]",
|
||||
"adminUsername": "[parameters('adminUsername')]",
|
||||
"adminPassword": "[parameters('adminPassword')]",
|
||||
"linuxOperatingSystemProfile": {
|
||||
"disableSshPasswordAuthentication": "false"
|
||||
"linuxConfiguration": {
|
||||
"disablePasswordAuthentication": "false"
|
||||
}
|
||||
},
|
||||
"storageProfile": {
|
||||
"sourceImage": {
|
||||
"id": "[variables('sourceImageName')]"
|
||||
},
|
||||
"destinationVhdsContainer": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/')]"
|
||||
"imageReference": {
|
||||
"publisher": "[parameters('imagePublisher')]",
|
||||
"offer": "[parameters('imageOffer')]",
|
||||
"sku" : "[parameters('imageSKU')]",
|
||||
"version":"latest"
|
||||
},
|
||||
"osDisk" : {
|
||||
"name": "osdiskjumpbox",
|
||||
"vhd": {
|
||||
"uri": "[concat('https://',parameters('storageAccountName'),'.blob.core.windows.net/',variables('vmStorageAccountContainerName'),'/','osdiskjumpbox.vhd')]"
|
||||
},
|
||||
"caching": "ReadWrite",
|
||||
"createOption": "FromImage"
|
||||
}
|
||||
},
|
||||
"networkProfile": {
|
||||
"networkInterfaces": [
|
||||
|
|
|
|||
|
|
@ -53,14 +53,14 @@
|
|||
{
|
||||
"type": "Microsoft.Storage/storageAccounts",
|
||||
"name": "[parameters('storageAccountName')]",
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"location": "[parameters('region')]",
|
||||
"properties": {
|
||||
"accountType": "Standard_LRS"
|
||||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Compute/availabilitySets",
|
||||
"name": "sparkCluserAS",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
@ -68,7 +68,7 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"apiVersion": "2014-12-01-preview",
|
||||
"apiVersion": "2015-05-01-preview",
|
||||
"type": "Microsoft.Network/virtualNetworks",
|
||||
"name": "[parameters('virtualNetworkName')]",
|
||||
"location": "[parameters('region')]",
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
### Cognosys Technologies
|
||||
###
|
||||
### Warning! This script partitions and formats disk information be careful where you run it
|
||||
### This script is currently under development and has only been tested on Ubuntu images in Azure
|
||||
### This script is not currently idempotent and only works for provisioning at the moment
|
||||
|
||||
### Remaining work items
|
||||
### -Alternate discovery options (Azure Storage)
|
||||
### -Implement Idempotency and Configuration Change Support
|
||||
### -Implement OS Disk Striping Option (Currenlty using multiple spark data paths)
|
||||
### -Implement Non-Durable Option (Put data on resource disk)
|
||||
### -Configure Work/Log Paths
|
||||
### -Recovery Settings (These can be changed via API)
|
||||
|
||||
|
||||
help()
|
||||
{
|
||||
#TODO: Add help text here
|
||||
echo "This script installs spark cluster on Ubuntu"
|
||||
echo "Parameters:"
|
||||
echo "-k spark version like 1.2.1"
|
||||
echo "-m master 1 slave 0"
|
||||
echo "-h view this help content"
|
||||
}
|
||||
|
||||
log()
|
||||
{
|
||||
# If you want to enable this logging add a un-comment the line below and add your account key
|
||||
#curl -X POST -H "content-type:text/plain" --data-binary "$(date) | ${HOSTNAME} | $1" https://logs-01.loggly.com/inputs/[account-key]/tag/redis-extension,${HOSTNAME}
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
echo "Begin execution of spark script extension on ${HOSTNAME}"
|
||||
|
||||
if [ "${UID}" -ne 0 ];
|
||||
then
|
||||
echo "Script executed without root permissions"
|
||||
echo "You must be root to run this program." >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
#Script Parameters
|
||||
SPK_VERSION="1.2.1"
|
||||
MASTER1SLAVE0="-1"
|
||||
|
||||
#Loop through options passed
|
||||
while getopts :k:m:h optname; do
|
||||
echo "Option $optname set with value ${OPTARG}"
|
||||
case $optname in
|
||||
k) #spark version
|
||||
SPK_VERSION=${OPTARG}
|
||||
;;
|
||||
m) #Master 1 Slave 0
|
||||
MASTER1SLAVE0=${OPTARG}
|
||||
;;
|
||||
h) #show help
|
||||
help
|
||||
exit 2
|
||||
;;
|
||||
\?) #unrecognized option - show help
|
||||
echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed."
|
||||
help
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
#cd /usr/local/spark/conf
|
||||
#echo "sparknode0" >> slaves
|
||||
#echo "sparknode1" >> slaves
|
||||
|
||||
cd /usr/local/spark/sbin
|
||||
|
||||
if [ ${MASTER1SLAVE0} -eq "1" ];
|
||||
then
|
||||
#
|
||||
#Start Master
|
||||
#-----------------------
|
||||
./start-master.sh
|
||||
else
|
||||
#
|
||||
#Start Slave
|
||||
#-----------------------
|
||||
./start-slave.sh
|
||||
fi
|
||||
|
||||
#========================= END ==================================
|
||||
|
||||
Загрузка…
Ссылка в новой задаче