Merge pull request #1 from magaal/patch-1

Adding shutdown and startup scripts
This commit is contained in:
Lara Rubbelke 2015-03-12 09:48:50 -05:00
Родитель 4fbf8d2b90 113a457690
Коммит 58d30cdd20
20 изменённых файлов: 235 добавлений и 27 удалений

1
bash/centos_config.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{"fileUris":["https://raw.githubusercontent.com/magaal/Azure-DDP/patch-1/bash/centosconfig.sh"], "commandToExecute":"sh centosconfig.sh", "timestamp":"1.0.0" }

23
bash/centosconfig.sh Normal file
Просмотреть файл

@ -0,0 +1,23 @@
mkdir /root/scripts
mv /root/makefilesystem.sh /root/scripts/makefilesystem.sh
chmod 755 /root/scripts/makefilesystem.sh
yum -y install dos2unix
dos2unix /root/scripts/makefilesystem.sh /root/scripts/makefilesystem.sh
#disable iptables
chkconfig iptables off
/etc/init.d/iptables stop
setenforce 0
#start ntp service
yum -y install ntp
chkconfig ntpd on
ntpdate pool.ntp.org
echo 'vm.swappiness = 0' >> /etc/sysctl.conf
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled">>/etc/rc.local
echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag">>/etc/rc.local
ip="$(ifconfig | grep -A 1 'eth0' | tail -1 | cut -d ':' -f 2 | cut -d ' ' -f 1)"
if [ $ip = "172.16.0.4" ] ; then
reboot now
fi

Просмотреть файл

@ -0,0 +1 @@
{"fileUris":["https://raw.githubusercontent.com/magaal/Azure-DDP/patch-1/bash/clouderainstall_2.sh"], "commandToExecute": "sh clouderainstall_2.sh", "timestamp": "4.0.0" }

3
bash/clouderainstall.sh Normal file
Просмотреть файл

@ -0,0 +1,3 @@
wget http://archive.cloudera.com/cm5/installer/latest/cloudera-manager-installer.bin
chmod u+x cloudera-manager-installer.bin
./cloudera-manager-installer.bin --i-agree-to-all-licenses --noprompt --noreadme

Просмотреть файл

@ -0,0 +1,6 @@
wget http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo -P /etc/yum.repos.d/
yum clean all
yum -y install oracle-j2sdk1.7
yum -y install cloudera-manager-server-db-2
service cloudera-scm-server-db start
service cloudera-scm-server start

77
bash/clouderanotes.txt Normal file
Просмотреть файл

@ -0,0 +1,77 @@
Prepare Development PC to provision to Azure
1. Install nodejs, npm, azure-cli
a. On Mac OS X Mavericks+ - Brew is required. To install brew, nodejs, npm, jq and the Azure CLI perform the following:
i. ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"
ii. brew update
iii. brew doctor
iv. brew install node
v. npm install -g azure-cli
vi. azure –v
vii. brew install jq
b. On Linux, install
2. Download the publishsettings file.
a. On your development PC, run the azure account download command to download your .publishsettings file from a browser.
3. Azure account import publishsettings.
4. Clone the github repository
a. https://github.com/magaal/Azure-DDP/tree/patch-1/bash
5. Edit the clustersetup.sh file with parameters for your deployment
a. Edit lines:
i. 5 to match the name of your Azure subscription
ii. 13 & 14 to be any name you want
iii. 19 to be any name you want
iv. 29 and 30 - these must be globally unique
v. 34 - this must be globally unique
vi. 37 - this must be globally unique
vii. 38 & 39
viii. 54 for your VM size of management node
ix. 60 & 61 for size and number of disks to be attached to management node
x. 68 & 69 for the name of your worker node images
xi. 77 number of nodes in your cluster
xii. 83 - VM size of your worker nodes
xiii. 87, 88 number and size of persistent disks to attach to cluster nodes
xiv. 90, 91 - name of your cluster cloud services, must be unique
6. sh preprovision.sh
Prepare the Nodes
1. ssh clusteradmin@#.cloudapp.net (replace # with name of cloud service as above - this cloud service will be based on what you put for line 34 in clustersetup.sh)
a. sudo -s
b. passwd
c. exit
d. exit
2. ssh clusteradmin@#.cloudapp.net -p 23 (NOTE: you may have to wait an extra minute or so for this to be active after provisioning)
a. sudo -s
b. passwd
c. exit
d. exit
3. ssh root@#.cloudapp.net
a. ssh-keygen
b. ssh-copy-id -i /root/.ssh/id_rsa.pub root@172.16.0.4
c. ssh-copy-id -i /root/.ssh/id_rsa.pub root@172.16.0.5
d. exit
4. scp makefilesystem.sh root@#.cloudapp.net:~
5. scp -P 23 makefilesystem.sh root@#.cloudapp.net:~
6. sh precapture.sh
Create Worker Node Clone Image
1. sh capturecloneimage.sh
Create Cluster
1. sh createclusternodes.sh
Initialize Cluster & Install Cloudera
1. scp clustersetup.sh hosts.txt hostscript.sh updatehosts.sh root@#.cloudapp.net:~
2. sh provisioncluster.sh
3. Navigate to http://#.cloudapp.net:7180
4. Create your cluster with Core + Spark. Ensure you assign a Spark Gateway to a host.
5. IMPORTANT: On the cluster config pages, remove any mention of any volumes that begin with /mnt/resource, these are the temporary SSD disks and we do not want any Cloudera components installed there.
spark-submit --class org.apache.spark.examples.SparkPi --deploy-mode client --master yarn --num-executors 4 /opt/cloudera/parcels/CDH/jars/spark-examples-*.jar 10000 1000
Pausing and Resuming the Cluster
1. Execute sh shutdownclu.sh from your development PC to stop all worker nodes.
2. Execute sh startupclu.sh from your development PC to start all worker nodes.

Просмотреть файл

@ -36,18 +36,18 @@ azure account set "$subscriptionName"
##############################################################
## Create affinity group if it does not exist
##############################################################
printf "affinty group name is %s, affinity group location is %s\n" "$affinityGroupName" "$affinityGroupLocation"
#printf "affinty group name is %s, affinity group location is %s\n" "$affinityGroupName" "$affinityGroupLocation"
result=$(azure account affinity-group show "$affinityGroupName" --json | jq '.name')
if [[ -z $result ]]; then
(azure account affinity-group create --location "$affinityGroupLocation" --label "$affinityGroupLabel" --description "$affinityGroupDescription" "$affinityGroupName") || { echo "Failed to create affinity group $affinityGroupName"; exit 1; }
else
echo "affinity group $affinityGroupName exists"
fi
#result=$(azure account affinity-group show "$affinityGroupName" --json | jq '.name')
#if [[ -z $result ]]; then
# (azure account affinity-group create --location "$affinityGroupLocation" --label "$affinityGroupLabel" --description "$affinityGroupDescription" "$affinityGroupName") || { echo "Failed to create affinity group $affinityGroupName"; exit 1; }
#else
# echo "affinity group $affinityGroupName exists"
#fi
#show the affinity group details
printf "######################################## Affinity Group Details #######################################\n"
azure account affinity-group show "$affinityGroupName" --json
#printf "######################################## Affinity Group Details #######################################\n"
#azure account affinity-group show "$affinityGroupName" --json
##############################################################
## Create the primary storage account if it does not exist
@ -56,7 +56,7 @@ printf "storage account name is %s\n" "$storageAccountName"
result=$(azure storage account show "$storageAccountName" --json | jq '.name')
if [[ -z $result ]]; then
(azure storage account create --affinity-group "$affinityGroupName" --disable-geoReplication $storageAccountName) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
(azure storage account create --location "$affinityGroupLocation" --disable-geoReplication $storageAccountName) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
else
echo "Storage account $storageAccountName exists"
fi
@ -75,7 +75,7 @@ for storageAccount in ${storageAccountList[@]}
do
result=$(azure storage account show "$storageAccount" --json | jq '.name')
if [[ -z $result ]]; then
(azure storage account create --affinity-group "$affinityGroupName" --disable-geoReplication $storageAccount) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
(azure storage account create --location "$affinityGroupLocation" --disable-geoReplication $storageAccount) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
else
echo "Storage account $storageAccount exists"
fi
@ -88,12 +88,12 @@ printf "virtual network is %s, subnet is %s\n" "$vnetName" "$subnetName"
result=$(azure network vnet show $vnetName --json | jq '.name')
if [[ -z $result ]]; then
printf "Virtual network %s\n" "$vnetName does not exist. Please open the Azure Portal to create the virtual network. After the virtual network is created rerun the createagstoragevnet.sh process."
# (azure network vnet create --vnet $vnetName --location "$affinityGroupLocation" --address-space $vnetAddressSpace --cidr $vnetCidr --subnet-name $subnetName --subnet-start-ip $subnetAddressSpace --subnet-cidr $subnetCidr) || { echo "Failed to create virtual network $vnetName"; exit 1;}
printf "Virtual network %s\n" "$vnetName does not exist."
(azure network vnet create --vnet $vnetName --location "$affinityGroupLocation" --address-space $vnetAddressSpace --cidr $vnetCidr --subnet-name $subnetName --subnet-start-ip $subnetAddressSpace --subnet-cidr $subnetCidr) || { echo "Failed to create virtual network $vnetName"; exit 1;}
else
printf "Virtual network $virtualNetworkName exists\n"
fi
#show the virtual network details
printf "######################################## Virtual Network Details #######################################\n"
azure network vnet show "$vnetName" --json
azure network vnet show "$vnetName" --json

Просмотреть файл

@ -32,8 +32,8 @@ dnsName=$cloudServiceName".cloudapp.net"
#Check to see if the cloud servide already exists
result=$(azure service show $cloudServiceName --json | jq '.serviceName')
if [[ -z $result ]]; then
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
printf "Service does not exist. About to create cloud service:$cloudServiceName in location:$affinityGroupLocation\n"
(azure service create --location "$affinityGroupLocation" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
else
printf "Cloud Service $cloudServiceName exists\n"
fi
@ -48,7 +48,7 @@ if [[ -z $result ]]; then
printf "Virtual machine $vnName does not exist. Creating ...\n"
#create the vm and attach data disks
(azure vm create --connect --affinity-group $affinityGroupName --vm-size $instanceSize --vm-name $vmName --ssh 23 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
(azure vm create --connect --location "$affinityGroupLocation" --vm-size $instanceSize --vm-name $vmName --ssh 23 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
else
printf "Virtual machine $vmName exists\n"
fi

Просмотреть файл

@ -43,8 +43,8 @@ do
#Check to see if the cloud servide already exists
result=$(azure service show $cloudServiceName --json | jq '.serviceName')
if [[ -z $result ]]; then
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
printf "Service does not exist. About to create cloud service:$cloudServiceName in location:$affinityGroupLocation\n"
(azure service create --location "$affinityGroupLocation" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
else
printf "Cloud Service $cloudServiceName exists\n"
fi
@ -59,7 +59,7 @@ do
printf "Virtual machine $vnName does not exist. Creating ...\n"
#create the vm and attach data disks
( azure vm create --connect --affinity-group $affinityGroupName --vm-size $clusterinstanceSize --vm-name $vmName --ssh $ssh --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $cloneImageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
( azure vm create --connect --location "$affinityGroupLocation" --vm-size $clusterinstanceSize --vm-name $vmName --ssh $ssh --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $cloneImageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
#add all the necessary data disks
index=0
@ -79,8 +79,8 @@ do
#or we can use the root user to overwrite the /etc/hosts file
#echo "scp /etc/hosts ${adminUserName}@${vmName}:/etc" >> $mntscript
#echo "ssh ${adminUserName}@${vmName}:/root/scripts/st.pl" >> $mntscript
echo "ssh root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
echo "scp /etc/hosts root@${vmName}:/etc" >> $mntscript
echo "ssh -o StrictHostKeyChecking=no root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
echo "scp -o StrictHostKeyChecking=no /etc/hosts root@${vmName}:/etc" >> $mntscript
printf "######################################## Virtual Machine Details #######################################\n"
#display the details about the newly created VM

Просмотреть файл

@ -38,8 +38,8 @@ dnsName=$cloudServiceName".cloudapp.net"
#Check to see if the cloud servide already exists
result=$(azure service show $cloudServiceName --json | jq '.serviceName')
if [[ -z $result ]]; then
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
printf "Service does not exist. About to create cloud service:$cloudServiceName in location $affinityGroupLocation\n"
(azure service create --location "$affinityGroupLocation" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
else
printf "Cloud Service $cloudServiceName exists\n"
fi
@ -53,7 +53,7 @@ result=$(azure vm show $vmName --json | jq '.VMName')
if [[ -z $result ]]; then
printf "Virtual machine $vnName does not exist. Creating ...\n"
#create the vm and attach data disks
(azure vm create --connect --affinity-group $affinityGroupName --vm-size $instanceSize --vm-name $vmName --ssh 22 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1; }
(azure vm create --connect --location "$affinityGroupLocation" --vm-size $instanceSize --vm-name $vmName --ssh 22 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1; }
#add all the necessary data disks
index=0
@ -79,5 +79,5 @@ fi
#remove the double quotes from the vm name and write to the hosts file and the mount disk file
echo "$ipaddress $vmName" | sed -e 's/\"//g' >> $hostsfile
echo "ssh root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
echo "scp /etc/hosts root@${vmName}:/etc" >> $mntscript
echo "ssh -o StrictHostKeyChecking=no root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
echo "scp -o StrictHostKeyChecking=no /etc/hosts root@${vmName}:/etc" >> $mntscript

1
bash/final_config.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{"fileUris":["https://raw.githubusercontent.com/magaal/Azure-DDP/patch-1/bash/finalconfig.sh"], "commandToExecute": "sh finalconfig.sh", "timestamp": "4.0.0" }

8
bash/finalconfig.sh Normal file
Просмотреть файл

@ -0,0 +1,8 @@
#!/bin/bash
#On your management node logged in as root
chmod 755 /root/hostscript.sh
chmod 755 /root/updatehosts.sh
#Reads the hosts.txt and updates /etc/hosts file on the management node
sh /root/updatehosts.sh
#Update the hosts file on each node in the cluster and mounts the data drives
sh /root/hostscript.sh

12
bash/precapture.sh Normal file
Просмотреть файл

@ -0,0 +1,12 @@
#!/bin/bash
source ./clustersetup.sh
mgmtnode="${vmNamePrefix}0"
clonenode="${vmNamePrefix}c"
azure vm extension set -c "./centos_config.json" $mgmtnode CustomScriptForLinux Microsoft.OSTCExtensions 1.*
azure vm extension set -c "./centos_config.json" $clonenode CustomScriptForLinux Microsoft.OSTCExtensions 1.*
echo Sleeping for 30 seconds...
sleep 30
azure vm extension set -c "./prepclone.json" $clonenode CustomScriptForLinux Microsoft.OSTCExtensions 1.*
echo Sleeping for 20 seconds...
sleep 20

1
bash/prepclone.json Normal file
Просмотреть файл

@ -0,0 +1 @@
{"fileUris":["https://raw.githubusercontent.com/magaal/Azure-DDP/patch-1/bash/prepclone.sh"], "commandToExecute": "sh prepclone.sh" }

3
bash/prepclone.sh Normal file
Просмотреть файл

@ -0,0 +1,3 @@
sed -i 's/Provisioning.DeleteRootPassword=y/Provisioning.DeleteRootPassword=n/g' /etc/waagent.conf
sed -i 's/Provisioning.RegenerateSshHostKeyPair=y/Provisioning.RegenerateSshHostKeyPair=n/g' /etc/waagent.conf
waagent -deprovision -force

6
bash/preprovision.sh Normal file
Просмотреть файл

@ -0,0 +1,6 @@
echo ###CREATING AFFINITY GROUPS, STORAGE ACCOUNTS, and VNET###
sh createagstoragevnet.sh
echo ###CREATING MANAGEMENT NODE###
sh createmgmtnode.sh
echo ###CREATING CLONE NODE###
sh createclonenode.sh

19
bash/provisioncluster.sh Normal file
Просмотреть файл

@ -0,0 +1,19 @@
#!/bin/sh
source ./clustersetup.sh
mgmtnode="${vmNamePrefix}0"
echo Sleeping for 30 seconds...
sleep 30
azure vm extension set -c "./final_config.json" $mgmtnode CustomScriptForLinux Microsoft.OSTCExtensions 1.*
echo Formatting and mounting disks on all cluster nodes...pausing 5 minutes
sleep 300
azure vm extension set -c "./cloudera_install.json" $mgmtnode CustomScriptForLinux Microsoft.OSTCExtensions 1.*
echo Cloudera Installation submitted - please wait up to 5 minutes and browse to http://$cloudServicePrefix.cloudapp.net:7180 to finalize configuration
echo Deleting SSH endpoints on cluster nodes for security...
countService=1
cloudServiceName="$clustercloudServicePrefix$countService"
vmlist=$(azure vm list -d $cloudServiceName --json | jq .[] | jq ."VMName")
vmlistclean=$(echo $vmlist | sed 's/\"//g')
for i in $vmlistclean; do
azure vm endpoint delete $i ssh
done

23
bash/shutdownclu.sh Normal file
Просмотреть файл

@ -0,0 +1,23 @@
#!/bin/bash
source ./clustersetup.sh
countService=1
cloudServiceName="$clustercloudServicePrefix$countService"
mgmtNodeCSName="$cloudServicePrefix"
vmlist=$(azure vm list -d $cloudServiceName --json | jq .[] | jq ."VMName")
vmlistclean=$(echo $vmlist | sed 's/\"//g')
mgmtnode=$(azure vm list -d $cloudServicePrefix --json | jq .[] | jq ."VMName")
mgmtnodeclean=$(echo $mgmtnode | sed 's/\"//g')
for i in $vmlistclean; do
echo "#########VM SHUTDOWN - WORKER NODES#########"
echo "Shutting down VM: $i"
echo "#########VM SHUTDOWN - WORKER NODES#########"
azure vm shutdown $i
done
echo "#########VM SHUTDOWN - MANAGEMENT NODE#########"
echo "Shutting down VM: $mgmtnodeclean"
echo "#########VM SHUTDOWN - MANAGEMENT NODE#########"
azure vm shutdown $mgmtnodeclean

23
bash/startupclu.sh Normal file
Просмотреть файл

@ -0,0 +1,23 @@
#!/bin/bash
source ./clustersetup.sh
countService=1
cloudServiceName="$clustercloudServicePrefix$countService"
mgmtNodeCSName="$cloudServicePrefix"
vmlist=$(azure vm list -d $cloudServiceName --json | jq .[] | jq ."VMName")
vmlistclean=$(echo $vmlist | sed 's/\"//g')
mgmtnode=$(azure vm list -d $cloudServicePrefix --json | jq .[] | jq ."VMName")
mgmtnodeclean=$(echo $mgmtnode | sed 's/\"//g')
echo "#########VM STARTUP - MANAGEMENT NODE#########"
echo "Starting up VM: $mgmtnodeclean"
echo "#########VM STARTUP - MANAGEMENT NODE#########"
azure vm start $mgmtnodeclean
for i in $vmlistclean; do
echo "#########VM STARTUP - WORKER NODES#########"
echo "Starting up VM: $i"
echo "#########VM STARTUP - WORKER NODES#########"
azure vm start $i
done

Просмотреть файл

@ -2,6 +2,7 @@
#This script reads the hosts file and mergest its content with /etc/hosts
source ./clustersetup.sh
hostsfile="/root/hosts.txt"
#check to make sure hosts file exists
echo "hosts file name is:$hostsfile"