зеркало из https://github.com/microsoft/Azure-DDP.git
Initial build
This commit is contained in:
Lara Rubbelke
Родитель
49e052baad
Коммит
525634ca3f
|
|
@ -0,0 +1,163 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the Affinity Group, Virtual Network and Storage Accounts for distributed data platform deployments on Azure virtual machines.
|
||||
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure infrastructure to support deploying a distributed data platform
|
||||
on Windows Azure Virtual Machines with Linux hosts.
|
||||
|
||||
Create the affinity group. If it exists, move on to the next step.
|
||||
Create the main storage account and create the data node storage accounts. If the storage accounts exist, move to the next step.
|
||||
Create the virtual network. If it exists, this step may produce an error that can be ignored.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Create_AG_Storage_VNet -affinityGroupLocation "East US" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-affinityGroupDescription "Affinity Group used for DDP on Azure VM" `
|
||||
-affinityGroupLabel "DDP on Azure VM AG" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-virtualSubnetname "App" `
|
||||
-storageAccountName "clustersa" `
|
||||
-storageAccountList "clustersa1", "clustersa2", "clustersa3", "clustersa4" `
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupLocation,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The description of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupDescription,
|
||||
|
||||
# The affinity group label.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupLabel,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The virtual network address space.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkAddressSpace,
|
||||
|
||||
# The name of the virtual network CIDR.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkCIDR,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The virtual subnet address space.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$subnetAddressSpace,
|
||||
|
||||
# The virtual subnet CIDR.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$subnetCIDR,
|
||||
|
||||
# The name of the primary storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage accounts for the data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[array]$storageAccountList,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
###########################################################################################################
|
||||
## Create the Affinity Group
|
||||
###########################################################################################################
|
||||
if ((Get-AzureAffinityGroup | where {$_.Name -eq $affinityGroupName}) -eq $NULL)
|
||||
{
|
||||
New-AzureAffinityGroup -Location $affinityGroupLocation -Name $affinityGroupName -Description $affinityGroupDescription -Label $affinityGroupLabel
|
||||
Write-Host "New Affinity Group" $affinityGroupName "Created"
|
||||
}
|
||||
else
|
||||
{
|
||||
Write-Host "Affinity Group" $affinityGroupName "Exists"
|
||||
}
|
||||
|
||||
###########################################################################################################
|
||||
## Create the Storage Accounts.
|
||||
## Set the initial storage account as the default storage account.
|
||||
## Additional storage accounts are generated to store data node data disks. The number of storage accounts will
|
||||
## equal the number of cloud services.
|
||||
###########################################################################################################
|
||||
# Initial storage account set as default and used to store management node, images, and data node OS disks.
|
||||
if ((Get-AzureStorageAccount | where {$_.StorageAccountName -eq $storageAccountName}) -eq $NULL)
|
||||
{.\0_Create_Storage_Account.ps1 -affinityGroupName $affinityGroupName -storageAccountName $storageAccountName -subscriptionName $subscriptionName
|
||||
}
|
||||
else
|
||||
{
|
||||
Write-Host "Storage account" $storageAccountName "Exists"
|
||||
}
|
||||
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName
|
||||
|
||||
# Create storage accounts for data node data disks.
|
||||
# Cleanup old jobs
|
||||
get-job | ? {($_.State -ne "Running") -and ($_.State -ne "Blocked")} | remove-job
|
||||
|
||||
$jobs = @()
|
||||
foreach ($storageAccount in $storageAccountList)
|
||||
{
|
||||
|
||||
$jobs += Start-Job -FilePath .\0_Create_Storage_Account.ps1 `
|
||||
-ArgumentList $affinityGroupName, `
|
||||
$storageAccount, `
|
||||
$subscriptionName
|
||||
|
||||
Write-Progress -Activity "Submitting storage account for creation"
|
||||
}
|
||||
Write-Progress "Submitting storage account for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -Completed
|
||||
|
||||
###########################################################################################################
|
||||
## Create the Virtual Network
|
||||
###########################################################################################################
|
||||
azure account set $subscriptionName
|
||||
|
||||
if ((Get-AzureVnetSite | where {$_.Name -eq $virtualNetworkName}) -eq $NULL)
|
||||
{
|
||||
Write-Host "Virtual Network is not found. Please create the virtual network before proceeding."
|
||||
}
|
||||
<## Removing the create process until we can validate the CLI process
|
||||
{azure network vnet create --vnet $virtualNetworkName --location $affinityGroupLocation --address-space $virtualNetworkAddressSpace --cidr $virtualNetworkCIDR --subnet-name $virtualSubnetname --subnet-start-ip $subnetAddressSpace --subnet-cidr $subnetCIDR}
|
||||
##>
|
||||
else
|
||||
{
|
||||
Write-Host "Virtual Network exists"
|
||||
}
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
param(
|
||||
# The name of the image used to create the vms.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmName,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the primary storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage accounts for the data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccount,
|
||||
|
||||
# The location of the hosts file.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostsfile = ".\hosts.txt",
|
||||
|
||||
# The location of the script to push updates to the cluster nodes.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostscript = ".\hostscript.sh",
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName,
|
||||
|
||||
# Path for the 0_Create_VM.ps1 script
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$path
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
###########################################################################################################
|
||||
#Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
|
||||
# Following modifies the Write-Output behavior to turn the messages on globally for this session
|
||||
$VerbosePreference = "SilentlyContinue"
|
||||
$DebugPreference = "SilentlyContinue"
|
||||
|
||||
Set-Location $path
|
||||
|
||||
|
||||
.\0_Create_VM.ps1 -imageName $imageName `
|
||||
-adminUserName $adminUserName `
|
||||
-adminPassword $adminPassword `
|
||||
-instanceSize $instanceSize `
|
||||
-diskSizeInGB $diskSizeInGB `
|
||||
-vmName $vmName `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualSubnetname $virtualSubnetname `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountNameDisk $storageAccount `
|
||||
-cloudServiceName $cloudServiceName `
|
||||
-numofDisks $numOfDisks `
|
||||
-subscriptionName $subscriptionName
|
||||
|
||||
# Capture vm variable
|
||||
$vm = Get-AzureVM -ServiceName $cloudServiceName -Name $vmName
|
||||
$IpAddress = $vm.IpAddress
|
||||
|
||||
# Write to the hostscript.sh file
|
||||
"scp /etc/hosts root@${vmName}:/etc" | Out-File $hostscript -encoding ASCII -append
|
||||
"ssh root@$vmName /root/scripts/makefilesystem.sh" | Out-File $hostscript -encoding ASCII -append
|
||||
|
||||
# Write to the hosts.txt file
|
||||
"$IpAddress`t$vmName" | Out-File $hostsfile -encoding ASCII -append
|
||||
|
||||
# Set Static IP on the VM
|
||||
Set-AzureStaticVNetIP -IPAddress $IpAddress -VM $vm | Update-AzureVM
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the storage accounts for distributed data platforms on Azure virtual machines with Linux OS.
|
||||
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure infrastructure to support deploying a distributed data platform
|
||||
on Windows Azure Virtual Machines with Linux hosts.
|
||||
|
||||
Create a single storage account.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Create_Storage_Account.ps1 -affinityGroupName "clusterag" -storageAccountName "clustersa"
|
||||
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param (
|
||||
# Affinity Group of the blob storage account
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$affinityGroupName,
|
||||
|
||||
# Blob storage account for storing vhds and scripts
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$storageAccountName,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
###########################################################################################################
|
||||
## Create storage account
|
||||
###########################################################################################################
|
||||
|
||||
# Storage accounts require lower case names. Convert to lower case.
|
||||
$storageAccountName = $storageAccountName.ToLower()
|
||||
|
||||
# Check if account already exists then use it
|
||||
if ((Get-AzureStorageAccount | where {$_.StorageAccountName -eq $storageAccountName}) -eq $NULL)
|
||||
{
|
||||
Write-Verbose "Creating new storage account $storageAccountName."
|
||||
$storageAccount = New-AzureStorageAccount –StorageAccountName $storageAccountName -AffinityGroup $affinityGroupName
|
||||
Set-AzureStorageAccount -StorageAccountName $storageAccountName –GeoReplicationEnabled $false
|
||||
}
|
||||
else
|
||||
{
|
||||
Write-Host "Storage Account $storageAccountName Exists"
|
||||
}
|
||||
|
|
@ -0,0 +1,180 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Creates a Linux Virtual Machine for use with distributed data platform deployments on Azure virtual machines.
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure VMs to support the deploying distributed data platforms
|
||||
on Windows Azure Virtual Machines. This script will be run from master scripts.
|
||||
|
||||
The virtual machines will be named based on a prefix. The VMs are distributed evenly across the cloud services.
|
||||
Each VM will have attached data disks that are written to a storage account defined in the variable array.
|
||||
All OS disks are written to the default storage account where the image is stored.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Create_VM.ps1 -imageName "5112500ae3b842c8b9c604889f8753c3__OpenLogic-CentOS-65-20140606" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 100 `
|
||||
-vmName "clusternode" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualSubnetname "App" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-storageAccountName "clustersa" `
|
||||
-$storageAccountNameDisk "clustersa1" `
|
||||
-cloudServiceName "clusternode" `
|
||||
-numofDisks 2
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
|
||||
param(
|
||||
# The name of the image. Can be wildcard pattern.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmName,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage account for data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountNameDisk,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Create the cloud service if it doesn't exist
|
||||
###########################################################################################################
|
||||
if (Test-AzureName -Service $cloudServiceName)
|
||||
{
|
||||
Write-Output "Service $cloudServiceName exists."
|
||||
}
|
||||
else
|
||||
{$result = New-AzureService `
|
||||
-ServiceName $cloudServiceName `
|
||||
-AffinityGroup $affinityGroupName `
|
||||
-ErrorVariable csError
|
||||
}
|
||||
|
||||
if($?)
|
||||
{
|
||||
Write-Output "Service $cloudServiceName was created successfully. result is $($result.OperationDescription) $cloudServiceName."
|
||||
}
|
||||
else
|
||||
{
|
||||
throw "Service $cloudServiceName could not be created - Error is: $($csError[0])"
|
||||
}
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
# Create overall configuration
|
||||
## Set the VM size, name and general configuration
|
||||
## Attach disks
|
||||
###########################################################################################################
|
||||
$vmtest = Get-AzureVM | Where {$_.Name -eq $vmName -and $_.ServiceName -eq $cloudServiceName}
|
||||
if ($vmtest -eq $null)
|
||||
{
|
||||
$vmConfig = New-AzureVMConfig -Name $vmName -InstanceSize $instanceSize -ImageName $imageName
|
||||
|
||||
$vmDetails = Add-AzureProvisioningConfig -Linux `
|
||||
-LinuxUser $adminUserName `
|
||||
-Password $adminPassword `
|
||||
-VM $vmConfig
|
||||
|
||||
# Add disks to the configuration
|
||||
for ($index = 0; $index -lt $numOfDisks; $index++)
|
||||
{
|
||||
$diskLabel = "$vmName$index"
|
||||
$vmConfig = $vmConfig | Add-AzureDataDisk -CreateNew `
|
||||
-DiskSizeInGB $diskSizeInGB `
|
||||
-DiskLabel $diskLabel `
|
||||
-HostCaching None `
|
||||
-LUN $index `
|
||||
-MediaLocation "https://$storageAccountNameDisk.blob.core.windows.net/vhd/$vmName$index.vhd"
|
||||
}
|
||||
|
||||
<#
|
||||
# Sets SSH endpoint to port 22 passthrough
|
||||
Remove-AzureEndpoint "SSH" -VM $vmConfig
|
||||
Add-AzureEndpoint -Protocol tcp `
|
||||
-PublicPort 22 `
|
||||
-LocalPort 22 -Name "SSH" -VM $vmConfig
|
||||
#>
|
||||
|
||||
# Adds the subnet to the configuration
|
||||
Set-AzureSubnet $virtualSubnetname -VM $vmConfig
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
# Create the virtual machine
|
||||
###########################################################################################################
|
||||
$result = New-AzureVM -ServiceName $cloudServiceName `
|
||||
-VMs @($vmDetails) `
|
||||
-VNetName $virtualNetworkName `
|
||||
-ErrorVariable creationError
|
||||
|
||||
if($?)
|
||||
{
|
||||
Write-Output "VM $vmName was created successfully. result is $($result.OperationDescription) $vmName."
|
||||
}
|
||||
else
|
||||
{
|
||||
throw "Service $cloudServiceName could not be created - Error is: $($creationError[0])"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#Set-ExecutionPolicy RemoteSigned
|
||||
|
||||
#what regions are available
|
||||
Get-AzureLocation | Select DisplayName
|
||||
|
||||
#what subscription is the current focus
|
||||
(Get-AzureSubscription -Current).SubscriptionName
|
||||
(Get-AzureSubscription -Current).CurrentStorageAccountName
|
||||
|
||||
#how many cores are available for VMs on this subscription (open billing case to increase quota per subscription)
|
||||
[int]$maxVMCores = (Get-AzureSubscription -current -ExtendedDetails).MaxCoreCount
|
||||
[int]$currentVMCores = (Get-AzureSubscription -current -ExtendedDetails).CurrentCoreCount
|
||||
[int]$availableCores = $maxVMCores - $currentVMCores
|
||||
Write-Host "Cores available for VMs:" $availableCores
|
||||
|
||||
#how many storage accounts are available on this subscription
|
||||
#CurrentStorageAccounts value is always 0 (bug)
|
||||
[int]$maxAvl = (Get-AzureSubscription -current -ExtendedDetails).MaxStorageAccounts
|
||||
#[int]$currentUsed = (Get-AzureSubscription -current -ExtendedDetails).CurrentStorageAccounts
|
||||
[int]$currentUsed = (Get-AzureStorageAccount).Count
|
||||
[int]$availableNow = $maxAvl - $currentUsed
|
||||
Write-Host "Storage Accounts available:" $availableNow
|
||||
|
||||
#how many cloud (hosted) services are available on this subscription
|
||||
[int]$maxAvl = (Get-AzureSubscription -current -ExtendedDetails).MaxHostedServices
|
||||
[int]$currentUsed = (Get-AzureSubscription -current -ExtendedDetails).CurrentHostedServices
|
||||
[int]$availableNow = $maxAvl - $currentUsed
|
||||
Write-Host "Cloud services available:" $availableNow
|
||||
|
||||
#List Azure VMs
|
||||
Get-AzureVM
|
||||
(Get-AzureVM).name
|
||||
|
||||
#generate SSH to IP using root
|
||||
$cloudServicePrefix1 = "clt3" + "*"
|
||||
$array = @(Get-AzureVM | Select-Object IpAddress, ServiceName, Name | Where-Object serviceName -Like "$cloudServicePrefix1" | Sort-Object Name)
|
||||
for ($i=0;$i -lt $array.length; $i++) {
|
||||
"ssh root@" + $array.IpAddress[$i] + " #" + $array.ServiceName[$i] + ", " + $array.Name[$i] }
|
||||
|
||||
#enumerate endpoints
|
||||
#(Get-AzureVM | Get-AzureEndpoint | Where-Object Name -match "SSH").Port
|
||||
#http://blog.tylerdoerksen.com/2013/09/06/quick-tip-powershell-function-to-output-external-rdp-ports/
|
||||
Get-AzureVM | Where-Object Name -Like "$cloudServicePrefix1" | Select-Object ServiceName, Name, IpAddress,
|
||||
@{ Name = "SSH Port"; Expression = { ($_ | Get-AzureEndpoint SSH).Port }
|
||||
},
|
||||
@{ Name = "Public VIP"; Expression = { ($_ | Get-AzureEndpoint SSH).VIP }
|
||||
}| Format-Table -AutoSize
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
<############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the Management node for distributed data platform deployments on Azure virtual machines. The script will create the virtual network,
|
||||
storage accounts, and affinity groups.
|
||||
|
||||
The Management Node may also be called an Edge Node by many software vendors. Management software and keys are installed on this node.
|
||||
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure infrastructure to support deploying a distributed data platform
|
||||
on Windows Azure Virtual Machines with Linux hosts.
|
||||
|
||||
The virtual machine will be named based on a prefix followed by the number 0.
|
||||
The script will accept a parameter specifying the number of disks to attach to the virtual machine.
|
||||
The virtual machine will be assigned a static IP.
|
||||
|
||||
Addiitonal manual updates are required after the initial VM creation. The manual updates will prepare the VM for the distributed data workloads,
|
||||
software installation and best practices for optimized performance and availability.
|
||||
|
||||
.EXAMPLE
|
||||
.\1_Management_Nodes.ps1 -imageName "OpenLogic" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 100 `
|
||||
-numofDisks 2 `
|
||||
-vmNamePrefix "clusternode" `
|
||||
-cloudServiceName "clusternode" `
|
||||
-storageAccountName "clustersa" `
|
||||
-storageAccountList "clustersa1", "clustersa2", "clustersa3", "clustersa4" `
|
||||
-affinityGroupLocation "East US" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-affinityGroupDescription "Affinity Group used for ddp on Azure VM" `
|
||||
-affinityGroupLabel "DDP on Azure VM AG" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-virtualSubnetname "App" `
|
||||
-installerPort 8080 `
|
||||
-hostsfile ".\hosts.txt" `
|
||||
-hostscript ".\hostscript.sh"
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the image used to create the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmNamePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# The name of the primary storage account for the vm os.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage accounts for the data disks used for the cluster vms.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[array]$storageAccountList,
|
||||
|
||||
# The location of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupLocation,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The description of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupDescription,
|
||||
|
||||
# The affinity group label.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupLabel,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The virtual network address space.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkAddressSpace,
|
||||
|
||||
# The name of the virtual network CIDR.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkCIDR,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The virtual subnet address space.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$subnetAddressSpace,
|
||||
|
||||
# The virtual subnet CIDR.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$subnetCIDR,
|
||||
|
||||
# The port for the installer endpoint. This is dictated by the software installer tools.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$installerPort,
|
||||
|
||||
# The location of the hosts file. All cluster machine names and private IPs are written to this file.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostsfile = ".\hosts.txt",
|
||||
|
||||
# The location of the hostscript. Used for pushing updates to the cluster machines from the management node.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostscript = ".\hostscript.sh",
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Remove previous versions of the files that manage the host file
|
||||
###########################################################################################################
|
||||
If (Test-Path $hostsfile) {
|
||||
Remove-Item $hostsfile
|
||||
}
|
||||
|
||||
If (Test-Path $hostscript) {
|
||||
Remove-Item $hostscript
|
||||
}
|
||||
|
||||
###########################################################################################################
|
||||
## Create the Affinity Group, the Virtual Network and the Storage Accounts
|
||||
### The main storage account will be used for all cluster OS disks, and management node disks (OS and Data)
|
||||
### Create the storage accounts to store the data node VHD files
|
||||
###########################################################################################################
|
||||
.\0_Create_AG_Storage_VNet.ps1 -affinityGroupLocation $affinityGroupLocation `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-affinityGroupDescription $affinityGroupDescription `
|
||||
-affinityGroupLabel $affinityGroupLabel `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualNetworkAddressSpace $virtualNetworkAddressSpace `
|
||||
-virtualNetworkCIDR $virtualNetworkCIDR `
|
||||
-virtualSubnetname $virtualSubnetname `
|
||||
-subnetAddressSpace $subnetAddressSpace `
|
||||
-subnetCIDR $subnetCIDR `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountList $storageAccountList `
|
||||
-subscriptionName $subscriptionName
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Select the image to provision
|
||||
## Update this logic to a specific image name (vs Like condition) when an image selection is finalized
|
||||
###########################################################################################################
|
||||
$image = Get-AzureVMImage |
|
||||
? label -Like "*$imageName*" | Sort-Object PublishedDate -Descending |
|
||||
select -First 1
|
||||
$imageName = $image.ImageName
|
||||
|
||||
###########################################################################################################
|
||||
## Create the management node virtual machine
|
||||
### The VM will be created in the main cloud service
|
||||
### The VHD files are stored in the default storage account
|
||||
### Write the hostscript and hosts file
|
||||
### Set static IP on the VM
|
||||
###########################################################################################################
|
||||
$vmName = $vmNamePrefix + "0"
|
||||
|
||||
.\0_Create_VM.ps1 -imageName $imageName `
|
||||
-adminUserName $adminUserName `
|
||||
-adminPassword $adminPassword `
|
||||
-instanceSize $instanceSize `
|
||||
-diskSizeInGB $diskSizeInGB `
|
||||
-vmName $vmName `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualSubnetname $virtualSubnetName `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountNameDisk $storageAccountName `
|
||||
-cloudServiceName $cloudServiceName `
|
||||
-numofDisks $numOfDisks `
|
||||
-subscriptionName $subscriptionName
|
||||
|
||||
|
||||
# Capture vm variable
|
||||
$vm = Get-AzureVM -ServiceName $cloudServiceName -Name $vmName
|
||||
$IpAddress = $vm.IpAddress
|
||||
|
||||
# Add endpoint for the distribution installation software
|
||||
Add-AzureEndpoint -Protocol tcp -PublicPort $installerPort -LocalPort $installerPort -Name "Installer" -VM $vm | Update-AzureVM
|
||||
|
||||
# Write to the hostscript.sh file
|
||||
"scp /etc/hosts root@${vmName}:/etc" | Out-File $hostscript -encoding ASCII -append
|
||||
"ssh root@$vmName /root/scripts/makefilesystem.sh" | Out-File $hostscript -encoding ASCII -append
|
||||
|
||||
# Write to the hosts.txt file
|
||||
"$IpAddress`t$vmName" | Out-File $hostsfile -encoding ASCII -append
|
||||
|
||||
# Set Static IP on the VM
|
||||
Set-AzureStaticVNetIP -IPAddress $IpAddress -VM $vm | Update-AzureVM
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the Management node for distributed data platform deployments on Azure virtual machines. This script assumes the
|
||||
virtual network, affinity group and storage accounts were created prior to executing this script.
|
||||
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure infrastructure to support deploying a distributed data platform
|
||||
on Windows Azure Virtual Machines with Linux hosts.
|
||||
|
||||
The virtual machines will be named based on a prefix.
|
||||
The script will accept a parameter specifying the number of disks to attach to each virtual machine. The clone node
|
||||
will not have attached disks. Disks are attached later in the process when the cluster nodes are generated.
|
||||
|
||||
.EXAMPLE
|
||||
.\2_Clone_Node.ps1 -imageName "OpenLogic" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 0 `
|
||||
-numofDisks 0 `
|
||||
-vmNamePrefix "clusternode" `
|
||||
-cloudServiceName "clusternode" `
|
||||
-storageAccountName "clustersa" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-virtualSubnetname "App"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the image used to create the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB = 0,
|
||||
|
||||
# Number of data disks to add to the virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks = 0,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmNamePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# The name of the primary storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Select the image to provision
|
||||
## Update this logic to a specific image name (vs Like condition) when an image selection is finalized
|
||||
###########################################################################################################
|
||||
$image = Get-AzureVMImage |
|
||||
? label -Like "*$imageName*" | Sort-Object PublishedDate -Descending |
|
||||
select -First 1
|
||||
$imageName = $image.ImageName
|
||||
|
||||
###########################################################################################################
|
||||
## Create the virtual machine to serve as the clone image used to generate the cluster nodes
|
||||
###########################################################################################################
|
||||
$vmName = $vmNamePrefix + "c"
|
||||
|
||||
.\0_Create_VM.ps1 -imageName $imageName `
|
||||
-adminUserName $adminUserName `
|
||||
-adminPassword $adminPassword `
|
||||
-instanceSize $instanceSize `
|
||||
-diskSizeInGB $diskSizeInGB `
|
||||
-vmName $vmName `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualSubnetname $virtualSubnetname `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountNameDisk $storageAccountName `
|
||||
-cloudServiceName $cloudServiceName `
|
||||
-numofDisks $numOfDisks `
|
||||
-subscriptionName $subscriptionName
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Used to automate the creation of an image from a virtual machine.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Capture_Image.ps1 -cloudServiceName "clusternodec" `
|
||||
-vmName "clusternodec" `
|
||||
-imageName "clusternodec" `
|
||||
-imageLabel "Cluster Clone"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmNamePrefix,
|
||||
|
||||
# The name of the new image.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The label for the new image.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageLabel,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
#Stop the virtual machine.
|
||||
###########################################################################################################
|
||||
$vmName = $vmNamePrefix + "c"
|
||||
|
||||
Get-AzureVM | where {$_.Name -eq $vmName} | Stop-AzureVM -Force
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Create an image from the new vhd file in the images container.
|
||||
###########################################################################################################
|
||||
Save-AzureVMImage -ServiceName $cloudServiceName `
|
||||
-Name $vmName `
|
||||
-ImageName $imageName `
|
||||
-ImageLabel $imageLabel
|
||||
#-OSState 'Generalized'
|
||||
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the Management node for distributed data platform deployments on Azure virtual machines. The script will create the virtual network,
|
||||
storage accounts, and affinity groups.
|
||||
|
||||
The virtual machines will be named based on a prefix.
|
||||
|
||||
.EXAMPLE
|
||||
.\3_Cluster_Nodes.ps1 -imageName "clusternodec" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 0 `
|
||||
-numofDisks 0 `
|
||||
-vmNamePrefix "clusternode" `
|
||||
-cloudServicePrefix "clusternode" `
|
||||
-numCloudServices 3 `
|
||||
-numNodes 6 `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-virtualSubnetname "App" `
|
||||
-storageAccountName "clustersa" `
|
||||
-storageAccountList "clustersa1", "clustersa2", "clustersa3", "clustersa4" `
|
||||
-hostsfile ".\hosts.txt" `
|
||||
-hostscript ".\hostscript.sh"
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the image used to create the vms.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmNamePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServicePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numCloudServices,
|
||||
|
||||
# The number of nodes.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$numNodes,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the primary storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage accounts for the data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[array]$storageAccountList,
|
||||
|
||||
# The location of the hosts file.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostsfile = ".\hosts.txt",
|
||||
|
||||
# The location of the script to push updates to the cluster nodes.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostscript = ".\hostscript.sh",
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Check if the storage accounts exist. If not, create the storage accounts.
|
||||
## Storage accounts should have been created in the step 1_Management_Node.
|
||||
###########################################################################################################
|
||||
get-job | ? {($_.State -ne "Running") -and ($_.State -ne "Blocked")} | remove-job
|
||||
|
||||
$jobs = @()
|
||||
foreach ($storageAccount in $storageAccountList)
|
||||
{
|
||||
|
||||
$jobs += Start-Job -FilePath .\0_Create_Storage_Account.ps1 `
|
||||
-ArgumentList $affinityGroupName, `
|
||||
$storageAccount, `
|
||||
$subscriptionName
|
||||
|
||||
Write-Progress -Activity "Submitting storage account for creation"
|
||||
}
|
||||
Write-Progress "Submitting storage account for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -Completed
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Select the image created in previous step. Image is used to provision
|
||||
## cluster nodes.
|
||||
###########################################################################################################
|
||||
$image = Get-AzureVMImage -ImageName $imageName
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Create the virtual machines for the cluster nodes
|
||||
### Write the hostscript and hosts file
|
||||
### Set static IP on the VM
|
||||
### First iteration will create the inital vm in each cloud service.
|
||||
### First vm in each cloud service will create the cloud service and require longer locks.
|
||||
###########################################################################################################
|
||||
$countStorageAccount = $storageAccountList.Length
|
||||
$countService = 1
|
||||
$countVM = 1
|
||||
$storageAccountIndex = 0
|
||||
|
||||
for ($countVM = 1; $countVM -le $numNodes; $countVM++)
|
||||
{
|
||||
if ($countService -gt $numCloudServices) {$countService = 1}
|
||||
if ($storageAccountIndex -eq $countStorageAccount) {$storageAccountIndex = 0}
|
||||
|
||||
$cloudServiceName = "$cloudServicePrefix$countService"
|
||||
$vmName = "$vmNamePrefix$countVM"
|
||||
$storageAccount = $storageAccountList[$storageAccountIndex]
|
||||
|
||||
.\0_Create_VM.ps1 -imageName $imageName `
|
||||
-adminUserName $adminUserName `
|
||||
-adminPassword $adminPassword `
|
||||
-instanceSize $instanceSize `
|
||||
-diskSizeInGB $diskSizeInGB `
|
||||
-vmName $vmName `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualSubnetname $virtualSubnetname `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountNameDisk $storageAccount `
|
||||
-cloudServiceName $cloudServiceName `
|
||||
-numofDisks $numOfDisks `
|
||||
-subscriptionName $subscriptionName
|
||||
|
||||
# Capture vm variable
|
||||
$vm = Get-AzureVM -ServiceName $cloudServiceName -Name $vmName
|
||||
$IpAddress = $vm.IpAddress
|
||||
|
||||
# Write to the hostscript.sh file
|
||||
"scp /etc/hosts root@${vmName}:/etc" | Out-File $hostscript -encoding ASCII -append
|
||||
"ssh root@$vmName /root/scripts/makefilesystem.sh" | Out-File $hostscript -encoding ASCII -append
|
||||
|
||||
# Write to the hosts.txt file
|
||||
"$IpAddress`t$vmName" | Out-File $hostsfile -encoding ASCII -append
|
||||
|
||||
# Set Static IP on the VM
|
||||
Set-AzureStaticVNetIP -IPAddress $IpAddress -VM $vm | Update-AzureVM
|
||||
|
||||
$countService++
|
||||
$storageAccountIndex++
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
<!--Set Cluster Variables-->
|
||||
|
||||
<Cluster>
|
||||
<!--Set the subscription-->
|
||||
<SubscriptionName>My Subscription</SubscriptionName>
|
||||
|
||||
<!--Set Affinity Group Settings-->
|
||||
<!--affinityGroupLocation is the location where the cluster will deploy. East US, West US, East Asia, Southeast Asia, North Europe, West Europe are examples of valid locations-->
|
||||
<!--affinityGroupName must be globally unique-->
|
||||
<affinityGroupLocation>West US</affinityGroupLocation>
|
||||
<affinityGroupName>ddpWest</affinityGroupName>
|
||||
<affinityGroupDescription>Affinity Group DDP Sample</affinityGroupDescription>
|
||||
<affinityGroupLabel>AG for DDP Sample</affinityGroupLabel>
|
||||
|
||||
<!--Set Virtual Network Settings-->
|
||||
<!--If the virtual network already exists, input the vnet settings -->
|
||||
<!--If the virtual networ doesn't exist, the scripts will generate the vnet based on the settings -->
|
||||
<!--Valid address space settings include 192.168.0.0, 10.0.0.0 and 172.16.0.0 -->
|
||||
<!--Network faq is here http://msdn.microsoft.com/en-us/library/windowsazure/dn133803.aspx -->
|
||||
<virtualNetworkName>clb01</virtualNetworkName>
|
||||
<virtualNetworkAddressSpace>172.16.0.0</virtualNetworkAddressSpace>
|
||||
<virtualNetworkCIDR>17</virtualNetworkCIDR>
|
||||
<virtualSubnetname>App</virtualSubnetname>
|
||||
<subnetAddressSpace>172.16.0.0</subnetAddressSpace>
|
||||
<subnetCIDR>17</subnetCIDR>
|
||||
|
||||
<!--Set Storage Account Settings-->
|
||||
<!--All storage account names must be globally unique-->
|
||||
<storageAccountName>clb0</storageAccountName>
|
||||
<storageAccountList>
|
||||
<Name>clb001</Name>
|
||||
<Name>clb002</Name>
|
||||
</storageAccountList>
|
||||
|
||||
<!--Set Virtual Machine Settings-->
|
||||
<!--cloudServicePrefix must be globally unique-->
|
||||
<adminUserName>clusteradmin</adminUserName>
|
||||
<adminPassword>Password.1</adminPassword>
|
||||
<vmNamePrefix>clb0</vmNamePrefix>
|
||||
<cloudServicePrefix>clb0</cloudServicePrefix>
|
||||
|
||||
<!--do not change the hosts/hostscript names-->
|
||||
<hostsfile>.\hosts.txt</hostsfile>
|
||||
<hostscript>.\hostscript.sh</hostscript>
|
||||
|
||||
<!--Set Management Node Variables-->
|
||||
<ManagementNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
<diskSizeInGB>500</diskSizeInGB>
|
||||
<numOfDisks>2</numOfDisks>
|
||||
<installerPort>8080</installerPort>
|
||||
</ManagementNode>
|
||||
|
||||
<!--Set Clone Node Variable-->
|
||||
<CloneNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
</CloneNode>
|
||||
|
||||
<!--Set Clone Image Variables-->
|
||||
<!--Set the name and label of the Clone Image. -->
|
||||
<CloneImage>
|
||||
<cloneimageName>clb0c</cloneimageName>
|
||||
<cloneimageLabel>Test 0 Clone</cloneimageLabel>
|
||||
</CloneImage>
|
||||
|
||||
<!--Set Cluster Nodes Variables-->
|
||||
<ClusterNodes>
|
||||
<instanceSize>A5</instanceSize>
|
||||
<diskSizeInGB>1000</diskSizeInGB>
|
||||
<numOfDisks>4</numOfDisks>
|
||||
<vmNamePrefix>clb0</vmNamePrefix>
|
||||
<cloudServicePrefix>clb0</cloudServicePrefix>
|
||||
<numNodes>8</numNodes>
|
||||
<numCloudServices>2</numCloudServices>
|
||||
</ClusterNodes>
|
||||
</Cluster>
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
<!--Set Cluster Variables-->
|
||||
<Cluster>
|
||||
<!--Set the subscription-->
|
||||
<SubscriptionName></SubscriptionName>
|
||||
|
||||
<!--Affinity Group Settings-->
|
||||
<!--affinityGroupLocation is the location where the cluster will deploy. East US, West US, East Asia, Southeast Asia, North Europ, West Europe are examples of valid locations-->
|
||||
<!--affinityGroupName must be globally unique-->
|
||||
<affinityGroupLocation></affinityGroupLocation>
|
||||
<affinityGroupName></affinityGroupName>
|
||||
<affinityGroupDescription></affinityGroupDescription>
|
||||
<affinityGroupLabel></affinityGroupLabel>
|
||||
|
||||
<!--Virtual Network Settings-->
|
||||
<!--If the virtual network already exists, input the vnet settings -->
|
||||
<!--If the virtual networ doesn't exist, the scripts will generate the vnet based on the settings -->
|
||||
<!--Valid address space settings include 192.168.0.0, 10.0.0.0 and 172.16.0.0 -->
|
||||
<!--Network faq is here http://msdn.microsoft.com/en-us/library/windowsazure/dn133803.aspx -->
|
||||
<virtualNetworkName></virtualNetworkName>
|
||||
<virtualNetworkAddressSpace></virtualNetworkAddressSpace>
|
||||
<virtualNetworkCIDR></virtualNetworkCIDR>
|
||||
<virtualSubnetname></virtualSubnetname>
|
||||
<subnetAddressSpace></subnetAddressSpace>
|
||||
<subnetCIDR></subnetCIDR>
|
||||
|
||||
<!--Storage Account Settings-->
|
||||
<!--All storage account names must be globally unique-->
|
||||
<!--Names must be in lower case-->
|
||||
<storageAccountName></storageAccountName>
|
||||
<storageAccountList>
|
||||
<Name></Name>
|
||||
<Name></Name>
|
||||
</storageAccountList>
|
||||
|
||||
<!--Virtual Machine Settings-->
|
||||
<!--cloudServicePrefix must be globally unique-->
|
||||
<adminUserName></adminUserName>
|
||||
<adminPassword></adminPassword>
|
||||
<vmNamePrefix></vmNamePrefix>
|
||||
<cloudServicePrefix></cloudServicePrefix>
|
||||
|
||||
<!--do not change the hosts/hostscript names-->
|
||||
<hostsfile>.\hosts.txt</hostsfile>
|
||||
<hostscript>.\hostscript.sh</hostscript>
|
||||
|
||||
<!--Set Management Node Variables-->
|
||||
<!--Set the galleryimagename to search on the image label. For example, OpenLogic in the galleryimagename will return the latest OpenLogic version of the CentOS image.-->
|
||||
<!--Valid instanceSize settings are available online: http://msdn.microsoft.com/en-us/library/azure/dn197896.aspx.
|
||||
Most distributed data platform software will require a minimum virtual machine size of ExtraLarge.-->
|
||||
<!--InstallerPort is the endpoint to open for the OSS software. For example, this may be 8080 for Ambari.-->
|
||||
<ManagementNode>
|
||||
<galleryimageName></galleryimageName>
|
||||
<instanceSize></instanceSize>
|
||||
<diskSizeInGB></diskSizeInGB>
|
||||
<numOfDisks></numOfDisks>
|
||||
<installerPort></installerPort>
|
||||
</ManagementNode>
|
||||
|
||||
<!--Set Clone Node Variable-->
|
||||
<CloneNode>
|
||||
<galleryimageName></galleryimageName>
|
||||
<instanceSize></instanceSize>
|
||||
</CloneNode>
|
||||
|
||||
<!--Set Clone Image Variables-->
|
||||
<!--Set the name and label of the Clone Image. -->
|
||||
<CloneImage>
|
||||
<cloneimageName></cloneimageName>
|
||||
<cloneimageLabel></cloneimageLabel>
|
||||
</CloneImage>
|
||||
|
||||
<!--Set Cluster Nodes Variables-->
|
||||
<!--numNodes is the total number of nodes to generate. This includes all nodes in the data cluster (except for the management node)-->
|
||||
<!--Valid instanceSize settings are available online: http://msdn.microsoft.com/en-us/library/azure/dn197896.aspx.
|
||||
Most distributed data platform software will require a minimum virtual machine size of ExtraLarge.-->
|
||||
<ClusterNodes>
|
||||
<instanceSize></instanceSize>
|
||||
<diskSizeInGB></diskSizeInGB>
|
||||
<numOfDisks></numOfDisks>
|
||||
<vmNamePrefix></vmNamePrefix>
|
||||
<cloudServicePrefix></cloudServicePrefix>
|
||||
<numNodes></numNodes>
|
||||
<numCloudServices></numCloudServices>
|
||||
</ClusterNodes>
|
||||
</Cluster>
|
||||
Двоичные данные
PoSH/Distributed Data Platform on Virtual Machines Process POSH. V3docx.docx
Normal file
Двоичные данные
PoSH/Distributed Data Platform on Virtual Machines Process POSH. V3docx.docx
Normal file
Двоичный файл не отображается.
|
|
@ -0,0 +1,120 @@
|
|||
|
||||
#################################################################################
|
||||
## Script Variables
|
||||
#################################################################################
|
||||
$PATH_TO_SCRIPTS = “<Path to scripts>”
|
||||
|
||||
#################################################################################
|
||||
## Load Script Configuration File
|
||||
## Script Configuration File will set the variables that are passed into each of
|
||||
## the commands
|
||||
#################################################################################
|
||||
CD $PATH_TO_SCRIPTS
|
||||
[xml] $ddpconfig = Get-Content "<Name of the config.xml file>"
|
||||
|
||||
Select-AzureSubscription -SubscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
<################################################################################
|
||||
Execute each of the following commands individually.
|
||||
|
||||
DO NOT EXECUTE THIS ENTIRE SCRIPT AT ONCE!
|
||||
Manual updates in the virtual machines must be completed between steps 2 and 3.
|
||||
|
||||
Highlight each individual section and choose execute selection in the toolbar
|
||||
or press F8.
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Management Node
|
||||
#################################################################################
|
||||
|
||||
.\1_Management_Node.ps1 -imageName $ddpconfig.Cluster.ManagementNode.galleryimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword`
|
||||
-instanceSize $ddpconfig.Cluster.ManagementNode.instanceSize`
|
||||
-diskSizeInGB $ddpconfig.Cluster.ManagementNode.diskSizeInGB `
|
||||
-numOfDisks $ddpconfig.Cluster.ManagementNode.numOfDisks `
|
||||
-vmNamePrefix $ddpconfig.Cluster.vmNamePrefix `
|
||||
-cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-storageAccountList $ddpconfig.Cluster.storageAccountList.Name `
|
||||
-affinityGroupLocation $ddpconfig.Cluster.affinityGroupLocation `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-affinityGroupDescription $ddpconfig.Cluster.affinityGroupDescription `
|
||||
-affinityGroupLabel $ddpconfig.Cluster.affinityGroupLabel `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualNetworkAddressSpace $ddpconfig.Cluster.virtualNetworkAddressSpace `
|
||||
-virtualNetworkCIDR $ddpconfig.Cluster.VirtualNetworkCIDR `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-subnetAddressSpace $ddpconfig.Cluster.SubnetAddressSpace `
|
||||
-subnetCIDR $ddpconfig.Cluster.SubnetCIDR `
|
||||
-installerPort 7180 `
|
||||
-hostscript $ddpconfig.Cluster.hostscript `
|
||||
-hostsfile $ddpconfig.Cluster.hostsfile `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Clone Node
|
||||
## Create the clone node used for generating the data nodes and name nodes.
|
||||
#################################################################################
|
||||
|
||||
.\2_Clone_Node.ps1 -imageName $ddpconfig.Cluster.CloneNode.galleryimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword `
|
||||
-instanceSize $ddpconfig.Cluster.CloneNode.instanceSize `
|
||||
-diskSizeInGB 0 `
|
||||
-numOfDisks 0 `
|
||||
-vmNamePrefix $ddpconfig.Cluster.vmNamePrefix `
|
||||
-cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Manual Updates
|
||||
#################################################################################
|
||||
## Before you continue, complete the manual updates from the documentation to prepare the
|
||||
## cluster nodes.
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Capture the image
|
||||
#################################################################################
|
||||
|
||||
.\3_Capture_Image.ps1 -cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-vmNamePrefix $ddpconfig.Cluster.vmNamePrefix `
|
||||
-imageName $ddpconfig.Cluster.CloneImage.cloneimageName `
|
||||
-imageLabel $ddpconfig.Cluster.CloneImage.cloneimageLabel `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Create the worker nodes
|
||||
#################################################################################
|
||||
|
||||
.\4_Cluster_Nodes.ps1 -imageName $ddpconfig.Cluster.CloneImage.cloneimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword `
|
||||
-instanceSize $ddpconfig.Cluster.ClusterNodes.instanceSize `
|
||||
-diskSizeInGB $ddpconfig.Cluster.ClusterNodes.diskSizeInGB `
|
||||
-numOfDisks $ddpconfig.Cluster.ClusterNodes.numOfDisks `
|
||||
-vmNamePrefix $ddpconfig.Cluster.ClusterNodes.vmNamePrefix `
|
||||
-cloudServicePrefix $ddpconfig.Cluster.ClusterNodes.cloudServicePrefix `
|
||||
-numCloudServices $ddpconfig.Cluster.ClusterNodes.numCloudServices `
|
||||
-numNodes $ddpconfig.Cluster.ClusterNodes.numNodes `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-storageAccountList $ddpconfig.Cluster.storageAccountList.Name `
|
||||
-hostsfile $ddpconfig.Cluster.hostsfile `
|
||||
-hostscript $ddpconfig.Cluster.hostscript `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
################################################################################>
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Creates a Linux Virtual Machine for use with distributed data platform deployments on Azure virtual machines.
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure VMs to support the deploying distributed data platforms
|
||||
on Windows Azure Virtual Machines. This script will be run from master scripts.
|
||||
|
||||
The virtual machines will be named based on a prefix. The VMs are distributed evenly across the cloud services.
|
||||
Each VM will have attached data disks that are written to a storage account defined in the variable array.
|
||||
All OS disks are written to the default storage account where the image is stored.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Create-VM.ps1 -imageName "b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu_DAILY_BUILD-saucy-13_10-amd64-server-20140119-en-us-30GB" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 100 `
|
||||
-vmName "clusternode" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualSubnetname "App" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-storageAccountName "clustersa" `
|
||||
-$storageAccountNameDisk "clustersa1" `
|
||||
-cloudServiceName "clusternode" `
|
||||
-numofDisks 2
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
|
||||
param(
|
||||
# The name of the image. Can be wildcard pattern.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmName,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage account for data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountNameDisk,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
# Create overall configuration
|
||||
## Set the VM size, name and general configuration
|
||||
## Attach disks
|
||||
###########################################################################################################
|
||||
$vmConfig = New-AzureVMConfig -Name $vmName -InstanceSize $instanceSize -ImageName $imageName
|
||||
|
||||
$vmDetails = Add-AzureProvisioningConfig -Linux `
|
||||
-LinuxUser $adminUserName `
|
||||
-Password $adminPassword `
|
||||
-VM $vmConfig
|
||||
|
||||
# Add disks to the configuration
|
||||
for ($index = 0; $index -lt $numOfDisks; $index++)
|
||||
{
|
||||
$diskLabel = "$vmName$index"
|
||||
$vmConfig = $vmConfig | Add-AzureDataDisk -CreateNew `
|
||||
-DiskSizeInGB $diskSizeInGB `
|
||||
-DiskLabel $diskLabel `
|
||||
-HostCaching None `
|
||||
-LUN $index `
|
||||
-MediaLocation "https://$storageAccountNameDisk.blob.core.windows.net/vhd/$vmName$index.vhd"
|
||||
}
|
||||
|
||||
<#
|
||||
# Sets SSH endpoint to port 22 passthrough
|
||||
Remove-AzureEndpoint "SSH" -VM $vmConfig
|
||||
Add-AzureEndpoint -Protocol tcp `
|
||||
-PublicPort 22 `
|
||||
-LocalPort 22 -Name "SSH" -VM $vmConfig
|
||||
#>
|
||||
|
||||
# Adds the subnet to the configuration
|
||||
Set-AzureSubnet $virtualSubnetname -VM $vmConfig
|
||||
|
||||
###########################################################################################################
|
||||
# Create the virtual machine
|
||||
## If the cloud service exists, create the VM in the existing service
|
||||
## If the cloud service doesn't exist, create the cloud service and the VM
|
||||
###########################################################################################################
|
||||
if (Test-AzureName -Service $cloudServiceName)
|
||||
{$result = New-AzureVM -ServiceName $cloudServiceName `
|
||||
-VMs @($vmDetails) `
|
||||
-VNetName $virtualNetworkName `
|
||||
#-ErrorVariable creationError
|
||||
}
|
||||
else
|
||||
{$result = New-AzureVM -ServiceName $cloudServiceName `
|
||||
-VMs @($vmDetails) `
|
||||
-AffinityGroup $affinityGroupName `
|
||||
-VNetName $virtualNetworkName `
|
||||
#-ErrorVariable creationError
|
||||
}
|
||||
|
||||
if($?)
|
||||
{
|
||||
Write-Output "Service $cloudServiceName was created successfully. Result is $($result.OperationDescription) $vmName."
|
||||
}
|
||||
else
|
||||
{
|
||||
throw "Service $cloudServiceName could not be created - Error is: $($creationError[0])"
|
||||
}
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Creates a Linux Virtual Machine for use with distributed data platform deployments on Azure virtual machines.
|
||||
.DESCRIPTION
|
||||
Used to automate the creation of Windows Azure VMs to support the deploying distributed data platforms
|
||||
on Windows Azure Virtual Machines. This script will be run from master scripts.
|
||||
|
||||
The virtual machines will be named based on a prefix. The VMs are distributed evenly across the cloud services.
|
||||
Each VM will have attached data disks that are written to a storage account defined in the variable array.
|
||||
All OS disks are written to the default storage account where the image is stored.
|
||||
|
||||
.EXAMPLE
|
||||
.\0_Create-VM.ps1 -imageName "b39f27a8b8c64d52b05eac6a62ebad85__Ubuntu_DAILY_BUILD-saucy-13_10-amd64-server-20140119-en-us-30GB" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 100 `
|
||||
-vmName "clusternode" `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualSubnetname "App" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-storageAccountName "clustersa" `
|
||||
-$storageAccountNameDisk "clustersa1" `
|
||||
-cloudServiceName "clusternode" `
|
||||
-numofDisks 2
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
|
||||
param(
|
||||
# The name of the image. Can be wildcard pattern.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmName,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage account for data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountNameDisk,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServiceName,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
## Set the default storage account
|
||||
###########################################################################################################
|
||||
$subscriptionInfo = Get-AzureSubscription -SubscriptionName $subscriptionName
|
||||
$subName = $subscriptionInfo | %{ $_.SubscriptionName }
|
||||
|
||||
Set-AzureSubscription -SubscriptionName $subName –CurrentStorageAccount $storageAccountName
|
||||
Select-AzureSubscription -SubscriptionName $subName -Current
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
# Create overall configuration
|
||||
## Set the VM size, name and general configuration
|
||||
## Attach disks
|
||||
###########################################################################################################
|
||||
$vmConfig = New-AzureVMConfig -Name $vmName -InstanceSize $instanceSize -ImageName $imageName
|
||||
|
||||
$vmDetails = Add-AzureProvisioningConfig -Linux `
|
||||
-LinuxUser $adminUserName `
|
||||
-Password $adminPassword `
|
||||
-VM $vmConfig
|
||||
|
||||
# Add disks to the configuration
|
||||
for ($index = 0; $index -lt $numOfDisks; $index++)
|
||||
{
|
||||
$diskLabel = "$vmName$index"
|
||||
$vmConfig = $vmConfig | Add-AzureDataDisk -CreateNew `
|
||||
-DiskSizeInGB $diskSizeInGB `
|
||||
-DiskLabel $diskLabel `
|
||||
-HostCaching None `
|
||||
-LUN $index `
|
||||
-MediaLocation "https://$storageAccountNameDisk.blob.core.windows.net/vhd/$vmName$index.vhd"
|
||||
}
|
||||
|
||||
<#
|
||||
# Sets SSH endpoint to port 22 passthrough
|
||||
Remove-AzureEndpoint "SSH" -VM $vmConfig
|
||||
Add-AzureEndpoint -Protocol tcp `
|
||||
-PublicPort 22 `
|
||||
-LocalPort 22 -Name "SSH" -VM $vmConfig
|
||||
#>
|
||||
|
||||
# Adds the subnet to the configuration
|
||||
Set-AzureSubnet $virtualSubnetname -VM $vmConfig
|
||||
|
||||
###########################################################################################################
|
||||
# Create the virtual machine
|
||||
## If the cloud service exists, create the VM in the existing service
|
||||
## If the cloud service doesn't exist, create the cloud service and the VM
|
||||
###########################################################################################################
|
||||
if (Test-AzureName -Service $cloudServiceName)
|
||||
{$result = New-AzureVM -ServiceName $cloudServiceName `
|
||||
-VMs @($vmDetails) `
|
||||
-VNetName $virtualNetworkName `
|
||||
#-ErrorVariable creationError
|
||||
}
|
||||
else
|
||||
{$result = New-AzureVM -ServiceName $cloudServiceName `
|
||||
-VMs @($vmDetails) `
|
||||
-AffinityGroup $affinityGroupName `
|
||||
-VNetName $virtualNetworkName `
|
||||
#-ErrorVariable creationError
|
||||
}
|
||||
|
||||
if($?)
|
||||
{
|
||||
Write-Output "Service $cloudServiceName was created successfully. Result is $($result.OperationDescription) $vmName."
|
||||
}
|
||||
else
|
||||
{
|
||||
throw "Service $cloudServiceName could not be created - Error is: $($creationError[0])"
|
||||
}
|
||||
|
|
@ -0,0 +1,232 @@
|
|||
<#############################################################################################################
|
||||
Distributed Data Platform on Azure Virtual Machines
|
||||
|
||||
.SYNOPSIS
|
||||
Create the Management node for distributed data platform deployments on Azure virtual machines. The script will create the virtual network,
|
||||
storage accounts, and affinity groups.
|
||||
|
||||
The virtual machines will be named based on a prefix.
|
||||
|
||||
.EXAMPLE
|
||||
.\3_Cluster_Nodes.ps1 -imageName "clusternodec" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeInGB 0 `
|
||||
-numofDisks 0 `
|
||||
-vmNamePrefix "clusternode" `
|
||||
-cloudServicePrefix "clusternode" `
|
||||
-numCloudServices 3 `
|
||||
-numNodes 6 `
|
||||
-affinityGroupName "clusterag" `
|
||||
-virtualNetworkName "DDP-Network" `
|
||||
-virtualSubnetname "App" `
|
||||
-storageAccountName "clustersa" `
|
||||
-storageAccountList "clustersa1", "clustersa2", "clustersa3", "clustersa4" `
|
||||
-hostsfile ".\hosts.txt" `
|
||||
-hostscript ".\hostscript.sh"
|
||||
-subscriptionName "MySubscription"
|
||||
|
||||
############################################################################################################>
|
||||
|
||||
param(
|
||||
# The name of the image used to create the vms.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$imageName,
|
||||
|
||||
# The administrator username.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminUserName,
|
||||
|
||||
# The administrator password.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$adminPassword,
|
||||
|
||||
# The size of the instances.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$instanceSize,
|
||||
|
||||
# The size of the disk(s).
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$diskSizeInGB,
|
||||
|
||||
# Number of data disks to add to each virtual machine
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numOfDisks,
|
||||
|
||||
# The name of the vm.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$vmNamePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$cloudServicePrefix,
|
||||
|
||||
# The name of the cloud service.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int]$numCloudServices,
|
||||
|
||||
# The number of nodes.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$numNodes,
|
||||
|
||||
# The name of the affinity group.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$affinityGroupName,
|
||||
|
||||
# The name of the virtual network.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualNetworkName,
|
||||
|
||||
# The name of the virtual subnet.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$virtualSubnetname,
|
||||
|
||||
# The name of the primary storage account.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$storageAccountName,
|
||||
|
||||
# The name of the storage accounts for the data disks.
|
||||
[Parameter(Mandatory = $true)]
|
||||
[array]$storageAccountList,
|
||||
|
||||
# The location of the hosts file.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostsfile = ".\hosts.txt",
|
||||
|
||||
# The location of the script to push updates to the cluster nodes.
|
||||
[Parameter(Mandatory = $false)]
|
||||
[string]$hostscript = ".\hostscript.sh",
|
||||
|
||||
# Subscription name for creating objects
|
||||
[Parameter(Mandatory = $true)]
|
||||
[String]$subscriptionName
|
||||
)
|
||||
###########################################################################################################
|
||||
## Select the subscription
|
||||
###########################################################################################################
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
###########################################################################################################
|
||||
## Check if the storage accounts exist. If not, create the storage accounts.
|
||||
## Storage accounts should have been created in the step 1_Management_Node.
|
||||
###########################################################################################################
|
||||
get-job | ? {($_.State -ne "Running") -and ($_.State -ne "Blocked")} | remove-job
|
||||
|
||||
$jobs = @()
|
||||
foreach ($storageAccount in $storageAccountList)
|
||||
{
|
||||
|
||||
$jobs += Start-Job -FilePath .\0_Create_Storage_Account.ps1 `
|
||||
-ArgumentList $affinityGroupName, `
|
||||
$storageAccount, `
|
||||
$subscriptionName
|
||||
|
||||
Write-Progress -Activity "Submitting storage account for creation"
|
||||
}
|
||||
Write-Progress "Submitting storage account for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for storage account creation jobs to finish..." -Completed
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Select the image created in previous step. Image is used to provision
|
||||
## cluster nodes.
|
||||
###########################################################################################################
|
||||
$image = Get-AzureVMImage -ImageName $imageName
|
||||
|
||||
|
||||
###########################################################################################################
|
||||
## Create the virtual machines for the cluster nodes
|
||||
### Write the hostscript and hosts file
|
||||
### Set static IP on the VM
|
||||
### First iteration will create the inital vm in each cloud service.
|
||||
### First vm in each cloud service will create the cloud service and require longer locks.
|
||||
###########################################################################################################
|
||||
$countStorageAccount = $storageAccountList.Length
|
||||
$countService = 1
|
||||
$countVM = 1
|
||||
$storageAccountIndex = 0
|
||||
|
||||
for ($countVM = 1; $countVM -le $numCloudServices; $countVM++)
|
||||
{
|
||||
# if ($countService -gt $numCloudServices) {$countService = 1}
|
||||
if ($storageAccountIndex -eq $countStorageAccount) {$storageAccountIndex = 0}
|
||||
|
||||
.\0_Create_Cluster_Nodes.ps1 -imageName $imageName `
|
||||
-adminUserName $adminUserName `
|
||||
-adminPassword $adminPassword `
|
||||
-instanceSize $instanceSize `
|
||||
-diskSizeInGB $diskSizeInGB `
|
||||
-numofDisks $numOfDisks `
|
||||
-vmNamePrefix $vmNamePrefix `
|
||||
-cloudServicePrefix $cloudServicePrefix `
|
||||
-affinityGroupName $affinityGroupName `
|
||||
-virtualNetworkName $virtualNetworkName `
|
||||
-virtualSubnetname $virtualSubnetname `
|
||||
-storageAccountName $storageAccountName `
|
||||
-storageAccountList $storageAccountList `
|
||||
-hostsfile $hostsfile `
|
||||
-hostscript $hostscript `
|
||||
-subscriptionName $subscriptionName `
|
||||
-countService $countService `
|
||||
-countVM $countVM `
|
||||
-storageAccountIndex $storageAccountIndex
|
||||
|
||||
$countService++
|
||||
$storageAccountIndex++
|
||||
}
|
||||
|
||||
###########################################################################################################
|
||||
## Create the virtual machines for the cluster nodes
|
||||
### Write the hostscript and hosts file
|
||||
### Set static IP on the VM
|
||||
### First iteration will create the inital vm in each cloud service.
|
||||
### First vm in each cloud service will create the cloud service and require longer locks.
|
||||
###########################################################################################################
|
||||
$countStorageAccount = $storageAccountList.Length
|
||||
$countService = 1
|
||||
$countVM = $numCloudServices + 1
|
||||
#$storageAccountIndex = 0
|
||||
|
||||
for ($countVM = 1; $countVM -le $numNodes; $countVM++)
|
||||
{
|
||||
if ($countService -gt $numCloudServices) {$countService = 1}
|
||||
if ($storageAccountIndex -eq $countStorageAccount) {$storageAccountIndex = 0}
|
||||
|
||||
$jobs += Start-Job -FilePath .\0_Create_Cluster_Nodes.ps1 `
|
||||
-ArgumentList $imageName, `
|
||||
$adminUserName, `
|
||||
$adminPassword, `
|
||||
$instanceSize, `
|
||||
$diskSizeInGB, `
|
||||
$numOfDisks, `
|
||||
$vmNamePrefix, `
|
||||
$cloudServicePrefix, `
|
||||
$affinityGroupName, `
|
||||
$virtualNetworkName, `
|
||||
$virtualSubnetname, `
|
||||
$storageAccountName, `
|
||||
$storageAccountList, `
|
||||
$hostsfile, `
|
||||
$hostscript, `
|
||||
$subscriptionName, `
|
||||
$countService, `
|
||||
$countVM, `
|
||||
$storageAccountIndex
|
||||
|
||||
|
||||
$countService++
|
||||
$storageAccountIndex++
|
||||
|
||||
Write-Progress -Activity "Submitting virtual machine for creation"
|
||||
}
|
||||
Write-Progress "Submitting virtual machine for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -Completed
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
<!--Set Cluster Variables-->
|
||||
|
||||
<Cluster>
|
||||
<!--Set the subscription-->
|
||||
<SubscriptionName>MTC Workshop</SubscriptionName>
|
||||
|
||||
<!--Affinity Group Settings-->
|
||||
<!--affinityGroupLocation is the location where the cluster will deploy. East US, West US, East Asia, Southeast Asia, North Europe, West Europe are examples of valid locations-->
|
||||
<!--affinityGroupName must be globally unique-->
|
||||
<affinityGroupLocation>West US</affinityGroupLocation>
|
||||
<affinityGroupName>clb0West</affinityGroupName>
|
||||
<affinityGroupDescription>Affinity Group DDP Sample</affinityGroupDescription>
|
||||
<affinityGroupLabel>AG for DDP Sample</affinityGroupLabel>
|
||||
|
||||
<!--Virtual Network Settings-->
|
||||
<!--If the virtual network already exists, input the vnet settings -->
|
||||
<!--If the virtual networ doesn't exist, the scripts will generate the vnet based on the settings -->
|
||||
<!--Valid address space settings include 192.168.0.0, 10.0.0.0 and 172.16.0.0 -->
|
||||
<!--Network faq is here http://msdn.microsoft.com/en-us/library/windowsazure/dn133803.aspx -->
|
||||
<virtualNetworkName>clb01</virtualNetworkName>
|
||||
<virtualNetworkAddressSpace>172.16.0.0</virtualNetworkAddressSpace>
|
||||
<virtualNetworkCIDR>17</virtualNetworkCIDR>
|
||||
<virtualSubnetname>App</virtualSubnetname>
|
||||
<subnetAddressSpace>172.16.0.0</subnetAddressSpace>
|
||||
<subnetCIDR>17</subnetCIDR>
|
||||
|
||||
<!--Storage Account Settings-->
|
||||
<!--All storage account names must be globally unique-->
|
||||
<storageAccountName>clb0</storageAccountName>
|
||||
<storageAccountList>
|
||||
<Name>clb001</Name>
|
||||
<Name>clb002</Name>
|
||||
</storageAccountList>
|
||||
|
||||
<!--Virtual Machine Settings-->
|
||||
<!--cloudServicePrefix must be globally unique-->
|
||||
<adminUserName>clusteradmin</adminUserName>
|
||||
<adminPassword>Password.1</adminPassword>
|
||||
<vmNamePrefix>clb0</vmNamePrefix>
|
||||
<cloudServicePrefix>clb0</cloudServicePrefix>
|
||||
|
||||
<!--do not change the hosts/hostscript names-->
|
||||
<hostsfile>.\hosts.txt</hostsfile>
|
||||
<hostscript>.\hostscript.sh</hostscript>
|
||||
|
||||
<!--Set Management Node Variables-->
|
||||
<ManagementNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
<diskSizeInGB>500</diskSizeInGB>
|
||||
<numOfDisks>2</numOfDisks>
|
||||
<installerPort>8080</installerPort>
|
||||
</ManagementNode>
|
||||
|
||||
<!--Set Clone Node Variable-->
|
||||
<CloneNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
</CloneNode>
|
||||
|
||||
<!--Set Clone Image Variables-->
|
||||
<CloneImage>
|
||||
<vmName>clb0c</vmName>
|
||||
<cloneimageName>clb0c</cloneimageName>
|
||||
<cloneimageLabel>Test 0 Clone</cloneimageLabel>
|
||||
</CloneImage>
|
||||
|
||||
<!--Set Cluster Nodes Variables-->
|
||||
<ClusterNodes>
|
||||
<instanceSize>A5</instanceSize>
|
||||
<diskSizeInGB>1000</diskSizeInGB>
|
||||
<numOfDisks>4</numOfDisks>
|
||||
<vmNamePrefix>clb0</vmNamePrefix>
|
||||
<cloudServicePrefix>clb0</cloudServicePrefix>
|
||||
<numNodes>8</numNodes>
|
||||
<numCloudServices>2</numCloudServices>
|
||||
</ClusterNodes>
|
||||
</Cluster>
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
<!--Set Cluster Variables-->
|
||||
|
||||
<Cluster>
|
||||
<!--Set the subscription-->
|
||||
<SubscriptionName>MTC Workshop</SubscriptionName>
|
||||
|
||||
<!--Affinity Group Settings-->
|
||||
<!--affinityGroupLocation is the location where the cluster will deploy. East US, West US, East Asia, Southeast Asia, North Europe, West Europe are examples of valid locations-->
|
||||
<!--affinityGroupName must be globally unique-->
|
||||
<affinityGroupLocation>West US</affinityGroupLocation>
|
||||
<affinityGroupName>ddptest3</affinityGroupName>
|
||||
<affinityGroupDescription>Affinity Group DDP Sample</affinityGroupDescription>
|
||||
<affinityGroupLabel>AG for DDP Sample</affinityGroupLabel>
|
||||
|
||||
<!--Virtual Network Settings-->
|
||||
<!--If the virtual network already exists, input the vnet settings -->
|
||||
<!--If the virtual networ doesn't exist, the scripts will generate the vnet based on the settings -->
|
||||
<!--Valid address space settings include 192.168.0.0, 10.0.0.0 and 172.16.0.0 -->
|
||||
<!--Network faq is here http://msdn.microsoft.com/en-us/library/windowsazure/dn133803.aspx -->
|
||||
<virtualNetworkName>ddptestv4</virtualNetworkName>
|
||||
<virtualNetworkAddressSpace>172.16.0.0</virtualNetworkAddressSpace>
|
||||
<virtualNetworkCIDR>17</virtualNetworkCIDR>
|
||||
<virtualSubnetname>App</virtualSubnetname>
|
||||
<subnetAddressSpace>172.16.0.0</subnetAddressSpace>
|
||||
<subnetCIDR>17</subnetCIDR>
|
||||
|
||||
<!--Storage Account Settings-->
|
||||
<!--All storage account names must be globally unique-->
|
||||
<storageAccountName>ddptest3</storageAccountName>
|
||||
<storageAccountList>
|
||||
<Name>ddptest31</Name>
|
||||
<Name>ddptest32</Name>
|
||||
</storageAccountList>
|
||||
|
||||
<!--Virtual Machine Settings-->
|
||||
<!--cloudServicePrefix must be globally unique-->
|
||||
<adminUserName>clusteradmin</adminUserName>
|
||||
<adminPassword>Password.1</adminPassword>
|
||||
<vmNamePrefix>ddptest3a</vmNamePrefix>
|
||||
<cloudServicePrefix>ddptest3</cloudServicePrefix>
|
||||
|
||||
<!--do not change the hosts/hostscript names-->
|
||||
<hostsfile>.\hosts.txt</hostsfile>
|
||||
<hostscript>.\hostscript.sh</hostscript>
|
||||
|
||||
<!--Set Management Node Variables-->
|
||||
<ManagementNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
<diskSizeInGB>100</diskSizeInGB>
|
||||
<numOfDisks>2</numOfDisks>
|
||||
<installerPort>8080</installerPort>
|
||||
</ManagementNode>
|
||||
|
||||
<!--Set Clone Node Variable-->
|
||||
<CloneNode>
|
||||
<galleryimageName>OpenLogic</galleryimageName>
|
||||
<instanceSize>A7</instanceSize>
|
||||
</CloneNode>
|
||||
|
||||
<!--Set Clone Image Variables-->
|
||||
<CloneImage>
|
||||
<vmName>ddptest3ac</vmName>
|
||||
<cloneimageName>ddptest3c</cloneimageName>
|
||||
<cloneimageLabel>Test 0 Clone</cloneimageLabel>
|
||||
</CloneImage>
|
||||
|
||||
<!--Set Cluster Nodes Variables-->
|
||||
<ClusterNodes>
|
||||
<instanceSize>A7</instanceSize>
|
||||
<diskSizeInGB>100</diskSizeInGB>
|
||||
<numOfDisks>4</numOfDisks>
|
||||
<vmNamePrefix>ddptest3</vmNamePrefix>
|
||||
<cloudServicePrefix>ddptest3</cloudServicePrefix>
|
||||
<numNodes>8</numNodes>
|
||||
<numCloudServices>2</numCloudServices>
|
||||
</ClusterNodes>
|
||||
</Cluster>
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
$subscriptionName = "MTC Workshop"
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample lara.xml"
|
||||
|
||||
#################################################################################
|
||||
## Management Node
|
||||
#################################################################################
|
||||
$subscriptionName = "MTC Workshop"
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample lara.xml"
|
||||
|
||||
.\1_Management_Node.ps1 -imageName $ddpconfig.Cluster.ManagementNode.galleryimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword`
|
||||
-instanceSize $ddpconfig.Cluster.ManagementNode.instanceSize`
|
||||
-diskSizeInGB $ddpconfig.Cluster.ManagementNode.diskSizeInGB `
|
||||
-numOfDisks $ddpconfig.Cluster.ManagementNode.numOfDisks `
|
||||
-vmNamePrefix $ddpconfig.Cluster.vmNamePrefix `
|
||||
-cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-storageAccountList $ddpconfig.Cluster.storageAccountList.Name `
|
||||
-affinityGroupLocation $ddpconfig.Cluster.affinityGroupLocation `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-affinityGroupDescription $ddpconfig.Cluster.affinityGroupDescription `
|
||||
-affinityGroupLabel $ddpconfig.Cluster.affinityGroupLabel `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualNetworkAddressSpace $ddpconfig.Cluster.virtualNetworkAddressSpace `
|
||||
-virtualNetworkCIDR $ddpconfig.Cluster.VirtualNetworkCIDR `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-subnetAddressSpace $ddpconfig.Cluster.SubnetAddressSpace `
|
||||
-subnetCIDR $ddpconfig.Cluster.SubnetCIDR `
|
||||
-installerPort 7180 `
|
||||
-hostscript $ddpconfig.Cluster.hostscript `
|
||||
-hostsfile $ddpconfig.Cluster.hostsfile `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Clone Node
|
||||
## Create the clone node used for generating the data nodes and name nodes.
|
||||
#################################################################################
|
||||
$subscriptionName = "MTC Workshop"
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample lara.xml"
|
||||
|
||||
.\2_Clone_Node.ps1 -imageName $ddpconfig.Cluster.CloneNode.galleryimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword `
|
||||
-instanceSize $ddpconfig.Cluster.CloneNode.instanceSize `
|
||||
-diskSizeInGB 0 `
|
||||
-numOfDisks 0 `
|
||||
-vmNamePrefix $ddpconfig.Cluster.vmNamePrefix `
|
||||
-cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Manual Updates
|
||||
#################################################################################
|
||||
## Before you continue, complete the manual updates from the documentation to prepare the
|
||||
## cluster nodes.
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Capture the image
|
||||
#################################################################################
|
||||
$subscriptionName = "MTC Workshop"
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample lara.xml"
|
||||
|
||||
$vmName = $ddpconfig.Cluster.vmNamePrefix + "c"
|
||||
|
||||
.\3_Capture_Image.ps1 -cloudServiceName $ddpconfig.Cluster.cloudServicePrefix `
|
||||
-vmName $vmName `
|
||||
-imageName $ddpconfig.Cluster.CloneImage.cloneimageName `
|
||||
-imageLabel $ddpconfig.Cluster.CloneImage.cloneimageLabel `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
#################################################################################
|
||||
## Create the worker nodes
|
||||
#################################################################################
|
||||
$subscriptionName = "MTC Workshop"
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample lara.xml"
|
||||
|
||||
.\4_Cluster_Nodes.ps1 -imageName $ddpconfig.Cluster.CloneImage.cloneimageName `
|
||||
-adminUserName $ddpconfig.Cluster.adminUserName `
|
||||
-adminPassword $ddpconfig.Cluster.adminPassword `
|
||||
-instanceSize $ddpconfig.Cluster.ClusterNodes.instanceSize `
|
||||
-diskSizeInGB $ddpconfig.Cluster.ClusterNodes.diskSizeInGB `
|
||||
-numOfDisks $ddpconfig.Cluster.ClusterNodes.numOfDisks `
|
||||
-vmNamePrefix $ddpconfig.Cluster.ClusterNodes.vmNamePrefix `
|
||||
-cloudServicePrefix $ddpconfig.Cluster.ClusterNodes.cloudServicePrefix `
|
||||
-numCloudServices $ddpconfig.Cluster.ClusterNodes.numCloudServices `
|
||||
-numNodes $ddpconfig.Cluster.ClusterNodes.numNodes `
|
||||
-affinityGroupName $ddpconfig.Cluster.affinityGroupName `
|
||||
-virtualNetworkName $ddpconfig.Cluster.virtualNetworkName `
|
||||
-virtualSubnetname $ddpconfig.Cluster.virtualSubnetname `
|
||||
-storageAccountName $ddpconfig.Cluster.storageAccountName `
|
||||
-storageAccountList $ddpconfig.Cluster.storageAccountList.Name `
|
||||
-hostsfile $ddpconfig.Cluster.hostsfile `
|
||||
-hostscript $ddpconfig.Cluster.hostscript `
|
||||
-subscriptionName $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
Sat Jun 28 2014 15:18:21 GMT-0500 (Central Daylight Time):
|
||||
[Error: Missing attribute value]
|
||||
Error: Missing attribute value
|
||||
at XMLFragment.attribute (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\node_modules\xmlbuilder\lib\XMLFragment.js:268:15)
|
||||
at C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\node_modules\azure\node_modules\azure-mgmt\lib\models\servicemanagementserialize.js:540:35
|
||||
at Array.forEach (native)
|
||||
at ServiceManagementSerialize.buildNetworkConfiguration (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\node_modules\azure\node_modules\azure-mgmt\lib\models\servicemanagementserialize.js:536:38)
|
||||
at Object.ServiceManagementService.setNetworkConfig (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\node_modules\azure\node_modules\azure-mgmt\lib\servicemanagementservice.js:2114:32)
|
||||
at Object.exports.doServiceManagementOperation (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\lib\util\utils.js:218:22)
|
||||
at setNetworkConfig (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\lib\commands\asm\network.js:1029:11)
|
||||
at __$createVNet (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\lib\commands\asm\network.js:791:30)
|
||||
at __tryCatch (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\node_modules\streamline\lib\callbacks\runtime.js:141:4)
|
||||
at ___ (C:\Program Files (x86)\Microsoft SDKs\Windows Azure\CLI\lib\commands\asm\network.js:789:141)
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
$jobs.ChildJobs |where {$_.state -eq "Failed" -and $_.Id -gt 200} | Receive-Job -Name $_.JobId -Keep
|
||||
|
||||
Receive-Job -Name Job203 -Keep
|
||||
Receive-Job -
|
||||
Select-AzureSubscription "MTC Workshop"
|
||||
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
|
||||
$jobs.ChildJobs |where {$_.state -eq "Failed" -and $_.Id -gt 200}| Format-List -Property *
|
||||
|
||||
Get-Job | Format-List -Property *
|
||||
$jobs.ChildJobs |Select -ExpandProperty $_.Command
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample 2.xml"
|
||||
|
||||
$imageName = $ddpconfig.Cluster.CloneImage.cloneimageName
|
||||
$adminUserName = $ddpconfig.Cluster.adminUserName
|
||||
$adminPassword = $ddpconfig.Cluster.adminPassword
|
||||
$instanceSize = $ddpconfig.Cluster.ClusterNodes.instanceSize
|
||||
$diskSizeInGB= $ddpconfig.Cluster.ClusterNodes.diskSizeInGB
|
||||
$numOfDisks= $ddpconfig.Cluster.ClusterNodes.numOfDisks
|
||||
$vmNamePrefix= $ddpconfig.Cluster.ClusterNodes.vmNamePrefix
|
||||
$cloudServicePrefix= $ddpconfig.Cluster.ClusterNodes.cloudServicePrefix
|
||||
$numCloudServices= $ddpconfig.Cluster.ClusterNodes.numCloudServices
|
||||
$numNodes= $ddpconfig.Cluster.ClusterNodes.numNodes
|
||||
$affinityGroupName= $ddpconfig.Cluster.affinityGroupName
|
||||
$virtualNetworkName= $ddpconfig.Cluster.virtualNetworkName
|
||||
$virtualSubnetname= $ddpconfig.Cluster.virtualSubnetname
|
||||
$storageAccountName= $ddpconfig.Cluster.storageAccountName
|
||||
$storageAccountList= $ddpconfig.Cluster.storageAccountList.Name
|
||||
$hostsfile= $ddpconfig.Cluster.hostsfile
|
||||
$hostscript= $ddpconfig.Cluster.hostscript
|
||||
$subscriptionName= $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
# Following modifies the Write-Output behavior to turn the messages on globally for this session
|
||||
$VerbosePreference = "Continue"
|
||||
$DebugPreference = "Continue"
|
||||
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
azure account set $subscriptionName
|
||||
|
||||
|
||||
$countStorageAccount = $storageAccountList.Count
|
||||
$countService = 1
|
||||
$countVM = 1
|
||||
$storageAccountIndex = 0
|
||||
$jobs = @()
|
||||
for ($countVM = 1; $countVM -le $numNodes; $countVM++)
|
||||
{
|
||||
if ($countService -gt [int]$numCloudServices) {$countService = 1}
|
||||
if ($storageAccountIndex -eq $countStorageAccount) {$storageAccountIndex = 0}
|
||||
|
||||
$cloudServiceName = "$cloudServicePrefix$countService"
|
||||
$vmName = "$vmNamePrefix$countVM"
|
||||
$storageAccount = $storageAccountList[$storageAccountIndex]
|
||||
|
||||
|
||||
$jobs += Start-Job -FilePath "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH\0_Create_Cluster_Nodes.ps1" `
|
||||
-ArgumentList "ncdv3c", `
|
||||
"clusteradmin", `
|
||||
"Password.1", `
|
||||
"extralarge", `
|
||||
10, `
|
||||
1, `
|
||||
$vmName, `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
"App", `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
".\hosts", `
|
||||
".\hostscript.sh", `
|
||||
"MTC Workshop"
|
||||
|
||||
$countService++
|
||||
$storageAccountIndex++
|
||||
|
||||
Write-Progress -Activity "Submitting machine for creation" -Status $vmName -PercentComplete ($countVM / $numNodes * 100)
|
||||
}
|
||||
Write-Progress "Submitting virtual machine for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -Completed
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample 2.xml"
|
||||
|
||||
$imageName = $ddpconfig.Cluster.CloneImage.cloneimageName
|
||||
$adminUserName = $ddpconfig.Cluster.adminUserName
|
||||
$adminPassword = $ddpconfig.Cluster.adminPassword
|
||||
$instanceSize = $ddpconfig.Cluster.ClusterNodes.instanceSize
|
||||
$diskSizeInGB= $ddpconfig.Cluster.ClusterNodes.diskSizeInGB
|
||||
$numOfDisks= $ddpconfig.Cluster.ClusterNodes.numOfDisks
|
||||
$vmNamePrefix= $ddpconfig.Cluster.ClusterNodes.vmNamePrefix
|
||||
$cloudServicePrefix= $ddpconfig.Cluster.ClusterNodes.cloudServicePrefix
|
||||
$numCloudServices= $ddpconfig.Cluster.ClusterNodes.numCloudServices
|
||||
$numNodes= $ddpconfig.Cluster.ClusterNodes.numNodes
|
||||
$affinityGroupName= $ddpconfig.Cluster.affinityGroupName
|
||||
$virtualNetworkName= $ddpconfig.Cluster.virtualNetworkName
|
||||
$virtualSubnetname= $ddpconfig.Cluster.virtualSubnetname
|
||||
$storageAccountName= $ddpconfig.Cluster.storageAccountName
|
||||
$storageAccountList= $ddpconfig.Cluster.storageAccountList.Name
|
||||
$hostsfile= $ddpconfig.Cluster.hostsfile
|
||||
$hostscript= $ddpconfig.Cluster.hostscript
|
||||
$subscriptionName= $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
# Following modifies the Write-Output behavior to turn the messages on globally for this session
|
||||
$VerbosePreference = "Continue"
|
||||
$DebugPreference = "Continue"
|
||||
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
azure account set $subscriptionName
|
||||
|
||||
|
||||
$countStorageAccount = $storageAccountList.Count
|
||||
$countService = 1
|
||||
$countVM = 1
|
||||
$storageAccountIndex = 0
|
||||
$jobs = @()
|
||||
for ($countVM = 1; $countVM -le $numNodes; $countVM++)
|
||||
{
|
||||
if ($countService -gt [int]$numCloudServices) {$countService = 1}
|
||||
if ($storageAccountIndex -eq $countStorageAccount) {$storageAccountIndex = 0}
|
||||
|
||||
$cloudServiceName = "$cloudServicePrefix$countService"
|
||||
$vmName = "$vmNamePrefix$countVM"
|
||||
$storageAccount = $storageAccountList[$storageAccountIndex]
|
||||
$path = Resolve-Path ".\" + "\0_Create_Cluster_Nodes.ps1"
|
||||
|
||||
|
||||
$jobs += Start-Job -FilePath "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH\0_Create_Cluster_Nodes.ps1" `
|
||||
-ArgumentList "ncdv3c", `
|
||||
"clusteradmin", `
|
||||
"Password.1", `
|
||||
"extralarge", `
|
||||
10, `
|
||||
1, `
|
||||
"ncdlara$countVM", `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
"App", `
|
||||
"ncdv3", `
|
||||
"ncdv3", `
|
||||
".\hosts", `
|
||||
".\hostscript.sh", `
|
||||
"MTC Workshop", `
|
||||
$path
|
||||
|
||||
$countService++
|
||||
$storageAccountIndex++
|
||||
|
||||
Write-Progress -Activity "Submitting machine for creation" -Status $vmName -PercentComplete ($countVM / $numNodes * 100)
|
||||
}
|
||||
Write-Progress "Submitting virtual machine for creation" -Completed
|
||||
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -PercentComplete -1
|
||||
$jobs | Wait-Job | Out-Null
|
||||
Write-Progress "Waiting for virtual machine creation jobs to finish..." -Completed
|
||||
|
||||
|
||||
$jobs.ChildJobs
|
||||
|
||||
Receive-Job -Name Job7 -Keep
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
|
||||
cd "C:\Users\larar\Source\Workspaces\Distributed Data Cluster on Azure\GitHubScripts\PoSH"
|
||||
[xml]$ddpconfig = Get-Content ".\ClusterConfig Sample.xml"
|
||||
|
||||
$imageName = $ddpconfig.Cluster.CloneImage.cloneimageName
|
||||
$adminUserName = $ddpconfig.Cluster.adminUserName
|
||||
$adminPassword = $ddpconfig.Cluster.adminPassword
|
||||
$instanceSize = $ddpconfig.Cluster.ClusterNodes.instanceSize
|
||||
$diskSizeInGB= $ddpconfig.Cluster.ClusterNodes.diskSizeInGB
|
||||
$numOfDisks= $ddpconfig.Cluster.ClusterNodes.numOfDisks
|
||||
$vmNamePrefix= $ddpconfig.Cluster.ClusterNodes.vmNamePrefix
|
||||
$cloudServicePrefix= $ddpconfig.Cluster.ClusterNodes.cloudServicePrefix
|
||||
$numCloudServices= $ddpconfig.Cluster.ClusterNodes.numCloudServices
|
||||
$numNodes= $ddpconfig.Cluster.ClusterNodes.numNodes
|
||||
$affinityGroupName= $ddpconfig.Cluster.affinityGroupName
|
||||
$virtualNetworkName= $ddpconfig.Cluster.virtualNetworkName
|
||||
$virtualSubnetname= $ddpconfig.Cluster.virtualSubnetname
|
||||
$storageAccountName= $ddpconfig.Cluster.storageAccountName
|
||||
$storageAccountList= $ddpconfig.Cluster.storageAccountList.Name
|
||||
$hostsfile= $ddpconfig.Cluster.hostsfile
|
||||
$hostscript= $ddpconfig.Cluster.hostscript
|
||||
$subscriptionName= $ddpconfig.Cluster.SubscriptionName
|
||||
|
||||
|
||||
Select-AzureSubscription -SubscriptionName $subscriptionName
|
||||
azure account set $subscriptionName
|
||||
|
||||
$countStorageAccount = $storageAccountList.Count
|
||||
$countService = 2
|
||||
$countVM = 1
|
||||
[int]$storageAccountIndex = 0
|
||||
|
||||
$cloudServiceName = $cloudServicePrefix+[string]$countService
|
||||
$vmName = $vmNamePrefix+[string]$countVM
|
||||
$storageAccount = $storageAccountList[$storageAccountIndex]
|
||||
|
||||
.\0_Create_Cluster_Nodes.ps1 `
|
||||
-imageName "ncdv3c" `
|
||||
-adminUserName "clusteradmin" `
|
||||
-adminPassword "Password.1" `
|
||||
-instanceSize "ExtraLarge" `
|
||||
-diskSizeinGB 10 `
|
||||
-numOfDisks 1 `
|
||||
-vmName $vmName `
|
||||
-cloudServiceName "ncdv3" `
|
||||
-affinityGroupName "ncdv3" `
|
||||
-virtualNetworkName "ncdv3" `
|
||||
-virtualSubnetName "App" `
|
||||
-storageAccountName "ncdv3" `
|
||||
-storageAccount "ncdv3" `
|
||||
-hostsfile ".\hostsfile" `
|
||||
-hostscript ".\hostscript" `
|
||||
-subscriptionName "MTC Workshop"
|
||||
|
||||
|
||||
.\0_Create_Cluster_Nodes.ps1
|
||||
"ncdv3c" `
|
||||
"clusteradmin" `
|
||||
"Password.1" `
|
||||
"extralarge" `
|
||||
10 `
|
||||
1 `
|
||||
$vmName `
|
||||
"ncdv3" `
|
||||
"ncdv3" `
|
||||
"ncdv3" `
|
||||
"App" `
|
||||
"ncdv3" `
|
||||
"ncdv3" `
|
||||
".\hosts" `
|
||||
".\hostscript.sh" `
|
||||
"MTC Workshop"
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/bash
|
||||
######## Mounting Drives ###############
|
||||
# Setting variables to allow quick customisation:
|
||||
FISYS="ext4"
|
||||
mtoptions="noatime,nodiratime"
|
||||
|
||||
echo "Attaching disks to $HOSTNAME"
|
||||
|
||||
cp /etc/fstab /etc/old.fstab`date +%d-%m-%y---%H-%M-%S`
|
||||
dmesg | grep -e "\[sd[a-z]" | awk '{print $3;}' | sort -u > /tmp/diskdeviceoutput
|
||||
|
||||
d=0
|
||||
for i in {b..z}
|
||||
do
|
||||
sdx=$(cat /tmp/diskdeviceoutput | grep -o sd$i)
|
||||
|
||||
if [[ $sdx ]]
|
||||
then
|
||||
if cat /etc/mtab | grep $sdx
|
||||
then
|
||||
unset $sdx
|
||||
fi
|
||||
|
||||
echo "Mounting $sdx"
|
||||
sleep 3
|
||||
|
||||
echo "y
|
||||
" | mkfs.$FISYS -m 1 -O dir_index,extent,sparse_super -E lazy_itable_init /dev/$sdx > /var/log/mkfs.$sdx
|
||||
mkdir -p /drive/$d
|
||||
mount -o $mtoptions /dev/$sdx /drive/$d
|
||||
echo "/dev/$sdx /drive/$((d++)) $FISYS $mtoptions 0 0" >> /etc/fstab
|
||||
|
||||
else
|
||||
echo " "
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
echo "Checking the disks:"
|
||||
fdisk -l | grep /dev/sd > /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo "Checking the mounts:"
|
||||
mount -l | grep /drive >> /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo "Checking /etc/fstab for the mounts:"
|
||||
cat /etc/fstab | grep /drive >> /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo " "
|
||||
|
||||
echo "The above commands are written to log file /var/log/automountandformat.log."
|
||||
sleep 3
|
||||
echo "Disks are attached to $HOSTNAME"
|
||||
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#!/bin/bash
|
||||
#This script reads the hosts file and merges its content with /etc/hosts
|
||||
|
||||
#check to make sure hosts file exists
|
||||
echo "hosts file name is:/root/hosts.txt"
|
||||
|
||||
if [ -e /etc/hosts.bak ]
|
||||
then
|
||||
rm /etc/hosts.bak
|
||||
fi
|
||||
|
||||
#backup the hosts files
|
||||
cp /etc/hosts /etc/hosts.bak
|
||||
echo "########################################################before updating"
|
||||
cat /etc/hosts
|
||||
|
||||
if [ -e /root/hosts.txt ]
|
||||
then
|
||||
#read the hosts file for ip addresses. Look for that ip address in /etc/hosts file and remove it.
|
||||
for line in `awk '{print $1}' < /root/hosts.txt`; do
|
||||
echo $line
|
||||
sed -i "/$line/d" /etc/hosts
|
||||
done
|
||||
|
||||
cat /root/hosts.txt >> /etc/hosts
|
||||
|
||||
echo "######################################################################After updating"
|
||||
cat /etc/hosts
|
||||
else
|
||||
printf "File /root/hosts.txt does not exist\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "#############################################################################################"
|
||||
echo "Script finished successfully"
|
||||
echo "#############################################################################################"
|
||||
exit 0
|
||||
Двоичный файл не отображается.
|
|
@ -0,0 +1,34 @@
|
|||
#!/bin/bash
|
||||
#Capture the clone as an image
|
||||
|
||||
source ./clustersetup.sh
|
||||
|
||||
#This script requires jq json processor
|
||||
#check to see if jq is available and download it if necessary
|
||||
result=$(which jq)
|
||||
if [[ -z $result ]]; then
|
||||
wget http://stedolan.github.io/jq/download/linux64/jq
|
||||
sudo chmod +x jq
|
||||
sudo mv jq /usr/local/bin
|
||||
else
|
||||
printf "jq json processor was found\n"
|
||||
fi
|
||||
|
||||
(which jq) || { echo "jq json processor is not available"; exit 1; }
|
||||
|
||||
##############################################################
|
||||
## Capture the image of the clone virtual machine
|
||||
##############################################################
|
||||
#Check to see if the cloud servide already exists
|
||||
result=$(azure vm image show $nodeImageName --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
vmSource=$vmNamePrefix"c"
|
||||
printf "Capturing image $nodeImageName from virtual machine $vmSource\n"
|
||||
azure vm shutdown $nodeImageName
|
||||
azure vm capture --delete --label $cloneImageLabel $vmSource $cloneImageName
|
||||
else
|
||||
printf "Image $nodeImageName exists\n"
|
||||
fi
|
||||
|
||||
printf "######################################## Clone Image Details #######################################\n"
|
||||
azure vm image show $nodeImageName --json
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
#This settings file stores all the settings related to CDH cluster you are setting up
|
||||
|
||||
#########Start Cluster Settings
|
||||
#Affinty group helps you keep your storage and compute in the same region
|
||||
#Identify the region where affinity group should be created.
|
||||
#choices are valid values are "East US", "West US", "East Asia", "Southeast Asia", "North Europe", "West Europe"
|
||||
export affinityGroupName=
|
||||
export affinityGroupLocation=""
|
||||
export affinityGroupLabel=""
|
||||
export affinityGroupDescription=""
|
||||
|
||||
#setting related to virtual network
|
||||
#address space allows 192.168.0.0, 10.0.0.0 and 172.16.0.0 ip address ranges
|
||||
#virtual network faq is here http://msdn.microsoft.com/en-us/library/windowsazure/dn133803.aspx
|
||||
export vnetName=
|
||||
export vnetAddressSpace=172.16.0.0
|
||||
export vnetCidr=24
|
||||
export subnetName=App
|
||||
export subnetAddressSpace=172.16.0.0
|
||||
export subnetCidr=24
|
||||
|
||||
#storage account settings
|
||||
#name of the primary storage account for the management node, images, and data node OS disks.
|
||||
#list the array of storage accounts to store the data disks for the cluster nodes
|
||||
export storageAccountName=
|
||||
export storageAccountList=()
|
||||
|
||||
#cloud service settings
|
||||
#Prefix for all cloud services. This will also be used as the name of the primary cloud service.
|
||||
export cloudServicePrefix=
|
||||
|
||||
#virtual machine settings
|
||||
export vmNamePrefix=
|
||||
export adminUserName=
|
||||
export adminPassword=
|
||||
|
||||
#This script will be generated and it will be used to mount data drives in each node in the cluster. It will also copy /etc/hosts file to each node
|
||||
mntscript="hostscript.sh"
|
||||
#This file will generate hosts file that can be appended to /etc/hosts on each node.
|
||||
hostsfile="hosts.txt"
|
||||
|
||||
#########End Cluster Settings
|
||||
|
||||
#########Start Management Node and Clone Node Settings
|
||||
#Name of the image you will use to create your management node and clone node virtual machines
|
||||
export galleryimageName=5112500ae3b842c8b9c604889f8753c3__OpenLogic-CentOS-65-20140415
|
||||
|
||||
#Size of the Virtual machine. Valid sizes are extrasmall, small, medium, large, extralarge, a5, a6, a7
|
||||
#we recommend extra large or higher for the cluster nodes
|
||||
export instanceSize=
|
||||
#endpoint port to open for software installers on the Management Node (ie Amabari, Cloudera Manager)
|
||||
export installerport=
|
||||
|
||||
#Size of the data disk you want to attach to the Management Node. You will typically attach at least 1 disk.
|
||||
#Number of disks you want to attach. Small VM can have 2 disks, medium can have 4, large can have 8 and extralarge can have 8 data disks.
|
||||
export diskSizeInGB=
|
||||
export numOfDisks=
|
||||
|
||||
#########End Management Node and Clone Node Settings
|
||||
|
||||
#########Start Clone Image Settings
|
||||
|
||||
#Name and label of the custom image you will use to create your cluster nodes
|
||||
export cloneImageName=
|
||||
export cloneImageLabel=
|
||||
|
||||
#########End Clone Image Settings
|
||||
|
||||
#########Start Cluster Node Settings
|
||||
|
||||
#These settings are for nodes in the cluster
|
||||
#Number of nodes in your cluster
|
||||
export nodeCount=
|
||||
|
||||
#Number of cloud services to create for the cluster nodes. One additional cloud service is created for the management node and clone image.
|
||||
export numCloudServices=
|
||||
|
||||
#Size of the nodes in the cluster. Valid sizes are extrasmall, small, medium, large, extralarge, a5, a6, a7
|
||||
export clusterinstanceSize=
|
||||
|
||||
#Size of the data disk you want to attach to the VM you are creating. You will typically attach at least 1 disk
|
||||
#Number of disks you want to attach. Small VM can have 2 disks, medium can have 4, large can have 8 and extralarge can have 8 data disks
|
||||
export clusterdiskSizeInGB=
|
||||
export clusternumOfDisks=
|
||||
|
||||
export clustervmNamePrefix=
|
||||
export clustercloudServicePrefix=
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
##############################################################
|
||||
#!/bin/bash
|
||||
# Set up the affinity group, storage accounts and virtual network
|
||||
##############################################################
|
||||
# Assign variables from the config file
|
||||
source ./clustersetup.sh
|
||||
|
||||
# Delete the mountscript and hosts files if they exist
|
||||
if [ -e $mntscript ]; then
|
||||
rm $mntscript
|
||||
fi
|
||||
|
||||
if [ -e $hostfile ]; then
|
||||
rm $hostsfile
|
||||
fi
|
||||
|
||||
# This script requires jq json processor
|
||||
# Check to see if jq is available and download it if necessary
|
||||
result=$(which jq)
|
||||
if [[ -z $result ]]; then
|
||||
wget http://stedolan.github.io/jq/download/linux64/jq
|
||||
sudo chmod +x jq
|
||||
sudo mv jq /usr/local/bin
|
||||
else
|
||||
printf "jq json processor was found\n"
|
||||
fi
|
||||
|
||||
(which jq) || { echo "jq json processor is not available"; exit 1; }
|
||||
|
||||
##############################################################
|
||||
## Create affinity group if it does not exist
|
||||
##############################################################
|
||||
printf "affinty group name is %s, affinity group location is %s\n" "$affinityGroupName" "$affinityGroupLocation"
|
||||
|
||||
result=$(azure account affinity-group show "$affinityGroupName" --json | jq '.Name')
|
||||
if [[ -z $result ]]; then
|
||||
(azure account affinity-group create --location "$affinityGroupLocation" --label "$affinityGroupLabel" --description "$affinityGroupDescription" "$affinityGroupName") || { echo "Failed to create affinity group $affinityGroupName"; exit 1; }
|
||||
else
|
||||
echo "affinity group $affinityGroupName exists"
|
||||
fi
|
||||
|
||||
#show the affinity group details
|
||||
printf "######################################## Affinity Group Details #######################################\n"
|
||||
azure account affinity-group show "$affinityGroupName" --json
|
||||
|
||||
##############################################################
|
||||
## Create the primary storage account if it does not exist
|
||||
##############################################################
|
||||
printf "storage account name is %s\n" "$storageAccountName"
|
||||
|
||||
result=$(azure storage account show "$storageAccountName" --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
(azure storage account create --affinity-group "$affinityGroupName" --disable-geoReplication $storageAccountName) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
|
||||
else
|
||||
echo "Storage account $storageAccountName exists"
|
||||
fi
|
||||
|
||||
#show the storage account details
|
||||
printf "######################################## Storage Account Details #######################################\n"
|
||||
azure storage account show "$storageAccountName" --json
|
||||
|
||||
##############################################################
|
||||
## Create data node data disk storage accounts if they do not exist
|
||||
##############################################################
|
||||
for storageAccount in ${storageAccountList[@]}
|
||||
do
|
||||
result=$(azure storage account show "$storageAccount" --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
(azure storage account create --affinity-group "$affinityGroupName" --disable-geoReplication $storageAccount) || { echo "Failed to create storage account $storageAccountName"; exit 1; }
|
||||
else
|
||||
echo "Storage account $storageAccount exists"
|
||||
fi
|
||||
done
|
||||
|
||||
##############################################################
|
||||
## Create virtual network if it does not exist
|
||||
##############################################################
|
||||
printf "virtual network is %s, subnet is %s\n" "$vnetName" "$subnetName"
|
||||
|
||||
result=$(azure network vnet show $vnetName --json | jq '.Name')
|
||||
if [[ -z $result ]]; then
|
||||
printf "Need to create virtual network %s\n" "$vnetName. Please open the Azure Portal to create the virtual network. After the virtual network is created rerun the process."
|
||||
# (azure network vnet create --vnet $vnetName --location "$affinityGroupLocation" --address-space $vnetAddressSpace --cidr $vnetCidr --subnet-name $subnetName --subnet-start-ip $subnetAddressSpace --subnet-cidr $subnetCidr) || { echo "Failed to create virtual network $vnetName"; exit 1;}
|
||||
else
|
||||
printf "Virtual network $virtualNetworkName exists\n"
|
||||
fi
|
||||
|
||||
#show the virtual network details
|
||||
printf "######################################## Virtual Network Details #######################################\n"
|
||||
azure network vnet show "$vnetName" --json
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
##############################################################
|
||||
#!/bin/bash
|
||||
# Create a virtual machine that will be customized to create an image for nodes in the cluster
|
||||
# You will need to make manual updates to the VM before creating an image
|
||||
# createclonenode.sh
|
||||
##############################################################
|
||||
# Assign variables from the config file
|
||||
|
||||
source ./clustersetup.sh
|
||||
|
||||
#This script requires jq json processor
|
||||
#check to see if jq is available and download it if necessary
|
||||
result=$(which jq)
|
||||
if [[ -z $result ]]; then
|
||||
wget http://stedolan.github.io/jq/download/linux64/jq
|
||||
sudo chmod +x jq
|
||||
sudo mv jq /usr/local/bin
|
||||
else
|
||||
printf "jq json processor was found\n"
|
||||
fi
|
||||
|
||||
(which jq) || { echo "jq json processor is not available"; exit 1; }
|
||||
|
||||
|
||||
##############################################################
|
||||
# Create the virtual machine to clone the cluster nodes
|
||||
##############################################################
|
||||
vmName=$vmNamePrefix"c"
|
||||
cloudServiceName=$cloudServicePrefix
|
||||
dnsName=$cloudServiceName".cloudapp.net"
|
||||
|
||||
#Check to see if the cloud servide already exists
|
||||
result=$(azure service show $cloudServiceName --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
|
||||
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
|
||||
else
|
||||
printf "Cloud Service $cloudServiceName exists\n"
|
||||
fi
|
||||
|
||||
#show the cloud service details
|
||||
printf "######################################## Cloud Service Clone Image Details #######################################\n"
|
||||
azure service show "$cloudServiceName" --json
|
||||
|
||||
#Check to see if the VM already exists
|
||||
result=$(azure vm show $vmName --json | jq '.VMName')
|
||||
if [[ -z $result ]]; then
|
||||
|
||||
printf "Virtual machine $vnName does not exist. Creating ...\n"
|
||||
#create the vm and attach data disks
|
||||
(azure vm create --connect --vm-size $instanceSize --vm-name $vmName --ssh 22 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
|
||||
else
|
||||
printf "Virtual machine $vmName exists\n"
|
||||
fi
|
||||
|
||||
printf "######################################## Virtual Machine Clone Image Details #######################################\n"
|
||||
#display the details about the newly created VM
|
||||
azure vm show $vmName --json
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
#!/bin/bash
|
||||
#Creates the nodes in the cluster from a custom image
|
||||
#Expectation is that affinity group, storage account, virtual network have already been setup
|
||||
|
||||
source ./clustersetup.sh
|
||||
|
||||
#This script requires jq json processor
|
||||
#check to see if jq is available and download it if necessary
|
||||
result=$(which jq)
|
||||
if [[ -z $result ]]; then
|
||||
wget http://stedolan.github.io/jq/download/linux64/jq
|
||||
sudo chmod +x jq
|
||||
sudo mv jq /usr/local/bin
|
||||
else
|
||||
printf "jq json processor was found\n"
|
||||
fi
|
||||
|
||||
(which jq) || { echo "jq json processor is not available"; exit 1; }
|
||||
|
||||
##############################################################
|
||||
## Loop to create nodes in the cluster
|
||||
##############################################################
|
||||
countVM=1
|
||||
storageAccountIndex=0
|
||||
countService=1
|
||||
countStorageAccount=${#storageAccountList[@]}
|
||||
while [ $countVM -le $nodeCount ]
|
||||
do
|
||||
if [[ $countService -gt $numCloudServices ]]; then let countService=1
|
||||
fi
|
||||
if [[ $storageAccountIndex -ge $countStorageAccount ]]; then let storageAccountIndex=0
|
||||
fi
|
||||
|
||||
cloudServiceName="$clustercloudServicePrefix$countService"
|
||||
vmName="$clustervmNamePrefix$countVM"
|
||||
storageAccount=$storageAccountList[$storageAccountIndex]
|
||||
ssh=$(( $countVM + $countVM ))
|
||||
dnsName=$cloudServiceName".cloudapp.net"
|
||||
|
||||
##############################################################
|
||||
## Create the cloud service if it does not exist
|
||||
##############################################################
|
||||
#Check to see if the cloud servide already exists
|
||||
result=$(azure service show $cloudServiceName --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
|
||||
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
|
||||
else
|
||||
printf "Cloud Service $cloudServiceName exists\n"
|
||||
fi
|
||||
|
||||
#show the cloud service details
|
||||
printf "######################################## Cloud Service Details #######################################\n"
|
||||
azure service show "$cloudServiceName" --json
|
||||
|
||||
#Check to see if the VM already exists
|
||||
result=$(azure vm show $vmName --json | jq '.VMName')
|
||||
if [[ -z $result ]]; then
|
||||
|
||||
printf "Virtual machine $vnName does not exist. Creating ...\n"
|
||||
#create the vm and attach data disks
|
||||
( azure vm create --connect --vm-size $clusterinstanceSize --vm-name $vmName --ssh $ssh --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $cloneImageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1;}
|
||||
|
||||
#add all the necessary data disks
|
||||
index=0
|
||||
while [ $index -lt $clusternumOfDisks ]; do
|
||||
azure vm disk attach-new --verbose $vmName $clusterdiskSizeInGB
|
||||
let index=index+1
|
||||
done
|
||||
else
|
||||
printf "Virtual machine $vmName exists\n"
|
||||
fi
|
||||
|
||||
#We can either add the adminUserName in the SUDOER group so it can overwrite the /etc/hosts file
|
||||
#or we can use the root user to overwrite the /etc/hosts file
|
||||
#echo "scp /etc/hosts ${adminUserName}@${vmName}:/etc" >> $mntscript
|
||||
#echo "ssh ${adminUserName}@${vmName}:/root/scripts/st.pl" >> $mntscript
|
||||
echo "ssh root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
|
||||
echo "scp /etc/hosts root@${vmName}:/etc" >> $mntscript
|
||||
|
||||
printf "######################################## Virtual Machine Details #######################################\n"
|
||||
#display the details about the newly created VM
|
||||
ipaddress=$(azure vm show $vmName --json | jq '.IPAddress')
|
||||
#remove the double quotes from the IP address
|
||||
echo "$ipaddress $vmName" | sed -e 's/\"//g' >> $hostsfile
|
||||
|
||||
countVM=$(( $countVM + 1 ))
|
||||
countService=$(( $countService + 1 ))
|
||||
storageAccountIndex=$(( $storageAccountIndex + 1 ))
|
||||
done
|
||||
#make the mntscript executable
|
||||
chmod a+x $mntscript
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
##############################################################
|
||||
#!/bin/bash
|
||||
# Set up the management node
|
||||
##############################################################
|
||||
# Assign variables from the config file
|
||||
source ./clustersetup.sh
|
||||
|
||||
# Delete the mountscript and hosts files if they exist
|
||||
if [ -e $mntscript ]; then
|
||||
rm $mntscript
|
||||
fi
|
||||
|
||||
if [ -e $hostfile ]; then
|
||||
rm $hostsfile
|
||||
fi
|
||||
|
||||
# This script requires jq json processor
|
||||
# Check to see if jq is available and download it if necessary
|
||||
result=$(which jq)
|
||||
if [[ -z $result ]]; then
|
||||
wget http://stedolan.github.io/jq/download/linux64/jq
|
||||
sudo chmod +x jq
|
||||
sudo mv jq /usr/local/bin
|
||||
else
|
||||
printf "jq json processor was found\n"
|
||||
fi
|
||||
|
||||
(which jq) || { echo "jq json processor is not available"; exit 1; }
|
||||
|
||||
|
||||
##############################################################
|
||||
## Create the management node if it does not exist
|
||||
##############################################################
|
||||
vmName=$vmNamePrefix"0"
|
||||
cloudServiceName=$cloudServicePrefix
|
||||
dnsName=$cloudServiceName".cloudapp.net"
|
||||
|
||||
#Check to see if the cloud servide already exists
|
||||
result=$(azure service show $cloudServiceName --json | jq '.ServiceName')
|
||||
if [[ -z $result ]]; then
|
||||
printf "Service does not exist. About to create cloud service:$cloudServiceName in affinity group:$affinityGroupName\n"
|
||||
(azure service create --affinitygroup "${affinityGroupName}" --serviceName $cloudServiceName) || { echo "Failed to create Cloud Service $cloudServiceName"; exit 1; }
|
||||
else
|
||||
printf "Cloud Service $cloudServiceName exists\n"
|
||||
fi
|
||||
|
||||
#show the cloud service details
|
||||
printf "######################################## Cloud Service Management Node Details #######################################\n"
|
||||
azure service show "$cloudServiceName" --json
|
||||
|
||||
#Check to see if the VM already exists
|
||||
result=$(azure vm show $vmName --json | jq '.VMName')
|
||||
if [[ -z $result ]]; then
|
||||
printf "Virtual machine $vnName does not exist. Creating ...\n"
|
||||
#create the vm and attach data disks
|
||||
(azure vm create --vm-size $instanceSize --vm-name $vmName --ssh 22 --virtual-network-name $vnetName --subnet-names $subnetName $dnsName $galleryimageName $adminUserName $adminPassword) || { echo "Failed to create vm $vmName"; exit 1; }
|
||||
|
||||
#add all the necessary data disks
|
||||
index=0
|
||||
while [ $index -lt $numOfDisks ]; do
|
||||
azure vm disk attach-new --verbose $vmName $diskSizeInGB
|
||||
let index=index+1
|
||||
done
|
||||
|
||||
#add endpoint for ambari web interface
|
||||
azure vm endpoint create --endpoint-name "Installer" $vmName $installerport $installerport
|
||||
|
||||
printf "######################################## Virtual Machine Management Node Details #######################################\n"
|
||||
#display the details about the newly created VM
|
||||
azure vm show $vmName --json
|
||||
else
|
||||
printf "Virtual machine $vmName exists\n"
|
||||
fi
|
||||
|
||||
#get the ip address for the newly create virtual machine
|
||||
ipaddress=$(azure vm show $vmName --json | jq '.IPAddress')
|
||||
#remove the double quotes from the IP address
|
||||
echo "$ipaddress $vmName" | sed -e 's/\"//g' >> $hostsfile
|
||||
echo "ssh root@${vmName} /root/scripts/makefilesystem.sh" >> $mntscript
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
#!/bin/bash
|
||||
#This script deletes all the services associated with the cluster defined in cdhsetup.sh files
|
||||
#It checks to see if it is running in an interactive mode. If it running in interactive mode it asks
|
||||
#the user to confirm that cluster will be deleted. If the script is not running in interactive mode
|
||||
#it deletes the cluster without prompting the user.
|
||||
#It does not delete VNET, Storage Account and Affinity Group.
|
||||
#VNET and Affinit group have no cost. Storage account may have images and ohter assets stored in it.
|
||||
#It does print the instruction to delete vnet, affinity group and storage account so use can copy and paste to delete them.
|
||||
source ./clustersetup.sh
|
||||
|
||||
|
||||
#display the services that will be deleted.
|
||||
printf "These are the services in this cluster that will be deleted:\n"
|
||||
|
||||
printf "Virtual network name %s\n" $vnetName
|
||||
loopIndex=0
|
||||
while [ $loopIndex -le $nodeCount ]; do
|
||||
vmName=$vmNamePrefix"$loopIndex"
|
||||
cloudServiceName=$cloudServicePrefix"$loopIndex"
|
||||
dnsName=$cloudServiceName".cloudapp.net"
|
||||
|
||||
printf "Cloud Service %s, Virtual Machine %s\n" $dnsName $vmName
|
||||
let loopIndex=loopIndex+1
|
||||
done
|
||||
|
||||
function deletecluster {
|
||||
printf "I am about to delete the cluster\n"
|
||||
local loopIndex=0
|
||||
while [ $loopIndex -le $nodeCount ]; do
|
||||
vmName=$vmNamePrefix"$loopIndex"
|
||||
cloudServiceName=$cloudServicePrefix"$loopIndex"
|
||||
dnsName=$cloudServiceName".cloudapp.net"
|
||||
printf "Cloud Service %s, Virtual Machine %s\n" $dnsName $vmName
|
||||
azure vm delete -q -b -v --json $vmName
|
||||
azure service delete -q -v --json $vmName
|
||||
let loopIndex=loopIndex+1
|
||||
done
|
||||
|
||||
#we will not delete the vnet, storage account and affinity group but we
|
||||
will echo these command so users can
|
||||
#copy these commands and run them manually.
|
||||
echo "azure network vnet delete -v -q $vnetName"
|
||||
echo "azure storage delete -q -v $storageAccount"
|
||||
echo "azure account affinity-group -v -q $affinityGroupName"
|
||||
}
|
||||
|
||||
function end {
|
||||
printf "Cluster was not deleted.\n"
|
||||
}
|
||||
|
||||
function choose {
|
||||
local defaults="$1"
|
||||
local prompt="$2"
|
||||
local choice_yes="$3"
|
||||
local choice_no="$4"
|
||||
local answer
|
||||
|
||||
read -p "$prompt" answer
|
||||
[ -z "$answer" ] && answer="$default"
|
||||
case "$answer" in
|
||||
[yY1] ) eval "$choice_yes"
|
||||
#error check
|
||||
;;
|
||||
[nN0] ) eval "$choice_no"
|
||||
#error check
|
||||
;;
|
||||
* ) printf "%b" "Unexpected answer '$answer'!" >&2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
#determine if you are running interactively and prompt for confirmation
|
||||
case "$-" in
|
||||
*i*) choose N "Do you want to delete all the services assocaited with th
|
||||
is cluster, select Y or N:" deletecluster end
|
||||
;;
|
||||
*) deletecluster
|
||||
;;
|
||||
esac
|
||||
echo "##########################################################################
|
||||
###################"
|
||||
echo "Script finished successfully"
|
||||
echo "##########################################################################
|
||||
###################"
|
||||
exit 0
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/bash
|
||||
######## Mounting Drives ###############
|
||||
# Setting variables to allow quick customisation:
|
||||
FISYS="ext4"
|
||||
mtoptions="noatime,nodiratime"
|
||||
|
||||
echo "Attaching disks to $HOSTNAME"
|
||||
|
||||
cp /etc/fstab /etc/old.fstab`date +%d-%m-%y---%H-%M-%S`
|
||||
dmesg | grep -e "\[sd[a-z]" | awk '{print $3;}' | sort -u > /tmp/diskdeviceoutput
|
||||
|
||||
d=0
|
||||
for i in {b..z}
|
||||
do
|
||||
sdx=$(cat /tmp/diskdeviceoutput | grep -o sd$i)
|
||||
|
||||
if [[ $sdx ]]
|
||||
then
|
||||
if cat /etc/mtab | grep $sdx
|
||||
then
|
||||
unset $sdx
|
||||
fi
|
||||
|
||||
echo "Mounting $sdx"
|
||||
sleep 3
|
||||
|
||||
echo "y
|
||||
" | mkfs.$FISYS -m 1 -O dir_index,extent,sparse_super -E lazy_itable_init /dev/$sdx > /var/log/mkfs.$sdx
|
||||
mkdir -p /drive/$d
|
||||
mount -o $mtoptions /dev/$sdx /drive/$d
|
||||
echo "/dev/$sdx /drive/$((d++)) $FISYS $mtoptions 0 0" >> /etc/fstab
|
||||
|
||||
else
|
||||
echo " "
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
echo "Checking the disks:"
|
||||
fdisk -l | grep /dev/sd > /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo "Checking the mounts:"
|
||||
mount -l | grep /drive >> /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo "Checking /etc/fstab for the mounts:"
|
||||
cat /etc/fstab | grep /drive >> /var/log/automountandformat.log
|
||||
sleep 2 ; if ls /var/log/ | grep auto > /dev/null ; then echo "Done." ; fi
|
||||
|
||||
echo " "
|
||||
|
||||
echo "The above commands are written to log file /var/log/automountandformat.log."
|
||||
sleep 3
|
||||
echo "Disks are attached to $HOSTNAME"
|
||||
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
#This script reads the hosts file and mergest its content with /etc/hosts
|
||||
|
||||
source ./clustersetup.sh
|
||||
|
||||
#check to make sure hosts file exists
|
||||
echo "hosts file name is:$hostsfile"
|
||||
|
||||
if [ -e $hostsfile ]
|
||||
then
|
||||
while read line;do
|
||||
echo "$line"
|
||||
|
||||
if grep -Fxq "$line" /etc/hosts
|
||||
then
|
||||
printf "$line already exists in hosts file\n"
|
||||
else
|
||||
echo $line >> /etc/hosts
|
||||
# code if not found
|
||||
fi
|
||||
done < $hostsfile
|
||||
else
|
||||
printf "File $hostsfile does not exist\n"
|
||||
exit 1
|
||||
fi
|
||||
Загрузка…
Ссылка в новой задаче