diff --git a/datastax-enterprise/dsenode.sh b/datastax-enterprise/dsenode.sh new file mode 100644 index 0000000..77ff102 --- /dev/null +++ b/datastax-enterprise/dsenode.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +######################################################### +# Script Name: dsenode.sh +# Author: Trent Swanson - Full Scale 180 Inc github:(trentmswanson) +# Version: 0.1 +# Last Modified By: Trent Swanson +# Description: +# This script prepares an Ubuntu VM image for Datastax OpsCenter Node Cluster Installation and Configure +# Parameters : +# Note : +# This script has only been tested on Ubuntu 14.04 LTS and must be root +######################################################### + +bash vm-disk-utils-0.1.sh + +# TEMP FIX - Re-evaluate and remove when possible +# This is an interim fix for hostname resolution in current VM (If it does not exist add it) +grep -q "${HOSTNAME}" /etc/hosts +if [ $? == 0 ]; +then + echo "${HOSTNAME}found in /etc/hosts" +else + echo "${HOSTNAME} not found in /etc/hosts" + # Append it to the hsots file if not there + echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts + log "hostname ${HOSTNAME} added to /etchosts" +fi + +#Install Java +add-apt-repository -y ppa:webupd8team/java +apt-get -y update +echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections +echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections +apt-get -y install oracle-java7-installer + +#Need to see if we can find a better solution +chmod 777 /mnt +chmod 777 /datadisks diff --git a/datastax-enterprise/ephemeral-nodes-resources.json b/datastax-enterprise/ephemeral-nodes-resources.json new file mode 100644 index 0000000..3d18ecd --- /dev/null +++ b/datastax-enterprise/ephemeral-nodes-resources.json @@ -0,0 +1,168 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "region": { + "type": "string" + }, + "storageAccountPrefix": { + "type": "string" + }, + "adminUsername": { + "type": "string" + }, + "adminPassword": { + "type": "securestring" + }, + "namespace": { + "type": "string" + }, + "vmSize": { + "type": "string" + }, + "osSettings": { + "type": "object" + }, + "subnet": { + "type": "object" + }, + "nodeCount": { + "type": "int" + }, + "staticIps": { + "type": "object" + } + }, + "variables": { + "subnetRef": "[concat(resourceId('Microsoft.Network/virtualNetworks', parameters('subnet').vnet), '/subnets/', parameters('subnet').name)]", + "storageAccountName": "[concat(parameters('storageAccountPrefix'), parameters('namespace'))]", + "vmName": "[concat(parameters('namespace'), 'vm')]" + }, + "resources": [ + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Compute/availabilitySets", + "name": "[concat(parameters('namespace'), 'set')]", + "location": "[parameters('region')]", + "properties": { + "platformFaultDomainCount": 3, + "platformUpdateDomainCount": 20 + } + }, + { + "type": "Microsoft.Storage/storageAccounts", + "name": "[variables('storageAccountName')]", + "apiVersion": "2015-05-01-preview", + "location": "[parameters('region')]", + "properties": { + "accountType": "Standard_LRS" + } + }, + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Network/networkInterfaces", + "name": "[concat(parameters('namespace'), 'nic', copyindex())]", + "location": "[parameters('region')]", + "copy": { + "name": "[concat(parameters('namespace'), 'nicLoop')]", + "count": "[parameters('nodeCount')]" + }, + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "privateIPAllocationMethod": "Static", + "privateIPAddress": "[concat(parameters('staticIps').base, copyindex(parameters('staticIps').start))]", + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + }, + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Compute/virtualMachines", + "name": "[concat(parameters('namespace'), 'vm', copyindex())]", + "location": "[parameters('region')]", + "copy": { + "name": "[concat(parameters('namespace'), 'vmLoop')]", + "count": "[parameters('nodeCount')]" + }, + "dependsOn": [ + "[concat('Microsoft.Network/networkInterfaces/', parameters('namespace'), 'nic', copyindex())]", + "[concat('Microsoft.Compute/availabilitySets/', parameters('namespace'), 'set')]", + "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]" + ], + "properties": { + "availabilitySet": { + "id": "[resourceId('Microsoft.Compute/availabilitySets', concat(parameters('namespace'), 'set'))]" + }, + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "osProfile": { + "computername": "[concat(parameters('namespace'), 'vm', copyIndex())]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]" + }, + "storageProfile": { + "imageReference": "[parameters('osSettings').imageReference]", + "osDisk": { + "name": "osdisk", + "vhd": { + "uri": "[concat('http://',variables('storageAccountName'),'.blob.core.windows.net/vhds/', variables('vmName'), copyindex(), '-osdisk.vhd')]" + }, + "caching": "ReadWrite", + "createOption": "FromImage" + }, + "dataDisks": [ + { + "name": "datadisk1", + "diskSizeGB": 1023, + "lun": 0, + "vhd": { + "Uri": "[concat('http://', variables('storageAccountName'),'.blob.core.windows.net/','vhds/', variables('vmName'), copyindex(), 'DataDisk1.vhd')]" + }, + "caching": "None", + "createOption": "Empty" + } + ] + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(parameters('namespace'), 'nic', copyindex()))]" + } + ] + } + } + }, + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat(parameters('namespace'), 'vm', copyindex(), '/installdsenode')]", + "apiVersion": "2015-05-01-preview", + "location": "[parameters('region')]", + "copy": { + "name": "[concat(parameters('namespace'), 'vmLoop')]", + "count": "[parameters('nodeCount')]" + }, + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/', parameters('namespace'), 'vm', copyindex())]", + "[concat('Microsoft.Network/networkInterfaces/', parameters('namespace'), 'nic', copyindex())]" + ], + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.2", + "settings": { + "fileUris": "[parameters('osSettings').scripts]", + "commandToExecute": "bash dsenode.sh" + } + } + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/datastax-enterprise/opscenter-install-resources.json b/datastax-enterprise/opscenter-install-resources.json new file mode 100644 index 0000000..a9a6fb5 --- /dev/null +++ b/datastax-enterprise/opscenter-install-resources.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "region": { + "type": "string" + }, + "adminUsername": { + "type": "string" + }, + "adminPassword": { + "type": "securestring" + }, + "namespace": { + "type": "string" + }, + "osSettings": { + "type": "object" + }, + "opsCenterSettings": { + "type": "object" + }, + "datastaxUsername": { + "type": "string" + }, + "datastaxPassword": { + "type": "securestring" + } + }, + "variables": { + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat(parameters('namespace'), 'vm', '/installopscenter')]", + "apiVersion": "2015-05-01-preview", + "location": "[parameters('region')]", + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.2", + "settings": { + "fileUris": "[parameters('osSettings').scripts]", + "commandToExecute": "[concat('bash opscenter.sh -n \"', parameters('opsCenterSettings').clusterName , '\" -u ', parameters('adminUsername'), ' -p ', parameters('adminPassword'), ' -k ' , parameters('opsCenterSettings').adminPassword , ' -U ', parameters('datastaxUsername), ' -P ', parameters('datastaxPassword), ' -ed ', parameters('opsCenterSettings').nodeList)]" + } + } + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/datastax-enterprise/opscenter-resources.json b/datastax-enterprise/opscenter-resources.json new file mode 100644 index 0000000..9689fbb --- /dev/null +++ b/datastax-enterprise/opscenter-resources.json @@ -0,0 +1,126 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2014-04-01-preview/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "region": { + "type": "string" + }, + "storageAccountName": { + "type": "string" + }, + "adminUsername": { + "type": "string" + }, + "adminPassword": { + "type": "securestring" + }, + "namespace": { + "type": "string" + }, + "osSettings": { + "type": "object" + }, + "vmSize": { + "type": "string" + }, + "subnet": { + "type": "object" + }, + "staticIp": { + "type": "string" + }, + "dnsName": { + "type": "string" + } + }, + "variables": { + "subnetRef": "[concat(resourceId('Microsoft.Network/virtualNetworks', parameters('subnet').vnet), '/subnets/', parameters('subnet').name)]", + "publicIpName": "[concat(parameters('namespace'), 'PublicIp')]", + "publicIpRef": "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIpName'))]" + }, + "resources": [ + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Network/publicIPAddresses", + "name": "[variables('publicIpName')]", + "location": "[parameters('region')]", + "properties": { + "publicIPAllocationMethod": "Dynamic", + "dnsSettings": { + "domainNameLabel": "[parameters('dnsName')]" + } + } + }, + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Compute/availabilitySets", + "name": "[concat(parameters('namespace'), 'set')]", + "location": "[parameters('region')]", + "properties": {} + }, + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Network/networkInterfaces", + "name": "[concat(parameters('namespace'), 'nic')]", + "location": "[parameters('region')]", + "dependsOn": [ + "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIpName'))]" + ], + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "publicIPAddress": { + "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIpName'))]" + }, + "privateIPAllocationMethod": "Static", + "privateIPAddress": "[parameters('staticIp')]", + "subnet": { + "id": "[variables('subnetRef')]" + } + } + } + ] + } + }, + { + "apiVersion": "2015-05-01-preview", + "type": "Microsoft.Compute/virtualMachines", + "name": "[concat(parameters('namespace'), 'vm')]", + "location": "[parameters('region')]", + "dependsOn": [ + "[concat('Microsoft.Network/networkInterfaces/', parameters('namespace'), 'nic')]" + ], + "properties": { + "hardwareProfile": { + "vmSize": "[parameters('vmSize')]" + }, + "osProfile": { + "computername": "[concat(parameters('namespace'), 'vm')]", + "adminUsername": "[parameters('adminUsername')]", + "adminPassword": "[parameters('adminPassword')]" + }, + "storageProfile": { + "imageReference": "[parameters('osSettings').imageReference]", + "osDisk": { + "name": "osdisk", + "vhd": { + "uri": "[concat('http://',parameters('storageAccountName'),'.blob.core.windows.net/vhds/', parameters('namespace'), 'vm-osdisk.vhd')]" + }, + "caching": "ReadWrite", + "createOption": "FromImage" + } + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(parameters('namespace'), 'nic'))]" + } + ] + } + } + } + ], + "outputs": {} +} \ No newline at end of file diff --git a/datastax-enterprise/opscenter.sh b/datastax-enterprise/opscenter.sh new file mode 100644 index 0000000..90a370a --- /dev/null +++ b/datastax-enterprise/opscenter.sh @@ -0,0 +1,288 @@ +#!/bin/bash + +######################################################### +# Script Name: opscenter.sh +# Author: Trent Swanson - Full Scale 180 Inc github:(trentmswanson) +# Version: 0.1 +# Last Modified By: Trent Swanson +# Description: +# This scrtipt installs and configures Datastax Operations Center and deploys a cluster +# Parameters : +# 1 - n: Cluster name +# 2 - u: Cluster node admin user that Operations Center uses for cluster provisioning +# 3 - p: Cluster node admin password that Operations Center uses for cluster provisioning +# 4 - d: List of successive cluster IP addresses represented as the starting address and a count used to increment the last octet (10.0.0.5-3) +# 6 - k: Sets the Operations Center 'admin' password +# 7 - v: Sets the DSC Version +# 8 - h Help +# Note : +# This script has only been tested on Ubuntu 12.04 LTS and must be root +######################################################### + +help() +{ + #TODO: Add help text here + echo "This script installs Datastax Opscenter and configures nodes" + echo "Parameters:" + echo "-u username used to connect to and configure data nodes" + echo "-p password used to connect to and configure data nodes" + echo "-d dse nodes to manage (suuccessive ip range 10.0.0.4-8 for 8 nodes)" + echo "-e use ephemeral storage (yes/no)" +} + +# Log method to control/redirect log output +log() +{ + # If you want to enable this logging add a un-comment the line below and add your account id + #curl -X POST -H "content-type:text/plain" --data-binary "${HOSTNAME} - $1" https://logs-01.loggly.com/inputs//tag/es-extension,${HOSTNAME} + echo "$1" +} + +log "Begin execution of cassandra script extension on ${HOSTNAME}" + +# You must be root to run this script +if [ "${UID}" -ne 0 ]; +then + log "Script executed without root permissions" + echo "You must be root to run this program." >&2 + exit 3 +fi + +# TEMP FIX - Re-evaluate and remove when possible +# This is an interim fix for hostname resolution in current VM (If it does not exist add it) +grep -q "${HOSTNAME}" /etc/hosts +if [ $? == 0 ]; +then + echo "${HOSTNAME}found in /etc/hosts" +else + echo "${HOSTNAME} not found in /etc/hosts" + # Append it to the hsots file if not there + echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts + log "hostname ${HOSTNAME} added to /etchosts" +fi + +#Script Parameters +CLUSTER_NAME="Test Cluster" +EPHEMERAL=0 +DSE_ENDPOINTS="" +ADMIN_USER="" +SSH_KEY_PATH="" +DSE_VERSION="2.1.1" +DSE_USERNAME="" +DSE_PASSWORD="" + +#Loop through options passed +while getopts :n:d:u:p:j:v:U:P:k:e optname; do + log "Option $optname set with value ${OPTARG}" + case $optname in + n) + CLUSTER_NAME=${OPTARG} + ;; + u) #Credentials used for node install + ADMIN_USER=${OPTARG} + ;; + p) #Credentials used for node install + ADMIN_PASSWORD=${OPTARG} + ;; + d) #Static dicovery endpoints + DSE_ENDPOINTS=${OPTARG} + ;; + k) # Ops Center Admin Password + OPS_CENTER_ADMIN_PASS=${OPTARG} + ;; + v) # DSC Version + DSE_VERSION=${OPTARG} + ;; + U) DSE_USERNAME=${OPTARG} + ;; + P) DSE_PASSWORD=${OPTARG} + ;; + e) #place data on local resource disk + EPHEMERAL=1 + ;; + h) #show help + help + exit 2 + ;; + \?) #unrecognized option - show help + echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed." + help + exit 2 + ;; + esac +done + +#Install Java +add-apt-repository -y ppa:webupd8team/java +apt-get -y update +echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections +echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections +apt-get -y install oracle-java7-installer + +#Install opscenter +echo "deb http://debian.datastax.com/community stable main" | sudo tee -a /etc/apt/sources.list.d/datastax.community.list +curl -L http://debian.datastax.com/debian/repo_key | sudo apt-key add - +apt-get update +apt-get install opscenter + +# Enable authentication in /etc/opscenter/opscenterd.conf +sed -i '/^\[authentication\]$/,/^\[/ s/^enabled = False/enabled = True/' /etc/opscenter/opscenterd.conf + +# Enable SSL - uncomment webserver SSL settings and leave them set to the default +sed -i '/^\[webserver\]$/,/^\[/ s/^#ssl_keyfile/ssl_keyfile/' /etc/opscenter/opscenterd.conf +sed -i '/^\[webserver\]$/,/^\[/ s/^#ssl_certfile/ssl_certfile/' /etc/opscenter/opscenterd.conf +sed -i '/^\[webserver\]$/,/^\[/ s/^#ssl_port/ssl_port/' /etc/opscenter/opscenterd.conf + +# Disable HTTP port +# sed -i '/^\[webserver\]$/,/^\[/ s/^port/#port/' /etc/opscenter/opscenterd.conf + +# Start Ops Center +sudo service opscenterd start + +# CONFIGURE NODES + +# Expand a list of successive ip range and filter my local local ip from the list +# This increments the last octet of an IP start range using a defined value +# 10.0.0.4-3 would be converted to "10.0.0.4 10.0.0.5 10.0.0.6" +expand_ip_range() { + IFS='-' read -a IP_RANGE <<< "$1" + BASE_IP=`echo ${IP_RANGE[0]} | cut -d"." -f1-3` + LAST_OCTET=`echo ${IP_RANGE[0]} | cut -d"." -f4-4` + + #Get the IP Addresses on this machine + declare -a MY_IPS=`ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'` + declare -a EXPAND_STATICIP_RANGE_RESULTS=() + + for (( n=LAST_OCTET; n<("${IP_RANGE[1]}"+LAST_OCTET) ; n++)) + do + HOST="${BASE_IP}.${n}" + if ! [[ "${MY_IPS[@]}" =~ "${HOST}" ]]; then + EXPAND_STATICIP_RANGE_RESULTS+=($HOST) + fi + done + echo "${EXPAND_STATICIP_RANGE_RESULTS[@]}" +} + +# Convert the DSE endpoint range to a list for the provisioniing configuration +NODE_IP_LIST=$(expand_ip_range "$DSE_ENDPOINTS") + +get_node_fingerprints() { + TR=($1) + + ACCEPTED_FINGERPRINTS="" + for HOST in "${TR[@]}"; + do + ssh-keyscan -p 22 -t rsa "$HOST" > /tmp/tmpsshkeyhost.pub + HOSTKEY=$(ssh-keygen -lf /tmp/tmpsshkeyhost.pub) + + # TODO - This is a bit of a formatting hack job, need to clean it up + HOSTKEY=`echo ${HOSTKEY} | cut -d" " -f1-2` + HOSTKEY+=" (RSA)" + ACCEPTED_FINGERPRINTS+="\"$HOST\": \"$HOSTKEY\"," + done + ACCEPTED_FINGERPRINTS="${ACCEPTED_FINGERPRINTS%?}" + + echo "$ACCEPTED_FINGERPRINTS" +} + +NODE_CONFIG_LIST="\"${NODE_IP_LIST// /\",\"}\"" +ACCEPTED_FINGERPRINTS=$(get_node_fingerprints "$NODE_IP_LIST") + +# Create node provisioning document +sudo tee provision.json > /dev/null <