From e7b78cedb9b4346c0d1ec50dba102d59abc5e21f Mon Sep 17 00:00:00 2001 From: Bruce Campbell Date: Wed, 7 Feb 2018 15:42:56 -0800 Subject: [PATCH] Enabled preprovisioning on windows dcos agents (#2228) --- extensions/hello-world-dcos-windows/README.md | 46 ++++++++++++++ .../v1/hello-world-dcos.ps1 | 20 ++++++ .../v1/supported-orchestrators.json | 1 + .../v1/template-link.json | 33 ++++++++++ .../hello-world-dcos-windows/v1/template.json | 62 ++++++++++++++++++ parts/dcos/dcosWindowsProvision.ps1 | 10 ++- parts/dcos/dcosagentvars.t | 4 +- parts/dcos/dcoscustomdata110.t | 2 +- pkg/acsengine/engine.go | 63 ++++++++++++++++++- 9 files changed, 235 insertions(+), 6 deletions(-) create mode 100644 extensions/hello-world-dcos-windows/README.md create mode 100644 extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1 create mode 100644 extensions/hello-world-dcos-windows/v1/supported-orchestrators.json create mode 100644 extensions/hello-world-dcos-windows/v1/template-link.json create mode 100644 extensions/hello-world-dcos-windows/v1/template.json diff --git a/extensions/hello-world-dcos-windows/README.md b/extensions/hello-world-dcos-windows/README.md new file mode 100644 index 000000000..c5bf8f5be --- /dev/null +++ b/extensions/hello-world-dcos-windows/README.md @@ -0,0 +1,46 @@ +# hello-world-dcos Extension + +Sample hello-world extension. Calls the following on the master: + +``` + curl -X post http://localhost:8080/v2/apps -d "{ \"id\": \"hello-marathon\", \"cmd\": \"while [ true ] ; do echo 'Hello World' ; sleep 5 ; done\", \"cpus\": 0.1, \"mem\": 10.0, \"instances\": 1 }" -H "Content-type:application/json" +``` + +You can validate that the extension was run by running (make sure you have tunneled into the master): +``` +dcos auth login +dcos task log hello-marathon +``` + +# Configuration +|Name|Required|Acceptable Value| +|---|---|---| +|name|yes|hello-world-k8s| +|version|yes|v1| +|extensionParameters|no|| +|rootURL|optional|| + +# Example +``` javascript + "masterProfile": { + ... + "extensions": [ + { + "name": "hello-world-dcos", + "singleOrAll": "single" + } + ] + }, + ... + "extensionProfiles": [ + { + "name": "hello-world-dcos", + "version": "v1" + } + ] + + +``` + +# Supported Orchestrators +"DCOS", "DCOS173", "DCOS184", "DCOS188" \ No newline at end of file diff --git a/extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1 b/extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1 new file mode 100644 index 000000000..dd88a0695 --- /dev/null +++ b/extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1 @@ -0,0 +1,20 @@ +# Script file to run hello-world in dcos + +#!/usr/bin/pwsh + + +Write-Host "$(date) - Starting Script" + +# Deploy container +Write-Host "$(date) - Deploying hello-world" + +$uri = "http://"+($env:DCOS_AGENT_IP)+":5051/metrics/snapshot" +while($true) { + $obj = ((Invoke-Webrequest -Method GET -URI $uri ).Content | ConvertFrom-JSON ) + Write-Host "$(date) - system/cpus_total = " ($obj.'system/cpus_total') ", mem free bytes = " ($obj.'system/mem_free_bytes') ", mem total bytes = " ($obj.'system/mem_total_bytes') + sleep 5 +} + +Write-Host "$(date) - view resources in mesos UI to validate" +Write-Host "$(date) - Script complete" + diff --git a/extensions/hello-world-dcos-windows/v1/supported-orchestrators.json b/extensions/hello-world-dcos-windows/v1/supported-orchestrators.json new file mode 100644 index 000000000..f033603f5 --- /dev/null +++ b/extensions/hello-world-dcos-windows/v1/supported-orchestrators.json @@ -0,0 +1 @@ +["DCOS", "DCOS173", "DCOS184", "DCOS188", "DCOS190"] \ No newline at end of file diff --git a/extensions/hello-world-dcos-windows/v1/template-link.json b/extensions/hello-world-dcos-windows/v1/template-link.json new file mode 100644 index 000000000..049177d4d --- /dev/null +++ b/extensions/hello-world-dcos-windows/v1/template-link.json @@ -0,0 +1,33 @@ +{ + "name": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET), 'HelloWorldDcos')]", + "type": "Microsoft.Resources/deployments", + "apiVersion": "[variables('apiVersionLinkDefault')]", + "dependsOn": [ + "[resourceId('Microsoft.Compute/virtualMachines/extensions', concat(variables('masterVMNamePrefix'), sub(variables('masterCount'), 1)), 'waitforleader')]" + ], + "copy": { + "count": "EXTENSION_LOOP_COUNT", + "name": "helloWorldExtensionLoop" + }, + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "EXTENSION_URL_REPLACEextensions/hello-world-dcos-windows/v1/template.json", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "artifactsLocation": { + "value": "EXTENSION_URL_REPLACE" + }, + "apiVersionDefault": { + "value": "[variables('apiVersionDefault')]" + }, + "targetVMName": { + "value": "[concat(EXTENSION_TARGET_VM_NAME_PREFIX, copyIndex(EXTENSION_LOOP_OFFSET))]" + }, + "extensionParameters": { + "value": "EXTENSION_PARAMETERS_REPLACE" + } + } + } +} diff --git a/extensions/hello-world-dcos-windows/v1/template.json b/extensions/hello-world-dcos-windows/v1/template.json new file mode 100644 index 000000000..ef3c72085 --- /dev/null +++ b/extensions/hello-world-dcos-windows/v1/template.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "artifactsLocation": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Artifacts Location - URL" + } + }, + "apiVersionDefault": { + "type": "string", + "minLength": 1, + "metadata": { + "description": "Compute API Version" + } + }, + "targetVMName":{ + "type": "string", + "minLength": 1, + "metadata": { + "description": "Name of the vm to run the " + } + }, + "extensionParameters": { + "type": "securestring", + "minLength": 0, + "metadata": { + "description": "Custom Parameter for Extension - for hello-world, this is empty" + } + } + }, + "variables": { + "singleQuote": "'", + "initScriptUrl": "[concat(parameters('artifactsLocation'), 'extensions/hello-world-dcos-windows/v1/hello-world-dcos.ps1')]" + }, + "resources": [ + { + "apiVersion": "[parameters('apiVersionDefault')]", + "dependsOn": [], + "location": "[resourceGroup().location]", + "type": "Microsoft.Compute/virtualMachines/extensions", + "name": "[concat(parameters('targetVMName'),'/waitforleader')]", + "properties": { + "publisher": "Microsoft.OSTCExtensions", + "type": "CustomScriptForLinux", + "typeHandlerVersion": "1.5", + "autoUpgradeMinorVersion": true, + "settings": { + "fileUris": [ + "[variables('initScriptUrl')]" + ] + }, + "protectedSettings": { + "commandToExecute": "[concat('powershell \"./hello-world-dcos.ps1 ', variables('singleQuote'), parameters('extensionParameters'), variables('singleQuote'), ' >> c:/azuredata/hello-world-dcos-provision.log 2>&1 &\" &')]" + } + } + } + ], + "outputs": { } +} diff --git a/parts/dcos/dcosWindowsProvision.ps1 b/parts/dcos/dcosWindowsProvision.ps1 index d2aa4509b..c1c02ffe6 100644 --- a/parts/dcos/dcosWindowsProvision.ps1 +++ b/parts/dcos/dcosWindowsProvision.ps1 @@ -37,7 +37,11 @@ param( [string] [AllowNull()] - $customAttrs = "" + $customAttrs = "", + + [string] + [AllowNull()] + $preprovisionExtensionParams = "" ) @@ -139,7 +143,7 @@ try # the output. Write-Log "Get the install script" - Write-Log ("Parameters = isAgent = ["+ $isAgent + "] mastercount = ["+$MasterCount + "] First master ip= [" + $firstMasterIp+ "] boostrap URI = ["+ $bootstrapUri+"] Subnet = ["+ $subnet +"]" + " -customAttrs " + $customAttrs ) + Write-Log ("Parameters = isAgent = ["+ $isAgent + "] mastercount = ["+$MasterCount + "] First master ip= [" + $firstMasterIp+ "] boostrap URI = ["+ $bootstrapUri+"] Subnet = ["+ $subnet +"]" + " -customAttrs " + $customAttrs + " -preprovisionExtensionParms = "+ $preprovisionExtensionParams ) # Get the boostrap script @@ -201,6 +205,8 @@ try Write-Log "run setup script $run_cmd" Invoke-Expression $run_cmd } + + PREPROVISION_EXTENSION } catch { diff --git a/parts/dcos/dcosagentvars.t b/parts/dcos/dcosagentvars.t index 22b705583..dbb06d81f 100644 --- a/parts/dcos/dcosagentvars.t +++ b/parts/dcos/dcosagentvars.t @@ -7,11 +7,11 @@ {{if IsPublic .Ports}} "{{.Name}}VMNamePrefix": "[concat('wp', variables('winResourceNamePrefix'), add(900,variables('{{.Name}}Index')))]", "{{.Name}}windowsAgentCustomAttributes": "[concat(' -customAttrs ', variables('doubleSingleQuote'), '{{GetDCOSWindowsAgentCustomNodeAttributes . }}', variables('doubleSingleQuote') )]", - "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $true ', variables('{{.Name}}windowsAgentCustomAttributes'), variables('singleQuote'), ' ; ')]", + "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $true ', variables('{{.Name}}windowsAgentCustomAttributes'), ' -preprovisionExtensionParams \"{{GetDCOSWindowsAgentPreprovisionParameters .}}\"', variables('singleQuote'), ' ; ')]", {{else}} "{{.Name}}VMNamePrefix": "[concat('w', variables('winResourceNamePrefix'), add(900,variables('{{.Name}}Index')))]", "{{.Name}}windowsAgentCustomAttributes": "[concat(' -customAttrs ', variables('doubleSingleQuote'), '{{GetDCOSWindowsAgentCustomNodeAttributes . }}', variables('doubleSingleQuote') )]", - "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $false ', variables('{{.Name}}windowsAgentCustomAttributes'), variables('singleQuote'), ' ; ')]", + "{{.Name}}windowsAgentCustomScriptArguments": "[concat('$arguments = ', variables('singleQuote'), '-subnet ', variables('{{.Name}}Subnet'), ' -MasterCount ', variables('masterCount'), ' -firstMasterIP ', parameters('firstConsecutiveStaticIP'), ' -bootstrapUri ', '\"', variables('dcosWindowsBootstrapURL'), '\"', ' -isAgent $true -isPublic $false ', variables('{{.Name}}windowsAgentCustomAttributes'), ' -preprovisionExtensionParams \"{{GetDCOSWindowsAgentPreprovisionParameters .}}\"', variables('singleQuote'), ' ; ')]", {{end}} "{{.Name}}windowsAgentCustomScript": "[concat('powershell.exe -ExecutionPolicy Unrestricted -command \"', variables('{{.Name}}windowsAgentCustomScriptArguments'), variables('windowsCustomScriptSuffix'), '\" > %SYSTEMDRIVE%\\AzureData\\dcosWindowsProvision.log 2>&1')]", diff --git a/parts/dcos/dcoscustomdata110.t b/parts/dcos/dcoscustomdata110.t index 00d0e2832..740e40231 100644 --- a/parts/dcos/dcoscustomdata110.t +++ b/parts/dcos/dcoscustomdata110.t @@ -71,7 +71,7 @@ mounts: - /dcos/volume2 - - /dev/sdf1 - /dcos/volume3 -runcmd: +runcmd: PREPROVISION_EXTENSION - - ln - -s - /bin/rm diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index faced2798..37d49be07 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -1021,12 +1021,33 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return fmt.Sprintf("\"customData\": \"[base64(concat('#cloud-config\\n\\n', '%s'))]\",", str) }, "GetDCOSWindowsAgentCustomData": func(profile *api.AgentPoolProfile) string { - str := getBase64CustomScript(dcosWindowsProvision) + agentPreprovisionExtension := "" + if profile.PreprovisionExtension != nil { + agentPreprovisionExtension += "\n" + agentPreprovisionExtension += makeAgentExtensionScriptCommands(cs, profile) + } + b, err := Asset(dcosWindowsProvision) + if err != nil { + // this should never happen and this is a bug + panic(fmt.Sprintf("BUG: %s", err.Error())) + } + // translate the parameters + csStr := string(b) + csStr = strings.Replace(csStr, "PREPROVISION_EXTENSION", agentPreprovisionExtension, -1) + csStr = strings.Replace(csStr, "\r\n", "\n", -1) + str := getBase64CustomScriptFromStr(csStr) return fmt.Sprintf("\"customData\": \"%s\"", str) }, "GetDCOSWindowsAgentCustomNodeAttributes": func(profile *api.AgentPoolProfile) string { return getDCOSWindowsAgentCustomAttributes(profile) }, + "GetDCOSWindowsAgentPreprovisionParameters": func(profile *api.AgentPoolProfile) string { + agentPreprovisionExtensionParameters := "" + if profile.PreprovisionExtension != nil { + agentPreprovisionExtensionParameters = getDCOSWindowsAgentPreprovisionParameters(cs, profile) + } + return agentPreprovisionExtensionParameters + }, "GetMasterAllowedSizes": func() string { if t.ClassicMode { return GetClassicAllowedSizes() @@ -1609,6 +1630,10 @@ func makeAgentExtensionScriptCommands(cs *api.ContainerService, profile *api.Age if profile.IsAvailabilitySets() { copyIndex = fmt.Sprintf("',copyIndex(variables('%sOffset')),'", profile.Name) } + if profile.OSType == api.Windows { + return makeWindowsExtensionScriptCommands(profile.PreprovisionExtension, + cs.Properties.ExtensionProfiles, copyIndex) + } return makeExtensionScriptCommands(profile.PreprovisionExtension, cs.Properties.ExtensionProfiles, copyIndex) } @@ -1633,6 +1658,42 @@ func makeExtensionScriptCommands(extension *api.Extension, extensionProfiles []* scriptFilePath, scriptURL, scriptFilePath, scriptFilePath, extensionsParameterReference, extensionProfile.Name) } +func makeWindowsExtensionScriptCommands(extension *api.Extension, extensionProfiles []*api.ExtensionProfile, copyIndex string) string { + var extensionProfile *api.ExtensionProfile + for _, eP := range extensionProfiles { + if strings.EqualFold(eP.Name, extension.Name) { + extensionProfile = eP + break + } + } + + if extensionProfile == nil { + panic(fmt.Sprintf("%s extension referenced was not found in the extension profile", extension.Name)) + } + + scriptURL := getExtensionURL(extensionProfile.RootURL, extensionProfile.Name, extensionProfile.Version, extensionProfile.Script, extensionProfile.URLQuery) + scriptFileDir := fmt.Sprintf("$env:SystemDrive:/AzureData/extensions/%s", extensionProfile.Name) + scriptFilePath := fmt.Sprintf("%s/%s", scriptFileDir, extensionProfile.Script) + return fmt.Sprintf("New-Item -ItemType Directory -Force -Path \"%s\" ; Invoke-WebRequest -Uri \"%s\" -OutFile \"%s\" ; powershell \"%s %s\"\n", scriptFileDir, scriptURL, scriptFilePath, scriptFilePath, "$preprovisionExtensionParams") +} + +func getDCOSWindowsAgentPreprovisionParameters(cs *api.ContainerService, profile *api.AgentPoolProfile) string { + extension := profile.PreprovisionExtension + parms := "" + + var extensionProfile *api.ExtensionProfile + + for _, eP := range cs.Properties.ExtensionProfiles { + if strings.EqualFold(eP.Name, extension.Name) { + extensionProfile = eP + break + } + } + + parms = extensionProfile.ExtensionParameters + return parms +} + func getPackageGUID(orchestratorType string, orchestratorVersion string, masterCount int) string { if orchestratorType == api.DCOS { switch orchestratorVersion {