Refactor files and add missing automation parts in the quickstart.

azure-pipelines/lib/templates/lib-build-publish.yml

@@ -2,7 +2,7 @@
 parameters:
 - name: libraryDirectory
   type: string
-  default: $(System.DefaultWorkingDirectory)/src/dataopslib
+  default: $(System.DefaultWorkingDirectory)/data-platform/src/dataopslib
 - name: artifactFeed
   type: string
 - name: variableGroupName

data-platform/src/dataopslib/setup.py

@@ -39,7 +39,7 @@ setup(
     setup_requires=setup_requirements,
     test_suite='tests',
     tests_require=test_requirements,
-    url='https://dev.azure.com/csu-devsquad/advworks-dataops/_git/data_platform',
+    url='https://dev.azure.com/csu-devsquad/advworks-dataops/_git/hol',
     version=version,
     python_requires='>=3.6'
 )

infrastructure-as-code/infrastructure/parameters/parameters.dev.json

@@ -39,7 +39,7 @@
             "value": "develop"
         },
         "dataFactoryRootFolder": {
-            "value": "data_platform/adf"
+            "value": "data-platform/adf"
         }
     }
 }

infrastructure-as-code/infrastructure/parameters/parameters.dev.template.json

@@ -0,0 +1,45 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "environment":{
+            "value": "dev"
+        },
+        "location": {
+            "value": "eastus2"
+        },
+        "resourceGroupData":{
+            "value": "rg-<project.alias>-data-dev"
+        },
+        "resourceGroupCompute":{
+            "value": "rg-<project.alias>-compute-dev"
+        },
+        "resourceGroupMachineLearning":{
+            "value": "rg-<project.alias>-ml-dev"
+        },
+        "resourceGroupManagedDatabricks":{
+            "value": "rg-<project.alias>-compute-dbw-dev"
+        },
+        "dataLakeSkuName":{
+            "value": "Standard_LRS"
+        },
+        "keyVaultSecretsAdminObjectId": {
+            "value": "c06d0482-8384-4205-a17f-edd2222dce1b"
+        },
+        "dataFactoryAccountName": {
+            "value": "<azureDevOps.organization>"
+        },
+        "dataFactoryProjectName": {
+            "value": "<project.name>"
+        },
+        "dataFactoryRepositoryName": {
+            "value": "<project.name>"
+        },
+        "dataFactoryCollaborationBranch": {
+            "value": "develop"
+        },
+        "dataFactoryRootFolder": {
+            "value": "data-platform/adf"
+        }
+    }
+}

infrastructure-as-code/infrastructure/parameters/parameters.prod.template.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "environment":{
+            "value": "prod"
+        },
+        "location": {
+            "value": "eastus2"
+        },
+        "resourceGroupData":{
+            "value": "rg-<project.alias>-data-prod"
+        },
+        "resourceGroupCompute":{
+            "value": "rg-<project.alias>-compute-prod"
+        },
+        "resourceGroupMachineLearning":{
+            "value": "rg-<project.alias>-ml-prod"
+        },
+        "resourceGroupManagedDatabricks":{
+            "value": "rg-<project.alias>-compute-dbw-prod"
+        },
+        "dataLakeSkuName":{
+            "value": "Standard_GRS"
+        },
+        "keyVaultSecretsAdminObjectId": {
+            "value": "ff1b0094-0e92-4e5b-9f21-502837003e85"
+        }
+    }
+}

infrastructure-as-code/infrastructure/parameters/parameters.qa.template.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "environment":{
+            "value": "qa"
+        },
+        "location": {
+            "value": "eastus2"
+        },
+        "resourceGroupData":{
+            "value": "rg-<project.alias>-data-qa"
+        },
+        "resourceGroupCompute":{
+            "value": "rg-<project.alias>-compute-qa"
+        },
+        "resourceGroupMachineLearning":{
+            "value": "rg-<project.alias>-ml-qa"
+        },
+        "resourceGroupManagedDatabricks":{
+            "value": "rg-<project.alias>-compute-dbw-qa"
+        },
+        "dataLakeSkuName":{
+            "value": "Standard_GRS"
+        },
+        "keyVaultSecretsAdminObjectId": {
+            "value": "4b0a1c5e-3f7c-4d81-9d2b-479a654b1c70"
+        }
+    }
+}

quickstart/README.md

@@ -38,3 +38,11 @@ $env:AZURE_DEVOPS_EXT_PAT="<my pat goes here>"
 ./quickstart/scripts/dataops/Deploy-AzureDevOps.ps1 -ConfigurationFile "./quickstart/outputs/hol.json" 
 
 ```
+
+# Install GitTools
+
+https://marketplace.visualstudio.com/items?itemName=gittools.gittools&targetId=0d8e54d4-e229-47bd-9dc5-9be0f116a5c0&utm_source=vstsproduct&utm_medium=ExtHubManageList
+
+# Authorize Build Services to use Variable Groups
+
+# Change IAC parameters.dev.json values

quickstart/configs/dataops/template.json

@@ -1,6 +1,7 @@
 {
-    "Domain": {
+    "Project": {
-        "Name": "<project.name>"
+        "Name": "<project.name>",
+        "Alias": "<project.alias>"
     },
     "RepoConfiguration": {
         "RepoName": "<project.name>",
@@ -27,7 +28,7 @@
                 "SourceYamlPath": "azure-pipelines/lib/lib-ci.yml",
                 "BuildPolicy": {
                     "Name": "<project.alias>-lib-ci build policy",
-                    "PathFilter": "/data_platform/src/dataopslib/dataopslib/*"
+                    "PathFilter": "/data-platform/src/dataopslib/dataopslib/*"
                 }
             },
             {
@@ -39,7 +40,7 @@
                 "SourceYamlPath": "azure-pipelines/databricks/databricks-ci.yml",
                 "BuildPolicy": {
                     "Name": "<project.alias>-databricks-ci build policy",
-                    "PathFilter": "/data_platform/notebooks/*"
+                    "PathFilter": "/data-platform/notebooks/*"
                 }
             },
             {

quickstart/schemas/dataops/config.schema.1.0.0.json

@@ -2,11 +2,14 @@
   "$schema": "http://json-schema.org/schema#",
   "type": "object",
   "properties": {
-    "Domain": {
+    "Project": {
       "type": "object",
       "properties": {
         "Name": {
           "type": "string"
+        },
+        "Alias": {
+          "type": "string"
         }        
       },
       "required": [
@@ -79,7 +82,7 @@
     }
   },
   "required": [
-    "Domain",
+    "Project",
     "RepoConfiguration"
   ]
 }

quickstart/scripts/dataops/Deploy-AzureDevOps.ps1

@@ -24,6 +24,7 @@ $repoInfo = CreateAzureDevopsRepository -RepoConfiguration $config.RepoConfigura
 
 $directory = CloneRepo -RepoInfo $repoInfo -UseSSH $UseSSH -UsePAT $UsePAT -Verbose:$VerbosePreference
 ImportTemplateRepoToDomainRepo -RepoConfiguration $config.RepoConfiguration -UsePAT $UsePAT -Directory $directory[0] -Verbose:$VerbosePreference
+UpdateIaCParameters -Configuration $config -Directory $directory[0] -Verbose:$VerbosePreference
 
 CreateAzDevOpsRepoApprovalPolicy -RepoInfo $repoInfo -RepoConfiguration $config.RepoConfiguration -Verbose:$VerbosePreference
 CreateAzDevOpsRepoCommentPolicy  -RepoInfo $repoInfo -RepoConfiguration $config.RepoConfiguration -Verbose:$VerbosePreference
@@ -35,3 +36,7 @@ CreateAzDevOpsRepoBuildPolicy -RepoInfo $repoInfo -RepoConfiguration $config.Rep
 CreateAzureDevOpsVariableGroup -VariableGroupName "dataops-iac-cd-output-dev"
 CreateAzureDevOpsVariableGroup -VariableGroupName "dataops-iac-cd-output-qa"
 CreateAzureDevOpsVariableGroup -VariableGroupName "dataops-iac-cd-output-prod"
+CreateAzureDevOpsVariableGroup -VariableGroupName "lib-versions"
+CreateAzureDevOpsVariable -VariableGroupName "lib-versions" -VariableName "MAJOR" -VariableValue "0"
+CreateAzureDevOpsVariable -VariableGroupName "lib-versions" -VariableName "MINOR" -VariableValue "1"
+CreateAzureDevOpsVariable -VariableGroupName "lib-versions" -VariableName "PATCH" -VariableValue "0"

quickstart/scripts/modules/AzureDevOps.psm1

@@ -44,6 +44,36 @@ function CreateAzureDevOpsVariableGroup {
     return $GroupId
 }
 
+function CreateAzureDevOpsVariable {
+    [cmdletbinding()]
+    param(
+        [Parameter(Mandatory)] [string] $VariableGroupName,
+        [Parameter(Mandatory)] [string] $VariableName,
+        [Parameter(Mandatory)] [string] $VariableValue
+    )
+    [Argument]::AssertIsNotNullOrEmpty("VariableGroupName", $VariableGroupName)
+    [Argument]::AssertIsNotNullOrEmpty("VariableName", $VariableName)
+    [Argument]::AssertIsNotNullOrEmpty("VariableValue", $VariableValue)
+
+    $GroupId = $(az pipelines variable-group list --query "[?name=='$VariableGroupName'].id" -o tsv)
+
+    if (! $GroupId) {
+        Write-Verbose "Creating variable group $VariableGroupName ..."
+        $GroupId = $(az pipelines variable-group create --name $VariableGroupName --authorize --variable createdAt="$(Get-Date)" --query "id" -o tsv)
+
+        if (! $GroupId) {
+            Write-Error "The build agent does not have permissions to create variable groups"
+            exit 1
+        }
+    }
+
+    Write-Verbose "Trying to update variable $VariableName..."
+    if (! (az pipelines variable-group variable update --group-id $GroupId --name $VariableName --value $VariableValue)) { 
+        Write-Verbose"Creating variable $key..."
+        az pipelines variable-group variable create --group-id $GroupId --name $VariableName --value $VariableValue
+    }
+}
+
 function CreateAzureDevopsRepository {
     param (
         [Parameter(Mandatory)] [hashtable] $RepoConfiguration

quickstart/scripts/modules/RepoOperations.psm1

@@ -27,6 +27,44 @@ function ImportTemplateRepoToDomainRepo {
     Set-Location -
 }
 
+function UpdateIaCParameters {
+    param (
+        [Parameter(Mandatory)] [hashtable] $Configuration,
+        [Parameter(Mandatory)] [string] $Directory
+    )
+    [Argument]::AssertIsNotNull("Configuration", $Configuration)
+
+    Write-Host "Updating IaC parameters..." -ForegroundColor Green
+
+    Set-Location $Directory
+
+	BeginScope -Scope "IaC parameters"
+
+	ReplaceTemplateTokens `
+		-Configuration $Configuration `
+		-InputFile infrastructure-as-code/infrastructure/parameters/parameters.dev.template.json `
+		-OutputFile infrastructure-as-code/infrastructure/parameters/parameters.dev.json
+
+	ReplaceTemplateTokens `
+		-Configuration $Configuration `
+		-InputFile infrastructure-as-code/infrastructure/parameters/parameters.qa.template.json `
+		-OutputFile infrastructure-as-code/infrastructure/parameters/parameters.qa.json 
+
+	ReplaceTemplateTokens `
+		-Configuration $Configuration `
+		-InputFile infrastructure-as-code/infrastructure/parameters/parameters.prod.template.json `
+		-OutputFile infrastructure-as-code/infrastructure/parameters/parameters.prod.json 
+		
+	EndScope
+
+	git add infrastructure-as-code/infrastructure/parameters/parameters.dev.json
+	git add infrastructure-as-code/infrastructure/parameters/parameters.qa.json
+	git add infrastructure-as-code/infrastructure/parameters/parameters.prod.json
+    git push -u origin $RepoConfiguration.DefaultBranchName
+	
+    Set-Location -
+}
+
 function PublishOutputs {
     param(
         [Parameter(Mandatory)] [hashtable] $Configuration