Merged PR 170: Solving paths pipelines and automatically modifying parameters

Solving paths pipelines  and automatically modifying parameters

Related work items: #286


Former-commit-id: 8807941944b9df8ebbd41235831ce2622b0be102

azure-pipelines/adf/adf-cd.yml

@@ -15,27 +15,27 @@ pool:
   vmImage: 'ubuntu-latest'
 
 variables:
-- template: templates/variable.environment.yml
+- template: ../variable.environment.yml
   
 stages:
-- template: templates/adf/build.yml
+- template: templates/build.yml
   parameters:
     environment: ${{variables.environment}}
     iacCdVariableGroupPrefix: ${{variables.iacCdVariableGroupPrefix}}
     iacCdDbwVariableGroupPrefix: ${{variables.iacCdDbwVariableGroupPrefix}}
-    workingDirectory: 'adf'
+    workingDirectory: 'data-platform/adf'
 
-- template: templates/adf/deploy.yml
+- template: templates/deploy.yml
   parameters:
     environment: ${{variables.environment}}
     iacCdVariableGroupPrefix: ${{variables.iacCdVariableGroupPrefix}}
     iacCdDbwVariableGroupPrefix: ${{variables.iacCdDbwVariableGroupPrefix}}
     pipelineArtifactDirectory: $(System.DefaultWorkingDirectory)/_dataops-adf-cd
 
-- template: templates/adf/test.yml
+- template: templates/test.yml
   parameters:
     environment: ${{variables.environment}}
     iacCdVariableGroupPrefix: ${{variables.iacCdVariableGroupPrefix}}
     iacCdDbwVariableGroupPrefix: ${{variables.iacCdDbwVariableGroupPrefix}}
-    workingDirectory: src/bdd-adf-pipelines
+    workingDirectory: data-platform/src/bdd-adf-pipelines
     testResultsDirectory: $(System.DefaultWorkingDirectory)/results

azure-pipelines/adf/templates/build.yml

@@ -56,7 +56,7 @@ stages:
         inputs:
           command: 'custom'
           workingDir: ${{ parameters.workingDirectory }}
-          customCommand: 'run build export $(Build.Repository.LocalPath)/${{ parameters.workingDirectory }} /subscriptions/$(SUBSCRIPTION_ID)/resourceGroups/$(resourceGroupCompute)/providers/Microsoft.DataFactory/factories/$(dataFactoryName) "../ArmTemplates"'
+          customCommand: 'run build export $(Build.Repository.LocalPath)/${{ parameters.workingDirectory }} /subscriptions/$(SUBSCRIPTION_ID)/resourceGroups/$(resourceGroupCompute)/providers/Microsoft.DataFactory/factories/$(dataFactoryName) "../../ArmTemplates"'
         displayName: 'Validate and Generate ARM template'
 
       # Publish the artifact to be used as a source for a release pipeline.

azure-pipelines/databricks/databricks-ci.yml

@@ -1,7 +1,7 @@
 trigger: none
 
 variables:
-  NOTEBOOK_WORKING_DIR: $(System.DefaultWorkingDirectory)/notebooks
+  NOTEBOOK_WORKING_DIR: $(System.DefaultWorkingDirectory)/data-platform/notebooks
 
 pool:
   vmImage: 'Ubuntu-18.04'

azure-pipelines/databricks/databricks-lib-cd.template.yml

@@ -0,0 +1,21 @@
+trigger: none
+
+resources:
+  pipelines:
+  - pipeline: lib
+    source: <Project.Alias>-lib-cd
+    trigger: 
+      branches:
+      - develop
+      - qa
+      - main
+
+variables:
+- template: ../variable.environment.yml
+
+stages:
+- template: templates/databricks-deploy-library-job-template.yml
+  parameters:
+    branch: ${{variables.branch}}
+    environment: ${{variables.environment}}
+    azureServiceConnection: ${{variables.azureServiceConnection}}

azure-pipelines/databricks/databricks-lib-cd.yml

@@ -3,29 +3,19 @@ trigger: none
 resources:
   pipelines:
   - pipeline: lib
-    source: dataops-lib-cd
+    source: <Project.Alias>-lib-cd
     trigger: 
       branches:
       - develop
       - qa
       - main
 
+variables:
+- template: ../variable.environment.yml
+
 stages:
-
 - template: templates/databricks-deploy-library-job-template.yml
   parameters:
-    branch: 'refs/heads/develop'
-    environment: 'dev'
-    azureServiceConnection: 'spn-iac-dev'
-
-- template: templates/databricks-deploy-library-job-template.yml
-  parameters:
-    branch: 'refs/heads/qa'
-    environment: 'qa'
-    azureServiceConnection: 'spn-iac-qa'    
-
-- template: templates/databricks-deploy-library-job-template.yml
-  parameters:
-    branch: 'refs/heads/main'
-    environment: 'prod'
-    azureServiceConnection: 'spn-iac-prod'
+    branch: ${{variables.branch}}
+    environment: ${{variables.environment}}
+    azureServiceConnection: ${{variables.azureServiceConnection}}

azure-pipelines/databricks/databricks-notebooks-cd.yml

@@ -9,7 +9,7 @@ trigger:
       - data-platform/notebooks/*
 
 variables:
-- template: templates/variable.environment.yml
+- template: ../variable.environment.yml
 
 stages:
 - template: templates/databricks-deploy-notebooks-job-template.yml

azure-pipelines/databricks/templates/databricks-deploy-notebooks-job-template.yml

@@ -24,7 +24,7 @@ stages:
           - checkout: self
           - task: PublishBuildArtifacts@1
             inputs:
-              PathtoPublish: notebooks
+              PathtoPublish: data-platform/notebooks
               ArtifactName: 'databricks-${{ parameters.environment }}'
             displayName: 'Publish Databricks Artifacts'
           - template: ./databricks-setup-environment-template.yml
@@ -37,6 +37,6 @@ stages:
             env:
               DATABRICKS_HOST: https://$(databricksWorkspaceUrl)
               DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
-              NOTEBOOKS_PATH: $(Pipeline.Workspace)/s/notebooks
+              NOTEBOOKS_PATH: $(Pipeline.Workspace)/s/data-platform/notebooks
               DATABRICKS_NOTEBOOK_PATH: '/'
             displayName: 'Deploy notebooks'

azure-pipelines/iac/templates/variable.environment.yml

@@ -4,9 +4,12 @@ variables:
   ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
     environmentName: prod
     azureServiceConnection: spn-iac-prod
+    branch: 'refs/heads/main'
   ${{ if eq(variables['Build.SourceBranchName'], 'qa') }}:
     environmentName: qa
     azureServiceConnection: spn-iac-qa
+    branch: 'refs/heads/qa'
   ${{ if and(ne(variables['Build.SourceBranchName'], 'main'), ne(variables['Build.SourceBranchName'], 'qa')) }}:
     environmentName: dev
     azureServiceConnection: spn-iac-dev
+    branch: 'refs/heads/develop'

azure-pipelines/lib/lib-cd.yml

@@ -10,8 +10,7 @@ trigger:
     - data-platform/src/dataopslib/dataopslib
 
 variables:
-  artifactFeed: advworks-dataops/lib-packages
-  variableGroupName: lib-versions
+  - template: ../variable.environment.yml
 
 stages:
 - template: templates/lib-stage-environment.yml
@@ -19,20 +18,20 @@ stages:
     branch: 'refs/heads/develop'
     environment: dev
     prereleaseAlias: beta
-    variableGroupName: $(variableGroupName)
-    artifactFeed: $(artifactFeed)
+    variableGroupName: ${{variables.variableGroupName}}
+    artifactFeed: ${{variables.artifactFeed}}
 
 - template: templates/lib-stage-environment.yml
   parameters:
     branch: 'refs/heads/qa'
     environment: qa
     prereleaseAlias: preview
-    variableGroupName: $(variableGroupName)
-    artifactFeed: $(artifactFeed)    
+    variableGroupName: ${{variables.variableGroupName}}
+    artifactFeed: ${{variables.artifactFeed}}   
 
 - template: templates/lib-stage-environment.yml
   parameters:
     branch: 'refs/heads/main'
     environment: prod
-    variableGroupName: $(variableGroupName)
-    artifactFeed: $(artifactFeed)
+    variableGroupName: ${{variables.variableGroupName}}
+    artifactFeed: ${{variables.artifactFeed}}

azure-pipelines/lib/lib-ci.yml

@@ -4,13 +4,12 @@ pool:
   vmImage: "ubuntu-latest"
 
 variables:
-  artifactFeed: advworks-dataops/lib-packages
-  variableGroupName: lib-versions
+  - template: ../variable.environment.yml
   
 jobs:
 - template: templates/lib-build-publish.yml
   parameters:
-    artifactFeed: $(artifactFeed)
-    variableGroupName: $(variableGroupName)
+    artifactFeed: ${{variables.artifactFeed}}
+    variableGroupName: ${{variables.variableGroupName}}
     prereleaseAlias: alpha
-    sourceBranchName: $(System.PullRequest.SourceBranch)
+    sourceBranchName: $(System.PullRequest.SourceBranch)

azure-pipelines/lib/templates/lib-stage-environment.yml

@@ -5,7 +5,7 @@ parameters:
   type: string
 - name: libraryDirectory
   type: string
-  default: $(System.DefaultWorkingDirectory)/src/dataopslib
+  default: $(System.DefaultWorkingDirectory)/data-platform/src/dataopslib
 - name: artifactFeed
   type: string
 - name: variableGroupName

azure-pipelines/variable.environment.template.yml

@@ -0,0 +1,18 @@
+variables:
+  iacCdVariableGroupPrefix: 'dataops-iac-cd-output'
+  iacCdDbwVariableGroupPrefix: 'dataops-iac-dbw-cd-output'
+  artifactFeed: <RepoConfiguration.AzureDevOpsProject>/lib-packages
+  variableGroupName: lib-versions
+  
+  ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
+    environment: prod
+    azureServiceConnection: spn-iac-prod
+    branch: 'refs/heads/main'
+  ${{ if eq(variables['Build.SourceBranchName'], 'qa') }}:    
+    environment: qa
+    azureServiceConnection: spn-iac-qa
+    branch: 'refs/heads/qa'
+  ${{ if and(ne(variables['Build.SourceBranchName'], 'main'), ne(variables['Build.SourceBranchName'], 'qa')) }}:
+    environment: dev
+    azureServiceConnection: spn-iac-dev
+    branch: 'refs/heads/develop'

azure-pipelines/variable.environment.yml

@@ -1,12 +1,18 @@
 variables:
   iacCdVariableGroupPrefix: 'dataops-iac-cd-output'
   iacCdDbwVariableGroupPrefix: 'dataops-iac-dbw-cd-output'
+  artifactFeed: <RepoConfiguration.AzureDevOpsProject>/lib-packages
+  variableGroupName: lib-versions
+  
   ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
     environment: prod
     azureServiceConnection: spn-iac-prod
+    branch: 'refs/heads/main'
   ${{ if eq(variables['Build.SourceBranchName'], 'qa') }}:    
     environment: qa
     azureServiceConnection: spn-iac-qa
+    branch: 'refs/heads/qa'
   ${{ if and(ne(variables['Build.SourceBranchName'], 'main'), ne(variables['Build.SourceBranchName'], 'qa')) }}:
     environment: dev
     azureServiceConnection: spn-iac-dev
+    branch: 'refs/heads/develop'

quickstart/scripts/modules/RepoOperations.psm1

@@ -54,12 +54,24 @@ function UpdateIaCParameters {
 		-Configuration $Configuration `
 		-InputFile infrastructure-as-code/infrastructure/parameters/parameters.prod.template.json `
 		-OutputFile infrastructure-as-code/infrastructure/parameters/parameters.prod.json 
+	
+	ReplaceTemplateTokens `
+		-Configuration $Configuration `
+		-InputFile azure-pipelines/databricks/databricks-lib-cd.template.yml `
+		-OutputFile azure-pipelines/databricks/databricks-lib-cd.yml 
+	
+	ReplaceTemplateTokens `
+		-Configuration $Configuration `
+		-InputFile azure-pipelines/variable.environment.template.yml `
+		-OutputFile azure-pipelines/variable.environment.yml 
 		
 	EndScope
 
 	git add infrastructure-as-code/infrastructure/parameters/parameters.dev.json
 	git add infrastructure-as-code/infrastructure/parameters/parameters.qa.json
 	git add infrastructure-as-code/infrastructure/parameters/parameters.prod.json
+	git add azure-pipelines/databricks/databricks-lib-cd.yml
+	git add azure-pipelines/variable.environment.yml 
 
 	git commit -m "Update IaC paramaters."