Update pipelines to support version-n-go process (#280)

* Pipeline updates to support configuration of skipping tests, always running jobs, and overriding Git credentials. Added App-specific pipeline to support version-n-go.

* Support env var > parameter > default for forcing the run and skipping tests. Mask PAT from output.

devops/providers/azure-devops/templates/infrastructure/azure-pipeline-build-stage.yml

@@ -1,11 +1,16 @@
 parameters:
   environment: ''
   configurationMatrix: []
+  forceRun: false
+  skipTests: false
 
 jobs:
 
 - job: TemplateChangeDetection
+  variables:
+  - group: '${{ parameters.environment }} Environment Variables'
   displayName: Determine CI Targets to Run
+  condition: not(coalesce(variables.FORCE_RUN, ${{ parameters.forceRun }}))
   steps:
   - ${{ each config in parameters.configurationMatrix }}:
     - task: Bash@3
@@ -21,7 +26,7 @@ jobs:
   - job: Build_${{ config.jobName }}_${{ parameters.environment }}
     dependsOn: TemplateChangeDetection
     pool: $(AGENT_POOL)
-    condition: eq(dependencies.TemplateChangeDetection.outputs['${{ config.jobName }}.needs_cicd'], 'true')
+    condition: or(coalesce(variables.FORCE_RUN, ${{ parameters.forceRun }}), eq(dependencies.TemplateChangeDetection.outputs['${{ config.jobName }}.needs_cicd'], 'true'))
 
     variables:
     - group: '${{ parameters.environment }} Environment Variables'
@@ -61,8 +66,19 @@ jobs:
       inputs:
         version: '$(GO_VERSION)'
 
+    - task: Bash@3
+      name: SetupGitCredentialInjection
+      displayName: Configure Git to use PAT
+      condition: ne(variables.TEMPLATE_REPO_PAT, '')
+      inputs:
+        targetType: 'inline'
+        script: '[[ -z $SecretPAT ]] && echo "No PAT provided; skipping." || git config --global url."https://cobalt:$SecretPAT@".insteadOf "https://"'
+      env:
+        SecretPAT: $(TEMPLATE_REPO_PAT)
+
     - task: AzureCLI@1
       displayName: 'Unit Test Terraform Template'
+      condition: not(coalesce(variables.SKIP_TESTS, ${{ parameters.skipTests }}))
       inputs:
         azureSubscription: '$(SERVICE_CONNECTION_NAME)'
         scriptPath: './$(BUILD_ARTIFACT_NAME)/$(PIPELINE_ROOT_DIR)/$(SCRIPTS_DIR)/test-unit.sh'
@@ -83,6 +99,17 @@ jobs:
         TF_VAR_remote_state_account: $(REMOTE_STATE_ACCOUNT)
         TF_VAR_remote_state_container: $(REMOTE_STATE_CONTAINER)
 
+    - task: Bash@3
+      name: TeardownGitCredentialInjection
+      displayName: Reset Git configuration
+      condition: and(always(), ne(variables.TEMPLATE_REPO_PAT, ''))
+      inputs:
+        targetType: 'inline'
+        script: '[[ -z $SecretPAT ]] && echo "No PAT provided; skipping." || git config --global --unset url."https://cobalt:$SecretPAT@".insteadOf'
+      env:
+        SecretPAT: $(TEMPLATE_REPO_PAT)
+
+
     - task: AzureCLI@1
       displayName: 'Create Terraform Execution Plan'
       inputs:

devops/providers/azure-devops/templates/infrastructure/azure-pipeline-release-stage.yml

@@ -1,11 +1,16 @@
 parameters:
   environment: ''
   configurationMatrix: []
+  forceRun: false
+  skipTests: false
 
 jobs:
 
 - job: TemplateChangeDetection
+  variables:
+  - group: '${{ parameters.environment }} Environment Variables'
   displayName: Determine CD Targets to Run
+  condition: not(coalesce(variables.FORCE_RUN, ${{ parameters.forceRun }}))
   steps:
   - ${{ each config in parameters.configurationMatrix }}:
     - task: Bash@3
@@ -20,7 +25,7 @@ jobs:
 - ${{ each config in parameters.configurationMatrix }}:
   - deployment: Provision_${{ config.jobName }}_${{ parameters.environment }}
     dependsOn: TemplateChangeDetection
-    condition: eq(dependencies.TemplateChangeDetection.outputs['${{ config.jobName }}.needs_cicd'], 'true')
+    condition: or(coalesce(variables.FORCE_RUN, ${{ parameters.forceRun }}), eq(dependencies.TemplateChangeDetection.outputs['${{ config.jobName }}.needs_cicd'], 'true'))
 
     ${{ if config.deploymentTimeoutInMinutes }}:
       timeoutInMinutes: '${{ config.deploymentTimeoutInMinutes }}'
@@ -91,6 +96,7 @@ jobs:
 
           - task: AzureCLI@1
             displayName: 'Integration Test Terraform Template'
+            condition: not(coalesce(variables.SKIP_TESTS, ${{ parameters.skipTests }}))
             inputs:
               azureSubscription: '$(SERVICE_CONNECTION_NAME)'
               scriptPath: './$(RELEASE_ARTIFACT_NAME)/$(PIPELINE_ROOT_DIR)/$(SCRIPTS_DIR)/test-integration.sh'

devops/providers/azure-devops/templates/infrastructure/azure-pipelines-app.yml

@@ -0,0 +1,63 @@
+parameters:
+  environments: []
+  configurationMatrix: []
+
+stages:
+
+- stage: PublishBuildArtifact
+  jobs:
+  - job: Publish
+    displayName: Pull scripts and validate agent readiness
+    steps:
+    - task: Bash@3
+      name: PullScripts
+      displayName: Download Cobalt scripts
+      inputs:
+        targetType: 'inline'
+        script: |
+          # Note: This is intentionally inline because applications should only have
+          # a minimal pipeline definition and Terraform module (for Cobalt) in their repo.
+          mkdir cobaltsrcripts
+          cd cobaltsrcripts
+          git init
+          git remote add origin https://github.com/microsoft/cobalt.git
+          git fetch --force --tags --prune --progress --no-recurse-submodules origin
+          git checkout --progress --force origin/master
+    - task: CopyFiles@2
+      displayName: Copy Pipeline Scripts to Artifact Directory
+      inputs:
+        contents: 'devops/**/scripts/*'
+        sourceFolder: $(Build.SourcesDirectory)/cobaltsrcripts
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: CopyFiles@2
+      displayName: Copy Terraform Files to Artifact Directory
+      inputs:
+        contents: 'infra/**'
+        sourceFolder: $(Build.SourcesDirectory)
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Artifact
+      inputs:
+        parallel: true
+        parallelCount: 8
+        artifactName: '$(BUILD_ARTIFACT_NAME)'
+        pathToPublish: $(Build.ArtifactStagingDirectory)
+
+- ${{ each environment in parameters.environments }}:
+  - stage: ${{ environment }}_Build
+    jobs:
+    - template: azure-pipeline-build-stage.yml
+      parameters:
+        environment: ${{ environment }}
+        configurationMatrix: ${{ parameters.configurationMatrix }}
+        skipTests: true
+        forceRun: true
+
+  - stage: ${{ environment }}_Release
+    jobs:
+    - template: azure-pipeline-release-stage.yml
+      parameters:
+        environment: ${{ environment }}
+        configurationMatrix: ${{ parameters.configurationMatrix }}
+        skipTests: true
+        forceRun: true