feat(pipelines): refactor separate pipelines per environment into single deployment pipeline (#52)

* import existing pipeline changes

* try refactoring steps, splitting plan and apply permissions

* refactor more

* fix syntax

* fix tf apply step

* ci: use ubuntu-latest

* more cleanup, remove unused code

* cd: add prod stage

* relative paths, adjust PR

* pr-syntax: adjust, test later

* feat: add makefile with all local tf commands

* chore: use main branch again

Makefile

@@ -0,0 +1,16 @@
+init-dev:
+	. ./.env.dev
+	terraform init -backend-config=backends/dev.hcl
+
+init-prod:
+	. ./.env
+	terraform init -backend-config=backends/prod.hcl
+
+plan-dev:
+	terraform plan -var-file=environments/dev.tfvars -out plan.tfplan
+
+plan-prod:
+	terraform plan -var-file=environments/prod.tfvars -out plan.tfplan
+
+apply:
+	terraform apply plan.tfplan

azure-pipelines/_vars.yaml

@@ -0,0 +1,10 @@
+variables:
+  terraformStateContainer: terraform
+  terraformPlanFile:       deployment.tfplan
+  isPR:                    ${{ eq(variables['Build.Reason'], 'PullRequest') }}
+
+  # Unused - but kept here for reference
+  isFork:        ${{ eq(variables['System.PullRequest.IsFork'], 'True') }}
+  isTrustedCode: ${{ eq(variables.isFork, 'False') }}
+  isScheduled:   ${{ eq(variables['Build.Reason'], 'Schedule') }}
+  isTrustedCI:   ${{ and( eq(variables.isFork,'False'), eq(variables.isPR,'False'), eq(variables.isScheduled,'False') ) }}

azure-pipelines/cd.yaml

@@ -7,49 +7,55 @@ trigger:
     - production
   paths:
     exclude:
-    - '*.md'
-    - 'backends/*'
-    - 'images/*'
     - '.github/*'
+    - 'backends/*'
+    - 'modules/cicd-setup'
+    - 'images/*'
+    - '*.md'
 
 pr: none
 
 pool:
-  vmImage: 'ubuntu-18.04'
+  vmImage: 'ubuntu-latest'
 
 variables:
-- template: vars/global.yaml
-- group: mask-ids
-
-# Conditional uses ${{ }} syntax in order to be processed at compile time
-# in order to work for loading variable groups. For details see:
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#understand-variable-syntax
-- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}:
-  - group: e2e-gov-demo-dev-kv
-- ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/production') }}:
-  - group: e2e-gov-demo-kv
+- template: ./_vars.yaml
+- group: e2e-gov-demo-kv
 
 stages:
-- template: stages/ci.yaml
+- template: ./stages/ci.yaml
 
-- stage: cd_stage
-  displayName: CD - Deployment
+- stage: devStage
+  displayName: Deploy - Dev
+  condition: and(succeeded('CI'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
+  variables:
+  - group: e2e-gov-demo-headless-owner
   jobs:
-  - job: deploy
-    displayName: Terraform Plan and Apply
+  - job: devJob
+    displayName: Terraform - Plan and Apply
+    variables:
+      terraformStateFile: dev-v2.tfstate
+      terraformVarFile: environments/dev.tfvars
+      devopsOrgUrl: "https://dev.azure.com/ado-gov-demo-dev"
     steps:
-    - template: steps/debug-vars.yaml
-    - template: steps/terraform-init.yaml
-    - template: steps/terraform-plan.yaml
-      parameters:
-        extraFlags: "-out=deployment.tfplan"
+    - template: ./steps/terraform-init.yaml
+    - template: ./steps/terraform-plan.yaml
+    - template: ./steps/terraform-apply.yaml
+
+- stage: prodStage
+  displayName: Deploy - Production
+  condition: and(succeeded('CI'), eq(variables['Build.SourceBranch'], 'refs/heads/production'))
+  variables:
+  - group: e2e-gov-demo-headless-owner
+  jobs:
+  - job: prodJob
+    displayName: Terraform - Plan and Apply
+    variables:
+      terraformStateFile: prod-v2.tfstate
+      terraformVarFile: environments/prod.tfvars
+      devopsOrgUrl: "https://dev.azure.com/ado-gov-demo"
+    steps:
+    - template: ./steps/terraform-init.yaml
+    - template: ./steps/terraform-plan.yaml
+    - template: ./steps/terraform-apply.yaml
 
-    - bash: terraform apply -auto-approve deployment.tfplan
-      displayName: Terraform Apply
-      env:
-        ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
-        ARM_CLIENT_ID:              $(kv-arm-client-id)
-        ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
-        ARM_TENANT_ID:              $(kv-arm-tenant-id)
-        AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)

azure-pipelines/ci.yaml

@@ -7,13 +7,13 @@ trigger:
     - fix/*
   paths:
     exclude:
-    - '*.md'
-    - 'backends/*'
-    - 'images/*'
     - '.github/*'
-
+    - 'backends/*'
+    - 'modules/cicd-setup'
+    - 'images/*'
+    - '*.md'
 pool:
-  vmImage: 'ubuntu-18.04'
+  vmImage: 'ubuntu-latest'
 
 stages:
-- template: stages/ci.yaml
+- template: ./stages/ci.yaml

azure-pipelines/pr-main.yaml

@@ -1,18 +0,0 @@
-name: $(BuildID)
-
-pool:
-  vmImage: 'ubuntu-18.04'
-
-trigger: none # PR only
-
-pr:
-- main
-
-variables:
-  - template: vars/global.yaml
-  - group: e2e-gov-demo-dev-kv # DEV
-  - group: mask-ids
-
-stages:
-  - template: stages/ci.yaml
-  - template: stages/pull-request.yaml

azure-pipelines/pr-production.yaml

@@ -1,18 +0,0 @@
-name: $(BuildID)
-
-pool:
-  vmImage: 'ubuntu-18.04'
-
-trigger: none # PR only
-
-pr:
-- production
-
-variables:
-  - template: vars/global.yaml
-  - group: e2e-gov-demo-kv # PROD
-  - group: mask-ids
-
-stages:
-  - template: stages/ci.yaml
-  - template: stages/pull-request.yaml

azure-pipelines/pull-request.yaml

@@ -1,24 +1,50 @@
+name: $(BuildID)
+
+pool:
+  vmImage: 'ubuntu-latest'
+
+trigger: none # PR only
+
+pr:
+- main
+- production
+
+variables:
+  githubRepoName:       Azure/devops-governance
+  githubConnectionName: e2e-governance-demo
+
+- ${{ if eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/main') }}:
+  terraformStateFile: dev-v2.tfstate
+  terraformVarFile: environments/dev.tfvars
+  devopsOrgUrl: "https://dev.azure.com/ado-gov-demo-dev"
+- ${{ if eq(variables['System.PullRequest.TargetBranch'], 'refs/heads/production') }}:
+  terraformStateFile: prod-v2.tfstate
+  terraformVarFile: environments/prod.tfvars
+  devopsOrgUrl: "https://dev.azure.com/ado-gov-demo"
+
 stages:
+- template: ./stages/ci.yaml
 
-# ------------
-# Detect Drift
-# ------------
-
-- stage: terraformStage
+- stage: driftStage
   displayName: Detect Drift
+  variables:
+  - template: ./_vars.yaml
+  - group: e2e-gov-demo-subscription-reader
+  - group: mask-ids
   jobs:
-  - job: terraformJob
+  - job: driftJob
     displayName: Terraform Plan
     steps:
-    # terraform plan
-    # --------------
-    - template: ../steps/terraform-init.yaml
+    - template: ./steps/terraform-init.yaml
     - bash: |
         # Remember Exit Code
         set -uo pipefail
 
         # Run `terraform plan` and save output (used later for posting to GitHub)
-        if terraform plan -detailed-exitcode -var superadmins_aad_object_id=$AAD_SUPERADMINS_GROUP_ID \
+        if terraform plan -detailed-exitcode \
+          -var superadmins_aad_object_id=$(kv-aad-superadmins-group-id) \
+          -var 'application_owners_ids=$(kv-aad-app-owners-ids)' \
+          -var-file=$(terraformVarFile) \
             | tee plan-output.txt \
             exit ${PIPESTATUS[0]}
         then
@@ -37,12 +63,10 @@ stages:
         ARM_CLIENT_ID:              $(kv-arm-client-id)
         ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
         ARM_TENANT_ID:              $(kv-arm-tenant-id)
-        AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
-        AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)
+        AZDO_ORG_SERVICE_URL:       $(devopsOrgUrl)
+        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-ado-pat)
 
     # Save plan output
-    # ----------------
     - publish: ./plan-output.txt
       artifact: terraformPlanOutput
       displayName: Save 'terraform plan' output
@@ -57,10 +81,7 @@ stages:
   displayName: Add Results to Pull Request
   condition: and(succeededOrFailed(), eq(variables.isPR, 'True'))
   variables:
-    githubRepoName:       Azure/devops-governance
-    githubConnectionName: e2e-governance-demo
-    exitCode:             $[ stageDependencies.terraformStage.terraformJob.outputs['planStep.exitCode'] ]
-    # artifactName:         terraformPlanOutput # Debugging, unused for now
+    exitCode: $[ stageDependencies.driftStage.driftJob.outputs['planStep.exitCode'] ]
   jobs:
   - job: postCommentJob
     displayName: Post to GitHub
@@ -72,7 +93,7 @@ stages:
       condition: eq(variables.exitCode, 'zero')
       displayName: Post - No Drift
       inputs:
-        gitHubConnection: ${{ variables.githubConnectionName }}
+        gitHubConnection: $(githubConnectionName)
         repositoryName: $(githubRepoName)
         comment: |
           ### 🟢  No configuration drift detected
@@ -85,7 +106,7 @@ stages:
       condition: eq(variables.exitCode, 'non-zero')
       displayName: Post - Has Drift
       inputs:
-        gitHubConnection: ${{ variables.githubConnectionName }}
+        gitHubConnection: $(githubConnectionName)
         repositoryName: $(githubRepoName)
         comment: |
           ### ⚠️  Configuration Drift Detected (_OR_ State File Locked)
@@ -95,36 +116,3 @@ stages:
           Approving this Pull Request may result in destructive changes to your Azure resources. Please review the `terraform plan` output diff at Azure Pipelines Build Result Page.
 
           Proceed with caution!
-
-  # Debugging - turns out multiline variables not supported
-  # -------------
-  # - job: debugOutput
-  #   displayName: Debug Output
-  #   steps:
-  #   - download: current  # current pipeline
-  #     artifact: $(artifactName)
-  #     patterns: '*'
-  #     displayName: Download 'terraform plan' output
-
-  #   - bash: |
-  #       ls $(Pipeline.Workspace)
-  #       cat $(Pipeline.Workspace)/$(artifactName)/plan-output.txt
-  #     displayName: output download file
-
-    # Step - work in progress
-    # We can pass output to downstream stage/job.
-    # But we lose newlines, making output useless for markdown.
-    # Todo: create own task that reads comment contents from a file.
-    # - bash: echo $(planOutput)
-    #   displayName: Debug - plan output
-
-    # Step - work in progress
-    # Multiline variables are not supported in Azure DevOps 😕
-    # - bash: |
-    #     oneliner=$(printf "%s " $(sed -e 's/$/&\\n/' ./plan-output.txt))
-    #     echo $oneliner
-    #     echo "====="
-    #     echo "##vso[task.setvariable variable=planOutput;isOutput=true]$(echo $oneliner)"
-    #   displayName: Save plan output (wip)
-    #   name: outputPlanStep
-    #   condition: always()

azure-pipelines/schedule-drift.yaml

@@ -26,6 +26,6 @@ steps:
 - template: steps/debug-vars.yaml
 - template: steps/confirm-kv-loaded.yaml
 - template: steps/terraform-init.yaml
-- template: steps/terraform-plan.yaml
+- template: steps/terraform-plan.yaml # changed, so now broken.
   parameters:
     extraFlags: -detailed-exitcode # Drift Detection

azure-pipelines/stages/ci.yaml

@@ -1,8 +1,8 @@
 stages:
-- stage: CIStage
+- stage: CI
   displayName: CI - Integration
   jobs:
-  - job: CIJob
+  - job:
     displayName: Terraform - Lint and Validate
     steps:
     - bash: terraform -version

azure-pipelines/steps/confirm-kv-loaded.yaml

@@ -1,17 +0,0 @@
-steps:
-- bash: |
-    echo "--------------------------------"
-    echo "Confirm Key Vault 🔑 Integration"
-    echo "--------------------------------"
-    echo "KV_DEBUG_ENV: $KV_DEBUG_ENV"
-    echo ""
-    if [ $KV_DEBUG_ENV == \$\(kv-debug-env\) ]; then
-      echo "⛔️ Key Vault not loaded"
-      echo "Please double check configuration Variable Groups in Azure Pipelines UI and that the YAML pipeline is running against the 'main' or 'production' branch."
-      exit 1
-    else
-      echo "✅ Key Vault loaded"
-    fi
-  displayName: Debug - Key Vault loaded?
-  env:
-    KV_DEBUG_ENV: $(kv-debug-env)

azure-pipelines/steps/debug-vars.yaml

@@ -1,16 +0,0 @@
-steps:
-- bash: |
-    echo ""
-    echo "---------"
-    echo "Debugging"
-    echo "---------"
-    echo "Build.SourceBranch: ${{ variables['Build.SourceBranch'] }}"
-    echo "isMain: $(isMain)"
-    echo "isProduction: $(isProduction)"
-    echo "isTag: $(isTag)"
-    echo "isFork: $(isFork)"
-    echo "isPR: $(isPR)"
-    echo "isTrustedCode: $(isTrustedCode)"
-    echo "isScheduled: $(isScheduled)"
-    echo "isTrustedCI: $(isTrustedCI)"
-  displayName: Debug - Custom Variables

azure-pipelines/steps/terraform-apply.yaml

@@ -0,0 +1,11 @@
+steps:
+- bash: |
+    terraform apply -auto-approve $(terraformPlanFile)
+  displayName: Terraform Apply
+  env:
+    ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
+    ARM_CLIENT_ID:              $(kv-arm-client-id)
+    ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
+    ARM_TENANT_ID:              $(kv-arm-tenant-id)
+    AZDO_ORG_SERVICE_URL:       $(devopsOrgUrl)
+    AZDO_PERSONAL_ACCESS_TOKEN: $(kv-ado-pat)

azure-pipelines/steps/terraform-init.yaml

@@ -2,13 +2,8 @@ steps:
 - bash: |
     terraform -version
     terraform init \
-      -backend-config="storage_account_name=$TF_STATE_BLOB_ACCOUNT_NAME" \
-      -backend-config="container_name=$TF_STATE_BLOB_CONTAINER_NAME" \
-      -backend-config="key=$TF_STATE_BLOB_FILE" \
-      -backend-config="sas_token=$TF_STATE_BLOB_SAS_TOKEN"
+      -backend-config="storage_account_name=$(kv-tf-state-blob-account)" \
+      -backend-config="sas_token=$(kv-tf-state-sas-token)" \
+      -backend-config="container_name=$(terraformStateContainer)" \
+      -backend-config="key=$(terraformStateFile)"
   displayName: Terraform Init
-  env:
-    TF_STATE_BLOB_ACCOUNT_NAME:   $(kv-tf-state-blob-account)
-    TF_STATE_BLOB_CONTAINER_NAME: $(kv-tf-state-blob-container)
-    TF_STATE_BLOB_FILE:           $(kv-tf-state-blob-file)
-    TF_STATE_BLOB_SAS_TOKEN:      $(kv-tf-state-sas-token)

azure-pipelines/steps/terraform-plan.yaml

@@ -1,16 +1,15 @@
-parameters:
-- name: extraFlags
-  type: string
-  default: ""
-
 steps:
-- bash: terraform plan -var superadmins_aad_object_id=$AAD_SUPERADMINS_GROUP_ID ${{ parameters.extraFlags }}
+- bash: |
+    terraform plan \
+      -var superadmins_aad_object_id=$(kv-aad-superadmins-group-id) \
+      -var 'application_owners_ids=$(kv-aad-app-owners-ids)' \
+      -var-file=$(terraformVarFile) \
+      -out $(terraformPlanFile)
   displayName: Terraform Plan
   env:
     ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
     ARM_CLIENT_ID:              $(kv-arm-client-id)
     ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
     ARM_TENANT_ID:              $(kv-arm-tenant-id)
-    AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-    AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
-    AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)
+    AZDO_ORG_SERVICE_URL:       $(devopsOrgUrl)
+    AZDO_PERSONAL_ACCESS_TOKEN: $(kv-ado-pat)

azure-pipelines/vars/global.yaml

@@ -1,9 +0,0 @@
-variables:
-  isMain:              ${{ eq(variables['Build.SourceBranch'], 'refs/heads/main') }}
-  isProduction:        ${{ eq(variables['Build.SourceBranch'], 'refs/heads/production') }}
-  isTag:               ${{ startsWith(variables['Build.SourceBranch'], 'refs/tags/v') }}
-  isFork:              ${{ eq(variables['System.PullRequest.IsFork'], 'True') }}
-  isPR:                ${{ eq(variables['Build.Reason'], 'PullRequest') }}
-  isTrustedCode:       ${{ eq(variables.isFork, 'False') }}
-  isScheduled:         ${{ eq(variables['Build.Reason'], 'Schedule') }}
-  isTrustedCI:         ${{ and( eq(variables.isFork,'False'), eq(variables.isPR,'False'), eq(variables.isScheduled,'False') ) }}