pipelines: refactor out more steps (#29)

* pipelines: refactor out more steps * pipeline-templates: maybe different relative path syntax * pipeline: rename pr template * vars: surface github vars * Revert "vars: surface github vars" - doesn't work 🤷‍♀️ This reverts commit bc850025d5.
2021-06-22 18:49:33 +02:00 · 2021-06-22 18:49:33 +02:00 · 81904ac0ca
--- a/azure-pipelines/cd.yaml
+++ b/azure-pipelines/cd.yaml
@ -25,5 +25,25 @@ variables:
    - group: e2e-gov-demo-kv

 stages:
- template: templates/ci.yaml
- template: templates/deploy.yaml
+- template: stages/ci.yaml
+
+- stage: cd_stage
+  displayName: CD Stage
+  jobs:
+  - job: deploy
+    displayName: Terraform Plan and Apply
+    steps:
+    - template: steps/terraform-init.yaml
+    - template: steps/terraform-plan.yaml
+      parameters:
+        extraFlags: "-out=deployment.tfplan"
+
+    - bash: terraform apply -auto-approve deployment.tfplan
+      displayName: Terraform Apply
+      env:
+        ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
+        ARM_CLIENT_ID:              $(kv-arm-client-id)
+        ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
+        ARM_TENANT_ID:              $(kv-arm-tenant-id)
+        AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
+        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
--- a/azure-pipelines/pr-main.yaml
+++ b/azure-pipelines/pr-main.yaml
@ -14,4 +14,4 @@ variables:

 stages:
  - template: stages/ci.yaml
-  - template: stages/pr-drift-comment.yaml
+  - template: stages/pull-request.yaml
--- a/azure-pipelines/pr-production.yaml
+++ b/azure-pipelines/pr-production.yaml
@ -14,4 +14,4 @@ variables:

 stages:
  - template: stages/ci.yaml
-  - template: stages/pr-drift-comment.yaml
+  - template: stages/pull-request.yaml
--- a/azure-pipelines/schedule-drift.yaml
+++ b/azure-pipelines/schedule-drift.yaml
@ -8,7 +8,7 @@ pr: none

 schedules:
  - cron: "0 0 * * *"
-    displayName: Daily midnight build
+    displayName: Daily Midnight Build
    always: true
    branches:
      include:
@ -23,30 +23,7 @@ variables:
    - group: e2e-gov-demo-kv

 steps:
- bash: |
-    terraform -version
-    terraform init \
-      -backend-config="storage_account_name=$TF_STATE_BLOB_ACCOUNT_NAME" \
-      -backend-config="container_name=$TF_STATE_BLOB_CONTAINER_NAME" \
-      -backend-config="key=$TF_STATE_BLOB_FILE" \
-      -backend-config="sas_token=$TF_STATE_BLOB_SAS_TOKEN"
-  displayName: Terraform Init
-  env:
-    TF_STATE_BLOB_ACCOUNT_NAME:   $(kv-tf-state-blob-account)
-    TF_STATE_BLOB_CONTAINER_NAME: $(kv-tf-state-blob-container)
-    TF_STATE_BLOB_FILE:           $(kv-tf-state-blob-file)
-    TF_STATE_BLOB_SAS_TOKEN:      $(kv-tf-state-sas-token)
-
- bash: |
-    terraform plan \
-      -detailed-exitcode \
-      -var superadmins_aad_object_id=$AAD_SUPERADMINS_GROUP_ID
-  displayName: Terraform - Detect Drift
-  env:
-    ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
-    ARM_CLIENT_ID:              $(kv-arm-client-id)
-    ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
-    ARM_TENANT_ID:              $(kv-arm-tenant-id)
-    AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-    AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
-    AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)
+- template: steps/terraform-init.yaml
+- template: steps/terraform-plan.yaml
+  parameters:
+    extraFlags: -detailed-exitcode # Drift Detection
--- a/azure-pipelines/stages/deploy.yaml
+++ b/azure-pipelines/stages/deploy.yaml
@ -1,41 +0,0 @@
-stages:
- stage: cd_stage
-  displayName: CD Stage
-  jobs:
-  - job: deploy
-    displayName: Terraform Plan and Apply
-    steps:
-    - bash: |
-        terraform -version
-        terraform init \
-          -backend-config="storage_account_name=$TF_STATE_BLOB_ACCOUNT_NAME" \
-          -backend-config="container_name=$TF_STATE_BLOB_CONTAINER_NAME" \
-          -backend-config="key=$TF_STATE_BLOB_FILE" \
-          -backend-config="sas_token=$TF_STATE_BLOB_SAS_TOKEN"
-      displayName: Terraform Init
-      env:
-        TF_STATE_BLOB_ACCOUNT_NAME:   $(kv-tf-state-blob-account)
-        TF_STATE_BLOB_CONTAINER_NAME: $(kv-tf-state-blob-container)
-        TF_STATE_BLOB_FILE:           $(kv-tf-state-blob-file)
-        TF_STATE_BLOB_SAS_TOKEN:      $(kv-tf-state-sas-token)
-
-    - bash: terraform plan -out=deployment.tfplan -var superadmins_aad_object_id=$AAD_SUPERADMINS_GROUP_ID
-      displayName: Terraform Plan (ignores drift)
-      env:
-        ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
-        ARM_CLIENT_ID:              $(kv-arm-client-id)
-        ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
-        ARM_TENANT_ID:              $(kv-arm-tenant-id)
-        AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
-        AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)
-
-    - bash: terraform apply -auto-approve deployment.tfplan
-      displayName: Terraform Apply
-      env:
-        ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
-        ARM_CLIENT_ID:              $(kv-arm-client-id)
-        ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
-        ARM_TENANT_ID:              $(kv-arm-tenant-id)
-        AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
-        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
--- a/azure-pipelines/stages/pr-drift-comment.yaml
+++ b/azure-pipelines/stages/pr-drift-comment.yaml
@ -1,30 +1,18 @@
 stages:

-# Stage: Terraform Plan
-# ---------------------
+# ------------
+# Detect Drift
+# ------------
+
 - stage: terraformStage
  displayName: Detect Drift
  jobs:
  - job: terraformJob
    displayName: Terraform Plan
    steps:
-    # Step - terraform init
-    - bash: |
-        terraform -version
-        terraform init \
-          -backend-config="storage_account_name=$TF_STATE_BLOB_ACCOUNT_NAME" \
-          -backend-config="container_name=$TF_STATE_BLOB_CONTAINER_NAME" \
-          -backend-config="key=$TF_STATE_BLOB_FILE" \
-          -backend-config="sas_token=$TF_STATE_BLOB_SAS_TOKEN"
-      displayName: Terraform - Init
-      # condition: eq(variables.isProduction, 'True') # never / Debugging
-      env:
-        TF_STATE_BLOB_ACCOUNT_NAME:   $(kv-tf-state-blob-account)
-        TF_STATE_BLOB_CONTAINER_NAME: $(kv-tf-state-blob-container)
-        TF_STATE_BLOB_FILE:           $(kv-tf-state-blob-file)
-        TF_STATE_BLOB_SAS_TOKEN:      $(kv-tf-state-sas-token)
-
-    # Step - terraform plan
+    # terraform plan
+    # --------------
+    - template: ../steps/terraform-init.yaml
    - bash: |
        # Remember Exit Code
        set -uo pipefail
@ -43,7 +31,6 @@ stages:
          exit $tf_exit_code
        fi
      displayName: Terraform - Detect configuration drift
-      # condition: eq(variables.isProduction, 'True') # never / Debugging
      name: planStep
      env:
        ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
@ -54,32 +41,33 @@ stages:
        AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
        AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)

-    # Step - pass `terraform plan` output to downstream jobs
+    # Save plan output
+    # ----------------
    - publish: ./plan-output.txt
      artifact: terraformPlanOutput
      displayName: Save 'terraform plan' output
      condition: succeededOrFailed()


-# Stage: Pull Request Comment
 # ---------------------------
+# Post Status to Pull Request
+# ---------------------------
+
 - stage: pullRequestCommentStage
  displayName: Add Results to Pull Request
  condition: and(succeededOrFailed(), eq(variables.isPR, 'True'))
  variables:
    githubRepoName:       Azure/devops-governance
    githubConnectionName: e2e-governance-demo
-    adoOrgName:           julie-msft
-    adoProjectName:       e2e-governance-demo
    exitCode:             $[ stageDependencies.terraformStage.terraformJob.outputs['planStep.exitCode'] ]
-    artifactName:         terraformPlanOutput
-    # planOutput:           $[ stageDependencies.terraformStage.terraformJob.outputs['outputPlanStep.planOutput'] ]
+    # artifactName:         terraformPlanOutput # Debugging, unused for now
  jobs:
  - job: postCommentJob
    displayName: Post to GitHub
    steps:

-    # Step - post, No Drift
+    # Add Comment - No Drift
+    # ----------------------
    - task: GitHubComment@0
      condition: eq(variables.exitCode, 'zero')
      displayName: Post - No Drift
@ -91,7 +79,8 @@ stages:

          Source Version: $(Build.SourceVersion)

-    # Step - post, Drift detected
+    # Add Comment - Drift detected
+    # ----------------------------
    - task: GitHubComment@0
      condition: eq(variables.exitCode, 'non-zero')
      displayName: Post - Has Drift
@ -107,19 +96,20 @@ stages:

          Proceed with caution!

-  # Debugging - Work in progress
-  - job: debugOutput
-    displayName: Debug Output
-    steps:
-    - download: current  # current pipeline
-      artifact: $(artifactName)
-      patterns: '*'
-      displayName: Download 'terraform plan' output
+  # Debugging - turns out multiline variables not supported
+  # -------------
+  # - job: debugOutput
+  #   displayName: Debug Output
+  #   steps:
+  #   - download: current  # current pipeline
+  #     artifact: $(artifactName)
+  #     patterns: '*'
+  #     displayName: Download 'terraform plan' output

-    - bash: |
-        ls $(Pipeline.Workspace)
-        cat $(Pipeline.Workspace)/$(artifactName)/plan-output.txt
-      displayName: output download file
+  #   - bash: |
+  #       ls $(Pipeline.Workspace)
+  #       cat $(Pipeline.Workspace)/$(artifactName)/plan-output.txt
+  #     displayName: output download file

    # Step - work in progress
    # We can pass output to downstream stage/job.
--- a/azure-pipelines/steps/terraform-init.yaml
+++ b/azure-pipelines/steps/terraform-init.yaml
@ -0,0 +1,14 @@
+steps:
+- bash: |
+    terraform -version
+    terraform init \
+      -backend-config="storage_account_name=$TF_STATE_BLOB_ACCOUNT_NAME" \
+      -backend-config="container_name=$TF_STATE_BLOB_CONTAINER_NAME" \
+      -backend-config="key=$TF_STATE_BLOB_FILE" \
+      -backend-config="sas_token=$TF_STATE_BLOB_SAS_TOKEN"
+  displayName: Terraform Init
+  env:
+    TF_STATE_BLOB_ACCOUNT_NAME:   $(kv-tf-state-blob-account)
+    TF_STATE_BLOB_CONTAINER_NAME: $(kv-tf-state-blob-container)
+    TF_STATE_BLOB_FILE:           $(kv-tf-state-blob-file)
+    TF_STATE_BLOB_SAS_TOKEN:      $(kv-tf-state-sas-token)
--- a/azure-pipelines/steps/terraform-plan.yaml
+++ b/azure-pipelines/steps/terraform-plan.yaml
@ -0,0 +1,16 @@
+parameters:
+- name: extraFlags
+  type: string
+  default: ""
+
+steps:
+- bash: terraform plan -var superadmins_aad_object_id=$AAD_SUPERADMINS_GROUP_ID ${{ parameters.extraFlags }}
+  displayName: Terraform Plan
+  env:
+    ARM_SUBSCRIPTION_ID:        $(kv-arm-subscription-id)
+    ARM_CLIENT_ID:              $(kv-arm-client-id)
+    ARM_CLIENT_SECRET:          $(kv-arm-client-secret)
+    ARM_TENANT_ID:              $(kv-arm-tenant-id)
+    AZDO_ORG_SERVICE_URL:       $(kv-azure-devops-org-url)
+    AZDO_PERSONAL_ACCESS_TOKEN: $(kv-azure-devops-pat)
+    AAD_SUPERADMINS_GROUP_ID:   $(kv-aad-superadmins-group-id)