Initial template (#1)

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

101-terraform-job/README.md

@@ -0,0 +1,27 @@
+# Separate Plan and Apply stages
+
+## About this template
+
+This template includes a multi-stage pipeline that deploys 
+an environment from Terraform configuration, and run
+a subsequent job configured from Terraform outputs.
+
+![pipeline jobs](/docs/images/terraform_starter/101-terraform-job.png)
+
+The Terraform definition only deploys an empty resource group.
+You can extend the definition with your custom infrastructure, such as Web Apps.
+
+## Walkthrough
+
+### Using the template
+
+To use the template, follow the section
+[How to use the templates](/README.md#how-to-use-the-templates)
+in the main README file.
+
+## Next steps
+
+* The next template, [201-plan-apply-stages](../201-plan-apply-stages) demonstrates
+  how to manually review and approve changes before they are applied on an environment.
+  It also shows you can structure your project to develop and test locally without an Azure
+  backend.

101-terraform-job/azure-pipelines.yml

@@ -0,0 +1,22 @@
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - 101-terraform-job/
+
+variables:
+- group: terraform-secrets
+
+stages:
+
+- template: terraform-stages-template.yml
+  parameters:
+    environment: test
+    environmentDisplayName: Test
+    # Pass variables as environment variables.
+    # Terraform recognizes TF_VAR prefixed environment variables.
+    TerraformEnvVariables:
+      TF_VAR_department: engineering

101-terraform-job/terraform-stages-template.yml

@@ -0,0 +1,61 @@
+parameters:
+  environment: test
+  environmentDisplayName: Test
+  TerraformArguments: ''
+  TerraformEnvVariables:
+
+stages:
+- stage: Terraform_${{ parameters.environment }}
+  displayName: Terraform ${{ parameters.environmentDisplayName }}
+  pool:
+    vmImage: ubuntu-latest
+  jobs:
+
+  - job: Terraform
+    displayName: Terraform
+    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
+    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
+    steps:
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        provisionStorage: true
+        TerraformDirectory: 101-terraform-job/terraform
+        environment: ${{ parameters.environment }}
+
+    # Using bash instead of Terraform extension because of following issues:
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
+    - bash: |
+        set -eu
+        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
+        terraform apply -input=false -auto-approve -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
+      displayName: Terraform apply
+      workingDirectory: 101-terraform-job/terraform
+      env:
+        ${{ parameters.TerraformEnvVariables }}
+
+- stage: PostTerraform_${{ parameters.environment }}
+  displayName: PostTerraform ${{ parameters.environmentDisplayName }}
+  pool:
+    vmImage: ubuntu-latest
+  jobs:
+  - job: ReadTerraform
+    displayName: Use Terraform outputs
+    steps:
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        TerraformDirectory: 101-terraform-job/terraform
+        environment: ${{ parameters.environment }}
+
+    - template: ../infrastructure/terraform-outputs-template.yml
+      parameters:
+        TerraformDirectory: 101-terraform-job/terraform
+
+    - bash: |
+        # Dummy job showing how to consume Terraform outputs
+        echo Subscription ID: $(subscription_id)
+        echo Resource group: $(resource_group_name)
+      displayName: Sample script

101-terraform-job/terraform/backend.tf

@@ -0,0 +1,5 @@
+#Set the terraform backend
+terraform {
+  # Backend variables are initialized by Azure DevOps
+  backend "azurerm" {}
+}

101-terraform-job/terraform/main.tf

@@ -0,0 +1,16 @@
+# Deploy a Resource Group with Azure resources.
+#
+# For suggested naming conventions, refer to:
+#   https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging
+
+# Sample Resource Group
+
+resource "azurerm_resource_group" "main" {
+  name     = "rg-${var.appname}-${var.environment}-main"
+  location = var.location
+  tags     = {
+    department = var.department
+  }
+}
+
+# Add additional modules...

101-terraform-job/terraform/outputs.tf

@@ -0,0 +1,7 @@
+output "subscription_id" {
+  value = data.azurerm_client_config.current.subscription_id
+}
+
+output "resource_group_name" {
+  value = azurerm_resource_group.main.name
+}

101-terraform-job/terraform/provider.tf

@@ -0,0 +1,15 @@
+#Set the terraform required version
+terraform {
+  required_version = ">= 0.12.6"
+}
+
+# Configure the Azure Provider
+provider "azurerm" {
+  # It is recommended to pin to a given version of the Provider
+  version = "=1.44.0"
+}
+
+# Data
+
+# Make client_id, tenant_id, subscription_id and object_id variables
+data "azurerm_client_config" "current" {}

101-terraform-job/terraform/variables.tf

@@ -0,0 +1,23 @@
+variable "appname" {
+  type = string
+  description = "Application name. Use only lowercase letters and numbers"
+  default = "starterterraform"
+}
+
+variable "environment" {
+  type    = string
+  description = "Environment name, e.g. 'dev' or 'stage'"
+  default = "dev"
+}
+
+variable "location" {
+  type    = string
+  description = "Azure region where to create resources."
+  default = "North Europe"
+}
+
+variable "department" {
+  type    = string
+  description = "A sample variable passed from the build pipeline and used to tag resources."
+  default = "Engineering"
+}

201-plan-apply-stages/README.md

@@ -0,0 +1,101 @@
+# Separate Plan and Apply stages
+
+## About this template
+
+This template includes a multi-stage pipeline allowing to manually review and approve infrastructure
+changes before they are deployed.
+
+![pipeline jobs](/docs/images/terraform_starter/pipeline_jobs.png)
+
+The Terraform definition only deploys a resource group and two empty SQL Server instances
+(to illustrate two different approaches to managing secrets, in this case the SQL Server
+password).
+You can extend the definition with your custom infrastructure, such as Web Apps.
+
+The project can be used in local development without a remote Terraform state backend.
+This allows quickly iterating while developing the Terraform configuration, and 
+good security practices.
+
+When the project is run in Azure DevOps, however, the pipeline adds the
+`infrastructure/terraform_backend/backend.tf` to the `infrastructure/terraform` 
+directory to enable the Azure Storage shared backend for additional resiliency.
+See the Terraform documentation to understand [why a state store is needed](https://www.terraform.io/docs/state/purpose.html).
+
+## Walkthrough
+
+### Using the template
+
+To use the template, follow the section
+[How to use the templates](/README.md#how-to-use-the-templates)
+in the main README file.
+
+### Manual approvals
+
+As of December 2019, there is no support for stage gates in Azure DevOps multi-stage pipelines, but
+*deployment environments* provide a basic mechanism for stage approvals.
+
+Create an environment with no resources. Name it `Staging`.
+
+![create environment](/docs/images/terraform_starter/create_environment.png)
+
+Define environment approvals. If you want to allow anyone out of a group a people to be able to individually approve, add a group.
+
+![create environment_approval1](/docs/images/terraform_starter/create_environment_approval1.png)
+
+![create environment approval2](/docs/images/terraform_starter/create_environment_approval2.png)
+
+![create environment approval3](/docs/images/terraform_starter/create_environment_approval3.png)
+
+![environment approval](/docs/images/terraform_starter/environment_approval.png)
+
+Repeat those steps for an environment named `QA`.
+
+Under Library, create a Variable Group named `terraform-secrets`. Create a secret
+named `SQL_PASSWORD` and give it a unique value (e.g. `Strong_Passw0rd!`). Make
+the variable secret using the padlock icon.
+
+![environment approval](/docs/images/terraform_starter/variable_group.png)
+
+### Running the pipeline
+
+As you run the pipeline, after running `terraform plan`, the next stage will be waiting for your approval.
+
+![pipeline stage waiting](/docs/images/terraform_starter/pipeline_stage_waiting.png)
+
+Review the detailed plan to ensure no critical resources or data will be lost.
+
+![terraform plan output](/docs/images/terraform_starter/terraform_plan_output.png)
+
+You can also review the plan and terraform configuration files by navigating to Pipeline Artifacts (rightmost column in the table below).
+
+![pipeline artifacts](/docs/images/terraform_starter/pipeline_artifacts.png)
+
+![pipeline artifacts detail](/docs/images/terraform_starter/pipeline_artifacts_detail.png)
+
+Approve or reject the deployment.
+
+![stage approval waiting](/docs/images/terraform_starter/stage_approval_waiting.png)
+
+The pipeline will proceed to `terraform apply`.
+
+At this stage you will have a new resource group deployed named `rg-starterterraform-stage-main`. 
+
+The pipeline will then proceed in the same manner for the `QA` environment.
+
+![pipeline completed](/docs/images/terraform_starter/pipeline_completed.png)
+
+If any changes have been performed on the infrastructure between the Plan and Apply stages, the pipeline will fail.
+You can rerun the Plan stage directly in the pipeline view to produce an updated plan.
+
+![plan changed](/docs/images/terraform_starter/plan_changed.png)
+
+## Next steps
+
+* It's not currently possible to skip approval and deployment if there are no
+  changes in the Terraform plan, because of limitations in multi-stage
+  pipelines (stages cannot be conditioned on the outputs of previous stages).
+  You could cancel the pipeline (through the REST API) in that case, but that
+  would prevent extending the pipeline to include activities beyond Terraform.
+* The next template, [301-deploy-agent-vms](../301-deploy-agent-vms) demonstrates
+  how you can use Terraform to manage infrastructure used for the build itself,
+  such as build agent VMs.

201-plan-apply-stages/azure-pipelines.yml

@@ -0,0 +1,32 @@
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - 201-plan-apply-stages/
+
+variables:
+- group: terraform-secrets
+
+stages:
+- template: terraform-stages-template.yml
+  parameters:
+    environment: stage
+    environmentDisplayName: Staging
+    TerraformArguments: >-
+      -var department=Engineering
+    # For additional security, pass secret through environment instead of command line.
+    # Terraform recognizes TF_VAR prefixed environment variables.
+    TerraformEnvVariables:
+      TF_VAR_sql2password: $(SQL_PASSWORD)
+
+- template: terraform-stages-template.yml
+  parameters:
+    environment: qa
+    environmentDisplayName: QA
+    TerraformArguments: >-
+      -var department=QA
+    TerraformEnvVariables:
+      TF_VAR_sql2password: $(SQL_PASSWORD)

201-plan-apply-stages/terraform-stages-template.yml

@@ -0,0 +1,160 @@
+parameters:
+  environment: stage
+  environmentDisplayName: Staging
+  TerraformArguments: ''
+  TerraformEnvVariables:
+
+stages:
+- stage: Terraform_Plan_${{ parameters.environment }}
+  displayName: Plan ${{ parameters.environmentDisplayName }}
+  jobs:
+  - job: Terraform_Plan
+    displayName: Plan Terraform
+    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
+    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
+    pool:
+      vmImage: ubuntu-latest
+    steps:
+
+    - bash: |
+        cp terraform_backend/* terraform
+      displayName: Configure backend
+      workingDirectory: 201-plan-apply-stages
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        provisionStorage: true
+        TerraformDirectory: 201-plan-apply-stages/terraform
+        environment: ${{ parameters.environment }}
+
+    # Using bash instead of Terraform extension because of following issues:
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
+    - bash: |
+        set -eu
+        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
+        terraform plan -input=false -out=tfplan -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
+      displayName: Terraform plan
+      workingDirectory: 201-plan-apply-stages/terraform
+      env:
+        ${{ parameters.TerraformEnvVariables }}
+
+    - bash: |
+        # Save a human-friendly version of the plan with passwords hidden
+        terraform show -no-color tfplan > plan.txt
+        # Remove terraform plan from published artifacts, as it contains clear-text secrets
+        rm tfplan
+        # Resource providers can be > 100MB large, we don't want them in the published artifacts.
+        rm -r .terraform
+      displayName: Save plan text
+      workingDirectory: 201-plan-apply-stages/terraform
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish plan artifact
+      inputs:
+        targetPath: 201-plan-apply-stages/terraform
+        artifact: terraform_resources_${{ parameters.environment }}
+
+- stage: Terraform_Apply_${{ parameters.environment }}
+  displayName: Apply ${{ parameters.environmentDisplayName }}
+  jobs:
+  - deployment: Apply
+    environment: ${{ parameters.environmentDisplayName }}
+    displayName: Apply Terraform
+    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
+    pool:
+      vmImage: ubuntu-latest
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+
+          - task: DownloadPipelineArtifact@2
+            displayName: Download plan
+            inputs:
+              artifactName: terraform_resources_${{ parameters.environment }}
+              targetPath: terraform_resources
+
+          - template: ../infrastructure/terraform-init-template.yml
+            parameters:
+              TerraformDirectory: terraform_resources
+              environment: ${{ parameters.environment }}
+
+          # As the Terraform extension plan task doesn't support -detailed-exitcode
+          # (to check if any changes are present), we define an equivalent bash
+          # task instead.
+          - bash: |
+              set -eu
+              export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
+              # terraform plan -detailed-exitcode exit codes:
+              # 0 - Succeeded, diff is empty (no changes)
+              # 1 - Errored
+              # 2 - Succeeded, there is a diff
+              # >2 - unexpected, crash or bug
+              if terraform plan -detailed-exitcode -input=false -out=tfplan -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}; then
+                echo "Terraform succeeded with no changes"
+                # NB terraform apply should still be run, e.g. if new outputs have been created
+              else
+                terraform_exitcode=$?
+                if [ $terraform_exitcode -eq 2 ]; then
+                  echo "Terraform succeeded with updates"
+                else
+                  echo "ERROR: terraform exited with code $terraform_exitcode"
+                  exit 1
+                fi
+              fi
+            displayName: Terraform plan
+            workingDirectory: terraform_resources
+            env:
+              ${{ parameters.TerraformEnvVariables }}
+
+          - bash: |
+              set -eux  # ensure pipeline stops if terraform fails or diff reports a difference
+              terraform show -no-color tfplan > newplan.txt
+              diff -u plan.txt newplan.txt
+            workingDirectory: terraform_resources
+            displayName: Check unchanged plan
+
+          - bash: |
+              set -eu
+              terraform apply -input=false -auto-approve tfplan
+            displayName: Terraform apply
+            workingDirectory: terraform_resources
+            env:
+              ARM_CLIENT_SECRET: $(ARM_CLIENT_SECRET)
+
+  - job: ReadTerraform
+    dependsOn: Apply
+    condition: always()
+    displayName: Read outputs
+    pool:
+      vmImage: ubuntu-latest
+    steps:
+
+    - bash: |
+        cp terraform_backend/* terraform
+      displayName: Configure backend
+      workingDirectory: 201-plan-apply-stages
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        TerraformDirectory: 201-plan-apply-stages/terraform
+        environment: ${{ parameters.environment }}
+
+    - template: ../infrastructure/terraform-outputs-template.yml
+      parameters:
+        TerraformDirectory: 201-plan-apply-stages/terraform
+ 
+  - job: DummySampleJob
+    displayName: Use Terraform outputs
+    dependsOn: ReadTerraform
+    variables:
+      sqlserver1_host: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_host'] ]
+      sqlserver1_user: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_user'] ]
+      sqlserver1_password: $[ dependencies.ReadTerraform.outputs['Outputs.sqlserver1_password'] ]
+    steps:
+      - bash: |
+          # Dummy job showing how to consume Terraform outputs
+          echo DB_CONN_STRING="User ID=$(sqlserver1_user);Password=$(sqlserver1_password)"
+        displayName: Sample script

201-plan-apply-stages/terraform/main.tf

@@ -0,0 +1,35 @@
+# Deploy a Resource Group with Azure resources.
+#
+# For suggested naming conventions, refer to:
+#   https://docs.microsoft.com/en-us/azure/cloud-adoption-framework/ready/azure-best-practices/naming-and-tagging
+
+# Sample Resource Group
+
+resource "azurerm_resource_group" "main" {
+  name     = "rg-${var.appname}-${var.environment}-main"
+  location = var.location
+  tags     = {
+    department = var.department
+  }
+}
+
+# Sample Resources
+
+module "sqlserver1_generated_password" {
+  source = "./sqlserver1_generated_password"
+  appname = var.appname
+  environment = var.environment
+  resource_group = azurerm_resource_group.main.name
+  location = azurerm_resource_group.main.location
+}
+
+module "sqlserver2_assigned_password" {
+  source = "./sqlserver2_assigned_password"
+  appname = var.appname
+  environment = var.environment
+  resource_group = azurerm_resource_group.main.name
+  location = azurerm_resource_group.main.location
+  sql_password = var.sql2password
+}
+
+# Add additional modules...

201-plan-apply-stages/terraform/outputs.tf

@@ -0,0 +1,16 @@
+output "subscription_id" {
+  value = data.azurerm_client_config.current.subscription_id
+}
+
+output "sqlserver1_host" {
+  value = module.sqlserver1_generated_password.fully_qualified_domain_name
+}
+
+output "sqlserver1_user" {
+  value = module.sqlserver1_generated_password.user
+}
+
+output "sqlserver1_password" {
+  value = module.sqlserver1_generated_password.password
+  sensitive = true
+}

201-plan-apply-stages/terraform/provider.tf

@@ -0,0 +1,19 @@
+#Set the terraform required version
+terraform {
+  required_version = ">= 0.12.6"
+}
+
+# Configure the Azure Provider
+provider "azurerm" {
+  # It is recommended to pin to a given version of the Provider
+  version = "=1.44.0"
+}
+
+provider "random" {
+  version = "~> 2.2"
+}
+
+# Data
+
+# Make client_id, tenant_id, subscription_id and object_id variables
+data "azurerm_client_config" "current" {}

201-plan-apply-stages/terraform/sqlserver1_generated_password/main.tf

@@ -0,0 +1,18 @@
+resource "random_password" "sql" {
+  length = 16
+  special = true
+  override_special = "!@#$%&*()-_=+[]:?"
+  min_upper = 1
+  min_lower = 1
+  min_numeric = 1
+  min_special = 1
+}
+
+resource "azurerm_sql_server" "example" {
+  name                         = "sqldb-${var.appname}-${var.environment}"
+  resource_group_name          = var.resource_group
+  location                     = var.location
+  version                      = "12.0"
+  administrator_login          = "sqladm"
+  administrator_login_password = random_password.sql.result
+}

201-plan-apply-stages/terraform/sqlserver1_generated_password/outputs.tf

@@ -0,0 +1,12 @@
+output "fully_qualified_domain_name" {
+  value = azurerm_sql_server.example.fully_qualified_domain_name
+}
+
+output "user" {
+  value = azurerm_sql_server.example.administrator_login
+}
+
+output "password" {
+  value = azurerm_sql_server.example.administrator_login_password
+  sensitive = true
+}

201-plan-apply-stages/terraform/sqlserver1_generated_password/variables.tf

@@ -0,0 +1,15 @@
+variable "appname" {
+  type = string
+}
+
+variable "environment" {
+  type    = string
+}
+
+variable "resource_group" {
+  type    = string
+}
+
+variable "location" {
+  type    = string
+}

201-plan-apply-stages/terraform/sqlserver2_assigned_password/main.tf

@@ -0,0 +1,8 @@
+resource "azurerm_sql_server" "example" {
+  name                         = "sqldb-${var.appname}-2-${var.environment}"
+  resource_group_name          = var.resource_group
+  location                     = var.location
+  version                      = "12.0"
+  administrator_login          = "sqladm"
+  administrator_login_password = var.sql_password
+}

201-plan-apply-stages/terraform/sqlserver2_assigned_password/variables.tf

@@ -0,0 +1,19 @@
+variable "appname" {
+  type = string
+}
+
+variable "environment" {
+  type    = string
+}
+
+variable "resource_group" {
+  type    = string
+}
+
+variable "location" {
+  type    = string
+}
+
+variable "sql_password" {
+  type    = string
+}

201-plan-apply-stages/terraform/variables.tf

@@ -0,0 +1,28 @@
+variable "appname" {
+  type = string
+  description = "Application name. Use only lowercase letters and numbers"
+  default = "starterterraform"
+}
+
+variable "environment" {
+  type    = string
+  description = "Environment name, e.g. 'dev' or 'stage'"
+  default = "dev"
+}
+
+variable "location" {
+  type    = string
+  description = "Azure region where to create resources."
+  default = "North Europe"
+}
+
+variable "department" {
+  type    = string
+  description = "A sample variable passed from the build pipeline and used to tag resources."
+  default = "Engineering"
+}
+
+variable "sql2password" {
+  type    = string
+  description = "A password for SQL Server #2"
+}

201-plan-apply-stages/terraform_backend/backend.tf

@@ -0,0 +1,5 @@
+#Set the terraform backend
+terraform {
+  # Backend variables are initialized by Azure DevOps
+  backend "azurerm" {}
+}

301-deploy-agent-vms/README.md

@@ -0,0 +1,45 @@
+# Deploy hosted agent VMs
+
+## About this template
+
+This template shows how to use Terraform to deploy a pool of agent VMs on which a subsequent job is run.
+
+![agent pool](/docs/images/terraform_starter/301-agent-pool.png)
+
+The Terraform definition does not contain any other resources.
+You can extend the definition with your custom infrastructure, such as Web Apps.
+
+## Walkthrough
+
+### Creating an agent pool
+
+In your Azure DevOps project settings, [create an Agent pool](https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/pools-queues).
+Name the pool `starterpool` (if you want to use a different name, change the value in [azure-pipelines.yml](azure-pipelines.yml)).
+
+starterpool
+
+### Creating a PAT token
+
+In Azure DevOps, [create a PAT token](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=preview-page).
+Click on *Show all scopes* and grant the token *Read and Manage* permissions on *Agent Pools*.
+
+![PAT token](/docs/images/terraform_starter/301-pat-token.png)
+
+Under Library, create a Variable Group named `terraform-secrets`. Create a secret
+named `AGENT_POOL_MANAGE_PAT_TOKEN` and paste the token value
+Make the variable secret using the padlock icon.
+
+### Using the template
+
+To use the template, follow the section
+[How to use the templates](/README.md#how-to-use-the-templates)
+in the main README file.
+
+### Automatic shutdown of agents
+
+The pipeline configures the agent VMs to automatically shutdown daily at 23:00 UTC.
+To use a different schedule, change `TF_VAR_az_devops_agent_vm_shutdown_time`
+in [azure-pipelines.yml](azure-pipelines.yml),
+or remove that line completely to disable automatic shutdown.
+
+The pipeline contains a task to start up the agent VMs again before running the agent job.

301-deploy-agent-vms/azure-pipelines.yml

@@ -0,0 +1,25 @@
+pr: none
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - 301-deploy-agent-vms/
+
+variables:
+- group: terraform-secrets
+
+stages:
+
+- template: terraform-stages-template.yml
+  parameters:
+    environment: dev
+    environmentDisplayName: Dev
+    # Pass variables as environment variables.
+    # Terraform recognizes TF_VAR prefixed environment variables.
+    TerraformEnvVariables:
+      TF_VAR_az_devops_url: $(System.TeamFoundationCollectionUri)
+      TF_VAR_az_devops_pat: $(AGENT_POOL_MANAGE_PAT_TOKEN)
+      TF_VAR_az_devops_agent_pool: starterpool
+      TF_VAR_az_devops_agent_vm_shutdown_time: 2300

301-deploy-agent-vms/terraform-stages-template.yml

@@ -0,0 +1,77 @@
+parameters:
+  environment: agents
+  TerraformArguments: ''
+  TerraformEnvVariables:
+
+stages:
+- stage: Terraform_${{ parameters.environment }}
+  displayName: Terraform ${{ parameters.environmentDisplayName }}
+  pool:
+    vmImage: ubuntu-latest
+  jobs:
+
+  - job: Terraform
+    displayName: Terraform
+    # Avoid concurrent Terraform runs on PRs, which would result in failures due to exclusive lock on remote state file.
+    condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/master'), variables['RUN_FLAG_TERRAFORM']))
+    steps:
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        provisionStorage: true
+        TerraformDirectory: 301-deploy-agent-vms/terraform
+        environment: ${{ parameters.environment }}
+
+    # Using bash instead of Terraform extension because of following issues:
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/748
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/725
+    # - https://github.com/microsoft/azure-pipelines-extensions/issues/747
+    - bash: |
+        set -eu
+        export ARM_CLIENT_SECRET=$(ARM_CLIENT_SECRET)
+        terraform apply -input=false -auto-approve -var environment=${{ parameters.environment }} ${{ parameters.TerraformArguments }}
+      displayName: Terraform apply
+      workingDirectory: 301-deploy-agent-vms/terraform
+      env:
+        ${{ parameters.TerraformEnvVariables }}
+
+- stage: PostTerraform_${{ parameters.environment }}
+  displayName: PostTerraform ${{ parameters.environmentDisplayName }}
+  pool:
+    vmImage: ubuntu-latest
+  jobs:
+  - job: ReadTerraform
+    displayName: Read Terraform outputs
+    steps:
+
+    - template: ../infrastructure/terraform-init-template.yml
+      parameters:
+        TerraformDirectory: 301-deploy-agent-vms/terraform
+        environment: ${{ parameters.environment }}
+
+    - template: ../infrastructure/terraform-outputs-template.yml
+      parameters:
+        TerraformDirectory: 301-deploy-agent-vms/terraform
+
+    - bash: env
+
+    - task: AzureCLI@1
+      displayName: Start agents
+      inputs:
+        azureSubscription: Terraform
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -eux  # fail on error
+          az vm start --ids $(echo $AGENT_VM_IDS | jq -r '.[]') -o none
+
+  - job: DummySampleJob
+    displayName: Run Agent job
+    dependsOn: ReadTerraform
+    variables:
+      pool_name: $[ dependencies.ReadTerraform.outputs['Outputs.pool_name'] ]
+    pool: $(pool_name)
+    steps:
+    - bash: |
+        echo This is running on agent
+        hostname
+      displayName: Sample script

301-deploy-agent-vms/terraform/backend.tf

@@ -0,0 +1,5 @@
+#Set the terraform backend
+terraform {
+  # Backend variables are initialized by Azure DevOps
+  backend "azurerm" {}
+}

301-deploy-agent-vms/terraform/devops-agent/devops_agent_init.sh

@@ -0,0 +1,94 @@
+#!/bin/sh
+
+
+test -n "$1" || { echo "The argument az_devops_url must be provided"; exit 1; }
+az_devops_url="$1"
+[[ "$az_devops_url" == */ ]] || { echo "The argument az_devops_url must end with /"; exit 1; }
+test -n "$2" || { echo "The argument az_devops_pat must be provided"; exit 1; }
+az_devops_pat="$2"
+test -n "$3" || { echo "The argument az_devops_agent_pool must be provided"; exit 1; }
+az_devops_agent_pool="$3"
+test -n "$4" || { echo "The argument az_devops_agents_per_vm must be provided"; exit 1; }
+az_devops_agents_per_vm="$4"
+
+
+#strict mode, fail on error
+set -euo pipefail
+
+
+echo "start"
+
+echo "install Ubuntu packages"
+
+# To make it easier for build and release pipelines to run apt-get,
+# configure apt to not require confirmation (assume the -y argument by default)
+export DEBIAN_FRONTEND=noninteractive
+echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90assumeyes
+echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/00usepty
+
+
+apt-get update
+apt-get install -y --no-install-recommends \
+        ca-certificates \
+        jq \
+        apt-transport-https \
+        docker.io
+
+
+echo "Allowing agent to run docker"
+
+usermod -aG docker azuredevopsuser
+
+echo "Installing Azure CLI"
+
+curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+
+echo "install VSTS Agent"
+
+cd /home/azuredevopsuser
+mkdir -p agent
+cd agent
+
+AGENTRELEASE="$(curl -s https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/latest | grep -oP '"tag_name": "v\K(.*)(?=")')"
+AGENTURL="https://vstsagentpackage.azureedge.net/agent/${AGENTRELEASE}/vsts-agent-linux-x64-${AGENTRELEASE}.tar.gz"
+echo "Release "${AGENTRELEASE}" appears to be latest" 
+echo "Downloading..."
+wget -q -O agent_package.tar.gz ${AGENTURL} 
+
+# Generate random prefix for agent names
+if ! test -e "host_uuid.txt"; then
+  uuidgen > host_uuid.txt.tmp
+  mv host_uuid.txt.tmp host_uuid.txt
+fi
+host_id=$(cat host_uuid.txt)
+
+
+for agent_num in $(seq 1 $az_devops_agents_per_vm); do
+  agent_dir="agent-$agent_num"
+  mkdir -p "$agent_dir"
+  pushd "$agent_dir"
+    agent_id="${agent_num}_${host_id}"
+    echo "installing agent $agent_id"
+    tar zxf ../agent_package.tar.gz
+    chmod -R 777 .
+    echo "extracted"
+    ./bin/installdependencies.sh
+    echo "dependencies installed"
+
+    if test -e .agent; then
+      echo "attempting to uninstall agent"
+      ./svc.sh stop || true
+      ./svc.sh uninstall || true
+      sudo -u azuredevopsuser ./config.sh remove --unattended --auth pat --token "$az_devops_pat" || true
+    fi
+
+    echo "running installation"
+    sudo -u azuredevopsuser ./config.sh --unattended --url "$az_devops_url" --auth pat --token "$az_devops_pat" --pool "$az_devops_agent_pool" --agent "$agent_id" --acceptTeeEula --work ./_work --runAsService
+    echo "configuration done"
+    ./svc.sh install
+    echo "service installed"
+    ./svc.sh start
+    echo "service started"
+    echo "config done"
+  popd
+done

301-deploy-agent-vms/terraform/devops-agent/main.tf

@@ -0,0 +1,187 @@
+resource "azurerm_resource_group" "devops" {
+  name   = "rg-${var.appname}-${var.environment}-devops"
+  location = var.location
+}
+
+# Create virtual network
+
+resource "azurerm_virtual_network" "devops" {
+  name                = "vnet-${var.appname}-devops-${var.environment}"
+  address_space       = ["10.100.0.0/16"]
+  location            = var.location
+  resource_group_name = azurerm_resource_group.devops.name
+}
+
+resource "azurerm_subnet" "devops" {
+  name                 = "agents-subnet"
+  resource_group_name  = azurerm_resource_group.devops.name
+  virtual_network_name = azurerm_virtual_network.devops.name
+  address_prefix       = "10.100.1.0/24"
+}
+
+resource "azurerm_storage_account" "devops" {
+  name                     = "stado${var.appname}${var.environment}"
+  resource_group_name      = azurerm_resource_group.devops.name
+  location                 = azurerm_resource_group.devops.location
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+}
+
+resource "azurerm_storage_container" "devops" {
+  name                  = "content"
+  storage_account_name  = azurerm_storage_account.devops.name
+  container_access_type = "private"
+}
+
+resource "azurerm_storage_blob" "devops" {
+  name                   = "devops_agent_init-${md5(file("${path.module}/devops_agent_init.sh"))}.sh"
+  storage_account_name   = azurerm_storage_account.devops.name
+  storage_container_name = azurerm_storage_container.devops.name
+  type                   = "Block"
+  source                 = "${path.module}/devops_agent_init.sh"
+}
+
+data "azurerm_storage_account_blob_container_sas" "devops_agent_init" {
+  connection_string = azurerm_storage_account.devops.primary_connection_string
+  container_name    = azurerm_storage_container.devops.name
+  https_only        = true
+
+  start  = "2000-01-01"
+  expiry = "2099-01-01"
+
+  permissions {
+    read   = true
+    add    = false
+    create = false
+    write  = false
+    delete = false
+    list   = false
+  }
+}
+
+
+# Create public IPs
+resource "azurerm_public_ip" "devops" {
+  name                = "pip-${var.appname}-devops-${var.environment}-${format("%03d", count.index + 1)}"
+  location            = var.location
+  resource_group_name = azurerm_resource_group.devops.name
+  allocation_method   = "Dynamic"
+  count               = var.az_devops_agent_vm_count
+}
+
+# Create network interface
+resource "azurerm_network_interface" "devops" {
+  name                      = "nic-${var.appname}-devops-${var.environment}-${format("%03d", count.index + 1)}"
+  location                  = var.location
+  resource_group_name       = azurerm_resource_group.devops.name
+
+  ip_configuration {
+    name                          = "AzureDevOpsNicConfiguration"
+    subnet_id                     = azurerm_subnet.devops.id
+    private_ip_address_allocation = "dynamic"
+    public_ip_address_id          = azurerm_public_ip.devops[count.index].id
+  }
+
+  count                     = var.az_devops_agent_vm_count
+}
+
+# Create virtual machine
+
+resource "random_password" "agent_vms" {
+  length = 24
+  special = true
+  override_special = "!@#$%&*()-_=+[]:?"
+  min_upper = 1
+  min_lower = 1
+  min_numeric = 1
+  min_special = 1
+}
+
+resource "azurerm_virtual_machine" "devops" {
+  name                  = "vm${var.appname}devops${var.environment}-${format("%03d", count.index + 1)}"
+  location              = var.location
+  resource_group_name   = azurerm_resource_group.devops.name
+  network_interface_ids = [azurerm_network_interface.devops[count.index].id]
+  vm_size               = var.az_devops_agent_vm_size
+
+  storage_os_disk {
+    name              = "osdisk${var.appname}devops${var.environment}${format("%03d", count.index + 1)}"
+    caching           = "ReadWrite"
+    create_option     = "FromImage"
+    managed_disk_type = "Premium_LRS"
+  }
+
+  storage_image_reference {
+    publisher = "Canonical"
+    offer     = "UbuntuServer"
+    sku       = "16.04.0-LTS"
+    version   = "latest"
+  }
+
+  os_profile {
+    computer_name  = "AzureDevOps"
+    admin_username = "azuredevopsuser"
+    admin_password = random_password.agent_vms.result
+  }
+
+  os_profile_linux_config {
+    disable_password_authentication = false
+
+    dynamic "ssh_keys" {
+      for_each = var.az_devops_agent_sshkeys
+      content {
+        key_data = each.key
+        path = "/home/azuredevopsuser/.ssh/authorized_keys"
+      }
+    }
+  }
+
+  boot_diagnostics {
+    enabled     = "true"
+    storage_uri = azurerm_storage_account.devops.primary_blob_endpoint
+  }
+
+  count = var.az_devops_agent_vm_count
+}
+
+resource "azurerm_virtual_machine_extension" "devops" {
+  name                 = format("install_azure_devops_agent-%03d", count.index + 1)
+  virtual_machine_id   = azurerm_virtual_machine.devops[count.index].id
+  publisher            = "Microsoft.Azure.Extensions"
+  type                 = "CustomScript"
+  type_handler_version = "2.0"
+
+  #timestamp: use this field only to trigger a re-run of the script by changing value of this field.
+  #           Any integer value is acceptable; it must only be different than the previous value.
+  settings = jsonencode({
+    "timestamp" : 1
+  })
+  protected_settings = jsonencode({
+  "fileUris": ["${azurerm_storage_blob.devops.url}${data.azurerm_storage_account_blob_container_sas.devops_agent_init.sas}"],
+  "commandToExecute": "bash ${azurerm_storage_blob.devops.name} '${var.az_devops_url}' '${var.az_devops_pat}' '${var.az_devops_agent_pool}' '${var.az_devops_agents_per_vm}'"
+  })
+  count = var.az_devops_agent_vm_count
+}
+
+resource "azurerm_template_deployment" "devops_shutdown" {
+  name = format("shutdown-vm-%03d", count.index + 1)
+  resource_group_name = azurerm_resource_group.devops.name
+
+  template_body = file("${path.module}/shutdown_schedule_arm_template.json")
+
+  parameters = {
+    name = "shutdown-computevm-${azurerm_virtual_machine.devops[count.index].name}"
+    shutdown_enabled = var.az_devops_agent_vm_shutdown_time != null ? "Enabled" : "Disabled"
+    shutdown_time = coalesce(var.az_devops_agent_vm_shutdown_time, "0000")
+    vm_id = azurerm_virtual_machine.devops[count.index].id
+  }
+
+  depends_on = [
+    azurerm_virtual_machine.devops
+  ]
+
+  deployment_mode = "Incremental"
+
+  count = var.az_devops_agent_vm_count
+}
+

301-deploy-agent-vms/terraform/devops-agent/outputs.tf

@@ -0,0 +1,3 @@
+output "agent_vm_ids" {
+  value = azurerm_virtual_machine.devops.*.id
+}

301-deploy-agent-vms/terraform/devops-agent/shutdown_schedule_arm_template.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "name": {
+      "type": "string"
+    },
+    "shutdown_enabled": {
+      "type": "string"
+    },
+    "shutdown_time": {
+      "type": "string"
+    },
+    "vm_id": {
+      "type": "string"
+    }
+  },
+  "resources": [
+    {
+      "type": "microsoft.devtestlab/schedules",
+      "apiVersion": "2018-09-15",
+      "name": "[parameters('name')]",
+      "location": "northeurope",
+      "properties": {
+        "status": "[parameters('shutdown_enabled')]",
+        "taskType": "ComputeVmShutdownTask",
+        "dailyRecurrence": {
+          "time": "[parameters('shutdown_time')]"
+        },
+        "timeZoneId": "UTC",
+        "notificationSettings": {
+          "status": "Disabled",
+          "timeInMinutes": 30,
+          "notificationLocale": "en"
+        },
+        "targetResourceId": "[parameters('vm_id')]"
+      }
+    }
+  ]
+}

301-deploy-agent-vms/terraform/devops-agent/variables.tf

@@ -0,0 +1,56 @@
+variable "appname" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "location" {
+  type = string
+}
+
+variable "az_devops_url" {
+  type = string
+  description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg"
+}
+
+variable "az_devops_pat" {
+  type = string
+  description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage"
+}
+
+variable "az_devops_agent_pool" {
+  type = string
+  description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools"
+  default = "pool001"
+}
+
+variable "az_devops_agent_sshkeys" {
+  type        = list(string)
+  description = "Optionally provide ssh public key(s) to logon to the VM"
+}
+
+variable "az_devops_agent_vm_size" {
+  type    = string
+  description = "Specify the size of the VM"
+  default = "Standard_D2s_v3"
+}
+
+variable "az_devops_agent_vm_count" {
+  type    = number
+  description = "Number of Azure DevOps agent VMs"
+  default = 1
+}
+
+variable "az_devops_agent_vm_shutdown_time" {
+  type    = string
+  description = "UTC Time at which to shutdown the agent VMs daily, for example '2000' for 8 PM"
+  default = null
+}
+
+variable "az_devops_agents_per_vm" {
+  type = number
+  description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix."
+  default = 4
+}

301-deploy-agent-vms/terraform/main.tf

@@ -0,0 +1,17 @@
+# Azure DevOps agent VMs
+
+module "devops-agent" {
+  source = "./devops-agent"
+  appname = var.appname
+  environment = var.environment
+  location = var.location
+  az_devops_url = var.az_devops_url
+  az_devops_pat = var.az_devops_pat
+  az_devops_agent_pool = var.az_devops_agent_pool
+  az_devops_agents_per_vm = var.az_devops_agents_per_vm
+  az_devops_agent_sshkeys = var.az_devops_agent_sshkeys
+  az_devops_agent_vm_size = var.az_devops_agent_vm_size
+  az_devops_agent_vm_count = var.az_devops_agent_vm_count
+  az_devops_agent_vm_shutdown_time = var.az_devops_agent_vm_shutdown_time
+}
+

301-deploy-agent-vms/terraform/outputs.tf

@@ -0,0 +1,7 @@
+output "pool_name" {
+  value = var.az_devops_agent_pool
+}
+
+output "agent_vm_ids" {
+  value = module.devops-agent.agent_vm_ids
+}

301-deploy-agent-vms/terraform/provider.tf

@@ -0,0 +1,19 @@
+#Set the terraform required version
+terraform {
+  required_version = ">= 0.12.6"
+}
+
+# Configure the Azure Provider
+provider "azurerm" {
+  # It is recommended to pin to a given version of the Provider
+  version = "=1.44.0"
+}
+
+provider "random" {
+  version = "~> 2.2"
+}
+
+# Data
+
+# Make client_id, tenant_id, subscription_id and object_id variables
+data "azurerm_client_config" "current" {}

301-deploy-agent-vms/terraform/variables.tf

@@ -0,0 +1,69 @@
+variable "appname" {
+  type = string
+  description = "Application name. Use only lowercase letters and numbers"
+  default = "starterterraform"
+}
+
+variable "environment" {
+  type    = string
+  description = "Environment name, e.g. 'dev' or 'stage'"
+  default = "dev"
+}
+
+variable "location" {
+  type    = string
+  description = "Azure region where to create resources."
+  default = "North Europe"
+}
+
+variable "department" {
+  type    = string
+  description = "A sample variable passed from the build pipeline and used to tag resources."
+  default = "Engineering"
+}
+
+variable "az_devops_url" {
+  type = string
+  description = "Specify the Azure DevOps url e.g. https://dev.azure.com/myorg"
+}
+
+variable "az_devops_pat" {
+  type = string
+  description = "Provide a Personal Access Token (PAT) for Azure DevOps. Create it at https://dev.azure.com/[Organization]/_usersSettings/tokens with permission Agent Pools > Read & manage"
+}
+
+variable "az_devops_agent_pool" {
+  type = string
+  description = "Specify the name of the agent pool - must exist before. Create it at https://dev.azure.com/[Organization]/_settings/agentpools"
+  default = "pool001"
+}
+
+variable "az_devops_agent_sshkeys" {
+  type        = list(string)
+  description = "Optionally provide ssh public key(s) to logon to the VM"
+  default     = []
+}
+
+variable "az_devops_agent_vm_size" {
+  type    = string
+  description = "Specify the size of the VM"
+  default = "Standard_D2s_v3"
+}
+
+variable "az_devops_agent_vm_count" {
+  type    = number
+  description = "Number of Azure DevOps agent VMs"
+  default = 2
+}
+
+variable "az_devops_agents_per_vm" {
+  type = number
+  description = "Number of Azure DevOps agents spawned per VM. Agents will be named with a random prefix."
+  default = 4
+}
+
+variable "az_devops_agent_vm_shutdown_time" {
+  type    = string
+  description = "UTC Time at which to shutdown the agent VMs daily, for example '2000' for 8 PM. If null, no shutdown will configured."
+  default = null
+}

README.md

@@ -1,64 +1,260 @@
----
-page_type: sample
-languages:
-- csharp
-products:
-- dotnet
-description: "Add 150 character max description"
-urlFragment: "update-this-to-unique-url-stub"
----
-
-# Official Microsoft Sample
-
-<!-- 
-Guidelines on README format: https://review.docs.microsoft.com/help/onboard/admin/samples/concepts/readme-template?branch=master
-
-Guidance on onboarding samples to docs.microsoft.com/samples: https://review.docs.microsoft.com/help/onboard/admin/samples/process/onboarding?branch=master
-
-Taxonomies for products and languages: https://review.docs.microsoft.com/new-hope/information-architecture/metadata/taxonomies?branch=master
--->
-
-Give a short description for your sample here. What does it do and why is it important?
-
-## Contents
-
-Outline the file contents of the repository. It helps users navigate the codebase, build configuration and any related assets.
-
-| File/folder       | Description                                |
-|-------------------|--------------------------------------------|
-| `src`             | Sample source code.                        |
-| `.gitignore`      | Define what to ignore at commit time.      |
-| `CHANGELOG.md`    | List of changes to the sample.             |
-| `CONTRIBUTING.md` | Guidelines for contributing to the sample. |
-| `README.md`       | This README file.                          |
-| `LICENSE`         | The license for the sample.                |
-
-## Prerequisites
-
-Outline the required components and tools that a user might need to have on their machine in order to run the sample. This can be anything from frameworks, SDKs, OS versions or IDE releases.
-
-## Setup
-
-Explain how to prepare the sample once the user clones or downloads the repository. The section should outline every step necessary to install dependencies and set up any settings (for example, API keys and output folders).
-
-## Running the sample
-
-Outline step-by-step instructions to execute the sample and see its output. Include steps for executing the sample from the IDE, starting specific services in the Azure portal or anything related to the overall launch of the code.
-
-## Key concepts
-
-Provide users with more context on the tools and services used in the sample. Explain some of the code that is being used and how services interact with each other.
-
-## Contributing
-
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
-
-When you submit a pull request, a CLA bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
-
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+---
+page_type: sample
+products:
+- devops
+description: "Started project for Azure Pipelines deploying resources on Terraform"
+---
+
+# Terraform starter project for Azure Pipelines
+
+<!-- 
+Guidelines on README format: https://review.docs.microsoft.com/help/onboard/admin/samples/concepts/readme-template?branch=master
+
+Guidance on onboarding samples to docs.microsoft.com/samples: https://review.docs.microsoft.com/help/onboard/admin/samples/process/onboarding?branch=master
+
+Taxonomies for products and languages: https://review.docs.microsoft.com/new-hope/information-architecture/metadata/taxonomies?branch=master
+-->
+
+This project can be used as a starter for Azure Pipelines deploying resources on Terraform.
+
+![pipeline jobs](/docs/images/terraform_starter/pipeline_jobs.png)
+
+## Contents
+
+| File/folder             | Description                                                  |
+|-------------------------|--------------------------------------------------------------|
+| `infrastructure`        | YAML pipeline templates shared across the samples.           |
+| `101-terraform-job`     | Sample YAML pipeline for a simple Terraform job.             |
+| `201-plan-apply-stages` | Sample YAML pipeline for a simple Terraform job.             |
+| `301-deploy-agent-vms`  | Sample YAML pipeline for a simple Terraform job.             |
+| `docs`                  | Resources related to documentation.                          |
+| `CODE_OF_CONDUCT.md`    | Microsoft Open Source Code of Conduct.                       |
+| `LICENSE`               | The license for the sample.                                  |
+| `README.md`             | This README file.                                            |
+| `SECURITY.md`           | Reporting security issues.                                   |
+
+# Templates
+
+## 101 Basic Terraform job
+
+The first template shows how to build an environment from Terraform configuration, and run
+a subsequent job configured from Terraform outputs.
+
+[101-terraform-job: Basic Terraform job](101-terraform-job)
+
+![pipeline job](/docs/images/terraform_starter/101-terraform-job.png)
+
+## 201 Separate Plan and Apply stages
+
+The next template shows how to build a multi-stage pipeline
+allowing to manually review and approve infrastructure changes before they are deployed.
+
+[201-plan-apply-stages: Separate Plan and Apply stages](201-plan-apply-stages)
+
+![pipeline jobs](docs/images/terraform_starter/pipeline_stage_waiting.png)
+
+## 301 Deploy hosted agent VMs
+
+The next template shows how to use Terraform to deploy a pool of agent VMs on which to run
+subsequent jobs.
+
+![agent job](/docs/images/terraform_starter/301-agent-job.png)
+
+[301-deploy-agent-vms: Deploy hosted agent VMs](301-deploy-agent-vms)
+
+# How to use the templates
+
+## Variables and state management
+
+Variables can be injected using `-var key=value` syntax in the `TerraformArguments` parameter.
+The pipeline demonstrates this by adding a custom tag named `department` to the
+created resource group, with distinct values in staging and QA.
+
+Rather than passing a Terraform plan between stages (which would contain clear-text secrets),
+the pipeline performs `terraform plan` again before applying changes and verifies that
+a textual representation of the plan (not including secrets values) is unchanged.
+
+The Terraform state is managed in a Azure Storage backend. Note that this backend contains
+secrets in cleartext.
+
+## Secrets management
+
+### Generate secrets with Terraform
+
+To demonstrate one approach to secrets management, the Terraform configuration
+generates a random password (per stage) for the SQL Server 1 instance, stored in
+Terraform state.
+You can adapt this to suit your lifecycle.
+
+### Manage secrets with Azure DevOps
+
+You might want to read credentials from an externally managed Key Vault
+or inject them via pipeline variables. This approach is demonstrated
+by defining a password for the SQL Server 2 instance and passing
+it to Terraform via an environment variable.
+
+## Getting started
+
+In `infrastructure/terraform/variables.tf`, change the `appname` default value from
+`starterterraform` to a globally unique name.
+
+## Azure DevOps pipeline
+
+Install the [Terraform extension for Azure DevOps](https://marketplace.visualstudio.com/items?itemName=ms-devlabs.custom-terraform-tasks).
+
+Create a Service Connection of type Azure Resource Manager at subscription scope. Name the Service Connection `Terraform`.
+Allow all pipelines to use the connection.
+
+In `infrastructure/terraform-init-template.yml`, update the `TerraformBackendStorageAccount` name to a globally unique storage account name.
+The pipeline will create the storage account.
+
+Create a build pipeline referencing `infrastructure/azure-pipelines.yml`.
+
+## Usage on non-master branch
+
+To avoid issues with concurrent access to the Terraform state file, the jobs running Terraform `plan` and `apply` commands
+run by default only on the `master` branch. On other branches, they are skipped by default:
+
+![run on non-master branch](/docs/images/terraform_starter/non_master_branch.png)
+
+You can set the `RUN_FLAG_TERRAFORM` variable (to any non-empty value)
+when running the pipeline, to trigger Terraform application on a non-`master` branch.
+
+## Local development
+
+In local development, no backend is configured so a local backend is used.
+
+Install Azure CLI and login. Terraform will use your Azure CLI credentials.
+
+```
+$ az login -o table
+You have logged in. Now let us find all the subscriptions to which you have access...
+CloudName    IsDefault    Name                                                  State    TenantId
+-----------  -----------  ----------------------------------------------------  -------  ------------------------------------
+AzureCloud   True         My Azure subscription                                 Enabled  xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+AzureCloud   False        My other Azure subscription                           Enabled  xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```
+
+Run `terraform init`.
+
+```
+$ terraform init
+
+Initializing the backend...
+
+Initializing provider plugins...
+- Checking for available provider plugins...
+- Downloading plugin for provider "azurerm" (hashicorp/azurerm) 1.38.0...
+
+Terraform has been successfully initialized!
+
+You may now begin working with Terraform. Try running "terraform plan" to see
+any changes that are required for your infrastructure. All Terraform commands
+should now work.
+
+If you ever set or change modules or backend configuration for Terraform,
+rerun this command to reinitialize your working directory. If you forget, other
+commands will detect it and remind you to do so if necessary.
+```
+
+Run `terraform plan`.
+
+```
+$ terraform plan -out tfplan
+Refreshing Terraform state in-memory prior to plan...
+The refreshed state will be used to calculate this plan, but will not be
+persisted to local or remote state storage.
+
+data.azurerm_client_config.current: Refreshing state...
+
+------------------------------------------------------------------------
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
+  + create
+
+Terraform will perform the following actions:
+
+  # azurerm_resource_group.main will be created
+  + resource "azurerm_resource_group" "main" {
+      + id       = (known after apply)
+      + location = "northeurope"
+      + name     = "rg-starterterraform-dev-main"
+      + tags     = (known after apply)
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+```
+
+Run `terraform apply tfplan`.
+
+```
+$ terraform apply tfplan
+data.azurerm_client_config.current: Refreshing state...
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
+  + create
+
+Terraform will perform the following actions:
+
+  # azurerm_resource_group.main will be created
+  + resource "azurerm_resource_group" "main" {
+      + id       = (known after apply)
+      + location = "northeurope"
+      + name     = "rg-starterterraform-dev-main"
+      + tags     = (known after apply)
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+
+Do you want to perform these actions?
+  Terraform will perform the actions described above.
+  Only 'yes' will be accepted to approve.
+
+  Enter a value: yes
+
+azurerm_resource_group.main: Creating...
+azurerm_resource_group.main: Creation complete after 1s [id=/subscriptions/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx/resourceGroups/rg-starterterraform-dev-main]
+
+Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
+
+Outputs:
+
+subscription_id = xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```
+
+At this stage you will have a new resource group deployed named `rg-starterterraform-dev-main`. 
+
+# Using Terraform outputs
+
+The pipeline automatically exports Terraform outputs into pipeline variables.
+
+The pipelines contain a sample job that consumes those variables:
+
+![output variables](/docs/images/terraform_starter/output_variables.png)
+
+- For example, in the template [301-deploy-agent-vms](301-deploy-agent-vms), the Terraform config has an output named [agent_vm_ids](301-deploy-agent-vms/terraform/outputs.tf). In the subsequent task used
+we use the bash variable [AGENT_VM_IDS](301-deploy-agent-vms/terraform-stages-template.yml) to pass the list of agent VMs to the `az start` command.
+
+This mechanism is useful for using generated resource names, access keys,
+and even [entire kube_config files](https://www.terraform.io/docs/providers/azurerm/r/kubernetes_cluster.html#kube_config_raw) (for Azure Kubernetes Service)
+in downstream testing or continuous delivery jobs.
+
+# Next steps
+
+* You can of course adapt the pipeline to other environments, such as Production.
+
+## Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
+
+When you submit a pull request, a CLA bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

infrastructure/terraform-init-template.yml

@@ -0,0 +1,96 @@
+parameters:
+  environment: stage
+  provisionStorage: false
+  TerraformVersion: 0.12.21
+  TerraformDirectory:
+  TerraformBackendServiceConnection: Terraform
+  TerraformEnvironmentServiceConnection: Terraform
+  TerraformBackendResourceGroup: terraform
+  TerraformBackendStorageAccount: terraformstarterstate
+  TerraformBackendStorageContainer: terraformstate
+  TerraformBackendLocation: North Europe
+
+steps:
+
+- task: AzureCLI@1
+  displayName: Set Terraform backend
+  condition: and(succeeded(), ${{ parameters.provisionStorage }})
+  inputs:
+    azureSubscription: ${{ parameters.TerraformBackendServiceConnection }}
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -eu  # fail on error
+      RG='${{ parameters.TerraformBackendResourceGroup }}'
+      export AZURE_STORAGE_ACCOUNT='${{ parameters.TerraformBackendStorageAccount }}'
+      export AZURE_STORAGE_KEY="$(az storage account keys list -g "$RG" -n "$AZURE_STORAGE_ACCOUNT" --query '[0].value' -o tsv)"
+      if test -z "$AZURE_STORAGE_KEY"; then
+        az configure --defaults group="$RG" location='${{ parameters.TerraformBackendLocation }}'
+        az group create -n "$RG" -o none
+        az storage account create -n "$AZURE_STORAGE_ACCOUNT" -o none
+        export AZURE_STORAGE_KEY="$(az storage account keys list -g "$RG" -n "$AZURE_STORAGE_ACCOUNT" --query '[0].value' -o tsv)"
+      fi
+
+      container='${{ parameters.TerraformBackendStorageContainer }}'
+      if ! az storage container show -n "$container" -o none 2>/dev/null; then
+        az storage container create -n "$container" -o none
+      fi
+      blob='${{ parameters.environment }}.tfstate'
+      if [[ $(az storage blob exists -c "$container" -n "$blob" --query exists) = "true" ]]; then
+        if [[ $(az storage blob show -c "$container" -n "$blob" --query "properties.lease.status=='locked'") = "true" ]]; then
+          echo "State is leased"
+          lock_jwt=$(az storage blob show -c "$container" -n "$blob" --query metadata.terraformlockid -o tsv)
+          if [ "$lock_jwt" != "" ]; then
+            lock_json=$(base64 -d <<< "$lock_jwt")
+            echo "State is locked"
+            jq . <<< "$lock_json"
+          fi
+          if [ "${TERRAFORM_BREAK_LEASE:-}" != "" ]; then
+            az storage blob lease break -c "$container" -b "$blob"
+          else
+            echo "If you're really sure you want to break the lease, rerun the pipeline with variable TERRAFORM_BREAK_LEASE set to 1."
+            exit 1
+          fi
+        fi
+      fi
+    addSpnToEnvironment: true
+
+- task: ms-devlabs.custom-terraform-tasks.custom-terraform-installer-task.TerraformInstaller@0
+  displayName: Install Terraform
+  inputs:
+    terraformVersion: ${{ parameters.TerraformVersion }}
+
+- task: AzureCLI@1
+  displayName: Terraform credentials
+  inputs:
+    azureSubscription: ${{ parameters.TerraformEnvironmentServiceConnection }}
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -eu
+      subscriptionId=$(az account show --query id -o tsv)
+      echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$servicePrincipalId"
+      echo "##vso[task.setvariable variable=ARM_CLIENT_SECRET;issecret=true]$servicePrincipalKey"
+      echo "##vso[task.setvariable variable=ARM_SUBSCRIPTION_ID]$subscriptionId"
+      echo "##vso[task.setvariable variable=ARM_TENANT_ID]$tenantId"
+    addSpnToEnvironment: true
+
+# Using bash instead of Terraform extension because of following issue:
+# - https://github.com/microsoft/azure-pipelines-extensions/issues/738
+- task: AzureCLI@1
+  displayName: Terraform init
+  inputs:
+    azureSubscription: ${{ parameters.TerraformBackendServiceConnection }}
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -eux  # fail on error
+      subscriptionId=$(az account show --query id -o tsv)
+      terraform init \
+        -backend-config=storage_account_name=${{ parameters.TerraformBackendStorageAccount }} \
+        -backend-config=container_name=${{ parameters.TerraformBackendStorageContainer }} \
+        -backend-config=key=${{ parameters.environment }}.tfstate \
+        -backend-config=resource_group_name=${{ parameters.TerraformBackendResourceGroup }} \
+        -backend-config=subscription_id=$subscriptionId \
+        -backend-config=tenant_id=$tenantId \
+        -backend-config=client_id=$servicePrincipalId \
+        -backend-config=client_secret="$servicePrincipalKey"
+    workingDirectory: ${{ parameters.TerraformDirectory }}
+    addSpnToEnvironment: true

infrastructure/terraform-outputs-template.yml

@@ -0,0 +1,29 @@
+parameters:
+  TerraformDirectory:
+
+steps:
+- bash: |
+    set -eu
+
+    echo "Setting job variables from Terraform outputs:"
+    terraform output -json | jq -r '
+      . as $in
+      | keys[]
+      | ["- " + .]
+      | @tsv'
+
+    terraform output -json | jq -r '
+      . as $in
+      | keys[]
+      | ($in[.].value | tostring) as $value
+      | ($in[.].sensitive | tostring) as $sensitive
+      | [
+        "- " + . + ": " + if $in[.].sensitive then "(sensitive)" else $value end,  # output name to console
+        "##vso[task.setvariable variable=" + . + ";isSecret=" + $sensitive + "]" + $value,  # set as ADO task variable
+        "##vso[task.setvariable variable=" + . + ";isOutput=true;isSecret=" + $sensitive + "]" + $value  # also set as ADO job variable
+        ]
+      | .[]'
+
+  name: Outputs
+  displayName: Read Terraform outputs
+  workingDirectory: ${{ parameters.TerraformDirectory }}