Switch artifact collection and some variables to ps1 scripts

.gitignore

@@ -55,11 +55,6 @@ dlldata.c
 # Benchmark Results
 BenchmarkDotNet.Artifacts/
 
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-
 # StyleCop
 StyleCopReport.xml
 
@@ -346,4 +341,4 @@ ASALocalRun/
 healthchecksdb
 
 # Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
+MigrationBackup/

azure-pipelines.yml

@@ -30,15 +30,13 @@ jobs:
     workingDirectory: src
 
   - template: azure-pipelines/dotnet.yml
-  - template: azure-pipelines/collect-deployables.yml
-  - template: azure-pipelines/collect-logs.yml
   - template: azure-pipelines/expand-template.yml
 
   - task: NuGetCommand@2
     displayName: Push packages to CI feed
     inputs:
       command: push
-      packagesToPush: $(Build.ArtifactStagingDirectory)/deployables/*.nupkg;$(Build.ArtifactStagingDirectory)/deployables/*.snupkg
+      packagesToPush: $(Build.ArtifactStagingDirectory)/deployables-Windows/*.nupkg;$(Build.ArtifactStagingDirectory)/deployables-Windows/*.snupkg
       nuGetFeedType: internal
       publishVstsFeed: $(ci_feed)
       allowPackageConflicts: true
@@ -50,7 +48,6 @@ jobs:
   steps:
   - template: azure-pipelines/install-dependencies.yml
   - template: azure-pipelines/dotnet.yml
-  - template: azure-pipelines/collect-logs.yml
   - template: azure-pipelines/expand-template.yml
 
 - job: macOS
@@ -59,7 +56,6 @@ jobs:
   steps:
   - template: azure-pipelines/install-dependencies.yml
   - template: azure-pipelines/dotnet.yml
-  - template: azure-pipelines/collect-logs.yml
   - template: azure-pipelines/expand-template.yml
 
 - job: MergeCoverage

azure-pipelines/artifacts/Variables.ps1

@@ -0,0 +1,40 @@
+# This artifact captures all variables defined in the ..\variables folder.
+# It "snaps" the values of these variables where we can compute them during the build,
+# and otherwise captures the scripts to run later during an Azure Pipelines environment release.
+
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+$ArtifactBasePath = "$RepoRoot\obj\_artifacts"
+$VariablesArtifactPath = "$ArtifactBasePath\variables"
+if (-not (Test-Path $VariablesArtifactPath)) { New-Item -ItemType Directory -Path $VariablesArtifactPath | Out-Null }
+
+# Copy variables, either by value if the value is calculable now, or by script
+Get-ChildItem -Path "$PSScriptRoot\..\variables" |% {
+    $value = $null
+    if (-not $_.BaseName.StartsWith('_')) { # Skip trying to interpret special scripts
+        # First check the environment variables in case the variable was set in a queued build
+        if (Test-Path env:$($_.BaseName)) {
+            $value = Get-Content "env:$($_.BaseName)"
+        }
+
+        # If that didn't give us anything, try executing the script right now from its original position
+        if (-not $value) {
+            $value = & $_.FullName
+        }
+
+        if ($value) {
+            # We got something, so wrap it with quotes so it's treated like a literal value.
+            $value = "'$value'"
+        }
+    }
+
+    # If that didn't get us anything, just copy the script itself
+    if (-not $value) {
+        $value = Get-Content -Path $_.FullName
+    }
+
+    Set-Content -Path "$VariablesArtifactPath\$($_.Name)" -Value $value
+}
+
+@{
+    "$VariablesArtifactPath" = (Get-ChildItem $VariablesArtifactPath -Recurse);
+}

azure-pipelines/artifacts/_all.ps1

@@ -0,0 +1,45 @@
+# This script returns all the artifacts that should be collected after a build.
+#
+# Each powershell artifact is expressed as an object with these properties:
+#  Source          - the full path to the source file
+#  ArtifactName    - the name of the artifact to upload to
+#  ContainerFolder - the relative path within the artifact in which the file should appear
+#
+# Each artifact aggregating .ps1 script should return a hashtable:
+#   Key = path to the directory from which relative paths within the artifact should be calculated
+#   Value = an array of paths (absolute or relative to the BaseDirectory) to files to include in the artifact.
+#           FileInfo objects are also allowed.
+
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+
+Function EnsureTrailingSlash($path) {
+    if ($path.length -gt 0 -and !$path.EndsWith('\') -and !$path.EndsWith('/')) {
+        $path = $path + [IO.Path]::DirectorySeparatorChar
+    }
+
+    $path.Replace('\', [IO.Path]::DirectorySeparatorChar)
+}
+
+Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" -Recurse |% {
+    $ArtifactName = $_.BaseName
+
+    (& $_).GetEnumerator() |% {
+        $BaseDirectory = New-Object Uri ((EnsureTrailingSlash $_.Key), [UriKind]::Absolute)
+        $_.Value |% {
+            if ($_.GetType() -eq [IO.FileInfo] -or $_.GetType() -eq [IO.DirectoryInfo]) {
+                $_ = $_.FullName
+            }
+
+            $artifact = New-Object -TypeName PSObject
+            Add-Member -InputObject $artifact -MemberType NoteProperty -Name ArtifactName -Value $ArtifactName
+
+            $SourceFullPath = New-Object Uri ($BaseDirectory, $_)
+            Add-Member -InputObject $artifact -MemberType NoteProperty -Name Source -Value $SourceFullPath.LocalPath
+
+            $RelativePath = [Uri]::UnescapeDataString($BaseDirectory.MakeRelative($SourceFullPath))
+            Add-Member -InputObject $artifact -MemberType NoteProperty -Name ContainerFolder -Value (Split-Path $RelativePath)
+
+            Write-Output $artifact
+        }
+    }
+}

azure-pipelines/artifacts/_pipelines.ps1

@@ -0,0 +1,52 @@
+# This script translates all the artifacts described by _all.ps1
+# into commands that instruct VSTS to actually collect those artifacts.
+
+param (
+    [string]$ArtifactNameSuffix
+)
+
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+if (!$env:BUILDCONFIGURATION) { throw "BUILDCONFIGURATION environment variable must be set." }
+if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) {
+    $ArtifactStagingFolder = $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+} else {
+    $ArtifactStagingFolder = "$RepoRoot\obj\_artifacts"
+    if (Test-Path $ArtifactStagingFolder) {
+        Remove-Item $ArtifactStagingFolder -Recurse -Force
+    }
+}
+
+function Create-SymbolicLink {
+    param (
+        $Link,
+        $Target
+    )
+
+    if ($Link -eq $Target) {
+        return
+    }
+
+    if (Test-Path $Link) { Remove-Item $Link }
+    $LinkContainer = Split-Path $Link -Parent
+    if (!(Test-Path $LinkContainer)) { mkdir $LinkContainer }
+    Write-Verbose "Linking $Link to $Target"
+    ln $Target $Link
+}
+
+# Stage all artifacts
+$Artifacts = & "$PSScriptRoot\_all.ps1"
+$Artifacts |% {
+    $DestinationFolder = (Join-Path (Join-Path $ArtifactStagingFolder "$($_.ArtifactName)$ArtifactNameSuffix") $_.ContainerFolder).TrimEnd('\')
+    $Name = "$(Split-Path $_.Source -Leaf)"
+
+    #Write-Host "$($_.Source) -> $($_.ArtifactName)\$($_.ContainerFolder)" -ForegroundColor Yellow
+
+    if (-not (Test-Path $DestinationFolder)) { New-Item -ItemType Directory -Path $DestinationFolder | Out-Null }
+    if (Test-Path -PathType Leaf $_.Source) { # skip folders
+        Create-SymbolicLink -Link "$DestinationFolder\$Name" -Target $_.Source
+    }
+}
+
+$Artifacts |% { $_.ArtifactName } | Get-Unique |% {
+    Write-Host "##vso[artifact.upload containerfolder=$_$ArtifactNameSuffix;artifactname=$_$ArtifactNameSuffix;]$ArtifactStagingFolder/$_$ArtifactNameSuffix"
+}

azure-pipelines/artifacts/build_logs.ps1

@@ -0,0 +1,10 @@
+if ($env:BUILD_ARTIFACTSTAGINGDIRECTORY) {
+    $artifactsRoot = $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+} else {
+    $RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+    $artifactsRoot = "$RepoRoot\bin"
+}
+
+@{
+    "$artifactsRoot/build_logs" = (Get-ChildItem -Recurse "$artifactsRoot/build_logs")
+}

azure-pipelines/artifacts/coverageResults.ps1

@@ -0,0 +1,20 @@
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+
+# Prepare code coverage reports for merging on another machine
+if ($env:SYSTEM_DEFAULTWORKINGDIRECTORY) {
+    Write-Host "Substituting $env:SYSTEM_DEFAULTWORKINGDIRECTORY with `"{reporoot}`""
+    $reports = Get-ChildItem "$RepoRoot/bin/coverage.cobertura.xml" -Recurse
+    $reports |% {
+        $content = Get-Content -Path $_ |% { $_ -Replace [regex]::Escape($env:SYSTEM_DEFAULTWORKINGDIRECTORY), "{reporoot}" }
+        Set-Content -Path $_ -Value $content -Encoding UTF8
+    }
+} else {
+    Write-Warning "Azure Pipelines not detected. Machine-neutral token replacement skipped."
+}
+
+@{
+    $RepoRoot = (
+        (Get-ChildItem "$RepoRoot\bin\coverage.cobertura.xml" -Recurse) +
+        (Get-ChildItem "$RepoRoot\obj\*.cs" -Recurse)
+    );
+}

azure-pipelines/artifacts/deployables.ps1

@@ -0,0 +1,11 @@
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+$BuildConfiguration = $env:BUILDCONFIGURATION
+if (!$BuildConfiguration) {
+    $BuildConfiguration = 'Debug'
+}
+
+$PackagesRoot = "$RepoRoot/bin/Packages/$BuildConfiguration"
+
+@{
+    "$PackagesRoot" = (Get-ChildItem $PackagesRoot -Recurse)
+}

azure-pipelines/artifacts/projectAssetsJson.ps1

@@ -0,0 +1,7 @@
+$RepoRoot = [System.IO.Path]::GetFullPath("$PSScriptRoot\..\..")
+
+@{
+    "$RepoRoot\obj" = (
+        (Get-ChildItem "$RepoRoot\obj\project.assets.json" -Recurse)
+    );
+}

azure-pipelines/collect-deployables.yml

@@ -1,15 +0,0 @@
-steps:
-- task: CopyFiles@1
-  inputs:
-    Contents: |
-      bin/Packages/$(BuildConfiguration)/*
-    TargetFolder: $(Build.ArtifactStagingDirectory)/deployables
-    flattenFolders: true
-  displayName: Collecting deployable artifacts
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)/deployables
-    ArtifactName: deployables
-    ArtifactType: Container
-  displayName: Publish deployables artifacts

azure-pipelines/collect-logs.yml

@@ -1,9 +0,0 @@
-steps:
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)/build_logs
-    ArtifactName: build_logs-$(Agent.JobName)
-    ArtifactType: Container
-  displayName: Publish build_logs artifacts
-  condition: succeededOrFailed()

azure-pipelines/dotnet.yml

@@ -36,48 +36,20 @@ steps:
     testRunTitle: netcoreapp2.2-$(Agent.JobName)
     workingDirectory: src
 
-- task: CopyFiles@1
+- task: PowerShell@2
   inputs:
-    Contents: |
-      bin/**/coverage.cobertura.xml
-      obj/**/*.cs
-    TargetFolder: $(Build.ArtifactStagingDirectory)/coverageResults
-  displayName: Collecting coverage.cobertura.xml artifacts
+    filePath: azure-pipelines/variables/_pipelines.ps1
+    failOnStderr: true
+  displayName: Update pipeline variables based on build outputs
   condition: succeededOrFailed()
 
-- powershell: |
-    Write-Host "Substituting $(System.DefaultWorkingDirectory) with {reporoot}"
-    $reports = Get-ChildItem "$(Build.ArtifactStagingDirectory)/coverageResults/coverage.cobertura.xml" -Recurse
-    $reports |% {
-        $content = Get-Content -Path $_ |% { $_ -Replace [regex]::Escape("$(System.DefaultWorkingDirectory)"), "{reporoot}" }
-        Set-Content -Path $_ -Value $content -Encoding UTF8
-    }
-  displayName: Preparing code coverage reports for merging on another machine
-
-- task: PublishBuildArtifacts@1
+- task: PowerShell@2
   inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)/coverageResults
-    ArtifactName: coverageResults-$(Agent.JobName)
-    ArtifactType: Container
-  displayName: Publish coverageResults artifacts
+    filePath: azure-pipelines/artifacts/_pipelines.ps1
+    arguments: -ArtifactNameSuffix "-$(Agent.JobName)"
+  displayName: Publish artifacts
   condition: succeededOrFailed()
 
 - bash: bash <(curl -s https://codecov.io/bash)
   displayName: Publish code coverage results to codecov.io
   condition: ne(variables['codecov_token'], '')
-
-- task: CopyFiles@1
-  inputs:
-    Contents: |
-      obj/**/project.assets.json
-    TargetFolder: $(Build.ArtifactStagingDirectory)/projectAssetsJson
-  displayName: Collecting project.assets.json artifacts
-  condition: succeededOrFailed()
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    PathtoPublish: $(Build.ArtifactStagingDirectory)/projectAssetsJson
-    ArtifactName: projectAssetsJson-$(Agent.JobName)
-    ArtifactType: Container
-  displayName: Publish projectAssetsJson artifacts
-  condition: succeededOrFailed()

azure-pipelines/install-dependencies.yml

@@ -1,13 +1,19 @@
 steps:
+- task: PowerShell@2
+  inputs:
+    filePath: azure-pipelines/variables/_pipelines.ps1
+    failOnStderr: true
+  displayName: Set pipeline variables based on source
+
 - script: dotnet --info
   displayName: .NET Core SDK/runtimes (on host)
   workingDirectory: $(Agent.HomeDirectory)
 
 - task: UseDotNet@2
-  displayName: Install .NET Core SDK 2.2.300
+  displayName: Install .NET Core SDK
   inputs:
     packageType: sdk
-    version: 2.2.300
+    version: $(DotNetSdkVersion)
 
 - task: UseDotNet@2
   displayName: Install .NET Core runtime 2.0.x

azure-pipelines/variables/DotNetSdkVersion.ps1

@@ -0,0 +1,2 @@
+$globalJson = Get-Content -Path "$PSScriptRoot\..\..\global.json" | ConvertFrom-Json
+$globalJson.sdk.version

azure-pipelines/variables/_all.ps1

@@ -0,0 +1,10 @@
+# This script returns a hashtable of build variables that should be set
+# at the start of a build or release definition's execution.
+
+$vars = @{}
+
+Get-ChildItem "$PSScriptRoot\*.ps1" -Exclude "_*" |% {
+    $vars[$_.BaseName] = & $_
+}
+
+$vars

azure-pipelines/variables/_pipelines.ps1

@@ -0,0 +1,15 @@
+# This script translates the variables returned by the _all.ps1 script
+# into commands that instruct VSTS to actually set those variables for other VSTS tasks to consume.
+
+# The build or release definition may have set these variables to override
+# what the build would do. So only set them if they have not already been set.
+
+(& "$PSScriptRoot\_all.ps1").GetEnumerator() |% {
+    if (Test-Path -Path "env:$($_.Key.ToUpper())") {
+        Write-Host "Skipping setting $($_.Key) because variable is already set." -ForegroundColor Cyan
+    } else {
+        Write-Host "$($_.Key)=$($_.Value)" -ForegroundColor Yellow
+        Write-Host "##vso[task.setvariable variable=$($_.Key);]$($_.Value)"
+        Set-Item -Path "env:$($_.Key)" -Value $_.Value
+    }
+}