Add Support for PGO Azure Pipeline (#525)

.azure/azure-pipelines.pgo.yml

@@ -0,0 +1,54 @@
+#
+# Continuous Integration (CI)
+# This pipeline builds MsQuic with PGO instrumentation and runs the performance tests with it.
+#
+
+trigger: none
+pr: none
+
+name: 0.$(Date:yyyy).$(Date:MM).$(DayOfMonth).$(Rev:rr).0
+
+#
+# Builds
+#
+
+stages:
+
+- stage: build_windows
+  displayName: Build Windows
+  dependsOn: []
+  jobs:
+  # Officially supported configurations.
+  - template: ./templates/build-config-user.yml
+    parameters:
+      image: windows-latest
+      platform: windows
+      arch: x86
+      tls: schannel
+      extraBuildArgs: -PGO -DisableTest
+  - template: ./templates/build-config-user.yml
+    parameters:
+      image: windows-latest
+      platform: windows
+      arch: x64
+      tls: schannel
+      extraBuildArgs: -PGO -DisableTest
+
+#
+# Performance Tests
+#
+
+- stage: performance
+  displayName: Performance Testing
+  dependsOn:
+  - build_windows
+  jobs:
+  - template: ./templates/run-performance-int.yml
+    parameters:
+      tls: schannel
+      extraArgs: -PGO
+  - template: ./templates/run-performance-int.yml
+    parameters:
+      tls: schannel
+      arch: x86
+      extraArgs: -PGO

.azure/templates/run-performance-int.yml

@@ -4,6 +4,7 @@ parameters:
   config: 'Release'
   arch: 'x64'
   tls: ''
+  extraArgs: ''
 
 jobs:
 - job: performance_windows_${{ parameters.arch }}_${{ parameters.tls }}
@@ -17,6 +18,11 @@ jobs:
   steps:
   - checkout: self
 
+  - task: DeleteFiles@1
+    displayName: Clear Old Artifacts
+    inputs:
+      contents: artifacts
+
   - template: ./download-artifacts.yml
     parameters:
       platform: windows
@@ -37,7 +43,7 @@ jobs:
     inputs:
       pwsh: true
       filePath: scripts/performance.ps1
-      arguments: -Config ${{ parameters.config }} -Arch ${{ parameters.arch }} -Tls ${{ parameters.tls }}
+      arguments: -Config ${{ parameters.config }} -Arch ${{ parameters.arch }} -Tls ${{ parameters.tls }} ${{ parameters.extraArgs }}
 
   - task: CopyFiles@2
     displayName: Move Performance Results

scripts/build.ps1

@@ -123,7 +123,6 @@ $RootDir = Split-Path $PSScriptRoot -Parent
 # Important directory paths.
 $BaseArtifactsDir = Join-Path $RootDir "artifacts"
 $BaseBuildDir = Join-Path $RootDir "build"
-$SrcDir = Join-Path $RootDir "src"
 
 $ArtifactsDir = Join-Path $BaseArtifactsDir $Platform
 $BuildDir = Join-Path $BaseBuildDir $Platform
@@ -232,6 +231,25 @@ function CMake-Build {
         if (!$DisableTools) {
             Copy-Item (Join-Path $BuildDir "obj" $Config "msquicetw.lib") $ArtifactsDir
         }
+        if ($PGO -and $Config -eq "Release") {
+            # TODO - Figure out a better way to get the path?
+            $VCToolsPath = "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.26.28801\bin\Host$Arch\$Arch"
+            if (!(Test-Path $VCToolsPath)) {
+                $VCToolsPath = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC\14.26.28801\bin\Host$Arch\$Arch"
+            }
+            if (!(Test-Path $VCToolsPath)) {
+                $VCToolsPath = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\MSVC\14.26.28801\bin\Host$Arch\$Arch"
+            }
+            if (Test-Path $VCToolsPath) {
+                Copy-Item (Join-Path $VCToolsPath "pgort140.dll") $ArtifactsDir
+                Copy-Item (Join-Path $VCToolsPath "pgodb140.dll") $ArtifactsDir
+                Copy-Item (Join-Path $VCToolsPath "mspdbcore.dll") $ArtifactsDir
+                Copy-Item (Join-Path $VCToolsPath "tbbmalloc.dll") $ArtifactsDir
+                Copy-Item (Join-Path $VCToolsPath "pgomgr.exe") $ArtifactsDir
+            } else {
+                Log "Failed to find VC Tools path!"
+            }
+        }
     }
 }
 

scripts/performance.ps1

@@ -12,12 +12,15 @@ This script runs performance tests locally for a period of time.
 .PARAMETER Tls
     The TLS library use.
 
+.PARAMETER PGO
+    Uses pgomgr to merge the resulting .pgc files back to the .pgd.
+
 #>
 
 param (
     [Parameter(Mandatory = $false)]
     [ValidateSet("Debug", "Release")]
-    [string]$Config = "Debug",
+    [string]$Config = "Release",
 
     [Parameter(Mandatory = $false)]
     [ValidateSet("x86", "x64", "arm", "arm64")]
@@ -25,7 +28,10 @@ param (
 
     [Parameter(Mandatory = $false)]
     [ValidateSet("schannel", "openssl", "stub", "mitls")]
-    [string]$Tls = ""
+    [string]$Tls = "",
+
+    [Parameter(Mandatory = $false)]
+    [switch]$PGO = $false
 )
 
 Set-StrictMode -Version 'Latest'
@@ -49,16 +55,19 @@ if ($IsWindows) {
 }
 $Platform = "$($OsPlat)_$($Arch)_$($Tls)"
 
-# Path to the spinquic exectuable.
-$PingClient = $null
+# Path to the build artifacts.
+$Artifacts = $null
+$QuicPing = $null
 if ($IsWindows) {
-    $PingClient = Join-Path $RootDir "\artifacts\windows\$($Arch)_$($Config)_$($Tls)\quicping.exe"
+    $Artifacts = Join-Path $RootDir "\artifacts\windows\$($Arch)_$($Config)_$($Tls)"
+    $QuicPing = "quicping.exe"
 } else {
-    $PingClient = Join-Path $RootDir "/artifacts/linux/$($Arch)_$($Config)_$($Tls)/quicping"
+    $Artifacts = Join-Path $RootDir "/artifacts/linux/$($Arch)_$($Config)_$($Tls)"
+    $QuicPing = "quicping"
 }
 
 # Make sure the build is present.
-if (!(Test-Path $PingClient)) {
+if (!(Test-Path (Join-Path $Artifacts $QuicPing))) {
     Write-Error "Build does not exist!`n `nRun the following to generate it:`n `n    $(Join-Path $RootDir "scripts" "build.ps1") -Config $Config -Arch $Arch -Tls $Tls`n"
 }
 
@@ -103,7 +112,7 @@ function Parse-Loopback-Results($Results) {
 function Get-Latest-Test-Results($Platform, $Test) {
     $Uri = "https://msquicperformanceresults.azurewebsites.net/performance/$Platform/$Test"
     Write-Host "Requesting: $Uri"
-    $LatestResult =  Invoke-RestMethod -Uri $Uri
+    $LatestResult = Invoke-RestMethod -Uri $Uri
     Write-Host "Result: $LatestResult"
     return $LatestResult
 }
@@ -126,48 +135,76 @@ $env:GIT_REDIRECT_STDERR = '2>&1'
 $CurrentCommitHash = git rev-parse HEAD
 Set-Location -Path $currentLoc
 
+function Merge-PGO-Counts($Path) {
+    $Command = "$Artifacts\pgomgr.exe /merge $Path $Artifacts\msquic.pgd"
+    Invoke-Expression $Command | Write-Debug
+    Remove-Item "$Path\*.pgc" | Out-Null
+}
+
 function Run-Loopback-Test() {
     Write-Host "Running Loopback Test"
-    $proc = Start-Background-Executable -File $PingClient -Arguments "-listen:* -selfsign:1 -peer_uni:1"
+
+    # Run server in it's own directory.
+    $ServerDir = "$($Artifacts)_server"
+    if (!(Test-Path $ServerDir)) { New-Item -Path $ServerDir -ItemType Directory -Force | Out-Null }
+    Copy-Item "$Artifacts\*" $ServerDir | Out-Null
+
+    $proc = Start-Background-Executable -File (Join-Path $ServerDir $QuicPing) -Arguments "-listen:* -selfsign:1 -peer_uni:1"
     Start-Sleep 4
 
     $allRunsResults = @()
 
     1..10 | ForEach-Object {
-        $runResult = Run-Foreground-Executable -File $PingClient -Arguments "-target:localhost -uni:1 -length:100000000"
+        $runResult = Run-Foreground-Executable -File (Join-Path $Artifacts $QuicPing) -Arguments "-target:localhost -uni:1 -length:100000000"
         $parsedRunResult = Parse-Loopback-Results -Results $runResult
         $allRunsResults += $parsedRunResult
-        Write-Host "Client $_ Finished"
+        if ($PGO) {
+            # Merge client PGO counts.
+            Merge-PGO-Counts $Artifacts
+        }
+        Write-Host "Client $_ Finished: $parsedRunResult kbps"
     }
 
     Stop-Background-Executable -Process $proc
+    if ($PGO) {
+        # Merge server PGO counts.
+        Merge-PGO-Counts $ServerDir
+    }
+    Remove-Item $ServerDir -Recurse -Force | Out-Null
 
     $MedianCurrentResult = Median-Test-Results -FullResults $allRunsResults
-
-    $fullLastResult = Get-Latest-Test-Results -Platform $Platform -Test "loopback"
-    $MedianLastResult = 0
-    if ($fullLastResult -ne "") {
-        $MedianLastResult = Median-Test-Results -FullResults $fullLastResult.individualRunResults
-    }
-
-    $ToPublishResults = [TestPublishResult]::new()
-
-    $ToPublishResults.CommitHash = $CurrentCommitHash.Substring(0, 7)
-    $ToPublishResults.PlatformName = $Platform
-    $ToPublishResults.TestName = "loopback"
-    $ToPublishResults.IndividualRunResults = $allRunsResults
-
-    $ResultsFolderRoot = "$Platform/loopback"
-    $ResultsFileName = "/results.json"
-
-    $NewFilePath = Join-Path $RootDir "artifacts/PerfDataResults/$ResultsFolderRoot"
-    $NewFileLocation = Join-Path $NewFilePath $ResultsFileName
-    New-Item $NewFilePath -ItemType Directory -Force
-
-    $ToPublishResults | ConvertTo-Json | Out-File $NewFileLocation
-
     Write-Host "Current Run: $MedianCurrentResult kbps"
-    Write-Host "Last Master Run: $MedianLastResult kbps"
+
+    if (!$PGO) {
+        $fullLastResult = Get-Latest-Test-Results -Platform $Platform -Test "loopback"
+        $MedianLastResult = 0
+        if ($fullLastResult -ne "") {
+            $MedianLastResult = Median-Test-Results -FullResults $fullLastResult.individualRunResults
+        }
+        Write-Host "Last Master Run: $MedianLastResult kbps"
+
+        $ToPublishResults = [TestPublishResult]::new()
+        $ToPublishResults.CommitHash = $CurrentCommitHash.Substring(0, 7)
+        $ToPublishResults.PlatformName = $Platform
+        $ToPublishResults.TestName = "loopback"
+        $ToPublishResults.IndividualRunResults = $allRunsResults
+
+        $ResultsFolderRoot = "$Platform/loopback"
+        $ResultsFileName = "/results.json"
+
+        $NewFilePath = Join-Path $RootDir "artifacts/PerfDataResults/$ResultsFolderRoot"
+        $NewFileLocation = Join-Path $NewFilePath $ResultsFileName
+        New-Item $NewFilePath -ItemType Directory -Force
+
+        $ToPublishResults | ConvertTo-Json | Out-File $NewFileLocation
+    }
 }
 
 Run-Loopback-Test
+
+if ($PGO) {
+    Write-Host "Copying msquic.pgd out for publishing."
+    $OutPath = Join-Path $RootDir "\artifacts\PerfDataResults\winuser\pgo_$($Arch)"
+    if (!(Test-Path $OutPath)) { New-Item -Path $OutPath -ItemType Directory -Force }
+    Copy-Item "$Artifacts\msquic.pgd" $OutPath
+}