Add a script to always clean up perf machines (#1997)

* Add a script to always clean up perf machines

* Fix wan perf

* Null session

.azure/templates/run-performance.yml

@@ -78,6 +78,14 @@ jobs:
       filePath: scripts/performance.ps1
       arguments: -Config ${{ parameters.config }} -LocalTls ${{ parameters.localTls }} -RemoteTls ${{ parameters.remoteTls }} -LocalArch ${{ parameters.arch }} -RemoteArch ${{ parameters.arch }} ${{ parameters.kernelMode }} ${{ parameters.extraArgs }} -FailOnRegression ${{ parameters.failOnRegression }} -Local -TestToRun '${{ parameters.testToRun }}' -Protocol ${{ parameters.protocol }} -LogProfile ${{ parameters.logProfile }}
 
+  - task: PowerShell@2
+    condition: always()
+    displayName: Clean up machines
+    timeoutInMinutes: 5
+    inputs:
+      pwsh: true
+      filePath: scripts/cancel-performance.ps1
+
   - task: CopyFiles@2
     displayName: Move Performance Results
     condition: succeededOrFailed()

.azure/templates/run-wanperf.yml

@@ -65,6 +65,15 @@ jobs:
       filePath: scripts/emulated-performance.ps1
       arguments: -Protocol ${{ parameters.protocol }} -NumIterations ${{ parameters.iterations }} -BottleneckMbps ${{ parameters.rateMbps }} -BottleneckQueueRatio ${{ parameters.bottleneckQueueRatio }} -DurationMs ${{ parameters.durationMs }} -RttMs ${{ parameters.rttMs }} -Pacing ${{ parameters.pacing }} -RandomLossDenominator ${{ parameters.randomLossDenominator }} -RandomReorderDenominator ${{ parameters.randomReorderDenominator }} -ReorderDelayDeltaMs ${{ parameters.reorderDelayDeltaMs }} -BaseRandomSeed ${{ parameters.baseRandomSeed }} -LogProfile ${{ parameters.logProfile }} -Config ${{ parameters.config }} -Arch ${{ parameters.arch }} -Tls ${{ parameters.tls }} ${{ parameters.extraArgs }}
 
+  - task: PowerShell@2
+    condition: always()
+    displayName: Clean up machines
+    timeoutInMinutes: 5
+    inputs:
+      pwsh: true
+      filePath: scripts/cancel-performance.ps1
+      arguments: -SkipRemote
+
   - task: CopyFiles@2
     displayName: Move Performance Results
     condition: succeededOrFailed()

scripts/cancel-performance.ps1

@@ -0,0 +1,110 @@
+<#
+
+.SYNOPSIS
+This script cleans up after a perf run
+
+.PARAMETER Remote
+    The remote to connect to. Must have ssh remoting enabled, and public key auth. username@ip
+
+#>
+
+param (
+    [Parameter(Mandatory = $false)]
+    [string]$Remote = "",
+
+    [Parameter(Mandatory = $false)]
+    [string]$ComputerName = "quic-server",
+
+    [Parameter(Mandatory = $false)]
+    [string]$WinRMUser = "",
+
+    [switch]$SkipRemote = $false
+)
+
+Set-StrictMode -Version 'Latest'
+$PSDefaultParameterValues['*:ErrorAction'] = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+
+if (!$IsWindows -and [string]::IsNullOrWhiteSpace($Remote)) {
+    $Remote = "quic-server"
+}
+
+# Remove any previous remote PowerShell sessions
+Get-PSSession | Remove-PSSession
+
+$Session = $null
+
+if (!$SkipRemote) {
+    if ($Remote -eq "") {
+        if ($WinRMUser -ne "") {
+            $Session = New-PSSession -ComputerName $ComputerName -Credential $WinRMUser -ConfigurationName PowerShell.7
+        } else {
+            $Session = New-PSSession -ComputerName $ComputerName -ConfigurationName PowerShell.7
+        }
+    } else {
+        $Session = New-PSSession -HostName "$Remote"
+    }
+}
+
+if ($null -eq $Session) {
+    Write-Host "Failed to create remote session"
+} else {
+    $RemoteAddress = $Session.ComputerName
+    Write-Output "Connected to: $RemoteAddress"
+}
+
+try {
+
+    if ($IsWindows) {
+        if ($null -ne (Get-Process -Name "secnetperf" -ErrorAction Ignore)) {
+            try {
+                Stop-Process -Name "secnetperf" -Force | Out-Null
+            }
+            catch {}
+        }
+        if ($null -ne (Get-Service -Name "secnetperfdrvpriv" -ErrorAction Ignore)) {
+            try {
+                net.exe stop secnetperfdrvpriv /y | Out-Null
+            }
+            catch {}
+            sc.exe delete secnetperfdrvpriv /y | Out-Null
+        }
+        if ($null -ne (Get-Service -Name "msquicpriv" -ErrorAction Ignore)) {
+            try {
+                net.exe stop msquicpriv /y | Out-Null
+            }
+            catch {}
+            sc.exe delete msquicpriv /y | Out-Null
+        }
+
+        if ($null -ne $Session) {
+            Invoke-Command -Session $Session -ScriptBlock {
+                if ($null -ne (Get-Process -Name "secnetperf" -ErrorAction Ignore)) {
+                    try {
+                        Stop-Process -Name "secnetperf" -Force | Out-Null
+                    }
+                    catch {}
+                }
+                if ($null -ne (Get-Service -Name "secnetperfdrvpriv" -ErrorAction Ignore)) {
+                    try {
+                        net.exe stop secnetperfdrvpriv /y | Out-Null
+                    }
+                    catch {}
+                    sc.exe delete secnetperfdrvpriv /y | Out-Null
+                }
+                if ($null -ne (Get-Service -Name "msquicpriv" -ErrorAction Ignore)) {
+                    try {
+                        net.exe stop msquicpriv /y | Out-Null
+                    }
+                    catch {}
+                    sc.exe delete msquicpriv /y | Out-Null
+                }
+            }
+        }
+    }
+
+} finally  {
+    if ($null -ne $Session) {
+        Remove-PSSession -Session $Session
+    }
+}

scripts/performance-helper.psm1

@@ -282,6 +282,12 @@ function Get-ExeName {
 
 function Remove-PerfServices {
     if ($IsWindows) {
+        if ($null -ne (Get-Process -Name "secnetperf" -ErrorAction Ignore)) {
+            try {
+                Stop-Process -Name "secnetperf" -Force | Out-Null
+            }
+            catch {}
+        }
         if ($null -ne (Get-Service -Name "secnetperfdrvpriv" -ErrorAction Ignore)) {
             try {
                 net.exe stop secnetperfdrvpriv /y | Out-Null
@@ -298,6 +304,12 @@ function Remove-PerfServices {
         }
 
         Invoke-TestCommand -Session $Session -ScriptBlock {
+            if ($null -ne (Get-Process -Name "secnetperf" -ErrorAction Ignore)) {
+                try {
+                    Stop-Process -Name "secnetperf" -Force | Out-Null
+                }
+                catch {}
+            }
             if ($null -ne (Get-Service -Name "secnetperfdrvpriv" -ErrorAction Ignore)) {
                 try {
                     net.exe stop secnetperfdrvpriv /y | Out-Null