Add full RPS matrix, and a pipeline to run it. (#1146)

* Add full RPS matrix, and a pipeline to run it.

Pipeline is manual only currently, and test suite likely needs more, but adding more is easy once this is going

* Fix name to periodic, trigger the blank space group

* fixup comments, add cron trigger

* Fix yaml

* Another yaml fix

* Turn into full run matrix

* Catch on missing old results

* Expand timeout to 5 hours

* Another timeout fix attempt

* 1 more attempt at timeout fixes

.azure/azure-pipelines.periodic.yml

@@ -0,0 +1,63 @@
+#
+# Continuous Integration (CI)
+# This pipeline builds and runs MsQuic performance RPS long run tests.
+#
+
+trigger: none
+pr: none
+
+schedules:
+- cron: "0 0 * * Sun"
+  displayName: Weekly Perf Run
+  branches:
+    include:
+    - main
+  always: true
+
+name: 0.$(Date:yyyy).$(Date:MM).$(DayOfMonth).$(Rev:rr).0
+stages:
+
+#
+# Builds
+#
+
+- stage: build_windows
+  displayName: Build Windows
+  dependsOn: []
+  variables:
+    runCodesignValidationInjection: false
+  jobs:
+  - template: ./templates/build-config-user.yml
+    parameters:
+      image: windows-latest
+      platform: windows
+      arch: x64
+      tls: schannel
+      config: Release
+      extraBuildArgs: -DisableTest -DisableTools
+  - template: ./templates/build-config-winkernel.yml
+    parameters:
+      arch: x64
+      config: Release
+
+#
+# Performance Tests
+#
+
+- stage: performance_rps
+  displayName: RPS Performance Testing
+  dependsOn:
+  - build_windows
+  jobs:
+  - template: ./templates/run-performance.yml
+    parameters:
+      pool: MsQuic-Win-Perf
+      platform: windows
+      localTls: schannel
+      remoteTls: schannel
+      arch: x64
+      failOnRegression: 0
+      kernelMode: -Kernel -TestsFile FullRunRPS.json
+      extraName: 'Kernel'
+      testTypes: 'Remote'
+      timeout: 300

.azure/templates/run-performance.yml

@@ -13,11 +13,13 @@ parameters:
   kernelMode: ''
   testTypes: 'Remote,Loopback'
   testToRun: ''
+  timeout: 45
   failOnRegression: 1
 
 jobs:
 - job: performance_${{ parameters.platform }}_${{ parameters.arch }}_${{ parameters.localTls }}_${{ parameters.remoteTls }}_${{ parameters.extraName }}
   displayName: ${{ parameters.platform }} ${{ parameters.arch }} ${{ parameters.localTls }} ${{ parameters.remoteTls }} ${{ parameters.extraName }}
+  timeoutInMinutes: 0
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
     workspace:
@@ -59,7 +61,7 @@ jobs:
   - task: PowerShell@2
     condition: contains('${{ parameters.testTypes }}', 'Remote')
     displayName: Run Performance Test
-    timeoutInMinutes: 45
+    timeoutInMinutes: ${{ parameters.timeout }}
     inputs:
       pwsh: true
       filePath: scripts/performance.ps1
@@ -68,7 +70,7 @@ jobs:
   - task: PowerShell@2
     condition: and(contains('${{ parameters.testTypes }}', 'Loopback'), succeededOrFailed())
     displayName: Run Performance Test - Loopback
-    timeoutInMinutes: 45
+    timeoutInMinutes: ${{ parameters.timeout }}
     inputs:
       pwsh: true
       filePath: scripts/performance.ps1

scripts/FullRunRPS.json

@@ -0,0 +1,56 @@
+{
+    "Remote": {
+        "Exe": "quicperf",
+        "Arguments": "-selfsign:1"
+    },
+    "FullMatrix": true,
+    "Tests": [
+        {
+            "TestName": "RPS",
+            "Local": {
+                "Platform": "Windows",
+                "Tls": ["stub", "schannel", "openssl", "mitls"],
+                "Arch": ["x64", "x86", "arm", "arm64"],
+                "Exe": "quicperf",
+                "Arguments": "-test:RPS -target:$RemoteAddress"
+            },
+            "Variables": [
+                {
+                    "Name": "ConnectionCount",
+                    "Local": {
+                        "50": "-conns:50 -requests:1500",
+                        "250": "-conns:250 -requests:7500",
+                        "500": "-conns:500 -requests:15000"
+                    },
+                    "Default": "250"
+                },
+                {
+                    "Name": "RequestSize",
+                    "Local": {
+                        "0": "-request:0",
+                        "512": "-request:512",
+                        "4096": "-request:4096",
+                        "16384": "-request:16384"
+                    },
+                    "Default": "0"
+                },
+                {
+                    "Name": "ResponseSize",
+                    "Local": {
+                        "0": "-response:0",
+                        "512": "-response:512",
+                        "4096": "-response:4096",
+                        "16384": "-response:16384"
+                    },
+                    "Default": "4096"
+                }
+            ],
+            "AllowLoopback": false,
+            "Iterations": 5,
+            "RemoteReadyMatcher": "Started!",
+            "ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
+            "Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
+            "RegressionThreshold": "-50.0"
+        }
+    ]
+}

scripts/performance-helper.psm1

@@ -629,15 +629,18 @@ class ThroughputRequest {
 }
 
 function Get-LatestThroughputRemoteTestResults($CpuData, [ThroughputRequest]$Request) {
-    $TestConfig = $Request.GetConfiguration()
-    foreach ($Test in $CpuData.Tests) {
-        if ($null -eq $Test.TputConfig) {
-            continue;
-        }
+    try {
+        $TestConfig = $Request.GetConfiguration()
+        foreach ($Test in $CpuData.Tests) {
+            if ($null -eq $Test.TputConfig) {
+                continue;
+            }
 
-        if ($TestConfig -eq $Test.TputConfig -and $Request.PlatformName -eq $Test.PlatformName) {
-            return $Test
+            if ($TestConfig -eq $Test.TputConfig -and $Request.PlatformName -eq $Test.PlatformName) {
+                return $Test
+            }
         }
+    } catch {
     }
     return $null
 }
@@ -776,15 +779,18 @@ class RPSRequest {
 }
 
 function Get-LatestRPSRemoteTestResults($CpuData, [RpsRequest]$Request) {
-    $TestConfig = $Request.GetConfiguration()
-    foreach ($Test in $CpuData.Tests) {
-        if ($null -eq $Test.RpsConfig) {
-            continue;
-        }
+    try {
+        $TestConfig = $Request.GetConfiguration()
+        foreach ($Test in $CpuData.Tests) {
+            if ($null -eq $Test.RpsConfig) {
+                continue;
+            }
 
-        if ($TestConfig -eq $Test.RpsConfig -and $Request.PlatformName -eq $Test.PlatformName) {
-            return $Test
+            if ($TestConfig -eq $Test.RpsConfig -and $Request.PlatformName -eq $Test.PlatformName) {
+                return $Test
+            }
         }
+    } catch {
     }
     return $null
 }
@@ -907,15 +913,18 @@ class HPSRequest {
 }
 
 function Get-LatestHPSRemoteTestResults($CpuData, [HpsRequest]$Request) {
-    $TestConfig = $Request.GetConfiguration()
-    foreach ($Test in $CpuData.Tests) {
-        if ($null -eq $Test.HpsConfig) {
-            continue;
-        }
+    try {
+        $TestConfig = $Request.GetConfiguration()
+        foreach ($Test in $CpuData.Tests) {
+            if ($null -eq $Test.HpsConfig) {
+                continue;
+            }
 
-        if ($TestConfig -eq $Test.HpsConfig -and $Request.PlatformName -eq $Test.PlatformName) {
-            return $Test
+            if ($TestConfig -eq $Test.HpsConfig -and $Request.PlatformName -eq $Test.PlatformName) {
+                return $Test
+            }
         }
+    } catch {
     }
     return $null
 }

scripts/performance.ps1

@@ -178,6 +178,12 @@ if (!$IsWindows) {
 
 if ($TestsFile -eq "") {
     $TestsFile = Join-Path $PSScriptRoot "RemoteTests.json"
+} elseif (-not (Test-Path $TestsFile)) {
+    $TestsFile = Join-Path $PSScriptRoot $TestsFile
+}
+
+if (-not (Test-Path $TestsFile)) {
+    Write-Error "Test file to run not found"
 }
 
 if ($Local) {