зеркало из https://github.com/microsoft/msquic.git
Restart Server on Each NetPerf Test Run (#4050)
This commit is contained in:
Nick Banks
GitHub
Родитель
5a9b9a7471
Коммит
b0edc581f4
|
|
@ -73,11 +73,11 @@ jobs:
|
|||
- name: Build
|
||||
if: inputs.build == '-Test'
|
||||
shell: pwsh
|
||||
run: msbuild msquic.kernel.sln /p:Configuration=${{ inputs.config }} /p:Platform=${{ inputs.arch }} /p:QUIC_VER_SUFFIX=-official /p:QUIC_VER_GIT_HASH=${{ github.sha }}
|
||||
run: msbuild msquic.kernel.sln /m /p:Configuration=${{ inputs.config }} /p:Platform=${{ inputs.arch }} /p:QUIC_VER_SUFFIX=-official /p:QUIC_VER_GIT_HASH=${{ github.sha }}
|
||||
- name: Build
|
||||
if: inputs.build == ''
|
||||
shell: pwsh
|
||||
run: msbuild msquic.kernel.sln /p:Configuration=${{ inputs.config }} /p:Platform=${{ inputs.arch }} /p:QUIC_VER_SUFFIX=-official
|
||||
run: msbuild msquic.kernel.sln /m /p:Configuration=${{ inputs.config }} /p:Platform=${{ inputs.arch }} /p:QUIC_VER_SUFFIX=-official
|
||||
- name: Sign Kernel
|
||||
shell: pwsh
|
||||
run: scripts/sign.ps1 -Config ${{ inputs.config }} -Arch ${{ inputs.arch }} -Tls ${{ inputs.tls }}
|
||||
|
|
|
|||
|
|
@ -330,7 +330,7 @@ function Install-TestCertificates {
|
|||
if (!$IsWindows -or !(Win-SupportsCerts)) { return } # Windows only
|
||||
$DnsNames = $env:computername,"localhost","127.0.0.1","::1","192.168.1.11","192.168.1.12","fc00::1:11","fc00::1:12"
|
||||
$NewRoot = $false
|
||||
Write-Host "Searching for MsQuicTestRoot certificate..."
|
||||
Write-Debug "Searching for MsQuicTestRoot certificate..."
|
||||
$RootCert = Get-ChildItem -path Cert:\LocalMachine\Root\* -Recurse | Where-Object {$_.Subject -eq "CN=MsQuicTestRoot"}
|
||||
if (!$RootCert) {
|
||||
Write-Host "MsQuicTestRoot not found! Creating new MsQuicTestRoot certificate..."
|
||||
|
|
@ -342,10 +342,10 @@ function Install-TestCertificates {
|
|||
$NewRoot = $true
|
||||
Write-Host "New MsQuicTestRoot certificate installed!"
|
||||
} else {
|
||||
Write-Host "Found existing MsQuicTestRoot certificate!"
|
||||
Write-Debug "Found existing MsQuicTestRoot certificate!"
|
||||
}
|
||||
|
||||
Write-Host "Searching for MsQuicTestServer certificate..."
|
||||
Write-Debug "Searching for MsQuicTestServer certificate..."
|
||||
$ServerCert = Get-ChildItem -path Cert:\LocalMachine\My\* -Recurse | Where-Object {$_.Subject -eq "CN=MsQuicTestServer"}
|
||||
if (!$ServerCert) {
|
||||
Write-Host "MsQuicTestServer not found! Creating new MsQuicTestServer certificate..."
|
||||
|
|
@ -356,10 +356,10 @@ function Install-TestCertificates {
|
|||
Remove-Item $TempServerPath
|
||||
Write-Host "New MsQuicTestServer certificate installed!"
|
||||
} else {
|
||||
Write-Host "Found existing MsQuicTestServer certificate!"
|
||||
Write-Debug "Found existing MsQuicTestServer certificate!"
|
||||
}
|
||||
|
||||
Write-Host "Searching for MsQuicTestExpiredServer certificate..."
|
||||
Write-Debug "Searching for MsQuicTestExpiredServer certificate..."
|
||||
$ExpiredServerCert = Get-ChildItem -path Cert:\LocalMachine\My\* -Recurse | Where-Object {$_.Subject -eq "CN=MsQuicTestExpiredServer"}
|
||||
if (!$ExpiredServerCert) {
|
||||
Write-Host "MsQuicTestExpiredServer not found! Creating new MsQuicTestExpiredServer certificate..."
|
||||
|
|
@ -370,10 +370,10 @@ function Install-TestCertificates {
|
|||
Remove-Item $TempExpiredServerPath
|
||||
Write-Host "New MsQuicTestExpiredServer certificate installed!"
|
||||
} else {
|
||||
Write-Host "Found existing MsQuicTestExpiredServer certificate!"
|
||||
Write-Debug "Found existing MsQuicTestExpiredServer certificate!"
|
||||
}
|
||||
|
||||
Write-Host "Searching for MsQuicTestClient certificate..."
|
||||
Write-Debug "Searching for MsQuicTestClient certificate..."
|
||||
$ClientCert = Get-ChildItem -path Cert:\LocalMachine\My\* -Recurse | Where-Object {$_.Subject -eq "CN=MsQuicTestClient"}
|
||||
if (!$ClientCert) {
|
||||
Write-Host "MsQuicTestClient not found! Creating new MsQuicTestClient certificate..."
|
||||
|
|
@ -384,10 +384,10 @@ function Install-TestCertificates {
|
|||
Remove-Item $TempClientPath
|
||||
Write-Host "New MsQuicTestClient certificate installed!"
|
||||
} else {
|
||||
Write-Host "Found existing MsQuicTestClient certificate!"
|
||||
Write-Debug "Found existing MsQuicTestClient certificate!"
|
||||
}
|
||||
|
||||
Write-Host "Searching for MsQuicTestExpiredClient certificate..."
|
||||
Write-Debug "Searching for MsQuicTestExpiredClient certificate..."
|
||||
$ExpiredClientCert = Get-ChildItem -path Cert:\LocalMachine\My\* -Recurse | Where-Object {$_.Subject -eq "CN=MsQuicTestExpiredClient"}
|
||||
if (!$ExpiredClientCert) {
|
||||
Write-Host "MsQuicTestExpiredClient not found! Creating new MsQuicTestExpiredClient certificate..."
|
||||
|
|
@ -398,7 +398,7 @@ function Install-TestCertificates {
|
|||
Remove-Item $TempExpiredClientPath
|
||||
Write-Host "New MsQuicTestExpiredClient certificate installed!"
|
||||
} else {
|
||||
Write-Host "Found existing MsQuicTestExpiredClient certificate!"
|
||||
Write-Debug "Found existing MsQuicTestExpiredClient certificate!"
|
||||
}
|
||||
|
||||
if ($NewRoot) {
|
||||
|
|
|
|||
|
|
@ -3,14 +3,78 @@
|
|||
Various helper functions for running secnetperf tests.
|
||||
#>
|
||||
|
||||
Set-StrictMode -Version 'Latest'
|
||||
$PSDefaultParameterValues['*:ErrorAction'] = 'Stop'
|
||||
|
||||
# Path to the WER registry key used for collecting dumps on Windows.
|
||||
$WerDumpRegPath = "HKLM:\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps\secnetperf.exe"
|
||||
|
||||
# Write a GitHub error message to the console.
|
||||
function Write-GHError($msg) {
|
||||
Write-Host "::error::$msg"
|
||||
}
|
||||
|
||||
# Write a GitHub warning message to the console.
|
||||
function Write-GHWarning($msg) {
|
||||
Write-Host "::warning::$msg"
|
||||
# Configured the remote machine to collect dumps on crash.
|
||||
function Configure-DumpCollection {
|
||||
param ($Session)
|
||||
if ($isWindows) {
|
||||
Invoke-Command -Session $Session -ScriptBlock {
|
||||
$DumpDir = "C:/_work/quic/artifacts/crashdumps"
|
||||
New-Item -Path $DumpDir -ItemType Directory -ErrorAction Ignore | Out-Null
|
||||
New-Item -Path $Using:WerDumpRegPath -Force -ErrorAction Ignore | Out-Null
|
||||
Set-ItemProperty -Path $Using:WerDumpRegPath -Name DumpFolder -Value $DumpDir | Out-Null
|
||||
Set-ItemProperty -Path $Using:WerDumpRegPath -Name DumpType -Value 2 | Out-Null
|
||||
}
|
||||
$DumpDir = Join-Path (Split-Path $PSScriptRoot -Parent) "artifacts/crashdumps"
|
||||
New-Item -Path $DumpDir -ItemType Directory -ErrorAction Ignore | Out-Null
|
||||
New-Item -Path $WerDumpRegPath -Force -ErrorAction Ignore | Out-Null
|
||||
Set-ItemProperty -Path $WerDumpRegPath -Name DumpFolder -Value $DumpDir | Out-Null
|
||||
Set-ItemProperty -Path $WerDumpRegPath -Name DumpType -Value 2 | Out-Null
|
||||
} else {
|
||||
# TODO: Configure Linux to collect dumps.
|
||||
}
|
||||
}
|
||||
|
||||
# Collects any crash dumps that were generated locally by secnetperf.
|
||||
function Collect-LocalDumps {
|
||||
param ($OutputDir)
|
||||
if ($isWindows) {
|
||||
$DumpFiles = (Get-ChildItem "./artifacts/crashdumps") | Where-Object { $_.Extension -eq ".dmp" }
|
||||
if ($DumpFiles) {
|
||||
mkdir $OutputDir | Out-Null
|
||||
foreach ($File in $DumpFiles) {
|
||||
Copy-Item -Path $File.FullName -Destination $OutputDir
|
||||
}
|
||||
# Delete all the files in the crashdumps folder.
|
||||
Remove-Item -Path "./artifacts/crashdumps/*" -Force
|
||||
return $true
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
# Collect any crash dumps that were generated on the remote machine.
|
||||
function Collect-RemoteDumps {
|
||||
param ($Session, $OutputDir)
|
||||
if ($isWindows) {
|
||||
$DumpFiles = Invoke-Command -Session $Session -ScriptBlock {
|
||||
Get-ChildItem "C:/_work/quic/artifacts/crashdumps" | Where-Object { $_.Extension -eq ".dmp" }
|
||||
}
|
||||
if ($DumpFiles) {
|
||||
mkdir $OutputDir | Out-Null
|
||||
foreach ($File in $DumpFiles) {
|
||||
Copy-Item -FromSession $Session -Path $File.FullName -Destination $OutputDir
|
||||
}
|
||||
# Delete all the files in the crashdumps folder.
|
||||
Invoke-Command -Session $Session -ScriptBlock {
|
||||
Remove-Item -Path "C:/_work/quic/artifacts/crashdumps/*" -Force
|
||||
}
|
||||
return $true
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
# Waits for a remote job to be ready based on looking for a particular string in
|
||||
|
|
@ -21,7 +85,7 @@ function Start-RemoteServer {
|
|||
$Job = Invoke-Command -Session $Session -ScriptBlock { iex $Using:Command } -AsJob
|
||||
# Poll the job for 10 seconds to see if it started.
|
||||
$StopWatch = [system.diagnostics.stopwatch]::StartNew()
|
||||
while ($StopWatch.ElapsedMilliseconds -lt 10000) {
|
||||
while ($StopWatch.ElapsedMilliseconds -lt 30000) {
|
||||
$CurrentResults = Receive-Job -Job $Job -Keep -ErrorAction Continue
|
||||
if (![string]::IsNullOrWhiteSpace($CurrentResults)) {
|
||||
$DidMatch = $CurrentResults -match "Started!" # Look for the special string to indicate success.
|
||||
|
|
@ -36,7 +100,7 @@ function Start-RemoteServer {
|
|||
Stop-Job -Job $Job
|
||||
$RemoteResult = Receive-Job -Job $Job -ErrorAction Stop
|
||||
$RemoteResult = $RemoteResult -join "`n"
|
||||
Write-GHWarning $RemoteResult.ToString()
|
||||
Write-Host $RemoteResult.ToString()
|
||||
throw "Server failed to start!"
|
||||
}
|
||||
|
||||
|
|
@ -63,117 +127,133 @@ function Stop-RemoteServer {
|
|||
Stop-Job -Job $Job
|
||||
$RemoteResult = Receive-Job -Job $Job -ErrorAction Stop
|
||||
$RemoteResult = $RemoteResult -join "`n"
|
||||
Write-GHWarning $RemoteResult.ToString()
|
||||
Write-Host $RemoteResult.ToString()
|
||||
throw "Server failed to stop!"
|
||||
}
|
||||
|
||||
# Invokes all the secnetperf tests.
|
||||
function Invoke-SecnetperfTest($MsQuicCommit, $commands, $exe, $start, $LogProfile) {
|
||||
class TestResult {
|
||||
[String]$Metric;
|
||||
[System.Object[]]$Values;
|
||||
[Boolean]$HasFailures;
|
||||
|
||||
Write-Host "Running Secnetperf tests..."
|
||||
TestResult (
|
||||
[String]$Metric,
|
||||
[System.Object[]]$Values,
|
||||
[Boolean]$HasFailures
|
||||
) {
|
||||
$this.Metric = $Metric;
|
||||
$this.Values = $Values;
|
||||
$this.HasFailures = $HasFailures;
|
||||
}
|
||||
}
|
||||
|
||||
$SQL = @"
|
||||
"@
|
||||
$json = @{}
|
||||
# Invokes secnetperf with the given arguments for both TCP and QUIC.
|
||||
function Invoke-Secnetperf {
|
||||
param ($Session, $RemoteName, $RemoteDir, $SecNetPerfPath, $LogProfile, $ExeArgs)
|
||||
|
||||
$values = @(@(), @())
|
||||
$hasFailures = $false
|
||||
|
||||
# TODO: This logic is pretty fragile. Needs improvement.
|
||||
$metric = "throughput-download"
|
||||
if ($exeArgs.Contains("plat:1")) {
|
||||
$metric = "latency"
|
||||
} elseif ($exeArgs.Contains("prate:1")) {
|
||||
$metric = "hps"
|
||||
} elseif ($exeArgs.Contains("-up")) {
|
||||
$metric = "throughput-upload"
|
||||
}
|
||||
|
||||
for ($i = 0; $i -lt $commands.Count; $i++) {
|
||||
for ($tcp = 0; $tcp -lt 2; $tcp++) {
|
||||
$testid = $i + 1 + $start
|
||||
$SQL += @"
|
||||
|
||||
INSERT OR IGNORE INTO Secnetperf_tests (Secnetperf_test_ID, Kernel_mode, Run_arguments) VALUES ($testid, 0, "$($commands[$i]) -tcp:$tcp");
|
||||
|
||||
"@
|
||||
$command = "$exe -target:netperf-peer $($commands[$i]) -tcp:$tcp -trimout"
|
||||
$execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(' ')) # First arg is the exec mode
|
||||
$command = "./$SecNetPerfPath -target:netperf-peer $ExeArgs -tcp:$tcp -trimout -watchdog:45000"
|
||||
Write-Host "> $command"
|
||||
|
||||
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") { # Start logging.
|
||||
Write-Host "Starting logging with log profile: $LogProfile..."
|
||||
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
|
||||
Invoke-Command -Session $Session -ScriptBlock {
|
||||
try { & "$Using:RemoteDir/scripts/log.ps1" -Cancel } catch {} # Cancel any previous logging
|
||||
& "$Using:RemoteDir/scripts/log.ps1" -Start -Profile $Using:LogProfile -ProfileInScriptDirectory
|
||||
}
|
||||
try { .\scripts\log.ps1 -Cancel } catch {} # Cancel any previous logging
|
||||
.\scripts\log.ps1 -Start -Profile $LogProfile
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
# Start the server running.
|
||||
$Job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath $execMode"
|
||||
|
||||
# Run the test multiple times, failing (for now) only if all tries fail.
|
||||
# TODO: Once all failures have been fixed, consider all errors fatal.
|
||||
$successCount = 0
|
||||
for ($try = 0; $try -lt 3; $try++) {
|
||||
try {
|
||||
$rawOutput = Invoke-Expression $command
|
||||
} catch {
|
||||
Write-GHError "Failed to run test: $($commands[$i])"
|
||||
Write-GHError $_
|
||||
$script:encounterFailures = $true
|
||||
continue
|
||||
}
|
||||
|
||||
if ($null -eq $rawOutput) {
|
||||
Write-GHError "RawOutput is null. Failed to run test: $($commands[$i])"
|
||||
$script:encounterFailures = $true
|
||||
continue
|
||||
}
|
||||
|
||||
if ($rawOutput.Contains("Error")) {
|
||||
$rawOutput = $rawOutput.Substring(7) # Skip over the 'Error: ' prefix
|
||||
Write-GHError $rawOutput
|
||||
$script:encounterFailures = $true
|
||||
continue
|
||||
}
|
||||
|
||||
$env = 2
|
||||
if ($isWindows) {
|
||||
$env = 1
|
||||
}
|
||||
|
||||
Write-Host $rawOutput
|
||||
|
||||
$transport = "quic"
|
||||
|
||||
if ($tcp -eq 1) {
|
||||
$transport = "tcp"
|
||||
}
|
||||
|
||||
if ($command.Contains("lowlat")) {
|
||||
$latency_percentiles = '(?<=\d{1,3}(?:\.\d{1,2})?th: )\d+'
|
||||
$Perc = [regex]::Matches($rawOutput, $latency_percentiles) | ForEach-Object {$_.Value}
|
||||
$json["latency-$transport"] = $Perc
|
||||
# TODO: SQL += ...
|
||||
continue
|
||||
}
|
||||
|
||||
$throughput = '@ (\d+) kbps'
|
||||
|
||||
$metric = "download"
|
||||
if ($command.Contains("-up")) {
|
||||
$metric = "upload"
|
||||
}
|
||||
|
||||
foreach ($line in $rawOutput) {
|
||||
if ($line -match $throughput) {
|
||||
|
||||
$num = $matches[1]
|
||||
|
||||
# Generate SQL statement. Assume LAST_INSERT_ROW_ID()
|
||||
$SQL += @"
|
||||
|
||||
INSERT INTO Secnetperf_test_runs (Secnetperf_test_ID, Secnetperf_commit, Client_environment_ID, Server_environment_ID, Result, Secnetperf_latency_stats_ID)
|
||||
VALUES ($testid, '$MsQuicCommit', $env, $env, $num, NULL);
|
||||
|
||||
"@
|
||||
|
||||
# Generate JSON as intermediary file for dashboard
|
||||
$json["throughput-$metric-$transport"] = $num
|
||||
break
|
||||
if ($null -eq $rawOutput) {
|
||||
throw "secnetperf: No console output (possibly crashed)!"
|
||||
}
|
||||
if ($rawOutput.Contains("Error")) {
|
||||
throw "secnetperf: $($rawOutput.Substring(7))" # Skip over the 'Error: ' prefix
|
||||
}
|
||||
Write-Host $rawOutput
|
||||
if ($metric -eq "latency") {
|
||||
$latency_percentiles = '(?<=\d{1,3}(?:\.\d{1,2})?th: )\d+'
|
||||
$values[$tcp] += [regex]::Matches($rawOutput, $latency_percentiles) | ForEach-Object {$_.Value}
|
||||
} elseif ($metric -eq "hps") {
|
||||
$rawOutput -match '(\d+) HPS'
|
||||
$values[$tcp] += $matches[1]
|
||||
} else { # throughput
|
||||
$rawOutput -match '@ (\d+) kbps'
|
||||
$values[$tcp] += $matches[1]
|
||||
}
|
||||
$successCount++
|
||||
} catch {
|
||||
Write-GHError $_
|
||||
#$hasFailures = $true
|
||||
}
|
||||
|
||||
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") { # Stop logging.
|
||||
Write-Host "Stopping logging..."
|
||||
.\scripts\log.ps1 -Stop -OutputPath ".\artifacts\logs\$command"
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 1
|
||||
Start-Sleep -Seconds 1 | Out-Null
|
||||
}
|
||||
if ($successCount -eq 0) {
|
||||
$hasFailures = $true # For now, consider failure only if all failed
|
||||
Write-GHError "secnetperf: All test tries failed!"
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-GHError "Exception while running test case!"
|
||||
Write-GHError $_
|
||||
$_ | Format-List *
|
||||
$hasFailures = $true
|
||||
} finally {
|
||||
$ArtifactName = $tcp -eq 0 ? "$metric-quic" : "$metric-tcp"
|
||||
|
||||
# Stop the server.
|
||||
try { Stop-RemoteServer $Job $RemoteName | Out-Null } catch { } # Ignore failures for now
|
||||
|
||||
# Stop logging and copy the logs to the artifacts folder.
|
||||
if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
|
||||
try { .\scripts\log.ps1 -Stop -OutputPath "./artifacts/logs/$ArtifactName/client" -RawLogOnly }
|
||||
catch { Write-Host "Failed to stop logging on client!" }
|
||||
Invoke-Command -Session $Session -ScriptBlock {
|
||||
try {
|
||||
& "$Using:RemoteDir/scripts/log.ps1" -Stop -OutputPath "$Using:RemoteDir/artifacts/logs/$Using:ArtifactName/server" -RawLogOnly
|
||||
dir "$Using:RemoteDir/artifacts/logs/$Using:ArtifactName"
|
||||
} catch { Write-Host "Failed to stop logging on server!" }
|
||||
}
|
||||
try { Copy-Item -FromSession $Session "$RemoteDir/artifacts/logs/$ArtifactName/*" "./artifacts/logs/$ArtifactName/" }
|
||||
catch { Write-Host "Failed to copy server logs!" }
|
||||
}
|
||||
|
||||
# Grab any crash dumps that were generated.
|
||||
if (Collect-LocalDumps "./artifacts/logs/$ArtifactName/clientdumps") {
|
||||
Write-Host "Dump file(s) generated locally"
|
||||
#$hasFailures = $true
|
||||
}
|
||||
if (Collect-RemoteDumps $Session "./artifacts/logs/$ArtifactName/serverdumps") {
|
||||
Write-Host "Dump file(s) generated on peer"
|
||||
#$hasFailures = $true
|
||||
}
|
||||
}}
|
||||
|
||||
return $SQL, $json
|
||||
return [TestResult]::new($metric, $values, $hasFailures)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,9 @@ param (
|
|||
[string]$RemoteName = "netperf-peer"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version 'Latest'
|
||||
$PSDefaultParameterValues['*:ErrorAction'] = 'Stop'
|
||||
|
||||
# Set up some important paths.
|
||||
$RemoteDir = "C:/_work/quic"
|
||||
if (!$isWindows) {
|
||||
|
|
@ -89,7 +92,19 @@ Copy-Item -ToSession $Session ./artifacts -Destination "$RemoteDir/artifacts" -R
|
|||
Copy-Item -ToSession $Session ./scripts -Destination "$RemoteDir/scripts" -Recurse
|
||||
Copy-Item -ToSession $Session ./src/manifest/MsQuic.wprp -Destination "$RemoteDir/scripts"
|
||||
|
||||
$encounterFailures = $false
|
||||
$SQL = @"
|
||||
INSERT OR IGNORE INTO Secnetperf_builds (Secnetperf_Commit, Build_date_time, TLS_enabled, Advanced_build_config)
|
||||
VALUES ('$MsQuicCommit', '$(Get-Date -Format "yyyy-MM-dd HH:mm:ss")', 1, 'TODO');
|
||||
"@
|
||||
$json = @{}
|
||||
$allTests = @(
|
||||
"-exec:maxtput -up:10s -ptput:1",
|
||||
"-exec:maxtput -down:10s -ptput:1",
|
||||
"-exec:maxtput -rconn:1 -share:1 -conns:100 -run:10s -prate:1",
|
||||
"-exec:lowlat -rstream:1 -up:512 -down:4000 -run:10s -plat:1"
|
||||
)
|
||||
$env = $isWindows ? 1 : 2
|
||||
$hasFailures = $false
|
||||
|
||||
try {
|
||||
|
||||
|
|
@ -106,6 +121,9 @@ if ($isWindows) { # TODO: Run on Linux too?
|
|||
}
|
||||
}
|
||||
|
||||
# Configure the dump collection.
|
||||
Configure-DumpCollection $Session
|
||||
|
||||
if (!$isWindows) {
|
||||
# Make sure the secnetperf binary is executable.
|
||||
Write-Host "Updating secnetperf permissions..."
|
||||
|
|
@ -117,88 +135,53 @@ if (!$isWindows) {
|
|||
chmod +x "./$SecNetPerfPath"
|
||||
}
|
||||
|
||||
# Logging to collect quic traces while running the tests.
|
||||
# if ($LogProfile -ne "" -and $LogProfile -ne "NULL") { # TODO: Linux back slash works?
|
||||
# Write-Host "Starting logging with log profile: $LogProfile..."
|
||||
# .\scripts\log.ps1 -Start -Profile $LogProfile
|
||||
# }
|
||||
|
||||
# Run secnetperf on the server.
|
||||
Write-Host "Starting secnetperf server..."
|
||||
$Job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath -exec:maxtput"
|
||||
|
||||
$PSDefaultParameterValues["Disabled"] = $true # TODO: Why?
|
||||
|
||||
####################################################################################################
|
||||
|
||||
# TEST EXECUTION
|
||||
|
||||
####################################################################################################
|
||||
|
||||
$SQL = @"
|
||||
|
||||
INSERT OR IGNORE INTO Secnetperf_builds (Secnetperf_Commit, Build_date_time, TLS_enabled, Advanced_build_config)
|
||||
VALUES ('$MsQuicCommit', '$(Get-Date -Format "yyyy-MM-dd HH:mm:ss")', 1, 'TODO');
|
||||
# Run all the test cases.
|
||||
Write-Host "Setup complete! Running all tests..."
|
||||
for ($i = 0; $i -lt $allTests.Count; $i++) {
|
||||
$ExeArgs = $allTests[$i]
|
||||
$Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $SecNetPerfPath $LogProfile $ExeArgs
|
||||
$Test = $Output[-1]
|
||||
if ($Test.HasFailures) { $hasFailures = $true }
|
||||
|
||||
# Process the results and add them to the SQL and JSON.
|
||||
$TestId = $i + 1
|
||||
$SQL += @"
|
||||
`nINSERT OR IGNORE INTO Secnetperf_tests (Secnetperf_test_ID, Kernel_mode, Run_arguments) VALUES ($TestId, 0, "$ExeArgs -tcp:0");
|
||||
INSERT OR IGNORE INTO Secnetperf_tests (Secnetperf_test_ID, Kernel_mode, Run_arguments) VALUES ($TestId, 0, "$ExeArgs -tcp:1");
|
||||
"@
|
||||
|
||||
$exe = "./$SecNetPerfPath"
|
||||
for ($tcp = 0; $tcp -lt $Test.Values.Length; $tcp++) {
|
||||
$transport = $tcp -eq 1 ? "tcp" : "quic"
|
||||
foreach ($item in $Test.Values[$tcp]) {
|
||||
$json["$($Test.Metric)-$transport"] = $item
|
||||
if ($Test.Metric.startsWith("throughput")) {
|
||||
# Generate SQL statement. Assume LAST_INSERT_ROW_ID()
|
||||
$SQL += @"
|
||||
`nINSERT INTO Secnetperf_test_runs (Secnetperf_test_ID, Secnetperf_commit, Client_environment_ID, Server_environment_ID, Result, Secnetperf_latency_stats_ID)
|
||||
VALUES ($TestId, '$MsQuicCommit', $env, $env, $item, NULL);
|
||||
"@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$json = @{}
|
||||
|
||||
$maxtput = @(
|
||||
"-exec:maxtput -up:10s -ptput:1",
|
||||
"-exec:maxtput -down:10s -ptput:1",
|
||||
"-exec:maxtput -rconn:1 -share:1 -conns:100 -run:10s -prate:1"
|
||||
)
|
||||
|
||||
$lowlat = @(
|
||||
"-exec:lowlat -rstream:1 -up:512 -down:4000 -run:10s -plat:1"
|
||||
)
|
||||
|
||||
$res = Invoke-SecnetperfTest $MsQuicCommit $maxtput $exe 0 $LogProfile
|
||||
$SQL += $res[0]
|
||||
$json += $res[1]
|
||||
|
||||
# Start and restart the SecNetPerf server without maxtput.
|
||||
Write-Host "Restarting server without maxtput..."
|
||||
Stop-RemoteServer $Job $RemoteName
|
||||
$Job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath -exec:lowlat"
|
||||
|
||||
$res = Invoke-SecnetperfTest $MsQuicCommit $lowlat $exe 3 $LogProfile
|
||||
$SQL += $res[0]
|
||||
$json += $res[1]
|
||||
|
||||
####################################################################################################
|
||||
|
||||
# END TEST EXECUTION
|
||||
|
||||
####################################################################################################
|
||||
|
||||
# Kill the server process.
|
||||
Write-Host "`Stopping server..."
|
||||
Stop-RemoteServer $Job $RemoteName
|
||||
|
||||
# if ($LogProfile -ne "" -and $LogProfile -ne "NULL") {
|
||||
# Write-Host "Stopping logging..."
|
||||
# .\scripts\log.ps1 -Stop -OutputPath .\artifacts\logs\quic
|
||||
# }
|
||||
|
||||
# Save the test results (sql and json).
|
||||
Write-Host "`Writing test-results-$plat-$os-$arch-$tls.sql..."
|
||||
$SQL | Set-Content -Path "test-results-$plat-$os-$arch-$tls.sql"
|
||||
|
||||
Write-Host "`Writing json-test-results-$plat-$os-$arch-$tls.json..."
|
||||
$json | ConvertTo-Json | Set-Content -Path "json-test-results-$plat-$os-$arch-$tls.json"
|
||||
Write-Host "Tests complete!"
|
||||
|
||||
} catch {
|
||||
Write-GHError "Exception occurred!"
|
||||
Write-GHError "Exception while running tests!"
|
||||
Write-GHError $_
|
||||
$encounterFailures = $true
|
||||
Get-Error
|
||||
$_ | Format-List *
|
||||
$hasFailures = $true
|
||||
} finally {
|
||||
# TODO: Do any further book keeping here.
|
||||
# Save the test results (sql and json).
|
||||
Write-Host "`Writing test-results-$plat-$os-$arch-$tls.sql..."
|
||||
$SQL | Set-Content -Path "test-results-$plat-$os-$arch-$tls.sql"
|
||||
|
||||
Write-Host "`Writing json-test-results-$plat-$os-$arch-$tls.json..."
|
||||
$json | ConvertTo-Json | Set-Content -Path "json-test-results-$plat-$os-$arch-$tls.json"
|
||||
}
|
||||
|
||||
if ($encounterFailures) {
|
||||
if ($hasFailures) {
|
||||
exit 1
|
||||
}
|
||||
|
|
|
|||
|
|
@ -207,20 +207,30 @@ public:
|
|||
|
||||
#ifdef CXPLAT_FRE_ASSERT
|
||||
|
||||
#ifndef _KERNEL_MODE
|
||||
#include <stdio.h> // For printf below
|
||||
#endif
|
||||
|
||||
class CxPlatWatchdog {
|
||||
CxPlatEvent ShutdownEvent {true};
|
||||
CxPlatThread WatchdogThread;
|
||||
uint32_t TimeoutMs;
|
||||
bool WriteToConsole;
|
||||
static CXPLAT_THREAD_CALLBACK(WatchdogThreadCallback, Context) {
|
||||
auto This = (CxPlatWatchdog*)Context;
|
||||
if (!This->ShutdownEvent.WaitTimeout(This->TimeoutMs)) {
|
||||
#ifndef _KERNEL_MODE // Not supported in kernel mode
|
||||
if (This->WriteToConsole) {
|
||||
printf("Error: Watchdog timeout fired!\n");
|
||||
}
|
||||
#endif
|
||||
CXPLAT_FRE_ASSERTMSG(FALSE, "Watchdog timeout fired!");
|
||||
}
|
||||
CXPLAT_THREAD_RETURN(0);
|
||||
}
|
||||
public:
|
||||
CxPlatWatchdog(uint32_t WatchdogTimeoutMs, const char* Name = "cxplat_watchdog") noexcept
|
||||
: TimeoutMs(WatchdogTimeoutMs) {
|
||||
CxPlatWatchdog(uint32_t WatchdogTimeoutMs, const char* Name = "cxplat_watchdog", bool WriteToConsole = false) noexcept
|
||||
: TimeoutMs(WatchdogTimeoutMs), WriteToConsole(WriteToConsole) {
|
||||
CXPLAT_THREAD_CONFIG Config;
|
||||
memset(&Config, 0, sizeof(CXPLAT_THREAD_CONFIG));
|
||||
Config.Name = Name;
|
||||
|
|
|
|||
|
|
@ -461,7 +461,7 @@ PerfClientWorker::OnConnectionComplete() {
|
|||
|
||||
PerfClientConnection::~PerfClientConnection() {
|
||||
if (Client.UseTCP) {
|
||||
if (TcpConn) { TcpConn->Close(); }
|
||||
if (TcpConn) { TcpConn->Close(); TcpConn = nullptr; }
|
||||
} else {
|
||||
if (Handle) { MsQuic->ConnectionClose(Handle); }
|
||||
}
|
||||
|
|
@ -472,17 +472,18 @@ PerfClientConnection::Initialize() {
|
|||
if (Client.UseTCP) {
|
||||
auto CredConfig = MsQuicCredentialConfig(QUIC_CREDENTIAL_FLAG_CLIENT | QUIC_CREDENTIAL_FLAG_NO_CERTIFICATE_VALIDATION);
|
||||
TcpConn = // TODO: replace new/delete with pool alloc/free
|
||||
new (std::nothrow) TcpConnection(
|
||||
Client.Engine.get(),
|
||||
&CredConfig,
|
||||
new (std::nothrow) TcpConnection(Client.Engine.get(), &CredConfig, this);
|
||||
if (!TcpConn->IsInitialized()) {
|
||||
Worker.ConnectionPool.Free(this);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TcpConn->Start(
|
||||
Client.TargetFamily,
|
||||
Worker.Target.get(),
|
||||
Worker.RemoteAddr.GetPort(),
|
||||
Worker.LocalAddr.GetFamily() != QUIC_ADDRESS_FAMILY_UNSPEC ? &Worker.LocalAddr.SockAddr : nullptr,
|
||||
&Worker.RemoteAddr.SockAddr,
|
||||
this);
|
||||
if (!TcpConn->IsInitialized()) {
|
||||
TcpConn->Close();
|
||||
&Worker.RemoteAddr.SockAddr)) {
|
||||
Worker.ConnectionPool.Free(this);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ PerfServer::Init(
|
|||
//
|
||||
QuicAddr TeardownLocalAddress {QUIC_ADDRESS_FAMILY_INET, (uint16_t)9999};
|
||||
CXPLAT_UDP_CONFIG UdpConfig = {&TeardownLocalAddress.SockAddr, 0};
|
||||
UdpConfig.CallbackContext = this;
|
||||
#ifdef QUIC_OWNING_PROCESS
|
||||
UdpConfig.OwningProcess = QuicProcessGetCurrentProcess();
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -218,7 +218,7 @@ QuicMainStart(
|
|||
|
||||
uint32_t WatchdogTimeout = 0;
|
||||
if (TryGetValue(argc, argv, "watchdog", &WatchdogTimeout) && WatchdogTimeout != 0) {
|
||||
Watchdog = new(std::nothrow) CxPlatWatchdog(WatchdogTimeout, "perf_watchdog");
|
||||
Watchdog = new(std::nothrow) CxPlatWatchdog(WatchdogTimeout, "perf_watchdog", true);
|
||||
}
|
||||
|
||||
const CXPLAT_UDP_DATAPATH_CALLBACKS DatapathCallbacks = {
|
||||
|
|
|
|||
|
|
@ -122,10 +122,12 @@ TcpEngine::TcpEngine(
|
|||
TcpEngine::~TcpEngine() noexcept
|
||||
{
|
||||
// Loop over all connections and shut them down.
|
||||
ShuttingDown = true;
|
||||
ConnectionLock.Acquire();
|
||||
while (!CxPlatListIsEmpty(&Connections)) {
|
||||
auto Connection = (TcpConnection*)CxPlatListRemoveHead(&Connections);
|
||||
Connection->EngineEntry.Flink = NULL;
|
||||
CXPLAT_LIST_ENTRY* Entry = Connections.Flink;
|
||||
while (Entry != &Connections) {
|
||||
auto Connection = (TcpConnection*)Entry;
|
||||
Entry = Entry->Flink;
|
||||
Connection->Shutdown = true;
|
||||
Connection->TotalSendCompleteOffset = UINT64_MAX;
|
||||
Connection->Queue();
|
||||
|
|
@ -140,16 +142,23 @@ TcpEngine::~TcpEngine() noexcept
|
|||
delete [] Workers;
|
||||
}
|
||||
|
||||
void TcpEngine::AddConnection(TcpConnection* Connection, uint16_t PartitionIndex)
|
||||
bool TcpEngine::AddConnection(TcpConnection* Connection, uint16_t PartitionIndex)
|
||||
{
|
||||
bool Added = false;
|
||||
CXPLAT_DBG_ASSERT(PartitionIndex < ProcCount);
|
||||
CXPLAT_DBG_ASSERT(!Connection->Worker);
|
||||
Connection->PartitionIndex = PartitionIndex;
|
||||
Connection->Worker = &Workers[PartitionIndex];
|
||||
CXPLAT_FRE_ASSERT(Rundown.Acquire());
|
||||
ConnectionLock.Acquire();
|
||||
CxPlatListInsertTail(&Connections, &Connection->EngineEntry);
|
||||
ConnectionLock.Release();
|
||||
if (Rundown.Acquire()) {
|
||||
Connection->HasRundownRef = true;
|
||||
ConnectionLock.Acquire();
|
||||
if (!ShuttingDown) {
|
||||
CxPlatListInsertTail(&Connections, &Connection->EngineEntry);
|
||||
Added = true;
|
||||
}
|
||||
ConnectionLock.Release();
|
||||
}
|
||||
return Added;
|
||||
}
|
||||
|
||||
void TcpEngine::RemoveConnection(TcpConnection* Connection)
|
||||
|
|
@ -157,10 +166,11 @@ void TcpEngine::RemoveConnection(TcpConnection* Connection)
|
|||
ConnectionLock.Acquire();
|
||||
if (Connection->EngineEntry.Flink) {
|
||||
CxPlatListEntryRemove(&Connection->EngineEntry);
|
||||
Connection->EngineEntry.Flink = NULL;
|
||||
}
|
||||
ConnectionLock.Release();
|
||||
Rundown.Release();
|
||||
if (Connection->HasRundownRef) {
|
||||
Rundown.Release();
|
||||
}
|
||||
}
|
||||
|
||||
// ############################# WORKER #############################
|
||||
|
|
@ -315,12 +325,6 @@ TcpServer::AcceptCallback(
|
|||
TcpConnection::TcpConnection(
|
||||
TcpEngine* Engine,
|
||||
const QUIC_CREDENTIAL_CONFIG* CredConfig,
|
||||
_In_ QUIC_ADDRESS_FAMILY Family,
|
||||
_In_reads_or_z_opt_(QUIC_MAX_SNI_LENGTH)
|
||||
const char* ServerName,
|
||||
_In_ uint16_t ServerPort,
|
||||
const QUIC_ADDR* LocalAddress,
|
||||
const QUIC_ADDR* RemoteAddress,
|
||||
void* Context) :
|
||||
IsServer(false), Engine(Engine), Context(Context)
|
||||
{
|
||||
|
|
@ -340,6 +344,22 @@ TcpConnection::TcpConnection(
|
|||
WriteOutput("SecConfig load FAILED\n");
|
||||
return;
|
||||
}
|
||||
Initialized = true;
|
||||
}
|
||||
|
||||
bool
|
||||
TcpConnection::Start(
|
||||
_In_ QUIC_ADDRESS_FAMILY Family,
|
||||
_In_reads_or_z_opt_(QUIC_MAX_SNI_LENGTH)
|
||||
const char* ServerName,
|
||||
_In_ uint16_t ServerPort,
|
||||
const QUIC_ADDR* LocalAddress,
|
||||
const QUIC_ADDR* RemoteAddress
|
||||
)
|
||||
{
|
||||
if (!Engine->AddConnection(this, (uint16_t)CxPlatProcCurrentNumber())) {
|
||||
return false;
|
||||
}
|
||||
if (LocalAddress) {
|
||||
Family = QuicAddrGetFamily(LocalAddress);
|
||||
}
|
||||
|
|
@ -353,12 +373,10 @@ TcpConnection::TcpConnection(
|
|||
ServerName,
|
||||
&Route.RemoteAddress))) {
|
||||
WriteOutput("CxPlatDataPathResolveAddress FAILED\n");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
QuicAddrSetPort(&Route.RemoteAddress, ServerPort);
|
||||
Engine->AddConnection(this, (uint16_t)CxPlatProcCurrentNumber());
|
||||
Initialized = true;
|
||||
if (QUIC_FAILED(
|
||||
CxPlatSocketCreateTcp(
|
||||
Datapath,
|
||||
|
|
@ -366,10 +384,10 @@ TcpConnection::TcpConnection(
|
|||
&Route.RemoteAddress,
|
||||
this,
|
||||
&Socket))) {
|
||||
Initialized = false;
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
Queue();
|
||||
return true;
|
||||
}
|
||||
|
||||
TcpConnection::TcpConnection(
|
||||
|
|
@ -389,7 +407,7 @@ TcpConnection::TcpConnection(
|
|||
this);
|
||||
Initialized = true;
|
||||
IndicateAccept = true;
|
||||
Engine->AddConnection(this, (uint16_t)CxPlatProcCurrentNumber());
|
||||
CXPLAT_FRE_ASSERT(Engine->AddConnection(this, (uint16_t)CxPlatProcCurrentNumber()));
|
||||
Queue();
|
||||
}
|
||||
|
||||
|
|
@ -719,7 +737,7 @@ bool TcpConnection::SendTlsData(const uint8_t* Buffer, uint16_t BufferLength, ui
|
|||
{
|
||||
auto SendBuffer = NewSendBuffer();
|
||||
if (!SendBuffer) {
|
||||
WriteOutput("NewSendBuffer FAILED\n");
|
||||
//WriteOutput("NewSendBuffer FAILED\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -898,7 +916,7 @@ bool TcpConnection::ProcessSend()
|
|||
do {
|
||||
auto SendBuffer = NewSendBuffer();
|
||||
if (!SendBuffer) {
|
||||
WriteOutput("NewSendBuffer FAILED\n");
|
||||
//WriteOutput("NewSendBuffer FAILED\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -1000,6 +1018,9 @@ bool TcpConnection::EncryptFrame(TcpFrame* Frame)
|
|||
|
||||
QUIC_BUFFER* TcpConnection::NewSendBuffer()
|
||||
{
|
||||
if (Shutdown || !Socket) { // Queue (from Engine shutdown) happened before socket creation finished
|
||||
return nullptr;
|
||||
}
|
||||
if (!BatchedSendData) {
|
||||
CXPLAT_SEND_CONFIG SendConfig = { &Route, TLS_BLOCK_SIZE, CXPLAT_ECN_NON_ECT, 0 };
|
||||
BatchedSendData = CxPlatSendDataAlloc(Socket, &SendConfig);
|
||||
|
|
@ -1064,7 +1085,8 @@ void TcpConnection::Close()
|
|||
"[perf][tcp][%p] App Close",
|
||||
this);
|
||||
if (!Initialized) {
|
||||
// no-op
|
||||
ClosedByApp = true;
|
||||
Closed = true;
|
||||
} else if (WorkerThreadID == CxPlatCurThreadID()) {
|
||||
ClosedByApp = true;
|
||||
Shutdown = true;
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ typedef TcpSendCompleteCallback* TcpSendCompleteHandler;
|
|||
class TcpEngine {
|
||||
friend class TcpWorker;
|
||||
bool Initialized{false};
|
||||
bool ShuttingDown{false};
|
||||
bool Shutdown{false};
|
||||
const uint16_t ProcCount;
|
||||
TcpWorker* Workers;
|
||||
|
|
@ -102,7 +103,7 @@ public:
|
|||
TcpSendCompleteHandler SendCompleteHandler) noexcept;
|
||||
~TcpEngine() noexcept;
|
||||
bool IsInitialized() const { return Initialized; }
|
||||
void AddConnection(TcpConnection* Connection, uint16_t PartitionIndex);
|
||||
bool AddConnection(TcpConnection* Connection, uint16_t PartitionIndex);
|
||||
void RemoveConnection(TcpConnection* Connection);
|
||||
};
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ class TcpConnection {
|
|||
bool IndicateAccept{false};
|
||||
bool IndicateConnect{false};
|
||||
bool IndicateSendComplete{false};
|
||||
bool HasRundownRef{false};
|
||||
TcpConnection* Next{nullptr};
|
||||
TcpEngine* Engine;
|
||||
TcpWorker* Worker{nullptr};
|
||||
|
|
@ -264,14 +266,15 @@ public:
|
|||
TcpConnection(
|
||||
_In_ TcpEngine* Engine,
|
||||
_In_ const QUIC_CREDENTIAL_CONFIG* CredConfig,
|
||||
_In_ void* Context = nullptr);
|
||||
bool IsInitialized() const { return Initialized; }
|
||||
void Close();
|
||||
bool Start(
|
||||
_In_ QUIC_ADDRESS_FAMILY Family,
|
||||
_In_reads_or_z_opt_(QUIC_MAX_SNI_LENGTH)
|
||||
const char* ServerName,
|
||||
_In_ uint16_t ServerPort,
|
||||
_In_ const QUIC_ADDR* LocalAddress = nullptr,
|
||||
_In_ const QUIC_ADDR* RemoteAddress = nullptr,
|
||||
_In_ void* Context = nullptr);
|
||||
bool IsInitialized() const { return Initialized; }
|
||||
void Close();
|
||||
_In_ const QUIC_ADDR* RemoteAddress = nullptr);
|
||||
bool Send(TcpSendData* Data);
|
||||
};
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче