microsoft
/
ebpf-for-windows
зеркало из https://github.com/microsoft/ebpf-for-windows.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

update scripts to handle test hangs. (#3947) 

* update scripts to handle test hangs.

* fix.
This commit is contained in:
Shankar Seal 2024-10-25 16:44:27 -07:00 коммит произвёл GitHub
Родитель a7ac1b54b6
Коммит 7005b7ff47
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
10 изменённых файлов: 579 добавлений и 657 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

281
scripts/Run-Self-Hosted-Runner-Test.ps1
Просмотреть файл

@ -1,281 +0,0 @@
# Copyright (c) eBPF for Windows contributors
# SPDX-License-Identifier: MIT
#
# This script executes the provided test command, waits for <TestHangTimeout> (in seconds) and then triggers user and
# kernel dumps if the test process is still running (This is typically the case when the test process is hung for some
# reason).
#
# The user mode dump is created using 'procdump64.exe' and the kernel dump using 'notmyfault64.exe' utilities
# respectively, both from the SysInternals Suite. The script assumes the presence of these utilities under the current
# working directory.
#
# (While 'procdump64.exe' also has the ability to generate a kernel dump, that dump is restricted to the kernel
# portions of the user mode app's threads _only_ and does not provide any visibility into other kernel threads. This
# script therefore uses the 'notmyfault64.exe' tool to generate a 'true' kernel dump which captures the state of all
# kernel threads and related data structures)
#
[CmdletBinding()]
Param(
[Parameter(Mandatory = $true)] [string] $TestCommand,
[Parameter(Mandatory = $false)] [string] $TestArguments = "",
[Parameter(Mandatory = $false)] [int] $TestHangTimeout = 3600,
[Parameter(Mandatory = $false)] [string] $UserModeDumpFolder = "C:\Dumps",
[Parameter(Mandatory = $false)] [bool] $NeedKernelDump = $true
)
function ThrowWithErrorMessage
{
Param(
[Parameter(Mandatory = $True)] [string] $ErrorMessage
)
Write-Log $ErrorMessage
# Wait a bit to let the above message to show up.
Start-Sleep -seconds 5
throw $ErrorMessage
}
# Finds and returns the specified tool's location under the current directory. If not found, throws an exception.
function GetToolLocationPath
{
param(
[Parameter(Mandatory = $True)] [string] $ToolName
)
$ToolLocationPath = Get-ChildItem -Path "$Pwd" `
-Recurse -Filter "$ToolName" -ErrorAction SilentlyContinue | Select-Object -First 1
if ($ToolLocationPath -eq $null) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** $ToolName not found under $Pwd."
}
return $ToolLocationPath.FullName
}
function GetDriveFreeSpaceGB
{
Param(
# Drive Specification in the form of a single alphabet string. (eg. "C:")
[Parameter(Mandatory = $True)] [string] $DriveSpecification
)
if (($DriveSpecification.Length -eq $Null) -or ($DriveSpecification.Length -ne 2)) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** No drive or Invalid drive specified."
}
# Convert drive to single letter (eg. "C:" to "C") for Get-Volume.
$DriveSpecification = $DriveSpecification -replace ".$"
$Volume = Get-Volume $DriveSpecification
if ($Volume -eq $Null) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** Drive $DriveSpecification not found."
}
$FreeSpaceGB = (($Volume.SizeRemaining) / 1GB).ToString("F2")
return $FreeSpaceGB
}
if ($VerbosePreference -eq 'Continue') {
Write-Log "Command : $TestCommand"
Write-Log "Arguments : $TestArguments"
Write-Log "Test Hang Timeout : $TestHangTimeout"
Write-Log "User mode dump Folder : $UserModeDumpFolder"
Write-Log "Kernel dump needed : $NeedKernelDump"
}
# Verify timeout is non-zero.
if ($TestHangTimeout -le 0) {
ThrowWithErrorMessage `
-ErrorMessage "*** ERROR *** Invalid test hang timeout value: $TestHangTimeout"
}
# Verify user mode dump folder name is not null.
if ($UserModeDumpFolder -eq $Null) {
ThrowWithErrorMessage `
-ErrorMessage "*** ERROR *** User mode dump folder cannot be NULL"
}
# Create dump folder if not present.
if (-not (Test-Path -Path $UserModeDumpFolder)) {
New-Item -Path $UserModeDumpFolder -ItemType Directory -Force | Out-Null
# Verify dump folder creation.
if (-not (Test-Path -Path $UserModeDumpFolder)) {
ThrowWithErrorMessage `
-ErrorMessage "*** ERROR *** User mode dump folder creation failed: $UserModeDumpFolder"
}
}
# Check if procdump64.exe and notmyfault64.exe are present on the system.
$ProcDumpBinary = "ProcDump64.exe"
Write-Log "Verifying $ProcDumpBinary presence in $Pwd..."
$ProcDumpBinaryPath = GetToolLocationPath -ToolName $ProcDumpBinary
Write-Log "$ProcDumpBinary location: $ProcDumpBinaryPath"
Write-Log "`n"
$NotMyFaultBinary = "NotMyFault64.exe"
Write-Log "Verifying $NotMyFaultBinary presence in $Pwd..."
$NotMyFaultBinaryPath = GetToolLocationPath -ToolName $NotMyFaultBinary
Write-Log "$NotMyFaultBinary location: $NotMyFaultBinaryPath"
Write-Log "`n"
# While at it, enable EULA for all SysInternals tools.
REG ADD HKCU\Software\Sysinternals /v EulaAccepted /t REG_DWORD /d 1 /f | Out-Null
# The following 'Set-ItemProperty' command enables a full memory dump.
# NOTE: This needs a VM with an explicitly created page file of *AT LEAST* (physical_memory + 1MB) in size.
# The default value of the 'CrashDumpEnabled' key is 7 ('automatic' sizing of dump file size (system determined)).
# https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/memory-dump-file-options
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl' -Name 'CrashDumpEnabled' -Value 1
if ($VerbosePreference -eq 'Continue') {
# Dump current kernel mode dump settings.
Write-Log "`n"
Write-Log "Current kernel dump configuration:`n"
$KernelDumpInfo = Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl'
$Lines = $KernelDumpInfo -split '; '
foreach($Line in $Lines) {
Write-Log "`t$Line"
}
Write-Log "`n"
}
# Get the available free space before test start (useful in investigating dump file creation failures)
try {
$BeforeTestFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive
} catch {
Write-Log "Error getting available disk space: $_"
$BeforeTestFreeSpaceGB = "Unknown"
# Continue with the test.
}
Write-Log "Available System disk space (Before test start): $BeforeTestFreeSpaceGB GB"
# Start the test process using the provided command and arguments.
$FullTestCommandSpec = Join-Path $Pwd $TestCommand
Write-Log "`n`n"
Write-Log "Starting Test command: $FullTestCommandSpec $TestArguments"
Write-Log "Test hang timeout: $TestHangTimeout (seconds)"
Write-Log "`n"
# Create a temporary file for standard output and error.
$TestTempOutFile = [System.IO.Path]::GetTempFileName()
# Form the complete command line with output redirection for cmd.exe.
$TestCommandLine = "$FullTestCommandSpec $($TestArguments) > $TestTempOutFile 2>&1"
# Start the test process and wait for it to exit or timeout.
$TestProcess = Start-Process -FilePath "cmd.exe" -ArgumentList "/c $TestCommandLine" -PassThru -NoNewWindow
# Cache the process handle. This has a side-effect that enables access to the process's exit code after termination.
# (https://stackoverflow.com/a/23797762)
$ProcessHandle = $TestProcess.Handle
$WaitResult = $TestProcess.WaitForExit($TestHangTimeout * 1000)
if (-not $WaitResult) {
Write-Log "`n"
Write-Log "*** ERROR *** Test execution hang timeout ($TestHangTimeout seconds) expired.`n"
# First, generate a user mode dump.
$UserModeDumpFileName = "$($TestCommand)_$(Get-Date -Format 'yyyy-MM-dd_HH-mm-ss').dmp"
$UserModeDumpFilePath = Join-Path $UserModeDumpFolder $UserModeDumpFileName
if ($VerbosePreference -eq 'Continue') {
Write-Log "User mode Dumpfile name: $UserModeDumpFileName"
Write-Log "User mode Dumpfile Path: $UserModeDumpFilePath"
}
# Get the available free space at this point in case the test creates its own files.
# (useful in investigating user and/or kernel dump file creation failures).
try {
$DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive
} catch {
Write-Log "Error getting available disk space: $_"
$DriveFreeSpaceGB = "Unknown"
# Continue with the test.
}
Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n"
# $TestProcess refers to 'cmd.exe' which ends up running the real test application.
# (This is done so that we can capture the stdout and stderr outputs from the test application
# itself. I have not been able to get Powershell's 'built-in' redirection mechanisms for the
# 'Process' object to play well in our scenario).
# We therefore need pass the test application's id to procdump64.exe
$TestCommandId = (Get-Process -Name $TestCommand).Id
Write-Log "Test Command:$TestCommand, Id:$TestCommandId"
Write-Log "Creating User mode dump @ $UserModeDumpFilePath"
$ProcDumpArguments = "-r -ma $($TestCommandId) $UserModeDumpFilePath"
Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments"
$ProcDumpProcess = Start-Process -NoNewWindow `
-FilePath $ProcDumpBinaryPath `
-ArgumentList $ProcDumpArguments `
-Wait -PassThru
Write-Log "Waiting for user mode dump to complete..."
$ProcDumpProcess.WaitForExit()
# Get procdump64.exe's exit code.
$ExitCode = $($ProcDumpProcess.ExitCode)
Write-Log "$ProcDumpBinaryPath completed with exit code: $ExitCode`n"
# Flush disk buffers to ensure user mode dump data is completely written to disk.
Write-Log "User mode dump completed. Flushing disk buffers..."
Write-VolumeCache -DriveLetter C
# Wait for a bit for things to settle down to ensure the user mode dump is completely flushed.
Start-Sleep -seconds 10
# Paranoia check: Make sure a (seemingly) valid dump file _is_ created.
$UserModeDumpSizeMB =
(((Get-ItemProperty -Path $UserModeDumpFilePath).Length) /1MB).ToString("F2")
if ($UserModeDumpSizeMB -eq 0) {
Write-Log "* WARNING * User mode dump $UserModeDumpFilePath NOT CREATED"
} else {
Write-Log "`n`n"
Write-Log "Created $UserModeDumpFilePath, size: $UserModeDumpSizeMB MB"
Write-Log "`n`n"
}
if ($NeedKernelDump) {
Write-Log "Creating kernel dump...`n"
# Wait a bit for the above message to show up in the log.
Start-Sleep -seconds 5
# This will/should not return (test system will/should bluescreen and reboot).
Start-Process -NoNewWindow -Wait -FilePath $NotMyFaultBinaryPath -ArgumentList "/crash"
# If we get here, notmyfault64.exe failed for some reason. Kill the hung process, throw error.
$TestProcess.Kill()
ThrowWithErrorMessage `
-ErrorMessage "*** ERROR *** $($PSCommandPath): kernel mode dump creation FAILED"
} else {
Write-Log "`n`n"
Write-Log "Kernel dump not needed, killing test process $($TestProcess.ProcessName)..."
$TestProcess.Kill()
Write-Log "`n`n"
$ErrorMessage = "*** ERROR *** $($PSCommandPath): $FullTestCommandSpec.exe exceeded test hang timout of " +
"$TestHangTimeout seconds"
ThrowWithErrorMessage -ErrorMessage $ErrorMessage
}
} else {
# Ensure the process has completely exited.
$TestProcess.WaitForExit()
# Read and display the output (if any) from the temporary output file.
$TestStandardOutput = Get-Content -Path $TestTempOutFile
if ($TestStandardOutput) {
Write-Log "`n`n"
Write-Log "Test Program output:`n"
foreach($Line in $TestStandardOutput) {
if ($Line -ne $Null) {
Write-Log $Line
}
}
}
Write-Log "`n"
if ($($TestProcess.ExitCode) -ne 0) {
$ErrorMessage = "*** ERROR *** $($PSCommandPath): $FullTestCommandSpec failed."
ThrowWithErrorMessage -ErrorMessage $ErrorMessage
}
}

4
scripts/cleanup_ebpf_cicd_tests.ps1
Просмотреть файл

@ -2,7 +2,7 @@
# SPDX-License-Identifier: MIT
param ([parameter(Mandatory=$false)][string] $Target = "TEST_VM",
[parameter(Mandatory=$true)][bool] $KmTracing,
[parameter(Mandatory=$false)][bool] $KmTracing = $true,
[parameter(Mandatory=$false)][string] $LogFileName = "TestLog.log",
[parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(),
[parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json",
@ -40,7 +40,7 @@ foreach ($VM in $VMList) {
}
if ($DumpFound -eq $True) {
Write-Host "`n=== Post-crash reboot detected on VM $VMName ===`n"
Write-Log "Post-crash reboot detected on VM $VMName"
} else {
# Stop eBPF Components on the test VM. (Un-install is not necessary.)
# We *MUST* be able to stop all drivers cleanly after a test. Failure to do so indicates a fatal bug in

12
scripts/common.psm1
Просмотреть файл

@ -3,7 +3,6 @@
param ([parameter(Mandatory=$True)] [string] $LogFileName)
#
# Common helper functions.
#
@ -24,6 +23,17 @@ function Write-Log
}
}
function ThrowWithErrorMessage
{
Param(
[Parameter(Mandatory = $True)] [string] $ErrorMessage
)
Write-Log $ErrorMessage -ForegroundColor Red
Start-Sleep -Milliseconds 100
throw $ErrorMessage
}
function New-Credential
{
param([Parameter(Mandatory=$True)][string] $UserName,

56
scripts/config_test_vm.psm1
Просмотреть файл

@ -188,9 +188,24 @@ function Export-BuildArtifactsToVMs
throw "Failed to create PowerShell session on $VMName."
} else {
Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
# Create working directory c:\eBPF.
if(!(Test-Path "$Env:SystemDrive\eBPF")) {
New-Item -ItemType Directory -Path "$Env:SystemDrive\eBPF"
}
# Enable EULA for all SysInternals tools.
$RegistryPath = 'HKCU:\Software\Sysinternals'
if (-not (Test-Path $RegistryPath)) {
# Create the registry key if it doesn't exist
New-Item -Path $RegistryPath -Force
}
Set-ItemProperty -Path $RegistryPath -Name 'EulaAccepted' -Value 1
# Enables full memory dump.
# NOTE: This needs a VM with an explicitly created page file of *AT LEAST* (physical_memory + 1MB) in size.
# The default value of the 'CrashDumpEnabled' key is 7 ('automatic' sizing of dump file size (system determined)).
# https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/memory-dump-file-options
Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl' -Name 'CrashDumpEnabled' -Value 1
return $Env:SystemDrive
}
$VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}
@ -223,7 +238,6 @@ function Install-eBPFComponentsOnVM
Write-Log "Installing eBPF components on $VMName"
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
[Parameter(Mandatory=$True)] [string] $LogFileName,
@ -435,15 +449,6 @@ function Import-ResultsFromVM
-Force `
-ErrorAction Ignore 2>&1 | Write-Log
Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..")
Copy-Item `
-FromSession $VMSession `
-Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" `
-Destination "$pwd\..\.." `
-Recurse `
-Force `
-ErrorAction Ignore 2>&1 | Write-Log
# Copy kernel mode traces, if enabled.
if ($KmTracing) {
$EtlFile = $LogFileName.Substring(0, $LogFileName.IndexOf('.')) + ".etl"
@ -525,7 +530,7 @@ function Import-ResultsFromVM
-ErrorAction Ignore 2>&1 | Write-Log
}
# Move runner test logs to TestLogs folder.
Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")
Write-Log "Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs"
Move-Item "$env:TEMP\$LogFileName" -Destination ".\TestLogs" -Force -ErrorAction Ignore 2>&1 | Write-Log
}
@ -544,17 +549,19 @@ function Initialize-NetworkInterfacesOnVMs
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory=$True)] [string] $WorkingDirectory)
param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $true)][string] $LogFileName)
Push-Location "$env:SystemDrive\$WorkingDirectory"
Import-Module .\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
Write-Host "Installing DuoNic driver"
Write-Log "Installing DuoNic driver"
.\duonic.ps1 -Install -NumNicPairs 2
# Disable Duonic's fake checksum offload and force TCP/IP to calculate it.
Set-NetAdapterAdvancedProperty duo? -DisplayName Checksum -RegistryValue 0
Pop-Location
} -ArgumentList ("eBPF") -ErrorAction Stop
} -ArgumentList ("eBPF", $LogFileName) -ErrorAction Stop
}
}
@ -677,36 +684,27 @@ function Get-RegressionTestArtifacts
}
# Copied from https://github.com/microsoft/msquic/blob/main/scripts/prepare-machine.ps1
function Get-Duonic {
function Get-CoreNetTools {
# Download and extract https://github.com/microsoft/corenet-ci.
$DownloadPath = "$pwd\corenet-ci"
mkdir $DownloadPath
Write-Host "Downloading CoreNet-CI to $DownloadPath"
Write-Log "Downloading CoreNet-CI to $DownloadPath"
Get-ZipFileFromUrl -Url "https://github.com/microsoft/corenet-ci/archive/refs/heads/main.zip" -DownloadFilePath "$DownloadPath\corenet-ci.zip" -OutputDir $DownloadPath
#DuoNic.
Move-Item -Path "$DownloadPath\corenet-ci-main\vm-setup\duonic\*" -Destination $pwd -Force
# Procdump.
Move-Item -Path "$DownloadPath\corenet-ci-main\vm-setup\procdump64.exe" -Destination $pwd -Force
# NotMyFault.
Move-Item -Path "$DownloadPath\corenet-ci-main\vm-setup\notmyfault64.exe" -Destination $pwd -Force
Remove-Item -Path $DownloadPath -Force -Recurse
}
# Download the Visual C++ Redistributable.
function Get-VCRedistributable {
$url = "https://aka.ms/vs/17/release/vc_redist.x64.exe"
$DownloadPath = "$pwd\vc-redist"
mkdir $DownloadPath
Write-Host "Downloading Visual C++ Redistributable from $url to $DownloadPath"
$ProgressPreference = 'SilentlyContinue'
Invoke-WebRequest -Uri $url -OutFile "$DownloadPath\vc_redist.x64.exe"
Move-Item -Path "$DownloadPath\vc_redist.x64.exe" -Destination $pwd -Force
Remove-Item -Path $DownloadPath -Force -Recurse
}
# Download and extract PSExec to run tests as SYSTEM.
function Get-PSExec {
$url = "https://download.sysinternals.com/files/PSTools.zip"
$DownloadPath = "$pwd\psexec"
mkdir $DownloadPath
Write-Host "Downloading PSExec from $url to $DownloadPath"
Write-Log "Downloading PSExec from $url to $DownloadPath"
$ProgressPreference = 'SilentlyContinue'
Invoke-WebRequest $url -OutFile "$DownloadPath\pstools.zip"
cd $DownloadPath

163
scripts/execute_ebpf_cicd_tests.ps1
Просмотреть файл

@ -1,86 +1,115 @@
# Copyright (c) eBPF for Windows contributors
# SPDX-License-Identifier: MIT
param ([parameter(Mandatory = $false)][string] $AdminTarget = "TEST_VM",
[parameter(Mandatory = $false)][string] $StandardUserTarget = "TEST_VM_STANDARD",
[parameter(Mandatory = $false)][string] $LogFileName = "TestLog.log",
[parameter(Mandatory = $false)][string] $WorkingDirectory = $pwd.ToString(),
[parameter(Mandatory = $false)][string] $TestExecutionJsonFileName = "test_execution.json",
[parameter(Mandatory = $false)][bool] $Coverage = $false,
[parameter(Mandatory = $false)][string] $TestMode = "CI/CD",
[parameter(Mandatory = $false)][string[]] $Options = @("None"),
[parameter(Mandatory = $false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(),
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps"
param ([Parameter(Mandatory = $false)][string] $AdminTarget = "TEST_VM",
[Parameter(Mandatory = $false)][string] $StandardUserTarget = "TEST_VM_STANDARD",
[Parameter(Mandatory = $false)][string] $LogFileName = "TestLog.log",
[Parameter(Mandatory = $false)][string] $WorkingDirectory = $pwd.ToString(),
[Parameter(Mandatory = $false)][string] $TestExecutionJsonFileName = "test_execution.json",
[Parameter(Mandatory = $false)][string] $TestMode = "CI/CD",
[Parameter(Mandatory = $false)][string[]] $Options = @("None"),
[Parameter(Mandatory = $false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(),
[Parameter(Mandatory = $false)][int] $TestHangTimeout = (10*60),
[Parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps",
[Parameter(Mandatory = $false)][int] $TestJobTimeout = (60*60)
)
Push-Location $WorkingDirectory
Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -ErrorAction Stop
$AdminTestVMCredential = Get-StoredCredential -Target $AdminTarget -ErrorAction Stop
$StandardUserTestVMCredential = Get-StoredCredential -Target $StandardUserTarget -ErrorAction Stop
# Load other utility modules.
Import-Module $PSScriptRoot\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
Import-Module $PSScriptRoot\vm_run_tests.psm1 `
-Force `
-ArgumentList (
$AdminTestVMCredential.UserName,
$AdminTestVMCredential.Password,
$StandardUserTestVMCredential.UserName,
$StandardUserTestVMCredential.Password,
$WorkingDirectory,
$LogFileName,
$TestHangTimeout,
$UserModeDumpFolder) `
-WarningAction SilentlyContinue
# Read the test execution json.
$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json
$VMList = $Config.VMMap.$SelfHostedRunnerName
# currently one VM runs per runner.
$TestVMName = $VMList[0].Name
# Run tests on test VMs.
foreach ($VM in $VMList) {
Invoke-CICDTestsOnVM `
-VMName $VM.Name `
-Coverage $Coverage `
-TestMode $TestMode `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Options $Options
$Job = Start-Job -ScriptBlock {
param (
[Parameter(Mandatory = $True)][string] $TestVMName,
[Parameter(Mandatory = $true)] [PSCustomObject] $Config,
[Parameter(Mandatory = $True)] [string] $Admin,
[Parameter(Mandatory = $True)] [SecureString] $AdminPassword,
[Parameter(Mandatory = $True)] [string] $StandardUser,
[Parameter(Mandatory = $True)] [SecureString] $StandardUserPassword,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[Parameter(Mandatory = $True)][string] $TestMode,
[Parameter(Mandatory = $True)][string[]] $Options,
[Parameter(Mandatory = $True)][int] $TestHangTimeout,
[Parameter(Mandatory = $True)][string] $UserModeDumpFolder)
Push-Location $WorkingDirectory
# Load other utility modules.
Import-Module $WorkingDirectory\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\vm_run_tests.psm1 `
-Force `
-ArgumentList (
$Admin,
$AdminPassword,
$StandardUser,
$StandardUserPassword,
$WorkingDirectory,
$LogFileName,
$TestMode,
$Options,
$TestHangTimeout,
$UserModeDumpFolder) `
-WarningAction SilentlyContinue
# Run Kernel tests on test VM.
Write-Log "Running kernel tests on $TestVMName"
Run-KernelTestsOnVM -VMName $TestVMName -Config $Config
# Stop eBPF components on test VMs.
Stop-eBPFComponentsOnVM -VMName $TestVMName
} -ArgumentList (
$TestVMName,
$Config,
$AdminTestVMCredential.UserName,
$AdminTestVMCredential.Password,
$StandardUserTestVMCredential.UserName,
$StandardUserTestVMCredential.Password,
$WorkingDirectory,
$LogFileName,
$TestMode,
$Options,
$TestHangTimeout,
$UserModeDumpFolder)
# Keep track of the last received output count
$LastCount = 0
$TimeElapsed = 0
# Loop to fetch and print job output in near real-time
while ($Job.State -eq 'Running') {
$JobOutput = Receive-Job -Job $job
$JobOutput | ForEach-Object { Write-Host $_ }
Start-Sleep -Seconds 2
$TimeElapsed += 2
if ($TimeElapsed -gt $TestJobTimeout) {
if ($Job.State -eq "Running") {
Write-Host "Running kernel tests on $TestVMName has timed out after one hour" -ForegroundColor Yellow
$Timestamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss'
$CheckpointName = "$TestVMName-Checkpoint-$Timestamp"
Write-Log "Taking snapshot $CheckpointName of $TestVMName"
Checkpoint-VM -Name $TestVMName -SnapshotName $CheckpointName
break
}
}
}
# This script is used to execute the various kernel mode tests. The required behavior is selected by the $TestMode
# parameter.
if (($TestMode -eq "CI/CD") -or ($TestMode -eq "Regression")) {
# Print any remaining output after the job completes
$JobOutput = Receive-Job -Job $job
$JobOutput | ForEach-Object { Write-Host $_ }
# Run XDP Tests.
Invoke-XDPTestsOnVM `
-Interfaces $Config.Interfaces `
-VMName $VMList[0].Name `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
# Run Connect Redirect Tests.
Invoke-ConnectRedirectTestsOnVM `
-Interfaces $Config.Interfaces `
-ConnectRedirectTestConfig $Config.ConnectRedirectTest `
-UserType "Administrator" `
-VMName $VMList[0].Name `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
Invoke-ConnectRedirectTestsOnVM `
-Interfaces $Config.Interfaces `
-ConnectRedirectTestConfig $Config.ConnectRedirectTest `
-UserType "StandardUser" `
-VMName $VMList[0].Name `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
}
# Stop eBPF components on test VMs.
foreach ($VM in $VMList) {
Stop-eBPFComponentsOnVM -VMName $VM.Name
}
# Clean up
Remove-Job -Job $Job -Force
Pop-Location

13
scripts/install_ebpf.psm1
Просмотреть файл

@ -7,7 +7,6 @@ param ([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
Push-Location $WorkingDirectory
Import-Module $PSScriptRoot\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
$VcRedistPath = Join-Path $WorkingDirectory "vc_redist.x64.exe"
$MsiPath = Join-Path $WorkingDirectory "ebpf-for-windows.msi"
# eBPF drivers and services.
@ -158,7 +157,6 @@ function Print-eBPFComponentsStatus([string] $message = "")
sc.exe query $_.Key 2>&1 | Write-Log
}
}
function Install-eBPFComponents
{
param([parameter(Mandatory=$true)] [bool] $KmTracing,
@ -170,17 +168,6 @@ function Install-eBPFComponents
# This is useful for detecting issues with the runner baselines.
Print-eBPFComponentsStatus "Querying the status of eBPF drivers and services before the installation (none should be present)..." | Out-Null
# Install the Visual C++ Redistributable (Release version, which is required for the MSI installation).
Write-Log("Installing Visual C++ Redistributable from '$VcRedistPath'...")
$process = Start-Process -FilePath $VcRedistPath -ArgumentList "/quiet", "/norestart" -Wait -PassThru
if ($process.ExitCode -ne 0) {
Write-Log("Visual C++ Redistributable installation FAILED. Exit code: $($process.ExitCode).") -ForegroundColor Red
throw ("Visual C++ Redistributable installation FAILED. Exit code: $($process.ExitCode).")
}
Write-Log("Cleaning up...")
Remove-Item $VcRedistPath -Force
Write-Log("Visual C++ Redistributable installation completed successfully!") -ForegroundColor Green
# Copy the VC debug runtime DLLs to the system32 directory,
# so that debug versions of the MSI can be installed (i.e., export_program_info.exe will not fail).
Write-Log("Copying VC debug runtime DLLs to the $system32Path directory...")

391
scripts/run_driver_tests.psm1
Просмотреть файл

@ -2,17 +2,224 @@
# SPDX-License-Identifier: MIT
param ([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
[Parameter(Mandatory=$True)] [string] $LogFileName)
[Parameter(Mandatory=$True)] [string] $LogFileName,
[parameter(Mandatory = $false)][int] $TestHangTimeout = (10*60),
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps")
Push-Location $WorkingDirectory
Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
Import-Module .\install_ebpf.psm1 -Force -ArgumentList ($WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue
$CodeCoverage = "$env:ProgramFiles\OpenCppCoverage\OpenCppCoverage.exe"
#
# Execute tests on VM.
# Utility functions.
#
# Finds and returns the specified tool's location under the current directory. If not found, throws an exception.
function GetToolLocationPath
{
param(
[Parameter(Mandatory = $True)] [string] $ToolName
)
$ToolLocationPath = Get-ChildItem -Path "$Pwd" `
-Recurse -Filter "$ToolName" -ErrorAction SilentlyContinue | Select-Object -First 1
if ($ToolLocationPath -eq $null) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** $ToolName not found under $Pwd."
}
return $ToolLocationPath.FullName
}
function GetDriveFreeSpaceGB
{
Param(
# Drive Specification in the form of a single alphabet string. (eg. "C:")
[Parameter(Mandatory = $True)] [string] $DriveSpecification
)
if (($DriveSpecification.Length -eq $Null) -or ($DriveSpecification.Length -ne 2)) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** No drive or Invalid drive specified."
}
# Convert drive to single letter (eg. "C:" to "C") for Get-Volume.
$DriveSpecification = $DriveSpecification -replace ".$"
$Volume = Get-Volume $DriveSpecification
if ($Volume -eq $Null) {
ThrowWithErrorMessage -ErrorMessage "*** ERROR *** Drive $DriveSpecification not found."
}
$FreeSpaceGB = (($Volume.SizeRemaining) / 1GB).ToString("F2")
return $FreeSpaceGB
}
function Generate-KernelDump
{
Push-Location $WorkingDirectory
$NotMyFaultBinary = "NotMyFault64.exe"
Write-Log "Verifying $NotMyFaultBinary presence in $Pwd..."
$NotMyFaultBinaryPath = GetToolLocationPath -ToolName $NotMyFaultBinary
Write-Log "$NotMyFaultBinary location: $NotMyFaultBinaryPath"
Write-Log "`n"
Write-Log "Creating kernel dump...`n"
# Wait a bit for the above message to show up in the log.
Start-Sleep -seconds 5
# This will/should not return (test system will/should bluescreen and reboot).
$NotMyFaultProc = Start-Process -NoNewWindow -Passthru -FilePath $NotMyFaultBinaryPath -ArgumentList "/crash"
# wait for 30 minutes to generate the kernel dump.
$NotMyFaultProc.WaitForExit(30*60*1000)
# If we get here, notmyfault64.exe failed for some reason. Kill the hung process, throw error.
ThrowWithErrorMessage `
-ErrorMessage "*** ERROR *** $($PSCommandPath): kernel mode dump creation FAILED"
}
function Generate-ProcessDump
{
param([Parameter(Mandatory = $true)] [int] $TestProcessId,
[Parameter(Mandatory = $true)] [string] $TestCommand,
[Parameter(Mandatory = $false)] [string] $UserModeDumpFolder = "C:\Dumps")
# Check if procdump64.exe and notmyfault64.exe are present on the system.
$ProcDumpBinary = "ProcDump64.exe"
Write-Log "Verifying $ProcDumpBinary presence in $Pwd..."
$ProcDumpBinaryPath = GetToolLocationPath -ToolName $ProcDumpBinary
Write-Log "$ProcDumpBinary location: $ProcDumpBinaryPath"
Write-Log "`n"
# Create dump folder if not present.
if (-not (Test-Path -Path $UserModeDumpFolder)) {
Write-Log "$UserModeDumpFolder created."
New-Item -Path $UserModeDumpFolder -ItemType Directory -Force -ErrorAction Stop
}
$UserModeDumpFileName = "$($TestCommand)_$(Get-Date -Format 'yyyy-MM-dd_HH-mm-ss').dmp"
$UserModeDumpFilePath = Join-Path $UserModeDumpFolder $UserModeDumpFileName
if ($VerbosePreference -eq 'Continue') {
Write-Log "User mode Dumpfile name: $UserModeDumpFileName"
Write-Log "User mode Dumpfile Path: $UserModeDumpFilePath"
}
# Get the available free space at this point in case the test creates its own files.
# (useful in investigating user and/or kernel dump file creation failures).
try {
$DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive
} catch {
Write-Log "Error getting available disk space: $_" -ForegroundColor Red
$DriveFreeSpaceGB = "Unknown"
# Continue with the test.
}
Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n"
Write-Log "Creating User mode dump @ $UserModeDumpFilePath"
$ProcDumpArguments = "-r -ma $($TestProcessId) $UserModeDumpFilePath"
Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments"
$ProcDumpProcess = Start-Process -NoNewWindow `
-FilePath $ProcDumpBinaryPath `
-ArgumentList $ProcDumpArguments `
-Wait -PassThru
Write-Log "Waiting for user mode dump to complete..."
$ProcDumpProcess.WaitForExit()
# Get procdump64.exe's exit code.
$ExitCode = $($ProcDumpProcess.ExitCode)
Write-Log "$ProcDumpBinaryPath completed with exit code: $ExitCode`n"
# Flush disk buffers to ensure user mode dump data is completely written to disk.
Write-Log "User mode dump completed. Flushing disk buffers..."
Write-VolumeCache -DriveLetter C
# Wait for a bit for things to settle down to ensure the user mode dump is completely flushed.
Start-Sleep -seconds 10
# Make sure a valid dump file is created.
$UserModeDumpSizeMB =
(((Get-ItemProperty -Path $UserModeDumpFilePath).Length) /1MB).ToString("F2")
if ($UserModeDumpSizeMB -eq 0) {
Write-Log "* WARNING * User mode dump $UserModeDumpFilePath NOT CREATED"
} else {
Write-Log "`n Created $UserModeDumpFilePath, size: $UserModeDumpSizeMB MB `n"
}
}
#
# Test Completion processing
#
function Process-TestCompletion
{
param([Parameter(Mandatory = $true)] [Object] $TestProcess,
[Parameter(Mandatory = $true)] [string] $TestCommand,
[Parameter(Mandatory = $false)] [bool] $NestedProcess,
[Parameter(Mandatory = $false)] [int] $TestHangTimeout = (10*60), # 10 minutes default timeout.
[Parameter(Mandatory = $false)] [bool] $NeedKernelDump = $true)
# Use Wait-Process for the process to terminate or timeout.
# See https://stackoverflow.com/a/23797762
Wait-Process -InputObject $TestProcess -Timeout $TestHangTimeout -ErrorAction SilentlyContinue
if (-not $TestProcess.HasExited) {
Write-Log "`n*** ERROR *** Test $TestCommand execution hang timeout ($TestHangTimeout seconds) expired.`n"
# Find the test process Id.
if ($NestedProcess) {
# The TestCommand is running nested inside another TestProcess.
$TestNameNoExt = [System.IO.Path]::GetFileNameWithoutExtension($TestCommand)
$TestProcessId = (Get-Process -Name $TestNameNoExt).Id
} else {
$TestProcessId = $TestProcess.Id
}
Write-Log "Potentially hung process PID:$TestProcessId running $TestCommand"
# Generate a user mode dump.
Generate-ProcessDump -TestProcessId $TestProcessId -TestCommand $TestCommand
# Next, kill the test process, if kernel dumps are not enabled.
if (-not $NeedKernelDump) {
Write-Log "Kernel dump not needed, killing process with PID:$TestProcessId..."
Stop-Process -Id $TestProcessId
ThrowWithErrorMessage -ErrorMessage "Test $TestCommand Hung!"
}
#finally, throw a new TestHung exception.
Write-Log "Throwing TestHungException for $TestCommand" -ForegroundColor Red
throw [System.TimeoutException]::new("Test $TestCommand execution hang timeout ($TestHangTimeout seconds) expired.")
} else {
# Ensure the process has completely exited.
Wait-Process -InputObject $TestProcess
# Read and display the output (if any) from the temporary output file.
$TempOutputFile = "$env:TEMP\app_output.log" # Log for standard output
# Process the log file line-by-line
if ((Test-Path $TempOutputFile) -and (Get-Item $TempOutputFile).Length -gt 0) {
Write-Log "$TestCommand Output:`n" -ForegroundColor Green
Get-Content -Path $TempOutputFile | ForEach-Object {
Write-Log -TraceMessage $_
}
Remove-Item -Path $TempOutputFile -Force -ErrorAction Ignore
}
$TestExitCode = $TestProcess.ExitCode
if ($TestExitCode -ne 0) {
$TempErrorFile = "$env:TEMP\app_error.log" # Log for standard error
if ((Test-Path $TempErrorFile) -and (Get-Item $TempErrorFile).Length -gt 0) {
Write-Log "$TestCommand Error Output:`n" -ForegroundColor Red
Get-Content -Path $TempErrorFile | ForEach-Object {
Write-Log -TraceMessage $_ -ForegroundColor Red
}
Remove-Item -Path $TempErrorFile -Force -ErrorAction Ignore
}
$ErrorMessage = "*** ERROR *** $TestCommand failed with $TestExitCode."
ThrowWithErrorMessage -ErrorMessage $ErrorMessage
}
}
}
#
# Execute tests.
#
function Invoke-NetshEbpfCommand
@ -48,94 +255,93 @@ function Invoke-NetshEbpfCommand
function Invoke-Test
{
param([Parameter(Mandatory = $True)][string] $TestName,
[Parameter(Mandatory = $False)][string] $TestArgs = "",
[Parameter(Mandatory = $False)][string] $InnerTestName = "",
[Parameter(Mandatory = $True)][bool] $VerboseLogs,
[Parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[Parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps",
[Parameter(Mandatory = $False)][bool] $Coverage)
[Parameter(Mandatory = $True)][int] $TestHangTimeout)
Write-Log "Preparing to run $Testname"
$LASTEXITCODE = 0
$OriginalTestName = $TestName
$ArgumentsList = @()
if ($Coverage) {
$ArgumentsList += @('-q', '--modules=C:\eBPF', '--export_type', ('binary:' + $TestName + '.cov'), '--', $TestName)
$TestName = $CodeCoverage
# Initialize arguments.
if ($TestArgs -ne "") {
$ArgumentsList = @($TestArgs)
}
# Execute Test.
if ($VerboseLogs -eq $true) {
$ArgumentsList += '-s'
}
$JoinedArgumentsList = $ArgumentsList -join " "
$TestRunScript = ".\Run-Self-Hosted-Runner-Test.ps1"
& $TestRunScript `
-TestCommand $TestName `
-TestArguments $JoinedArgumentsList `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-NeedKernelDump $True `
-Verbose
# Execute Test.
Write-Log "Executing $TestName $TestArgs"
$TestFilePath = "$pwd\$TestName"
$TempOutputFile = "$env:TEMP\app_output.log" # Log for standard output
$TempErrorFile = "$env:TEMP\app_error.log" # Log for standard error
if ($ArgumentsList) {
$TestProcess = Start-Process -FilePath $TestFilePath -ArgumentList $ArgumentsList -PassThru -NoNewWindow -RedirectStandardOutput $TempOutputFile -RedirectStandardError $TempErrorFile -ErrorAction Stop
} else {
$TestProcess = Start-Process -FilePath $TestFilePath -PassThru -NoNewWindow -RedirectStandardOutput $TempOutputFile -RedirectStandardError $TempErrorFile -ErrorAction Stop
}
if ($InnerTestName -ne "") {
Process-TestCompletion -TestProcess $TestProcess -TestCommand $InnerTestName -NestedProcess $True -TestHangTimeout $TestHangTimeout
} else {
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestName -TestHangTimeout $TestHangTimeout
}
Write-Log "$TestName Passed" -ForegroundColor Green
Write-Log "`n`n"
Write-Log "Test `"$TestName $TestArgs`" Passed" -ForegroundColor Green
Write-Log "`n==============================`n"
}
# Function to create a tuple with default values for Arguments and Timeout
function New-TestTuple {
param (
[string]$Test,
[string]$Arguments = "", # Default value: ""
[int]$Timeout = 300 # Default value: 5 minutes
)
# Return a custom object (tuple)
[pscustomobject]@{
Test = $Test
Arguments = $Arguments
Timeout = $Timeout
}
}
function Invoke-CICDTests
{
param([parameter(Mandatory = $true)][bool] $VerboseLogs,
[parameter(Mandatory = $false)][bool] $Coverage = $false,
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps",
[parameter(Mandatory = $true)][bool] $ExecuteSystemTests
)
[parameter(Mandatory = $true)][bool] $ExecuteSystemTests)
Push-Location $WorkingDirectory
$env:EBPF_ENABLE_WER_REPORT = "yes"
# Now create an array of test tuples, overriding only the necessary values
# load_native_program_invalid4 has been deleted from the test list, but 0.17 tests still have this test.
# That causes the regression test to fail. So, we are skipping this test for now.
$TestList = @(
"api_test.exe ~`"load_native_program_invalid4`"",
"bpftool_tests.exe",
"sample_ext_app.exe",
"socket_tests.exe")
$SystemTestList = @("api_test.exe")
$TestList = @(
(New-TestTuple -Test "api_test.exe" -Arguments "~`"load_native_program_invalid4`""),
(New-TestTuple -Test "bpftool_tests.exe"),
(New-TestTuple -Test "sample_ext_app.exe"),
(New-TestTuple -Test "socket_tests.exe" -Timeout 1800)
)
foreach ($Test in $TestList) {
Invoke-Test -TestName $Test -VerboseLogs $VerboseLogs -Coverage $Coverage
Invoke-Test -TestName $($Test.Test) -TestArgs $($Test.Arguments) -VerboseLogs $VerboseLogs -TestHangTimeout $($Test.Timeout)
}
# Now run the system tests. No coverage is needed for these tests.
# Now run the system tests.
$SystemTestList = @((New-TestTuple -Test "api_test.exe"))
if ($ExecuteSystemTests) {
foreach ($Test in $SystemTestList) {
$TestCommand = "PsExec64.exe -accepteula -nobanner -s -w `"$pwd`" `"$pwd\$Test`" `"-d yes`""
Invoke-Test -TestName $TestCommand -VerboseLogs $VerboseLogs -Coverage $false
$TestCommand = "PsExec64.exe"
$TestArguments = "-accepteula -nobanner -s -w `"$pwd`" `"$pwd\$($Test.Test) $($Test.Arguments)`" `"-d yes`""
Invoke-Test -TestName $TestCommand -TestArgs $TestArguments -InnerTestName $($Test.Test) -VerboseLogs $VerboseLogs -TestHangTimeout $($Test.Timeout)
}
}
if ($Coverage) {
# Combine code coverage reports
$ArgumentsList += @()
foreach ($Test in $TestList) {
$ArgumentsList += @('--input_coverage', ($Test + '.cov'))
}
$ArgumentsList += @('--export_type', 'cobertura:c:\eBPF\ebpf_for_windows.xml', '--')
$JoinedArgumentsList = $ArgumentsList -join " "
$TestRunScript = ".\Run-Self-Hosted-Runner-Test.ps1"
& $TestRunScript `
-TestCommand $CodeCoverage `
-TestArguments $JoinedArgumentsList `
-TestHangTimeout = $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
}
if ($Env:BUILD_CONFIGURATION -eq "Release") {
Invoke-Test -TestName "ebpf_performance.exe" -VerboseLogs $VerboseLogs
}
@ -148,10 +354,7 @@ function Invoke-XDPTest
param([parameter(Mandatory = $true)][string] $RemoteIPV4Address,
[parameter(Mandatory = $true)][string] $RemoteIPV6Address,
[parameter(Mandatory = $true)][string] $XDPTestName,
[parameter(Mandatory = $true)][string] $WorkingDirectory,
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps"
)
[parameter(Mandatory = $true)][string] $WorkingDirectory)
Push-Location $WorkingDirectory
@ -159,26 +362,15 @@ function Invoke-XDPTest
$TestRunScript = ".\Run-Self-Hosted-Runner-Test.ps1"
$TestCommand = ".\xdp_tests.exe"
$TestArguments = "$XDPTestName --remote-ip $RemoteIPV4Address"
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
Start-Sleep -seconds 5
Write-Log "Executing $XDPTestName with remote address: $RemoteIPV6Address"
$TestRunScript = ".\Run-Self-Hosted-Runner-Test.ps1"
$TestCommand = ".\xdp_tests.exe"
$TestArguments = "$XDPTestName --remote-ip $RemoteIPV6Address"
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
Write-Log "$XDPTestName Test Passed" -ForegroundColor Green
Write-Log "`n`n"
@ -199,9 +391,7 @@ function Invoke-ConnectRedirectTest
[parameter(Mandatory = $true)][string] $StandardUserName,
[parameter(Mandatory = $true)][string] $StandardUserPassword,
[parameter(Mandatory = $true)][string] $UserType,
[parameter(Mandatory = $true)][string] $WorkingDirectory,
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps")
[parameter(Mandatory = $true)][string] $WorkingDirectory)
Push-Location $WorkingDirectory
@ -223,13 +413,9 @@ function Invoke-ConnectRedirectTest
" --user-type $UserType"
Write-Log "Executing connect redirect tests with v4 and v6 programs. Arguments: $TestArguments"
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
## Run test with only v4 program attached.
$TestArguments =
@ -244,13 +430,9 @@ function Invoke-ConnectRedirectTest
" [connect_authorize_redirect_tests_v4]"
Write-Log "Executing connect redirect tests with v4 programs. Arguments: $TestArguments"
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
## Run tests with only v6 program attached.
$TestArguments =
@ -265,13 +447,9 @@ function Invoke-ConnectRedirectTest
" [connect_authorize_redirect_tests_v6]"
Write-Log "Executing connect redirect tests with v6 programs. Arguments: $TestArguments"
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-Verbose
Write-Log "Connect-Redirect Test Passed" -ForegroundColor Green
@ -281,7 +459,6 @@ function Invoke-ConnectRedirectTest
function Invoke-CICDStressTests
{
param([parameter(Mandatory = $true)][bool] $VerboseLogs,
[parameter(Mandatory = $false)][bool] $Coverage = $false,
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps",
[parameter(Mandatory = $false)][bool] $NeedKernelDump = $true,
@ -301,15 +478,9 @@ function Invoke-CICDStressTests
} else {
$TestArguments = "-tt=8 -td=5 -erd=1000 -er=1"
}
$TestProcess = Start-Process -FilePath $TestCommand -ArgumentList $TestArguments -PassThru -NoNewWindow
Process-TestCompletion -TestProcess $TestProcess -TestCommand $TestCommand
$TestRunScript = ".\Run-Self-Hosted-Runner-Test.ps1"
& $TestRunScript `
-TestCommand $TestCommand `
-TestArguments $TestArguments `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-NeedKernelDump $True `
-Verbose
Pop-Location
}

7
scripts/setup_build/setup_build.vcxproj
Просмотреть файл

@ -186,13 +186,6 @@ popd
<DeploymentContent Condition="'$(Configuration)'=='Release'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)'=='NativeOnlyRelease'">true</DeploymentContent>
</CopyFileToFolders>
<CopyFileToFolders Include="..\Run-Self-Hosted-Runner-Test.ps1">
<DeploymentContent Condition="'$(Configuration)'=='Debug'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)'=='FuzzerDebug'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)'=='NativeOnlyDebug'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)'=='Release'">true</DeploymentContent>
<DeploymentContent Condition="'$(Configuration)'=='NativeOnlyRelease'">true</DeploymentContent>
</CopyFileToFolders>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>16.0</VCProjectVersion>

3
scripts/setup_ebpf_cicd_tests.ps1
Просмотреть файл

@ -47,8 +47,7 @@ if ($TestMode -eq "CI/CD" -or $TestMode -eq "Regression") {
Get-LegacyRegressionTestArtifacts
}
Get-Duonic
Get-VCRedistributable
Get-CoreNetTools
Get-PSExec
# Export build artifacts to the test VMs.

306
scripts/vm_run_tests.psm1
Просмотреть файл

@ -7,25 +7,49 @@ param ([Parameter(Mandatory = $True)] [string] $Admin,
[Parameter(Mandatory = $True)] [SecureString] $StandardUserPassword,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $false)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $false)][string] $TestMode = "CI/CD",
[Parameter(Mandatory = $false)][string[]] $Options = @("None"),
[Parameter(Mandatory = $false)][int] $TestHangTimeout = (10*60),
[Parameter(Mandatory = $false)][string] $UserModeDumpFolder = "C:\Dumps"
)
Push-Location $WorkingDirectory
Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
#
# Generate kernel dump.
#
function Generate-KernelDumpOnVM
{
param([Parameter(Mandatory = $true)] [string] $VMName,
[Parameter(Mandatory = $False)] [bool] $VerboseLogs)
Write-Log "Generating kernel dump on $VMName."
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[Parameter(Mandatory = $True)] [bool] $VerboseLogs)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
Generate-KernelDump
} -ArgumentList("eBPF", $LogFileName, $VerboseLogs) -ErrorAction Stop
}
#
# Execute tests on VM.
#
function Invoke-CICDTestsOnVM
{
param([parameter(Mandatory = $True)] [string] $VMName,
[parameter(Mandatory = $False)] [bool] $VerboseLogs = $false,
[parameter(Mandatory = $False)] [bool] $Coverage = $false,
[parameter(Mandatory = $False)][string] $TestMode = "CI/CD",
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps",
[parameter(Mandatory = $False)][string[]] $Options = @())
param([Parameter(Mandatory = $True)] [string] $VMName,
[Parameter(Mandatory = $False)] [bool] $VerboseLogs = $false,
[Parameter(Mandatory = $False)][string] $TestMode = "CI/CD",
[Parameter(Mandatory = $False)][string[]] $Options = @())
Write-Log "Running eBPF $TestMode tests on $VMName"
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -34,15 +58,14 @@ function Invoke-CICDTestsOnVM
param([Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[Parameter(Mandatory = $True)] [bool] $VerboseLogs,
[Parameter(Mandatory = $True)] [bool] $Coverage,
[parameter(Mandatory = $True)][string] $TestMode,
[parameter(Mandatory = $true)][int] $TestHangTimeout,
[parameter(Mandatory = $true)][string] $UserModeDumpFolder,
[parameter(Mandatory = $True)][string[]] $Options)
[Parameter(Mandatory = $True)][string] $TestMode,
[Parameter(Mandatory = $true)][int] $TestHangTimeout,
[Parameter(Mandatory = $true)][string] $UserModeDumpFolder,
[Parameter(Mandatory = $True)][string[]] $Options)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName, $TestHangTimeout, $UserModeDumpFolder) -Force -WarningAction SilentlyContinue
$TestMode = $TestMode.ToLower()
switch ($TestMode)
@ -50,18 +73,12 @@ function Invoke-CICDTestsOnVM
"ci/cd" {
Invoke-CICDTests `
-VerboseLogs $VerboseLogs `
-Coverage $Coverage `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-ExecuteSystemTests $true `
2>&1 | Write-Log
}
"regression" {
Invoke-CICDTests `
-VerboseLogs $VerboseLogs `
-Coverage $Coverage `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
-ExecuteSystemTests $false `
2>&1 | Write-Log
}
@ -70,10 +87,7 @@ function Invoke-CICDTestsOnVM
$RestartExtension = $Options -contains "RestartExtension"
Invoke-CICDStressTests `
-VerboseLogs $VerboseLogs `
-Coverage $Coverage `
-RestartExtension $RestartExtension `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder `
2>&1 | Write-Log
}
"performance" {
@ -90,7 +104,6 @@ function Invoke-CICDTestsOnVM
"eBPF",
$LogFileName,
$VerboseLogs,
$Coverage,
$TestMode,
$TestHangTimeout,
$UserModeDumpFolder,
@ -99,9 +112,9 @@ function Invoke-CICDTestsOnVM
function Add-eBPFProgramOnVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] [string] $Program,
[string] $Interface,
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] [string] $Program,
[Parameter(Mandatory = $false)] [string] $Interface = "",
[Parameter(Mandatory = $True)] [string] $LogFileName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -109,8 +122,8 @@ function Add-eBPFProgramOnVM
# Load program on VM.
$ProgId = Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $true)] [string] $Program,
[string] $Interface,
[Parameter(Mandatory = $true)] [string] $Program,
[Parameter(Mandatory = $false)] [string] $Interface,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
@ -133,9 +146,9 @@ function Add-eBPFProgramOnVM
function Set-eBPFProgramOnVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] $ProgId,
[parameter(Mandatory = $true)] [string] $Interface,
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] $ProgId,
[Parameter(Mandatory = $true)] [string] $Interface,
[Parameter(Mandatory = $True)] [string] $LogFileName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -143,8 +156,8 @@ function Set-eBPFProgramOnVM
# Set program on VM.
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $true)] $ProgId,
[parameter(Mandatory = $true)] [string] $Interface,
[Parameter(Mandatory = $true)] $ProgId,
[Parameter(Mandatory = $true)] [string] $Interface,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
@ -157,8 +170,8 @@ function Set-eBPFProgramOnVM
}
function Remove-eBPFProgramFromVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] $ProgId,
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] $ProgId,
[Parameter(Mandatory = $True)] [string] $LogFileName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -181,8 +194,8 @@ function Remove-eBPFProgramFromVM
function Start-ProcessOnVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] [string] $ProgramName,
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] [string] $ProgramName,
[string] $Parameters)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -190,7 +203,7 @@ function Start-ProcessOnVM
# Start process on VM.
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory= $True)] [string] $VM,
[parameter(Mandatory= $true)] [string] $ProgramName,
[Parameter(Mandatory= $true)] [string] $ProgramName,
[string] $Parameters,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory)
@ -203,15 +216,15 @@ function Start-ProcessOnVM
function Stop-ProcessOnVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] [string] $ProgramName)
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] [string] $ProgramName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
# Stop process on VM.
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $true)] [string] $ProgramName)
[Parameter(Mandatory = $true)] [string] $ProgramName)
$ProgramName = [io.path]::GetFileNameWithoutExtension($ProgramName)
Stop-Process -Name $ProgramName
@ -220,16 +233,16 @@ function Stop-ProcessOnVM
function Add-StandardUserOnVM
{
param ([parameter(Mandatory = $true)] [string] $VM,
[parameter(Mandatory = $true)] [string] $UserName,
[parameter(Mandatory = $true)] [string] $Password)
param ([Parameter(Mandatory = $true)] [string] $VM,
[Parameter(Mandatory = $true)] [string] $UserName,
[Parameter(Mandatory = $true)] [string] $Password)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
# Create standard user.
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([parameter(Mandatory = $true)] [string] $UserName,
[parameter(Mandatory = $true)] [string] $Password)
param([Parameter(Mandatory = $true)] [string] $UserName,
[Parameter(Mandatory = $true)] [string] $Password)
$SecurePassword = ConvertTo-SecureString -String $Password -AsPlainText -Force
New-LocalUser -Name $UserName -Password $SecurePassword
@ -238,13 +251,13 @@ function Add-StandardUserOnVM
function Remove-StandardUserOnVM
{
param ([parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $True)] [string] $UserName)
param ([Parameter(Mandatory = $True)] [string] $VM,
[Parameter(Mandatory = $True)] [string] $UserName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([parameter(Mandatory = $true)] [string] $UserName)
param([Parameter(Mandatory = $true)] [string] $UserName)
Remove-LocalUser -Name $UserName
} -ArgumentList ($UserName, $Password) -ErrorAction Stop
@ -252,40 +265,34 @@ function Remove-StandardUserOnVM
function Invoke-XDPTestOnVM
{
param ([parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $True)] [string] $XDPTestName,
param ([Parameter(Mandatory = $True)] [string] $VM,
[Parameter(Mandatory = $True)] [string] $XDPTestName,
[Parameter(Mandatory = $True)] [string] $RemoteIPV4Address,
[Parameter(Mandatory = $True)] [string] $RemoteIPV6Address,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $True)] [string] $LogFileName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
Invoke-Command -VMName $VM -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $true)] [string] $XDPTestName,
[Parameter(Mandatory = $true)] [string] $XDPTestName,
[Parameter(Mandatory = $True)] [string] $RemoteIPV4Address,
[Parameter(Mandatory = $True)] [string] $RemoteIPV6Address,
[Parameter(Mandatory = $True)] [string] $WorkingDirectory,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $True)][int] $TestHangTimeout,
[parameter(Mandatory = $True)][string] $UserModeDumpFolder)
[Parameter(Mandatory = $True)] [int] $TestHangTimeout,
[Parameter(Mandatory = $True)] [string] $UserModeDumpFolder)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName, $TestHangTimeout, $UserModeDumpFolder) -Force -WarningAction SilentlyContinue
Write-Log "Invoking $XDPTestName on $VM"
Invoke-XDPTest `
-RemoteIPV4Address $RemoteIPV4Address `
-RemoteIPV6Address $RemoteIPV6Address `
-XDPTestName $XDPTestName `
-WorkingDirectory $WorkingDirectory `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-WorkingDirectory $WorkingDirectory
} -ArgumentList (
$VM,
$XDPTestName,
@ -298,9 +305,9 @@ function Invoke-XDPTestOnVM
}
function Add-FirewallRuleOnVM {
param ([parameter(Mandatory = $True)] [string] $VM,
[parameter(Mandatory = $True)] [string] $ProgramName,
[parameter(Mandatory = $True)] [string] $RuleName,
param ([Parameter(Mandatory = $True)] [string] $VM,
[Parameter(Mandatory = $True)] [string] $ProgramName,
[Parameter(Mandatory = $True)] [string] $RuleName,
[Parameter(Mandatory = $True)] [string] $LogFileName)
$TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
@ -327,9 +334,7 @@ function Invoke-XDPTest1
[Parameter(Mandatory = $True)] [string] $VM1Interface1V6Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V4Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V6Address,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $True)] [string] $LogFileName)
Write-Log "Running XDP Test1 on $VM ..."
@ -342,18 +347,13 @@ function Invoke-XDPTest1
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface1V4Address `
-RemoteIPV6Address $VM1Interface1V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
Invoke-XDPTestOnVM `
-VM $VM `
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface2V4Address `
-RemoteIPV6Address $VM1Interface2V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
# Unload program from VM.
Remove-eBPFProgramFromVM $VM $ProgId $LogFileName
@ -370,9 +370,7 @@ function Invoke-XDPTest2
[Parameter(Mandatory = $True)] [string] $VM1Interface1V6Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V4Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V6Address,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $True)] [string] $LogFileName)
Write-Log "Running XDP Test2 ..."
@ -388,18 +386,14 @@ function Invoke-XDPTest2
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface1V4Address `
-RemoteIPV6Address $VM1Interface1V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
Invoke-XDPTestOnVM `
-VM $VM `
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface2V4Address `
-RemoteIPV6Address $VM1Interface2V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
# Unload program from VM1.
Remove-eBPFProgramFromVM $VM $ProgId $LogFileName
@ -416,9 +410,7 @@ function Invoke-XDPTest3
[Parameter(Mandatory = $True)] [string] $VM1Interface1V6Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V4Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface2V6Address,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $True)] [string] $LogFileName)
Write-Log "Running XDP Test3 ..."
@ -435,9 +427,7 @@ function Invoke-XDPTest3
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface1V4Address `
-RemoteIPV6Address $VM1Interface1V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
# Run XDP encap reflect test from VM2 targeting second interface of VM1.
Invoke-XDPTestOnVM `
@ -445,9 +435,7 @@ function Invoke-XDPTest3
-XDPTestName "xdp_encap_reflect_test" `
-RemoteIPV4Address $VM1Interface2V4Address `
-RemoteIPV6Address $VM1Interface2V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
# Unload programs from VM1.
Remove-eBPFProgramFromVM $VM $ProgId1 $LogFileName
@ -463,9 +451,7 @@ function Invoke-XDPTest4
[Parameter(Mandatory = $True)] [string] $VM1Interface1V6Address,
[Parameter(Mandatory = $True)] [string] $VM1Interface1Alias,
[Parameter(Mandatory = $True)] [string] $VM2Interface1Alias,
[Parameter(Mandatory = $True)] [string] $LogFileName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
[Parameter(Mandatory = $True)] [string] $LogFileName)
Write-Log "Running XDP Test4 ..."
@ -481,9 +467,7 @@ function Invoke-XDPTest4
-XDPTestName "xdp_reflect_test" `
-RemoteIPV4Address $VM1Interface1V4Address `
-RemoteIPV6Address $VM1Interface1V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
# Unload program from VM1.
Remove-eBPFProgramFromVM $VM $ProgId1 $LogFileName
@ -495,10 +479,8 @@ function Invoke-XDPTest4
function Invoke-XDPTestsOnVM
{
param([parameter(Mandatory = $True)] $Interfaces,
[parameter(Mandatory = $True)] [string] $VMName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
param([Parameter(Mandatory = $True)] $Interfaces,
[Parameter(Mandatory = $True)] [string] $VMName)
# NIC pairs are duo1-duo2 and duo3-duo4.
# VM1 is interfaces duo1 and duo3.
@ -527,9 +509,7 @@ function Invoke-XDPTestsOnVM
-VM1Interface1V6Address $VM1Interface1V6Address `
-VM1Interface2V4Address $VM1Interface2V4Address `
-VM1Interface2V6Address $VM1Interface3V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
Invoke-XDPTest2 `
-VM $VMName `
@ -539,9 +519,7 @@ function Invoke-XDPTestsOnVM
-VM1Interface1V6Address $VM1Interface1V6Address `
-VM1Interface2V4Address $VM1Interface2V4Address `
-VM1Interface2V6Address $VM1Interface3V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
Invoke-XDPTest3 `
-VM $VMName `
@ -551,9 +529,7 @@ function Invoke-XDPTestsOnVM
-VM1Interface1V6Address $VM1Interface1V6Address `
-VM1Interface2V4Address $VM1Interface2V4Address `
-VM1Interface2V6Address $VM1Interface3V6Address `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
Invoke-XDPTest4 `
-VM $VMName `
@ -561,19 +537,15 @@ function Invoke-XDPTestsOnVM
-VM1Interface1V6Address $VM1Interface1V6Address `
-VM1Interface1Alias $VM1Interface1Alias `
-VM2Interface1Alias $VM2Interface1Alias `
-LogFileName $LogFileName `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-LogFileName $LogFileName
}
function Invoke-ConnectRedirectTestsOnVM
{
param([parameter(Mandatory = $true)] $Interfaces,
[parameter(Mandatory = $true)] $ConnectRedirectTestConfig,
[parameter(Mandatory = $true)][ValidateSet("Administrator", "StandardUser")] $UserType = "Administrator",
[parameter(Mandatory = $true)] [string] $VMName,
[parameter(Mandatory = $False)][int] $TestHangTimeout = 3600,
[parameter(Mandatory = $False)][string] $UserModeDumpFolder = "C:\Dumps")
param([Parameter(Mandatory = $true)] $Interfaces,
[Parameter(Mandatory = $true)] $ConnectRedirectTestConfig,
[Parameter(Mandatory = $true)][ValidateSet("Administrator", "StandardUser")] $UserType = "Administrator",
[Parameter(Mandatory = $true)] [string] $VMName)
$VM1Interface = $Interfaces[0]
$VM1V4Address = $VM1Interface.V4Address
@ -620,7 +592,7 @@ function Invoke-ConnectRedirectTestsOnVM
-ScriptBlock {param ($StandardUser) Get-LocalUser -Name "$StandardUser"} `
-Argumentlist $StandardUser -ErrorAction SilentlyContinue
if($UserId) {
Write-Host "Deleting existing standard user:" $StandardUser "on" $VMName
Write-Log "Deleting existing standard user: $StandardUser on $VMName"
Remove-StandardUserOnVM -VM $VMName -UserName $StandardUser
}
@ -629,25 +601,26 @@ function Invoke-ConnectRedirectTestsOnVM
Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
param([Parameter(Mandatory = $True)][string] $VM,
[parameter(Mandatory = $true)][string] $LocalIPv4Address,
[parameter(Mandatory = $true)][string] $LocalIPv6Address,
[parameter(Mandatory = $true)][string] $RemoteIPv4Address,
[parameter(Mandatory = $true)][string] $RemoteIPv6Address,
[parameter(Mandatory = $true)][string] $VirtualIPv4Address,
[parameter(Mandatory = $true)][string] $VirtualIPv6Address,
[parameter(Mandatory = $true)][int] $DestinationPort,
[parameter(Mandatory = $true)][int] $ProxyPort,
[parameter(Mandatory = $true)][string] $StandardUserName,
[parameter(Mandatory = $true)][string] $StandardUserPassword,
[parameter(Mandatory = $true)][string] $UserType,
[parameter(Mandatory = $true)][string] $WorkingDirectory,
[Parameter(Mandatory = $true)][string] $LocalIPv4Address,
[Parameter(Mandatory = $true)][string] $LocalIPv6Address,
[Parameter(Mandatory = $true)][string] $RemoteIPv4Address,
[Parameter(Mandatory = $true)][string] $RemoteIPv6Address,
[Parameter(Mandatory = $true)][string] $VirtualIPv4Address,
[Parameter(Mandatory = $true)][string] $VirtualIPv6Address,
[Parameter(Mandatory = $true)][int] $DestinationPort,
[Parameter(Mandatory = $true)][int] $ProxyPort,
[Parameter(Mandatory = $true)][string] $StandardUserName,
[Parameter(Mandatory = $true)][string] $StandardUserPassword,
[Parameter(Mandatory = $true)][string] $UserType,
[Parameter(Mandatory = $true)][string] $WorkingDirectory,
[Parameter(Mandatory = $true)][string] $LogFileName,
[parameter(Mandatory = $false)][int] $TestHangTimeout,
[parameter(Mandatory = $false)][string] $UserModeDumpFolder)
[Parameter(Mandatory = $false)][int] $TestHangTimeout,
[Parameter(Mandatory = $false)][string] $UserModeDumpFolder)
$WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName, $TestHangTimeout, $UserModeDumpFolder) -Force -WarningAction SilentlyContinue
Write-Log "Invoking connect redirect tests [Mode=$UserType] on $VM"
Invoke-ConnectRedirectTest `
@ -662,9 +635,7 @@ function Invoke-ConnectRedirectTestsOnVM
-StandardUserName $StandardUserName `
-StandardUserPassword $StandardUserPassword `
-UserType $UserType `
-WorkingDirectory $WorkingDirectory `
-TestHangTimeout $TestHangTimeout `
-UserModeDumpFolder $UserModeDumpFolder
-WorkingDirectory $WorkingDirectory
Write-Log "Invoke-ConnectRedirectTest finished on $VM"
} -ArgumentList (
@ -692,8 +663,8 @@ function Invoke-ConnectRedirectTestsOnVM
function Stop-eBPFComponentsOnVM
{
param([parameter(Mandatory = $true)] [string] $VMName,
[parameter(Mandatory = $false)] [bool] $VerboseLogs = $false)
param([Parameter(Mandatory = $true)] [string] $VMName,
[Parameter(Mandatory = $false)] [bool] $VerboseLogs = $false)
# Stop the components, so that Driver Verifier can catch memory leaks etc.
Write-Log "Stopping eBPF components on $VMName"
@ -710,4 +681,49 @@ function Stop-eBPFComponentsOnVM
} -ArgumentList ("eBPF", $LogFileName) -ErrorAction Stop
}
function Run-KernelTestsOnVM
{
param([Parameter(Mandatory = $true)] [string] $VMName,
[Parameter(Mandatory = $true)] [PSCustomObject] $Config)
try {
# Run CICD tests on test VM.
Invoke-CICDTestsOnVM `
-VMName $VMName `
-TestMode $TestMode `
-Options $Options
# The required behavior is selected by the $TestMode
# parameter.
if (($TestMode -eq "CI/CD") -or ($TestMode -eq "Regression")) {
# Run XDP Tests.
Invoke-XDPTestsOnVM `
-Interfaces $Config.Interfaces `
-VMName $VMName
# Run Connect Redirect Tests.
Invoke-ConnectRedirectTestsOnVM `
-Interfaces $Config.Interfaces `
-ConnectRedirectTestConfig $Config.ConnectRedirectTest `
-UserType "Administrator" `
-VMName $VMName
Invoke-ConnectRedirectTestsOnVM `
-Interfaces $Config.Interfaces `
-ConnectRedirectTestConfig $Config.ConnectRedirectTest `
-UserType "StandardUser" `
-VMName $VMName
}
} catch [System.Management.Automation.RemoteException] {
# Next, generate kernel dump.
Write-Log $_.Exception.Message
Write-Log $_.ScriptStackTrace
if ($_.CategoryInfo.Reason -eq "TimeoutException") {
Generate-KernelDumpOnVM($VMName)
}
}
}
Pop-Location