1370 строки
87 KiB
PowerShell
1370 строки
87 KiB
PowerShell
using module .\helpers\prerequisites.psm1
|
|
using module .\helpers\internal.psm1
|
|
using module .\helpers\icmp.psm1
|
|
using module .\helpers\tcp.psm1
|
|
using module .\helpers\ndk.psm1
|
|
|
|
Function Test-NetStack {
|
|
<#
|
|
.SYNOPSIS
|
|
Test-NetStack performs ICMP, TCP, and RDMA traffic testing of networks. Test-NetStack can help you identify misconfigured networks, hotspots, asymmetry across cluster nodes, and more.
|
|
|
|
.DESCRIPTION
|
|
Test-NetStack performs ICMP, TCP, and RDMA traffic testing of networks. Test-NetStack can help you identify misconfigured networks, hotspots, asymmetry across cluster nodes, and more.
|
|
Specifically, Test-NetStack:
|
|
- Performs connectivity mapping across a cluster, specific nodes, or IP targets
|
|
- Stage1: ICMP Connectivity, Reliability, and PMTUD
|
|
- Stage2: TCP Stress 1:1
|
|
- Stage3: RDMA Connectivity
|
|
- Stage4: RDMA Stress 1:1
|
|
- Stage5: RDMA Stress N:1
|
|
- Stage6: RDMA Stress N:N
|
|
- Stage7: *EXPERIMENTAL* RDMA Stress N:1 Parallel
|
|
|
|
.PARAMETER Nodes
|
|
- Specifies the machines by DNS Name to test.
|
|
- PowerShell remoting without credentials is required; no credentials are stored or entered into Test-NetStack.
|
|
- Minimum 2 nodes, maximum of 16 nodes.
|
|
|
|
If part of a failover cluster, and neither the IPTarget or Node parameters are specified, get-clusternode will be run to attempt to gather nodenames.
|
|
|
|
.PARAMETER Stage
|
|
List of stages that specifies the tests to be run by Test-NetStack. By default, all stages will be run.
|
|
|
|
Tests will always occur in order of lowest stage first. It is highly recommended that you always run the preceeding tests.
|
|
|
|
Currently included stages for Test-NetStack:
|
|
- Stage1: ICMP Connectivity, Reliability, and PMTUD
|
|
- Stage2: TCP Stress 1:1
|
|
- Stage3: RDMA Connectivity
|
|
- Stage4: RDMA Stress 1:1
|
|
- Stage5: RDMA Stress N:1
|
|
- Stage6: RDMA Stress N:N
|
|
|
|
.PARAMETER EnableFirewallRules
|
|
* This command is best-effort and may fail for a variety of reasons *
|
|
|
|
Works with:
|
|
- The Windows Firewall
|
|
- The built-in firewall rules for ICMP, WinRM, and iWARP
|
|
- NTTTCP rules (by application path) needed for Stage 2
|
|
|
|
Note: if you upgrade the module version, firewall rules should be revoked, then re-enabled.
|
|
|
|
.PARAMETER RevokeFirewallRules
|
|
* This command is best-effort and may fail for a variety of reasons *
|
|
|
|
Works with:
|
|
- The Windows Firewall
|
|
- The built-in firewall rules for ICMP and iWARP
|
|
- NTTTCP rules defined by the EnableFirewallRules parameter
|
|
- Will not disable WinRM
|
|
|
|
.PARAMETER OnlyPrerequisites
|
|
Use if you want to review the connectivity map detection. This is useful if you're troubleshooting why some networks are or are not included in testing.
|
|
|
|
.PARAMETER OnlyConnectivityMap
|
|
Use if you want to review the connectivity map detection. This is useful if you're troubleshooting why some networks are or are not included in testing.
|
|
|
|
.PARAMETER LogPath
|
|
Defines the path for the logfile. By default, this will be in the path of the module under the results folder
|
|
|
|
.PARAMETER ContinueOnFailure
|
|
By default, Test-NetStack will stop processing later stages if a failure is incurred during an earlier stage. This switch will continue testing later stages.
|
|
|
|
The following lists the dependent stages:
|
|
- Stage 1 -> Stage 2
|
|
- Stage 3 -> Stage 4 -> Stage 5 -> Stage 6
|
|
|
|
.EXAMPLE
|
|
Run all tests in the local node's failover cluster. Review results from Stage2 and Stage6
|
|
$Results = Test-NetStack
|
|
|
|
$Results.Stage2
|
|
$Results.Stage6
|
|
|
|
.EXAMPLE
|
|
4-domain joined nodes; all tests run
|
|
$Results = Test-NetStack -Nodes 'AzStackHCI01', 'AzStackHCI02', 'AzStackHCI03', AzStackHCI04'
|
|
|
|
.EXAMPLE
|
|
2-node tests; ICMP and TCP tests only. Review results from Stage1 and Stage2
|
|
$Results = Test-NetStack -MachineList 'AzStackHCI01', 'AzStackHCI02' -Stage 1, 2
|
|
|
|
$Results.Stage1
|
|
$Results.Stage2
|
|
|
|
.NOTES
|
|
Author: Windows Core Networking team @ Microsoft
|
|
Please file issues on GitHub @ GitHub.com/Microsoft/Test-NetStack
|
|
|
|
.LINK
|
|
Networking Blog : https://aka.ms/MSFTNetworkBlog
|
|
HCI Host Guidance : https://docs.microsoft.com/en-us/azure-stack/hci/deploy/network-atc
|
|
#>
|
|
|
|
[CmdletBinding(DefaultParameterSetName = 'FullNodeMap')]
|
|
param (
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap' , position = 0)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes' , position = 0)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyConMapNodes' , position = 0)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes', position = 0)]
|
|
[ValidateScript({[System.Uri]::CheckHostName($_) -eq 'DNS'})]
|
|
[ValidateCount(2, 16)]
|
|
[String[]] $Nodes,
|
|
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'IPAddress' , position = 0)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqIPTarget' , position = 0)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapIPTarget' , position = 0)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesIPTarget', position = 0)]
|
|
[ValidateCount(2, 16)]
|
|
[IPAddress[]] $IPTarget,
|
|
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap' , position = 1)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'IPAddress' , position = 1)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes' , position = 1)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqIPTarget' , position = 1)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes' , position = 1)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesIPTarget', position = 1)]
|
|
[ValidateSet('1', '2', '3', '4', '5', '6', '7', '8')]
|
|
[Int32[]] $Stage = @('1', '2', '3', '4', '5', '6'),
|
|
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap', position = 2)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'IPAddress' , position = 2)]
|
|
[Switch] $EnableFirewallRules = $false,
|
|
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesNodes' , position = 2)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'RevokeFWRulesIPTarget', position = 2)]
|
|
[Switch] $RevokeFirewallRules = $false,
|
|
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqNodes' , position = 2)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyPrereqIPTarget', position = 2)]
|
|
[Switch] $OnlyPrerequisites = $false,
|
|
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapNodes' , position = 2)]
|
|
[Parameter(Mandatory = $true, ParameterSetName = 'OnlyConMapIPTarget', position = 2)]
|
|
[Switch] $OnlyConnectivityMap = $false,
|
|
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap', position = 3)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'IPAddress' , position = 3)]
|
|
[Parameter(Mandatory = $false)]
|
|
[switch] $ContinueOnFailure = $false,
|
|
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'FullNodeMap' , position = 4)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'IPAddress' , position = 4)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqNodes' , position = 4)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'OnlyPrereqIPTarget' , position = 4)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesNodes' , position = 4)]
|
|
[Parameter(Mandatory = $false, ParameterSetName = 'RevokeFWRulesIPTarget', position = 4)]
|
|
[Switch] $Experimental = $false,
|
|
|
|
[Parameter(Mandatory = $false)]
|
|
[String] $LogPath = "$(Join-Path -Path $((Get-Module -Name Test-Netstack -ListAvailable | Select-Object -First 1).ModuleBase) -ChildPath "Results\NetStackResults-$(Get-Date -f yyyy-MM-dd-HHmmss).txt")"
|
|
)
|
|
|
|
$LogFileParentPath = Split-Path -Path $LogPath -Parent -ErrorAction SilentlyContinue
|
|
|
|
if (-not (Test-Path $LogFileParentPath -ErrorAction SilentlyContinue)) {
|
|
$null = New-Item -Path $LogFileParentPath -ItemType Directory -Force -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
$LogFile = New-Item -Path $LogPath -ItemType File -Force -ErrorAction SilentlyContinue
|
|
|
|
$ExperimentalStages = @('7', '8')
|
|
$ChosenStages = $Stages | Where-Object {$ExperimentalStages -contains $_}
|
|
if ($Experimental -eq $false -and $ChosenStages.Length -gt 0) {
|
|
Write-Error "The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages."
|
|
"The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
break
|
|
}
|
|
|
|
$Global:ProgressPreference = 'SilentlyContinue'
|
|
|
|
Write-LogMessage -Message "Starting Test-NetStack" -LogFile $LogFile
|
|
|
|
# Each stages adds their results to this and is eventually returned by this function
|
|
$NetStackResults = New-Object -TypeName psobject
|
|
|
|
# Since FullNodeMap is the default, we can check if the customer entered Nodes or IPTarget. If neither, check for cluster membership, and use that for the nodes.
|
|
if ( $PsCmdlet.ParameterSetName -eq 'FullNodeMap' -or $PsCmdlet.ParameterSetName -eq 'OnlyPrereqNodes' -or $PsCmdlet.ParameterSetName -eq 'OnlyConMapNodes' -or $PsCmdlet.ParameterSetName -eq 'RevokeFWRulesNodes' ) {
|
|
if (-not($PSBoundParameters.ContainsKey('Nodes'))) {
|
|
try { $Nodes = (Get-ClusterNode -ErrorAction SilentlyContinue -WarningAction SilentlyContinue).Name }
|
|
catch {
|
|
Write-Host 'To run this cmdlet without parameters, join a cluster then try again. Otherwise, specify the Nodes or IPTarget parameters' -ForegroundColor Red
|
|
"To run this cmdlet without parameters, join a cluster then try again. Otherwise, specify the Nodes or IPTarget parameters." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
break
|
|
}
|
|
}
|
|
|
|
If ($EnableFirewallRules) { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -Nodes $Nodes -Stage $Stage -EnableFirewallRules }
|
|
else { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -Nodes $Nodes -Stage $Stage }
|
|
}
|
|
else { # Function returns both the target information and the results of the prerequisite testing
|
|
If ($EnableFirewallRules) { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -IPTarget $IPTarget -Stage $Stage -EnableFirewallRules }
|
|
else { $TargetInfo, $PrereqStatus = Test-NetStackPrerequisites -IPTarget $IPTarget -Stage $Stage }
|
|
}
|
|
|
|
if ( $RevokeFirewallRules ) {
|
|
if ($IPTarget) { $Targets = $IPTarget }
|
|
else { $Targets = $Nodes }
|
|
|
|
Revoke-FirewallRules -Stage $Stage -Targets $Targets
|
|
|
|
Return
|
|
}
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Prerequisites -Value $TargetInfo
|
|
Remove-Variable TargetInfo -ErrorAction SilentlyContinue
|
|
|
|
"Prerequisite Test Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$NetStackResults.Prerequisites | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
if ($OnlyPrerequisites) {
|
|
return $NetStackResults
|
|
}
|
|
elseif ($false -in $PrereqStatus) {
|
|
Write-LogMessage -Message "Prerequsite tests have failed. Review the NetStack results for more details." -LogFile $LogFile
|
|
$NetStackResults.Prerequisites | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
|
|
#region Connectivity Maps
|
|
|
|
$RDMAStages = @(3, 4, 5, 6, 7)
|
|
if ($OnlyConnectivityMap) {
|
|
Write-LogMessage -Message "'Connectivity Map Only' option specified. Checking for Network ATC." -LogFile $LogFile
|
|
$StorageIntentDeployment = Get-StorageIntentDeploymentStatus -LogFile $LogFile
|
|
}
|
|
elseif (($Stage | Where-Object {$RDMAStages -contains $_}).Count -gt 0) {
|
|
Write-LogMessage -Message "RDMA stages specified. Checking for Network ATC." -LogFile $LogFile
|
|
$StorageIntentDeployment = Get-StorageIntentDeploymentStatus -LogFile $LogFile
|
|
} else {
|
|
Write-LogMessage -Message "No RDMA stages specified, skipping check for Network ATC." -LogFile $LogFile
|
|
$StorageIntentDeployment = New-Object -TypeName psobject
|
|
$StorageIntentDeployment | Add-Member -MemberType NoteProperty -Name ClusterName -Value $null
|
|
$StorageIntentDeployment | Add-Member -MemberType NoteProperty -Name StorageIntent -Value $null
|
|
$StorageIntentDeployment | Add-Member -MemberType NoteProperty -Name DeploymentStatus -Value [StorageIntentDeploymentStatus]::NotDeployed
|
|
}
|
|
|
|
# If ATC is deployed, generate a list of adapters associated with a storage intent
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess) {
|
|
$StorageIntentNICMapping = Get-StorageIntentNICMapping -StorageIntentDeployment $StorageIntentDeployment
|
|
}
|
|
|
|
Write-LogMessage -Message "Generating Connectivity Map" -LogFile $LogFile
|
|
if ($Nodes) {
|
|
# If Network ATC is deployed, pass the storage intent adapter mapping to the connectivity mapper to fill in StorageIntentSet property
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess -and $StorageIntentNICMapping.Count -gt 0) {
|
|
$Mapping = Get-ConnectivityMapping -Nodes $Nodes -StorageIntentNICMapping $StorageIntentNICMapping
|
|
}
|
|
else { # Otherwise, run connectivity mapper as usual
|
|
$Mapping = Get-ConnectivityMapping -Nodes $Nodes
|
|
}
|
|
}
|
|
else {
|
|
$Mapping = Get-ConnectivityMapping -IPTarget $IPTarget
|
|
}
|
|
|
|
Write-LogMessage -Message "Determining Testable Networks" -LogFile $LogFile
|
|
$TestableNetworks = Get-TestableNetworksFromMapping -Mapping $Mapping
|
|
Write-LogMessage -Message "Determining Disqualified Networks" -LogFile $LogFile
|
|
$DisqualifiedNetworks = Get-DisqualifiedNetworksFromMapping -Mapping $Mapping
|
|
|
|
Write-LogMessage -Message "Connectivity Map Complete" -LogFile $LogFile
|
|
"`r`n####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
# If at least one note property doesn't exist, then no disqualified networks were identified
|
|
if (($DisqualifiedNetworks | Get-Member -MemberType NoteProperty).Count) {
|
|
"Disqualified Networks" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$DisqualifiedNetworks.PSObject.Properties | ForEach-Object {
|
|
$DisqualificationCategory = $_
|
|
"`r`nDisqualification Category: $($DisqualificationCategory.Name)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$DisqualificationCategory.Value | ForEach-Object {
|
|
$_.Name | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$_.Group | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name DisqualifiedNetworks -Value $DisqualifiedNetworks
|
|
}
|
|
else { Remove-Variable -Name DisqualifiedNetworks -ErrorAction SilentlyContinue }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name TestableNetworks -Value $TestableNetworks
|
|
|
|
if ($TestableNetworks -eq 'None Available' -and (-not($OnlyConnectivityMap))) {
|
|
Write-Error 'No Testable Networks Found. Aborting Test-NetStack.'
|
|
"No Testable Networks Found. Aborting Test-NetStack." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
|
|
"Testable Networks`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$TestableNetworks | ForEach-Object {
|
|
$_.Values | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$_.Group | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
if ($OnlyConnectivityMap) { return $NetStackResults }
|
|
#endregion Connectivity Maps
|
|
|
|
$runspaceGroups = Get-RunspaceGroups -TestableNetworks $TestableNetworks
|
|
|
|
# Defines the stage requirements - internal.psm1
|
|
$Definitions = [Analyzer]::new()
|
|
|
|
$ResultsSummary = New-Object -TypeName psobject
|
|
$StageFailures = 0
|
|
|
|
$MaxRunspaces = [int]$env:NUMBER_OF_PROCESSORS * 2
|
|
|
|
Switch ( $Stage | Sort-Object ) {
|
|
'1' { # ICMP Connectivity, Reliability, and PMTUD
|
|
$thisStage = $_
|
|
Write-Host "Beginning Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)"
|
|
"Stage 1`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Beginning Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
|
|
$NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
|
|
$NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }
|
|
|
|
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
|
|
$RunspacePool.Open()
|
|
|
|
$AllJobs = @()
|
|
$StageResults = @()
|
|
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
$thisTestableNet.Group | ForEach-Object {
|
|
$thisSource = $_
|
|
$thisSourceResult = @()
|
|
|
|
$thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$PowerShell = [powershell]::Create()
|
|
$PowerShell.RunspacePool = $RunspacePool
|
|
|
|
[void] $PowerShell.AddScript({
|
|
param ( $thisComputerName, $thisSource, $thisTarget, $Definitions, $LogFile )
|
|
|
|
if ($thisSource.NodeName -eq $Env:COMPUTERNAME) {
|
|
$thisSourceResult = Invoke-ICMPPMTUD -Source $thisSource.IPAddress -Destination $thisTarget.IPAddress
|
|
}
|
|
else {
|
|
$thisSourceResult = Invoke-Command -ComputerName $thisComputerName `
|
|
-ArgumentList $thisSource.IPAddress, $thisTarget.IPAddress `
|
|
-ScriptBlock ${Function:\Invoke-ICMPPMTUD}
|
|
}
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name SourceHostName -Value $thisComputerName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Source -Value $thisSource.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Destination -Value $thisTarget.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Connectivity -Value $thisSourceResult.Connectivity
|
|
$Result | Add-Member -MemberType NoteProperty -Name MTU -Value $thisSourceResult.MTU
|
|
$Result | Add-Member -MemberType NoteProperty -Name MSS -Value $thisSourceResult.MSS
|
|
|
|
if ($thisSource.NodeName -eq $Env:COMPUTERNAME) {
|
|
$thisSourceResult = Invoke-ICMPPMTUD -Source $thisSource.IPAddress -Destination $thisTarget.IPAddress -StartBytes $thisSourceMSS -Reliability
|
|
}
|
|
else {
|
|
$thisSourceResult = Invoke-Command -ComputerName $thisComputerName `
|
|
-ArgumentList $thisSource.IPAddress, $thisTarget.IPAddress, $thisSourceResult.MSS , $null, $true `
|
|
-ScriptBlock ${Function:\Invoke-ICMPPMTUD}
|
|
}
|
|
|
|
$TotalSent = $thisSourceResult.Count
|
|
$TotalFailed = ($thisSourceResult -eq '-1').Count
|
|
$SuccessPercentage = ([Math]::Round((100 - (($TotalFailed / $TotalSent) * 100)), 2))
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name TotalSent -Value $TotalSent
|
|
$Result | Add-Member -MemberType NoteProperty -Name TotalFailed -Value $TotalFailed
|
|
$Result | Add-Member -MemberType NoteProperty -Name Reliability -Value $SuccessPercentage
|
|
|
|
# -1 (no response) will be ignored for LAT and JIT
|
|
$Latency = Get-Latency -RoundTripTime ($thisSourceResult -ne -1)
|
|
$Jitter = Get-Jitter -RoundTripTime ($thisSourceResult -ne -1)
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name Latency -Value $Latency
|
|
$Result | Add-Member -MemberType NoteProperty -Name Jitter -Value $Jitter
|
|
|
|
# Pass or fail Stage 1 based on connectivity
|
|
if ($Result.Connectivity -eq $true) {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass'
|
|
}
|
|
else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
}
|
|
|
|
# Evaluate reliability results if collected
|
|
if ($TotalSent -and $SuccessPercentage -and $Latency -and $Jitter -and
|
|
$Definitions.Reliability.ICMPSent -and $Definitions.Reliability.ICMPReliability -and
|
|
$Definitions.Reliability.ICMPLatency -and $Definitions.Reliability.ICMPJitter) {
|
|
|
|
if ($TotalSent -ge $Definitions.Reliability.ICMPSent -and
|
|
$SuccessPercentage -ge $Definitions.Reliability.ICMPReliability -and
|
|
$Latency -le $Definitions.Reliability.ICMPLatency -and
|
|
$Jitter -le $Definitions.Reliability.ICMPJitter ) {
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReliabilityStatus -Value 'Pass'
|
|
}
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name ReliabilityStatus -Value 'Fail' }
|
|
}
|
|
else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReliabilityStatus -Value 'Fail'
|
|
"ERROR: Data failed to be collected for path ($($thisComputerName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
return $Result
|
|
})
|
|
|
|
$param = @{
|
|
thisComputerName = $thisSource.NodeName
|
|
thisSource = $thisSource
|
|
thisTarget = $thisTarget
|
|
Definitions = $Definitions
|
|
LogFile = $LogFile
|
|
}
|
|
|
|
[void] $PowerShell.AddParameters($param)
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] ($($thisSource.NodeName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] ($($thisSource.NodeName)) $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$asyncJobObj = @{ JobHandle = $PowerShell; AsyncHandle = $PowerShell.BeginInvoke() }
|
|
|
|
$AllJobs += $asyncJobObj
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
}
|
|
|
|
While ($AllJobs -ne $null) {
|
|
$AllJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
|
|
$thisJob = $_
|
|
$StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
|
|
$thisSourceHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).SourceHostName
|
|
$thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Source
|
|
$thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Destination
|
|
|
|
$AllJobs = $AllJobs -ne $thisJob
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] ($thisSourceHostName) $($thisSource) -> $($thisTarget)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] ($thisSourceHostName) $($thisSource) -> $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
|
|
$RunspacePool.Close()
|
|
$RunspacePool.Dispose()
|
|
|
|
if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage1 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage1 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage1 -Value $StageResults
|
|
|
|
Write-Host "Completed Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - Connectivity and PMTUD - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 1 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'2' { # TCP Stress 1:1
|
|
if ( $ContinueOnFailure -eq $false ) {
|
|
if ('fail' -in $NetStackResults.Stage1.PathStatus) {
|
|
|
|
$Stage -gt 1 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
|
|
"Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
}
|
|
|
|
$thisStage = $_
|
|
|
|
"Stage 2`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$NTttcpLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NTttcp"
|
|
foreach ($Node in $Nodes) {
|
|
if ($Node -ne $Env:ComputerName) {
|
|
$NTttcpLogDir = Invoke-Command -ComputerName $Node { New-Item -Path $Using:NTttcpLogPath -ItemType Directory -Force -ErrorAction SilentlyContinue }
|
|
}
|
|
else { # Machine is local
|
|
|
|
$NTttcpLogDir = New-Item -Path $NTttcpLogPath -ItemType Directory -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
# Using this variable as a placeholder for potential future debug mode options
|
|
# Set to true for now while monitoring lab runs after switching to NTttcp
|
|
$Sequential = $true
|
|
if ( $Sequential -eq $false ) {
|
|
Write-Host "Beginning Stage: $thisStage - TCP - Parallel - $([System.DateTime]::Now)"
|
|
"Beginning Stage: $thisStage - TCP - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
|
|
$NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
|
|
$NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }
|
|
|
|
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
|
|
$RunspacePool.Open()
|
|
|
|
$StageResults = @()
|
|
foreach ($group in $runspaceGroups) {
|
|
$GroupedJobs = @()
|
|
foreach ($pair in $group) {
|
|
$PowerShell = [powershell]::Create()
|
|
$PowerShell.RunspacePool = $RunspacePool
|
|
|
|
[void] $PowerShell.AddScript({
|
|
param ( $thisComputerName, $thisSource, $thisTarget, $Definitions, $LogFile )
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
$thisTargetResult = Invoke-TCP -Receiver $thisTarget -Sender $thisSource
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisTargetResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisTargetResult.ReceivedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisTargetResult.ReceivedPctgOfLinkSpeed
|
|
$Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.TCPPerf.TPUT
|
|
|
|
if ($thisTargetResult.ReceivedPctgOfLinkSpeed -and $Definitions.TCPPerf.TPUT) {
|
|
if ($thisTargetResult.ReceivedPctgOfLinkSpeed -ge $Definitions.TCPPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
|
|
}
|
|
else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
"ERROR: Data failed to be collected for path $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisTargetResult.RawData
|
|
|
|
Return $Result
|
|
})
|
|
|
|
$param = @{
|
|
thisComputerName = $pair.Source.NodeName
|
|
thisSource = $pair.Source
|
|
thisTarget = $pair.Target
|
|
Definitions = $Definitions
|
|
LogFile = $LogFile
|
|
}
|
|
|
|
[void] $PowerShell.AddParameters($param)
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$asyncJobObj = @{ JobHandle = $PowerShell
|
|
AsyncHandle = $PowerShell.BeginInvoke() }
|
|
|
|
$GroupedJobs += $asyncJobObj
|
|
}
|
|
|
|
While ($GroupedJobs -ne $null) {
|
|
$GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
|
|
$thisJob = $_
|
|
$StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
|
|
$thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
|
|
$thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
|
|
$thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver
|
|
|
|
$GroupedJobs = $GroupedJobs -ne $thisJob
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
}
|
|
|
|
$RunspacePool.Close()
|
|
$RunspacePool.Dispose()
|
|
}
|
|
else { # Run sequentially
|
|
Write-Host "Beginning Stage 2 - TCP - Sequential - $([System.DateTime]::Now)"
|
|
"Beginning Stage 2 - TCP - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
$thisTestableNet.Group | ForEach-Object {
|
|
$thisSource = $_
|
|
$thisSourceResult = @()
|
|
|
|
$thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$thisSourceResult = Invoke-TCP -Receiver $thisTarget -Sender $thisSource -LogDir $NTttcpLogDir
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IpAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
|
|
$Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.TCPPerf.TPUT
|
|
|
|
$ThroughputPercentageDec = $Definitions.TCPPerf.TPUT / 100.0
|
|
$AcceptableThroughput = $thisSourceResult.RawData.MinLinkSpeedbps * $ThroughputPercentageDec
|
|
|
|
if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.TCPPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
|
|
|
|
$StageResults += $Result
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage2 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage2 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage2 -Value $StageResults
|
|
Write-Host "Completed Stage: $thisStage - TCP - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - TCP - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 2 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'3' { # RDMA Connectivity
|
|
$thisStage = $_
|
|
"Stage 3`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
# Using this variable as a placeholder for potential future debug mode options
|
|
# Set to true for now while NDKPerf does not support parallel connections
|
|
$Sequential = $true
|
|
if ($Sequential -eq $false) { # Run in parallel (re-enable when supported by NDKPerf)
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Ping - Parallel - $([System.DateTime]::Now)"
|
|
"Beginning Stage: $thisStage - RDMA Ping - Parallel - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
|
|
$NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
|
|
$NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }
|
|
|
|
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
|
|
$RunspacePool.Open()
|
|
|
|
$AllJobs = @()
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
$thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
|
|
$thisSource = $_
|
|
$thisSourceResult = @()
|
|
|
|
$thisTestableNet.Group | Where-Object NodeName -ne $thisSource.NodeName | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$PowerShell = [powershell]::Create()
|
|
$PowerShell.RunspacePool = $RunspacePool
|
|
|
|
[void] $PowerShell.AddScript({
|
|
param ( $thisSource, $thisTarget, $Definitions )
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
$thisSourceResult = Invoke-NDKPing -Server $thisTarget -Client $thisSource
|
|
|
|
if ($thisSourceResult.ServerSuccess) {
|
|
$Result | Add-Member -MemberType NoteProperty -Name Connectivity -Value $true
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass'
|
|
}
|
|
else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name Connectivity -Value $false
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
}
|
|
|
|
Return $Result
|
|
})
|
|
|
|
$param = @{
|
|
thisSource = $thisSource
|
|
thisTarget = $thisTarget
|
|
Definitions = $Definitions
|
|
}
|
|
|
|
[void] $PowerShell.AddParameters($param)
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$asyncJobObj = @{ JobHandle = $PowerShell
|
|
AsyncHandle = $PowerShell.BeginInvoke() }
|
|
|
|
$AllJobs += $asyncJobObj
|
|
}
|
|
}
|
|
}
|
|
|
|
While ($AllJobs -ne $null) {
|
|
$AllJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
|
|
$thisJob = $_
|
|
$StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
|
|
$thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
|
|
$thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
|
|
$thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver
|
|
|
|
$AllJobs = $AllJobs -ne $thisJob
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
|
|
$RunspacePool.Close()
|
|
$RunspacePool.Dispose()
|
|
} else { # Run sequentially
|
|
Write-Host "Beginning Stage 3 - NDK Ping - Sequential - $([System.DateTime]::Now)"
|
|
"Beginning Stage: $thisStage - RDMA Ping - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$NDKLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
|
|
$NDKLog = New-Item -Path $NDKLogPath -ItemType File -Force -ErrorAction SilentlyContinue
|
|
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
# If Network ATC is deployed, filter by StorageIntentSet. Otherwise, filter by RDMAEnabled
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess) {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.StorageIntentSet }
|
|
if (($GroupToTest | Measure-Object).Count -eq 0) {
|
|
# This network is not a storage network - skip to the next network ("Return" since we are in ForEach-Object script block)
|
|
Return
|
|
}
|
|
} else {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled }
|
|
}
|
|
|
|
if (($GroupToTest | Measure-Object).Count -lt 2) {
|
|
Write-LogMessage -Message "RDMA stage $thisStage requested, but no RDMA-enabled adapters were found on this network. Skipping this network." -LogFile $LogFile
|
|
$Result = New-Object -TypeName psobject
|
|
# Fill in typical columns so fields will be available for other networks able to be tested, and so it's clear which network failed
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTestableNet.Group.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Skipped'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'RDMA Misconfiguration - Not Tested'
|
|
$StageResults += $Result
|
|
}
|
|
|
|
$GroupToTest | ForEach-Object {
|
|
$thisSource = $_
|
|
$thisSourceResult = @()
|
|
|
|
$GroupToTest | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
# If Network ATC is deployed but failed, skip testing and mark failed
|
|
if ($StorageIntentDeployment.DeploymentStatus -ne [StorageIntentDeploymentStatus]::DeploymentFail) {
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$thisSourceResult = Invoke-NDKPing -Server $thisTarget -Client $thisSource -NDKLog $NDKLog
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
if ($thisSourceResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
|
|
} else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'Network ATC Misconfiguration - Not Tested'
|
|
}
|
|
|
|
$StageResults += $Result
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage3 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage3 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage3 -Value $StageResults
|
|
|
|
Write-Host "Completed Stage: $thisStage - RDMA Ping - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - RDMA Ping - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 3 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'4' { # RDMA Stress 1:1
|
|
if ( $ContinueOnFailure -eq $false ) {
|
|
if ('fail' -in $NetStackResults.Stage3.PathStatus) {
|
|
|
|
$Stage -ge 4 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
|
|
"Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
}
|
|
|
|
$thisStage = $_
|
|
"Stage 4`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
# Using this variable as a placeholder for potential future debug mode options
|
|
# Set to true for now while NDKPerf does not support parallel connections
|
|
$Sequential -eq $true
|
|
if ($Sequential -eq $false) { # Run in parallel (re-enable when supported by NDKPerf)
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Perf 1:1 - Parallel - $([System.DateTime]::Now)"
|
|
"Beginning Stage: $thisStage - RDMA Perf 1:1 - Parallel - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
|
|
$NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
|
|
$NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }
|
|
|
|
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
|
|
$RunspacePool.Open()
|
|
|
|
$StageResults = @()
|
|
foreach ($group in $runspaceGroups) {
|
|
$GroupedJobs = @()
|
|
foreach ($pair in ($group | Where {($group.Source.RDMAEnabled) -and ($group.Target.RDMAEnabled)})) {
|
|
|
|
$PowerShell = [powershell]::Create()
|
|
$PowerShell.RunspacePool = $RunspacePool
|
|
|
|
[void] $PowerShell.AddScript({
|
|
param ( $thisSource, $thisTarget, $Definitions )
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
$thisSourceResult = Invoke-NDKPerf1to1 -Server $thisTarget -Client $thisSource -ExpectedTPUT $Definitions.NDKPerf.TPUT
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
|
|
$Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.NDKPerf.TPUT
|
|
|
|
if ($thisSourceResult.ReceivedPctgOfLinkSpeed -and $Definitions.NDKPerf.TPUT) {
|
|
if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.NDKPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
|
|
}
|
|
else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
"ERROR: Data failed to be collected for path $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
|
|
|
|
Return $Result
|
|
})
|
|
|
|
$param = @{
|
|
thisSource = $pair.Source
|
|
thisTarget = $pair.Target
|
|
Definitions = $Definitions
|
|
}
|
|
|
|
[void] $PowerShell.AddParameters($param)
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Started] $($pair.Source.IPAddress) -> ($($pair.Target.NodeName)) $($pair.Target.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$asyncJobObj = @{ JobHandle = $PowerShell
|
|
AsyncHandle = $PowerShell.BeginInvoke() }
|
|
|
|
$GroupedJobs += $asyncJobObj
|
|
}
|
|
|
|
While ($GroupedJobs -ne $null) {
|
|
$GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
|
|
$thisJob = $_
|
|
$StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
|
|
$thisReceiverHostName = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).ReceiverHostName
|
|
$thisSource = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Sender
|
|
$thisTarget = ($thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)).Receiver
|
|
|
|
$GroupedJobs = $GroupedJobs -ne $thisJob
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource) -> ($thisReceiverHostName) $($thisTarget)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
}
|
|
|
|
$RunspacePool.Close()
|
|
$RunspacePool.Dispose()
|
|
} else { # Run sequentially
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Perf 1:1 - Sequential - $([System.DateTime]::Now)"
|
|
"Beginning Stage: $thisStage - RDMA Perf 1:1 - Sequential - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$NDKServerLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKPerfServerOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
|
|
$NDKClientLogPath = Join-Path -Path $LogFileParentPath -ChildPath "\NDK\NDKPerfClientOutput-$(Get-Date -f yyyy-MM-dd-HHmmss).txt"
|
|
$NDKServerLog = New-Item -Path $NDKServerLogPath -ItemType File -Force -ErrorAction SilentlyContinue
|
|
$NDKClientLog = New-Item -Path $NDKClientLogPath -ItemType File -Force -ErrorAction SilentlyContinue
|
|
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
# If Network ATC is deployed, filter by StorageIntentSet. Otherwise, filter by RDMAEnabled
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess) {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.StorageIntentSet }
|
|
if (($GroupToTest | Measure-Object).Count -eq 0) {
|
|
# This network is not a storage network - skip to the next network ("Return" since we are in ForEach-Object script block)
|
|
Return
|
|
}
|
|
} else {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled }
|
|
}
|
|
|
|
if (($GroupToTest | Measure-Object).Count -lt 2) {
|
|
Write-LogMessage -Message "RDMA stage $thisStage requested, but no RDMA-enabled adapters were found on this network. Skipping this network." -LogFile $LogFile
|
|
$Result = New-Object -TypeName psobject
|
|
# Fill in typical columns so fields will be available for other networks able to be tested, and so it's clear which network failed
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTestableNet.Group.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.NDKPerf.TPUT
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Skipped'
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'RDMA Misconfiguration - Not Tested'
|
|
$StageResults += $Result
|
|
}
|
|
|
|
$GroupToTest | ForEach-Object {
|
|
$thisSource = $_
|
|
$thisSourceResult = @()
|
|
|
|
$GroupToTest | Where-Object NodeName -ne $thisSource.NodeName | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Sender -Value $thisSource.IPaddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
# If Network ATC is deployed but failed, skip testing and mark failed
|
|
if ($StorageIntentDeployment.DeploymentStatus -ne [StorageIntentDeploymentStatus]::DeploymentFail) {
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Starting] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$thisSourceResult = Invoke-NDKPerf1to1 -Server $thisTarget -Client $thisSource -ExpectedTPUT $Definitions.NDKPerf.TPUT -NDKServerLog $NDKServerLog -NDKClientLog $NDKClientLog
|
|
|
|
Write-Host ":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)"
|
|
":: Stage $thisStage : $([System.DateTime]::Now) :: [Completed] $($thisSource.IPAddress) -> ($($thisTarget.NodeName)) $($thisTarget.IPAddress)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.ReceivedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisSourceResult.ReceivedPctgOfLinkSpeed
|
|
$Result | Add-Member -MemberType NoteProperty -Name MinExpectedPctgOfLinkSpeed -Value $Definitions.NDKPerf.TPUT
|
|
|
|
if ($thisSourceResult.ReceivedPctgOfLinkSpeed -ge $Definitions.NDKPerf.TPUT) { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail' }
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
|
|
} else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name PathStatus -Value 'Fail'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'Network ATC Misconfiguration - Not Tested'
|
|
}
|
|
|
|
$StageResults += $Result
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if ('Fail' -in $StageResults.PathStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage4 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage4 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage4 -Value $StageResults
|
|
Write-Host "Completed Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - RDMA Perf 1:1 - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 4 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'5' { # RDMA Stress N:1
|
|
if ( $ContinueOnFailure -eq $false ) {
|
|
if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus) {
|
|
|
|
$Stage -ge 5 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
|
|
"Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
}
|
|
|
|
$thisStage = $_
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)"
|
|
"Stage 5`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Beginning Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
# If ATC is deployed, filter by StorageIntentSet. Otherwise, filter by RDMAEnabled
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess) {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.StorageIntentSet }
|
|
if (($GroupToTest | Measure-Object).Count -eq 0) {
|
|
# This network is not a storage network - skip to the next network ("Return" since we are in ForEach-Object script block)
|
|
Return
|
|
}
|
|
} else {
|
|
$GroupToTest = $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled }
|
|
}
|
|
|
|
if (($GroupToTest | Measure-Object).Count -lt 2) {
|
|
Write-LogMessage -Message "RDMA stage $thisStage requested, but no RDMA-enabled adapters were found on this network. Skipping this network." -LogFile $LogFile
|
|
$Result = New-Object -TypeName psobject
|
|
# Fill in typical columns so fields will be available for other networks able to be tested, and so it's clear which network failed
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTestableNet.Group.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Skipped'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'RDMA Misconfiguration - Not Tested'
|
|
$Result | Add-Member -MemberType NoteProperty -Name ClientNetworkTested -Value $thisTestableNet.Group.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value "N/A"
|
|
$StageResults += $Result
|
|
}
|
|
|
|
$GroupToTest | ForEach-Object {
|
|
$thisTarget = $_
|
|
|
|
$ClientNetwork = @($GroupToTest | Where-Object NodeName -ne $thisTarget.NodeName)
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisTarget.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisTarget.IPAddress
|
|
|
|
# If Network ATC is deployed but failed, skip testing and mark failed
|
|
if ($StorageIntentDeployment.DeploymentStatus -ne [StorageIntentDeploymentStatus]::DeploymentFail) {
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))"
|
|
":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$thisTargetResult = Invoke-NDKPerfNto1 -Server $thisTarget -ClientNetwork $ClientNetwork -ExpectedTPUT $Definitions.NDKPerf.TPUT
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisTargetResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisTargetResult.RxGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxPctgOfLinkSpeed -Value $thisTargetResult.ReceivedPctgOfLinkSpeed
|
|
|
|
if ($thisTargetResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Fail' }
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name ClientNetworkTested -Value $thisTargetResult.ClientNetworkTested
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisTargetResult.RawData
|
|
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))"
|
|
":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisTarget.InterfaceIndex) ($($thisTarget.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
} else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Fail'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'Network ATC Misconfiguration - Not Tested'
|
|
}
|
|
|
|
$StageResults += $Result
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
|
|
if ('Fail' -in $StageResults.ReceiverStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage5 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage5 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage5 -Value $StageResults
|
|
|
|
Write-Host "Completed Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - RDMA Perf N:1 - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 5 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'6' { # RDMA Stress N:N
|
|
if ( $ContinueOnFailure -eq $false ) {
|
|
if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus -or 'fail' -in $NetStackResults.Stage5.ReceiverStatus) {
|
|
|
|
$Stage -ge 6 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$ResultsSummary | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
|
|
"Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
}
|
|
|
|
$thisStage = $_
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)"
|
|
"Stage 6`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Beginning Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$StageResults = @()
|
|
$TestableNetworks | ForEach-Object {
|
|
$thisTestableNet = $_
|
|
|
|
# If Network ATC is deployed, filter by StorageIntentSet. Otherwise, filter by RDMAEnabled
|
|
if ($StorageIntentDeployment.DeploymentStatus -eq [StorageIntentDeploymentStatus]::DeploymentSuccess) {
|
|
$ServerList = $thisTestableNet.Group | Where-Object -FilterScript { $_.StorageIntentSet }
|
|
if (($ServerList | Measure-Object).Count -eq 0) {
|
|
# This network is not a storage network - skip to the next network ("Return" since we are in ForEach-Object script block)
|
|
Return
|
|
}
|
|
} else {
|
|
$ServerList = $thisTestableNet.Group | Where-Object -FilterScript { $_.RDMAEnabled }
|
|
}
|
|
|
|
$thisSubnet = ($ServerList | Select-Object -First 1).subnet
|
|
$thisVLAN = ($ServerList | Select-Object -First 1).VLAN
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$thisSubnet = $thisTestableNet.Name.Split(',')[0]
|
|
$thisVLAN = $thisTestableNet.Name.Split(',')[1].Trim()
|
|
$Result | Add-Member -MemberType NoteProperty -Name Subnet -Value $thisSubnet
|
|
$Result | Add-Member -MemberType NoteProperty -Name VLAN -Value $thisVLAN
|
|
|
|
if (($ServerList | Measure-Object).Count -lt 2) {
|
|
Write-LogMessage -Message "RDMA stage $thisStage requested, but no RDMA-enabled adapters were found on this network. Skipping this network." -LogFile $LogFile
|
|
# Fill in typical columns so fields will be available for other networks able to be tested, and so it's clear which network failed
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value "N/A"
|
|
$Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Skipped'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'RDMA Misconfiguration - Not Tested'
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value "N/A"
|
|
$StageResults += $Result
|
|
} else {
|
|
# If Network ATC is deployed but failed, skip testing and mark failed
|
|
if ($StorageIntentDeployment.DeploymentStatus -ne [StorageIntentDeploymentStatus]::DeploymentFail) {
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)"
|
|
":: $([System.DateTime]::Now) :: [Started] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$thisSourceResult = Invoke-NDKPerfNtoN -ServerList $ServerList -ExpectedTPUT $Definitions.NDKPerf.TPUT
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.RxGbps
|
|
|
|
if ($thisSourceResult.ServerSuccess) { $Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Fail' }
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
|
|
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)"
|
|
":: $([System.DateTime]::Now) :: [Completed] N -> N on subnet $($thisSubnet) and VLAN $($thisVLAN)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
} else {
|
|
$Result | Add-Member -MemberType NoteProperty -Name NetworkStatus -Value 'Fail'
|
|
$Result | Add-Member -MemberType NoteProperty -Name FailureReason -Value 'Network ATC Misconfiguration - Not Tested'
|
|
}
|
|
}
|
|
|
|
$StageResults += $Result
|
|
Remove-Variable Result -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
if ('Fail' -in $StageResults.NetworkStatus) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage6 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage6 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage6 -Value $StageResults
|
|
|
|
Write-Host "Completed Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)"
|
|
"Completed Stage: $thisStage - RDMA Perf N:N - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 6 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
'7' { # RDMA Stress N:1 Parallel
|
|
|
|
if ($Experimental -eq $false) {
|
|
Write-Error "Stage $_ is experimental. The experimental flag has not been set. Please enable it to run experimental stages."
|
|
"The experimental stage(s) $ChosenStages have been selected to be run, but the experimental flag has not been set. Please enable it to run experimental stages." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
return $NetStackResults
|
|
}
|
|
|
|
$IsVDiskUnhealthy = Get-VDiskStatus($LogFile)
|
|
if ($IsVDiskUnhealthy) {
|
|
$Stage -ge 7 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to unhealthy VDisk.' | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
if ( $ContinueOnFailure -eq $false ) {
|
|
if ('fail' -in $NetStackResults.Stage3.PathStatus -or 'fail' -in $NetStackResults.Stage4.PathStatus -or 'fail' -in $NetStackResults.Stage5.ReceiverStatus -or 'fail' -in $NetStackResults.Stage6.NetworkStatus) {
|
|
|
|
$Stage -ge 7 | ForEach-Object {
|
|
$AbortedStage = $_
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name "Stage$AbortedStage" -Value 'Aborted'; $StageFailures++
|
|
}
|
|
|
|
Write-Warning 'Aborted due to failures in earlier stage(s). To continue despite failures, use the ContinueOnFailure parameter.'
|
|
return $NetStackResults
|
|
}
|
|
}
|
|
|
|
$NodeGroups = $Mapping | Where-Object VLAN -ne 'Unsupported' | Group-Object NodeName
|
|
$thisStage = $_
|
|
Write-Host "Beginning Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)"
|
|
"Stage 7`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Console Output" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Beginning Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$StageResults = @()
|
|
$ISS = [System.Management.Automation.Runspaces.InitialSessionState]::CreateDefault()
|
|
$NetStackHelperModules = Get-ChildItem (Join-Path -Path $PSScriptRoot -ChildPath 'Helpers\*') -Include '*.psm1'
|
|
$NetStackHelperModules | ForEach-Object { $ISS.ImportPSModule($_.FullName) }
|
|
|
|
$NodeGroups | ForEach-Object {
|
|
$GroupedJobs = @()
|
|
$RunspacePool = [runspacefactory]::CreateRunspacePool(1, $MaxRunspaces, $ISS, $host)
|
|
$RunspacePool.Open()
|
|
$testNodeGroup = $_
|
|
$testNodeGroup.Group | Where-Object -FilterScript { $_.RDMAEnabled } | ForEach-Object {
|
|
|
|
$thisSource = $_
|
|
$PowerShell = [powershell]::Create()
|
|
$PowerShell.RunspacePool = $RunspacePool
|
|
$ClientNodes = @($Mapping | Where-Object NodeName -ne $thisSource.NodeName | Where-Object VLAN -eq $thisSource.VLAN | Where-Object Subnet -eq $thisSource.Subnet | Where-Object -FilterScript { $_.RDMAEnabled })
|
|
|
|
[void] $PowerShell.AddScript({
|
|
param ( $thisSource, $ClientNodes, $Definitions, $LogFile )
|
|
$StartTime = Get-Date
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))"
|
|
":: $([System.DateTime]::Now) :: [Started] N -> Interface $($thisTarget.InterfaceIndex) ($($thisSource.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$events = @()
|
|
|
|
$thisSourceResult = Invoke-NDKPerfNto1 -Server $thisSource -ClientNetwork $ClientNodes -ExpectedTPUT $Definitions.NDKPerf.TPUT
|
|
|
|
$events = (Get-EventLog System -InstanceId 0x466,0x467,0x469,0x46a -After $StartTime)
|
|
|
|
$Result = New-Object -TypeName psobject
|
|
$Result | Add-Member -MemberType NoteProperty -Name ReceiverHostName -Value $thisSource.NodeName
|
|
$Result | Add-Member -MemberType NoteProperty -Name Receiver -Value $thisSource.IPAddress
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxLinkSpeedGbps -Value $thisSourceResult.ReceiverLinkSpeedGbps
|
|
$Result | Add-Member -MemberType NoteProperty -Name RxGbps -Value $thisSourceResult.RxGbps
|
|
|
|
if ($events) {
|
|
Write-Host "Found cluster membership lost events when testing node $($thisSource.NodeName). They can be found in the test log."
|
|
"Found cluster membership lost events when testing node $($thisSource.NodeName). They can be found in the test log." | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$events | Select-Object Time, EntryType, InstanceID, Message | Format-Table -AutoSize | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
|
|
if ($thisSourceResult.ServerSuccess -and $events.count -eq 0) { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Pass' }
|
|
else { $Result | Add-Member -MemberType NoteProperty -Name ReceiverStatus -Value 'Fail' }
|
|
|
|
$Result | Add-Member -MemberType NoteProperty -Name ClientNetworkTested -Value $thisSourceResult.ClientNetworkTested
|
|
$Result | Add-Member -MemberType NoteProperty -Name RawData -Value $thisSourceResult.RawData
|
|
|
|
Write-Host ":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))"
|
|
":: $([System.DateTime]::Now) :: [Completed] N -> Interface $($thisSource.InterfaceIndex) ($($thisSource.IPAddress))" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
return $Result
|
|
})
|
|
|
|
$param = @{
|
|
thisSource = $thisSource
|
|
ClientNodes = $ClientNodes
|
|
Definitions = $Definitions
|
|
LogFile = $LogFile
|
|
}
|
|
|
|
[void] $PowerShell.AddParameters($param)
|
|
|
|
$asyncJobObj = @{ JobHandle = $PowerShell
|
|
AsyncHandle = $PowerShell.BeginInvoke() }
|
|
|
|
$GroupedJobs += $asyncJobObj
|
|
|
|
}
|
|
|
|
While ($null -ne $GroupedJobs) {
|
|
$GroupedJobs | Where-Object { $_.AsyncHandle.IsCompleted } | ForEach-Object {
|
|
$thisJob = $_
|
|
$StageResults += $thisJob.JobHandle.EndInvoke($thisJob.AsyncHandle)
|
|
|
|
$GroupedJobs = $GroupedJobs | Where-Object { $_ -ne $thisJob }
|
|
}
|
|
}
|
|
|
|
$RunspacePool.close()
|
|
$RunspacePool.Dispose()
|
|
|
|
}
|
|
|
|
$IsVDiskUnhealthy = Get-VDiskStatus($LogFile)
|
|
|
|
if ('Fail' -in $StageResults.ReceiverStatus -or $IsVDiskUnhealthy) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage7 -Value 'Fail'; $StageFailures++ }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name Stage7 -Value 'Pass' }
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Stage7 -Value $StageResults
|
|
Write-Host "Completed Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)`r`n"
|
|
"Completed Stage: $thisStage - RDMA Perf VMSwitch Stress - $([System.DateTime]::Now)`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"Stage 7 Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$StageResults | Select-Object -Property * -ExcludeProperty RawData | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
}
|
|
}
|
|
|
|
if ($StageFailures -gt 0) { $ResultsSummary | Add-Member -MemberType NoteProperty -Name NetStack -Value 'Fail' }
|
|
else { $ResultsSummary | Add-Member -MemberType NoteProperty -Name NetStack -Value 'Pass' }
|
|
|
|
"Net Stack Results" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
$ResultsSummary | ft * | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
"####################################`r`n" | Out-File $LogFile -Append -Encoding utf8 -Width 2000
|
|
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name ResultsSummary -Value $ResultsSummary
|
|
|
|
$Failures = Get-Failures -NetStackResults $NetStackResults
|
|
if (@($Failures.PSObject.Properties).Count -gt 0) {
|
|
$NetStackResults | Add-Member -MemberType NoteProperty -Name Failures -Value $Failures
|
|
Write-RecommendationsToLogFile -NetStackResults $NetStackResults -LogFile $LogFile
|
|
}
|
|
Write-Verbose "Log file stored at: $LogPath"
|
|
|
|
Return $NetStackResults
|
|
}
|