321 строка
12 KiB
PowerShell
321 строка
12 KiB
PowerShell
#author - vhisav@microsoft.com
|
|
|
|
Import-Module .\TestLibs\RDFELibs.psm1 -Force
|
|
$result = ""
|
|
$testResult = ""
|
|
$resultArr = @()
|
|
$isDeployed = DeployVMS -setupType $currentTestData.setupType -Distro $Distro -xmlConfig $xmlConfig
|
|
if ($isDeployed)
|
|
{
|
|
try
|
|
{
|
|
$noServer = $true
|
|
$noClient = $true
|
|
$clientMachines = @()
|
|
$slaveHostnames = ""
|
|
foreach ( $vmData in $allVMData )
|
|
{
|
|
if ( $vmData.RoleName -imatch "Server" )
|
|
{
|
|
$serverVMData = $vmData
|
|
$noServer = $false
|
|
|
|
}
|
|
elseif ( $vmData.RoleName -imatch "Client" )
|
|
{
|
|
$clientMachines = $vmData
|
|
$noClient = $fase
|
|
if ( $slaveHostnames )
|
|
{
|
|
$slaveHostnames += "," + $vmData.RoleName
|
|
}
|
|
else
|
|
{
|
|
$slaveHostnames = $vmData.RoleName
|
|
}
|
|
}
|
|
}
|
|
if ( $noServer )
|
|
{
|
|
Throw "No any server VM defined. Be sure that, server VM role name matches with the pattern `"*server*`". Aborting Test."
|
|
}
|
|
if ( $noSlave )
|
|
{
|
|
Throw "No any client VM defined. Be sure that, client machine role names matches with pattern `"*client*`" Aborting Test."
|
|
}
|
|
if ($serverVMData.InstanceSize -imatch "Standard_NC")
|
|
{
|
|
LogMsg "Waiting 5 minutes to finish RDMA update for NC series VMs."
|
|
sleep -Seconds 300
|
|
}
|
|
#region CONFIGURE VMs for TEST
|
|
|
|
LogMsg "SERVER VM details :"
|
|
LogMsg " RoleName : $($serverVMData.RoleName)"
|
|
LogMsg " Public IP : $($serverVMData.PublicIP)"
|
|
LogMsg " SSH Port : $($serverVMData.SSHPort)"
|
|
$i = 1
|
|
foreach ( $clientVMData in $clientMachines )
|
|
{
|
|
LogMsg "CLIENT VM #$i details :"
|
|
LogMsg " RoleName : $($clientVMData.RoleName)"
|
|
LogMsg " Public IP : $($clientVMData.PublicIP)"
|
|
LogMsg " SSH Port : $($clientVMData.SSHPort)"
|
|
}
|
|
#
|
|
# PROVISION VMS FOR LISA WILL ENABLE ROOT USER AND WILL MAKE ENABLE PASSWORDLESS AUTHENTICATION ACROSS ALL VMS IN SAME HOSTED SERVICE.
|
|
#
|
|
|
|
ProvisionVMsForLisa -allVMData $allVMData -installPackagesOnRoleNames "none"
|
|
|
|
#endregion
|
|
|
|
#region Provision VMs for RDMA tests
|
|
|
|
#region Generate constants.sh
|
|
|
|
LogMsg "Generating constansts.sh ..."
|
|
$constantsFile = ".\$LogDir\constants.sh"
|
|
#foreach ($testParam in $currentTestData.params )
|
|
#{
|
|
# Add-Content -Value "$testParam" -Path $constantsFile
|
|
# LogMsg "$testParam added to constansts.sh"
|
|
#}
|
|
|
|
Add-Content -Value "master=`"$($serverVMData.RoleName)`"" -Path $constantsFile
|
|
LogMsg "master=$($serverVMData.RoleName) added to constansts.sh"
|
|
|
|
|
|
Add-Content -Value "slaves=`"$slaveHostnames`"" -Path $constantsFile
|
|
LogMsg "slaves=$slaveHostnames added to constansts.sh"
|
|
|
|
Add-Content -Value "rdmaRun=`"yes`"" -Path $constantsFile
|
|
LogMsg "rdmaRun=yes added to constansts.sh"
|
|
|
|
LogMsg "constanst.sh created successfully..."
|
|
#endregion
|
|
|
|
#region Generate etc-hosts.txt file
|
|
$hostsFile = ".\$LogDir\etc-hosts.txt"
|
|
foreach ( $vmDetails in $allVMData )
|
|
{
|
|
Add-Content -Value "$($vmDetails.InternalIP)`t$($vmDetails.RoleName)" -Path "$hostsFile"
|
|
LogMsg "$($vmDetails.InternalIP)`t$($vmDetails.RoleName) added to etc-hosts.txt"
|
|
}
|
|
#endregion
|
|
|
|
#region Upload files to master VM...
|
|
Set-Content -Value "/root/TestRDMA.sh &> rdmaConsole.txt" -Path "$LogDir\StartRDMA.sh"
|
|
Set-Content -Value "* hard memlock unlimited" -Path "$LogDir\limits.conf"
|
|
Add-Content -Value "* soft memlock unlimited" -Path "$LogDir\limits.conf"
|
|
$out = .\tools\dos2unix.exe "$LogDir\limits.conf" 2>&1
|
|
LogMsg $out
|
|
RemoteCopy -uploadTo $serverVMData.PublicIP -port $serverVMData.SSHPort -files "$constantsFile,$hostsFile,.\remote-scripts\TestRDMA.sh,.\$LogDir\StartRDMA.sh,.\$LogDir\limits.conf" -username "root" -password $password -upload
|
|
#endregion
|
|
|
|
#region Install LIS-RDMA drivers..
|
|
|
|
$osRelease = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "cat /etc/*release*" -ignoreLinuxExitCode
|
|
$modinfo_hv_vmbus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "lsmod | grep hv_network_direct"
|
|
if (!( $modinfo_hv_vmbus -imatch "hv_network_direct" ))
|
|
{
|
|
LogMsg "RDMA LIS Drivers not installed. RDMA drivers will be installed now..."
|
|
LogMsg $modinfo_hv_vmbus
|
|
Add-Content -Value "rdmaPrepare=`"yes`"" -Path $constantsFile
|
|
LogMsg "rdmaPrepare=yes added to constansts.sh"
|
|
|
|
if ( $osRelease -imatch "CentOS Linux release 7.1.")
|
|
{
|
|
$LIS4folder = "RHEL71"
|
|
}
|
|
if ( $osRelease -imatch "CentOS Linux release 7.0.")
|
|
{
|
|
$LIS4folder = "RHEL70"
|
|
}
|
|
if ( $osRelease -imatch "CentOS Linux release 6.5")
|
|
{
|
|
$LIS4folder = "RHEL65"
|
|
}
|
|
|
|
$LIS4IntallCommand = "tar -xf lis-4.0.11-RDMA.tar && chmod +x $LIS4folder/install.sh && cd $LIS4folder && ./install.sh"
|
|
$LIS4InstallJobs = @()
|
|
foreach ( $vm in $allVMData )
|
|
{
|
|
#Install LIS4 RDMA drivers...
|
|
LogMsg "Setting contents of /etc/security/limits.conf..."
|
|
$out = .\tools\dos2unix.exe ".\$LogDir\limits.conf" 2>&1
|
|
LogMsg $out
|
|
RemoteCopy -uploadTo $vm.PublicIP -port $vm.SSHPort -files ".\$LogDir\limits.conf" -username "root" -password $password -upload
|
|
$out = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "cat limits.conf >> /etc/security/limits.conf"
|
|
|
|
LogMsg "Downlaoding LIS-RDMA drivers ..."
|
|
$out = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "wget https://ciwestus.blob.core.windows.net/linuxbinaries/lis-4.0.11-RDMA.tar"
|
|
|
|
LogMsg "Executing $LIS4IntallCommand ..."
|
|
$jobID = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "$LIS4IntallCommand" -RunInBackground
|
|
$LIS4InstallObj = New-Object PSObject
|
|
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name ID -Value $jobID
|
|
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name RoleName -Value $vm.RoleName
|
|
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name PublicIP -Value $vm.PublicIP
|
|
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name SSHPort -Value $vm.SSHPort
|
|
$LIS4InstallJobs += $LIS4InstallObj
|
|
}
|
|
|
|
$LIS4InstallJobsRunning = $true
|
|
$lisInstallErrorCount = 0
|
|
while ($LIS4InstallJobsRunning)
|
|
{
|
|
$LIS4InstallJobsRunning = $false
|
|
foreach ( $job in $LIS4InstallJobs )
|
|
{
|
|
if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
|
|
{
|
|
LogMsg "LIS4-rdma Installation Status for $($job.RoleName) : Running"
|
|
$LIS4InstallJobsRunning = $true
|
|
}
|
|
else
|
|
{
|
|
$jobOut = Receive-Job -ID $($job.ID)
|
|
if ( $jobOut -imatch "Please reboot your system")
|
|
{
|
|
LogMsg "LIS-rdma installed successfully for $($job.RoleName)"
|
|
}
|
|
else
|
|
{
|
|
#LogErr "LIS-rdma installation failed $($job.RoleName)"
|
|
#$lisInstallErrorCount += 1
|
|
}
|
|
}
|
|
|
|
}
|
|
if ( $LIS4InstallJobsRunning )
|
|
{
|
|
WaitFor -seconds 10
|
|
}
|
|
#else
|
|
#{
|
|
# if ( $lisInstallErrorCount -ne 0 )
|
|
# {
|
|
# Throw "LIS-rdma installation failed for some VMs.Aborting Test."
|
|
# }
|
|
#}
|
|
}
|
|
|
|
$isRestarted = RestartAllDeployments -allVMData $allVMData
|
|
if ( ! $isRestarted )
|
|
{
|
|
Throw "Failed to restart deployments in $isDeployed. Aborting Test."
|
|
}
|
|
}
|
|
else
|
|
{
|
|
LogMsg "RDMA LIS Drivers are already installed."
|
|
LogMsg $modinfo_hv_vmbus
|
|
Add-Content -Value "rdmaPrepare=`"no`"" -Path $constantsFile
|
|
LogMsg "rdmaPrepare=no added to constansts.sh"
|
|
}
|
|
#endregion
|
|
|
|
RemoteCopy -uploadTo $serverVMData.PublicIP -port $serverVMData.SSHPort -files "$constantsFile" -username "root" -password $password -upload
|
|
$out = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "chmod +x *.sh"
|
|
if ( $modinfo_hv_vmbus -imatch "microsoft-hyper-v-rdma" )
|
|
{
|
|
$testOut = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "/root/StartRDMA.sh"
|
|
}
|
|
else
|
|
{
|
|
#region EXECUTE TEST
|
|
$testJob = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "/root/StartRDMA.sh" -RunInBackground
|
|
#endregion
|
|
|
|
#region MONITOR TEST
|
|
while ( (Get-Job -Id $testJob).State -eq "Running" )
|
|
{
|
|
$currentStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "tail -n 1 /root/rdmaConsole.txt"
|
|
LogMsg "Current Test Staus : $currentStatus"
|
|
WaitFor -seconds 10
|
|
}
|
|
}
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/rdmaConsole.txt"
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/summary.log"
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/pingPongTestIntraNodeTestOut.txt"
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/pingPongTestInterNodeTestOut.txt"
|
|
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/var/log/waagent.log"
|
|
$out= RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "dmesg > /var/log/dmesg.txt"
|
|
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/var/log/dmesg.txt"
|
|
$finalStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "cat /root/state.txt"
|
|
$rdmaSummary = Get-Content -Path "$LogDir\summary.log" -ErrorAction SilentlyContinue
|
|
|
|
if ($finalStatus -imatch "TestCompleted")
|
|
{
|
|
LogMsg "Test finished successfully."
|
|
LogMsg "PINGPONG INTRA NODE TEST OUTPUT-"
|
|
$pingPongTestIntraNodeTestOut = ( Get-Content -Path "$LogDir\pingPongTestIntraNodeTestOut.txt" | Out-String )
|
|
LogMsg $pingPongTestIntraNodeTestOut
|
|
LogMsg "PINGPONG INTER NODE TEST OUTPUT-"
|
|
$pingPongTestInterNodeTestOut = ( Get-Content -Path "$LogDir\pingPongTestInterNodeTestOut.txt" | Out-String )
|
|
LogMsg $pingPongTestInterNodeTestOut
|
|
}
|
|
else
|
|
{
|
|
LogErr "Test did not finished successfully. Please check $LogDir\rdmaConsole.txt for detailed results."
|
|
}
|
|
#endregion
|
|
|
|
|
|
if ( $finalStatus -imatch "TestFailed")
|
|
{
|
|
LogErr "Test failed. Last known status : $currentStatus."
|
|
$testResult = "FAIL"
|
|
}
|
|
elseif ( $finalStatus -imatch "TestAborted")
|
|
{
|
|
LogErr "Test Aborted. Last known status : $currentStatus."
|
|
$testResult = "ABORTED"
|
|
}
|
|
elseif ( $finalStatus -imatch "TestCompleted")
|
|
{
|
|
LogMsg "Test Completed. Result : $finalStatus."
|
|
$testResult = "PASS"
|
|
}
|
|
elseif ( $finalStatus -imatch "TestRunning")
|
|
{
|
|
LogMsg "Powershell backgroud job for test is completed but VM is reporting that test is still running. Please check $LogDir\mdConsoleLogs.txt"
|
|
LogMsg "Contests of state.txt : $finalStatus"
|
|
$testResult = "PASS"
|
|
}
|
|
LogMsg "Test result : $testResult"
|
|
LogMsg "Test Completed"
|
|
}
|
|
catch
|
|
{
|
|
$ErrorMessage = $_.Exception.Message
|
|
LogMsg "EXCEPTION : $ErrorMessage"
|
|
}
|
|
Finally
|
|
{
|
|
$metaData = "PingPong"
|
|
if (!$testResult)
|
|
{
|
|
$testResult = "Aborted"
|
|
}
|
|
$resultArr += $testResult
|
|
$resultSummary += CreateResultSummary -testResult $finalStatus -metaData $metaData -checkValues "PASS,FAIL,ABORTED" -testName $currentTestData.testName# if you want to publish all result then give here all test status possibilites. if you want just failed results, then give here just "FAIL". You can use any combination of PASS FAIL ABORTED and corresponding test results will be published!
|
|
}
|
|
}
|
|
|
|
else
|
|
{
|
|
$testResult = "Aborted"
|
|
$resultArr += $testResult
|
|
}
|
|
|
|
$result = GetFinalResultHeader -resultarr $resultArr
|
|
|
|
#Clean up the setup
|
|
DoTestCleanUp -result $result -testName $currentTestData.testName -deployedServices $isDeployed -ResourceGroups $isDeployed
|
|
|
|
#Return the result and summery to the test suite script..
|
|
return $result, $resultSummary
|