Added 16 VM RDMA Test
+Fixes for 2VM RDMA test.
This commit is contained in:
Shital Savekar
Родитель
e69a2777af
Коммит
21157daa95
|
|
@ -700,7 +700,7 @@ Function GenerateCommand ($Setup, $serviceName, $osImage, $HSData)
|
|||
return $createSetupCommand, $serviceName, $vmCount
|
||||
}
|
||||
|
||||
Function CreateDeployment ($DeploymentCommand, $NewServiceName , $vmCount, [string]$storageaccount="")
|
||||
Function CreateDeployment ($DeploymentCommand, $NewServiceName , $vmCount, [string]$storageaccount="", $timeOutSeconds)
|
||||
{
|
||||
|
||||
$FailCounter = 0
|
||||
|
|
@ -710,7 +710,7 @@ Function CreateDeployment ($DeploymentCommand, $NewServiceName , $vmCount, [stri
|
|||
try
|
||||
{
|
||||
$FailCounter++
|
||||
$out = RunAzureCmd -AzureCmdlet "$DeploymentCommand" -storageaccount $storageaccount
|
||||
$out = RunAzureCmd -AzureCmdlet "$DeploymentCommand" -storageaccount $storageaccount -maxWaitTimeSeconds $timeOutSeconds
|
||||
#LogMsg $DeploymentCommand
|
||||
$retValue = $?
|
||||
LogMsg "VM's deployed. Verifying.."
|
||||
|
|
@ -803,7 +803,7 @@ Function CheckVMsInService($serviceName)
|
|||
return $allVMsReady
|
||||
}
|
||||
|
||||
Function CreateAllDeployments($setupType, $xmlConfig, $Distro, [string]$region ="", [string]$storageAccount="")
|
||||
Function CreateAllDeployments($setupType, $xmlConfig, $Distro, [string]$region ="", [string]$storageAccount="", $timeOutSeconds)
|
||||
{
|
||||
|
||||
$hostedServiceCount = 0
|
||||
|
|
@ -881,7 +881,7 @@ Function CreateAllDeployments($setupType, $xmlConfig, $Distro, [string]$region =
|
|||
$DeploymentCommand = GenerateCommand -Setup $Setup -serviceName $serviceName -osImage $img -HSData $HS
|
||||
Set-AzureSubscription -SubscriptionName $xmlConfig.config.Azure.General.SubscriptionName -CurrentStorageAccountName $currentStorageAccount
|
||||
$DeploymentStartTime = (Get-Date)
|
||||
$isDeployed = CreateDeployment -DeploymentCommand $DeploymentCommand[0] -NewServiceName $DeploymentCommand[1] -vmCount $DeploymentCommand[2] -storageaccount $currentStorageAccount
|
||||
$isDeployed = CreateDeployment -DeploymentCommand $DeploymentCommand[0] -NewServiceName $DeploymentCommand[1] -vmCount $DeploymentCommand[2] -storageaccount $currentStorageAccount -timeOutSeconds $timeOutSeconds
|
||||
$DeploymentEndTime = (Get-Date)
|
||||
$DeploymentElapsedTime = $DeploymentEndTime - $DeploymentStartTime
|
||||
if ( $isDeployed -eq "True" )
|
||||
|
|
@ -1254,7 +1254,7 @@ Function SetDistroSpecificVariables($detectedDistro)
|
|||
}
|
||||
}
|
||||
|
||||
Function DeployManagementServices ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false, $GetDeploymentStatistics = $false, [string]$region ="", [string]$storageAccount="")
|
||||
Function DeployManagementServices ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false, $GetDeploymentStatistics = $false, [string]$region ="", [string]$storageAccount="",[int]$timeOutSeconds)
|
||||
{
|
||||
if( (!$EconomyMode) -or ( $EconomyMode -and ($xmlConfig.config.Azure.Deployment.$setupType.isDeployed -eq "NO")))
|
||||
{
|
||||
|
|
@ -1267,7 +1267,7 @@ Function DeployManagementServices ($xmlConfig, $setupType, $Distro, $getLogsIfFa
|
|||
$i = 0
|
||||
$role = 1
|
||||
$setupTypeData = $xmlConfig.config.Azure.Deployment.$setupType
|
||||
$isAllDeployed = CreateAllDeployments -xmlConfig $xmlConfig -setupType $setupType -Distro $Distro -region $region -storageAccount $storageAccount
|
||||
$isAllDeployed = CreateAllDeployments -xmlConfig $xmlConfig -setupType $setupType -Distro $Distro -region $region -storageAccount $storageAccount -timeOutSeconds $timeOutSeconds
|
||||
$isAllVerified = "False"
|
||||
$isAllConnected = "False"
|
||||
if($isAllDeployed[0] -eq "True")
|
||||
|
|
@ -1366,7 +1366,7 @@ Function DeployManagementServices ($xmlConfig, $setupType, $Distro, $getLogsIfFa
|
|||
}
|
||||
}
|
||||
|
||||
Function DeployVMs ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false, $GetDeploymentStatistics = $false, [string]$region = "", [string]$storageAccount = "")
|
||||
Function DeployVMs ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false, $GetDeploymentStatistics = $false, [string]$region = "", [string]$storageAccount = "", [int]$timeOutSeconds = 600)
|
||||
{
|
||||
$AzureSetup = $xmlConfig.config.Azure.General
|
||||
|
||||
|
|
@ -1384,7 +1384,7 @@ Function DeployVMs ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false,
|
|||
{
|
||||
LogMsg "CurrentStorageAccount : $($storageAccount)"
|
||||
}
|
||||
$retValue = DeployManagementServices -xmlConfig $xmlConfig -setupType $setupType -Distro $Distro -getLogsIfFailed $getLogsIfFailed -GetDeploymentStatistics $GetDeploymentStatistics -region $region -storageAccount $storageAccount
|
||||
$retValue = DeployManagementServices -xmlConfig $xmlConfig -setupType $setupType -Distro $Distro -getLogsIfFailed $getLogsIfFailed -GetDeploymentStatistics $GetDeploymentStatistics -region $region -storageAccount $storageAccount -timeOutSeconds $timeOutSeconds
|
||||
}
|
||||
return $retValue
|
||||
}
|
||||
|
|
@ -1958,7 +1958,15 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
|
|||
LogMsg "Removing compressed file : $tarFileName"
|
||||
Remove-Item -Path $tarFileName -Force 2>&1 | Out-Null
|
||||
LogMsg "Decompressing files in VM ..."
|
||||
$out = RunLinuxCmd -username $username -password $password -ip $uploadTo -port $port -command "tar -xf $tarFileName" -runAsSudo
|
||||
if ( $username -eq "root" )
|
||||
{
|
||||
$out = RunLinuxCmd -username $username -password $password -ip $uploadTo -port $port -command "tar -xf $tarFileName"
|
||||
}
|
||||
else
|
||||
{
|
||||
$out = RunLinuxCmd -username $username -password $password -ip $uploadTo -port $port -command "tar -xf $tarFileName" -runAsSudo
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,370 @@
|
|||
#author - vhisav@microsoft.com
|
||||
Import-Module .\TestLibs\RDFELibs.psm1 -Force
|
||||
$result = ""
|
||||
$testResult = ""
|
||||
$resultArr = @()
|
||||
$isDeployed = DeployVMS -setupType $currentTestData.setupType -Distro $Distro -xmlConfig $xmlConfig -timeOutSeconds 4200
|
||||
if ($isDeployed)
|
||||
{
|
||||
try
|
||||
{
|
||||
$allVMData = GetAllDeployementData -DeployedServices $isDeployed
|
||||
$noServer = $true
|
||||
$noClient = $true
|
||||
$clientMachines = @()
|
||||
$slaveHostnames = ""
|
||||
foreach ( $vmData in $allVMData )
|
||||
{
|
||||
if (( $vmData.RoleName -imatch "Server" ) -or ( $vmData.RoleName -imatch "master" ))
|
||||
{
|
||||
$serverVMData = $vmData
|
||||
$noMaster = $false
|
||||
|
||||
}
|
||||
elseif (( $vmData.RoleName -imatch "Client" ) -or ( $vmData.RoleName -imatch "slave" ))
|
||||
{
|
||||
$clientMachines = $vmData
|
||||
$noSlave = $fase
|
||||
if ( $slaveHostnames )
|
||||
{
|
||||
$slaveHostnames += "," + $vmData.RoleName
|
||||
}
|
||||
else
|
||||
{
|
||||
$slaveHostnames = $vmData.RoleName
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( $noMaster )
|
||||
{
|
||||
Throw "No any master VM defined. Be sure that, server VM role name matches with the pattern `"*master*`". Aborting Test."
|
||||
}
|
||||
if ( $noSlave )
|
||||
{
|
||||
Throw "No any slave VM defined. Be sure that, client machine role names matches with pattern `"*slave*`" Aborting Test."
|
||||
}
|
||||
#region CONFIGURE VMs for TEST
|
||||
|
||||
|
||||
LogMsg "SERVER VM details :"
|
||||
LogMsg " RoleName : $($serverVMData.RoleName)"
|
||||
LogMsg " Public IP : $($serverVMData.PublicIP)"
|
||||
LogMsg " SSH Port : $($serverVMData.SSHPort)"
|
||||
$i = 1
|
||||
foreach ( $clientVMData in $clientMachines )
|
||||
{
|
||||
LogMsg "CLIENT VM #$i details :"
|
||||
LogMsg " RoleName : $($clientVMData.RoleName)"
|
||||
LogMsg " Public IP : $($clientVMData.PublicIP)"
|
||||
LogMsg " SSH Port : $($clientVMData.SSHPort)"
|
||||
}
|
||||
#
|
||||
# PROVISION VMS FOR LISA WILL ENABLE ROOT USER AND WILL MAKE ENABLE PASSWORDLESS AUTHENTICATION ACROSS ALL VMS IN SAME HOSTED SERVICE.
|
||||
#
|
||||
|
||||
ProvisionVMsForLisa -allVMData $allVMData -installPackagesOnRoleNames "none"
|
||||
|
||||
#endregion
|
||||
|
||||
#region Provision VMs for RDMA tests
|
||||
|
||||
#region Generate constants.sh
|
||||
|
||||
LogMsg "Generating constansts.sh ..."
|
||||
$constantsFile = ".\$LogDir\constants.sh"
|
||||
|
||||
Set-Content -Value "master=`"$($serverVMData.RoleName)`"" -Path $constantsFile
|
||||
LogMsg "master=$($serverVMData.RoleName) added to constansts.sh"
|
||||
|
||||
|
||||
Add-Content -Value "slaves=`"$slaveHostnames`"" -Path $constantsFile
|
||||
LogMsg "slaves=$slaveHostnames added to constansts.sh"
|
||||
|
||||
Add-Content -Value "rdmaPrepare=`"yes`"" -Path $constantsFile
|
||||
LogMsg "rdmaPrepare=yes added to constansts.sh"
|
||||
|
||||
Add-Content -Value "rdmaRun=`"no`"" -Path $constantsFile
|
||||
LogMsg "rdmaRun=no added to constansts.sh"
|
||||
|
||||
Add-Content -Value "installLocal=`"yes`"" -Path $constantsFile
|
||||
LogMsg "installLocal=yes added to constansts.sh"
|
||||
|
||||
LogMsg "constanst.sh created successfully..."
|
||||
#endregion
|
||||
|
||||
#region Generate etc-hosts.txt file
|
||||
$hostsFile = ".\$LogDir\etc-hosts.txt"
|
||||
foreach ( $vmDetails in $allVMData )
|
||||
{
|
||||
Add-Content -Value "$($vmDetails.InternalIP)`t$($vmDetails.RoleName)" -Path "$hostsFile"
|
||||
LogMsg "$($vmDetails.InternalIP)`t$($vmDetails.RoleName) added to etc-hosts.txt"
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Set contents of limits.conf...
|
||||
Set-Content -Value "/root/TestRDMA.sh &> rdmaConsole.txt" -Path "$LogDir\StartRDMA.sh"
|
||||
Set-Content -Value "* hard memlock unlimited" -Path "$LogDir\limits.conf"
|
||||
Add-Content -Value "* soft memlock unlimited" -Path "$LogDir\limits.conf"
|
||||
RemoteCopy -uploadTo $serverVMData.PublicIP -port $serverVMData.SSHPort -files "$constantsFile,$hostsFile,.\remote-scripts\TestRDMA.sh,.\$LogDir\StartRDMA.sh,.\$LogDir\limits.conf" -username "root" -password $password -upload
|
||||
#endregion
|
||||
|
||||
#region Install LIS-RDMA drivers..
|
||||
|
||||
$osRelease = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "cat /etc/*release*"
|
||||
if ( $osRelease -imatch "CentOS Linux release 7.1.")
|
||||
{
|
||||
$LIS4folder = "RHEL71"
|
||||
}
|
||||
if ( $osRelease -imatch "CentOS Linux release 7.0.")
|
||||
{
|
||||
$LIS4folder = "RHEL70"
|
||||
}
|
||||
if ( $osRelease -imatch "CentOS Linux release 6.5")
|
||||
{
|
||||
$LIS4folder = "RHEL65"
|
||||
}
|
||||
|
||||
$lisRDMAFileUrl = "https://ciwestus.blob.core.windows.net/linuxbinaries/lis-4.0.11-RDMA.tar"
|
||||
Set-Content -Value "tar -xf lis-4.0.11-RDMA.tar" -Path "$LogDir\InstallLIS.sh"
|
||||
Add-Content -Value "chmod +x $LIS4folder/install.sh" -Path "$LogDir\InstallLIS.sh"
|
||||
Add-Content -Value "cd $LIS4folder" -Path "$LogDir\InstallLIS.sh"
|
||||
Add-Content -Value "./install.sh > /root/LIS4InstallStatus.txt 2>&1" -Path "$LogDir\InstallLIS.sh"
|
||||
$LIS4IntallCommand = "./InstallLIS.sh"
|
||||
$LIS4InstallJobs = @()
|
||||
foreach ( $vm in $allVMData )
|
||||
{
|
||||
#Install LIS4 RDMA drivers...
|
||||
LogMsg "Setting contents of /etc/security/limits.conf..."
|
||||
$out = .\tools\dos2unix.exe ".\$LogDir\limits.conf" 2>&1
|
||||
LogMsg $out
|
||||
RemoteCopy -uploadTo $vm.PublicIP -port $vm.SSHPort -files ".\$LogDir\limits.conf,.\$LogDir\InstallLIS.sh" -username "root" -password $password -upload
|
||||
$out = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "cat limits.conf >> /etc/security/limits.conf"
|
||||
|
||||
LogMsg "Downlaoding LIS-RDMA drivers in $($vm.RoleName)..."
|
||||
$out = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "wget $lisRDMAFileUrl"
|
||||
$out = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "chmod +x InstallLIS.sh"
|
||||
LogMsg "Executing $LIS4IntallCommand ..."
|
||||
$jobID = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "$LIS4IntallCommand" -RunInBackground
|
||||
$LIS4InstallObj = New-Object PSObject
|
||||
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name ID -Value $jobID
|
||||
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name RoleName -Value $vm.RoleName
|
||||
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name PublicIP -Value $vm.PublicIP
|
||||
Add-member -InputObject $LIS4InstallObj -MemberType NoteProperty -Name SSHPort -Value $vm.SSHPort
|
||||
$LIS4InstallJobs += $LIS4InstallObj
|
||||
}
|
||||
|
||||
#Monitor LIS installation...
|
||||
$LIS4InstallJobsRunning = $true
|
||||
$lisInstallErrorCount = 0
|
||||
while ($LIS4InstallJobsRunning)
|
||||
{
|
||||
$LIS4InstallJobsRunning = $false
|
||||
foreach ( $job in $LIS4InstallJobs )
|
||||
{
|
||||
if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
|
||||
{
|
||||
LogMsg "lis-4.0.11-RDMA Installation Status for $($job.RoleName) : Running"
|
||||
$LIS4InstallJobsRunning = $true
|
||||
}
|
||||
else
|
||||
{
|
||||
$jobOut = Receive-Job -ID $($job.ID)
|
||||
$LIS4out = RunLinuxCmd -ip $job.PublicIP -port $job.SSHPort -username "root" -password $password -command "cat LIS4InstallStatus.txt"
|
||||
if ( $LIS4out -imatch "Please reboot your system")
|
||||
{
|
||||
LogMsg "lis-4.0.11-RDMA installed successfully for $($job.RoleName)"
|
||||
}
|
||||
else
|
||||
{
|
||||
#LogErr "LIS-rdma installation failed $($job.RoleName)"
|
||||
#$lisInstallErrorCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if ( $LIS4InstallJobsRunning )
|
||||
{
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
#else
|
||||
#{
|
||||
# if ( $lisInstallErrorCount -ne 0 )
|
||||
# {
|
||||
# Throw "LIS-rdma installation failed for some VMs.Aborting Test."
|
||||
# }
|
||||
#}
|
||||
}
|
||||
|
||||
$isRestarted = RestartAllDeployments -allVMData $allVMData
|
||||
if ( ! $isRestarted )
|
||||
{
|
||||
Throw "Failed to restart deployments in $isDeployed. Aborting Test."
|
||||
}
|
||||
#endregion
|
||||
|
||||
#region Prepare VMs for test
|
||||
$packageInstallJobs = @()
|
||||
Set-Content -Value "/root/TestRDMA.sh &> prepareForRDMAConsole.txt" -Path "$LogDir\PrepareForRDMA.sh"
|
||||
$packageIntallCommand = "/root/PrepareForRDMA.sh"
|
||||
foreach ( $vm in $allVMData )
|
||||
{
|
||||
#Install Intel and IBM MPI libraries...
|
||||
RemoteCopy -uploadTo $vm.PublicIP -port $vm.SSHPort -files "$constantsFile,$hostsFile,.\remote-scripts\TestRDMA.sh,.\$LogDir\PrepareForRDMA.sh" -username "root" -password $password -upload
|
||||
$jobID = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "chmod +x PrepareForRDMA.sh"
|
||||
LogMsg "Executing $packageIntallCommand ..."
|
||||
|
||||
$jobID = RunLinuxCmd -ip $vm.PublicIP -port $vm.SSHPort -username "root" -password $password -command "$packageIntallCommand" -RunInBackground
|
||||
$packageInstallObj = New-Object PSObject
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name ID -Value $jobID
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name RoleName -Value $vm.RoleName
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name PublicIP -Value $vm.PublicIP
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name SSHPort -Value $vm.SSHPort
|
||||
$packageInstallJobs += $packageInstallObj
|
||||
}
|
||||
|
||||
$packageInstallJobsRunning = $true
|
||||
$packageInstallErrorCount = 0
|
||||
while ($packageInstallJobsRunning)
|
||||
{
|
||||
$packageInstallJobsRunning = $false
|
||||
foreach ( $job in $packageInstallJobs )
|
||||
{
|
||||
if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
|
||||
{
|
||||
LogMsg "RDMA preparation Status for $($job.RoleName) : Running"
|
||||
$packageInstallJobsRunning = $true
|
||||
}
|
||||
else
|
||||
{
|
||||
$jobOut = Receive-Job -ID $($job.ID)
|
||||
if ( $jobOut -imatch "Please reboot your system")
|
||||
{
|
||||
LogMsg "RDMA preparation completed for $($job.RoleName)"
|
||||
}
|
||||
else
|
||||
{
|
||||
#LogErr "RDMA preparation failed $($job.RoleName)"
|
||||
#$packageInstallErrorCount += 1
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if ( $packageInstallJobsRunning )
|
||||
{
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
#else
|
||||
#{
|
||||
# if ( $packageInstallErrorCount -ne 0 )
|
||||
# {
|
||||
# Throw "RDMA preparation failed for some VMs.Aborting Test."
|
||||
# }
|
||||
#}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
Set-Content -Value "master=`"$($serverVMData.RoleName)`"" -Path $constantsFile
|
||||
LogMsg "master=$($serverVMData.RoleName) added to constansts.sh"
|
||||
|
||||
|
||||
Add-Content -Value "slaves=`"$slaveHostnames`"" -Path $constantsFile
|
||||
LogMsg "slaves=$slaveHostnames added to constansts.sh"
|
||||
|
||||
Add-Content -Value "rdmaPrepare=`"no`"" -Path $constantsFile
|
||||
LogMsg "rdmaPrepare=no added to constansts.sh"
|
||||
|
||||
Add-Content -Value "rdmaRun=`"yes`"" -Path $constantsFile
|
||||
LogMsg "rdmaRun=yes added to constansts.sh"
|
||||
|
||||
Add-Content -Value "installLocal=`"no`"" -Path $constantsFile
|
||||
LogMsg "installLocal=no added to constansts.sh"
|
||||
|
||||
LogMsg "constanst.sh created successfully..."
|
||||
|
||||
Set-Content -Value "/root/TestRDMA.sh &> rdmaConsole.txt" -Path "$LogDir\StartRDMA.sh"
|
||||
|
||||
#region EXECUTE TEST
|
||||
RemoteCopy -uploadTo $serverVMData.PublicIP -port $serverVMData.SSHPort -files "$constantsFile,$LogDir\StartRDMA.sh" -username "root" -password $password -upload
|
||||
$out = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "chmod +x *.sh"
|
||||
$testJob = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "/root/StartRDMA.sh" -RunInBackground
|
||||
#endregion
|
||||
|
||||
#region MONITOR TEST
|
||||
while ( (Get-Job -Id $testJob).State -eq "Running" )
|
||||
{
|
||||
$currentStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "tail -n 1 /root/rdmaConsole.txt"
|
||||
LogMsg "Current Test Staus : $currentStatus"
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
|
||||
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/rdmaConsole.txt"
|
||||
RemoteCopy -downloadFrom $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/summary.log"
|
||||
$finalStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "cat /root/state.txt"
|
||||
$rdmaSummary = Get-Content -Path "$LogDir\summary.log" -ErrorAction SilentlyContinue
|
||||
|
||||
if ($finalStatus -imatch "TestCompleted")
|
||||
{
|
||||
LogMsg "Test finished successfully. Please check $LogDir\rdmaConsole.txt for detailed results."
|
||||
}
|
||||
else
|
||||
{
|
||||
LogErr "Test did not finished successfully. Please check $LogDir\rdmaConsole.txt for detailed results."
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
||||
if ( $finalStatus -imatch "TestFailed")
|
||||
{
|
||||
LogErr "Test failed. Last known status : $currentStatus."
|
||||
$testResult = "FAIL"
|
||||
}
|
||||
elseif ( $finalStatus -imatch "TestAborted")
|
||||
{
|
||||
LogErr "Test Aborted. Last known status : $currentStatus."
|
||||
$testResult = "ABORTED"
|
||||
}
|
||||
elseif ( $finalStatus -imatch "TestCompleted")
|
||||
{
|
||||
LogMsg "Test Completed. Result : $finalStatus."
|
||||
$testResult = "PASS"
|
||||
}
|
||||
elseif ( $finalStatus -imatch "TestRunning")
|
||||
{
|
||||
LogMsg "Powershell backgroud job for test is completed but VM is reporting that test is still running. Please check $LogDir\mdConsoleLogs.txt"
|
||||
LogMsg "Contests of state.txt : $finalStatus"
|
||||
$testResult = "PASS"
|
||||
}
|
||||
LogMsg "Test result : $testResult"
|
||||
LogMsg "Test Completed"
|
||||
}
|
||||
catch
|
||||
{
|
||||
$ErrorMessage = $_.Exception.Message
|
||||
LogMsg "EXCEPTION : $ErrorMessage"
|
||||
}
|
||||
Finally
|
||||
{
|
||||
$metaData = "Status"
|
||||
if (!$testResult)
|
||||
{
|
||||
$testResult = "Aborted"
|
||||
}
|
||||
$resultArr += $testResult
|
||||
$resultSummary += CreateResultSummary -testResult $finalStatus -metaData $metaData -checkValues "PASS,FAIL,ABORTED" -testName $currentTestData.testName# if you want to publish all result then give here all test status possibilites. if you want just failed results, then give here just "FAIL". You can use any combination of PASS FAIL ABORTED and corresponding test results will be published!
|
||||
}
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
$testResult = "Aborted"
|
||||
$resultArr += $testResult
|
||||
}
|
||||
|
||||
$result = GetFinalResultHeader -resultarr $resultArr
|
||||
|
||||
#Clean up the setup
|
||||
DoTestCleanUp -result $result -testName $currentTestData.testName -deployedServices $isDeployed -ResourceGroups $isDeployed
|
||||
|
||||
#Return the result and summery to the test suite script..
|
||||
return $result, $resultSummary
|
||||
|
|
@ -207,7 +207,7 @@ if ($isDeployed)
|
|||
#region MONITOR TEST
|
||||
while ( (Get-Job -Id $testJob).State -eq "Running" )
|
||||
{
|
||||
$currentStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "tail -n 1 /root/rdmaConsole.txtq"
|
||||
$currentStatus = RunLinuxCmd -ip $serverVMData.PublicIP -port $serverVMData.SSHPort -username "root" -password $password -command "tail -n 1 /root/rdmaConsole.txt"
|
||||
LogMsg "Current Test Staus : $currentStatus"
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
|
|
|
|||
|
|
@ -218,19 +218,21 @@ else
|
|||
exit 10
|
||||
fi
|
||||
|
||||
if [ ! ${master} ]; then
|
||||
errMsg="Please add/provide value for master in constants.sh. master=<Master VM hostname>"
|
||||
LogMsg "${errMsg}"
|
||||
echo "${errMsg}" >> ~/summary.log
|
||||
UpdateTestState $ICA_TESTABORTED
|
||||
exit 1
|
||||
fi
|
||||
if [ ! ${slaves} ]; then
|
||||
errMsg="Please add/provide value for slaves in constants.sh. slaves=<hostname1,hostname2,hostname3>"
|
||||
LogMsg "${errMsg}"
|
||||
echo "${errMsg}" >> ~/summary.log
|
||||
UpdateTestState $ICA_TESTABORTED
|
||||
exit 1
|
||||
if [ ${installLocal} != "yes" ]; then
|
||||
if [ ! ${master} ]; then
|
||||
errMsg="Please add/provide value for master in constants.sh. master=<Master VM hostname>"
|
||||
LogMsg "${errMsg}"
|
||||
echo "${errMsg}" >> ~/summary.log
|
||||
UpdateTestState $ICA_TESTABORTED
|
||||
exit 1
|
||||
fi
|
||||
if [ ! ${slaves} ]; then
|
||||
errMsg="Please add/provide value for slaves in constants.sh. slaves=<hostname1,hostname2,hostname3>"
|
||||
LogMsg "${errMsg}"
|
||||
echo "${errMsg}" >> ~/summary.log
|
||||
UpdateTestState $ICA_TESTABORTED
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${rdmaPrepare}" != "yes" ] && [ "${rdmaPrepare}" != "no" ]; then
|
||||
|
|
@ -253,27 +255,39 @@ slavesArr=`echo ${slaves} | tr ',' ' '`
|
|||
if [ "${rdmaPrepare}" == "yes" ]; then
|
||||
|
||||
# master VM preparation for RDMA tests...
|
||||
ssh root@${slave} test -e rdmaPrepared
|
||||
if [ $? -ne 0 ]; then
|
||||
LogMsg "Info : Running config on master : '${master}'"
|
||||
PrepareForRDMA "${master}"
|
||||
else
|
||||
LogMsg "${master} is already prepared for RDMA tests."
|
||||
fi
|
||||
|
||||
# slave VMs preparation for RDMA tests...
|
||||
for slave in $slavesArr
|
||||
do
|
||||
ssh root@${slave} test -e rdmaPrepared
|
||||
|
||||
if [ ${installLocal} == "yes" ]; then
|
||||
localVM=`hostname`
|
||||
ssh root@${localVM} test -e rdmaPrepared
|
||||
if [ $? -ne 0 ]; then
|
||||
LogMsg "Info : Running config on slave '${slave}'"
|
||||
PrepareForRDMA "${slave}"
|
||||
LogMsg "Info : mounting ${master}:/mirror NFS directory to /mirror on '${slave}'"
|
||||
ssh root@${slave} mount ${master}:/mirror /mirror
|
||||
LogMsg "Info : Running config on current machine : '${localVM}'"
|
||||
PrepareForRDMA "${localVM}"
|
||||
else
|
||||
LogMsg "${slave} is already prepared for RDMA tests."
|
||||
LogMsg "${localVM} is already prepared for RDMA tests."
|
||||
fi
|
||||
done
|
||||
else
|
||||
ssh root@${master} test -e rdmaPrepared
|
||||
if [ $? -ne 0 ]; then
|
||||
LogMsg "Info : Running config on master : '${master}'"
|
||||
PrepareForRDMA "${master}"
|
||||
else
|
||||
LogMsg "${master} is already prepared for RDMA tests."
|
||||
fi
|
||||
|
||||
# slave VMs preparation for RDMA tests...
|
||||
for slave in $slavesArr
|
||||
do
|
||||
ssh root@${slave} test -e rdmaPrepared
|
||||
if [ $? -ne 0 ]; then
|
||||
LogMsg "Info : Running config on slave '${slave}'"
|
||||
PrepareForRDMA "${slave}"
|
||||
LogMsg "Info : mounting ${master}:/mirror NFS directory to /mirror on '${slave}'"
|
||||
ssh root@${slave} mount ${master}:/mirror /mirror
|
||||
else
|
||||
LogMsg "${slave} is already prepared for RDMA tests."
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
LogMsg "Info : Skipping RDMA preparation. (Source : constants.sh)"
|
||||
fi
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче