Automation Fixes

+ Fixed enableRoot.sh for Ubuntu 1404
+ Fixed InstallCustomKernel using try catch.
+ RunLinuxCmd(): Continue for next attempt in case of timeout.

TestLibs/LISALibrary.psm1

@@ -139,89 +139,100 @@ Function ProvisionVMsForLisa($allVMData, $installPackagesOnRoleNames)
 
 function InstallCustomKernel ($customKernel, $allVMData, [switch]$RestartAfterUpgrade)
 {
-    $customKernel = $customKernel.Trim()
+    try
-    if( ($customKernel -ne "linuxnext") -and ($customKernel -ne "netnext") )
     {
-        Throw "Only linuxnext and netnext version is supported. Other version will be added soon. Use -customKernel linuxnext"
+        $customKernel = $customKernel.Trim()
-    }
+        if( ($customKernel -ne "linuxnext") -and ($customKernel -ne "netnext") )
-    $scriptName = "customKernelInstall.sh"
-    $jobCount = 0
-    $kernelSuccess = 0
-	$packageInstallJobs = @()
-	foreach ( $vmData in $allVMData )
-	{
-        RemoteCopy -uploadTo $vmData.PublicIP -port $vmData.SSHPort -files ".\remote-scripts\$scriptName,.\SetupScripts\DetectLinuxDistro.sh" -username $user -password $password -upload
-        $out = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "chmod +x *.sh" -runAsSudo
-        $currentKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
-		LogMsg "Executing $scriptName ..."
-		$jobID = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "/home/$user/$scriptName -customKernel $customKernel" -RunInBackground -runAsSudo
-		$packageInstallObj = New-Object PSObject
-		Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name ID -Value $jobID
-		Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name RoleName -Value $vmData.RoleName
-		Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name PublicIP -Value $vmData.PublicIP
-		Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name SSHPort -Value $vmData.SSHPort
-		$packageInstallJobs += $packageInstallObj
-        $jobCount += 1
-		#endregion
-	}
-	
-	$packageInstallJobsRunning = $true
-	while ($packageInstallJobsRunning)
-	{
-		$packageInstallJobsRunning = $false
-		foreach ( $job in $packageInstallJobs )
-		{
-			if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
-			{
-				$currentStatus = RunLinuxCmd -ip $job.PublicIP -port $job.SSHPort -username $user -password $password -command "tail -n 1 build-customKernel.txt"
-				LogMsg "Package Installation Status for $($job.RoleName) : $currentStatus"
-				$packageInstallJobsRunning = $true
-			}
-			else
-			{
-                if ( !(Test-Path -Path "$LogDir\$($job.RoleName)-build-customKernel.txt" ) )
-                {
-				    RemoteCopy -download -downloadFrom $job.PublicIP -port $job.SSHPort -files "build-customKernel.txt" -username $user -password $password -downloadTo $LogDir
-                    if ( ( Get-Content "$LogDir\build-customKernel.txt" ) -imatch "CUSTOM_KERNEL_SUCCESS" )
-                    {
-                        $kernelSuccess += 1
-                    }
-				    Rename-Item -Path "$LogDir\build-customKernel.txt" -NewName "$($job.RoleName)-build-customKernel.txt" -Force | Out-Null
-                }
-			}
-		}
-		if ( $packageInstallJobsRunning )
-		{
-			WaitFor -seconds 10
-		}
-	}
-    
-    if ( $kernelSuccess -eq $jobCount )
-    {
-        LogMsg "Kernel upgraded to `"$customKernel`" successfully in all VMs."
-        if ( $RestartAfterUpgrade )
         {
-            LogMsg "Now restarting VMs..."
+            LogErr "Only linuxnext and netnext version is supported. Other version will be added soon. Use -customKernel linuxnext"
-            $restartStatus = RestartAllDeployments -allVMData $allVMData
+        }
-            if ( $restartStatus -eq "True")
+        else
+        {
+            $scriptName = "customKernelInstall.sh"
+            $jobCount = 0
+            $kernelSuccess = 0
+	        $packageInstallJobs = @()
+	        foreach ( $vmData in $allVMData )
+	        {
+                RemoteCopy -uploadTo $vmData.PublicIP -port $vmData.SSHPort -files ".\remote-scripts\$scriptName,.\SetupScripts\DetectLinuxDistro.sh" -username $user -password $password -upload
+                $out = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "chmod +x *.sh" -runAsSudo
+                $currentKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
+		        LogMsg "Executing $scriptName ..."
+		        $jobID = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "/home/$user/$scriptName -customKernel $customKernel" -RunInBackground -runAsSudo
+		        $packageInstallObj = New-Object PSObject
+		        Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name ID -Value $jobID
+		        Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name RoleName -Value $vmData.RoleName
+		        Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name PublicIP -Value $vmData.PublicIP
+		        Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name SSHPort -Value $vmData.SSHPort
+		        $packageInstallJobs += $packageInstallObj
+                $jobCount += 1
+		        #endregion
+	        }
+	
+	        $packageInstallJobsRunning = $true
+	        while ($packageInstallJobsRunning)
+	        {
+		        $packageInstallJobsRunning = $false
+		        foreach ( $job in $packageInstallJobs )
+		        {
+			        if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
+			        {
+				        $currentStatus = RunLinuxCmd -ip $job.PublicIP -port $job.SSHPort -username $user -password $password -command "tail -n 1 build-customKernel.txt"
+				        LogMsg "Package Installation Status for $($job.RoleName) : $currentStatus"
+				        $packageInstallJobsRunning = $true
+			        }
+			        else
+			        {
+                        if ( !(Test-Path -Path "$LogDir\$($job.RoleName)-build-customKernel.txt" ) )
+                        {
+				            RemoteCopy -download -downloadFrom $job.PublicIP -port $job.SSHPort -files "build-customKernel.txt" -username $user -password $password -downloadTo $LogDir
+                            if ( ( Get-Content "$LogDir\build-customKernel.txt" ) -imatch "CUSTOM_KERNEL_SUCCESS" )
+                            {
+                                $kernelSuccess += 1
+                            }
+				            Rename-Item -Path "$LogDir\build-customKernel.txt" -NewName "$($job.RoleName)-build-customKernel.txt" -Force | Out-Null
+                        }
+			        }
+		        }
+		        if ( $packageInstallJobsRunning )
+		        {
+			        WaitFor -seconds 10
+		        }
+	        }
+    
+            if ( $kernelSuccess -eq $jobCount )
             {
-                $upgradedKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
+                LogMsg "Kernel upgraded to `"$customKernel`" successfully in all VMs."
-                LogMsg "Old kernel: $currentKernelVersion"
+                if ( $RestartAfterUpgrade )
-                LogMsg "New kernel: $upgradedKernelVersion"
+                {
-                Add-Content -Value "Old kernel: $currentKernelVersion" -Path .\report\AdditionalInfo.html -Force
+                    LogMsg "Now restarting VMs..."
-                Add-Content -Value "New kernel: $upgradedKernelVersion" -Path .\report\AdditionalInfo.html -Force
+                    $restartStatus = RestartAllDeployments -allVMData $allVMData
+                    if ( $restartStatus -eq "True")
+                    {
+                        $upgradedKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
+                        LogMsg "Old kernel: $currentKernelVersion"
+                        LogMsg "New kernel: $upgradedKernelVersion"
+                        Add-Content -Value "Old kernel: $currentKernelVersion" -Path .\report\AdditionalInfo.html -Force
+                        Add-Content -Value "New kernel: $upgradedKernelVersion" -Path .\report\AdditionalInfo.html -Force
+                        return $true
+                    }
+                    else
+                    {
+                        return $false
+                    }
+                }
                 return $true
             }
             else
             {
+                LogErr "Kernel upgrade failed in $($jobCount-$kernelSuccess) VMs."
                 return $false
             }
         }
-        return $true
     }
-    else
+    catch
     {
-        LogErr "Kernel upgrade failed in $($jobCount-$kernelSuccess) VMs."
+        LogErr "Exception in InstallCustomKernel."
         return $false
     }
 }

TestLibs/RDFELibs.psm1

@@ -1968,6 +1968,7 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
 								sleep -Seconds 1
 								$uploadJobStatus = Get-Job -Id $uploadJob.Id
 							}
+							Write-Host ""
 							$returnCode = Get-Content -Path $uploadStatusRandomFile
 							Remove-Item -Force $uploadStatusRandomFile | Out-Null
 							Remove-Job -Id $uploadJob.Id -Force | Out-Null
@@ -2052,6 +2053,7 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
 								sleep -Seconds 1
 								$uploadJobStatus = Get-Job -Id $uploadJob.Id
 							}
+							Write-Host ""
 							$returnCode = Get-Content -Path $uploadStatusRandomFile
 							Remove-Item -Force $uploadStatusRandomFile | Out-Null
 							Remove-Job -Id $uploadJob.Id -Force | Out-Null
@@ -2143,9 +2145,20 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
 
 Function WrapperCommandsToFile([string] $username,[string] $password,[string] $ip,[string] $command, [int] $port)
 {
-	$command | out-file -encoding ASCII -filepath "$LogDir\runtest.sh"
+    if ( ( $lastLinuxCmd -eq $command) -and ($lastIP -eq $ip) -and ($lastPort -eq $port) -and ($lastUser -eq $username) )
-	RemoteCopy -upload -uploadTo $ip -username $username -port $port -password $password -files ".\$LogDir\runtest.sh"
+    {
-	del "$LogDir\runtest.sh"
+        #Skip upload if current command is same as last command.
+    }
+    else
+    {
+        Set-Variable -Name lastLinuxCmd -Value $command -Scope Global
+        Set-Variable -Name lastIP -Value $ip -Scope Global
+        Set-Variable -Name lastPort -Value $port -Scope Global
+        Set-Variable -Name lastUser -Value $username -Scope Global
+	    $command | out-file -encoding ASCII -filepath "$LogDir\runtest.sh"
+	    RemoteCopy -upload -uploadTo $ip -username $username -port $port -password $password -files ".\$LogDir\runtest.sh"
+	    del "$LogDir\runtest.sh"
+    }
 }
 
 Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string] $command, [int] $port, [switch]$runAsSudo, [Boolean]$WriteHostOnly, [Boolean]$NoLogsPlease, [switch]$ignoreLinuxExitCode, [int]$runMaxAllowedTime = 300, [switch]$RunInBackGround)
@@ -2447,7 +2460,7 @@ Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string]
 					if($timeOut)
 					{
 						$retValue = ""
-						Throw "Tmeout while executing command : $command"
+						LogErr "Tmeout while executing command : $command"
 					}
 					LogErr "Linux machine returned exit code : $($LinuxExitCode.Split("-")[4])"
 					if ($attemptswt -eq $maxRetryCount -and $attemptswot -eq $maxRetryCount)
@@ -2458,7 +2471,7 @@ Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string]
 					{
 						if ($notExceededTimeLimit)
 						{
-							LogMsg "Failed to execute : $command. Retrying..."
+							LogErr "Failed to execute : $command. Retrying..."
 						}
 					}
 				}

remote-scripts/enableRoot.sh

@@ -20,6 +20,10 @@ if [ $? == 0 ]; then
         echo "ROOT_PASSWRD_SET"
         service $sshdServiceName restart
         sshdServiceStatus=$?
+        if [ $sshdServiceStatus != 0 ]; then
+                service ssh restart
+                sshdServiceStatus=$?
+        fi
     else
         echo "$sshd_configFilePath verification failed for root login."
         echo "ROOT_PASSWORD_SET_SSHD_CONFIG_FAIL"