Automation Fixes
+ Fixed enableRoot.sh for Ubuntu 1404 + Fixed InstallCustomKernel using try catch. + RunLinuxCmd(): Continue for next attempt in case of timeout.
This commit is contained in:
Shital Savekar
Родитель
db2effcda6
Коммит
7ba12677ec
|
|
@ -139,89 +139,100 @@ Function ProvisionVMsForLisa($allVMData, $installPackagesOnRoleNames)
|
|||
|
||||
function InstallCustomKernel ($customKernel, $allVMData, [switch]$RestartAfterUpgrade)
|
||||
{
|
||||
$customKernel = $customKernel.Trim()
|
||||
if( ($customKernel -ne "linuxnext") -and ($customKernel -ne "netnext") )
|
||||
try
|
||||
{
|
||||
Throw "Only linuxnext and netnext version is supported. Other version will be added soon. Use -customKernel linuxnext"
|
||||
}
|
||||
$scriptName = "customKernelInstall.sh"
|
||||
$jobCount = 0
|
||||
$kernelSuccess = 0
|
||||
$packageInstallJobs = @()
|
||||
foreach ( $vmData in $allVMData )
|
||||
{
|
||||
RemoteCopy -uploadTo $vmData.PublicIP -port $vmData.SSHPort -files ".\remote-scripts\$scriptName,.\SetupScripts\DetectLinuxDistro.sh" -username $user -password $password -upload
|
||||
$out = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "chmod +x *.sh" -runAsSudo
|
||||
$currentKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
|
||||
LogMsg "Executing $scriptName ..."
|
||||
$jobID = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "/home/$user/$scriptName -customKernel $customKernel" -RunInBackground -runAsSudo
|
||||
$packageInstallObj = New-Object PSObject
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name ID -Value $jobID
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name RoleName -Value $vmData.RoleName
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name PublicIP -Value $vmData.PublicIP
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name SSHPort -Value $vmData.SSHPort
|
||||
$packageInstallJobs += $packageInstallObj
|
||||
$jobCount += 1
|
||||
#endregion
|
||||
}
|
||||
|
||||
$packageInstallJobsRunning = $true
|
||||
while ($packageInstallJobsRunning)
|
||||
{
|
||||
$packageInstallJobsRunning = $false
|
||||
foreach ( $job in $packageInstallJobs )
|
||||
{
|
||||
if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
|
||||
{
|
||||
$currentStatus = RunLinuxCmd -ip $job.PublicIP -port $job.SSHPort -username $user -password $password -command "tail -n 1 build-customKernel.txt"
|
||||
LogMsg "Package Installation Status for $($job.RoleName) : $currentStatus"
|
||||
$packageInstallJobsRunning = $true
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( !(Test-Path -Path "$LogDir\$($job.RoleName)-build-customKernel.txt" ) )
|
||||
{
|
||||
RemoteCopy -download -downloadFrom $job.PublicIP -port $job.SSHPort -files "build-customKernel.txt" -username $user -password $password -downloadTo $LogDir
|
||||
if ( ( Get-Content "$LogDir\build-customKernel.txt" ) -imatch "CUSTOM_KERNEL_SUCCESS" )
|
||||
{
|
||||
$kernelSuccess += 1
|
||||
}
|
||||
Rename-Item -Path "$LogDir\build-customKernel.txt" -NewName "$($job.RoleName)-build-customKernel.txt" -Force | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( $packageInstallJobsRunning )
|
||||
{
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
}
|
||||
|
||||
if ( $kernelSuccess -eq $jobCount )
|
||||
{
|
||||
LogMsg "Kernel upgraded to `"$customKernel`" successfully in all VMs."
|
||||
if ( $RestartAfterUpgrade )
|
||||
$customKernel = $customKernel.Trim()
|
||||
if( ($customKernel -ne "linuxnext") -and ($customKernel -ne "netnext") )
|
||||
{
|
||||
LogMsg "Now restarting VMs..."
|
||||
$restartStatus = RestartAllDeployments -allVMData $allVMData
|
||||
if ( $restartStatus -eq "True")
|
||||
LogErr "Only linuxnext and netnext version is supported. Other version will be added soon. Use -customKernel linuxnext"
|
||||
}
|
||||
else
|
||||
{
|
||||
$scriptName = "customKernelInstall.sh"
|
||||
$jobCount = 0
|
||||
$kernelSuccess = 0
|
||||
$packageInstallJobs = @()
|
||||
foreach ( $vmData in $allVMData )
|
||||
{
|
||||
RemoteCopy -uploadTo $vmData.PublicIP -port $vmData.SSHPort -files ".\remote-scripts\$scriptName,.\SetupScripts\DetectLinuxDistro.sh" -username $user -password $password -upload
|
||||
$out = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "chmod +x *.sh" -runAsSudo
|
||||
$currentKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
|
||||
LogMsg "Executing $scriptName ..."
|
||||
$jobID = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "/home/$user/$scriptName -customKernel $customKernel" -RunInBackground -runAsSudo
|
||||
$packageInstallObj = New-Object PSObject
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name ID -Value $jobID
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name RoleName -Value $vmData.RoleName
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name PublicIP -Value $vmData.PublicIP
|
||||
Add-member -InputObject $packageInstallObj -MemberType NoteProperty -Name SSHPort -Value $vmData.SSHPort
|
||||
$packageInstallJobs += $packageInstallObj
|
||||
$jobCount += 1
|
||||
#endregion
|
||||
}
|
||||
|
||||
$packageInstallJobsRunning = $true
|
||||
while ($packageInstallJobsRunning)
|
||||
{
|
||||
$packageInstallJobsRunning = $false
|
||||
foreach ( $job in $packageInstallJobs )
|
||||
{
|
||||
if ( (Get-Job -Id $($job.ID)).State -eq "Running" )
|
||||
{
|
||||
$currentStatus = RunLinuxCmd -ip $job.PublicIP -port $job.SSHPort -username $user -password $password -command "tail -n 1 build-customKernel.txt"
|
||||
LogMsg "Package Installation Status for $($job.RoleName) : $currentStatus"
|
||||
$packageInstallJobsRunning = $true
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( !(Test-Path -Path "$LogDir\$($job.RoleName)-build-customKernel.txt" ) )
|
||||
{
|
||||
RemoteCopy -download -downloadFrom $job.PublicIP -port $job.SSHPort -files "build-customKernel.txt" -username $user -password $password -downloadTo $LogDir
|
||||
if ( ( Get-Content "$LogDir\build-customKernel.txt" ) -imatch "CUSTOM_KERNEL_SUCCESS" )
|
||||
{
|
||||
$kernelSuccess += 1
|
||||
}
|
||||
Rename-Item -Path "$LogDir\build-customKernel.txt" -NewName "$($job.RoleName)-build-customKernel.txt" -Force | Out-Null
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( $packageInstallJobsRunning )
|
||||
{
|
||||
WaitFor -seconds 10
|
||||
}
|
||||
}
|
||||
|
||||
if ( $kernelSuccess -eq $jobCount )
|
||||
{
|
||||
$upgradedKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
|
||||
LogMsg "Old kernel: $currentKernelVersion"
|
||||
LogMsg "New kernel: $upgradedKernelVersion"
|
||||
Add-Content -Value "Old kernel: $currentKernelVersion" -Path .\report\AdditionalInfo.html -Force
|
||||
Add-Content -Value "New kernel: $upgradedKernelVersion" -Path .\report\AdditionalInfo.html -Force
|
||||
LogMsg "Kernel upgraded to `"$customKernel`" successfully in all VMs."
|
||||
if ( $RestartAfterUpgrade )
|
||||
{
|
||||
LogMsg "Now restarting VMs..."
|
||||
$restartStatus = RestartAllDeployments -allVMData $allVMData
|
||||
if ( $restartStatus -eq "True")
|
||||
{
|
||||
$upgradedKernelVersion = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username $user -password $password -command "uname -r"
|
||||
LogMsg "Old kernel: $currentKernelVersion"
|
||||
LogMsg "New kernel: $upgradedKernelVersion"
|
||||
Add-Content -Value "Old kernel: $currentKernelVersion" -Path .\report\AdditionalInfo.html -Force
|
||||
Add-Content -Value "New kernel: $upgradedKernelVersion" -Path .\report\AdditionalInfo.html -Force
|
||||
return $true
|
||||
}
|
||||
else
|
||||
{
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
else
|
||||
{
|
||||
LogErr "Kernel upgrade failed in $($jobCount-$kernelSuccess) VMs."
|
||||
return $false
|
||||
}
|
||||
}
|
||||
return $true
|
||||
}
|
||||
else
|
||||
catch
|
||||
{
|
||||
LogErr "Kernel upgrade failed in $($jobCount-$kernelSuccess) VMs."
|
||||
LogErr "Exception in InstallCustomKernel."
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
|
@ -1968,6 +1968,7 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
|
|||
sleep -Seconds 1
|
||||
$uploadJobStatus = Get-Job -Id $uploadJob.Id
|
||||
}
|
||||
Write-Host ""
|
||||
$returnCode = Get-Content -Path $uploadStatusRandomFile
|
||||
Remove-Item -Force $uploadStatusRandomFile | Out-Null
|
||||
Remove-Job -Id $uploadJob.Id -Force | Out-Null
|
||||
|
|
@ -2052,6 +2053,7 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
|
|||
sleep -Seconds 1
|
||||
$uploadJobStatus = Get-Job -Id $uploadJob.Id
|
||||
}
|
||||
Write-Host ""
|
||||
$returnCode = Get-Content -Path $uploadStatusRandomFile
|
||||
Remove-Item -Force $uploadStatusRandomFile | Out-Null
|
||||
Remove-Job -Id $uploadJob.Id -Force | Out-Null
|
||||
|
|
@ -2143,9 +2145,20 @@ Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $usern
|
|||
|
||||
Function WrapperCommandsToFile([string] $username,[string] $password,[string] $ip,[string] $command, [int] $port)
|
||||
{
|
||||
$command | out-file -encoding ASCII -filepath "$LogDir\runtest.sh"
|
||||
RemoteCopy -upload -uploadTo $ip -username $username -port $port -password $password -files ".\$LogDir\runtest.sh"
|
||||
del "$LogDir\runtest.sh"
|
||||
if ( ( $lastLinuxCmd -eq $command) -and ($lastIP -eq $ip) -and ($lastPort -eq $port) -and ($lastUser -eq $username) )
|
||||
{
|
||||
#Skip upload if current command is same as last command.
|
||||
}
|
||||
else
|
||||
{
|
||||
Set-Variable -Name lastLinuxCmd -Value $command -Scope Global
|
||||
Set-Variable -Name lastIP -Value $ip -Scope Global
|
||||
Set-Variable -Name lastPort -Value $port -Scope Global
|
||||
Set-Variable -Name lastUser -Value $username -Scope Global
|
||||
$command | out-file -encoding ASCII -filepath "$LogDir\runtest.sh"
|
||||
RemoteCopy -upload -uploadTo $ip -username $username -port $port -password $password -files ".\$LogDir\runtest.sh"
|
||||
del "$LogDir\runtest.sh"
|
||||
}
|
||||
}
|
||||
|
||||
Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string] $command, [int] $port, [switch]$runAsSudo, [Boolean]$WriteHostOnly, [Boolean]$NoLogsPlease, [switch]$ignoreLinuxExitCode, [int]$runMaxAllowedTime = 300, [switch]$RunInBackGround)
|
||||
|
|
@ -2447,7 +2460,7 @@ Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string]
|
|||
if($timeOut)
|
||||
{
|
||||
$retValue = ""
|
||||
Throw "Tmeout while executing command : $command"
|
||||
LogErr "Tmeout while executing command : $command"
|
||||
}
|
||||
LogErr "Linux machine returned exit code : $($LinuxExitCode.Split("-")[4])"
|
||||
if ($attemptswt -eq $maxRetryCount -and $attemptswot -eq $maxRetryCount)
|
||||
|
|
@ -2458,7 +2471,7 @@ Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string]
|
|||
{
|
||||
if ($notExceededTimeLimit)
|
||||
{
|
||||
LogMsg "Failed to execute : $command. Retrying..."
|
||||
LogErr "Failed to execute : $command. Retrying..."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ if [ $? == 0 ]; then
|
|||
echo "ROOT_PASSWRD_SET"
|
||||
service $sshdServiceName restart
|
||||
sshdServiceStatus=$?
|
||||
if [ $sshdServiceStatus != 0 ]; then
|
||||
service ssh restart
|
||||
sshdServiceStatus=$?
|
||||
fi
|
||||
else
|
||||
echo "$sshd_configFilePath verification failed for root login."
|
||||
echo "ROOT_PASSWORD_SET_SSHD_CONFIG_FAIL"
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче