azure-linux-automation/TestLibs/RDFELibs.psm1

#region Azure RDFE Methods
<#
.SYNOPSIS 
List All OS Images in Cluster
#>


Function ListOSImages()
{
	$images = Get-AzureVMImage
	$i=0
	foreach ( $t in $images )
	{
		if (!$t.MediaLink)
		{
			LogMsg $t.Label
			LogMsg $t.ImageName
		}
	}
}

Function DeleteUnwantedVMImages([switch]$preserveVHDs)
{
	$images = Get-AzureVMImage
	$imageCount = 0
	foreach ($image in $images)
	{
		if ($image.ImageName -imatch "CentOS-6-20-2013")
		{
			$imageCount = $imageCount + 1
			$retryCount =  3
			$isDeleted = $false
			$stillNotDeleted=$true
			While (($retryCount -gt 0) -and ($stillNotDeleted))
			{
				$stillNotDeleted=$true
				$retryCount = $retryCount - 1
				if ($preserveVHDs)
				{
					LogMsg "Deleting image $($image.ImageName), keeping its VHD file inact in storage account.."
					Remove-AzureVMImage -ImageName $image.ImageName -Verbose
					$stillNotDeleted = !$?
				}
				else
				{
					LogMsg "Deleting image $($image.ImageName) & deleting its VHD file from storage account.."
					Remove-AzureVMImage -ImageName $image.ImageName -Verbose -DeleteVHD
					$stillNotDeleted = !$?
				}
			}
			if ($stillNotDeleted)
			{
				Write-Host "Failed.. :-("
			}
			else
			{
				Write-Host "Deleted successfully :-)"
			}
		}
	}
}
<#
.SYNOPSIS 
Set the Azure Subscription based subcription id on Host machine 
.PARAMETER subscription
Specifies the subsciption id
#>

Function DetectLinuxDistro($VIP, $SSHport, $testVMUser, $testVMPassword)
{

	$tempout = RemoteCopy  -upload -uploadTo $VIP -port $SSHport -files ".\SetupScripts\DetectLinuxDistro.sh" -username $testVMUser -password $testVMPassword 2>&1 | Out-Null
	$tempout = RunLinuxCmd -username $testVMUser -password $testVMPassword -ip $VIP -port $SSHport -command "chmod +x *.sh" -runAsSudo 2>&1 | Out-Null
	$DistroName = RunLinuxCmd -username $testVMUser -password $testVMPassword -ip $VIP -port $SSHport -command "/home/$user/DetectLinuxDistro.sh" -runAsSudo
	if(($DistroName -imatch "Unknown") -or (!$DistroName))
	{
		LogErr "Linux distro detected : $DistroName"
		Throw "Unable to detect distro."
	}
	else
	{
		if ($DistroName -imatch "UBUNTU")
		{
			$CleanedDistroName = "UBUNTU" 
		}
		elseif ($DistroName -imatch "CENTOS")
		{
			$CleanedDistroName = "CENTOS"
		}
		elseif ($DistroName -imatch "SLES")
		{
			$CleanedDistroName = "SLES"
		}
		elseif ($DistroName -imatch "SUSE")
		{
			$CleanedDistroName = "SUSE"
		}
		elseif ($DistroName -imatch "ORACLELINUX")
		{
			$CleanedDistroName = "ORACLELINUX"
		}
		elseif ($DistroName -imatch "REDHAT")
		{
			$CleanedDistroName = "REDHAT"
		}
		elseif ($DistroName -imatch "FEDORA")
		{
			$CleanedDistroName = "FEDORA"
		}
		elseif ($DistroName -imatch "COREOS")
		{
			$CleanedDistroName = "COREOS"
		}
		else
		{
			$CleanedDistroName = "UNKNOWN"
		}
		Set-Variable -Name detectedDistro -Value $CleanedDistroName -Scope Global
		SetDistroSpecificVariables -detectedDistro $detectedDistro
		LogMsg "Linux distro detected : $CleanedDistroName"	
	}
	return $CleanedDistroName
}

Function GetCurrentPackageData ($packageXml, $packageName)
{
	$failed = $true
	foreach ($packageDefinition in $packageXml.data.packageDefinition.package)
	{
		if ($packageDefinition.name -eq $packageName)
		{
			$expectedDefinition = $packageDefinition
			$failed = $false
			break
		}
	}
	if ($failed)
	{
		Throw "Unable to find $packageName in package definitions.."
	}
	return $expectedDefinition
}



Function InstallPackages ($VMIpAddress, $VMSshPort, $VMUserName, $VMPassword)
{
	$installError=0
	$installCount=0
	$installSuccess=0
	Set-Content -Value "" -Path .\VHD_Provision.log
	$password = "redhat"
	$packageXml = [xml](Get-Content .\XML\packageInstall.xml)
	$detectedDistro = DetectLinuxDistro -VIP $VMIpAddress -SSHport $VMSshPort -testVMUser $VMUserName -testVMPassword $VMPassword
	
	Write-Host "Detected Distro : $detectedDistro.."
	foreach ($package in $packageXml.data.installPackages.$detectedDistro.package)
	{
		#Write-Host "In the loop now.."
		#Write-Host "$($package.name)"
		$installCount=$installCount+1
		$currentPackageName =  $package.name

		$currentPackageData = GetCurrentPackageData -packageXml $packageXml -packageName $currentPackageName
		$currentDistroPackageData = $currentPackageData.$detectedDistro
		$currentPackageFile = $currentDistroPackageData.file
		LogMsg "Now installing .. $currentPackageName"
		Add-Content -Value "START--------------- INSTALL : $currentPackageName --------------------" -Path .\VHD_Provision.log
		if ($currentDistroPackageData.Location -eq "Remote")
		{

#LogMsg "Invoking command : /root/packageInstall.sh -install $currentPackageName -isLocal no"
			try
			{
				$out = RunLinuxCmd -username $VMUserName -password $VMPassword -ip $VMIpAddress -port $VMSshPort -command "./packageInstall.sh -install $currentPackageName -isLocal no" -runAsSudo
			}
			catch
			{

			}
			if ($out -imatch "InstallSuccess")
			{
				LogMsg "Package : $currentPackageName : Installed successfully."
				$installSuccess=$installSuccess+1
			}
			else
			{
				LogErr "Package : $currentPackageName : Installation failed."
				$installError=$installError+1
			}


		}
		elseif ($currentDistroPackageData.Location -eq "local")
		{
			LogMsg "Uploading install file : $currentPackageFile..."

			RemoteCopy -upload -uploadTo $VMIpAddress -port $VMSshPort -username $VMUserName -password $VMPassword -files ".\tools\Packages\$currentPackageFile"

#LogMsg "Invoking command : /root/packageInstall.sh -install $currentPackageName -isLocal yes -file $currentPackageFile"
			if ($currentDistroPackageData.supportingFiles)
			{
				foreach ($supprotFile in ($currentDistroPackageData.supportingFiles).Split(",") )
				{
					LogMsg "Uploading support file - $supprotFile .. "
					RemoteCopy -upload -uploadTo $VMIpAddress -port $VMSshPort -username $VMUserName -password $VMPassword -files ".\tools\Packages\$supprotFile"
				}
			}
			$out = RunLinuxCmd -username $VMUserName -password $VMPassword -ip $VMIpAddress -port $VMSshPort -command "./packageInstall.sh -install $currentPackageName -isLocal yes -file $currentPackageFile" -runAsSudo
			if ($out -imatch "InstallSuccess")
			{
				LogMsg "Package : $currentPackageName : Installed successfully."
				$installSuccess=$installSuccess+1
			}
			else
			{
				LogErr "Package : $currentPackageName : Installation failed."
				$installError=$installError+1
			}

		}
		Add-Content -Value $out -Path .\VHD_Provision.log
		Add-Content -Value "END----------------- INSTALL : $currentPackageName --------------------" -Path .\VHD_Provision.log
	}

	LogMsg "Total packages  : $installCount."
	LogMsg "Total installed : $installSuccess"
	LogMsg "Total failed	: $installError"

	if ($installError -gt 0)
	{
		LogErr "$installError out of $installCount packages failed to install"
		$retValue=$false
	}
	else
	{
		LogMsg "All packages installed successfully."
		$retValue=$true
	}
	return $retValue
}

Function IsEnvironmentSupported()
{
	$version = (Get-Module -Name "Azure").Version
	If ($version.Major -GT 0 -OR
		$version.Minor -GT 8 -OR
		(($version.Minor -EQ 8) -And ($version.Build -GE 8)))
	{
		return $true
	}
	Else
	{
		return $false
	}
}

Function SetSubscription ($subscriptionID, $subscriptionName, $certificateThumbprint, $managementEndpoint, $storageAccount, $environment = "AzureCloud")
{
	$myCert = Get-Item cert:\CurrentUser\My\$certificateThumbprint

	# For Azure Powershell Version >= 0.8.8, Environment is used in Set-AzureSubscription for replacing ManagementEndpoint
	if (IsEnvironmentSupported)
	{
		Set-AzureSubscription -SubscriptionName $subscriptionName -Certificate $myCert -SubscriptionID $subscriptionID `
							  -CurrentStorageAccountName $storageAccount -Environment $environment
	}
	Else
	{
		Set-AzureSubscription -SubscriptionName $subscriptionName -Certificate $myCert -SubscriptionID $subscriptionID `
							  -CurrentStorageAccountName $storageAccount -ServiceEndpoint $managementEndpoint
	}
	Select-AzureSubscription -Current $subscriptionName
}

<#
.SYNOPSIS 
Gets the Azure Hosted Service VIP 
.PARAMETER servicename
Specifies the servicename
#>
Function GetHsVmVip([string]$servicename)
{
	$endpoints = Get-AzureService $serviceName |  Get-AzureVM | Get-AzureEndpoint
	$vip = $endpoints[0].Vip
	return $VIP
}

<#
.SYNOPSIS 
Deletes Azure Service 
.PARAMETER servicename
Specifies the servicename
#>
Function DeleteService ($serviceName, [switch]$KeepDisks)
{
	$j= 0 
	$ExistingServices = Get-AzureService

	foreach ( $eachService in $ExistingServices )
	{
		if( $eachService.ServiceName -eq $serviceName )
		{
			$j = $j + 1

			LogMsg "$serviceName ..service exists!"
			if ($eachService.Description -imatch "DONOTDISTURB")
			{
				Write-Host "Not Removing $($eachService.ServiceName). As it is labelled as DO NOT DISTURB. Please try Not to remove it..." -ForegroundColor Red
				$retValue = "True"
				break
			}
			LogWarn "Warning : Deleting All Virtual Machines in $serviceName in 5 seconds. Interrupt to break.."
			WaitFor -seconds 5
			LogMsg "Deleting $serviceName..."
			$retValue = "False"
			$retryCount = 1
			while (($retValue -eq "False") -and ($retryCount -lt 10))
			{
				if ( $KeepDisks )
				{
					$out = Remove-AzureService -ServiceName $serviceName -Force  -Verbose
				}
				else
				{
					$out = Remove-AzureService -ServiceName $serviceName -DeleteAll -Force  -Verbose
				}
				$RemoveServiceExitCode =  $?
				if(($out -imatch "Complete") -or $RemoveServiceExitCode)
				{
					if($retryCount -lt 1)
					{
						LogMsg "Deleted $serviceName"
					}
					else
					{
						LogMsg "Deleted $serviceName after $retryCount Attempt"
					}
					$retValue = "True"
				}
				else
				{
					$retryCount = $retryCount + 1
					LogWarn "Error in deletion. Retry Attempt $retryCount "
				}
			}
		}
	}

	if ($j -eq 0 )
	{
		LogMsg "$serviceName not found!"
		$retValue = "True"
	}

	return $retValue
}

<#
.SYNOPSIS 
Creates Azure Service 
.PARAMETER servicename
Specifies the servicename
#>
Function CreateService($serviceName, $location, $AffinityGroup)
{
	$FailCounter = 0
	$retValue = "False"
	While(($retValue -eq "False") -and ($FailCounter -lt 5))
	{
	try{
		$FailCounter++

		if($location) {
			LogMsg "Using location : $location"
			$out = RunAzureCmd -AzureCmdlet "New-AzureService -ServiceName $serviceName -Location $location"
		}
		else {
			if($AffinityGroup) {
			LogMsg "Using Affinity Group : $AffinityGroup" 
			$out = RunAzureCmd -AzureCmdlet "New-AzureService -ServiceName $serviceName -AffinityGroup $AffinityGroup"
			}
		}

		$operationID = $out.OperationID
		$operationStatus = $out.OperationStatus
		LogMsg "New-AzureService`t" -NoNewline 
		if ($operationStatus  -eq "Succeeded"){
			LogMsg "Hosted Service Created."
			$retValue = "True"
		}
		else {
			LogErr "Failed"
			$retValue = "False"
		  
		}
		}
		catch
		{
		$retValue = "False"
		}
	}
	return $retValue
}

Function AddCertificate($serviceName)
{
	$FailCounter = 0
	$retValue = $false

	#Added try catch to handle "Unable to Send Request error"
	While(($retValue -eq $false) -and ($FailCounter -lt 5))
	{
		try
		{
		$FailCounter++
		$currentDirectory = (pwd).Path
		LogMsg "Adding Certificate to hosted service.."
		$out = RunAzureCmd -AzureCmdlet "Add-AzureCertificate -CertToDeploy `"$currentDirectory\ssh\myCert.cer`" -ServiceName $serviceName"
		$retValue = $?
		}
		catch
		{
			$retValue = $false	  
		}
	}
	return $retValue
}

Function GenerateCommand ($Setup, $serviceName, $osImage, $HSData)
{
	$role = 0
	$HS = $HSData
	$setupType = $Setup
	$defaultuser = $xml.config.Azure.Deployment.Data.UserName
	$defaultPassword = $xml.config.Azure.Deployment.Data.Password
	$totalVMs = 0
	$totalHS = 0
	$extensionCounter = 0
	$vmCommands = @()
	$vmCount = 0
	if ( $CurrentTestData.ProvisionTimeExtensions )
	{
		$extensionString = [string](Get-Content .\XML\Extensions.xml)
		foreach ($line in $extensionString.Split())
		{
			if ($line -imatch "EXECUTE-PS-")
			{
				$line = $line.Trim()
				$line = $line.Replace("EXECUTE-PS-","")
				$line = $line.Split(">")
				$line = $line.Split("<")
				$PSoutout = Invoke-Expression -Command $line[2]
				$extensionString = $extensionString.Replace("EXECUTE-PS-$($line[2])",$PSoutout)
				Sleep -Milliseconds 500
			}
		}
		$extensionXML = [xml]$extensionString
	}
	foreach ( $newVM in $HS.VirtualMachine)
	{
		$vmCount = $vmCount + 1
		$VnetName = $HS.VnetName
		$instanceSize = $newVM.InstanceSize
		$SubnetName = $newVM.SubnetName
		$DnsServerIP = $HS.DnsServerIP
#...............................................
# LIST OUT THE TOTAL PORTS TO BE OPENED AND ADD THEM ACCORDINGLY...
#'''''''''''''''''''''''''''''''''''''''''''''''
		$portCommand = ""   
		$portNo = 0
		$Endpoints = $newVM.Endpoints
		foreach ( $openedPort in $newVM.EndPoints)
		{
			if($openedPort.Name -eq "SSH")
			{
				$portCommand =  $portCommand + " Set-AzureEndpoint -Name `"" + $openedPort.Name + "`" -LocalPort `"" + $openedPort.LocalPort + "`" -PublicPort `"" +  $openedPort.PublicPort + "`" -Protocol `"" + $openedPort.Protocol + "`""
			}
			else
			{
				$portCommand =  $portCommand + " Add-AzureEndpoint" + " -Name `"" + $openedPort.Name  + "`"" + " -LocalPort `"" + $openedPort.LocalPort + "`" -PublicPort `"" +  $openedPort.PublicPort + "`" -Protocol `"" + $openedPort.Protocol + "`""
				if ($openedPort.LoadBalanced -eq "True")
				{
					if ($openedPort.ProbePort)
					{
						$portCommand = $portCommand + " -LBSetName `"" + $openedPort.Name + "`"" + "-ProbePort " +  $openedPort.ProbePort + " -ProbeProtocol tcp"
					}
					else
					{
						$portCommand = $portCommand + " -LBSetName `"" + $openedPort.Name + "`"" + " -NoProbe"
					}
				}
			}
			$portCommand = $portCommand + " |"
			$portNo = $portNo + 1
		}
		if($newVM.RoleName)
		{
			$vmName = $newVM.RoleName
		}
		else
		{
			$vmName = $serviceName +"-role-"+$role
		}
		$diskCommand = ""
		foreach ( $dataDisk in $newVM.DataDisk)
		{
			if ( $dataDisk.LUN -and $dataDisk.DiskSizeInGB -and $dataDisk.HostCaching )
			{
				if ($diskCommand)
				{
					$diskCommand = $diskCommand + " | " + "Add-AzureDataDisk -CreateNew -DiskSizeInGB $($dataDisk.DiskSizeInGB) -LUN $($dataDisk.LUN) -HostCaching $($dataDisk.HostCaching) -DiskLabel `"$vmName-Disk-$($dataDisk.LUN)`"" 
				}
				else
				{
					$diskCommand = "Add-AzureDataDisk -CreateNew -DiskSizeInGB $($dataDisk.DiskSizeInGB) -LUN $($dataDisk.LUN) -HostCaching $($dataDisk.HostCaching) -DiskLabel `"$vmName-Disk-$($dataDisk.LUN)`""
				}
			}
		}
		if ( $CurrentTestData.ProvisionTimeExtensions )
		{
			$ExtensionCommand = ""
			foreach ( $extn in $CurrentTestData.ProvisionTimeExtensions.Split(","))
			{
				$extn = $extn.Trim()
				foreach ( $newExtn in $extensionXML.Extensions.Extension )
				{
					if ($newExtn.Name -eq $extn)
					{
						[hashtable]$extensionHashTable = @{};
						$newExtn.Params.ChildNodes | foreach {$extensionHashTable[$_.Name] = $_.'#text'};
						$PublicConfiguration += $extensionHashTable | ConvertTo-Json
						[hashtable]$extensionHashTable = @{};
						$PrivateConfiguration += $extensionHashTable | ConvertTo-Json
						if ( $ExtensionCommand )
						{
							$ExtensionCommand = $ExtensionCommand + " | Set-AzureVMExtension -ExtensionName $($newExtn.OfficialName) -ReferenceName $extn -Publisher $($newExtn.Publisher) -Version $($newExtn.Version) -PublicConfiguration `$PublicConfiguration[$extensionCounter] -PrivateConfiguration `$PrivateConfiguration[$extensionCounter]"
						}
						else
						{
							$ExtensionCommand = "Set-AzureVMExtension -ExtensionName $($newExtn.OfficialName) -ReferenceName $extn -Publisher $($newExtn.Publisher) -Version $($newExtn.Version) -PublicConfiguration `$PublicConfiguration[$extensionCounter] -PrivateConfiguration `$PrivateConfiguration[$extensionCounter]"
						}
						LogMsg "Extension $extn (OfficialName : $($newExtn.OfficialName)) added to deployment command."
						$extensionCounter += 1
					}
				}
			}
			if ( $PublicConfiguration )
			{
				Set-Variable -Name PublicConfiguration -Value $PublicConfiguration -Scope Global
			}
			if ( $PrivateConfiguration )
			{
				Set-Variable -Name PrivateConfiguration -Value $PrivateConfiguration -Scope Global
			}
		}
		$sshPath = '/home/' + $defaultuser + '/.ssh/authorized_keys'		 	
		$vmRoleConfig = "New-AzureVMConfig -Name $vmName -InstanceSize $instanceSize -ImageName $osImage"
		$vmProvConfig = "Add-AzureProvisioningConfig -Linux -LinuxUser $defaultuser -Password $defaultPassword -SSHPublicKeys (New-AzureSSHKey -PublicKey -Fingerprint 690076D4C41C1DE677CD464EA63B44AE94C2E621 -Path $sshPath)"
		if($SubnetName)
		{
			$vmProvConfig = $vmProvConfig + "| Set-AzureSubnet -SubnetNames $SubnetName"
		}
		$vmPortConfig =  $portCommand.Substring(0,$portCommand.Length-1)
		
		#Start building SingleVM Config command..

		$singleVMCommand = "( " + $vmRoleConfig + " | " + $vmProvConfig + " | " + $vmPortConfig
		
		if ( $diskCommand )
		{
			$singleVMCommand = $singleVMCommand + " | " + $diskCommand
		}
		if ( $ExtensionCommand )
		{
			$singleVMCommand = $singleVMCommand + " | " + $ExtensionCommand
		}

		$singleVMCommand = $singleVMCommand + " )"
		
		#Finished building SingleVM Config command..

		$totalVMs = $totalVMs + 1
		$role = $role + 1

		if ($totalVMs -gt 1)
		{
			$finalVMcommand = $finalVMcommand + ', ' + $singleVMCommand
		}
		else
		{
			$finalVMcommand = $singleVMCommand
		}
	}			

	if ($VnetName)
	{
		$newDNS  = "(New-AzureDns -Name `"DNSsettings`" -IPAddress $DnsServerIP)"
		$createSetupCommand = "New-AzureVM -ServiceName $serviceName -VMs ($finalVMcommand)  -VNetName $VnetName -DnsSettings $newDNS"
	}
	else
	{
		$createSetupCommand = "New-AzureVM -ServiceName $serviceName -VMs ($finalVMcommand)"
	}   

	return $createSetupCommand,  $serviceName, $vmCount
} 

Function CreateDeployment ($DeploymentCommand, $NewServiceName , $vmCount)
{
	
	$FailCounter = 0
	$retValue = "False"
	#While(($retValue -eq "False") -and ($FailCounter -lt 5))
	#{
		try
		{
			$FailCounter++
			$out = RunAzureCmd -AzureCmdlet "$DeploymentCommand"
			#LogMsg $DeploymentCommand
			$retValue = $?
			LogMsg "VM's deployed. Verifying.."
			$VMDetails = Get-AzureVM -ServiceName $NewServiceName
			$tempcounter = 0
			foreach ($vm in $VMDetails)
			{
				$VMcounter = $VMcounter + 1
			}
			if ($VMcounter -eq $vmCount)
			{
				LogMsg "Expected VMs : $vmCount. Deployed VMs : $vmCounter"
				$retValue = "True"
			}
			else
			{
				$retValue = "False"
				LogErr "Expected VMs : $vmCount. Deployed VMs : $vmCounter"
			}
		 }
		 catch
		 {
		   $retValue = "False"
		 }   
	#}
	return $retValue 
}

Function CheckVMsInService($serviceName)
{
	try
	{
		$DeployedVMs = RetryOperation -operation { Get-AzureVM -ServiceName $serviceName } -retryInterval 1 -maxRetryCount 10 -NoLogsPlease
		$allVMsReady = "False"
		$isTimedOut = "False"
		$VMCheckStarted = Get-Date
		While (($allVMsReady -eq "False") -and ($isTimedOut -eq "False"))
		{
			$i = 0
			$VMStatus = @()
	#$DeployedVMs = Get-AzureService -ServiceName $serviceName  | Get-AzureVM 
			$DeployedVMs = RetryOperation -operation { Get-AzureVM -ServiceName $serviceName } -retryInterval 1 -maxRetryCount 10 -NoLogsPlease
			$Recheck = 0
			$VMStatuString = ""
			foreach ( $VM in $DeployedVMs )
			{
				$VMStatuString += "$($VM.Name) : $($VM.InstanceStatus) "
				if ( $VM.InstanceStatus -ne "ReadyRole" )
				{
					$VMStatus = $VM.InstanceStatus
	#Write-Host $VMStatus
					$Recheck = $Recheck + 1
				}
				else
				{
	#Write-Host $VMStatus
				}
			}

			$VMcheckTimeNow = Get-Date
			$VMtime= $VMcheckTimeNow - $VMCheckStarted
			if ($VMtime.TotalSeconds -gt 1800 )
			{
				$isTimedOut = "True"
			}   
			if ($Recheck -eq 0 )
			{
				$allVMsReady = "True"
			}
			$remainigSeconds = 1800 - $VMtime.TotalSeconds
			Write-Progress -Id 500 -Activity "Checking Deployed VM in Service : $serviceName. Seconds Remaining : $remainigSeconds" -Status "$VMStatuString"
			Write-Host "." -NoNewline
			#Write-Host $VMStatus -NoNewline
			sleep 1
		}   
		Write-Progress -Id 500 -Activity "Checking Deployed VM in Service : $serviceName. Seconds Remaining : $remainigSeconds" -Status "$VMStatuString" -Completed
	}
	catch
	{
		$ErrorMessage =  $_.Exception.Message
		LogMsg "EXCEPTION in CheckVMsInService() : $ErrorMessage"
		$allVMsReady = "False"
	}
	return $allVMsReady
}

Function CreateAllDeployments($setupType, $xmlConfig, $Distro)
{

	$hostedServiceCount = 0
	$xml = $xmlConfig
	LogMsg $setupType
	$setupTypeData = $xml.config.Azure.Deployment.$setupType
	$allsetupServices = $setupTypeData
	if ($allsetupServices.HostedService[0].Location -or $allsetupServices.HostedService[0].AffinityGroup)
	{
		$isMultiple = 'True'
		$hostedServiceCount = 0
	}
	else
	{
		$isMultiple = 'False'
	}

	foreach ($newDistro in $xml.config.Azure.Deployment.Data.Distro)
	{

		if ($newDistro.Name -eq $Distro)
		{
			$osImage = $newDistro.OsImage
		}
	}

	$location = $xml.config.Azure.General.Location
	$AffinityGroup = $xml.config.Azure.General.AffinityGroup

	foreach ($HS in $setupTypeData.HostedService )
	{
		$curtime = Get-Date
		$isServiceDeployed = "False"
		$retryDeployment = 0
		if ( $HS.Tag -ne $null )
		{
			$serviceName = "ICA-" + $HS.Tag + "-" + $Distro + "-" + $curtime.Month + "-" +  $curtime.Day  + "-" + $curtime.Hour + "-" + $curtime.Minute + "-" + $curtime.Second
		}
		else
		{
			$serviceName = "ICA-" + $setupType + "-" + $Distro + "-" + $curtime.Month + "-" +  $curtime.Day  + "-" + $curtime.Hour + "-" + $curtime.Minute + "-" + $curtime.Second
		}
		if($isMultiple -eq "True")
		{
			$serviceName = $serviceName + "-" + $hostedServiceCount
		}

		while (($isServiceDeployed -eq "False") -and ($retryDeployment -lt 5))
		{
			LogMsg "Creating Hosted Service : $serviceName."
			LogMsg "Verifying that service name is not in use."
			$isServiceDeleted = DeleteService -serviceName $serviceName
#$isServiceDeleted = "True"
			if ($isServiceDeleted -eq "True")
			{	 
				$isServiceCreated = CreateService -serviceName $serviceName -location $location -AffinityGroup $AffinityGroup
#$isServiceCreated = "True"
				if ($isServiceCreated -eq "True")
				{
					$isCertAdded = AddCertificate -serviceName $serviceName
#$isCertAdded = "True"
					if ($isCertAdded -eq "True")
					{
						LogMsg "Certificate added successfully."
						$DeploymentCommand = GenerateCommand -Setup $Setup -serviceName $serviceName -osImage $osImage -HSData $HS
						Set-AzureSubscription -SubscriptionName $xmlConfig.config.Azure.General.SubscriptionName  -CurrentStorageAccountName $xmlConfig.config.Azure.General.StorageAccount
						$DeploymentStartTime = (Get-Date)
						$isDeployed = CreateDeployment -DeploymentCommand $DeploymentCommand[0] -NewServiceName $DeploymentCommand[1] -vmCount $DeploymentCommand[2]
						$DeploymentEndTime = (Get-Date)
						$DeploymentElapsedTime = $DeploymentEndTime - $DeploymentStartTime
						if ( $isDeployed -eq "True" )
						{
							LogMsg "Deployment Created!"
							$retValue = "True"
							$isServiceDeployed = "True"
							$hostedServiceCount = $hostedServiceCount + 1
							if ($hostedServiceCount -eq 1)
							{
								$deployedServices = $serviceName
							}
							else
							{
								$deployedServices = $deployedServices + "^" + $serviceName
							}

						}
						else
						{
							LogErr "Unable to Deploy one or more VM's"
							$retryDeployment = $retryDeployment + 1
							$retValue = "False"
							$isServiceDeployed = "False"
						}
					}
					else
					{
						LogErr "Unable to Add certificate to $serviceName"
						$retryDeployment = $retryDeployment + 1
						$retValue = "False"
						$isServiceDeployed = "False"
					}

				}
				else
				{
					LogErr "Unable to create $serviceName"
					$retryDeployment = $retryDeployment + 1
					$retValue = "False"
					$isServiceDeployed = "False"
				}
			}	
			else
			{
				LogErr "Unable to delete existing service - $serviceName"
				$retryDeployment = 3
				$retValue = "False"
				$isServiceDeployed = "False"
			}

		}
	}
	return $retValue, $deployedServices, $DeploymentElapsedTime
}

Function VerifyAllDeployments($servicesToVerify, [Switch]$GetVMProvisionTime)
{
	$VMProvisionElapsedTime = $null
	$VMProvisionStarted = Get-Date
	LogMsg "Waiting for VM(s) to become Ready."
	foreach ($service in  $servicesToVerify)
	{
		$serviceName = $service
		LogMsg "checking $serviceName.. "
		$isDeploymentReady = CheckVMsInService ($serviceName)
		$VMProvisionFinished = Get-Date
		$VMProvisionElapsedTime = $VMProvisionFinished - $VMProvisionStarted
		if ($isDeploymentReady -eq "True")
		{
			LogMsg ""
			LogMsg "$serviceName is Ready.."
			Write-Host ""
			$retValue = "True"
		}
		else
		{
			LogErr "$serviceName provision Failed.."
			$retValue = "False"
			break
		}
	}
	if($GetVMProvisionTime)
	{
		return $retValue, $VMProvisionElapsedTime
	}
	else
	{
		return $retValue
	}
}

Function WaitFor($seconds,$minutes,$hours)
{
	if(!$hours -and !$minutes -and !$seconds)
	{
		Write-Host "Come on.. Mention at least one second bro ;-)"
	}
	else
	{
		if(!$hours)
		{
			$hours = 0
		}
		if(!$minutes)
		{
			$minutes = 0
		}
		if(!$seconds)
		{
			$seconds = 0
		}

		$timeToSleepInSeconds = ($hours*60*60) + ($minutes*60) + $seconds
		$secondsRemaining = $timeToSleepInSeconds 
		$secondsRemainingPercentage = (100 - (($secondsRemaining/$timeToSleepInSeconds)*100))
		for ($i = 1; $i -le $timeToSleepInSeconds; $i++)
		{
			write-progress -Id 27 -activity SLEEPING -Status "$($secondsRemaining) seconds remaining..." -percentcomplete $secondsRemainingPercentage
			$secondsRemaining = $timeToSleepInSeconds - $i
			$secondsRemainingPercentage = (100 - (($secondsRemaining/$timeToSleepInSeconds)*100))
			sleep -Seconds 1
		}
		write-progress -Id 27 -activity SLEEPING -Status "Wait Completed..!" -Completed
	}

}

Function RemoveICAUnusedServices([switch]$removePreservedServices, $pattern="ICA-", [switch]$onlyPreservedServices)
{
	$ExistingServices = Get-AzureService
	foreach ($service in $ExistingServices)
	{
		if ($service.ServiceName -imatch $pattern) 
		{
			if (!($service.Description -imatch "DONOTDISTURB"))
			{
				if(($service.Description -imatch "Preserving"))
				{
					if($removePreservedServices -or $onlyPreservedServices)
					{
						Write-Host "ATTENTION : Removing preserved service : $($service.ServiceName).." -ForegroundColor Red
						DeleteService -ServiceName $service.ServiceName 
					}
					else
					{
						Write-Host "Skipping preserved hosted service : $($service.ServiceName).." -ForegroundColor Green
					}
				}
				else
				{
					if(!$onlyPreservedServices)
					{
						Write-Host "Removing $($service.ServiceName).." -ForegroundColor Red
						DeleteService -ServiceName $service.ServiceName 
					}
				}
			}
			else
			{
				Write-Host "Not Removing $($service.ServiceName). As it is labelled as DO NOT DISTURB. Please try Not to remove it..." -ForegroundColor Red
			}
		}		
	}
}
Function RemoveICAUnusedDataDisks()
{
	$dataDisks = Get-AzureDisk
	$removeDiskId = 55
	$totalIcaDisks = 0
	$dataDiskToRemove = @()
	foreach ($disk in $dataDisks)
	{
		if (($disk.DiskName -imatch "ICA-") -and (!$disk.AttachedTo))
		{
			$totalIcaDisks +=  1
			$dataDiskToRemove += $disk.DiskName
		}
	}

	if ($totalIcaDisks -ge 1)
	{
		Write-Progress -Id $removeDiskId -Activity "Removing unused ICA data disks.." -PercentComplete 0
		$totalRemaining = $totalIcaDisks
		$index = 1
		foreach ($disk in $dataDiskToRemove)
		{
			Write-Progress -Id $removeDiskId -Activity "Removing unused ICA data disks.." -PercentComplete (100 - ($totalRemaining*100/$totalIcaDisks)) -Status "[$index/$totalIcaDisks] $disk"
			try
			{
				Write-Host "Removing - $disk.." -NoNewline
				sleep 3
				$out = Remove-AzureDisk -DiskName $disk -DeleteVHD
				Write-Host ".Ok!" -ForegroundColor Green
				$totalRemaining -= 1
				$index += 1
			}
			catch
			{
				Write-Host ".Failed" -ForegroundColor Red
			}

		}
		Write-Progress -Id $removeDiskId -Activity "Removed!" -Completed
	}
	else
	{
		Write-Host "No unused ICA disks found." -ForegroundColor Green
	}
}

#function to collect and compare kernel logs
Function GetAndCheckKernelLogs($DeployedServices, $status, $vmUser, $vmPassword)
{
	if ( !$vmUser )
	{
		$vmUser = $user
	}
	if ( !$vmPassword )
	{
		$vmPassword = $password
	}
	$retValue = $false
	$hsNames = $DeployedServices.Split('^')
	foreach ($hsName in $hsNames)
	{
		#$ErrCount = 0
		$hsDetails =  RunAzureCmd -AzureCmdlet "Get-AzureService -ServiceName $hsName"
		$VMs =  Get-AzureVM -ServiceName $hsName
		
		foreach ($VM in $VMs)
		{
			$BootLogDir="$Logdir\$($VM.Name)"
			mkdir $BootLogDir -Force | Out-Null			
			LogMsg "Collecting $($VM.Name) VM Kernel $status Logs.."
			$InitailBootLog="$BootLogDir\InitialBootLogs.txt"
			$FinalBootLog="$BootLogDir\FinalBootLogs.txt"
			$KernelLogStatus="$BootLogDir\KernelLogStatus.txt"
			$VMEndpoints = Get-AzureEndpoint -VM $VM
			$VMSSHPort = GetPort -Endpoints $VMEndpoints -usage "SSH"
			if($status -imatch "Initial")
			{
				$randomFileName = [System.IO.Path]::GetRandomFileName()
				Set-Content -Value "A Random file." -Path "$Logdir\$randomFileName"
				$out = RemoteCopy -uploadTo $VMEndpoints[0].Vip -port $VMSSHPort  -files "$Logdir\$randomFileName" -username $vmUser -password $vmPassword -upload
				Remove-Item -Path "$Logdir\$randomFileName" -Force
				$out = RunLinuxCmd -ip $VMEndpoints[0].Vip -port $VMSSHPort -username $vmUser -password $vmPassword -command "dmesg > /home/$vmUser/InitialBootLogs.txt" -runAsSudo
				$out = RemoteCopy -download -downloadFrom $VMEndpoints[0].Vip -port $VMSSHPort -files "/home/$vmUser/InitialBootLogs.txt" -downloadTo $BootLogDir -username $vmUser -password $vmPassword
				LogMsg "$($VM.Name): $status Kernel logs collected ..SUCCESSFULLY"
				$detectedDistro = DetectLinuxDistro -VIP $VMEndpoints[0].Vip -SSHport $VMSSHPort -testVMUser $vmUser -testVMPassword $vmPassword
				SetDistroSpecificVariables -detectedDistro $detectedDistro
				$retValue = $true
			}
			elseif($status -imatch "Final")
			{
				$out = RunLinuxCmd -ip $VMEndpoints[0].Vip -port $VMSSHPort -username $vmUser -password $vmPassword -command "dmesg > /home/$vmUser/FinalBootLogs.txt" -runAsSudo
				$out = RemoteCopy -download -downloadFrom $VMEndpoints[0].Vip -port $VMSSHPort -files "/home/$vmUser/FinalBootLogs.txt" -downloadTo $BootLogDir -username $vmUser -password $vmPassword
				$KernelDiff = Compare-Object -ReferenceObject (Get-Content $FinalBootLog) -DifferenceObject (Get-Content $InitailBootLog)
				#Removing final dmesg file from logs to reduce the size of logs. We can alwayas see complete Final Logs as : Initial Kernel Logs + Difference in Kernel Logs
				Remove-Item -Path $FinalBootLog -Force | Out-Null
				if($KernelDiff -eq $null)
				{
					LogMsg "** Initial and Final Kernel Logs has same content **"  
					Set-Content -Value "*** Initial and Final Kernel Logs has same content ***" -Path $KernelLogStatus
					$retValue = $true
				}
				else
				{
					$errorCount = 0
					Set-Content -Value "Following lines were added in the kernel log during execution of test." -Path $KernelLogStatus
					LogMsg "Following lines were added in the kernel log during execution of test." 
					Add-Content -Value "-------------------------------START----------------------------------" -Path $KernelLogStatus
					foreach ($line in $KernelDiff)
					{
						Add-Content -Value $line.InputObject -Path $KernelLogStatus
						if ( ($line.InputObject -imatch "fail") -or ($line.InputObject -imatch "error") -or ($line.InputObject -imatch "warning"))
						{
							$errorCount += 1
							LogErr $line.InputObject
						}
						else
						{
							LogMsg $line.InputObject
						}
					}
					Add-Content -Value "--------------------------------EOF-----------------------------------" -Path $KernelLogStatus
				}
				LogMsg "$($VM.Name): $status Kernel logs collected and Compared ..SUCCESSFULLY"
				if ($errorCount -gt 0)
				{
					LogErr "Found $errorCount fail/error/warning messages in kernel logs during execution."
					$retValue = $false
				}
			}
			else
			{
				LogMsg "pass value for status variable either final or initial"
				$retValue = $false
			}
		}
	}
	return $retValue
}

Function SetDistroSpecificVariables($detectedDistro)
{
	$python_cmd = "python"
	LogMsg "Set `$python_cmd > python"	
	Set-Variable -Name python_cmd -Value $python_cmd -Scope Global
	Set-Variable -Name ifconfig_cmd -Value "ifconfig" -Scope Global
	if(($detectedDistro -eq "SLES") -or ($detectedDistro -eq "SUSE"))
	{
		Set-Variable -Name ifconfig_cmd -Value "/sbin/ifconfig" -Scope Global
		Set-Variable -Name fdisk -Value "/sbin/fdisk" -Scope Global
		LogMsg "Set `$ifconfig_cmd > $ifconfig_cmd for $detectedDistro"
		LogMsg "Set `$fdisk > /sbin/fdisk for $detectedDistro"
	}
	else
	{
		Set-Variable -Name fdisk -Value "fdisk" -Scope Global
		LogMsg "Set `$fdisk > fdisk for $detectedDistro"
	}
}

Function DeployVMs ($xmlConfig, $setupType, $Distro, $getLogsIfFailed = $false, [Switch]$GetDeploymentStatistics)
{
	if( (!$EconomyMode) -or ( $EconomyMode -and ($xmlConfig.config.Azure.Deployment.$setupType.isDeployed -eq "NO")))
	{
		try
		{
			$position = 0
			$VerifiedServices =  $NULL
			$retValue = $NULL
			$ExistingServices = Get-AzureService
			$position = 1
			$i = 0
			$role = 1
			$setupTypeData = $xmlConfig.config.Azure.Deployment.$setupType
			$isAllDeployed = CreateAllDeployments -xmlConfig $xmlConfig -setupType $setupType -Distro $Distro
			$isAllVerified = "False"
			$isAllConnected = "False"
			if($isAllDeployed[0] -eq "True")
			{
				$deployedServices = $isAllDeployed[1]
				$DeploymentElapsedTime = $isAllDeployed[2]
				$servicesToVerify = $deployedServices.Split('^') ########
				if ( $GetDeploymentStatistics )
				{
					$VMBooTime = GetVMBootTime -DeployedServices $deployedServices -TimeoutInSeconds 1800
					$verifyAll = VerifyAllDeployments -servicesToVerify $servicesToVerify -GetVMProvisionTime $GetDeploymentStatistics
					$isAllVerified = $verifyAll[0]
					$VMProvisionTime = $verifyAll[1]
				}
				else
				{
					$isAllVerified = VerifyAllDeployments -servicesToVerify $servicesToVerify
				}
				if ($isAllVerified -eq "True")
				{
					$isAllConnected = isAllSSHPortsEnabled -DeployedServices $deployedServices
					if ($isAllConnected -eq "True")
					{
			#Set-Content .\temp\DeployedServicesFile.txt "$deployedServices"
						$VerifiedServices = $deployedServices
						$retValue = $VerifiedServices
						$vnetIsAllConfigured = $false
						$xmlConfig.config.Azure.Deployment.$setupType.isDeployed = $retValue
					#Collecting Initial Kernel
						$user=$xmlConfig.config.Azure.Deployment.Data.UserName
						$KernelLogOutput= GetAndCheckKernelLogs -DeployedServices $deployedServices -status "Initial"
					}
					else
					{
						Write-Host "Unable to connect Some/All SSH ports.."
						$retValue = $NULL  
					}
				}
				else
				{
					Write-Host "Provision Failed for one or more VMs"
					$retValue = $NULL
				}

			}
			else
			{
				Write-Host "One or More Deployments are Failed..!"
				$retValue = $NULL
			}
			# get the logs of the first provision-failed VM
			if ($retValue -eq $NULL -and $getLogsIfFailed -and $DebugOsImage)
			{
				foreach ($service in $servicesToVerify)
				{
					$VMs = Get-AzureVM -ServiceName $service
					foreach ($vm in $VMs)
					{
						if ($vm.InstanceStatus -ne "ReadyRole" )
						{
							$out = GetLogsFromProvisionFailedVM -vmName $vm.Name -serviceName $service -xmlConfig $xmlConfig
							return $NULL
						}
					}
				}
			}
		}
		catch
		{
			if ($position -eq 0)
			{
				Write-Host "Failed to execute Get-AzureService. Source : DeployVMs()"
			}
			else
			{
				Write-Host "Exception detected. Source : DeployVMs()"
			}
			$retValue = $NULL
		}
	}
	else
	{
		$retValue = $xmlConfig.config.Azure.Deployment.$setupType.isDeployed
		$KernelLogOutput= GetAndCheckKernelLogs -DeployedServices $retValue -status "Initial"
	}
	Set-Variable -Name setupType -Value $setupType -Scope Global
	if ( $GetDeploymentStatistics )
	{
		return $retValue, $DeploymentElapsedTime, $VMBooTime, $VMProvisionTime
	}
	else
	{
		return $retValue
	}
}

function GetLogsFromProvisionFailedVM ($vmName, $serviceName, $xmlConfig)
{
	try
	{
		LogMsg "Stopping the provision-failed VM : $vmName"
		$tmp = Stop-AzureVM -ServiceName $serviceName -Name $vmName -Force
		LogMsg "Stopped the VM succussfully"
		
		LogMsg "Capturing the provision-failed VM Image"
		$ErrorImageName = "$serviceName-fail"
		$tmp = Save-AzureVMImage -ServiceName $serviceName -Name $vmName -NewImageName $ErrorImageName -NewImageLabel $ErrorImageName
		LogMsg "Successfully captured VM image : $ErrorImageName"
		$vhdLink = (Get-AzureVMImage -ImageName $ErrorImageName).MediaLink

		$debugVMName = "$serviceName-debug"
		$debugVMUser = $xmlConfig.config.Azure.Deployment.Data.UserName
		$debugVMPasswd = $xmlConfig.config.Azure.Deployment.Data.Password

		$debugSshPath = "/home/$debugVMUser/.ssh/authorized_keys"

		LogMsg "Creating debug VM $debugVMName in service $serviceName"
		$newVmConfigCmd = "New-AzureVMConfig -Name $debugVMName -InstanceSize `"Basic_A1`" -ImageName $DebugOsImage | Add-AzureProvisioningConfig -Linux -LinuxUser $debugVMUser -Password $debugVMPasswd -SSHPublicKeys (New-AzureSSHKey -PublicKey -Fingerprint `"690076D4C41C1DE677CD464EA63B44AE94C2E621`" -Path $debugSshPath) | Set-AzureEndpoint -Name `"SSH`" -LocalPort 22 -PublicPort 22 -Protocol `"TCP`""
		$newVmCmd = "New-AzureVM -ServiceName $serviceName -VMs ($newVmConfigCmd)"
		
		$out = RunAzureCmd -AzureCmdlet $newVmCmd

		$isVerified = VerifyAllDeployments -servicesToVerify @($serviceName)
		if ($isVerified -eq "True")
		{
			$isConnected = isAllSSHPortsEnabled -DeployedServices $serviceName
			if ($isConnected -ne "True")
			{
				return
			}
		}

		LogMsg "Removing image $ErrorImageName, keep the VHD $vhdLink"
		Remove-AzureVMImage -ImageName $ErrorImageName

		LogMsg "Attaching VHD $vhdLink to VM $debugVMName"	
		$vm = Get-AzureVM -ServiceName $serviceName -Name $debugVMName
		$vm | Add-AzureDataDisk -ImportFrom -MediaLocation $vhdLink -DiskLabel "main" -LUN 0 | Update-AzureVM

		$ip = (Get-AzureEndpoint -VM $vm)[0].Vip

		$runFile = "remote-scripts\GetLogFromDataDisk.py"
		$out = RemoteCopy -uploadTo $ip -port 22  -files "$runFile" -username $debugVMUser -password $debugVMPasswd -upload

		$out = RunLinuxCmd -ip $ip -port 22 -username $debugVMUser -password $debugVMPasswd -command "chmod +x *" -runAsSudo
		$out = RunLinuxCmd -ip $ip -port 22 -username $debugVMUser -password $debugVMPasswd -command "./GetLogFromDataDisk.py -u $debugVMUser" -runAsSudo

		$dir = "$LogDir\$vmName"
		if (-not (Test-Path $dir))
		{
			mkdir $dir
		}
		LogMsg "Downloading logs from the VHD"
		$out = RemoteCopy -download -downloadFrom $ip -port 22 -files "/home/$debugVMUser/waagent.log" -downloadTo $dir -username $debugVMUser -password $debugVMPasswd
		$out = RemoteCopy -download -downloadFrom $ip -port 22 -files "/home/$debugVMUser/messages.log" -downloadTo $dir -username $debugVMUser -password $debugVMPasswd
		$out = RemoteCopy -download -downloadFrom $ip -port 22 -files "/home/$debugVMUser/dmesg.log" -downloadTo $dir -username $debugVMUser -password $debugVMPasswd

		LogMsg "Stopping VM $debugVMName"
		$tmp = Stop-AzureVM -ServiceName $serviceName -Name $debugVMName -Force

		# Remove the Cloud Service
		LogMsg "Executing: Remove-AzureService -ServiceName $serviceName -Force -DeleteAll"
		Remove-AzureService -ServiceName $serviceName -Force -DeleteAll
	}
	catch
	{
		$ErrorMessage =  $_.Exception.Message
		LogMsg "EXCEPTION in GetLogsFromProvisionFailedVM() : $ErrorMessage"
	}
}
Function Test-TCP($testIP, $testport)
{
	$socket = new-object Net.Sockets.TcpClient
	$isConnected = "False"
	try
	{
		$socket.Connect($testIP, $testPort) 
	}
	catch [System.Net.Sockets.SocketException]
	{
	}
	if ($socket.Connected) 
	{
		$isConnected = "True"
	}
	$socket.Close()
	return $isConnected
}

Function GetOSImageFromDistro($Distro, $xmlConfig)
{
	foreach ($newDistro in $xmlConfig.config.Azure.Deployment.Data.Distro)  #........v-shisav... needed for getting the OS image.. Removed hardcoding...
	{ 
		if ($newDistro.Name -eq $Distro)
		{
			$osImage = $newDistro.OsImage
			break
		}
	}
	return $osImage
}

Function SetOSImageToDistro($Distro, $xmlConfig, $ImageName)
{
	$i = 0
	foreach ($newDistro in $xmlConfig.config.Azure.Deployment.Data.Distro)  #........v-shisav... needed for getting the OS image.. Removed hardcoding...
	{ 
		if ($newDistro.Name -eq $Distro)
		{
			$xmlConfig.config.Azure.Deployment.Data.Distro[$i].OsImage = $ImageName
			break
		}
		$i = $i + 1
	}
	Set-Variable -Name xmlConfig -Value $xmlConfig -Scope Global
	return $true
}

Function GetPort($Endpoints, $usage)
{
	foreach ($port in $Endpoints)
	{
		if ($port.Name -imatch $usage)
		{
			$tcpPort = $port.Port
			return $tcpPort
			break
		}
	}
}

Function isAllSSHPortsEnabled($DeployedServices)
{
	LogMsg "Trying to Connect to deployed VM(s)"

	$TestIPPOrts = ""

	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$AllEndpoints = Get-AzureEndpoint -VM $testVM
			$HSVIP = GetHsVmVip -servicename $hostedservice
			$HSport = GetPort -Endpoints $AllEndpoints -usage SSH
			if($TestIPPOrts)
			{
				$TestIPPOrts = $TestIPPOrts + "^$HSVIP" + ':' +"$HSport"
			}
			else
			{
				$TestIPPOrts = "$HSVIP" + ':' +"$HSport"
			}


		}
	}
	$timeout = 0
	do
	{
		$WaitingForConnect = 0
		foreach ($IPPORT in $TestIPPOrts.Split("^"))
		{
			$IPPORT = $IPPORT.Split(":")
			$testIP = $IPPORT[0]
			$testPort = $IPPORT[1]
			Write-Host "Connecting to  $TestIP : $testPort" -NoNewline
			$out = Test-TCP  -testIP $TestIP -testport $testPort
			if ($out -ne "True")
			{ 
				Write-Host " : Failed"
				$WaitingForConnect = $WaitingForConnect + 1
			}
			else
			{
				Write-Host " : Connected"
			}
		}
		if($WaitingForConnect -gt 0)
		{
			$timeout = $timeout + 1
			Write-Host "$WaitingForConnect VM(s) still awaiting to open SSH port.." -NoNewline
			Write-Host "Retry $timeout/100"
			sleep 3
			$retValue = "False"
		}
		else
		{
			LogMsg "ALL VM's SSH port is/are open now.."
			$retValue = "True"
		}

	}
	While (($timeout -lt 100) -and ($WaitingForConnect -gt 0))

	return $retValue
}
Function GetVMBootTime($DeployedServices, $TimeoutInSeconds)
{
	$VMBootStarted = Get-Date
	$TestIPPOrts = ""
	$sleepTime = 3
	$maxRetryCount = $TimeoutInSeconds / $sleepTime
	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$AllEndpoints = Get-AzureEndpoint -VM $testVM
			$HSVIP = GetHsVmVip -servicename $hostedservice
			$HSport = GetPort -Endpoints $AllEndpoints -usage SSH
			if($TestIPPOrts)
			{
				$TestIPPOrts = $TestIPPOrts + "^$HSVIP" + ':' +"$HSport"
			}
			else
			{
				$TestIPPOrts = "$HSVIP" + ':' +"$HSport"
			}
		}
	}
	$timeout = 0
	do
	{
		$WaitingForConnect = 0
		foreach ($IPPORT in $TestIPPOrts.Split("^"))
		{
			$IPPORT = $IPPORT.Split(":")
			$testIP = $IPPORT[0]
			$testPort = $IPPORT[1]
			Write-Host "Connecting to  $TestIP : $testPort" -NoNewline
			$out = Test-TCP  -testIP $TestIP -testport $testPort
			if ($out -ne "True")
			{ 
				Write-Host " : Failed"
				$WaitingForConnect = $WaitingForConnect + 1
			}
			else
			{
				Write-Host " : Connected"
			}
			
		}
		
		if($WaitingForConnect -gt 0)
		{
			$timeout = $timeout + 1
			Write-Host "$WaitingForConnect VM(s) still awaiting to open SSH port.." -NoNewline
			Write-Host "Retry $timeout/$maxRetryCount"
			sleep $sleepTime
			$retValue = "False"
		}
		else
		{
			LogMsg "ALL VM's SSH port is/are open now.."
			$retValue = "True"
		}
	}
	While (($timeout -lt $maxRetryCount) -and ($WaitingForConnect -gt 0))
	$VMBootFinished =  Get-Date
	$VMBootElapsedTime = $VMBootFinished - $VMBootStarted
	return $VMBootElapsedTime
}
Function GetHsVmVip($servicename)
{
	$endpoints = Get-AzureVM  -ServiceName $servicename | Get-AzureEndpoint
	$vip = $endpoints[0].Vip
	return $VIP
}

Function GetProbePort($Endpoints, $usage)
{
	foreach ($port in $Endpoints)
	{
		if ($port.Name -imatch $usage)
		{
			$tcpPort = $port.ProbePort
			return $tcpPort
			break
		}
	}
}

Function ApplyDONOTDISTURBtoHostedServices($DeployedServices)
{
	$DeployedServices = $DeployedServices.split("^")
	foreach ( $serviceName in $DeployedServices )
	{
		Set-AzureService -ServiceName $serviceName -Description "DONOTDISTURB"
	}
}

Function GetPlatfromImages()
{
	$allImages = Get-AzureVMImage
	$count = 1
	foreach ($image in $allImages)
	{
		if((!$image.MediaLink) -and ($image.OS -eq "Linux"))
		{
			Write-Host "$count. " -NoNewline -ForegroundColor Gray
			Write-Host "$($image.ImageName)"
			$count += 1
		}
	}
}	

Function RemoveEmptyHostedServices()
{
	$allServices = Get-AzureService
	$TotalServices = $allServices.Length
	$ServicesRemaining = $TotalServices
	$count = 1
	foreach ($service in $allServices)
	{
		$ServicesRemaining = $ServicesRemaining - 1
		$Percentage = (100 - (($ServicesRemaining/$TotalServices)*100))
		write-progress -Id 47 -activity Cheking.. -Status "$($service.ServiceName) ... " -percentcomplete $Percentage
		Write-Host "Checking $($service.ServiceName) ... " -NoNewline
		$out = Get-AzureDeployment -ServiceName $service.ServiceName 2>&1
		if ($out -imatch  "No deployments were found")
		{
			Write-Host "Deleting now... " -ForegroundColor Red -NoNewline
			write-progress -Id 47 -activity Deleting.. -Status "$($service.ServiceName) ... " -percentcomplete $Percentage
			$out = Remove-AzureService -ServiceName $service.ServiceName -Force
			Write-Host "Deleted!" -ForegroundColor Green
			write-progress -Id 47 -activity Deleted.. -Status "$($service.ServiceName)!" -percentcomplete $Percentage
		}
		else
		{
			write-progress -Id 47 -activity Cheking.. -Status "$($service.ServiceName) ... OK" -percentcomplete $Percentage
			Write-Host "OK!" -ForegroundColor Green
		}
		write-progress -Id 47 -activity SLEEPING -Status "Wait Completed..!" -percentcomplete $Percentage
	}
	write-progress -Id 47 -activity "Empty Services Removed..!"  -percentcomplete 100
}


Function RemoveICAOsImages([switch]$removeUploaded, [switch]$removeCaptured, [switch]$removeVHD)
{
	$allImages = Get-AzureVMImage
	$IcaUploaded = @()
	$IcaCaptured = @()
	$totalCount = 0
	$uploadedCount = 0
	$capturedCount = 0

	foreach ($image in $allImages)
	{
		if ( ($image.ImageName -imatch "ICA-UPLOADED") -or  ($image.ImageName -imatch "ICA-CAPTURED") )
		{
			if ( ($removeUploaded) -and ($image.ImageName -imatch "ICA-UPLOADED") )
			{
				$totalCount += 1
			}
			elseif (($removeCaptured) -and ($image.ImageName -imatch "ICA-CAPTURED"))
			{
				$totalCount += 1
			}
		}
	}
	Write-Host "Total Count =  $totalCount"
	$totalRemaining = $totalCount
	Write-Progress -Activity "Removing ICA OS Images" -Id 50 -PercentComplete 0
	foreach ($image in $allImages)
	{
		$SrNo = $totalCount - $totalRemaining + 1
		$tempImage =   $image.ImageName	  
		if ($image.ImageName -imatch "ICA-UPLOADED")
		{

			if($removeUploaded)
			{
				Write-Progress  -Id 50 -Activity "Removing ICA OS Images" -CurrentOperation "$SrNo/$totalCount. $tempImage" -PercentComplete (100 - (($totalRemaining/$totalCount)*100))
				if ($removeVHD)
				{
					Write-Host "Removing $tempImage with VHD"
					$out = Remove-AzureVMImage -ImageName $tempImage  -DeleteVHD

				}
				else
				{
					Write-Host "Removing $tempImage keeping VHD"
					$out = Remove-AzureVMImage -ImageName $tempImage

				}
				$totalRemaining -= 1
			}
		}
		if ($image.ImageName -imatch "ICA-CAPTURED")
		{
			if($removeCaptured)
			{
				Write-Progress  -Id 50 -Activity "Removing ICA OS Images" -CurrentOperation "$SrNo/$totalCount. $tempImage" -PercentComplete (100 - (($totalRemaining/$totalCount)*100))
				if ($removeVHD)
				{
					Write-Host "Removing $tempImage with VHD"
					$out = Remove-AzureVMImage -ImageName $tempImage  -DeleteVHD

				}
				else
				{
					Write-Host "Removing $tempImage keeping VHD"
					$out = Remove-AzureVMImage -ImageName $tempImage

				}
				$totalRemaining -= 1
			} 
		}
	}
	Write-Progress  -Id 50 -Activity "Removing ICA OS Images" -Completed 
}

Function GetTestVMHardwareDetails ($xmlConfigFile, $setupType, [switch]$VCPU, [switch]$RAM)
{
	$switchCount = 0
	if ($VCPU)
	{
		$switchCount += 1
	}
	if ($RAM)
	{
		$switchCount += 1
	}

	if ($switchCount -lt 1)
	{
		LogErr "Use atleast one switch from `$VCPU or `$RAM.."
		$retValue = "Error"
	}
	elseif ($switchCount -gt 1 )
	{
		LogErr "Use only one switch from `$VCPU or `$RAM.."
		$retValue = "Error"
	}
	elseif ($switchCount -eq 1)
	{
		try
		{
			$testVMSize = $xmlConfigFile.config.Azure.Deployment.$setupType.HostedService.VirtualMachine.InstanceSize
			foreach ($size in $testVMSize)
			{
				$testVMDetails = $xmlConfigFile.config.Azure.Deployment.Data.VMsizes.$size 
				if ($VCPU)
				{
					if (!$retValue)
					{
						$retValue = $($testVMDetails.VCPU)
					}
					else
					{
						$retValue = "$retValue," + $($testVMDetails.VCPU)
					}
				}
				elseif ($RAM)
				{
					if (!$retValue)
					{
						$retValue = $($testVMDetails.RAM)
					}
					else
					{
						$retValue = "$retValue," + $($testVMDetails.RAM)
					}				
				}
			}
		}
		catch
		{
			LogErr "Unable to find $setupType details."
			$retValue = "Unknown"
		}
	}

	return $retValue
}
#endregion

#region Linux Commands Methods
Function RemoteCopy($uploadTo, $downloadFrom, $downloadTo, $port, $files, $username, $password, [switch]$upload, [switch]$download, [switch]$usePrivateKey) #Removed XML config
{
	$retry=1
	$maxRetry=3
	if($upload)
	{
#LogMsg "Uploading the files"
		if ($files)
		{
			$files = $files.split(",")
			foreach ($f in $files)
			{
				if ( !$f )
				{
					continue
				}
				$retry=1
				$maxRetry=10
				$testFile = $f.trim()
				if ( ( $f -imatch ".sh" ) -or ( $f -imatch ".py" ))
				{
					$out = .\tools\dos2unix.exe $f 2>&1
					LogMsg $out
				}
				$recurse = ""
				while($retry -le $maxRetry)
				{
					if($usePrivateKey)
					{
						LogMsg "Uploading $testFile to $username : $uploadTo, port $port using PrivateKey authentication"
						echo y | .\tools\pscp -i .\ssh\$sshKey -q -P $port $testFile $username@${uploadTo}:
						$returnCode = $LASTEXITCODE
					}
					else
					{
						LogMsg "Uploading $testFile to $username : $uploadTo, port $port using Password authentication"
						echo y | .\tools\pscp -pw $password -q -P $port $testFile $username@${uploadTo}:
						$returnCode = $LASTEXITCODE
					}
					if(($returnCode -ne 0) -and ($retry -ne $maxRetry))
					{
						LogWarn "Error in upload, Attempt $retry. Retrying for upload"
						$retry=$retry+1
					}
					elseif(($returnCode -ne 0) -and ($retry -eq $maxRetry))
					{
						Write-Host "Error in upload after $retry Attempt,Hence giving up"
						$retry=$retry+1
						Throw "Error in upload after $retry Attempt,Hence giving up"
					}
					elseif($returnCode -eq 0)
					{
						LogMsg "Upload Success after $retry Attempt"
						$retry=$maxRetry+1
					}
				}
			}
		}
		else
		{
			LogMsg "No Files to upload...!"
			Throw "No Files to upload...!"
		}

	}
	elseif($download)
	{
#Downloading the files
		if ($files)
		{
			$files = $files.split(",")
			foreach ($f in $files)
			{
				$retry=1
				$maxRetry=10
				$testFile = $f.trim()
				$recurse = ""
				while($retry -le $maxRetry)
				{
					if($usePrivateKey)
					{
						LogMsg "Downloading $testFile from $username : $downloadFrom,port $port to $downloadTo using PrivateKey authentication"
						echo y | .\tools\pscp -i .\ssh\$sshKey -q -P $port $username@${downloadFrom}:$testFile $downloadTo
						$returnCode = $LASTEXITCODE
					}
					else
					{
						LogMsg "Downloading $testFile from $username : $downloadFrom,port $port to $downloadTo using Password authentication"
						echo y | .\tools\pscp -pw $password -q -P $port $username@${downloadFrom}:$testFile $downloadTo
						$returnCode = $LASTEXITCODE
					}
					if(($returnCode -ne 0) -and ($retry -ne $maxRetry))
					{
						LogWarn "Error in download, Attempt $retry. Retrying for download"
						$retry=$retry+1
					}
					elseif(($returnCode -ne 0) -and ($retry -eq $maxRetry))
					{
						Write-Host "Error in download after $retry Attempt,Hence giving up"
						$retry=$retry+1
						Throw "Error in download after $retry Attempt,Hence giving up."
					}
					elseif($returnCode -eq 0)
					{
						LogMsg "Download Success after $retry Attempt"
						$retry=$maxRetry+1
					}
				}
			}
		}
		else
		{
			LogMsg "No Files to download...!"
			Throw "No Files to download...!"
		}
	}
	else
	{
		LogMsg "Error: Upload/Download switch is not used!"
	}
}

Function RunLinuxCmd([string] $username,[string] $password,[string] $ip,[string] $command, [int] $port, [switch]$runAsSudo, [Boolean]$WriteHostOnly, [Boolean]$NoLogsPlease, [switch]$ignoreLinuxExitCode, [int]$runMaxAllowedTime = 300, [switch]$RunInBackGround)
{
	$randomFileName = [System.IO.Path]::GetRandomFileName()
	$maxRetryCount = 10
	$currentDir = $PWD.Path
	$RunStartTime = Get-Date
	
	if($runAsSudo)
	{
		$plainTextPassword = $password.Replace('"','');
		if ( $detectedDistro -eq "COREOS" )
		{
			$linuxCommand = "`"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/share/oem/bin:/usr/share/oem/python/bin:/opt/bin && echo $plainTextPassword | sudo -S env `"PATH=`$PATH`" $command && echo AZURE-LINUX-EXIT-CODE-`$? || echo AZURE-LINUX-EXIT-CODE-`$?`""
			$logCommand = "`"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/share/oem/bin:/usr/share/oem/python/bin:/opt/bin && echo $plainTextPassword | sudo -S env `"PATH=`$PATH`" $command`""
		}
		else
		{
			$linuxCommand = "`"echo $plainTextPassword | sudo -S $command && echo AZURE-LINUX-EXIT-CODE-`$? || echo AZURE-LINUX-EXIT-CODE-`$?`""
			$logCommand = "`"echo $plainTextPassword | sudo -S $command`""
		}
	}
	else
	{
		if ( $detectedDistro -eq "COREOS" )
		{
			$linuxCommand = "`"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/share/oem/bin:/usr/share/oem/python/bin:/opt/bin && $command && echo AZURE-LINUX-EXIT-CODE-`$? || echo AZURE-LINUX-EXIT-CODE-`$?`""
			$logCommand = "`"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/share/oem/bin:/usr/share/oem/python/bin:/opt/bin && $command`""		
		}
		else
		{
			$linuxCommand = "`"$command && echo AZURE-LINUX-EXIT-CODE-`$? || echo AZURE-LINUX-EXIT-CODE-`$?`""
			$logCommand = "`"$command`""
		}
	}
	LogMsg ".\tools\plink.exe -t -pw $password -P $port $username@$ip $logCommand"
	$returnCode = 1
	$attempts = 0
	$notExceededTimeLimit = $true
	$isBackGroundProcessStarted = $false
	while ( ($returnCode -ne 0) -and ($attempts -lt $maxRetryCount) -and $notExceededTimeLimit)
	{
		$attempts += 1
		$runLinuxCmdJob = Start-Job -ScriptBlock `
		{ `
			$username = $args[1]; $password = $args[2]; $ip = $args[3]; $port = $args[4]; $jcommand = $args[5]; `
			cd $args[0]; `
			#Write-Host ".\tools\plink.exe -t -C -v -pw $password -P $port $username@$ip $jcommand";`
			.\tools\plink.exe -t -C -v -pw $password -P $port $username@$ip $jcommand;`
		} `
		-ArgumentList $currentDir, $username, $password, $ip, $port, $linuxCommand
		$RunLinuxCmdOutput = ""
		$debugOutput = ""
		$LinuxExitCode = ""
		if ( $RunInBackGround )
		{
			While(($runLinuxCmdJob.State -eq "Running") -and ($isBackGroundProcessStarted -eq $false ) -and $notExceededTimeLimit)
			{
				$SSHOut = Receive-Job $runLinuxCmdJob 2> $LogDir\$randomFileName
				$JobOut = Get-Content $LogDir\$randomFileName
				if($jobOut)
				{
					foreach($outLine in $jobOut)
					{
						if($outLine -imatch "Started a shell")
						{
							$LinuxExitCode = $outLine
							$isBackGroundProcessStarted = $true
							$returnCode = 0
						}
						else
						{
							$RunLinuxCmdOutput += "$outLine`n"
						}
					}
				}
				$debugLines = Get-Content $LogDir\$randomFileName
				if($debugLines)
				{
					$debugString = ""
					foreach ($line in $debugLines)
					{
						$debugString += $line
					}
					$debugOutput += "$debugString`n"
				}
				Write-Progress -Activity "Attempt : $attempts : Initiating command in Background Mode : $logCommand on $ip : $port" -Status "Timeout in $($RunMaxAllowedTime - $RunElaplsedTime) seconds.." -Id 87678 -PercentComplete (($RunElaplsedTime/$RunMaxAllowedTime)*100) -CurrentOperation "SSH ACTIVITY : $debugString"
				$RunCurrentTime = Get-Date
				$RunDiffTime = $RunCurrentTime - $RunStartTime
				$RunElaplsedTime =  $RunDiffTime.TotalSeconds
				if($RunElaplsedTime -le $RunMaxAllowedTime)
				{
					$notExceededTimeLimit = $true
				}
				else
				{
					$notExceededTimeLimit = $false
					Stop-Job $runLinuxCmdJob
					$timeOut = $true
				}
			}
			WaitFor -seconds 5
			$SSHOut = Receive-Job $runLinuxCmdJob 2> $LogDir\$randomFileName
			if($SSHOut )
			{
				foreach ($outLine in $SSHOut)
				{
					if($outLine -imatch "AZURE-LINUX-EXIT-CODE-")
					{
						$LinuxExitCode = $outLine
						$isBackGroundProcessTerminated = $true
					}
					else
					{
						$RunLinuxCmdOutput += "$outLine`n"
					}
				}
			}
			
			$debugLines = Get-Content $LogDir\$randomFileName
			if($debugLines)
			{
				$debugString = ""
				foreach ($line in $debugLines)
				{
					$debugString += $line
				}
				$debugOutput += "$debugString`n"
			}
			Write-Progress -Activity "Attempt : $attempts : Executing $logCommand on $ip : $port" -Status $runLinuxCmdJob.State -Id 87678 -SecondsRemaining ($RunMaxAllowedTime - $RunElaplsedTime) -Completed
			if ( $isBackGroundProcessStarted -and !$isBackGroundProcessTerminated )
			{
				LogMsg "$command is running in background with ID $($runLinuxCmdJob.Id) ..."
				Add-Content -Path $LogDir\CurrentTestBackgroundJobs.txt -Value $runLinuxCmdJob.Id
			}
			else
			{
				Remove-Job $runLinuxCmdJob 
				if (!$isBackGroundProcessStarted)
				{
					LogErr "Failed to start process in background.."
				}
				if ( $isBackGroundProcessTerminated )
				{
					LogErr "Background Process terminated from Linux side with error code :  $($LinuxExitCode.Split("-")[4])"
					$returnCode = $($LinuxExitCode.Split("-")[4])
					LogErr $SSHOut
				}
				if($debugOutput -imatch "Unable to authenticate")
				{
					LogMsg "Unable to authenticate. Not retrying!"
					Throw "Unable to authenticate"

				}
				if($timeOut)
				{
					$retValue = ""
					Throw "Tmeout while executing command : $command"
				}
				LogErr "Linux machine returned exit code : $($LinuxExitCode.Split("-")[4])"
				if ($attempts -eq $maxRetryCount)
				{
					Throw "Failed to execute : $command."
				}
				else
				{
					if ($notExceededTimeLimit)
					{
						LogMsg "Failed to execute : $command. Retrying..."
					}
				}
			}
			Remove-Item $LogDir\$randomFileName -Force | Out-Null   
		}
		else
		{
			While($notExceededTimeLimit -and ($runLinuxCmdJob.State -eq "Running"))
			{
				$jobOut = Receive-Job $runLinuxCmdJob 2> $LogDir\$randomFileName
				if($jobOut)
				{
					foreach ($outLine in $jobOut)
					{
						if($outLine -imatch "AZURE-LINUX-EXIT-CODE-")
						{
							$LinuxExitCode = $outLine
						}
						else
						{
							$RunLinuxCmdOutput += "$outLine`n"
						}
					}
				}
				$debugLines = Get-Content $LogDir\$randomFileName
				if($debugLines)
				{
					$debugString = ""
					foreach ($line in $debugLines)
					{
						$debugString += $line
					}
					$debugOutput += "$debugString`n"
				}
				Write-Progress -Activity "Attempt : $attempts : Executing $logCommand on $ip : $port" -Status "Timeout in $($RunMaxAllowedTime - $RunElaplsedTime) seconds.." -Id 87678 -PercentComplete (($RunElaplsedTime/$RunMaxAllowedTime)*100) -CurrentOperation "SSH ACTIVITY : $debugString"
				$RunCurrentTime = Get-Date
				$RunDiffTime = $RunCurrentTime - $RunStartTime
				$RunElaplsedTime =  $RunDiffTime.TotalSeconds
				if($RunElaplsedTime -le $RunMaxAllowedTime)
				{
					$notExceededTimeLimit = $true
				}
				else
				{
					$notExceededTimeLimit = $false
					Stop-Job $runLinuxCmdJob
					$timeOut = $true
				}
			}
			$jobOut = Receive-Job $runLinuxCmdJob 2> $LogDir\$randomFileName
			if($jobOut)
			{
				foreach ($outLine in $jobOut)
				{
					if($outLine -imatch "AZURE-LINUX-EXIT-CODE-")
					{
						$LinuxExitCode = $outLine
					}
					else
					{
						$RunLinuxCmdOutput += "$outLine`n"
					}
				}
			}
			$debugLines = Get-Content $LogDir\$randomFileName
			if($debugLines)
			{
				$debugString = ""
				foreach ($line in $debugLines)
				{
					$debugString += $line
				}
				$debugOutput += "$debugString`n"
			}
			Write-Progress -Activity "Attempt : $attempts : Executing $logCommand on $ip : $port" -Status $runLinuxCmdJob.State -Id 87678 -SecondsRemaining ($RunMaxAllowedTime - $RunElaplsedTime) -Completed
			Remove-Job $runLinuxCmdJob 
			Remove-Item $LogDir\$randomFileName -Force | Out-Null
			if ($LinuxExitCode -imatch "AZURE-LINUX-EXIT-CODE-0") 
			{
				$returnCode = 0
				LogMsg "$command executed successfully in $RunElaplsedTime seconds." -WriteHostOnly $WriteHostOnly -NoLogsPlease $NoLogsPlease
				$retValue = $RunLinuxCmdOutput.Trim()
			}
			else
			{
				if (!$ignoreLinuxExitCode)
				{
					$debugOutput = ($debugOutput.Split("`n")).Trim()
					foreach ($line in $debugOutput)
					{
						if($line)
						{
							LogErr $line
						}
					}
				}
				if($debugOutput -imatch "Unable to authenticate")
					{
						LogMsg "Unable to authenticate. Not retrying!"
						Throw "Unable to authenticate"

					}
				if(!$ignoreLinuxExitCode)
				{
					if($timeOut)
					{
						$retValue = ""
						Throw "Tmeout while executing command : $command"
					}
					LogErr "Linux machine returned exit code : $($LinuxExitCode.Split("-")[4])"
					if ($attempts -eq $maxRetryCount)
					{
						Throw "Failed to execute : $command."
					}
					else
					{
						if ($notExceededTimeLimit)
						{
							LogMsg "Failed to execute : $command. Retrying..."
						}
					}
				}
				else
				{
					LogMsg "Command execution returned return code $($LinuxExitCode.Split("-")[4]) Ignoring.."
					$retValue = $RunLinuxCmdOutput.Trim()
					break
				}
			}
		}
	}
	return $retValue
}
#endregion

#region Test Case Logging
Function DoTestCleanUp($result, $testName, $DeployedServices, [switch]$keepUserDirectory, [switch]$SkipVerifyKernelLogs)
{
	try
	{
		if($DeployedServices)
		{
			$currentTestBackgroundJobs = Get-Content $LogDir\CurrentTestBackgroundJobs.txt -ErrorAction SilentlyContinue
			if ( $currentTestBackgroundJobs )
			{
				$currentTestBackgroundJobs = $currentTestBackgroundJobs.Split()
			}
			foreach ( $taskID in $currentTestBackgroundJobs )
			{
				#Removal of background 
				LogMsg "Removing Background Job ID : $taskID..."
				Remove-Job -Id $taskID -Force
			}
			$user=$xmlConfig.config.Azure.Deployment.Data.UserName
			if ( !$SkipVerifyKernelLogs )
			{
				try
				{
					$KernelLogOutput=GetAndCheckKernelLogs -DeployedServices $deployedServices -status "Final" #Collecting kernel logs after execution of test case : v-sirebb
				}
				catch 
				{
					$ErrorMessage =  $_.Exception.Message
					LogMsg "EXCEPTION in GetAndCheckKernelLogs(): $ErrorMessage"	
				}
			}			
			$isClened = @()
			$hsNames = $DeployedServices
			$hsNames = $hsNames.Split("^")
			foreach ($hs in $hsNames)
			{
				$hsDetails = Get-AzureService -ServiceName $hs
				if (!($hsDetails.Description -imatch "DONOTDISTURB"))
				{
					if($result -eq "PASS")
					{
						if($EconomyMode -and (-not $IsLastCaseInCycle))
						{
							LogMsg "Skipping cleanup of $hs."
							if(!$keepUserDirectory)
							{
								RemoveAllFilesFromHomeDirectory -DeployedServices $hs
							}
						}
						else
						{
							LogMsg "Cleaning up deployed test virtual machines."
							$isClened = DeleteService -serviceName $hsDetails.ServiceName
						
							if ($isClened -contains "False")
							{
								LogMsg "CleanUP unsuccessful for $($hsDetails.ServiceName).. Please delete the services manually."
							}
							else
							{
								LogMsg "CleanUP Successful for $($hsDetails.ServiceName).."
							}
						}
					}
					else
					{
						LogMsg "Preserving the hosted service(s) $hsNames"
						LogMsg "Integrating Test Case Name in the `"Description`" of preserved setups.."
						foreach ($service in $hsNames)
						{
							$suppressedOut = RetryOperation -operation { RunAzureCmd -AzureCmdlet "Set-AzureService -ServiceName $service -Description `"Preserving this setup for FAILED/ABORTED test : $testName`"" -maxWaitTimeSeconds 120 } -maxRetryCount 5 -retryInterval 5
						}
						LogMsg "Collecting VM logs.."
						GetVMLogs -DeployedServices $hs
						if(!$keepUserDirectory -and !$keepReproInact -and $EconomyMode)
							{
								RemoveAllFilesFromHomeDirectory -DeployedServices $hs
							}
						if($keepReproInact)
						{
							$xmlConfig.config.Azure.Deployment.$setupType.isDeployed = "NO"
						}
					}
				}
				else
				{
					if ($result -ne "PASS")
					{
						LogMsg "Collecting VM logs.."
						GetVMLogs -DeployedServices $hs
						if($keepReproInact)
						{
							$xmlConfig.config.Azure.Deployment.$setupType.isDeployed = "NO"
						}
					}
					LogMsg "Skipping cleanup, as service is marked as DO NOT DISTURB.."
				}
			}
		}
		else
		{
			LogMsg "Skipping cleanup, as No services deployed for cleanup!"
		}
	}
	catch
	{
		$ErrorMessage =  $_.Exception.Message
		Write-Host "EXCEPTION in DoTestCleanUp : $ErrorMessage"  
	}
}

Function GetFinalizedResult($resultArr, $checkValues, $subtestValues, $currentTestData)
{
	$result = "", ""
	if (($resultArr -contains "FAIL") -or ($resultArr -contains "Aborted")) {
		$result[0] = "FAIL"
	}
	else{
		$result[0] = "PASS"
	}
	$i = 0
	$subtestLen = $SubtestValues.Length
	while ($i -lt $subtestLen)
	{
		$currentTestValue = $SubtestValues[$i]
		$currentTestResult = $resultArr[$i]
		$currentTestName = $currentTestData.testName
		if ($checkValues -imatch $currentTestResult)
		{				
			$result[1] += "		  $currentTestName : $currentTestValue : $currentTestResult <br />"
		}
		$i = $i + 1
	}

	return $result
}

Function CreateResultSummary($testResult, $checkValues, $testName, $metaData)
{
	if ($checkValues -imatch $testResult)
	{				
		$resultString = "		  $testName : $metaData : $testResult <br />"
	}

	return $resultString
}

Function GetFinalResultHeader($resultArr){
	if(($resultArr -imatch "FAIL" ) -or ($resultArr -imatch "Aborted"))
	{
		$result = "FAIL"
		if($resultArr -imatch "Aborted")
		{
			$result = "Aborted"
		}

	}
	else
	{
		$result = "PASS"
	}
	return $result
}

Function SetStopWatch()
{
	$sw = [system.diagnostics.stopwatch]::startNew()
	return $sw
}

Function GetStopWatchElapasedTime([System.Diagnostics.Stopwatch]$sw, [string] $format)
{
	if ($format -eq "ss")
	{
		$num=$sw.Elapsed.TotalSeconds
	}
	elseif ($format -eq "hh")
	{
		$num=$sw.Elapsed.TotalHours
	}
	elseif ($format -eq "mm")
	{
		$num=$sw.Elapsed.TotalMinutes
	}
	return [System.Math]::Round($Num, 2)

}

Function GetVMLogs($DeployedServices)
{
	$TestIPPOrts = ""

	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$AllEndpoints = Get-AzureEndpoint -VM $testVM
			$HSVIP = GetHsVmVip -servicename $hostedservice
			$HSport = GetPort -Endpoints $AllEndpoints -usage SSH
			$testIP = $HSVIP
			$testPort = $HSport
			#$LisLogFile = "LIS-Logs-" + $testVM.InstanceName + ".tgz"
			$LisLogFile = "LIS-Logs" + ".tgz"
			try
			{
				LogMsg "Collecting logs from IP : $testIP PORT : $testPort"	
				RemoteCopy -upload -uploadTo $testIP -username $user -port $testPort -password $password -files '.\remote-scripts\LIS-LogCollector.sh'
				RunLinuxCmd -username $user -password $password -ip $testIP -port $testPort -command 'chmod +x LIS-LogCollector.sh'
				$out = RunLinuxCmd -username $user -password $password -ip $testIP -port $testPort -command './LIS-LogCollector.sh -v' -runAsSudo
				LogMsg $out
				RemoteCopy -download -downloadFrom $testIP -username $user -password $password -port $testPort -downloadTo $LogDir -files $LisLogFile
				LogMsg "Logs collected successfully from IP : $testIP PORT : $testPort"
				Rename-Item -Path "$LogDir\$LisLogFile" -NewName ("LIS-Logs-" + $testVM.InstanceName + ".tgz") -Force
			}
			catch
			{
				$ErrorMessage =  $_.Exception.Message
				LogErr "EXCEPTION : $ErrorMessage"
				LogErr "Unable to collect logs from IP : $testIP PORT : $testPort"  		
			}
		}
	}
}

Function RemoveAllFilesFromHomeDirectory($DeployedServices)
{
	$TestIPPOrts = ""

	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$AllEndpoints = Get-AzureEndpoint -VM $testVM
			$HSVIP = GetHsVmVip -servicename $hostedservice
			$HSport = GetPort -Endpoints $AllEndpoints -usage SSH
			$testIP = $HSVIP
			$testPort = $HSport
			try
			{
				LogMsg "Removing all files logs from IP : $testIP PORT : $testPort"	
				$out = RunLinuxCmd -username $user -password $password -ip $testIP -port $testPort -command 'rm -rf *' -runAsSudo
				LogMsg "All files removed from /home/$user successfully. VM IP : $testIP PORT : $testPort"  
			}
			catch
			{
				$ErrorMessage =  $_.Exception.Message
				Write-Host "EXCEPTION : $ErrorMessage"
				Write-Host "Unable to remove files from IP : $testIP PORT : $testPort"  		
			}
		}
	}
}

Function CaptureVMImage ($ServiceName)
{
	$retryCount = 5
	$retryRequired = $true
		LogMsg "Stopping the Virtual Machine.."
		$tempVM = Get-AzureVM -ServiceName $ServiceName
		$out = $tempVM | Stop-AzureVM -Force -Verbose
		WaitFor -seconds 60
		LogMsg "Capturing the VM image from service : $ServiceName..."
	#LogMsg ".\SetupScripts\CSUploadVHD.ps1 -Destination $Destination -Label $VHDName -LiteralPath $LiteralPath\$VHDName"
		$curtime = Get-Date
		$ImageName = "ICA-CAPTURED-" + $Distro + "-" + $curtime.Month + "-" +  $curtime.Day  + "-" + $curtime.Year + "-" + $curtime.Hour + "-" + $curtime.Minute + ".vhd"
		$ImageDestination =  $DestinationURL + "/" + $ImageName
		$ImageLabel = $ImageName
		While($retryRequired -and ($retryCount -gt 0))
		{
			try
			{
				$retryCount = $retryCount - 1
				$out = $tempVM | Save-AzureVMImage -NewImageLabel $ImageLabel -NewImageName $ImageName -Verbose
				$retryRequired = $false
				LogMsg "VHD captured with Image Name : $ImageName"
			}
			catch
			{
				$retryRequired = $true
			}
		}
		LogMsg "Removing empty service : $ServiceName"
		$out = DeleteService -ServiceName $ServiceName
		
	return $ImageName
}

function ParseAndAddSubtestResultsToDB($resultSummary, $conn, $testCaseRunObj)
{
	$newResultSummary = $resultSummary.Replace(" ","")
	$newResultSummary = $newResultSummary.Replace("<br/>","^")
	$newResultSummary = $newResultSummary.Remove(($newResultSummary.Length - 1))
	$newResultSummary = $newResultSummary.Split("^")
	$tempCounter = 1
	foreach ($testSummary in $newResultSummary)
	{
		$tempCounter += 1
		$testSummary = $testSummary.Split(":")

		$SubtestName = $testSummary[0]
		$SubtestResult = $testSummary[($testSummary.Length - 1)]
		Function GetResultMetadata ($testSummary)
		{
			$metadata = ""
			$counter = 0
			$firstEntry=$true
			foreach ($word in $testSummary)
			{
				if(($counter -ne 0) -and ($counter -ne ($testSummary.Length - 1)))
				{
					if($firstEntry)
					{
						$metadata += $word
						$firstEntry = $false
					}
					else
					{
						$metadata += " : $word"
					}
				}
				$counter += 1
			}
			return $metadata
		}
		$subtestMetadata = GetResultMetadata -testSummary $testSummary
#Write-Host "$SubtestName : $subtestMetadata : $SubtestResult"

#Now Add this subtest data to DB
		$subTestRunObject = CreateSubTestCaseObject -testSuiteRunId $testCaseRunObj.testSuiteRunId -testCaseId $testCaseRunObj.testCaseId -testName "$SubtestName : $subtestMetadata" -result $SubtestResult -startTime $testStartTime -endTime $testCaseRunObj.endTime -SubtestStartTime $SubTestStartTime -SubtestEndTime $SubTestEndTime
		$temp = AddSubTestResultinDB -conn $Conn -subTestCaseObj $subTestRunObject -testSuiteRunId $testCaseRunObj.testSuiteRunId 
	}
}
#endregion

#region Network FTM TestCase Methods
#################################################################
Function IperfClientServertcpNonConnectivity($server,$client)
{
	RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening TCP port $($server.tcpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Complete >> iperf-client.txt" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Complete >> iperf-server.txt" -runAsSudo
		if($isClientStarted -eq $false)
		{
			$serverState = IsIperfServerRunning $server
			if($serverState -eq $false)
			{
#LogMsg "Test Finished..!"
				$testResult = "PASS"
			}
			else
			{
				LogErr "Connections observed on server. Test Finished..!"
				$testResult = "FAIL"
			}
		}
		else
		{
			$testResult = "FAIL"
#LogMsg "Failured detected in client connection."
			LogErr "Ohh, client connected.. Verifying that it connected to server.."
			$serverState = IsIperfServerRunning $server
#$serverState = $false
			if($serverState -eq $false)
			{
				LogMsg "Not Connected to server. Please check the logs.. where the client was connected. ;-)"
			}
			else
			{
				LogMsg "Connections observed on server. Test Finished..!"
			}
		}
	}
	else
	{
		LogMsg "Unable to start iperf-server. Aborting test."
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerUDPNonConnectivity($server,$client)
{
	RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening UDP port $($server.tcpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Complete >> iperf-client.txt" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Complete >> iperf-server.txt" -runAsSudo
		$serverState = IsIperfServerRunning $server
		if($serverState -eq $false)
		{
#LogMsg "Test Finished..!"
			$testResult = "PASS"
		}
		else
		{
			LogErr "Connections observed on server. Test Finished..!"
			$testResult = "FAIL"
		}
	}
	else
	{
		LogMsg "Unable to start iperf-server. Aborting test."
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerUDPTest($server,$client)
{
	RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo

	$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening UDP port $($server.tcpPort) ..."

#>>>On confirmation, of server starting, let's start iperf client...
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Complete >> iperf-client.txt" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Complete >> iperf-server.txt" -runAsSudo
		$serverState = IsIperfServerRunning $server
		if($serverState -eq $true)
		{
#LogMsg "Test Finished..!"
			$testResult = "PASS"
		}
		else
		{
			LogErr "Connections not observed on server. Test Finished..!"
			$testResult = "FAIL"
		}
	}
	else
	{
		LogMsg "Unable to start iperf-server. Aborting test."
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerTest($server,$client)
{
	RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
	$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo

	StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening TCP port $($client.tcpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client

		if($isClientStarted -eq $true)
		{
			$serverState = IsIperfServerRunning $server
			if($serverState -eq $true)
			{
				LogMsg "Test Finished..!"
				$testResult = "PASS"
			}
			else
			{
				LogMsg "Test Finished..!"
				$testResult = "FAIL"
			}
		}
		else
		{
			LogErr "Failured detected in client connection."
			LogMsg "Test Finished..!"
			$testResult = "FAIL"
		}
	}
	else
	{
		LogMsg "Unable to start iperf-server. Aborting test."
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerTestParallel($server,$client)
{
	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo ServerStarted > iperf-server.txt" -runAsSudo
	$out = StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening TCP port $($client.tcpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo ClientStarted > iperf-client.txt" -runAsSudo
		$out = StartIperfClient $client
		$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo ClientStopped >> iperf-client.txt" -runAsSudo
		$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo ServerStopped >> iperf-server.txt" -runAsSudo
		$isClientStarted = IsIperfClientStarted $client
		
		if($isClientStarted -eq $true)
		{
			$serverState = IsIperfServerRunning $server
			if($serverState -eq $true)
			{
				LogMsg "Test Finished..!"
				$testResult = "PASS"
			}
			else
			{
				LogErr "Test Finished ..!"
				$testResult = "FAIL"
			}
			$clientLog= $client.LogDir + "\iperf-client.txt"
			$isClientConnected = AnalyseIperfClientConnectivity -logFile $clientLog -beg "ClientStarted" -end "ClientStopped"
			$clientConnCount = GetParallelConnectionCount -logFile $clientLog -beg "ClientStarted" -end "ClientStopped"
			If ($isClientConnected) {
				$testResult = "PASS"
				$serverLog= $server.LogDir + "\iperf-server.txt"
				$isServerConnected = AnalyseIperfServerConnectivity $serverLog "ServerStarted" "ServerStopped"
				If ($isServerConnected) {
					$testResult = "PASS"
					$serverConnCount = GetParallelConnectionCount -logFile $serverLog -beg "ServerStarted" -end "ServerStopped"
					LogMsg "Server Parallel Connection Count is $serverConnCount"
					LogMsg "Client Parallel Connection Count is $clientConnCount"
					If ($serverConnCount -eq $clientConnCount) {
						$testResult = "PASS"
						LogMsg "Connection Counts are Same in both Server and Client Logs"
						$clientDt= GetTotalDataTransfer -logFile $clientLog -beg "ClientStarted" -end "ClientStopped"
						$serverDt= GetTotalDataTransfer -logFile $serverLog -beg "ServerStarted" -end "ServerStopped"
						LogMsg "Server Total Data Transfer is $serverDt"
						LogMsg "Client Total Data Transfer is $clientDt"
						If ($serverDt  -eq $clientDt) {
							$testResult = "PASS"
							LogMsg "Total DataTransfer is equal on both Server and Client"
						} else {
							$testResult = "FAIL"
							LogErr "Total DataTransfer is NOT equal on both Server and Client"
						}
					} else {
						$testResult = "FAIL"
						LogErr "Connection Counts are NOT Same in both Server and Client Logs"
					}
				} else {
					$testResult = "FAIL"
					LogErr "Server is not Connected to Client"
				}
			} else {
				$testResult = "FAIL"
				LogErr "Client is not Connected to Server"
			}	
		} else {
			LogErr "Failures detected in client connection."
			RemoteCopy -download -downloadFrom $server.ip -files "/home/$user/iperf-server.txt" -downloadTo $server.LogDir -port $server.sshPort -username $server.user -password $server.password
			LogMsg "Test Finished..!"
			$testResult = "FAIL"
		}

	} else	{
		LogErr "Unable to start iperf-server. Aborting test."
		RemoteCopy -download -downloadFrom $server.ip -files "/home/$user/iperf-server.txt" -downloadTo $server.LogDir -port $server.sshPort -username $server.user -password $server.password
		RemoteCopy -download -downloadFrom $client.ip -files "/home/$user/iperf-server.txt" -downloadTo $client.LogDir -port $client.sshPort -username $client.user -password $client.password
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerUDPTestParallel($server,$client)
{
	RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
	$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt *.log" -runAsSudo
	$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo

	$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening UDP port $($client.udpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$out = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$out = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo TestComplete >> iperf-server.txt" -runAsSudo
		if($isClientStarted -eq $true)
		{
			$serverState = IsIperfServerRunning $server
			if($serverState -eq $true)
			{
				LogMsg "Test Finished..!"
				$testResult = "PASS"
			}
			else
			{
				LogErr "Test Finished..!"
				$testResult = "FAIL"
			}
			$clientLog= $client.LogDir + "\iperf-client.txt"
			$isClientConnected = AnalyseIperfClientConnectivity -logFile $clientLog -beg "Test Started" -end "TestComplete"
			$clientConnCount = GetParallelConnectionCount -logFile $clientLog -beg "Test Started" -end "TestComplete"
			If ($isClientConnected) {
				$testResult = "PASS"
				$serverLog= $server.LogDir + "\iperf-server.txt"
				$isServerConnected = AnalyseIperfServerConnectivity $serverLog "Test Started" "TestComplete"
				If ($isServerConnected) {
					$testResult = "PASS"
					$serverConnCount = GetParallelConnectionCount -logFile $serverLog -beg "Test Started" -end "TestComplete"
					LogMsg "Server Parallel Connection Count is $serverConnCount"
					LogMsg "Client Parallel Connection Count is $clientConnCount"
					If ($serverConnCount -eq $clientConnCount) {
						$testResult = "PASS"
						LogMsg "Connection Counts are Same in both Server and Client Logs"
						$clientDt= GetTotalDataTransfer -logFile $clientLog -beg "Test Started" -end "TestComplete"
						$serverDt= GetTotalUdpServerTransfer -logFile $serverLog -beg "Test Started" -end "TestComplete"
						LogMsg "Server Total Data Transfer is $serverDt"
						LogMsg "Client Total Data Transfer is $clientDt"
						$dataLoss = 0
						$diff= (($clientDt.Split("K")[0]) - ($serverDt.Split("K")[0]))
						If ($diff -gt 0) {
							$dataLoss= (($diff/($clientDt.Split("K")[0]))*100)
						}
						If ($dataLoss -lt 30) {
							$testResult = "PASS"
							LogMsg "DataTransfer Loss $dataLoss % is Less Than 30%"
							$udpLoss= GetUdpLoss -logFile $serverLog -beg "Test Started" -end "TestComplete"
							LogMsg "UDP Loss is $udpLoss"
							If ($udpLoss -gt 30) {
								$testResult = "FAIL"
								LogErr "UDP Loss $udpLoss is greater than 30%"
							} 
						} else {
							$testResult = "FAIL"
							LogErr "DataTransfer Loss $dataLoss % is Less Than 30%"
						}				
					} else {
						$testResult = "FAIL"
						LogErr "Connection Counts are NOT Same in both Server and Client Logs"
					}
				} else {
					$testResult = "FAIL"
					LogErr "Server is not Connected to Client"
				}
			} else {
				$testResult = "FAIL"
				LogErr "Client is not Connected to Client"
			}	
		} else {
			LogErr "Failured detected in client connection."
			RemoteCopy -download -downloadFrom $server.ip -files "/home/$user/iperf-server.txt" -downloadTo $server.LogDir -port $server.sshPort -username $server.user -password $server.password
			LogMsg "Test Finished..!"
			$testResult = "FAIL"
		}

	} else	{
		LogErr "Unable to start iperf-server. Aborting test."
		RemoteCopy -download -downloadFrom $server.ip -files "/home/$user/iperf-server.txt" -downloadTo $server.LogDir -port $server.sshPort -username $server.user -password $server.password
		RemoteCopy -download -downloadFrom $client.ip -files "/home/$user/iperf-server.txt" -downloadTo $client.LogDir -port $client.sshPort -username $client.user -password $client.password
		$testResult = "Aborted"
	}
	return $testResult
}

Function IperfClientServerUDPDatagramTest($server,$client, [switch] $VNET)
{
	if(!$VNET)
	{
		RemoteCopy -uploadTo $server.ip -port $server.sshPort -files $server.files -username $server.user -password $server.password -upload
		RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload
		$tmp = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo
		$tmp = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "chmod +x *" -runAsSudo
	}

	$tmp = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "rm -rf *.txt && rm -rf *.log" -runAsSudo
	$tmp = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshPort -command "rm -rf *.txt && rm -rf *.log" -runAsSudo
	
	$tmp = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	$tmp = StartIperfServer $server
	$isServerStarted = IsIperfServerStarted $server
	if($isServerStarted -eq $true)
	{
		LogMsg "iperf Server started successfully. Listening UDP port $($server.udpPort) ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$tmp = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		$tmp = StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$tmp = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Complete >> iperf-client.txt" -runAsSudo
		$tmp = RunLinuxCmd -username $server.user -password $server.password -ip $server.ip -port $server.sshport -command "echo Test Complete >> iperf-server.txt" -runAsSudo
		if($isClientStarted -eq $true)
		{
			$serverState = IsIperfServerRunning $server
			if($serverState -eq $true)
			{
#LogMsg "Test Finished..!"
				$testResult = "PASS"
				$iperfclientLogPath = $client.logDir + "\iperf-client.txt"
				$iperfserverLogPath = $server.logDir + "\iperf-server.txt"
				$udpLoss = GetUdpLoss  -logfile $iperfclientLogPath -beg "Test Started" -end "TestComplete"
				$isServerConnected = AnalyseIperfServerConnectivity -logfile $iperfserverLogPath -beg "Test Started" -end "Test Complete"
				
				if ($udpLoss -gt  30)
				{
					LogErr "UDP loss is greater than 30%"
					$testResult = "FAIL"
				}
				if (!$isServerConnected)
				{
					LogErr "Server Not Connected. [Error source : Function IperfClientServerUDPDatagramTest]"
					$testResult = "FAIL"
				}
			}
			else
			{
				LogErr "Test Finished..!"
				$testResult = "FAIL"
			}
		}
		else
		{
			LogErr "Failured detected in client connection."
			LogMsg "Test Finished..!"
			$testResult = "FAIL"
		}
	}
	else
	{
		LogMsg "Unable to start iperf-server. Aborting test."
		$testResult = "Aborted"
	}
	return $testResult
}

Function VerifyCustomProbe ($server1,$server2, [string] $probe) {
	RemoteCopy -uploadTo $server1.ip -port $server1.sshPort -files $server1.files -username $server1.user -password $server1.password -upload
	RemoteCopy -uploadTo $server2.Ip -port $server2.sshPort -files $server2.files -username $server2.user -password $server2.password -upload

	$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshPort -command "chmod +x *" -runAsSudo

	$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "echo Test Started > iperf-server.txt" -runAsSudo

	StartIperfServer $server1
	StartIperfServer $server2

	$isServerStarted = IsIperfServerStarted $server1
	$isServerStarted = IsIperfServerStarted $server2
	sleep(60)

	if(($isServerStarted -eq $true) -and ($isServerStarted -eq $true)) {
		LogMsg "Iperf Server1 and Server2 started successfully. Listening TCP port $client.tcpPort ..."

		$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshport -command "echo TestComplete >> iperf-server.txt" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "echo TestComplete >> iperf-server.txt" -runAsSudo
		RemoteCopy -download -downloadFrom $server1.ip -files "/home/$user/iperf-server.txt" -downloadTo $server1.LogDir -port $server1.sshPort -username $server1.user -password $server1.password
		RemoteCopy -download -downloadFrom $server2.ip -files "/home/$user/iperf-server.txt" -downloadTo $server2.LogDir -port $server1.sshPort -username $server2.user -password $server2.password

#	$server1State = IsIperfServerRunning $server1
#	$server2State = IsIperfServerRunning $server2
#	
#	if(($server1State -eq $true) -and ($server2State -eq $true)) {
#		LogMsg "Both Servers Started!"
#		$testResult = "PASS"
#	} else {
#		LogMsg "Both Servers NOT Started!!"
#		$testResult = "FAIL"
#	}
#	
		$server1CpConnCount = 0
		$server2CpConnCount = 0

		$server1Log= $server1.LogDir + "\iperf-server.txt"
		$server2Log= $server2.LogDir + "\iperf-server.txt"

		$isServerConnected1 = AnalyseIperfServerConnectivity $server1Log "Test Started" "TestComplete"
		$isServerConnected2 = AnalyseIperfServerConnectivity $server2Log "Test Started" "TestComplete"

		If ( $probe -eq "no" ) {
			if(!$isServerConnected1) {
				$isServerConnected1=!$isServerConnected1
			}
			if(!$isServerConnected2) {
				$isServerConnected2=!$isServerConnected2
			}
		}
		If (($isServerConnected1) -and ($isServerConnected2)) {
			If ( $probe -eq "yes" ) {
				$testResult = "PASS"
			} elseif ( $probe -eq "no" ) {
				$testResult = "FAIL"
			}
#Verify Custom Probe Messages on both server
			If ( $probe -eq "yes" ) {
				If (( IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete") -and (IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
					$server1CpConnCount= GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
					$server2CpConnCount= GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
					LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
					LogMsg "$server2CpConnCount Custom Probe Messages observed on Server1"
					$testResult = "PASS"
				} else {
					if (!( IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete") ) {
						LogErr "NO Custom Probe Messages observed on Server1"
						$testResult = "FAIL"
					}
					if (!(IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
						LogErr "NO Custom Probe Messages observed on Server1"
						$testResult = "FAIL"
					} 
				}
			} elseif ( $probe -eq "no" ) {
				If ((!(IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete")) -and (!(IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete"))) {
					LogMsg "NO Custom Probe Messages observed on Server1"
					LogMsg "NO Custom Probe Messages observed on Server2"
					$testResult = "PASS"
				} else {
					if((IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete")) {
						$server1CpConnCount = GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
						LogErr "$server1CpConnCount Custom Probe Messages observed on Server1"
						$testResult = "FAIL"
					} 
					if((IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
						$server2CpConnCount = GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
						LogErr "$server2CpConnCount Custom Probe Messages observed on Server2"
						$testResult = "FAIL"
					} 
				}	
#		If ( $probe -eq "yes" ) {
#			If ( IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete") {
#				$server1CpConnCount = GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
#				LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
#				$testResult = "PASS"
#			} else {
#				LogMsg "$server1CpConnCount NO Custom Probe Messages observed on Server1"
#				$testResult = "FAIL"
#			}
#			If ( IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete") {
#				$server2CpConnCount = GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
#				LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
#				$testResult = "PASS"
#			} else {
#			LogMsg "$server1CpConnCount NO Custom Probe Messages observed on Server1"
#			$testResult = "FAIL"
#			}
#		} elseif ( $probe -eq "no" ) {
#			If ( !(IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete")) {
#				$server1CpConnCount = GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
#				LogMsg "$server1CpConnCount NO Custom Probe Messages observed on Server1"
#				$testResult = "PASS"
#			} else {
#				LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
#				$testResult = "FAIL"
#			}
#			If (!(IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
#				$server2CpConnCount = GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
#				LogMsg "$server1CpConnCount NO Custom Probe Messages observed on Server1"
#				$testResult = "PASS"
#			} else {
#			LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
#			$testResult = "FAIL"
#			}
#		}
			}
		}

		return $testResult
	}
}

Function VerifyLBTCPConnectivity ($server1,$server2, $client, [string] $probe, [string] $pSize) {
	RemoteCopy -uploadTo $server1.ip -port $server1.sshPort -files $server1.files -username $server1.user -password $server1.password -upload
	RemoteCopy -uploadTo $server2.Ip -port $server2.sshPort -files $server2.files -username $server2.user -password $server2.password -upload
	RemoteCopy -uploadTo $client.Ip -port $client.sshPort -files $client.files -username $client.user -password $client.password -upload

	$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshPort -command "chmod +x *" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshPort -command "chmod +x *" -runAsSudo

	$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshport -command "echo Test Started > iperf-server.txt" -runAsSudo
	$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "echo Test Started > iperf-server.txt" -runAsSudo
	StartIperfServer $server1
	StartIperfServer $server2
	$isServerStarted = IsIperfServerStarted $server1
	$isServerStarted = IsIperfServerStarted $server2
	sleep(30)
	if(($isServerStarted -eq $true) -and ($isServerStarted -eq $true)) {
		LogMsg "Iperf Server1 and Server2 started successfully. Listening TCP port $client.tcpPort ..."
#>>>On confirmation, of server starting, let's start iperf client...
		$suppressedOut = RunLinuxCmd -username $client.user -password $client.password -ip $client.ip -port $client.sshport -command "echo Test Started > iperf-client.txt" -runAsSudo
		StartIperfClient $client
		$isClientStarted = IsIperfClientStarted $client
		$suppressedOut = RunLinuxCmd -username $server1.user -password $server1.password -ip $server1.ip -port $server1.sshport -command "echo TestComplete >> iperf-server.txt" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $server2.user -password $server2.password -ip $server2.ip -port $server2.sshPort -command "echo TestComplete >> iperf-server.txt" -runAsSudo
		if($isClientStarted -eq $true) {
			$server1State = IsIperfServerRunning $server1
			$server2State = IsIperfServerRunning $server2
			if(($server1State -eq $true) -and ($server2State -eq $true)) {
				LogMsg "Test Finished..!"
				$testResult = "PASS"
			} else {
				LogMsg "Test Finished..!"
				$testResult = "FAIL"
			}
			$clientLog= $client.LogDir + "\iperf-client.txt"
			$isClientConnected = AnalyseIperfClientConnectivity -logFile $clientLog -beg "Test Started" -end "TestComplete"
			$clientConnCount = GetParallelConnectionCount -logFile $clientLog -beg "Test Started" -end "TestComplete"
			$server1CpConnCount = 0
			$server2CpConnCount = 0
			If ($isClientConnected) {
				$testResult = "PASS"
				$server1Log= $server1.LogDir + "\iperf-server.txt"
				$server2Log= $server2.LogDir + "\iperf-server.txt"
				$isServerConnected1 = AnalyseIperfServerConnectivity $server1Log "Test Started" "TestComplete"
				$isServerConnected2 = AnalyseIperfServerConnectivity $server2Log "Test Started" "TestComplete"
				If (($isServerConnected1) -and ($isServerConnected2)) {
					$testResult = "PASS"

					$connectStr1="$($server1.DIP)\sport\s\d*\sconnected with $($client.ip)\sport\s\d"
					$connectStr2="$($server2.DIP)\sport\s\d*\sconnected with $($client.ip)\sport\s\d"

					$server1ConnCount = GetStringMatchCount -logFile $server1Log -beg "Test Started" -end "TestComplete" -str $connectStr1
					$server2ConnCount = GetStringMatchCount -logFile $server2Log -beg "Test Started" -end "TestComplete" -str $connectStr2
#Verify Custom Probe Messages on both server
					If ( $probe -eq "yes" ) {
						If (( IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete") -and (IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
							$server1CpConnCount= GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
							$server2CpConnCount= GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
							LogMsg "$server1CpConnCount Custom Probe Messages observed on Server1"
							LogMsg "$server2CpConnCount Custom Probe Messages observed on Server1"
							$testResult = "PASS"
							LogMsg "Server1 Parallel Connection Count is $server1ConnCount"
							LogMsg "Server2 Parallel Connection Count is $server2ConnCount"
							$diff = [Math]::Abs($server1ConnCount - $server2ConnCount)
							If ((($diff/$pSize)*100) -lt 20) {
								$testResult = "PASS"
								LogMsg "Connection Counts are distributed evenly in both Servers"
								LogMsg "Diff between server1 and server2 is $diff"
								$server1Dt= GetTotalDataTransfer -logFile $server1Log -beg "Test Started" -end "TestComplete"
								$server2Dt= GetTotalDataTransfer -logFile $server2Log -beg "Test Started" -end "TestComplete"
								$clientDt= GetTotalDataTransfer -logFile $clientLog -beg "Test Started" -end "TestComplete"
								LogMsg "Server1 Total Data Transfer is $server1Dt"
								LogMsg "Server2 Total Data Transfer is $server2Dt"
								LogMsg "Client Total Data Transfer is $clientDt"
								$totalServerDt = ([int]($server1Dt.Split("K")[0]) + [int]($server2Dt.Split("K")[0]))
								LogMsg "All Servers Total Data Transfer is $totalServerDt"
								If (([int]($clientDt.Split("K")[0])) -eq [int]($totalServerDt)) {
									$testResult = "PASS"
									LogMsg "Total DataTransfer is equal on both Server and Client"
								} else {
									$testResult = "FAIL"
									LogErr "Total DataTransfer is NOT equal on both Server and Client"
								}
							} else {
								$testResult = "FAIL"
								LogErr "Connection Counts are not distributed correctly"
								LogErr "Diff between server1 and server2 is $diff"
							}
						} else {
							if (!( IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete") ) {
								LogErr "NO Custom Probe Messages observed on Server1"
								$testResult = "FAIL"
							}
							if (!(IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
								$server2CpConnCount= GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
								LogErr "$server2CpConnCount Custom Probe Messages observed on Server1"
								$testResult = "FAIL"
							} 
						}
					} elseif ( $probe -eq "no" ) {
						If ((!(IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete")) -and (!(IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete"))) {
							LogMsg "NO Custom Probe Messages observed on Server1"
							LogMsg "NO Custom Probe Messages observed on Server2"
							$testResult = "PASS"
							LogMsg "Server1 Parallel Connection Count is $server1ConnCount"
							LogMsg "Server2 Parallel Connection Count is $server2ConnCount"
							$diff = [Math]::Abs($server1ConnCount - $server2ConnCount)
							If ((($diff/$pSize)*100) -lt 20) {
								$testResult = "PASS"
								LogMsg "Connection Counts are distributed evenly in both Servers"
								LogMsg "Diff between server1 and server2 is $diff"
								$server1Dt= GetTotalDataTransfer -logFile $server1Log -beg "Test Started" -end "TestComplete"
								$server2Dt= GetTotalDataTransfer -logFile $server2Log -beg "Test Started" -end "TestComplete"
								$clientDt= GetTotalDataTransfer -logFile $clientLog -beg "Test Started" -end "TestComplete"
								LogMsg "Server1 Total Data Transfer is $server1Dt"
								LogMsg "Server2 Total Data Transfer is $server2Dt"
								LogMsg "Client Total Data Transfer is $clientDt"
								$totalServerDt = ([int]($server1Dt.Split("K")[0]) + [int]($server2Dt.Split("K")[0]))
								LogMsg "All Servers Total Data Transfer is $totalServerDt"
								If (([int]($clientDt.Split("K")[0])) -eq [int]($totalServerDt)) {
									$testResult = "PASS"
									LogMsg "Total DataTransfer is equal on both Server and Client"
								} else {
									$testResult = "FAIL"
									LogErr "Total DataTransfer is NOT equal on both Server and Client"
								}
							} else {
								$testResult = "FAIL"
								LogErr "Connection Counts are not distributed correctly"
								LogErr "Diff between server1 and server2 is $diff"
							}
						} else {
							if((IsCustomProbeMsgsPresent -logFile $server1Log -beg "Test Started" -end "TestComplete")) {
								$server1CpConnCount = GetCustomProbeMsgsCount -logFile $server1Log -beg "Test Started" -end "TestComplete"
								LogErr "$server1CpConnCount Custom Probe Messages observed on Server1"
								$testResult = "FAIL"
							} 
							if((IsCustomProbeMsgsPresent -logFile $server2Log -beg "Test Started" -end "TestComplete")) {
								$server2CpConnCount = GetCustomProbeMsgsCount -logFile $server2Log -beg "Test Started" -end "TestComplete"
								LogErr "$server2CpConnCount Custom Probe Messages observed on Server2"
								$testResult = "FAIL"
							} 
						}
					}

				}	else {
					$testResult = "FAIL"
					LogErr "Server is not Connected to Client"
				}
			} else {
				$testResult = "FAIL"
				LogErr "Client is not Connected to Client"
			}	
		} else {
			LogErr "Failured detected in client connection."
			RemoteCopy -download -downloadFrom $server1.ip -files "/home/$user/iperf-server.txt" -downloadTo $server1.LogDir -port $server1.sshPort -username $server1.user -password $server1.password
			LogMsg "Test Finished..!"
			$testResult = "FAIL"
		}
	} else	{
		LogMsg "Unable to start iperf-server. Aborting test."
		RemoteCopy -download -downloadFrom $server1.ip -files "/home/$user/iperf-server.txt" -downloadTo $server1.LogDir -port $server1.sshPort -username $server1.user -password $server1.password
		RemoteCopy -download -downloadFrom $server2.ip -files "/home/$user/iperf-server.txt" -downloadTo $server2.LogDir -port $server2.sshPort -username $server2.user -password $server2.password
		$testResult = "Aborted"
	}
	return $testResult
}

Function CreateIperfNode
{
	param(
			[string] $nodeIp,
			[string] $nodeSshPort,
			[string] $nodeTcpPort,
			[string] $nodeUdpPort,
			[string] $nodeProbePort,
			[string] $nodeIperfCmd,
			[string] $user,
			[string] $password,
			[string] $files,
			[string] $nodeDip,
			[string] $nodeHostname,
			[string] $nodeUrl,
			[string] $logDir)

	$objNode = New-Object -TypeName PSObject
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ip -Value $nodeIp -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name sshPort -Value $nodeSshPort -Force 
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name tcpPort -Value $nodeTcpPort -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name probePort -Value $nodeProbePort -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name cmd -Value $nodeIperfCmd -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name user -Value $user -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name password -Value $password -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name files -Value $files -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name logDir -Value $LogDir -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name DIP -Value $nodeDip -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name Hostname -Value $nodeHostname -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name URL -Value $nodeUrl -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name udpPort -Value $nodeUdpPort -Force
	return $objNode
}

Function CreateIdnsNode
{
	param(
			[string] $nodeIp,
			[string] $nodeSshPort,
			[string] $user,
			[string] $password,
			[string] $nodeDip,
			[string] $nodeUrl,
			[string] $nodeDefaultHostname,
			[string] $nodeNewHostname,
			[string] $fqdn,
			[string] $logDir)

	$objNode = New-Object -TypeName PSObject
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ip -Value $nodeIp -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name sshPort -Value $nodeSshPort -Force 
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name user -Value $user -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name password -Value $password -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name logDir -Value $LogDir -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name DIP -Value $nodeDip -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name URL -Value $nodeUrl -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name hostname -Value $nodeDefaultHostname -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name Newhostname -Value $nodeNewHostname -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name fqdn -Value $fqdn -Force
	return $objNode
}

Function CreatePingNode
{
	param(
			[string] $nodeIp,
			[string] $nodeSshPort,
			[string] $user,
			[string] $password,
			[string] $nodeDip,
			[string] $nodeUrl,
			[string] $cmd,
			[string] $files,
			[string] $logDir)


	$objNode = New-Object -TypeName PSObject
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ip -Value $nodeIp -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name sshPort -Value $nodeSshPort -Force 
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name user -Value $user -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name password -Value $password -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name logDir -Value $LogDir -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name DIP -Value $nodeDip -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name URL -Value $nodeUrl -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name cmd -Value $cmd -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name files -Value $files -Force
	return $objNode
}

Function StartIperfServer($node)
{	
	$currentDir = $PWD.Path
	LogMsg "Starting iperf Server on $($node.ip)"
	$out = RunLinuxCmd -username $node.user -password $node.password -ip $node.ip -port $node.sshport -command $node.cmd -runAsSudo -RunInBackGround
	sleep 1
}

Function StartIperfClient($node)
{
	LogMsg "Starting iperf Client on $($node.ip)"
	$out = RunLinuxCmd -username $node.user -password $node.password -ip $node.ip -port $node.sshport -command $node.cmd -runAsSudo
	sleep 3
}

Function IsIperfServerStarted($node, $expectedServerInstances = 1)
{
	#RemoteCopy -download -downloadFrom $node.ip -files "/home/$user/start-server.py.log" -downloadTo $node.LogDir -port $node.sshPort -username $node.user -password $node.password
	LogMsg "Verifying if server is started or not.."
	$iperfout = RunLinuxCmd -username $node.user -password $node.password -ip $node.ip -port $node.sshPort -command "ps -ef | grep iperf -s | grep -v grep | wc -l" -runAsSudo
	$iperfout = [int]$iperfout[-1].ToString()
	LogMsg "Total iperf server running instances : $($iperfout)"	
	if($iperfout -ge $expectedServerInstances)
	{
		return $true
	}
	else
	{
		return $false
	}
}

Function IsIperfServerRunning($node)
{
	$out = RunLinuxCmd -username $node.user -password $node.password -ip $node.ip -port $node.sshport -command "python check-server.py && mv Runtime.log check-server.py.log -f" -runAsSudo
	RemoteCopy -download -downloadFrom $node.ip -files "/home/$user/check-server.py.log, /home/$user/iperf-server.txt" -downloadTo $node.LogDir -port $node.sshPort -username $node.user -password $node.password
	RemoteCopy -download -downloadFrom $node.ip -files "/home/$user/state.txt, /home/$user/Summary.log" -downloadTo $node.logdir -port $node.sshPort -username $node.user -password $node.password
	$serverState = Get-Content "$($node.Logdir)\state.txt"
	$serverSummary =  Get-Content "$($node.Logdir)\Summary.log"

#>>>Remove Temporary files..
	Remove-Item "$($node.Logdir)\state.txt" -Force
	Remove-Item "$($node.Logdir)\Summary.log" -Force
#>>>Verify client connections appeared on server...
	if($serverState -eq "TestCompleted" -and $serverSummary -eq "PASS"){
		return $true
	}
	else{
		return $false
	}
}

Function IsIperfClientStarted($node, [string]$beginningText, [string]$endText)
{
	sleep 1
	$out = RunLinuxCmd -username $node.user -password $node.password -ip $node.ip -port $node.sshPort -command "mv Runtime.log start-client.py.log -f" -runAsSudo
	RemoteCopy -download -downloadFrom $node.ip -files "/home/$user/start-client.py.log, /home/$user/iperf-client.txt" -downloadTo $node.LogDir -port $node.sshPort -username $node.user -password $node.password
	RemoteCopy -download -downloadFrom $node.ip -files "/home/$user/state.txt, /home/$user/Summary.log" -downloadTo $node.Logdir -port $node.sshPort -username $node.user -password $node.password
	$clientState = Get-Content "$($node.Logdir)\state.txt"
	$clientSummary = Get-Content "$($node.Logdir)\Summary.log"
	Write-Host "Client State : $clientState"
	Write-Host "Client Summary : $clientSummary"
#>>>Remove Temporary files..
	Remove-Item "$($node.Logdir)\state.txt" -Force
	Remove-Item "$($node.Logdir)\Summary.log" -Force
	if ($beginningText -and $endText)
	{
		$connectStingCount = GetStringMatchCount -logFile "$($node.LogDir)\iperf-client.txt" -beg $beginningText -end $endText -str "connected with"
		if ($connectStingCount -gt 0)
		{
			$connFailureCount = GetStringMatchCount -logFile "$($node.LogDir)\iperf-client.txt" -beg $beginningText -end $endText -str "Connection refused"
			$connFailureCount += GetStringMatchCount -logFile "$($node.LogDir)\iperf-client.txt" -beg $beginningText -end $endText -str "connect failed"
			Write-Host "connection failures found:" $connFailureCount
			if($connFailureCount -gt 0)
			{
				$retVal = $false
			}
			else
			{
				$retVal = $true
			}
		}
		else
		{
			$retVal = $false
		}
	}
	else
	{
		if(($clientState -eq "TestCompleted") -and ($clientSummary -eq "PASS")){
			$retVal = $true
		}
		else{

			$retVal = $false
		}
	}
	return $retVal
}

Function DoNslookupTest ($vm1, $vm2)
{
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo TestStarted > nslookup12.log" -runAsSudo
	if ($detectedDistro -eq "COREOS")
	{
		$nslookupCommand = "python nslookup.py -n $($vm2.Hostname)"
	}
	else
	{
		$nslookupCommand = "nslookup $($vm2.Hostname)"
	}
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo Executing : $nslookupCommand >> nslookup12.log" -runAsSudo
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "$nslookupCommand >> nslookup12.log" -runAsSudo -ignoreLinuxExitCode
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo TestCompleted >> nslookup12.log" -runAsSudo
	RemoteCopy -download -downloadTo $vm1.logDir -downloadFrom $vm1.ip -port $vm1.sshport  -username $vm1.user -password $vm1.password -files "nslookup12.log"
	$nslookuplog12 = $vm1.logDir + "\nslookup12.log"
	$nslookupResult = GetStringMatchCount -logFile $nslookuplog12 -beg TestStarted -end TestCompleted -str $vm2.DIP
	$ErrorLine = GetStringMatchCount -logFile $nslookuplog12 -beg TestStarted -end TestCompleted -str "server can`'t find"
	
	$nslookuplog12 = $vm1.logDir + "\nslookup12.log"
	Write-host "Hostname match count : $nslookupResult"
	if(($nslookupResult -gt 0) -and ($ErrorLine -eq 0))
	{
		$testResult = "PASS"
	}
	else
	{
		$testResult = "FAIL"
		$nslookupError = GetStringMatchObject -logFile $nslookuplog12 -beg TestStarted -end TestCompleted -str "server can`'t find"
		LogErr "NSLOOKUP FAILED : $nslookupError"
	}
	LogMsg "NSLOOKUP Result : $testResult"
	return $testResult
}

Function DoDigTest ($vm1, $vm2)
{
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo TestStarted > dig12.log" -runAsSudo
	if ($detectedDistro -eq "COREOS")
	{
		$digCommand = "python dig.py -n $($vm2.fqdn)"
	}
	else
	{
		$digCommand = "dig $($vm2.fqdn)"
	}
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo Executing : $digCommand >> dig12.log" -runAsSudo
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "$digCommand >> dig12.log" -runAsSudo -ignoreLinuxExitCode
	$out = RunLinuxCmd -username $vm1.user -password $vm1.password -ip $vm1.Ip -port $vm1.SshPort -command "echo TestCompleted >> dig12.log" -runAsSudo
	RemoteCopy -download -downloadTo $vm1.logDir -downloadFrom $vm1.ip -port $vm1.sshport  -username $vm1.user -password $vm1.password -files "dig12.log"
	$diglog12 = $vm1.logDir + "\dig12.log"
	$digResult = GetStringMatchCount -logFile $diglog12 -beg TestStarted -end TestCompleted -str $vm2.DIP
	if(($digResult -eq 1))
	{
		$testResult = "PASS"
		$digOutput = GetStringMatchObject -logFile $diglog12 -beg TestStarted -end TestCompleted -str $vm2.DIP
		LogMsg "DIG Result : DIP RESOLVED. : PASS"
		LogMsg "DIG Output : $digOutput"
	}
	else
	{
		$testResult = "FAIL"
		LogErr "DIG Result : DIP NOT RESOLVED. : FAIL"
	}
	return $testResult
}

Function GetVMFQDN ($vm)
{
	$out = RunLinuxCmd -username $vm.user -password $vm.password -ip $vm.Ip -port $vm.SshPort -command "hostname --fqdn > vmfqdn.txt" -runAsSudo
	RemoteCopy -download -downloadTo $vm.logDir -downloadFrom $vm.ip -port $vm.sshport  -username $vm.user -password $vm.password -files "vmfqdn.txt"
	$vmFQDN = Get-Content "$($vm.LogDir)\vmfqdn.txt"
	if($vmFQDN)
	{
		return $vmFQDN
	}
	else
	{
		Throw "Unable to get VM FQDN.."
	}
}

Function DoPingTest ($pingFrom, [switch]$isVNET, [switch]$fromLocal, $intermediateVM)
{
#Skeep the file upload process for VNET test case.
	if(!$isVNET)
	{
		RemoteCopy -upload -uploadTo $pingFrom.IP -port $pingFrom.sshport -username $pingFrom.user -password $pingFrom.password -files $pingFrom.files
	}
	$pingLog = $pingFrom.logDir + "\ping.log"
	if(!$fromLocal)
	{
		$suppressedOut = RunLinuxCmd -username $pingFrom.user -password $pingFrom.password -ip $pingFrom.Ip -port $pingFrom.SshPort -command "echo TestStarted > ping.log" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $pingFrom.user -password $pingFrom.password -ip $pingFrom.Ip -port $pingFrom.SshPort -command "chmod +x *.py" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $pingFrom.user -password $pingFrom.password -ip $pingFrom.Ip -port $pingFrom.SshPort -command "echo Executing : $($pingFrom.cmd) >> ping.log" -runAsSudo
		$suppressedOut = RunLinuxCmd -username $pingFrom.user -password $pingFrom.password -ip $pingFrom.Ip -port $pingFrom.SshPort -command "$($pingFrom.cmd)" -runAsSudo 
		$suppressedOut = RunLinuxCmd -username $pingFrom.user -password $pingFrom.password -ip $pingFrom.Ip -port $pingFrom.SshPort -command "echo TestCompleted >> ping.log" -runAsSudo
		RemoteCopy -download -downloadTo $pingFrom.logDir -downloadFrom $pingFrom.ip -port $pingFrom.sshport  -username $pingFrom.user -password $pingFrom.password -files "ping.log"
	}
	else
	{
		Set-Content -Value "TestStarted" -Path $pingLog
		$out = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $pingFrom -runAsSudo -remoteCommand "rm -rf ping.log"
		$out = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $pingFrom -runAsSudo -remoteCommand "chmod +x /home/$user/*.py"		
		#$newPingCmd = ($pingFrom.cmd).Replace(" ","\ ")
		$newPingCmd = $pingFrom.cmd
		$pingoutput = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $pingFrom -runAsSudo -remoteCommand "$newPingCmd" 
		Write-Host $pingoutput	 
#$out = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $pingFrom -runAsSudo -remoteCommand "echo\ TestCompleted\ >>\ ping.log"	  
		$pingoutput = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $pingFrom -runAsSudo -remoteCommand "cat ping.log"	  
		Add-Content -Value $pingoutput  -Path $pingLog 
		Add-Content -Value "TestCompleted" -Path $pingLog 
	}

	$pingResult = GetStringMatchCount -logFile $pinglog -beg TestStarted -end TestCompleted -str "ttl"

	if(($pingResult -ge 1))
	{
		$testResult = "PASS"
#I'm working on adding more checks here like packet loss, time to travel. will do it soon. Just now checking for connectivity.
	}
	else
	{
		$testResult = "FAIL"
	}
	return $testResult
}
#endregion

#region ResultLog Parser Code
###############################################################

Function GetStringMatchCount([string] $logFile,[string] $beg,[string] $end, [string] $str)
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str $str
	return $match.Count
}

Function GetTotalDataTransfer([string] $logFile,[string] $beg,[string] $end)
{
	$dataTransfer=GetDataTxed -logFile $logFile -ptrn "[bits,bytes]/sec$"
	return $dataTransfer
}

Function GetTotalUdpServerTransfer([string] $logFile,[string] $beg,[string] $end)
{
	$dataTransfer=GetDataTxed -logFile $logFile -ptrn "[bits,bytes]/sec"
	return $dataTransfer
}

Function GetTotalUdpServerTransfer([string] $logFile,[string] $beg,[string] $end)
{
	$dataTransfer=GetDataTxed -logFile $logFile -ptrn "[bits,bytes]/sec"
	return $dataTransfer
}

Function GetUdpLoss([string] $logFile, [string] $beg,[string] $end)
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str "\([\d,\d.\d]*%\)$"
	$arr = @()
	foreach ($item in $match) {
		$item = $item.ToString()
		$str2=@($item.Split(" ",[StringSplitOptions]'RemoveEmptyEntries'))
		foreach ($a in $str2) {
			if($a.Contains("%"))
			{
#$i=$str2.IndexOf($a)
#$a=$str2[$i]
				$a= $a.Replace("%", "")
				$a= $a.Replace("(", "")
				$a= $a.Replace(")", "")
#$num=$b[0].Split("(")
				$arr += $a
			}
		}
		$sum = ($arr | Measure-Object -Sum).Sum
	}
	return $sum
}

Function GetDataTxed([string] $logFile,[string] $beg,[string] $end, [string] $ptrn) 
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str $ptrn
	$match= $match | Select-String -Pattern "0.00 KBytes/sec" -NotMatch
	$lastItem=$match[-1]
	$lastItem=$lastItem.ToString()
	$str1=@($lastItem.Split(" ",[StringSplitOptions]'RemoveEmptyEntries'))
	foreach ($a in $str1) {
		if($a.Contains("Bytes") -and !($a.Contains("Bytes/sec")))
		{
			$i=$str1.IndexOf($a)
			$result=$str1[$i-1]+$str1[$i]
		}
	}
	return $result
}

Function AnalyseIperfServerConnectivity([string] $logFile,[string] $beg,[string] $end)
{
	$connectStr="\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d*\sconnected with \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d"
	$match=IsContainString -logFile $logFile -beg $beg -end $end -str $connectStr
	$failure = 0
	If($match) {
		$RefusedConnections = GetStringMatchCount -logFile $logFile -beg $beg -end $end -str "Connection refused"
		if($RefusedConnections -gt 0){
			LogErr "Server connected to client. But `"Connection Refused`" observed $RefusedConnections times in the logs. Marking test as FAIL."
			return $false
		}
		else{
			LogMsg "Server Connected successfully"
			return $true
		}
	}
	else {
		LogErr "Server connection Fails!"
		return $false
	}
}

Function AnalyseIperfClientConnectivity([string] $logFile,[string] $beg,[string] $end)
{
	$connectStr="\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d*\sconnected with \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d"
	$match=IsContainString -logFile $logFile -beg $beg -end $end -str $connectStr
	$failure = 0
	If ($match) 
	{
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "failed"
		If($match) 
		{
			$failure = $failure + 1
			LogErr "Client connected with some failed connections!"
		}	
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "error"	
		If($match)
		{
			$failure = $failure + 1
			LogErr "There were some errors in the connections!"
		}
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "refused"	
		If($match)
		{
			$failure = $failure + 1
			LogErr "Some connections were refused!" 
		}
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "Broken pipe"	
		If($match)
		{
			$failure = $failure + 1
			LogErr "Broken Pipe Message observed in Iperf Client Output" 
		}
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "TestIncomplete"	
		If($match)
		{
			$failure = $failure + 1
			LogErr "Client was successfully connected to server but, client process failed to exit!" 
		}
		if ($failure -eq 0)
		{
			LogMsg "Client was successfully connected to server"
			return $true
		} else {
			LogErr "Client connection fails"
			return $false
		}
	} 
	else
	{
		$match = IsContainString -logFile $logFile -beg $beg -end $end -str "No address associated"	
		If($match) {
			LogErr "Client was not connected to server!"
			LogErr "No address associated with hostname"
			return $false
		} 
		elseif (($match= IsContainString -logFile $logFile -beg $beg -end $end -str "Connection refused"))
		{
			LogErr "Client was not connected to server."
			LogErr "Connection refused by the server."
			return $false
		}
		elseif (($match= IsContainString -logFile $logFile -beg $beg -end $end -str "Name or service not known"))
		{
			LogErr "Client was not connected to server."
			LogErr "Name or service not known."
			return $false
		} 
		else
		{
			LogErr "Client was not connected to server."
			LogErr "Unlisted error. Check logs for more information...!."
			return $false
		}
	}
}

Function IsContainString([string] $logFile,[string] $beg,[string] $end, [string] $str)
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str $str
	If ($match.count -gt 0) {
		return $true
	} else {
		return $false
	}
}

Function GetStringMatchObject([string] $logFile,[string] $beg,[string] $end, [string] $str) 
{
	if ($beg -eq "0") {
		$begPos = 1
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr | Select-String -Pattern $str
	} elseif ($beg -ne ""  -and $end -ne "") {
		$match=Select-String -Pattern $beg -Path $logFile
		$begPos= $match.LineNumber
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr | Select-String -Pattern $str
	} else {
		$match=Select-String -Pattern $str -Path $logFile
	}
	return $match
}

Function GetAllStringMatchObject([string] $logFile,[string] $beg,[string] $end)
{
	if ($beg -eq "0") {
		$begPos = 1
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr | Select-String
	} elseif ($beg -ne ""  -and $end -ne "") {
		$match=Select-String -Pattern $beg -Path $logFile
		$begPos= $match.LineNumber
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr | Select-String
	} else {
		$match=Select-String -Path $logFile
	}
	return $match
}

Function GetParallelConnectionCount([string] $logFile,[string] $beg,[string] $end)
{
	$connectStr="\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d*\sconnected with \d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\sport\s\d"
	$p=GetStringMatchCount -logFile $logFile -beg $beg -end $end -str $connectStr
	return $p
}

Function GetMSSSize([string] $logFile,[string] $beg,[string] $end)
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str "MSS size \d*\sbytes"
	If ($match.Length -gt 1) {
		$lastItem=$match.Item($match.Length-1) 
	} else {
		$lastItem=$match
	}
	$lastItem=$lastItem.ToString()
#$str1=@($lastItem.Split(']'))
	$str2=@($lastItem.Split(" ",[StringSplitOptions]'RemoveEmptyEntries'))
	foreach ($a in $str2) {
		if($a.Contains("size"))
		{
			$i=$str2.IndexOf($a)
			$result=$str2[$i+1]+$str2[$i+2]
		}
	}
	return $result
}

Function GetParallelConnectionsCount($iperfClientoutput)
{
	$iperfClientOutput = $iperfClientOutput.Split("")
	$uniqueIds = @()
	$AllConnectedIDs = @()
	$NoofUniqueIds = 0
	foreach ($word in $iperfClientOutput)
	{
		if ($word -imatch "]")
		{
			$word = $word.Replace("]","")
			$word = $word.Replace("[","")
			$word = $word -as [int]
			if ($word)
			{
				$AllConnectedIDs += $word
				$NotUnique = 0
				foreach ($id in $uniqueIds)
				{
					if ($word -eq $id)
					{
						$NotUnique = $NotUnique + 1
					}
				}
				if ($NotUnique -eq 0)
				{
					$uniqueIds += $word
				}
			}

		}
	}
	$NoofUniqueIds = $uniqueIds.Count
#return $AllConnectedIDs, $uniqueIds, $NoofUniqueIds
	return $NoofUniqueIds
}

Function GetCustomProbeMsgsCount([string] $logFile,[string] $beg,[string] $end) 
{
	$match=GetStringMatchObject -logFile $logFile -beg $beg -end $end -str "0.00 KBytes  0.00 KBytes/sec"
	return $match.Count
}

Function IsCustomProbeMsgsPresent ([string] $logFile,[string] $beg,[string] $end) 
{
	$cpStr="0.00 KBytes  0.00 KBytes/sec"
	$match=IsContainString -logFile $logFile -beg $beg -end $end -str $cpStr
	return $match
}

Function GetAllStringMatchObject([string] $logFile,[string] $beg,[string] $end)
{
	if ($beg -eq "0") {
		$begPos = 1
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr
	} elseif ($beg -ne ""  -and $end -ne "") {
		$match=Select-String -Pattern $beg -Path $logFile 
		$begPos=  $match.LineNumber
		$match=Select-String -Pattern $end -Path $logFile
		$endPos= $match.LineNumber
		$lineArr = ($begPos-1)..($endPos-1)
		$match=Get-Content -Path $logFile | Select-Object -Index $lineArr
	} else {
		$match=Select-String -Path $logFile
	}
	return $match
}

Function GetUdpDatagramSize ([string] $logFile,[string] $beg,[string] $end)
{
	$datagramLine = GetStringMatchObject -beg $beg -end $end -logFile $logFile -str Sending
	$datagramLine = $datagramLine.ToString()
	$datagramLine = $datagramLine.Split(" ")
	$datagramSize = $datagramLine[1]
	return $datagramSize
}
#endregion

#region PerfTest Methods
Function GetIozoneResultAllValues 
{
	param
	(
	 [Parameter(Mandatory=$true)]
	 [string] $logFile
	)
	$iozoneData= @{"write"="";"rewrite"="";"read"="";"reread"="";"randread"="";"randwrite"="";"bkwdread"="";"recrewrite"="";"strideread"="";}
	foreach ($key in @($iozoneData.keys))
	{
		$iozoneData[$key]=GetIozoneResult $logFile $key
	}
	return $iozoneData
}

Function GetIozoneResult ([string] $logFile,[string] $str)
{
	$str="^"+$str
	$match=GetStringMatchObject -logFile $logFile -str $str
	$match=$match.ToString()
	$value=$match.Split(" ")
	return $value[$value.lenght-1]
}
#endregion

#region VNET TESTS Methods

Function Get-AllVMHostnameAndDIP($DeployedServices)
{
	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$VMhostname = $testVM.InstanceName
			$VMDIP = $testVM.IpAddress
			if($HostnameDIP)
			{
				$HostnameDIP = $HostnameDIP + "^$VMhostname" + ':' +"$VMDIP"
			}
			else
			{
				$HostnameDIP = "$VMhostname" + ':' +"$VMDIP"
			}
		}
	}

	return $HostnameDIP
}

Function Get-SSHDetailofVMs($DeployedServices)
{
	foreach ($hostedservice in $DeployedServices.Split("^"))
	{
		$DeployedVMs = Get-AzureVM -ServiceName $hostedService
		foreach ($testVM in $DeployedVMs)
		{
			$AllEndpoints = Get-AzureEndpoint -VM $testVM
			$HSVIP = GetHsVmVip -servicename $hostedservice
			$HSport = GetPort -Endpoints $AllEndpoints -usage SSH
			if($TestIPPOrts)
			{
				$TestIPPOrts = $TestIPPOrts + "^$HSVIP" + ':' +"$HSport"
			}
			else
			{
				$TestIPPOrts = "$HSVIP" + ':' +"$HSport"
			}


		}
	}

	return $TestIPPOrts
}

Function CreateVMNode
{
	param(
			[string] $nodeIp,
			[string] $nodeSshPort,
			[string] $user,
			[string] $password,
			[string] $nodeDip,
			[string] $nodeHostname,
			[string] $cmd,
			[string] $files,
			[string] $logDir)


	$objNode = New-Object -TypeName PSObject
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ip -Value $nodeIp -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name sshPort -Value $nodeSshPort -Force 
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name user -Value $user -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name password -Value $password -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name logDir -Value $LogDir -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name DIP -Value $nodeDip -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name Hostname -Value $nodeHostname -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name URL -Value $nodeUrl -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name cmd -Value $cmd -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name files -Value $files -Force
	return $objNode
}

Function ConfigureDnsServer($intermediateVM, $DnsServer, $HostnameDIPDetails)
{
#Get VNETVM details using - Get-AllVMHostnameAndDIP() function. This will generate the string of all VMs IP and hostname.
	$HostnameDIP = $HostnameDIPDetails
	$DnsConfigureCommand = "python /home/test/ConfigureDnsServer.py -v `"$HostnameDIP`""
	$out = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $DnsServer -remoteCommand $DnsConfigureCommand
	if($out -imatch 'ExitCode : 0')
	{
		LogMsg  "DNS server configured successfully."
	}
	else
	{
		Throw "DNS server configuration failed."
	}
}

Function UploadFilesToAllDeployedVMs($SSHDetails,$files)
{
	$TestIPPOrts = $SSHDetails

	foreach ($IPPORT in $TestIPPOrts.Split("^"))
	{
		$IPPORT = $IPPORT.Split(":")
		$testIP = $IPPORT[0]
		$testPort = $IPPORT[1]
		RemoteCopy -upload -uploadTo $testIP  -port $testPort -username $user -password $password -files $files
	}
}

Function RunLinuxCmdOnAllDeployedVMs($SSHDetails,$command)
{
	$TestIPPOrts = $SSHDetails

	foreach ($IPPORT in $TestIPPOrts.Split("^"))
	{
		$IPPORT = $IPPORT.Split(":")
		$testIP = $IPPORT[0]
		$testPort = $IPPORT[1]
		$suppressedOut = RunLinuxCmd -ip $testIP -port $testPort -username $user -password $password -command "$command" -runAsSudo
	}
}

Function RunLinuxCmdOnRemoteVM($intermediateVM,$remoteVM, [switch] $runAsSudo, $remoteCommand, [switch]$hostnameMode)
{
#Assuming that all py scripts  will be in the remoteVM
	$newPass = ($remoteVM.password).Replace("`"","")
#RunLinuxCmd -intermediateVM -c 'RunSSHcmd.py -remoteVM'
#RunLinuxCmd -intermediateVM -c 'RemoteCopy.py' -files 'isConnected.txt'
#RemoteCopy -from intermediateVM -isConnected.txt to automation server.
#if(isConnected = true)
# {
#RunLinuxCmd -intermediateVM -c 'RemoteCopy.py' -files 'RunSSHcmd-out.txt, RunSSHcmd-err.txt'
#RemoteCopy -from intermediateVM -isConnected.txt to automation server.
# }

#Decide whether desired work is accomplished or not.. and return true or false.

#Generate the Full command that will be actually executed on intermediate VM.
	if(!$hostnameMode)
	{
		$RunSSHremoteCommand = "python /home/test/RunSSHCmd.py -s `'$($remoteVM.ip)`' -u $($remoteVM.user) -p`'$newPass`' -P$($remoteVM.sshPort) -c `'$remoteCommand`'"
	}
	else
	{
		$RunSSHremoteCommand = "python /home/test/RunSSHCmd.py -s `'$($remoteVM.Hostname)`' -u $($remoteVM.user) -p`'$newPass`' -P$($remoteVM.sshPort) -c `'$remoteCommand`'"
	}
	if($runAsSudo)
	{
		$RunSSHremoteCommand = $RunSSHremoteCommand  + " -o yes"
	}
#Write-Host $RunSSHremoteCommand
#Now Run this command..
	$remoteOutput = RunLinuxCmd -ip $intermediateVM.ip -username $intermediateVM.user -password $intermediateVM.password -port $intermediateVM.SSHport -command $RunSSHremoteCommand -runAsSudo
#Write-Host $remoteOutput
	if($remoteOutput -imatch 'ExitCode : 0')
	{
		LogMsg "$remoteCommand executed successfully on $($remoteVM.ip)."
	}
	else
	{
		Write-host $remoteOutput
		Throw "$remoteCommand Failed to execute on $($remoteVM.ip)."
	}
	return $remoteOutput
}

Function RemoteCopyRemoteVM($intermediateVM,$remoteVM,$remoteFiles, [switch]$upload, [switch]$download, [switch]$hostnameMode )
{
	$remoteFiles = $remoteFiles.Replace(" ","")
	$tempFiles = $remoteFiles.Split(",")
	$fileCount = $tempFiles.Length
	($remoteVM.password) = ($remoteVM.password).Replace("`"","")
	if($upload)
	{
#$allFiles = "/home/test/azuremodules.py,/home/test/ConfigureDnsServer.py,/home/test/CleanupDnsServer.py,/home/test/ConfigureResolvConf.py,/home/test/RunSSHCmd.py,/home/test/RemoteCopy.py"
		if($hostnameMode)
		{
			$uploadCommand = "python RemoteCopy.py  -m upload -c `'$($remoteVM.Hostname)`' -u $($remoteVM.user) -p `'$($remoteVM.password)`' -P$($remoteVM.sshPort) -r `'/home/$user`' -f `'$remoteFiles`'"
		}
		else
		{
			$uploadCommand = "python RemoteCopy.py  -m upload -c `'$($remoteVM.ip)`' -u $($remoteVM.user) -p `'$($remoteVM.password)`' -P$($remoteVM.sshPort) -r `'/home/$user`' -f `'$remoteFiles`'"
		}
		$remoteFiles = $remoteFiles.Replace(" ",'')
		$uploadOutput = RunLinuxCmd -ip $intermediateVM.ip -port $intermediateVM.sshPort -username $intermediateVM.user -password $intermediateVM.password -command $uploadCommand -runAsSudo
		$uploadCount = (Select-String -InputObject $uploadOutput -Pattern "...OK!").Length
		LogMsg "Uploaded $uploadCount files to $($remoteVM.ip)"
		$uploadErrorCount = (Select-String -InputObject $uploadOutput -Pattern "...Error!").Length
		if ($uploadErrorCount -gt 0)
		{
			LogErr $uploadOutput
			Throw "Failed to upload $uploadErrorCount files to $($remoteVM.ip)"
		}
		elseif ($uploadCount -ne $fileCount)
		{
			LogErr $uploadOutput
			Throw "File count doensn't match"
		}
		else
		{
			$retValue = "True"
		}
	}
	if($download)
	{
#$allFiles = "/home/test/azuremodules.py,/home/test/ConfigureDnsServer.py,/home/test/CleanupDnsServer.py,/home/test/ConfigureResolvConf.py,/home/test/RunSSHCmd.py,/home/test/RemoteCopy.py"
		if($hostnameMode)
		{
			$downloadCommand = "python RemoteCopy.py  -m download -c `'$($remoteVM.Hostname)`' -u $($remoteVM.user) -p `'$($remoteVM.password)`' -P$($remoteVM.sshPort) -l `'/home/$user`' -f `'$remoteFiles`'"
		}
		else
		{
			$downloadCommand = "python RemoteCopy.py  -m download -c `'$($remoteVM.ip)`' -u $($remoteVM.user) -p `'$($remoteVM.password)`' -P$($remoteVM.sshPort) -l `'/home/$user`' -f `'$remoteFiles`'"
		}
		$remoteFiles = $remoteFiles.Replace(" ",'')
		$downloadOutput = RunLinuxCmd -ip $intermediateVM.ip -port $intermediateVM.sshPort -username $intermediateVM.user -password $intermediateVM.password -command $downloadCommand -runAsSudo
		$downloadCount = (Select-String -InputObject $downloadOutput -Pattern "...OK!").Length
		LogMsg "downloaded $downloadCount files from $($remoteVM.ip)"
		$downloadErrorCount = (Select-String -InputObject $downloadOutput -Pattern "...Error!").Length
		if ($downloadErrorCount -gt 0)
		{
			LogErr $downloadOutput
			Throw "Failed to download $downloadErrorCount files to $($remoteVM.ip)"
		}
		elseif ($downloadCount -ne $fileCount)
		{
			LogErr $downloadOutput
			Throw "File count doensn't match"
		}
		else
		{
			$retValue = "True"
		}
	}
	return $retValue
}

Function ConfigureVNETVMs($SSHDetails)
{
	UploadFilesToAllDeployedVMs -SSHDetails $SSHDetails -files ".\remote-scripts\ConfigureResolvConf.py,.\remote-scripts\azuremodules.py"
	$suppressedOut = RunLinuxCmdOnAllDeployedVMs -SSHDetails $SSHDetails -command "chmod +x /home/$user/*.py"
	$TestIPPOrts = $SSHDetails
	foreach ($IPPORT in $TestIPPOrts.Split("^"))
	{
		$IPPORT = $IPPORT.Split(":")
		$testIP = $IPPORT[0]
		$testPort = $IPPORT[1]
		LogMsg "$testIP : $testPort configuration in progress.."

		$out = RunLinuxCmd -ip $testIP -port $testPort -username $user -password $password -command "python /home/$user/ConfigureResolvConf.py" -runAsSudo
		if (-not ($out -imatch 'ExitCode : 0'))
		{
			Throw "$testIP : $testPort ConfigureResolvConf.py failed..."
		}

#Enable TCP MTU probing, requried for using WS2012 RRAS as VPN device
#TODO how to check the return value?
		$suppressedOut = RunLinuxCmd -ip $testIP -port $testPort -username $user -password $password -command "sh -c 'echo 1 >/proc/sys/net/ipv4/tcp_mtu_probing'" -runAsSudo

		LogMsg "$testIP : $testPort configuration finished"
	}
}

Function ConvertFileNames([switch]$ToLinux, [switch]$ToWindows, $currentWindowsFiles, $currentLinuxFiles, $expectedLinuxPath, $expectedWindowsPath)
{
	if ($ToLinux)
	{
		$remotefiles = ""
		$files = $currentWindowsFiles.Split(',')
		foreach ($newFile in $files)
		{
			$newFile = $newFile.Split("\")
			$newFileLen = $newFile.Length
			$exactFile= $newFile[$newFileLen -1]
			if(!$remotefiles)
			{
				$remotefiles = $expectedLinuxPath + "/" +$exactFile
			}
			else
			{
				$remotefiles = $remotefiles + ", $expectedLinuxPath" + "/" +$exactFile
			}
		}
		return $remotefiles
	}

	if ($ToWindows)
	{
		$remotefiles = ""
		$files = $currentLinuxFiles.Split(',')
		foreach ($newFile in $files)
		{
			$newFile = $newFile.Split("/")
			$newFileLen = $newFile.Length
			$exactFile= $newFile[$newFileLen -1]

			if(!$remotefiles)
			{
				$remotefiles = $expectedWindowsPath + "\" +$exactFile
			}
			else
			{
				$remotefiles = $remotefiles + ", $expectedWindowsPath" + "\" +$exactFile
			}
		}
		return $remotefiles
	}

}

Function VerifyDIPafterInitialDeployment($DeployedServices)
{
	$hsNames = $DeployedServices.Split('^')

	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			LogMsg "Checking : $($VM.Name)"

			$VMEndpoints = Get-AzureEndpoint -VM $VM
			$VMSSHPort = GetPort -Endpoints $VMEndpoints -usage "SSH"
			$out = RunLinuxCmd -ip $VMEndpoints[0].Vip -port $VMSSHPort -username $user -password $password -command "$ifconfig_cmd -a" -runAsSudo
			if ($out -imatch $VM.IpAddress)
			{
				LogMsg "Expected DIP : $($VM.IpAddress); Recorded DIP : $($VM.IpAddress);"
				LogMsg "$($VM.Name) has correct DIP.."
			}
			else
			{
				LogErr "INCORRECT DIP DETAIL : $($VM.Name)"
				$ErrCount = $ErrCount + 1
			}
		}

		if ($ErrCount -eq 0)
		{
			$testRusult = "True"
		}
		else 
		{
			$testRusult = "False"
		}
	}
	return $testRusult
}

Function VerifyDNSServerInResolvConf($DeployedServices, $dnsServerIP)
{
	$hsNames = $DeployedServices.Split('^')

	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			LogMsg "Checking resolv.conf file of : $($VM.Name)"

			$VMEndpoints = Get-AzureEndpoint -VM $VM
			$VMSSHPort = GetPort -Endpoints $VMEndpoints -usage "SSH"
			$out = RunLinuxCmd -ip $VMEndpoints[0].Vip -port $VMSSHPort -username $user -password $password -command "cat /etc/resolv.conf" -runAsSudo
			if ($out -imatch $dnsServerIP)
			{
				LogMsg "Expected DNS IP : $dnsServerIP; Recorded DNS IP : $dnsServerIP;"
				LogMsg "$($VM.Name) has correct DNS SERVER IP.."
			}
			else
			{
				LogErr "INCORRECT DNS SERVER IP : $($VM.Name)"
				$ErrCount = $ErrCount + 1
			}
		}

		if ($ErrCount -eq 0)
		{
			$testRusult = "True"
		}
		else 
		{
			$testRusult = "False"
		}
	}
	return $testRusult
}

Function RestartAllDeployments($DeployedServices)
{
	$hsNames = $DeployedServices.Split('^')
	
	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			$isRestarted = ""
			$retryCount = 3
			While(($retryCount -gt 0) -and !($isRestarted))
			{
				LogMsg "Restaring : $($VM.Name)"
				$out = Restart-AzureVM -ServiceName $hsName -Name $VM.Name
				$isRestarted = $?
				if ($isRestarted)
				{
					LogMsg "Restarted : $($VM.Name)"
				}
				else
				{
					LogErr "FAILED TO RESTART : $($VM.Name)"
					$retryCount = $retryCount + 1
					if ($retryCount -gt 0)
					{
						LogMsg "Retrying..."
					}
					if ($retryCount -eq 0)
					{
						Throw "Unable to Restart : $($VM.Name)"
					}
				}
			}
		}
	}
	
	$isAllVerified = VerifyAllDeployments -servicesToVerify $hsNames
	if ($isAllVerified -eq "True")
	{
		$isAllConnected = isAllSSHPortsEnabled -DeployedServices $deployedServices
		if ($isAllConnected -eq "True")
		{
#Set-Content .\temp\DeployedServicesFile.txt "$deployedServices"
			$retValue = "True"
		}
		else
		{
			LogErr "Unable to connect Some/All SSH ports.."
			$retValue = "False"  
		}
	}
	else
	{
		LogErr "Provision Failed for one or more VMs"
		$retValue = "False"
	}

	return $retValue
}

Function StopAllDeployments($DeployedServices)
{
	$hsNames = $DeployedServices.Split('^')

	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			$isStopped = ""
			$retryCount = 3
			While(($retryCount -gt 0) -and !($isStopped))
			{
				LogMsg "Stopping : $($VM.Name)"
				$out = Stop-AzureVM -ServiceName $hsName -Name $VM.Name -StayProvisioned -Force
				$isStopped = $?
				if ($isStopped)
				{
					LogMsg "Stopped : $($VM.Name)"
					$retValue = $true
				}
				else
				{
					LogErr "FAILED TO STOP : $($VM.Name)"
					$retryCount = $retryCount + 1
					if ($retryCount -gt 0)
					{
						LogMsg "Retrying..."
					}
					if ($retryCount -eq 0)
					{
						$retValue = $false
						Throw "Unable to Restart : $($VM.Name)"
					}
				}
			}
		}
	}
	return $retValue
}

Function StartAllDeployments($DeployedServices)
{
	$hsNames = $DeployedServices.Split('^')

	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			$isRestarted = ""
			$retryCount = 3
			While(($retryCount -gt 0) -and !($isRestarted))
			{
				LogMsg "Starting : $($VM.Name)"
				$out = Start-AzureVM -ServiceName $hsName -Name $VM.Name
				$isRestarted = $?
				if ($isRestarted)
				{
					LogMsg "Started : $($VM.Name)"
				}
				else
				{
					LogErr "FAILED TO START : $($VM.Name)"
					$retryCount = $retryCount + 1
					if ($retryCount -gt 0)
					{
						LogMsg "Retrying..."
					}
					if ($retryCount -eq 0)
					{
						Throw "Unable to Start : $($VM.Name)"
					}
				}
			}
		}
	}

	$isAllVerified = VerifyAllDeployments -servicesToVerify $hsNames
	if ($isAllVerified -eq "True")
	{
		$isAllConnected = isAllSSHPortsEnabled -DeployedServices $deployedServices
		if ($isAllConnected -eq "True")
		{
#Set-Content .\temp\DeployedServicesFile.txt "$deployedServices"
			$retValue = "True"
		}
		else
		{
			LogErr "Unable to connect Some/All SSH ports.."
			$retValue = "False"  
		}
	}
	else
	{
		LogErr "Provision Failed for one or more VMs"
		$retValue = "False"
	}

	return $retValue
}

Function Get-IPV4NetworkStartIP ($IpAddressCIDR)
{
#source = http://blog.tyang.org/2011/05/01/powershell-functions-get-ipv4-network-start-and-end-address/

	$StrNetworkAddress = ($IpAddressCIDR.split("/"))[0]
	$NetworkIP = ([System.Net.IPAddress]$StrNetworkAddress).GetAddressBytes()
	[Array]::Reverse($NetworkIP)
	$NetworkIP = ([System.Net.IPAddress]($NetworkIP -join ".")).Address
	$StartIP = $NetworkIP + 0
#Convert To Double
	If (($StartIP.Gettype()).Name -ine "double")
	{
		$StartIP = [Convert]::ToDouble($StartIP)
	}
	$StartIP = [System.Net.IPAddress]$StartIP
	Return $StartIP
}

Function Get-IPV4NetworkEndIP ($IpAddressCIDR)
{
#source = http://blog.tyang.org/2011/05/01/powershell-functions-get-ipv4-network-start-and-end-address/

	$StrNetworkAddress = ($IpAddressCIDR.split("/"))[0]
	[int]$NetworkLength = ($IpAddressCIDR.split("/"))[1]
	$IPLength = 32-$NetworkLength
	$NumberOfIPs = ([System.Math]::Pow(2, $IPLength)) -1
	$NetworkIP = ([System.Net.IPAddress]$StrNetworkAddress).GetAddressBytes()
	[Array]::Reverse($NetworkIP)
	$NetworkIP = ([System.Net.IPAddress]($NetworkIP -join ".")).Address
	$EndIP = $NetworkIP + $NumberOfIPs
	If (($EndIP.Gettype()).Name -ine "double")
	{
		$EndIP = [Convert]::ToDouble($EndIP)
	}
	$EndIP = [System.Net.IPAddress]$EndIP
	Return $EndIP
}

Function Get-IPV4NetworkRange($IpAddressCIDR)
{
	$startIP = Get-IPV4NetworkStartIP -IpAddressCIDR $IpAddressCIDR
	$EndIP = Get-IPV4NetworkEndIP -IpAddressCIDR $IpAddressCIDR
	$ipStream = ''
# created by Dr. Tobias Weltner, MVP PowerShell source = http://ficility.net/tag/ip-address-powershell/
	$ip1 = ([System.Net.IPAddress]$startIP).GetAddressBytes()
	[Array]::Reverse($ip1)
	$ip1 = ([System.Net.IPAddress]($ip1 -join '.')).Address
	$ip2 = ([System.Net.IPAddress]$EndIP).GetAddressBytes()
	[Array]::Reverse($ip2)
	$ip2 = ([System.Net.IPAddress]($ip2 -join '.')).Address
	for ($x=$ip1; $x -le $ip2; $x++) 
	{
		$ip = ([System.Net.IPAddress]$x).GetAddressBytes()
		[Array]::Reverse($ip)
		$ip = $ip -join '.'
		$ipStr = $ip.ToString()
		if(!$ipStream)
		{
			$ipStream = $ipStr
		}
		else
		{
			$ipStream = $ipStream + "^" + $ipStr
		}
	}
	return $ipStream
}

Function DetectSubnet($inputString)
{
	$subnet1 = '192.168.4.192/26'
	$subnet2 = '192.168.4.128/26'

	$subnet1Range = Get-IPV4NetworkRange -IpAddressCIDR $subnet1
	$subnet2Range = Get-IPV4NetworkRange -IpAddressCIDR $subnet2

	$subnet1Range = $subnet1Range.split('^')
	$subnet2Range = $subnet2Range.split('^')

	$isDetected = 'False'

	if ($isDetected = "false")
	{
		foreach ($IP in $subnet1Range)
		{
			if ($inputString -imatch $IP)
			{
				$isDetected = "True"
				$detectedSubnet = "subnet1"
				break
			}
		}
	}
	if ($isDetected = "false")
	{
		foreach ($IP in $subnet2Range)
		{
			if ($inputString -imatch $IP)
			{
				$isDetected = "True"
				$detectedSubnet = "subnet2"
				break
			}
		}
	}
	return $detectedSubnet
}

Function VerifyGatewayVMsInHostedService($DeployedServices)
{
	$hsNames = $DeployedServices.Split('^')

	foreach ($hsName in $hsNames)
	{
		$ErrCount = 0
		$hsDetails =  Get-AzureService -ServiceName $hsName
		$VMs =  Get-AzureVM -ServiceName $hsName

		foreach ($VM in $VMs)
		{
			LogMsg "Checking Gateway : $($VM.Name)"

			$VMEndpoints = Get-AzureEndpoint -VM $VM
			$VMSSHPort = GetPort -Endpoints $VMEndpoints -usage "SSH"
			$currentVMGateway = RunLinuxCmd -ip $VMEndpoints[0].Vip -port $VMSSHPort -username $user -password $password -command "route" -runAsSudo
			$currentVMDIP = $VM.IpAddress
			$currentVMDIPSubnet = DetectSubnet -inputString $currentVMDIP
			$currentVMGatewaySubnet = DetectSubnet -inputString $currentVMGateway
			LogMsg "DIP subnet subnet detected : $currentVMDIPSubnet"
			LogMsg "Gateway subnet detected	: $currentVMGatewaySubnet"

			if ($currentVMDIPSubnet -eq $currentVMGatewaySubnet)
			{
				LogMsg "PASS"
			}
			else
			{
				LogErr "FAIL"
				$ErrCount = $ErrCount + 1
			}
		}

		if ($ErrCount -eq 0)
		{
			$testRusult = "True"
		}
		else 
		{
			$testRusult = "False"
		}
	}
	return $testRusult
}

Function DoSSHTest($fromVM, $toVM, $command, [switch]$runAsSudo, [switch]$hostnameMode)
{
	if($runAsSudo)
	{
		if($hostnameMode)
		{
			$sshOutput = RunLinuxCmdOnRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteCommand $command -runAsSudo -hostnameMode
		}
		else
		{
			$sshOutput = RunLinuxCmdOnRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteCommand $command -runAsSudo 
		}
	}
	else
	{
		if($hostnameMode)
		{
			$sshOutput = RunLinuxCmdOnRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteCommand $command -hostnameMode
		}
		else
		{
			$sshOutput = RunLinuxCmdOnRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteCommand $command
		}
	}
#Write-host "Printing output"

#Write-Host "$sshOutput"
	if ($sshOutput -imatch "ExitCode : 0")
	{
		return "PASS"
	}
	else
	{
		return "FAIL"
	}
}

Function DoSCPTest($fromVM, $toVM, $filesToCopy, [switch]$hostnameMode)
{
	if($hostnameMode)
	{
		$scpUpload = RemoteCopyRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteFiles $filesToCopy -upload -hostnameMode
#Write-Host $scpUpload
	}
	else
	{
		$scpUpload = RemoteCopyRemoteVM -intermediateVM $fromVM -remoteVM $toVM -remoteFiles $filesToCopy -upload
#Write-Host $scpUpload
	}

	if(($scpUpload -eq "True"))
	{
		return "PASS"
	}
	else
	{
		return "FAIL"
	}
}

Function RunMysqlCmd ($intermediateVM, $mysqlServer, $MysqlUsername, $MysqlPassword, $mysqlCommand, [switch]$HostnameMode)
{
	if($HostnameMode)

	{
		$mysqlOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshPort -command "mysql -u`'$MysqlUsername`' -p`'$MysqlPassword`' -h `'$($mysqlServer.hostname)`' -e `'$mysqlCommand`'" -runAsSudo
	}
	else
	{
		$mysqlOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshPort -command "mysql -u`'$MysqlUsername`' -p`'$MysqlPassword`' -h`'$($mysqlServer.ip)`' -e `'$mysqlCommand`'" -runAsSudo

	}
	return $mysqlOutput
}

Function DoMysqlAccessTest ($fromVM, $mysqlServer, $MysqlUsername, $MysqlPassword, [switch]$HostnameMode)
{
	try
	{
		if($HostnameMode)
		{
			$mysqlOutput = RunMysqlCmd -intermediateVM $fromVM -mysqlServer $mysqlServer -MysqlUsername $MysqlUsername -MysqlPassword $MysqlPassword -mysqlCommand "\s" -HostnameMode
		}
		else
		{
			$mysqlOutput = RunMysqlCmd -intermediateVM $fromVM -mysqlServer $mysqlServer -MysqlUsername $MysqlUsername -MysqlPassword $MysqlPassword -mysqlCommand "\s"
		}
		if ($mysqlOutput -imatch "TCP/IP")
		{
			$retValue = "PASS"
		}
		else
		{
			$retValue = "FAIL"
		}
	}
	catch
	{
		$retValue = "FAIL"
	}
	$logfilepath  = "$($FromVM.LogDir)\mysqlOutput.log"
	Set-Content -Path $logfilepath -Value $mysqlOutput
	return $retValue
}

Function DoMysqlOperationsTest ($fromVM, $mysqlServer, $MysqlUsername, $MysqlPassword, [switch]$HostnameMode)
{
	$logfilepath  = "$($FromVM.LogDir)\mysqlOutput.log"

	$mySqlOperations = @()
	$mySqlOperations = $mySqlOperations + "use test;"
	$mySqlOperations = $mySqlOperations + "CREATE TABLE IF NOT EXISTS products (productID	INT UNSIGNED  NOT NULL AUTO_INCREMENT,productCode  CHAR(3)	   NOT NULL DEFAULT `"`",name		 VARCHAR(30)   NOT NULL DEFAULT `"`",quantity	 INT UNSIGNED  NOT NULL DEFAULT 0,price		DECIMAL(7,2)  NOT NULL DEFAULT 99999.99,PRIMARY KEY  (productID));"
	$mySqlOperations = $mySqlOperations + "SHOW TABLES;"
	$mySqlOperations = $mySqlOperations + "DESCRIBE products;"
	$mySqlOperations = $mySqlOperations + "INSERT INTO products VALUES (1001, `"PEN`", `"Pen Red`", 5000, 1.23);"
	$mySqlOperations = $mySqlOperations + "INSERT INTO products VALUES (NULL, `"PEN`", `"Pen Blue`",  8000, 1.25),(NULL, `"PEN`", `"Pen Black`", 2000, 1.25);"
	$mySqlOperations = $mySqlOperations + "INSERT INTO products (productCode, name, quantity, price) VALUES (`"PEC`", `"Pencil 2B`", 10000, 0.48), (`"PEC`", `"Pencil 2H`", 8000, 0.49);"
	$mySqlOperations = $mySqlOperations + "SELECT * FROM products;"
	$mySqlOperations = $mySqlOperations + "DELETE FROM products;"
	$mySqlOperations = $mySqlOperations + "SHOW TABLES;"
	$mySqlOperations = $mySqlOperations + "DROP TABLE products;"
	$allOperations = ''
	foreach ($mysqlCommand in $mySqlOperations)
	{
		$allOperations = "use test; SHOW TABLES; SHOW DATABASES; describe products"
	}
	$allOperations =  $allOperations 
	if($HostnameMode)
	{
		Add-Content -Path $logfilepath -Value "Executing : $allOperations"
		$mysqlOutput = RunMysqlCmd -intermediateVM $fromVM -mysqlServer $mysqlServer -MysqlUsername $MysqlUsername -MysqlPassword $MysqlPassword -mysqlCommand $allOperations -HostnameMode
		set-Content -Path $logfilepath -Value "$mysqlOutput"
	}
	else
	{
		Add-Content -Path $logfilepath -Value "Executing : $allOperations"
		$mysqlOutput = RunMysqlCmd -intermediateVM $fromVM -mysqlServer $mysqlServer -MysqlUsername $MysqlUsername -MysqlPassword $MysqlPassword -mysqlCommand $allOperations
		set-Content -Path $logfilepath -Value "$mysqlOutput"
	}

	$CompleteMysqlOutput = Get-Content -Path $logfilepath
	if ($CompleteMysqlOutput -imatch "Error")
	{
		$retValue = "FAIL"
	}
	else
	{
		$retValue = "PASS"
	}
	return $retValue
}

Function DoNfsShareAccessTest ($fromVM, $nfsServer, $nfsServerDirctory, [switch]$HostnameMode)
{
	try
	{
		if($HostnameMode)
		{
			LogMsg "Mounting $($nfsServer.Hostname):$nfsServerDirctory to home directory .."
			$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "mount $($nfsServer.Hostname):$nfsServerDirctory ~" -runAsSudo 
			LogMsg "Unmounting.."
			$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "umount ~ -lf" -runAsSudo
																																											$retValue = "PASS"
		}

		else
		{
			LogMsg "Mounting $($nfsServer.ip):$nfsServerDirctory to home directory .."
			$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "mount $($nfsServer.ip):$nfsServerDirctory ~" -runAsSudo																								LogMsg "Unmounting.."
			$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "umount ~ -lf" -runAsSudo
			$retValue = "PASS"
		}
	}
	catch
	{
		LogErr "Failed."
		$retValue = "FAIL"
	}
	return $retValue
}

Function DoNfsShareFileTranferTest ($fromVM, $nfsServer, $nfsServerDirctory, [switch]$HostnameMode)
{
	$logfilepath = "$($fromVM.LogDir)\nfsFileCreate.log"
	try
	{
		if($HostnameMode)
		{
			LogMsg "Mounting $($nfsServer.Hostname):$nfsServerDirctory to home directory .."
			$output = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "mount $($nfsServer.Hostname):$nfsServerDirctory ~" -runAsSudo 2>&1 
		}

		else
		{
			LogMsg "Mounting $($nfsServer.ip):$nfsServerDirctory to home directory .."
			$output =  RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "mount $($nfsServer.ip):$nfsServerDirctory ~" -runAsSudo 2>&1 
		}
		LogMsg "Tranferring 1MB data.."
		$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "dd if=/dev/zero of=~/testfile bs=1M count=1" -runAsSudo
		LogMsg "Unmounting.."
		$suppressedOut = RunLinuxCmd -ip $fromVM.ip -port $fromVM.sshport -username $fromVM.user -password $fromVM.password -command "umount ~ -lf" -runAsSudo
		$retValue = "PASS"
	}
	catch
	{
		LogErr "Failed."
		$retValue = "FAIL"
	}
	return $retValue
}

Function DoSSHTestFromLocalVM($intermediateVM, $LocalVM, $toVM,[switch]$hostnameMode)
{

	if($hostnameMode)
	{
		LogMsg "Executing - date - command on $($toVM.Hostname) .."
		$sshOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshport -runAsSudo -command "python /home/$user/RunSSHCmd.py -s `'$($LocalVM.ip)`' -u $($LocalVM.user) -p`'$($LocalVM.password)`' -P $($LocalVM.sshPort) -c `'python /home/$user/RunSSHCmd.py -s `"$($toVM.hostname)`" -u test -p `"$($toVM.password)`"`"  -P 22 -c `"date`" -o yes`'"

	}
	else
	{
		LogMsg "Executing - date - command on $($toVM.DIP) .."
		$sshOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshport -runAsSudo -command "python /home/$user/RunSSHCmd.py -s `'$($LocalVM.ip)`' -u $($LocalVM.user) -p`'$($LocalVM.password)`' -P $($LocalVM.sshPort) -c `'python /home/$user/RunSSHCmd.py -s `"$($toVM.dip)`" -u test -p `"$($toVM.password)`"`"  -P 22 -c `"date`" -o yes`'"
	}
	LogMsg "Verifying output.."
	$logfilepath = $toVM.logDir + "\sshOutput.log"
	LogMsg "Writing output to $logfilepath ..."
	Set-Content -Path $logfilepath -Value $sshOutput
#Write-host "Printing output"

#Write-Host "$sshOutput"
	if ($sshOutput -imatch (Get-Date).Year)
	{
		return "PASS"
	}
	else
	{
		return "FAIL"
	}
}

Function DoSCPTestFromLocalVM( $intermediateVM, $LocalVM, $toVM, [switch]$hostnameMode)
{
	$logFilepath = $toVM.logDir + "\scpOutput.log"
	LogMsg "Creating 1MB file on local VM..."
	$out = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $LocalVM -remoteCommand "`'dd\ if=/dev/zero\ of=/home/$user/testfile\ bs=1M\ count=1`'" -runAsSudo

	if($hostnameMode)
	{
		LogMsg "File Created. Now copying it to $($toVM.Hostname) ..."
		$scpOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshport -runAsSudo -command "python /home/$user/RunSSHCmd.py -s `'$($LocalVM.ip)`' -u $($LocalVM.user) -p`'$($LocalVM.password)`' -P $($LocalVM.sshPort) -c `'python /home/$user/RemoteCopy.py -c `"$($toVM.Hostname)`" -m upload -u `"$($toVM.user)`" -p `"$($toVM.password)`"`" -P 22 -r `"/home/$user`" -f `"/home/$user/testfile`"`'"
	}
	else
	{
		LogMsg "File Created. Now copying it to $($toVM.DIP) ..."
		$scpOutput = RunLinuxCmd -username $intermediateVM.user -password $intermediateVM.password -ip $intermediateVM.ip -port $intermediateVM.sshport -runAsSudo -command "python /home/$user/RunSSHCmd.py -s `'$($LocalVM.ip)`' -u $($LocalVM.user) -p`'$($LocalVM.password)`' -P $($LocalVM.sshPort) -c `'python /home/$user/RemoteCopy.py -c `"$($toVM.DIP)`" -m upload -u `"$($toVM.user)`" -p `"$($toVM.password)`"`" -P 22 -r `"/home/$user`" -f `"/home/$user/testfile`"`'"
	}
	LogMsg "Writing output to $logfilepath ..."
	Set-Content -Path $logFilepath -Value $scpOutput

	if(($scpOutput -imatch "OK!") -and ($scpOutput -imatch "Connected"))
	{
		LogMsg "File transferred successfully."
		return "PASS"
	}
	else
	{
		return "FAIL"
	}
}

Function StartIperfServerOnRemoteVM($remoteVM, $intermediateVM)
{
	#$NewremoteVMcmd = ($remoteVM.cmd).Replace(" ","\ ")
	$NewremoteVMcmd = $remoteVM.cmd
	Write-Host $NewremoteVMcmd 
	LogMsg "Deleting any previous server logs ..."
	$DeletePreviousLogs = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "rm -rf /root/*.txt /root/*.log" -runAsSudo
	$CommandOutput = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand $NewremoteVMcmd -runAsSudo
	LogMsg "Checking if server started successfully or not ..."
	$isServerStarted = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "cat  /root/isServerStarted.txt" -runAsSudo
	if(($isServerStarted -imatch "yes") -and ($CommandOutput -imatch "ExitCode : 0"))
	{
		LogMsg "Server started successfully ..."
		$retValue = "True"
	}
	else
	{
		$retValue = "False"
		LogErr "Server Failed to start ..."
	}
	return $retValue
}

Function StartIperfClientOnRemoteVM($remoteVM, $intermediateVM)
{
	#$NewremoteVMcmd = ($remoteVM.cmd).Replace(" ","\ ")
	$NewremoteVMcmd = $remoteVM.cmd
	Write-Host $NewremoteVMcmd 
	LogMsg "Deleting any previous client logs ..."
	$DeletePreviousLogs = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "rm -rf /root/*.txt /root/*.log" -runAsSudo
	$CommandOutput = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand $NewremoteVMcmd -runAsSudo
	LogMsg "Checking if client connected successfully or not ..."

	$DeletePreviousLogs = RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "cp /root/Runtime.log /root/start-client.py.log" -runAsSudo

	Set-Content -Value (RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "cat /root/start-client.py.log" -runAsSudo) -Path ("$($remoteVM.logDir)" + "\start-client.py.log")
	Set-Content -Value (RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "cat /root/state.txt" -runAsSudo) -Path ("$($remoteVM.logDir)" + "\state.txt")
	Set-Content -Value (RunLinuxCmdOnRemoteVM -intermediateVM $intermediateVM -remoteVM $remoteVM -remoteCommand "cat /root/Summary.log" -runAsSudo) -Path ("$($remoteVM.logDir)" + "\Summary.log")


	$clientState = Get-Content "$($remoteVM.Logdir)\state.txt"
	$clientSummary = Get-Content "$($remoteVM.Logdir)\Summary.log"
	Write-Host $clientState 
	Write-Host $clientSummary
#>>>Remove Temporary files..
	Remove-Item "$($remoteVM.Logdir)\state.txt" -Force
	Remove-Item "$($remoteVM.Logdir)\Summary.log" -Force
	if($clientState -imatch "TestCompleted" -and $clientSummary -imatch "PASS")
	{
		return $true
	}
	else
	{
		return $false
	}
}

Function IperfLocalToVNETUdpTest ($vnetAsServer, $localAsClient, $intermediateVM)
{
	LogMsg "Removing any previous Server logs.."
	$suppressedOut = RunLinuxCmd -username $vnetAsServer.user -password $vnetAsServer.password -ip $vnetAsServer.ip -port $vnetAsServer.sshPort -command "rm -rf *.txt *.log"  -runAsSudo
	StartIperfServer -node $vnetAsServer
	$isServerStarted = IsIperfServerStarted -node $vnetAsServer
	if($isServerStarted -eq $true)
	{
		LogMsg "Starting iperf client ..."
		$isClientConnected = StartIperfClientOnRemoteVM -remoteVM $localAsClient -intermediateVM $intermediateVM

		if ($isClientConnected -eq $true)
		{
			LogMsg "Checking if server received connections from client of not ..."
			$checkServer = IsIperfServerRunning -node $vnetAsServer
			if($checkServer -eq $true)
			{
				LogMsg "Server was successfully connected to client.."
				$retValue = "PASS"
			}
			else
			{
				LogMsg "Failures detected on server .."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Client failed to connect to server..."
			$retValue = "Fail"
		}
	}
	else
	{
		LogErr "Failed to start iperf server."
		$retValue = "Fail"
	}
	return $retValue
}

Function IperfVnetToLocalUdpTest ($vnetAsClient, $localAsServer)
{

	$isServerStarted = StartIperfServerOnRemoteVM -remoteVM $localAsServer -intermediateVM $vnetAsClient
	if($isServerStarted -eq "True")
	{
		LogMsg "Starting iperf client ..."
		LogMsg "Removing any previous client logs.."
		$suppressedOut = RunLinuxCmd -username $vnetAsClient.user -password $vnetAsClient.password -ip $vnetAsClient.ip -port $vnetAsClient.sshPort -command "rm -rf *.txt *.log"  -runAsSudo
		StartIperfClient -node $vnetAsClient
		$isClientConnected = IsIperfClientStarted -node $vnetAsClient
		if ($isClientConnected -eq $true)
		{
			LogMsg "Checking if server received connections from client of not ..."
			$temp = RunLinuxCmdOnRemoteVM -intermediateVM $vnetAsClient -remoteVM $localAsServer -runAsSudo -remoteCommand "cp /root/iperf-server.txt /home/$user/"
			$checkServer = RunLinuxCmdOnRemoteVM -intermediateVM $vnetAsClient -remoteVM $localAsServer -runAsSudo -remoteCommand "/home/$user/check-server.py"
			$checkServerSummary = RunLinuxCmdOnRemoteVM -intermediateVM $vnetAsClient -remoteVM $localAsServer -runAsSudo -remoteCommand "cat ~/Summary.log"		
			if($checkServerSummary -imatch "PASS")
			{
				LogMsg "Server was successfully connected to client.."
				$retValue = "PASS"
			}
			else
			{
				LogMsg "Failures detected on server .."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Client failed to connect to server..."
			$retValue = "Fail"
		}
	}
	else
	{
		LogErr "Failed to start iperf server."
		$retValue = "Fail"
	}
	return $retValue
}
#endregion

#region E2E Feature Tests Methods

Function GetTotalPhysicalDisks($FdiskOutput)
{
	$physicalDiskNames = ("sda","sdb","sdc","sdd","sde","sdf","sdg","sdh","sdi","sdj","sdk","sdl","sdm","sdn",
			"sdo","sdp","sdq","sdr","sds","sdt","sdu","sdv","sdw","sdx","sdy","sdz", "sdaa", "sdab", "sdac", "sdad","sdae", "sdaf", "sdag", "sdah", "sdai")
	$diskCount = 0
	foreach ($physicalDiskName in $physicalDiskNames)
	{
		if ($FdiskOutput -imatch "Disk /dev/$physicalDiskName")
		{
			$diskCount += 1
		}
	}
	return $diskCount
}

Function GetNewPhysicalDiskNames($FdiskOutputBeforeAddingDisk, $FdiskOutputAfterAddingDisk)
{
	$availableDisksBeforeAddingDisk = ""
	$availableDisksAfterAddingDisk = ""
	$physicalDiskNames = ("sda","sdb","sdc","sdd","sde","sdf","sdg","sdh","sdi","sdj","sdk","sdl","sdm","sdn",
			"sdo","sdp","sdq","sdr","sds","sdt","sdu","sdv","sdw","sdx","sdy","sdz", "sdaa", "sdab", "sdac", "sdad","sdae", "sdaf", "sdag", "sdah", "sdai")
	foreach ($physicalDiskName in $physicalDiskNames)
	{
		if ($FdiskOutputBeforeAddingDisk -imatch "Disk /dev/$physicalDiskName")
		{
			if ( $availableDisksBeforeAddingDisk -eq "" )
			{
				$availableDisksBeforeAddingDisk = "/dev/$physicalDiskName"
			}
			else
			{
				$availableDisksBeforeAddingDisk = $availableDisksBeforeAddingDisk + "^" + "/dev/$physicalDiskName"
			}
		}
	}
	foreach ($physicalDiskName in $physicalDiskNames)
	{
		if ($FdiskOutputAfterAddingDisk -imatch "Disk /dev/$physicalDiskName")
		{
			if ( $availableDisksAfterAddingDisk -eq "" )
			{
				$availableDisksAfterAddingDisk = "/dev/$physicalDiskName"
			}
			else
			{
				$availableDisksAfterAddingDisk = $availableDisksAfterAddingDisk + "^" + "/dev/$physicalDiskName"
			}
		}
	}
	$newDisks = ""
	foreach ($afterDisk in $availableDisksAfterAddingDisk.Split("^"))
	{
		if($availableDisksBeforeAddingDisk -imatch $afterDisk)
		{

		}
		else
		{
			if($newDisks -eq "")
			{
				$newDisks = $afterDisk
			}
			else
			{
				$newDisks = $newDisks + "^" + $afterDisk
			}
		}
	}
	return $newDisks
}
Function CreateHotAddRemoveDataDiskNode
{
	param(
			[string] $ServiceName,
			[string] $nodeIp,
			[string] $nodeSshPort,
			[string] $user,
			[string] $password,
			[string] $files,
			[int] $Lun,
			[string] $InstanceSize,
			[string] $ExistingDiskMediaLink,
			$allExistingDisks,
			[string] $logDir)

	$objNode = New-Object -TypeName PSObject
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ServiceName -Value $ServiceName -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ip -Value $nodeIp -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name sshPort -Value $nodeSshPort -Force 
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name user -Value $user -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name password -Value $password -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name files -Value $files -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name logDir -Value $LogDir -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name Lun -Value $Lun -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name InstanceSize -Value $InstanceSize -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name AttachedDisks -Value @() -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name ExistingDiskMediaLink -Value $ExistingDiskMediaLink -Force
	Add-Member -InputObject $objNode -MemberType NoteProperty -Name allExistingDisks -Value $allExistingDisks -Force
	return $objNode
}

Function PerformIOTestOnDisk($testVMObject, [string]$attachedDisk, [string]$diskFileSystem)
{
	$retValue = "Aborted"
   	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	if ( $diskFileSystem -imatch "xfs" )
	{
		 $diskFileSystem = "xfs -f"
	}
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
	if ($isVMAlive -eq "True")
	{
		$retValue = "FAIL"
		$mountPoint = "/mnt/datadisk"
		LogMsg "Performing I/O operations on $attachedDisk.."
		$LogPath = "$LogDir\VerifyIO$($attachedDisk.Replace('/','-')).txt"
		$dmesgBefore = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "dmesg" -runAsSudo 
		#CREATE A MOUNT DIRECTORY
		$out = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "mkdir -p $mountPoint" -runAsSudo 
		$partitionNumber=1
		$PartitionDiskOut = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "./ManagePartitionOnDisk.sh -diskName $attachedDisk -create yes -forRaid no" -runAsSudo 
		$FormatDiskOut = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "time mkfs.$diskFileSystem $attachedDisk$partitionNumber" -runAsSudo -runMaxAllowedTime 2400 
		$out = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "mount -o nobarrier $attachedDisk$partitionNumber $mountPoint" -runAsSudo 
		Add-Content -Value $formatDiskOut -Path $LogPath -Force
		$ddOut = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "dd if=/dev/zero bs=1024 count=1000000 of=$mountPoint/file_1GB" -runAsSudo -runMaxAllowedTime 1200
		WaitFor -seconds 10
		Add-Content -Value $ddOut -Path $LogPath
		try
		{
			$out = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "umount $mountPoint" -runAsSudo 
		}
		catch
		{
			LogMsg "umount failed. Trying umount -l"
			$out = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "umount -l $mountPoint" -runAsSudo 
		}
		$dmesgAfter = RunLinuxCmd -username $testVMUsername -password $testVMPassword -ip $testVMVIP -port $testVMSSHport -command "dmesg" -runAsSudo
		$addedLines = $dmesgAfter.Replace($dmesgBefore,$null)
		LogMsg "Kernel Logs : $($addedLines.Replace('[32m','').Replace('','').Replace('',''))" -LinuxConsoleOuput
		$retValue = "PASS"	
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	return $retValue
}
Function DoHotAddNewDataDiskTest ($testVMObject, [int]$diskSizeInGB )
{
	
	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$HotAddLogFile = "$($testVMObject.logDir)\Hot-Add-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
	$retValue = "ABORTED"
	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------ADD DISK TO LUN $testLun : START----------------------" -Path $HotAddLogFile -Encoding UTF8
#GetCurrentDiskInfo

		$fdiskOutputBeforeAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		Add-Content  -Value "Before Adding New Disk : " -Path $HotAddLogFile -Encoding UTF8
		Add-Content  -Value $fdiskOutputBeforeAddingDisk -Path $HotAddLogFile -Encoding UTF8
		$disksBeforeAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $fdiskOutputBeforeAddingDisk
		

#Add datadisk to VM
		$supressedOut = RetryOperation -operation { Get-AzureVM -ServiceName $testVMServiceName | Add-AzureDataDisk -CreateNew -DiskSizeInGB $diskSizeInGB -DiskLabel "TestDisk-$testLun" -LUN $testLun | Update-AzureVM  } -maxRetryCount 5 -retryInterval 5 -description "Attaching $diskSizeInGB GB disk to LUN : $testLun."
		if ( ( $supressedOut.OperationDescription -eq "Update-AzureVM"  ) -and ( $supressedOut.OperationStatus -eq "Succeeded" ))
		{
			LogMsg "Disk Attached Successfully.."
			WaitFor -seconds 10
			LogMsg "Checking VM status.."
			$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$newDiskAdded = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($newDiskAdded -eq "FAIL"))
				{
					LogMsg "Attempt : $retryCount : Checking for new disk."
					$fdiskOutputAfterAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $fdiskOutputAfterAddingDisk
					if ( ($disksBeforeAddingNewDisk + 1) -eq $disksafterAddingNewDisk )
					{
						$newDiskAdded = "PASS"
						LogMsg "New Disk detected."
						$newDisknames = GetNewPhysicalDiskNames -FdiskOutputBeforeAddingDisk $fdiskOutputBeforeAddingDisk -FdiskOutputAfterAddingDisk $fdiskOutputAfterAddingDisk
						if($detectedDistro -imatch "SLES" )
						{
							$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "ext3"
						}
						else
						{
							$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "ext4"
						}
						$xfsResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "xfs"
						if ( ($extResult  -eq "PASS") -and ($xfsResult  -eq "PASS") )
						{
							$retValue = "PASS"
						}
						else
						{
							$retValue = "FAIL"
						}
					}
					else
					{
						Write-Host "New disk not detected."
						WaitFor -seconds 10
						$newDiskAdded = "FAIL"
						$retryCount += 1
					}
				}
				Add-Content  -Value "After Adding New Disk : " -Path $HotAddLogFile -Encoding UTF8
				Add-Content  -Value $fdiskOutputAfterAddingDisk -Path $HotAddLogFile -Encoding UTF8
			}
			else
			{
				LogErr "VM Status : OFF."
				LogErr "VM is not Alive after adding new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Failed to Attach disk."
			$retValue = "FAIL"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------ADD DISK TO LUN $testLun : END : $retValue----------------------" -Path $HotAddLogFile 
	return $retValue
}
Function DoHotRemoveDataDiskTest ($testVMObject)
{

	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$HotRemoveLogFile = "$($testVMObject.logDir)\Hot-Remove-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
	$retValue = "ABORTED"
	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------REMOVE DISK FROM LUN $testLun : START----------------------" -Path $HotRemoveLogFile -Encoding UTF8
#GetCurrentDiskInfo

		$out = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		$disksBeforeRemovingDisk = GetTotalPhysicalDisks -FdiskOutput $out
		Add-Content  -Value "Before Removing Disk : " -Path $HotRemoveLogFile -Encoding UTF8
		Add-Content  -Value $out -Path $HotRemoveLogFile  -Encoding UTF8
#Add datadisk to VM
		$supressedOut = RetryOperation -operation { Get-AzureVM -ServiceName $testVMServiceName | Remove-AzureDataDisk -LUN $testLun | Update-AzureVM } -maxRetryCount 5 -retryInterval 5 -description "Removing disk from LUN : $testLun."
		if ( ( $supressedOut.OperationDescription -eq "Update-AzureVM"  ) -and ( $supressedOut.OperationStatus -eq "Succeeded" ))
		{
			LogMsg "Disk Removed Successfully.."
			WaitFor -seconds 10
			$isVMAlive = RetryOperation -operation {Test-TCP -testIP $testVMVIP -testport $testVMSSHport} -description "Checking VM status.." -expectResult "True"
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$retValue = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($retValue -eq "FAIL"))
				{
					$out = ""
					LogMsg "Attempt : $retryCount : Verifying removal of disk."
					$out = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterRemovingDisk = GetTotalPhysicalDisks -FdiskOutput $out
					if ( ($disksBeforeRemovingDisk - 1) -eq $disksafterRemovingDisk )
					{
						LogMsg "Disk removed successfully.."
						$retValue = "PASS"
					}
					else
					{
						LogErr "Disk can be still visible in VM."
						WaitFor -seconds 10
						$retValue = "FAIL"
						$retryCount += 1
					}
				}
				Add-Content  -Value "After removing Disk : " -Path $HotRemoveLogFile -Encoding UTF8
				Add-Content  -Value $out -Path $HotRemoveLogFile -Encoding UTF8
			}
			else
			{
				LogMsg "VM Status : OFF."
				LogErr "VM is not Alive after removing new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Failed to remove disk."
			$retValue = "FAIL"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------REMOVE DISK FROM LUN $testLun : END : $retValue----------------------" -Path $HotRemoveLogFile -Encoding UTF8
	return $retValue
}

Function DoHotAddNewDataDiskTestParallel ($testVMObject, $TotalLuns)
{

	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$retValue = "ABORTED"
	$HotAddLogFile = "$($testVMObject.logDir)\Hot-Add-Disk.log"
#$HotAddLogFile = ".\temp\Hot-Add-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport

	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------ADD $TotalLuns DISKS : START----------------------" -Path $HotAddLogFile -Encoding UTF8
#GetCurrentDiskInfo

		$FdiskOutputBeforeAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		Add-Content  -Value "Before Adding Disks : " -Path $HotAddLogFile -Encoding UTF8
		Add-Content  -Value $FdiskOutputBeforeAddingDisk -Path $HotAddLogFile -Encoding UTF8
		$disksBeforeAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $FdiskOutputBeforeAddingDisk

#Add datadisk to VM
		$lunCounter = 0
		$HotAddCommand = "Get-AzureVM -ServiceName $testVMServiceName"
		while ($lunCounter -lt $TotalLuns)
		{
			$diskSizeInGB = ($lunCounter+1)*10
			$HotAddCommand += " | Add-AzureDataDisk -CreateNew -DiskSizeInGB $diskSizeInGB -DiskLabel `"TestDisk-$lunCounter`" -LUN $lunCounter"
			$lunCounter += 1
		}
		$HotAddCommand += " | Update-AzureVM"
		
		$suppressedOut = RetryOperation -operation {Invoke-Expression $HotAddCommand } -maxRetryCount 5 -retryInterval 5 -description "Attaching $TotalLuns disks parallely."
		if(($suppressedOut.OperationDescription -eq "Update-AzureVM") -and ( $suppressedOut.OperationStatus -eq "Succeeded"))
		{
			LogMsg "$TotalLuns Disks Attached Successfully.."

			WaitFor -seconds 10
			$isVMAlive = RetryOperation -operation {Test-TCP -testIP $testVMVIP -testport $testVMSSHport} -description "Checking VM status.." -expectResult "True"
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$isAllDiskDetected = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($isAllDiskDetected -eq "FAIL"))
				{
					$out = ""
					LogMsg "Attempt : $retryCount : Checking for new disk."
					$FdiskOutputAfterAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $FdiskOutputAfterAddingDisk
					if ( ($disksBeforeAddingNewDisk + $TotalLuns) -eq $disksafterAddingNewDisk )
					{
						LogMsg "All $TotalLuns New Disks detected."
						$newDisks = GetNewPhysicalDiskNames -FdiskOutputBeforeAddingDisk $FdiskOutputBeforeAddingDisk -FdiskOutputAfterAddingDisk $FdiskOutputAfterAddingDisk
						$isAllDiskDetected = "PASS"
						$successCount = 0
						$errorCount = 0
						foreach ( $newDisk in $newDisks.split("^"))
						{
							$extResult = $null
							$xfsResult = $null
							if($detectedDistro -imatch "SLES" )
							{
								$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "ext3"
							}
							else
							{
								$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "ext4"
							}
							$xfsResult = PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "xfs"
							if ( ($extResult  -eq "PASS") -and ($xfsResult  -eq "PASS") )
							{
								$successCount += 1
							}
							else
							{
								LogErr "IO Operations failed for $newDisk"
								$errorCount += 1
							}
						}
						if ($successCount -eq $TotalLuns)
						{
							$retValue = "PASS"
						}
						else
						{
							$retValue = "FAIL"
							LogErr "I/O operations on $errorCount disks."
						}
					}
					else
					{
						$NotDetectedDisks = ( ($disksBeforeAddingNewDisk + $TotalLuns) - $disksafterAddingNewDisk )
						LogErr "Total undetected disks : $NotDetectedDisks"
						WaitFor -seconds 10
						$isAllDiskDetected = "FAIL"
						$retryCount += 1
					}

				}
				Add-Content  -Value "After Adding New Disk : " -Path $HotAddLogFile -Encoding UTF8
				Add-Content  -Value $FdiskOutputAfterAddingDisk -Path $HotAddLogFile -Encoding UTF8
			}
			else
			{
				LogMsg "VM Status : OFF."
				LogErr "VM is not Alive after adding new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Failed to attach disks."
			$retValue = "FAIL"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------ADD DISK TO LUN $testLun : END : $retValue----------------------" -Path $HotAddLogFile 
	return $retValue
}
Function DoHotRemoveNewDataDiskTestParallel ($testVMObject, $TotalLuns)
{

	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$retValue = "ABORTED"
	$HotRemoveLogFile = "$($testVMObject.logDir)\Hot-Remove-Disk.log"
#$HotRemoveLogFile = ".\temp\Hot-Remove-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport

	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------REMOVE $TotalLuns DISKS : START----------------------" -Path $HotRemoveLogFile -Encoding UTF8
#GetCurrentDiskInfo

		$out = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		Add-Content  -Value "Before Adding Disks : " -Path $HotRemoveLogFile -Encoding UTF8
		Add-Content  -Value $out -Path $HotRemoveLogFile -Encoding UTF8
		$disksBeforeRemovingNewDisk = GetTotalPhysicalDisks -FdiskOutput $out

#Add datadisk to VM
		
		$lunCounter = 0
		$HotRemoveCommand = "Get-AzureVM -ServiceName $testVMServiceName"
		while ($lunCounter -lt $TotalLuns)
		{
			$diskSizeInGB = ($lunCounter+1)*10
			$HotRemoveCommand += " | Remove-AzureDataDisk -LUN $lunCounter"
			$lunCounter += 1
		}
		$HotRemoveCommand += " | Update-AzureVM"
		LogMsg $HotRemoveCommand
		#$supressedOut = ( Invoke-Expression $HotAddCommand )
		$RemoveOut = RetryOperation -operation {Invoke-Expression $HotRemoveCommand}  -maxRetryCount 5 -retryInterval 5 -description "Removing $TotalLuns disks parallely."
		if($RemoveOut.OperationStatus -eq "Succeeded")
		{
			LogMsg "$TotalLuns Disks removed Successfully.."

			WaitFor -seconds 10
			$isVMAlive = RetryOperation -operation {Test-TCP -testIP $testVMVIP -testport $testVMSSHport} -description "Checking VM status.." -expectResult "True"
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$retValue = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($retValue -eq "FAIL"))
				{
					$out = ""
					LogMsg "Attempt : $retryCount : Checking for new disk."
					$out = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterRemovingNewDisk = GetTotalPhysicalDisks -FdiskOutput $out
					if ( ($disksBeforeRemovingNewDisk - $TotalLuns) -eq $disksafterRemovingNewDisk )
					{
						LogMsg "All Disks removed."
						$retValue = "PASS"

					}
					else
					{
						$UnexpectedDisks = ( $disksafterRemovingNewDisk - ($disksBeforeRemovingNewDisk - $TotalLuns) )
						LogErr "Total unexpected disks : $UnexpectedDisks"
						WaitFor -seconds 10
						$retValue = "FAIL"
						$retryCount += 1
					}

				}
				Add-Content  -Value "After removing Disks : " -Path $HotRemoveLogFile -Encoding UTF8
				Add-Content  -Value $out -Path $HotRemoveLogFile -Encoding UTF8
			}
			else
			{
				LogMsg "VM Status : OFF."
				LogErr "VM is not Alive after adding new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Error in  removing disks."
			LogErr "Aborting Test."
			$retValue = "Aborted"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------REMOVE $testLun DISKS : END : $retValue----------------------" -Path $HotRemoveLogFile 
	return $retValue
}
Function DoHotAddExistingDataDiskTest($testVMObject)
{
	$retValue = "ABORTED"
	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$testExistingDisk = $testVMObject.ExistingDiskMediaLink
	$HotAddLogFile = "$($testVMObject.logDir)\Hot-Add-Existing-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------ADD EXISTING DISK TO LUN $testLun : START----------------------" -Path $HotAddLogFile -Encoding UTF8
#GetCurrentDiskInfo
		$fdiskOutputBeforeAddingDisk =  RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		Add-Content  -Value "Before Adding Existing Disk : " -Path $HotAddLogFile -Encoding UTF8
		Add-Content  -Value $fdiskOutputBeforeAddingDisk -Path $HotAddLogFile -Encoding UTF8
		$disksBeforeAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $fdiskOutputBeforeAddingDisk
#Add datadisk to VM
		$addExistingDisk = RetryOperation -operation { Get-AzureVM -ServiceName $testVMServiceName | Add-AzureDataDisk  -ImportFrom -MediaLocation $testExistingDisk -DiskLabel "TestDisk-$testLun" -LUN $testLun | Update-AzureVM  } -maxRetryCount 15 -retryInterval 10 -Description "Attaching $testExistingDisk disk to LUN : $testLun."
		if ( ( $addExistingDisk.OperationDescription -eq "Update-AzureVM"  ) -and ( $addExistingDisk.OperationStatus -eq "Succeeded" ))
		{
			LogMsg "Disk Attached Successfully.."
			WaitFor -seconds 10
			LogMsg "Checking VM status.."
			$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$newDiskAdded = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($newDiskAdded -eq "FAIL"))
				{
					$fdiskOutputAfterAddingDisk = ""
					LogMsg "Attempt : $retryCount : Checking for new disk."
					$fdiskOutputAfterAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $fdiskOutputAfterAddingDisk
					if ( ($disksBeforeAddingNewDisk + 1) -eq $disksafterAddingNewDisk )
					{
						LogMsg "Existing Disk detected."
						$newDiskAdded = "PASS"
						$newDisknames = GetNewPhysicalDiskNames -FdiskOutputBeforeAddingDisk $fdiskOutputBeforeAddingDisk -FdiskOutputAfterAddingDisk $fdiskOutputAfterAddingDisk
						if($detectedDistro -imatch "SLES" )
						{
							$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "ext3"
						}
						else
						{
							$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "ext4"
						}
						$xfsResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisknames -diskFileSystem "xfs"
						if ( ($extResult -eq "PASS") -and ($xfsResult -eq "PASS"))
						{
							$retValue = "PASS"
						}
						else
						{
							$retValue = "FAIL"
						}
					}
					else
					{
						LogErr "Existing disk not detected."
						WaitFor -seconds 10
						$newDiskAdded = "FAIL"
						$retryCount += 1
					}

				}
				Add-Content  -Value "After Adding Existing Disk : " -Path $HotAddLogFile -Encoding UTF8
				Add-Content  -Value $fdiskOutputAfterAddingDisk -Path $HotAddLogFile -Encoding UTF8
			}
			else
			{
				LogMsg "VM Status : OFF."
				LogErr "VM is not Alive after adding new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Error while attaching disk."
			LogErr "Aborting Test."
			$retValue = "Aborted"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------ADD EXISTING DISK TO LUN $testLun : END : $retValue----------------------" -Path $HotAddLogFile 
	return $retValue
}
Function DoHotAddExistingDataDiskTestParallel ($testVMObject, $TotalLuns)
{

	$testVMSSHport = $testVMObject.sshPort
	$testVMVIP = $testVMObject.ip
	$testVMServiceName = $testVMObject.ServiceName
	$testVMUsername = $testVMObject.user 
	$testVMPassword = $testVMObject.password
	$testLun = $testVMObject.Lun
	$existingDisks = $testVMObject.AllExistingDisks
	$retValue = "ABORTED"
	$HotAddLogFile = "$($testVMObject.logDir)\Hot-Add-Disk.log"
#$HotAddLogFile = ".\temp\Hot-Add-Disk.log"
	$isVMAlive = Test-TCP -testIP $testVMVIP -testport $testVMSSHport

	if ($isVMAlive -eq "True")
	{
		Add-Content  -Value "--------------------ADD EXISTING $TotalLuns DISKS : START----------------------" -Path $HotAddLogFile -Encoding UTF8
#GetCurrentDiskInfo

		$FdiskOutputBeforeAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo
		Add-Content  -Value "Before Adding Disks : " -Path $HotAddLogFile -Encoding UTF8
		Add-Content  -Value $FdiskOutputBeforeAddingDisk -Path $HotAddLogFile -Encoding UTF8
		$disksBeforeAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $FdiskOutputBeforeAddingDisk

#Add datadisk to VM
		$lunCounter = 0
		$HotAddCommand = "Get-AzureVM -ServiceName $testVMServiceName"
		while ($lunCounter -lt $TotalLuns)
		{
			$HotAddCommand += " | Add-AzureDataDisk -ImportFrom -MediaLocation $($existingDisks[$lunCounter]) -DiskLabel TestDisk-$lunCounter -LUN $lunCounter "
			$lunCounter += 1
		}
		$HotAddCommand += " | Update-AzureVM"
		LogMsg "$HotAddCommand"
		$AttachDiskOut = RetryOperation -operation {Invoke-Expression $HotAddCommand } -maxRetryCount 10 -retryInterval 10 -description "Attaching $TotalLuns disks parallely."
		if ($AttachDiskOut.OperationStatus -eq "Succeeded")
		{
			LogMsg "$TotalLuns Disks Attached Successfully.."
			WaitFor -seconds 10
			$isVMAlive = RetryOperation -operation {Test-TCP -testIP $testVMVIP -testport $testVMSSHport} -description "Checking VM status.." -expectResult "True"
			if ($isVMAlive -eq "True")
			{
				LogMsg "VM Status : RUNNING."
				$retryCount = 1
				$MaxRetryCount = 20
				$isAllDiskDetected = "FAIL"
				While (($retryCount -le $MaxRetryCount) -and ($isAllDiskDetected -eq "FAIL"))
				{
					$FdiskOutputAfterAddingDisk = ""
					LogMsg "Attempt : $retryCount : Checking for existing disk."
					$FdiskOutputAfterAddingDisk = RunLinuxCmd -username $testVMUsername -password $testVMpassword -ip $testVMVIP -port $testVMSSHport -command "$fdisk -l" -runAsSudo -ignoreLinuxExitCode
					$disksafterAddingNewDisk = GetTotalPhysicalDisks -FdiskOutput $FdiskOutputAfterAddingDisk
					if ( ($disksBeforeAddingNewDisk + $TotalLuns) -eq $disksafterAddingNewDisk )
					{
						LogMsg "All $TotalLuns existing Disks detected."
						$isAllDiskDetected = "PASS"
						$newDisks = GetNewPhysicalDiskNames -FdiskOutputBeforeAddingDisk $FdiskOutputBeforeAddingDisk -FdiskOutputAfterAddingDisk $FdiskOutputAfterAddingDisk
						$isAllDiskDetected = "PASS"
						$successCount = 0
						$errorCount = 0
						foreach ( $newDisk in $newDisks.split("^"))
						{
							$extResult = $null
							$xfsResult = $null
							if($detectedDistro -imatch "SLES" )
							{
								$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "ext3"
							}
							else
							{
								$extResult =  PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "ext4"
							}
							$xfsResult = PerformIOTestOnDisk -testVMObject $testVMObject -attachedDisk $newDisk -diskFileSystem "xfs"
							if (($extResult -eq "PASS") -and ($xfsResult -eq "PASS"))
							{
								$successCount += 1
							}
							else
							{
								LogErr "IO Operations failed for $newDisk"
								$errorCount += 1
							}
						}
						if ($successCount -eq $TotalLuns)
						{
							$retValue = "PASS"
						}
						else
						{
							$retValue = "FAIL"
							LogErr "I/O operations on $errorCount disks."
						}
					}
					else
					{
						$NotDetectedDisks = ( ($disksBeforeAddingNewDisk + $TotalLuns) - $disksafterAddingNewDisk )
						LogErr "Total undetected disks : $NotDetectedDisks"
						WaitFor -seconds 10
						$isAllDiskDetected = "FAIL"
						$retryCount += 1
					}

				}
				Add-Content  -Value "After Adding New Disk : " -Path $HotAddLogFile -Encoding UTF8
				Add-Content  -Value $FdiskOutputAfterAddingDisk -Path $HotAddLogFile -Encoding UTF8
			}
			else
			{
				LogMsg "VM Status : OFF."
				LogErr "VM is not Alive after adding new disk."
				$retValue = "FAIL"
			}
		}
		else
		{
			LogErr "Error in Attaching disks.."
			LogErr "Aborting Test."
			$retValue = "Aborted"
		}
	}
	else
	{
		LogErr "VM is not Alive."
		LogErr "Aborting Test."
		$retValue = "Aborted"
	}
	Add-Content  -Value "--------------------ADD EXISTING $TotalLuns DISKS : END : $retValue----------------------" -Path $HotAddLogFile 
	return $retValue
}
Function CleanUpExistingDiskReferences($ExistingDiskMediaLinks)
{
	$existingDisks = $ExistingDiskMediaLinks
	$RetryCount2 = 1
	$falseDetections = 1
	$RetryCount3 = 0
	#Get information about all disks..
	do
	{
		$RetryCount3 += 1
		LogMsg "ATTEMPT : $RetryCount3 : Checking if Existing Disks are attached to any VM or not.."
		while(($RetryCount2 -le 20) -and ($falseDetections -gt 0))
		{
			$disksToBreakRefrences = @()
			$UnableToDetachDisks =  @()
			$UnableToBreakDisks = @()
			$BreakReferenceFailCounter = 0
			$DetachFailCounter = 0
			$totalAttachedDisks = 0
			$diskReferenceCounter = 0
			$totalOrphanedDisks = 0
			$falseAlarm = $true
			$falseDetections = 0
			WaitFor -seconds 15
			$allDiskReferences = Get-AzureDisk
			$RetryCount2 += 1
			foreach ($diskReference in $allDiskReferences)
			{
				if($existingDisks -match $diskReference.MediaLink.AbsoluteUri)
				{
					$diskReferenceCounter += 1
					if ($diskReference.AttachedTo -ne $null)
					{
						LogMsg "$(($diskReference.MediaLink.AbsoluteUri).ToUpper()) is in use by $(($diskReference.AttachedTo.HostedServiceName).ToUpper())"
						$targetVM = Get-AzureVM -ServiceName $diskReference.AttachedTo.HostedServiceName -Name $diskReference.AttachedTo.RoleName 
						$attachedDisks = $targetVM | Get-AzureDataDisk
						foreach ($disk in $attachedDisks)
						{
							if($diskReference.MediaLink.AbsoluteUri -imatch $disk.MediaLink.AbsoluteUri)
							{
								$diskToRemove = $disk
							}
						}
						if ($diskToRemove -ne $null)
						{
							$totalAttachedDisks += 1
							$falseAlarm = $false
							LogMsg "Disk is attached to $(($targetVM.Name).ToUpper()) at LUN : $($diskToRemove.Lun)"
							$outDetachDisk = RetryOperation -operation { Remove-AzureDataDisk -VM $targetVM -LUN $diskToRemove.Lun -Verbose | Update-AzureVM -Verbose } -description "Operation : Remove Disk : STARTED" -maxRetryCount 10 -retryInterval 5
							if ( $outDetachDisk.OperationStatus -eq "Succeeded" )
							{
							
								LogMsg "Operation : Remove Disk : FINISHED"
								$disksToBreakRefrences += $diskToRemove.DiskName
							}
							else
							{
								LogMsg "Operation : Remove Disk : FAIED"
								$DetachFailCounter += 1
								$UnableToDetachDisks += $diskToRemove.MediaLink.AbsoluteUri
							}
						}
						else
						{
							$falseAlarm = $true
						}
						if($falseAlarm)
						{
							LogMsg "False Detection. Disk is not in use by any VM in service $(($diskReference.AttachedTo.HostedServiceName).ToUpper())."
							$falseDetections += 1
					
						}
					}
					if ($diskReference.AttachedTo -eq $null)
					{
						$falseAlarm = $false
						$totalOrphanedDisks += 1
						LogMsg "$(($diskReference.MediaLink.AbsoluteUri).ToUpper()) is referenced as $(($diskReference.DiskName).ToUpper()) but not in use."
						$disksToBreakRefrences += $diskReference.DiskName
					}
				}
			}	
		}
		if($diskReferenceCounter -gt 0)
		{
			WaitFor -seconds 30
			foreach ($diskToBreakReference in $disksToBreakRefrences)
			{
				$RetryCount = 0
				do
				{
					$RetryCount += 1
					LogMsg "Operation : Break Reference of Disk : $(($diskToBreakReference).ToUpper()) : STARTED"
					try
					{
						$outBreak = Remove-AzureDisk -DiskName $diskToBreakReference
						$exceptionGenerated = $false
					}
					catch
					{
						LogErr "Operation : Break Reference : FAILED"
						$exceptionGenerated = $true
					}
				}
				while ((($outBreak.OperationStatus -ne "Succeeded") -or $exceptionGenerated) -and ($RetryCount -lt 10) )
			
				if($outBreak.OperationStatus -eq "Succeeded") 
				{
					LogMsg "Operation : Break Reference : FINISHED"
				}
				else
				{
					LogErr "Operation : Break Reference : FAILED"
					$BreakReferenceFailCounter += 1
					$UnableToBreakDisks += $diskToBreakReference
				}
			}
			if (($BreakReferenceFailCounter -eq 0) -and ($DetachFailCounter -eq 0))
			{
				LogMsg "All existing disks are free to use now."
				$retValue = $true
			}
			else
			{
				$retValue = $false
				foreach ($detachDisk in $UnableToDetachDisks)
				{
					LogErr "Unable to detach : $(($detachDisk).ToUpper())"
				}
				foreach ($breakDisk in $UnableToBreakDisks)
				{
					LogErr "Unable to Break Reference : $(($breakDisk).ToUpper())"
				}
				WaitFor -seconds 15
			}

		}
		else
		{
			LogMsg "All Existing Disks are already free to use."
			$retValue = $true
		}
	}
	while(($retValue -eq $false) -and ($RetryCount3 -lt 3))
	return $retValue
}


<#
.SYNOPSIS 
Retry to do the operation until it is executed successfully or has reached the maximum number of retry attempts. It returns the result of the operation if success. Otherwise, it returns null.
.PARAMETER operation
Specifies the operation which you want to retry. It is a script block. The format shoulde be {OPERATION}.
.PARAMETER description
Specifies the description of the operation.
.PARAMETER expectResult
Specifies the expect result. The default value is "$null" and it means not to check the result.
.PARAMETER maxRetryCount
Specifies the maximum retry count. The default value is 18.
.PARAMETER retryInterval
Specifies the retry interval. The default value is 10 seconds.
#>
Function RetryOperation($operation, $description, $expectResult=$null, $maxRetryCount=10, $retryInterval=10, [switch]$NoLogsPlease)
{
	$retryCount = 1
	
	do
	{
		LogMsg "Attempt : $retryCount/$maxRetryCount : $description" -NoLogsPlease $NoLogsPlease
		$ret = $null
		$oldErrorActionValue = $ErrorActionPreference
		$ErrorActionPreference = "Stop"
		
		try
		{
			$ret = Invoke-Command -ScriptBlock $operation
			if ($expectResult -ne $null)
			{
				if ($ret -match $expectResult)
				{
					return $ret
				}
				else
				{
					$ErrorActionPreference = $oldErrorActionValue
				}
			}
			else
			{
				return $ret
			}
		}
		catch
		{
			$retryCount ++
			WaitFor -seconds $retryInterval
			if ( $retryCount -le $maxRetryCount )
			{
				continue
			}
		}
		finally
		{
			$ErrorActionPreference = $oldErrorActionValue
		}
		if ($retryCount -ge $maxRetryCount)
		{
			LogErr "Opearation Failed." 
			break;
		}
	} while ($True)
	
	return $null
}#endregion