Added ZooKeeper performance test

Current Test Bed Configuration -
2 servers and 1 client.
(can be changed "n" servers and 1 client via XML file)

Azure_ICA_all.xml

@@ -2819,6 +2819,50 @@
 					</VirtualMachine>					
 				</HostedService>
 			</M1S5>	
+			<S2C1>
+				<isDeployed>NO</isDeployed>
+				<HostedService>
+					<VirtualMachine>
+						<state></state>
+						<InstanceSize>Standard_G5</InstanceSize>
+						<ARMInstanceSize>Standard_G5</ARMInstanceSize>
+						<RoleName>Server1</RoleName>
+						<EndPoints>
+							<Name>SSH</Name>
+							<Protocol>tcp</Protocol>
+							<LocalPort>22</LocalPort>
+							<PublicPort>12122</PublicPort>
+						</EndPoints>
+						<DataDisk></DataDisk>
+					</VirtualMachine>
+					<VirtualMachine>
+						<state></state>
+						<InstanceSize>Standard_G5</InstanceSize>
+						<ARMInstanceSize>Standard_G5</ARMInstanceSize>
+						<RoleName>Server2</RoleName>
+						<EndPoints>
+							<Name>SSH</Name>
+							<Protocol>tcp</Protocol>
+							<LocalPort>22</LocalPort>
+							<PublicPort>12222</PublicPort>
+						</EndPoints>
+						<DataDisk></DataDisk>
+					</VirtualMachine>					
+					<VirtualMachine>
+						<state></state>
+						<InstanceSize>Standard_G5</InstanceSize>
+						<ARMInstanceSize>Standard_G5</ARMInstanceSize>
+						<RoleName>Client</RoleName>
+						<EndPoints>
+							<Name>SSH</Name>
+							<Protocol>tcp</Protocol>
+							<LocalPort>22</LocalPort>
+							<PublicPort>12322</PublicPort>
+						</EndPoints>
+						<DataDisk></DataDisk>
+					</VirtualMachine>					
+				</HostedService>
+			</S2C1>			
 		</Deployment>
 		
 
@@ -4815,6 +4859,28 @@
 		  <TestID>AzureRHUIStress_001</TestID>
 		</test>
 
+			<test>
+				<testName>ICA-PERF-ZOOKEEPER-TEST</testName>
+				<testScript></testScript>
+				<testScriptps1>ICA-PERF-ZOOKEEPER-TEST.ps1</testScriptps1>
+				<setupType>S2C1</setupType>
+				<SupportedExecutionModes>AzureServiceManagement,AzureResourceManager</SupportedExecutionModes>
+				<TestParameters>
+					<param>ZK_VERSION=3.4.6</param>
+					<param>ZK_TIMEOUT=1000000</param>
+					<param>ZK_ZNODE_SIZE=100</param>
+					<param>ZK_ZNODE_COUNT=100</param>
+					<param>ZK_FORCE=true</param>
+					<param>ZK_SYNCHRONOUS=true</param>
+					<param>ZK_VERBOSE=false</param>				
+				</TestParameters>
+				<!--Remote Files will be directly downloaded to test VM.-->
+				<remoteFiles>https://raw.githubusercontent.com/phunt/zk-smoketest/master/zkclient.py,https://raw.githubusercontent.com/phunt/zk-smoketest/master/zk-latencies.py,https://github.com/phunt/zk-smoketest/raw/master/lib.linux-x86_64-2.6/libzookeeper_mt.so.2,https://github.com/phunt/zk-smoketest/raw/master/lib.linux-x86_64-2.6/zookeeper.so</remoteFiles>
+				<!--Remote XML will be downloaded to automation server for analysis-->
+				<remoteXML></remoteXML>
+				<Priority>P0</Priority>
+			</test>			
+			
 		</testsDefinition>
 	<testCycles>
 		<!-- Setup related tests -->
@@ -5531,6 +5597,12 @@
 				<Name>ICA-PERF-APACHE-BENCHMARK-TEST</Name>
 			</test>
 		</Cycle>		
+		<Cycle>
+			<cycleName>LISA-ZK-PERF</cycleName>
+			<test>
+				<Name>ICA-PERF-ZOOKEEPER-TEST</Name>
+			</test>
+		</Cycle>		
 		
 		<Cycle>
 			<cycleName>RHUI-Stress</cycleName>

TestLibs/LISALibrary.psm1

@@ -40,7 +40,7 @@
 		while ( (Get-Job -Id $provisionJob).State -eq "Running" )
 		{
 			$currentStatus = RunLinuxCmd -ip $vmData.PublicIP -port $vmData.SSHPort -username "root" -password $password -command "tail -n 1 /root/provisionLinux.log"
-			LogMsg "Current Staus : $currentStatus"
+			LogMsg "Current Package Installation Status : $currentStatus"
 			WaitFor -seconds 10
 		}
 		RemoteCopy -download -downloadFrom $vmData.PublicIP -port $vmData.SSHPort -files "/root/provisionLinux.log" -username "root" -password $password -downloadTo $LogDir

remote-scripts/ICA-PERF-ZOOKEEPER-TEST.ps1

@@ -0,0 +1,217 @@
+<#-------------Create Deployment Start------------------#>
+Import-Module .\TestLibs\RDFELibs.psm1 -Force
+$result = ""
+$testResult = ""
+$resultArr = @()
+$isDeployed = DeployVMS -setupType $currentTestData.setupType -Distro $Distro -xmlConfig $xmlConfig
+if ($isDeployed)
+{
+	try
+	{
+		$serverMachines = @()
+		$serverMachinesHostNames = ""
+		$noClient = $true
+		$noServer = $true
+		foreach ( $vmData in $allVMData )
+		{
+			if ( $vmData.RoleName -imatch "client" )
+			{
+				$clientVMData = $vmData
+				$noClient = $false
+			}
+			elseif ( $vmData.RoleName -imatch "server" )
+			{
+				$serverMachines += $vmData
+				$noServer = $fase
+				if ( $serverMachinesHostNames )
+				{
+					$serverMachinesHostNames += ",$($vmData.RoleName):2181"
+				}
+				else
+				{
+					$serverMachinesHostNames += "$($vmData.RoleName):2181"
+				}
+			}
+		}
+		$serverMachinesHostNames = $serverMachinesHostNames.Trim()
+		if ( $noClient )
+		{
+			Throw "No any master VM defined. Be sure that, Client VM role name matches with the pattern `"*master*`". Aborting Test."
+		}
+		if ( $noServer )
+		{
+			Throw "No any slave VM defined. Be sure that, Server machine role names matches with pattern `"*slave*`" Aborting Test."
+		}
+		#region CONFIGURE VM FOR TERASORT TEST
+		LogMsg "CLIENT VM details :"
+		LogMsg "  RoleName : $($clientVMData.RoleName)"
+		LogMsg "  Public IP : $($clientVMData.PublicIP)"
+		LogMsg "  SSH Port : $($clientVMData.SSHPort)"
+		$i = 1
+		foreach ( $vmData in $serverMachines )
+		{
+			LogMsg "SERVER #$i VM details :"
+			LogMsg "  RoleName : $($vmData.RoleName)"
+			LogMsg "  Public IP : $($vmData.PublicIP)"
+			LogMsg "  SSH Port : $($vmData.SSHPort)"
+			$i += 1
+		}
+
+
+		#
+		# PROVISION VMS FOR LISA WILL ENABLE ROOT USER AND WILL MAKE ENABLE PASSWORDLESS AUTHENTICATION ACROSS ALL VMS IN SAME HOSTED SERVICE.	
+		#
+		ProvisionVMsForLisa -allVMData $allVMData
+		
+		foreach ( $vmData in $allVMData )
+		{
+			LogMsg "Adding $($vmData.InternalIP) $($vmData.RoleName) to /etc/hosts of $($clientVMData.RoleName) "
+			$out = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "echo $($vmData.InternalIP) $($vmData.RoleName) >> /etc/hosts"
+		}
+		#endregion
+
+		LogMsg "Generating constansts.sh ..."
+		$constantsFile = "$LogDir\constants.sh"
+		Set-Content -Value "#Generated by Azure Automation." -Path $constantsFile
+		foreach ($zookeeperParam in $currentTestData.TestParameters.param )
+		{
+			Add-Content -Value "$zookeeperParam" -Path $constantsFile
+			LogMsg "$zookeeperParam added to constansts.sh"
+		}		
+		Add-Content -Value "ZK_SERVERS=$serverMachinesHostNames" -Path $constantsFile
+		LogMsg "ZK_SERVERS=$serverMachinesHostNames added to constants.sh"
+
+		LogMsg "constanst.sh created successfully..."
+		#endregion
+
+		#region Download remote files needed to run tests
+		LogMsg "Downloading remote files ..."
+
+		$fileToUpload = ""
+		foreach ( $fileURL in  $($currentTestData.remoteFiles).Split(",") )
+		{
+			LogMsg "Downloading $fileURL ..."
+			$start_time = Get-Date
+			$fileName =  $fileURL.Split("/")[$fileURL.Split("/").Count-1]
+			$out = Invoke-WebRequest -Uri $fileURL -OutFile "$LogDir\$fileName"
+			LogMsg "Time taken: $((Get-Date).Subtract($start_time).Seconds) second(s)"
+			if ( $fileToUpload )
+			{
+				$fileToUpload += ",.\$LogDir\$fileName"
+			}
+			else
+			{
+				$fileToUpload = ".\$LogDir\$fileName"
+			}
+		}
+		#endregion
+		
+		#region EXECUTE TEST
+		Set-Content -Value "/root/performance_zk.sh &> zkConsoleLogs.txt" -Path "$LogDir\StartZookeperTest.sh"
+		RemoteCopy -uploadTo $clientVMData.PublicIP -port $clientVMData.SSHPort -files "$fileToUpload,.\$constantsFile,.\remote-scripts\performance_zk.sh,.\$LogDir\StartZookeperTest.sh" -username "root" -password $password -upload
+		Remove-Item -Path "$LogDir\zookeeper.so" -Force 
+		LogMsg "Removed zookeeper.so from $LogDir directory..."
+		Remove-Item -Path "$LogDir\libzookeeper_mt.so.2" -Force 
+		LogMsg "Removed libzookeeper_mt.so.2 from $LogDir directory..."
+		Remove-Item -Path "$LogDir\zkclient.py" -Force 
+		LogMsg "Removed zkclient.py from $LogDir directory..."
+		Remove-Item -Path "$LogDir\zk-latencies.py" -Force 
+		LogMsg "Removed zk-latencies.py from $LogDir directory..."
+		$out = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "chmod +x *.sh"
+		$out = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "mkdir -p lib.linux-x86_64-2.6 && mv zookeeper.so lib.linux-x86_64-2.6/zookeeper.so && mv libzookeeper_mt.so.2 lib.linux-x86_64-2.6/libzookeeper_mt.so.2"
+		$testJob = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "/root/StartZookeperTest.sh" -RunInBackground
+		#endregion
+
+		#region MONITOR TEST
+		while ( (Get-Job -Id $testJob).State -eq "Running" )
+		{
+			$currentStatus = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "tail -n 1 /root/zkConsoleLogs.txt"
+			LogMsg "Current Test Staus : $currentStatus"
+			WaitFor -seconds 10
+		}
+		
+		$finalStatus = RunLinuxCmd -ip $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -command "cat /root/state.txt"
+		
+		RemoteCopy -downloadFrom $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/zkConsoleLogs.txt"
+		RemoteCopy -downloadFrom $clientVMData.PublicIP -port $clientVMData.SSHPort -username "root" -password $password -download -downloadTo $LogDir -files "/root/summary.log"
+		$zookeeperSummary = Get-Content -Path "$LogDir\summary.log" -ErrorAction SilentlyContinue
+		$zookeeperConsoleLogs = Get-Content -Path "$LogDir\zkConsoleLogs.txt" -ErrorAction SilentlyContinue
+		LogMsg "************************ZOOKEEPER REPORT************************"
+		foreach ( $line in $zookeeperConsoleLogs.Split("`n") )
+		{
+			if ( $line -imatch "Connected in ")
+			{
+				$printConsole = $true
+			}
+			elseif ( $line -imatch "Latency test complete")
+			{
+				LogMsg $line -LinuxConsoleOuput
+				$printConsole = $false
+			}
+			if ( $printConsole )
+			{
+				LogMsg $line -LinuxConsoleOuput
+			}
+		}
+		LogMsg "************************ZOOKEEPER REPORT************************"
+		#endregion
+
+		if (!$zookeeperSummary)
+		{
+			LogMsg "summary.log file is empty."
+			$zookeeperSummary = "<EMPTY>"
+		}
+		if ( $finalStatus -imatch "TestFailed")
+		{
+			LogErr "Test failed. Last known status : $currentStatus."
+			$testResult = "FAIL"
+		}
+		elseif ( $finalStatus -imatch "TestAborted")
+		{
+			LogErr "Test Aborted. Last known status : $currentStatus."
+			$testResult = "ABORTED"
+		}
+		elseif ( $finalStatus -imatch "TestCompleted")
+		{
+			LogMsg "Test Completed."
+			$testResult = "PASS"
+		}
+		elseif ( $finalStatus -imatch "TestRunning")
+		{
+			LogMsg "Powershell backgroud job for test is completed but VM is reporting that test is still running. Please check $LogDir\zkConsoleLogs.txt"
+			LogMsg "Contests of summary.log : $zookeeperSummary"
+			$testResult = "PASS"
+		}
+		LogMsg "Test result : $testResult"
+		LogMsg "Test Completed"
+	}
+	catch
+	{
+		$ErrorMessage =  $_.Exception.Message
+		LogMsg "EXCEPTION : $ErrorMessage"   
+	}
+	Finally
+	{
+		$metaData = "ZooKeeper RESULT"
+		if (!$testResult)
+		{
+			$testResult = "Aborted"
+		}
+		$resultArr += $testResult
+		$resultSummary +=  CreateResultSummary -testResult $zookeeperResult -metaData $metaData -checkValues "PASS,FAIL,ABORTED" -testName $currentTestData.testName# if you want to publish all result then give here all test status possibilites. if you want just failed results, then give here just "FAIL". You can use any combination of PASS FAIL ABORTED and corresponding test results will be published!
+	}   
+}
+
+else
+{
+	$testResult = "Aborted"
+	$resultArr += $testResult
+}
+
+$result = GetFinalResultHeader -resultarr $resultArr
+
+#Clean up the setup
+DoTestCleanUp -result $result -testName $currentTestData.testName -deployedServices $isDeployed -ResourceGroups $isDeployed
+
+#Return the result and summery to the test suite script..
+return $result

remote-scripts/performance_zk.sh

@@ -0,0 +1,511 @@
+#!/bin/bash
+
+########################################################################
+#
+# Linux on Hyper-V and Azure Test Code, ver. 1.0.0
+# Copyright (c) Microsoft Corporation
+#
+# All rights reserved. 
+# Licensed under the Apache License, Version 2.0 (the ""License"");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0  
+#
+# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION
+# ANY IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR
+# PURPOSE, MERCHANTABLITY OR NON-INFRINGEMENT.
+#
+# See the Apache Version 2.0 License for specific language governing
+# permissions and limitations under the License.
+#
+########################################################################
+
+#######################################################################
+#
+# performance_zk.sh
+#
+# Description:
+#     This tool uses the ZooKeeper (zk) python binding to test various operation latencies
+#
+#
+# In general the script does the following:
+#
+# 1.create a root znode for the test, i.e. /zk-latencies
+# 2.attach a zk session to each server in the ensemble (the --servers list)
+# 3.run various (create/get/set/delete) operations against each server, note the latencies of operations
+# 4.client then cleans up, removing /zk-latencies znode
+#
+#
+# Parameters:
+#	ZK_VERSION			zookeeper-server version
+#      	ZK_SERVERS:               	comma separated list of host:port (default localhost:2181)
+#	ZK_TIMEOUT;			session timeout in milliseconds (default 5000)
+#	ZK_ZNODE_SIZE;			data size when creating/setting znodes (default 25)
+#	ZK_ZNODE_COUNT;			the number of znodes to operate on in each performance section (default 10000)
+#	ZK_FORCE; (optional)		force the test to run, even if root_znode exists -WARNING! don't run this on a real znode or you'll lose it !!!
+#	ZK_SYNCHRONOUS;			by default asynchronous ZK api is used, this forces synchronous calls
+#	ZK_VERBOSE;			verbose output, include more detail
+#
+#######################################################################
+
+
+
+ICA_TESTRUNNING="TestRunning"
+ICA_TESTCOMPLETED="TestCompleted"
+ICA_TESTABORTED="TestAborted"
+ICA_TESTFAILED="TestFailed"
+
+
+#
+# Function definitions
+#
+
+LogMsg()
+{
+    echo `date "+%a %b %d %T %Y"` ": ${1}"
+}
+
+UpdateTestState()
+{
+    echo $1 > ~/state.txt
+}
+
+#######################################################################
+#
+# LinuxRelease()
+#
+#######################################################################
+LinuxRelease()
+{
+    DISTRO=`grep -ihs "buntu\|Suse\|Fedora\|Debian\|CentOS\|Red Hat Enterprise Linux" /etc/{issue,*release,*version}`
+
+    case $DISTRO in
+        *buntu*)
+            echo "UBUNTU";;
+        Fedora*)
+            echo "FEDORA";;
+        CentOS*)
+            echo "CENTOS";;
+        *SUSE*)
+            echo "SLES";;
+        Red*Hat*)
+            echo "RHEL";;
+        Debian*)
+            echo "DEBIAN";;
+    esac
+}
+
+######################################################################
+#
+# DoSlesAB()
+#
+# Description:
+#    Perform distro specific Apache and tool installation steps for SLES
+#    and then run the benchmark tool
+#
+#######################################################################
+ConfigSlesZK()
+{
+    #
+    # Note: A number of steps will use SSH to issue commands to the
+    #       APACHE_SERVER.  This requires that the SSH keys be provisioned
+    #       in advanced, and strict mode be disabled for both the SSH
+    #       server and client.
+    #
+
+	LogMsg "Info: Running SLES"
+	arr=$(echo $ZK_SERVERS | tr "," "\n")
+	for ZK_SERVER in $arr
+	do
+	#echo "${ZK_SERVER}"
+	SERVER=(`echo "${ZK_SERVER}" | awk -F':' '{print $1}'`)
+	#echo "${SERVER}"
+	#ssh root@${SERVER} "mkdir /root/kk"
+	#exit 1
+	LogMsg "Info: -----------------------------------------"
+	LogMsg "Info: Zookeeper-Server installation on server ${SERVER}"
+    	LogMsg "Info: Installing required packages first"
+	ssh root@${SERVER} "zypper --non-interactive install wget"
+	if [ $? -ne 0 ]; then
+		msg="Error: Unable to install package to server ${SERVER}"
+		LogMsg "${msg}"
+		echo "${msg}" >> ./summary.log
+		UpdateTestState $ICA_TESTFAILED
+		exit 1
+	fi
+	
+	#
+    # Install Java
+    #
+    LogMsg "Check if Java is installed"
+
+    javaInstalled=`which java`
+    if [ ! $javaInstalled ]; then
+        LogMsg "Installing Java"
+
+        ssh root@${SERVER} "zypper --non-interactive install jre-1.7.0"
+        if [ $? -ne 0 ]; then
+            LogMsg "Error: Unable to install java"
+            UpdateTestState $ICA_TESTFAILED
+            exit 1
+        fi
+    fi
+        LogMsg "Info: Download ZK package"
+	pkg=(`ssh root@${SERVER} "ls /root/ | grep ${ZK_ARCHIVE}"`)
+	echo $pkg
+	if [ -z "$pkg" ]; then
+		LogMsg "Downloading ZK package ${ZK_ARCHIVE}"
+		#exit 1	
+        	ssh root@${SERVER} "wget ${ZK_URL}"
+        	if [ $? -ne 0 ]; then
+                	msg="Error: Unable to download ZK package to server ${SERVER}"
+                	LogMsg "${msg}"
+                	echo "${msg}" >> ./summary.log
+                	UpdateTestState $ICA_TESTFAILED
+                	exit 1
+		fi	
+        fi
+
+	LogMsg "Info: Untar and Config ZK server"
+        ssh root@${SERVER} "tar -xzf ./${ZK_ARCHIVE}"
+        ssh root@${SERVER} "cp zookeeper-${ZK_VERSION}/conf/zoo_sample.cfg zookeeper-${ZK_VERSION}/conf/zoo.cfg"
+
+	LogMsg "Info: Starting Zookeeper-Server ${SERVER}"
+        ssh root@${SERVER} "zookeeper-${ZK_VERSION}/bin/zkServer.sh start"
+	if [ $? -ne 0 ]; then
+                msg="Error: Unable to start Zookeeper-Server on ${SERVER}"
+                LogMsg "${msg}"
+                echo "${msg}" >> ./summary.log
+                UpdateTestState $ICA_TESTFAILED
+                exit 1
+        fi
+
+	LogMsg "Info: Server started successfully"
+
+	done	
+}
+
+ConfigUbuntuZK()
+{
+    #
+    # Note: A number of steps will use SSH to issue commands to the
+    #       ZK_SERVER.  This requires that the SSH keys be provisioned
+    #       in advanced, and strict mode be disabled for both the SSH
+    #       server and client.
+    #
+
+	LogMsg "Info: Running Ubuntu"
+	arr=$(echo $ZK_SERVERS | tr "," "\n")
+	for ZK_SERVER in $arr
+	do	
+	SERVER=(`echo "${ZK_SERVER}" | awk -F':' '{print $1}'`)
+	LogMsg "Info: -----------------------------------------"
+	LogMsg "Info: Zookeeper-Server installation on server ${SERVER}"
+    	LogMsg "Info: Installing required packages first"
+	ssh root@${SERVER} "apt-get install -y wget"
+	if [ $? -ne 0 ]; then
+		msg="Error: Unable to install package to server ${SERVER}"
+		LogMsg "${msg}"
+		echo "${msg}" >> ./summary.log
+		UpdateTestState $ICA_TESTFAILED
+		exit 1
+	fi
+	
+	#
+    # Install Java
+    #
+    LogMsg "Check if Java is installed"
+
+    javaInstalled=`which java`
+    if [ ! $javaInstalled ]; then
+        LogMsg "Installing Java"
+
+        ssh root@${SERVER} "apt-get -y install default-jdk"
+        if [ $? -ne 0 ]; then
+            LogMsg "Error: Unable to install java"
+            UpdateTestState $ICA_TESTFAILED
+            exit 1
+        fi
+    fi
+
+        LogMsg "Info: Download ZK package"
+	pkg=(`ssh root@${SERVER} "ls /root/ | grep ${ZK_ARCHIVE}"`)
+	echo $pkg
+	if [ -z "$pkg" ]; then
+		LogMsg "Downloading ZK package ${ZK_ARCHIVE}"
+		#exit 1	
+        	ssh root@${SERVER} "wget ${ZK_URL}"
+        	if [ $? -ne 0 ]; then
+                	msg="Error: Unable to download ZK package to server ${SERVER}"
+                	LogMsg "${msg}"
+                	echo "${msg}" >> ./summary.log
+                	UpdateTestState $ICA_TESTFAILED
+                	exit 1
+		fi	
+        fi
+
+	LogMsg "Info: Untar and Config ZK server"
+        ssh root@${SERVER} "tar -xzf ./${ZK_ARCHIVE}"
+        ssh root@${SERVER} "cp zookeeper-${ZK_VERSION}/conf/zoo_sample.cfg zookeeper-${ZK_VERSION}/conf/zoo.cfg"
+
+	LogMsg "Info: Starting Zookeeper-Server ${SERVER}"
+        ssh root@${SERVER} "zookeeper-${ZK_VERSION}/bin/zkServer.sh start"
+	if [ $? -ne 0 ]; then
+                msg="Error: Unable to start Zookeeper-Server on ${SERVER}"
+                LogMsg "${msg}"
+                echo "${msg}" >> ./summary.log
+                UpdateTestState $ICA_TESTFAILED
+                exit 1
+        fi
+
+	LogMsg "Info: Server started successfully"
+
+	done	
+}
+
+ConfigRHELZK()
+{
+    #
+    # Note: A number of steps will use SSH to issue commands to the
+    #       ZK_SERVER.  This requires that the SSH keys be provisioned
+    #       in advanced, and strict mode be disabled for both the SSH
+    #       server and client.
+    #
+
+	LogMsg "Info: Running RHEL"
+	arr=$(echo $ZK_SERVERS | tr "," "\n")
+	for ZK_SERVER in $arr
+	do	
+	SERVER=(`echo "${ZK_SERVER}" | awk -F':' '{print $1}'`)
+	LogMsg "Info: -----------------------------------------"
+	LogMsg "Info: Zookeeper-Server installation on server ${SERVER}"
+    	LogMsg "Info: Installing required packages first"
+	ssh root@${SERVER} "yum install -y wget"
+	if [ $? -ne 0 ]; then
+		msg="Error: Unable to install package to server ${SERVER}"
+		LogMsg "${msg}"
+		echo "${msg}" >> ./summary.log
+		UpdateTestState $ICA_TESTFAILED
+		exit 1
+	fi
+	
+	#
+    # Install Java
+    #
+    LogMsg "Check if Java is installed"
+
+    javaInstalled=`which java`
+    if [ ! $javaInstalled ]; then
+        LogMsg "Installing Java"
+
+        ssh root@${SERVER} "yum -y install java-1.7.0-openjdk"
+        if [ $? -ne 0 ]; then
+            LogMsg "Error: Unable to install Java"
+            UpdateTestState $ICA_TESTFAILED
+            exit 1
+        fi
+    fi
+	
+        LogMsg "Info: Download ZK package"
+	pkg=(`ssh root@${SERVER} "ls /root/ | grep ${ZK_ARCHIVE}"`)
+	echo $pkg
+	if [ -z "$pkg" ]; then
+		LogMsg "Downloading ZK package ${ZK_ARCHIVE}"
+		#exit 1	
+        	ssh root@${SERVER} "wget ${ZK_URL}"
+        	if [ $? -ne 0 ]; then
+                	msg="Error: Unable to download ZK package to server ${SERVER}"
+                	LogMsg "${msg}"
+                	echo "${msg}" >> ./summary.log
+                	UpdateTestState $ICA_TESTFAILED
+                	exit 1
+		fi	
+        fi
+
+	LogMsg "Info: Untar and Config ZK server"
+        ssh root@${SERVER} "tar -xzf ./${ZK_ARCHIVE}"
+        ssh root@${SERVER} "cp zookeeper-${ZK_VERSION}/conf/zoo_sample.cfg zookeeper-${ZK_VERSION}/conf/zoo.cfg"
+
+	LogMsg "Info: Starting Zookeeper-Server ${SERVER}"
+        ssh root@${SERVER} "zookeeper-${ZK_VERSION}/bin/zkServer.sh start"
+	if [ $? -ne 0 ]; then
+                msg="Error: Unable to start Zookeeper-Server on ${SERVER}"
+                LogMsg "${msg}"
+                echo "${msg}" >> ./summary.log
+                UpdateTestState $ICA_TESTFAILED
+                exit 1
+        fi
+
+	LogMsg "Info: Server started successfully"
+
+	done	
+}
+
+#######################################################################
+#
+# Main script body
+#
+#######################################################################
+
+cd ~
+UpdateTestState $ICA_TESTRUNNING
+LogMsg "Starting test"
+
+#
+# Delete any old summary.log file
+#
+LogMsg "Cleaning up old summary.log"
+if [ -e ~/summary.log ]; then
+    rm -f ~/summary.log
+fi
+
+touch ~/summary.log
+
+#
+# Source the constants.sh file
+#
+LogMsg "Sourcing constants.sh"
+if [ -e ~/constants.sh ]; then
+    . ~/constants.sh
+else
+    msg="Error: ~/constants.sh does not exist"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    UpdateTestState $ICA_TESTABORTED
+    exit 10
+fi
+
+#
+# Make sure the required test parameters are defined
+#
+if [ "${ZK_VERSION:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_VERSION test parameter is missing"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    UpdateTestState $ICA_TESTFAILED
+    exit 20
+fi
+
+if [ "${ZK_SERVERS:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_SERVER test parameter is missing"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    UpdateTestState $ICA_TESTFAILED
+    exit 20
+fi
+
+if [ "${ZK_TIMEOUT:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_TIMEOUT test parameter is missing"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    ZK_TIMEOUT=100000
+fi
+
+if [ "${ZK_ZNODE_SIZE:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_ZNODE_SIZE test parameter is missing"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    ZK_ZNODE_SIZE=100
+fi
+
+if [ "${ZK_ZNODE_COUNT:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_ZNODE_COUNT test parameter is missing"
+    LogMsg "${msg}"
+    echo "${msg}" >> ~/summary.log
+    ZK_ZNODE_SIZE=100
+fi
+
+if [ "${ZK_FORCE:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_FORCE test parameter is missing"
+    LogMsg "${msg}"
+    ZK_FORCE=$false
+fi
+
+if [ "${ZK_VERBOSE:="UNDEFINED"}" = "UNDEFINED" ]; then
+    msg="Error: the ZK_VERBOSE test parameter is missing"
+    LogMsg "${msg}"
+    ZK_VERBOSE=$false
+fi
+
+LogMsg "Info: Test run parameters"
+echo "ZK_VERSION               = ${ZK_VERSION}"
+echo "ZK_SERVERS               = ${ZK_SERVERS}"
+echo "ZK_TIMEOUT    = ${ZK_TIMEOUT}"
+echo "ZK_ZNODE_SIZE = ${ZK_ZNODE_SIZE}"
+echo "ZK_ZNODE_COUNT = ${ZK_ZNODE_COUNT}"
+echo "ZK_FORCE = ${ZK_FORCE}"
+echo "ZK_SYNCHRONOUS = ${ZK_SYNCHRONOUS}"
+echo "ZK_VERBOSE = ${ZK_VERBOSE}"
+
+
+LogMsg "Info : ZK_VERSION = ${ZK_VERSION}"
+ZK_ARCHIVE="zookeeper-${ZK_VERSION}.tar.gz"
+ZK_URL=http://apache.spinellicreations.com/zookeeper/zookeeper-${ZK_VERSION}/${ZK_ARCHIVE}
+
+#
+# Configure ZK server - this has distro specific behaviour
+#
+distro=`LinuxRelease`
+case $distro in
+    "CENTOS" | "RHEL")
+        ConfigRHELZK
+    ;;
+    "UBUNTU")
+        ConfigUbuntuZK
+    ;;
+    "DEBIAN")
+        ConfigDebianZK
+    ;;
+    "SLES")
+        ConfigSlesZK
+    ;;
+     *)
+        msg="Error: Distro '${distro}' not supported"
+        LogMsg "${msg}"
+        echo "${msg}" >> ~/summary.log
+        UpdateTestState "TestAborted"
+        exit 1
+    ;; 
+esac
+
+IFS=',' read -ra ADDR <<< "$ZK_SERVERS"
+for zk_server in "${ADDR[@]}"; 
+do
+	LogMsg "Prepare ZK test cmd for ${zk_server}"
+	testBaseString='PYTHONPATH="lib.linux-x86_64-2.6" LD_LIBRARY_PATH="lib.linux-x86_64-2.6" python ./zk-latencies.py '
+	#testBaseString='./zk-latencies.py PYTHONPATH=lib.linux-x86_64-2.6 LD_LIBRARY_PATH=lib.linux-x86_64-2.6 '
+	testString="${testBaseString}""--servers=${zk_server} --timeout=${ZK_TIMEOUT} --znode_count=${ZK_ZNODE_COUNT} --znode_size=${ZK_ZNODE_SIZE}"
+
+	if [ "${ZK_FORCE}" = true ]; then
+		testString="${testString}"" --force"
+	fi
+
+	if [ "${ZK_SYNCHRONOUS}" = true ]; then
+			testString="${testString}"" --synchronous"
+	fi
+
+	if [ "${ZK_VERBOSE}" = true ]; then
+		testString="${testString}"" --verbose"
+	fi
+
+	LogMsg "Running zookeeper tests with cmd: ${testString}"
+	eval $testString
+	if [ $? -ne 0 ]; then
+		msg="Error: Unable to run test"
+		LogMsg "${msg}"
+		echo "${msg}" >> ./summary.log
+		UpdateTestState $ICA_TESTFAILED
+		exit 1
+	fi
+done
+
+
+#
+# If we made it here, everything worked.
+# Indicate success
+#
+LogMsg "Test completed successfully"
+UpdateTestState $ICA_TESTCOMPLETED
+
+exit 0