Support ssh auth with private key file to enable to deploy pai to AWS… (#1308)

* Support ssh auth with private key file to enable to deploy pai to AWS EC2 (#1295)

* Check the ssh-key-filename as it is optional (#1295)

* Check the ssh-key-filename is not None (#1295)

* Rename configuration key-filename to keyfile-path; Make keyfile path option to pass CI

* Use k8s secret to pass ssh key to watchdog

* Create k8s secret automatically; fixed the quick start config

* Fixed the watchdog keyfile path config name

* Fixed the format of the watchdog yaml template

pai-management/bootstrap/prometheus/service.yaml

@@ -22,6 +22,7 @@ prerequisite:
 template-list:
   - prometheus-configmap.yaml
   - prometheus-deployment.yaml
+  - start.sh
   - refresh.sh
   - delete.yaml
 

pai-management/bootstrap/prometheus/start.sh.template

@@ -21,6 +21,9 @@
 
 pushd $(dirname "$0") > /dev/null
 
+{% if 'ssh-keyfile-path' in clusterinfo['prometheusinfo'] and 'ssh-secret-name' in clusterinfo['prometheusinfo']-%}
+kubectl create secret generic {{ clusterinfo['prometheusinfo']['ssh-secret-name'] }} --from-file=pkey={{ clusterinfo['prometheusinfo']['ssh-keyfile-path'] }}
+{% endif -%}
 kubectl create configmap prometheus-alert --from-file=../../../prometheus/prometheus-alert --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $?
 kubectl apply --overwrite=true -f prometheus-configmap.yaml || exit $?
 kubectl apply --overwrite=true -f prometheus-deployment.yaml || exit $?

pai-management/bootstrap/watchdog/watchdog-configmap.yaml.template

@@ -7,5 +7,9 @@ data:
 
     hosts:
     {% for _, hostInfo in machinelist.items() %}
+    {% if 'ssh-secret-name' in clusterinfo['prometheusinfo'] -%}
+    - {hostip: {{ hostInfo['ip'] }}, port: {{ hostInfo['sshport'] }}, username: {{ hostInfo['username'] }}, password: {{ hostInfo['password'] }}, keyfile-path: "/etc/secret-volume/pkey" }
+    {% else -%}
     - {hostip: {{ hostInfo['ip'] }}, port: {{ hostInfo['sshport'] }}, username: {{ hostInfo['username'] }}, password: {{ hostInfo['password'] }}  }
+    {% endif -%}
     {% endfor %}

pai-management/bootstrap/watchdog/watchdog.yaml.template

@@ -52,6 +52,11 @@ spec:
           name: collector-mount
         - mountPath: /etc/watchdog
           name: config-volume
+        {% if 'ssh-secret-name' in clusterinfo['prometheusinfo'] %}
+        - mountPath: /etc/secret-volume
+          readOnly: true
+          name: secret-volume
+        {% endif %}
         command:
           - "python"
           - "/watchdog.py"
@@ -67,6 +72,11 @@ spec:
         - name: config-volume
           configMap:
             name: watchdog
+        {% if 'ssh-secret-name' in clusterinfo['prometheusinfo'] %}
+        - name: secret-volume
+          secret:
+            secretName: {{ clusterinfo['prometheusinfo']['ssh-secret-name'] }}
+        {% endif %}
       imagePullSecrets:
       - name: {{ clusterinfo["dockerregistryinfo"]["secretname"] }}
       tolerations:

pai-management/doc/cluster-bootup.md

@@ -138,10 +138,13 @@ machines:
   - 192.168.1.13
 
 # (Required) Log-in info of all machines. System administrator should guarantee
-# that the username/password pair is valid and has sudo privilege.
+# that the username/password pair or username/key-filename is valid and has sudo privilege.
 ssh-username: pai
 ssh-password: pai-password
 
+# (Optional, default=None) the key file that ssh client uses, that has higher priority then password.
+#ssh-keyfile-path: <keyfile-path>
+
 # (Optional, default=22) Port number of ssh service on each machine.
 #ssh-port: 22
 
@@ -337,9 +340,10 @@ machines:
   - 192.168.1.11
 
 # (Required) Log-in info of all machines. System administrator should guarantee
-# that the username/password pair is valid and has sudo privilege.
+# that the username/password pair or username/key-filename is valid and has sudo privilege.
 ssh-username: pai
 ssh-password: pai-password
+ssh-key-filename: key-filename
 
 # (Optional, default=22) Port number of ssh service on each machine.
 #ssh-port: 22

pai-management/k8sPaiLibrary/maintainlib/common.py

@@ -167,11 +167,16 @@ def sftp_paramiko(src, dst, filename, host_config):
         if port_validation(host_config['sshport']) == False:
             return False
         port = int(host_config['sshport'])
-
+    key_filename = None
+    if 'keyfile-path' in host_config:
+        if os.path.isfile(str(host_config['keyfile-path'])) and host_config['keyfile-path'] is not None:
+            key_filename = str(host_config['keyfile-path'])
+        else:
+            logger.warn("The key file: {0} specified doesn't exist".format(host_config['keyfile-path']))
     # First make sure the folder exist.
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    ssh.connect(hostname=hostip, port=port, username=username, password=password)
+    ssh.connect(hostname=hostip, port=port, key_filename=key_filename, username=username, password=password)
 
     stdin, stdout, stderr = ssh.exec_command("sudo mkdir -p {0}".format(dst), get_pty=True)
     stdin.write(password + '\n')
@@ -183,7 +188,10 @@ def sftp_paramiko(src, dst, filename, host_config):
 
     # Put the file to target Path.
     transport = paramiko.Transport((hostip, port))
-    transport.connect(username=username, password=password)
+    pkey = None
+    if key_filename is not None:
+        pkey = paramiko.RSAKey.from_private_key_file(key_filename)
+    transport.connect(username=username, pkey=pkey, password=password)
 
     sftp = paramiko.SFTPClient.from_transport(transport)
     sftp.put('{0}/{1}'.format(src, filename), '{0}/{1}'.format(dst, filename))
@@ -216,9 +224,15 @@ def ssh_shell_paramiko_with_result(host_config, commandline):
         if port_validation(host_config['sshport']) == False:
             return (None, None)
         port = int(host_config['sshport'])
+    key_filename = None
+    if 'keyfile-path' in host_config and host_config['keyfile-path'] is not None:
+        if os.path.isfile(str(host_config['keyfile-path'])):
+            key_filename = str(host_config['keyfile-path'])
+        else:
+            logger.warn("The key file: {0} specified doesn't exist".format(host_config['keyfile-path']))
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    ssh.connect(hostname=hostip, port=port, username=username, password=password)
+    ssh.connect(hostname=hostip, port=port, key_filename=key_filename, username=username, password=password)
     stdin, stdout, stderr = ssh.exec_command(commandline, get_pty=True)
     logger.info("Executing the command on host [{0}]: {1}".format(hostip, commandline))
     result_stdout = ""
@@ -252,10 +266,16 @@ def ssh_shell_with_password_input_paramiko(host_config, commandline):
         if port_validation(host_config['sshport']) == False:
             return False
         port = int(host_config['sshport'])
+    key_filename = None
+    if 'keyfile-path' in host_config:
+        if os.path.isfile(str(host_config['keyfile-path'])) and host_config['keyfile-path'] is not None:
+            key_filename = str(host_config['keyfile-path'])
+        else:
+            logger.warn("The key file: {0} specified doesn't exist".format(host_config['keyfile-path']))
 
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-    ssh.connect(hostname=hostip, port=port, username=username, password=password)
+    ssh.connect(hostname=hostip, port=port, key_filename=key_filename, username=username, password=password)
     stdin, stdout, stderr = ssh.exec_command(commandline, get_pty=True)
     stdin.write(password + '\n')
     stdin.flush()

pai-management/paiLibrary/paiCluster/cluster_util.py

@@ -51,6 +51,10 @@ def generate_configuration(quick_start_config_file, configuration_directory, for
     # Prepare config of ssh info.
     quick_start_config["ssh-username"] = quick_start_config_raw["ssh-username"]
     quick_start_config["ssh-password"] = quick_start_config_raw["ssh-password"]
+    if "ssh-keyfile-path" in quick_start_config_raw:
+        quick_start_config["ssh-keyfile-path"] = quick_start_config_raw["ssh-keyfile-path"]
+    if "ssh-secret-name" in quick_start_config_raw:
+        quick_start_config["ssh-secret-name"] = quick_start_config_raw["ssh-secret-name"]
     quick_start_config["ssh-port"] = \
         22 if "ssh-port" not in quick_start_config_raw \
         else quick_start_config_raw["ssh-port"]
@@ -70,25 +74,31 @@ def generate_configuration(quick_start_config_file, configuration_directory, for
         quick_start_config["dns"] = quick_start_config_raw["dns"]
     else:
         m0 = quick_start_config["machines"][0]
-        result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
-            {
+        host_config = {
             "hostip": m0["ip"],
             "username": quick_start_config["ssh-username"],
             "password": quick_start_config["ssh-password"],
             "sshport": quick_start_config["ssh-port"]
-            },
+        }
+        if "ssh-keyfile-path" in quick_start_config:
+            host_config["keyfile-path"] = quick_start_config["ssh-keyfile-path"]
+        result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
+            host_config,
             "cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2 | head -n 1")
         quick_start_config["dns"] = result_stdout.strip()
     #
     # Auto-complete missing configuration items: Part 2 -- hostnames.
     for m in quick_start_config["machines"]:
-        result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
-            {
+        host_config = {
             "hostip": m["ip"],
             "username": quick_start_config["ssh-username"],
             "password": quick_start_config["ssh-password"],
             "sshport": quick_start_config["ssh-port"]
-            },
+        }
+        if "ssh-keyfile-path" in quick_start_config:
+            host_config["keyfile-path"] = quick_start_config["ssh-keyfile-path"]
+        result_stdout, result_stderr = pai_common.ssh_shell_paramiko_with_result(
+            host_config,
             "hostname")
         m["hostname"] = result_stdout.strip()
     #

pai-management/quick-start/quick-start-example.yaml

@@ -25,10 +25,15 @@ machines:
   - <ip-of-worder2>
 
 # (Required) Log-in info of all machines. System administrator should guarantee
-# that the username/password pair is valid and has sudo privilege.
+# that the username/password pair or username/key-filename is valid and has sudo privilege.
 ssh-username: <username>
 ssh-password: <password>
 
+# (Optional, default=None) the key file that ssh client uses, that has higher priority then password.
+# Secret name is used by kubernetes to create secret for the key file in cluster so that service could use it.
+#ssh-keyfile-path: <keyfile-path>
+#ssh-secret-name: <secret-name>
+
 # (Optional, default=22) Port number of ssh service on each machine.
 #ssh-port: 22
 

pai-management/quick-start/services-configuration.yaml.template

@@ -117,6 +117,15 @@ prometheus:
   #    smtp_auth_username: user@gmail.com
   #    smtp_auth_password: gmail_password
 
+  # if you want to use key file to login nodes
+  {% if 'ssh-keyfile-path' in env -%}
+  ssh-keyfile-path: {{ env['ssh-keyfile-path'] }}
+  {% if 'ssh-secret-name' in env -%}
+  ssh-secret-name: {{ env['ssh-secret-name'] }}
+  {% else -%}
+  ssh-secret-name: ssh-key-secret
+  {% endif %}
+  {% endif %}
 
 pylon:
   # port of pylon

prometheus/exporter/watchdog.py

@@ -125,12 +125,18 @@ def ssh_exec(host_config, command, histogram=ssh_histogram):
         port = 22
         if "sshport" in host_config:
             port = int(host_config["sshport"])
+        key_filename = None
+        if 'keyfile-path' in host_config:
+            if os.path.isfile(str(host_config['keyfile-path'])) and host_config['keyfile-path'] is not None:
+                key_filename = str(host_config['keyfile-path'])
+            else:
+                logger.warn("The key file: {0} specified doesn't exist".format(host_config['keyfile-path']))
 
         ssh = paramiko.SSHClient()
 
         try:
             ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-            ssh.connect(hostname=hostip, port=port, username=username, password=password)
+            ssh.connect(hostname=hostip, port=port, key_filename=key_filename, username=username, password=password)
 
             logger.info("Executing the command on host [{0}]: {1}".format(hostip, command))