udate cluster

config/ssh.yaml

@@ -14,5 +14,9 @@ web_ui_port: 8080
 # jupyter_port: <local port which where jupyter is forwarded to>
 jupyter_port: 8888
 
-# connect: <true/false, connect to spark master or print connection string (--no-connect)> 
-connect: true
+# connect: <true/false, connect to spark master or print connection string (--no-connect)>
+connect: true
+
+# List of additional ports to forward. List in the format <local>:localhost:<remote>
+ports:
+#  - 1234:localhost:5678 # This will forward local port 1234 to the port 5678 on the master node

dtde/config.py

@@ -190,6 +190,7 @@ class SshConfig:
         self.web_ui_port = None
         self.jupyter_port = None
         self.connect = True
+        self.ports = []
 
     def _read_config_file(self, path: str=constants.DEFAULT_SSH_CONFIG_PATH):
         """
@@ -230,7 +231,11 @@ class SshConfig:
         if 'connect' in config and config['connect'] is False:
             self.connect = False
 
-    def merge(self, cluster_id, username, job_ui_port, web_ui_port, jupyter_port, connect):
+        if 'ports' in config:
+            self.ports += config['ports'] or []
+
+
+    def merge(self, cluster_id, username, job_ui_port, web_ui_port, jupyter_port, ports, connect):
         """
             Merges fields with args object
         """
@@ -242,6 +247,7 @@ class SshConfig:
                 job_ui_port=job_ui_port,
                 web_ui_port=web_ui_port,
                 jupyter_port=jupyter_port,
+                ports=ports,
                 connect=connect
             )
         )

dtde/spark/cli/cluster_ssh.py

@@ -34,6 +34,7 @@ def execute(args: typing.NamedTuple):
         job_ui_port=args.jobui,
         web_ui_port=args.webui,
         jupyter_port=args.jupyter,
+        ports=args.ports,
         connect=args.connect
     )
 
@@ -52,7 +53,7 @@ def execute(args: typing.NamedTuple):
         webui=ssh_conf.web_ui_port,
         jobui=ssh_conf.job_ui_port,
         jupyter=ssh_conf.jupyter_port,
-        ports=args.ports,
+        ports=ssh_conf.ports,
         username=ssh_conf.username,
         connect=ssh_conf.connect)