better logging and port mapping

dtde/cli.py

@@ -6,10 +6,12 @@
 """
 import argparse
 from typing import NamedTuple
-from dtde import constants, version, logger, log
+from dtde import constants, version, logger, log, util
 from dtde.spark.cli import spark
 from dtde.models import Software
 
+import azure.batch.models as batch_models
+
 def main():
     parser = argparse.ArgumentParser(prog=constants.CLI_EXE)
 
@@ -25,7 +27,10 @@ def main():
     args = parser.parse_args()
 
     parse_common_args(args)
-    run_software(args)
+    try:
+        run_software(args)
+    except batch_models.BatchErrorException as e:
+        util.print_batch_exception(e)
 
 
 def setup_common_args(parser: argparse.ArgumentParser):

dtde/clusterlib.py

@@ -188,9 +188,9 @@ def create_cluster(
                 batch_models.PoolEndpointConfiguration(
                     inbound_nat_pools = [
                         batch_models.InboundNATPool(
-                            backend_port=8080,
-                            frontend_port_range_start=8080,
-                            frontend_port_range_end=8880,
+                            backend_port=constants.DOCKER_SPARK_MASTER_UI_PORT,
+                            frontend_port_range_start=constants.DOCKER_SPARK_MASTER_UI_PORT,
+                            frontend_port_range_end=constants.DOCKER_SPARK_MASTER_UI_PORT + vm_count + vm_low_pri_count,
                             name="master_ui",
                             protocol="tcp",
                             network_security_group_rules=[
@@ -202,9 +202,9 @@ def create_cluster(
                             ]
                         ),
                         batch_models.InboundNATPool(
-                            backend_port=4040,
-                            frontend_port_range_start=4040,
-                            frontend_port_range_end=5040,
+                            backend_port=constants.DOCKER_SPARK_WEB_UI_PORT,
+                            frontend_port_range_start=constants.DOCKER_SPARK_WEB_UI_PORT,
+                            frontend_port_range_end=constants.DOCKER_SPARK_WEB_UI_PORT + vm_count + vm_low_pri_count,
                             name="job_ui",
                             protocol="tcp",
                             network_security_group_rules=[
@@ -216,9 +216,9 @@ def create_cluster(
                             ]
                         ),
                         batch_models.InboundNATPool(
-                            backend_port=8888,
-                            frontend_port_range_start=8888,
-                            frontend_port_range_end=9800,
+                            backend_port=constants.DOCKER_SPARK_JUPYTER_PORT,
+                            frontend_port_range_start=constants.DOCKER_SPARK_JUPYTER_PORT,
+                            frontend_port_range_end=constants.DOCKER_SPARK_JUPYTER_PORT + vm_count + vm_low_pri_count,
                             name="jupyter",
                             protocol="tcp",
                             network_security_group_rules=[

dtde/util.py

@@ -219,7 +219,6 @@ def create_pool_if_not_exist(pool, wait=True):
             return True
     except batch_models.BatchErrorException as e:
         if e.error.code != "PoolExists":
-            print_batch_exception(e)
             raise
         else:
             return False
@@ -391,17 +390,17 @@ def print_batch_exception(batch_exception):
     Prints the contents of the specified Batch exception.
     :param batch_exception:
     """
-    log.info("-------------------------------------------")
-    log.info("Exception encountered:")
+
+    log.error("Exception encountered:")
     if batch_exception.error and \
             batch_exception.error.message and \
             batch_exception.error.message.value:
-        log.info(batch_exception.error.message.value)
+        log.error(batch_exception.error.message.value)
         if batch_exception.error.values:
-            log.info("")
+            log.error("")
             for mesg in batch_exception.error.values:
-                log.info("%s:\t%s", mesg.key, mesg.value)
-    log.info("-------------------------------------------")
+                log.error("%s:\t%s", mesg.key, mesg.value)
+    log.error("-------------------------------------------")
 
 
 def get_cluster_total_target_nodes(pool):