Fix: windows path given in the command line (#41)

* Fix windows path

* Upload custom-script

* Custom script works with windows path

* Fix util

* 24 Hours

dtde/clusterlib.py

@@ -81,7 +81,7 @@ def generate_cluster_start_task(
     # Upload custom script file if given
     if custom_script is not None:
         resource_files.append(
-            util.upload_file_to_container(
+            util.upload_shell_script_to_container(
                 container_name=cluster_id,
                 file_path=custom_script,
                 node_path="custom-scripts/{0}".format(os.path.basename(custom_script))))

dtde/joblib.py

@@ -96,9 +96,9 @@ def submit_app(
 
     resource_files = []
 
+    app_resource_file = util.upload_file_to_container(container_name=name, file_path=app, use_full_path=True)
     # Upload application file
-    resource_files.append(
-        util.upload_file_to_container(container_name=name, file_path=app, use_full_path=True))
+    resource_files.append(app_resource_file)
 
     # Upload dependent JARS
     for jar in jars:
@@ -119,7 +119,7 @@ def submit_app(
     cmd = app_submit_cmd(
         cluster_id=cluster_id,
         name=name,
-        app=app,
+        app=app_resource_file.file_path,
         app_args=app_args,
         main_class=main_class,
         jars=jars,

dtde/util.py

@@ -76,7 +76,7 @@ def wait_for_master_to_be_ready(cluster_id: str):
     time.sleep(5)
 
 
-def upload_file_to_container(container_name, file_path, use_full_path = False, node_path = None) -> batch_models.ResourceFile:
+def upload_file_to_container(container_name, file_path, use_full_path=False, node_path=None) -> batch_models.ResourceFile:
     """
     Uploads a local file to an Azure Blob storage container.
     :param block_blob_client: A blob service client.
@@ -89,7 +89,7 @@ def upload_file_to_container(container_name, file_path, use_full_path = False, n
     tasks.
     """
     block_blob_client = azure_api.get_blob_client()
-
+    file_path = normalize_path(file_path)
     blob_name = None
     if use_full_path:
         blob_name = file_path
@@ -120,6 +120,48 @@ def upload_file_to_container(container_name, file_path, use_full_path = False, n
                                      blob_source=sas_url)
 
 
+def upload_text_to_container(container_name: str, content: str, local_path: str) -> batch_models.ResourceFile:
+    local_path = normalize_path(local_path)
+    blob_name = local_path
+    block_blob_client = azure_api.get_blob_client()
+    block_blob_client.create_container(container_name, fail_on_exist=False)
+    block_blob_client.create_blob_from_text(container_name,
+                                            blob_name,
+                                            content)
+
+    sas_token = block_blob_client.generate_blob_shared_access_signature(
+        container_name,
+        blob_name,
+        permission=blob.BlobPermissions.READ,
+        expiry=datetime.datetime.utcnow() + datetime.timedelta(hours=24))
+
+    sas_url = block_blob_client.make_blob_url(container_name,
+                                              blob_name,
+                                              sas_token=sas_token)
+
+    return batch_models.ResourceFile(file_path=blob_name,
+                                     blob_source=sas_url)
+
+
+def upload_shell_script_to_container(
+        container_name: str,
+        file_path: str,
+        node_path: str=None) -> batch_models.ResourceFile:
+    """
+    Upload a shell(.sh) file to the container.
+    It will make sure the windows line ending are replaced with unix line ending to prevent execution errors
+    :param container_name: Azure blob container
+    :param file_path: Local file path
+    :param node_path: Optional path on the node. By default will use the same as the file_path
+    """
+    if not node_path:
+        node_path = normalize_path(file_path)
+
+    with io.open(file_path, 'r') as f:
+        content = f.read().replace('\r\n', '\n')
+        return upload_text_to_container(container_name, content, node_path)
+
+
 def print_configuration(config):
     """
     Prints the configuration being used as a dictionary
@@ -375,6 +417,18 @@ def get_cluster_total_current_nodes(pool):
     return pool.current_dedicated_nodes + pool.current_low_priority_nodes
 
 
+def normalize_path(path: str)-> str:
+    """
+    Convert a path in a path that will work well with blob storage and unix
+    It will replace \ with / and remove relative .
+    """
+    path = path.replace('\\', '/')
+    if path.startswith('./'):
+        return path[2:]
+    else:
+        return path
+
+
 def get_file_properties(job_id: str, task_id: str, file_path: str):
     batch_client = azure_api.get_batch_client()