Merge master

.gitignore

@@ -1,5 +1,6 @@
 # custom
 configuration.cfg
+my-custom-scripts/
 
 # Virtual environments
 ENV/

custom-scripts/wasb.sh.template

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+export SPARK_HOME=/dsvm/tools/spark/current
+cd $SPARK_HOME/conf
+
+cp spark-defaults.conf.template spark-defaults.conf
+
+cat >> spark-defaults.conf <<EOF
+spark.jars                 /dsvm/tools/spark/current/jars/azure-storage-2.0.0.jar,/dsvm/tools/spark/current/jars/hadoop-azure-2.7.3.jar
+EOF
+
+cat >> core-site.xml <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+<name>fs.AbstractFileSystem.wasb.Impl</name>
+<value>org.apache.hadoop.fs.azure.Wasb</value>
+</property>
+<property>
+<name>fs.azure.account.key.STORAGE_ACCOUNT_NAME.blob.core.windows.net</name>
+<value>STORAGE_ACCOUNT_KEY</value>
+</property>
+</configuration>
+EOF
+
+sudo rm $SPARK_HOME/jars/azure-storage-4.4.0.jar
+
+cd $SPARK_HOME/jars
+sudo wget http://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/2.0.0/azure-storage-2.0.0.jar

dtde/joblib.py

@@ -94,7 +94,6 @@ def submit_app(
     Submit a spark app 
     """
     batch_client = azure_api.get_batch_client()
-    blob_client = azure_api.get_blob_client()
 
     resource_files = []