From b09f4fe1721ed7f0b6994a976ef9d9976d4a1c7f Mon Sep 17 00:00:00 2001
From: jiata <jiata@microsoft.com>
Date: Thu, 6 Jul 2017 18:15:58 -0700
Subject: [PATCH 1/3] custom-script for wasb connection

---
 .gitignore                      |  1 +
 custom-scripts/wasb.sh.template | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 custom-scripts/wasb.sh.template

diff --git a/.gitignore b/.gitignore
index fad4c51c..3e21168f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 # custom
 configuration.cfg
+my-custom-scripts/
 
 # Virtual environments
 ENV/
diff --git a/custom-scripts/wasb.sh.template b/custom-scripts/wasb.sh.template
new file mode 100644
index 00000000..b2a7e753
--- /dev/null
+++ b/custom-scripts/wasb.sh.template
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+export SPARK_HOME=/dsvm/tools/spark/current
+cd $SPARK_HOME/conf
+
+cp spark-defaults.conf.template spark-defaults.conf
+
+cat >> spark-defaults.conf <<EOF
+spark.jars                 /dsvm/tools/spark/current/jars/azure-storage-4.4.0.jar,/dsvm/tools/spark/current/jars/hadoop-azure-2.7.3.jar
+EOF
+
+cat >> core-site.xml <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+<name>fs.AbstractFileSystem.wasb.Impl</name>
+<value>org.apache.hadoop.fs.azure.Wasb</value>
+</property>
+<property>
+<name>fs.azure.account.key.YOURSTORAGEACCOUNT.blob.core.windows.net</name>
+<value>YOURSTORAGEACCOUNTKEY</value>
+</property>
+</configuration>
+EOF
+
+cd /opt/azure-storage-jars
+sudo wget http://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/2.0.0/azure-storage-2.0.0.jar
+sudo ln -sf /opt/azure-storage-jars/azure-storage-2.0.0.jar $SPARK_HOME/jars/azure-storage-4.4.0.jar

From 818bb041c5ff52f0909eba00fa95f79f53712c53 Mon Sep 17 00:00:00 2001
From: jiata <jiata@microsoft.com>
Date: Tue, 11 Jul 2017 18:36:00 -0400
Subject: [PATCH 2/3] updates to custom wasb template

---
 custom-scripts/wasb.sh.template | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/custom-scripts/wasb.sh.template b/custom-scripts/wasb.sh.template
index b2a7e753..eddbfb3e 100644
--- a/custom-scripts/wasb.sh.template
+++ b/custom-scripts/wasb.sh.template
@@ -6,7 +6,7 @@ cd $SPARK_HOME/conf
 cp spark-defaults.conf.template spark-defaults.conf
 
 cat >> spark-defaults.conf <<EOF
-spark.jars                 /dsvm/tools/spark/current/jars/azure-storage-4.4.0.jar,/dsvm/tools/spark/current/jars/hadoop-azure-2.7.3.jar
+spark.jars                 /dsvm/tools/spark/current/jars/azure-storage-2.0.0.jar,/dsvm/tools/spark/current/jars/hadoop-azure-2.7.3.jar
 EOF
 
 cat >> core-site.xml <<EOF
@@ -18,12 +18,13 @@ cat >> core-site.xml <<EOF
 <value>org.apache.hadoop.fs.azure.Wasb</value>
 </property>
 <property>
-<name>fs.azure.account.key.YOURSTORAGEACCOUNT.blob.core.windows.net</name>
-<value>YOURSTORAGEACCOUNTKEY</value>
+<name>fs.azure.account.key.STORAGE_ACCOUNT_NAME.blob.core.windows.net</name>
+<value>STORAGE_ACCOUNT_KEY</value>
 </property>
 </configuration>
 EOF
 
-cd /opt/azure-storage-jars
+sudo rm $SPARK_HOME/jars/azure-storage-4.4.0.jar
+
+cd $SPARK_HOME/jars
 sudo wget http://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/2.0.0/azure-storage-2.0.0.jar
-sudo ln -sf /opt/azure-storage-jars/azure-storage-2.0.0.jar $SPARK_HOME/jars/azure-storage-4.4.0.jar

From 4358042e8cc9b9e257e03cb102701ffb062eb85c Mon Sep 17 00:00:00 2001
From: Timothee Guerin <timothee.guerin@outlook.com>
Date: Wed, 12 Jul 2017 10:38:53 -0700
Subject: [PATCH 3/3] Remove blob client

---
 dtde/joblib.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dtde/joblib.py b/dtde/joblib.py
index f5cbba90..d2db670f 100644
--- a/dtde/joblib.py
+++ b/dtde/joblib.py
@@ -93,25 +93,25 @@ def submit_app(
     # Upload application file
     resource_files.append(
         util.upload_file_to_container(
-            blob_client, container_name = name, file_path = app, use_full_path = True))
+                container_name = name, file_path = app, use_full_path = True))
 
     # Upload dependent JARS
     for jar in jars:
         resource_files.append(
             util.upload_file_to_container(
-                blob_client, container_name = name, file_path = jar, use_full_path = True))
+                container_name = name, file_path = jar, use_full_path = True))
 
     # Upload dependent python files 
     for py_file in py_files:
         resource_files.append(
             util.upload_file_to_container(
-                blob_client, container_name = name, file_path = py_file, use_full_path = True))
+                container_name = name, file_path = py_file, use_full_path = True))
 
     # Upload other dependent files 
     for file in files:
         resource_files.append(
             util.upload_file_to_container(
-                blob_client, container_name = name, file_path = file, use_full_path = True))
+                ontainer_name = name, file_path = file, use_full_path = True))
 
     # create command to submit task
     cmd = app_submit_cmd(