updating scripts to download & use spark-csv and its dependency

build/Build.cmd

@@ -130,6 +130,15 @@ copy /y Samples\Microsoft.Spark.CSharp\bin\Release\* "%SPARKCLR_HOME%\samples\"
 copy /y Samples\Microsoft.Spark.CSharp\data\* "%SPARKCLR_HOME%\data\"
 popd
 
+@echo Download external dependencies
+pushd "%CMDHOME%"
+set DEPENDENCIES_DIR=dependencies
+if NOT EXIST "%DEPENDENCIES_DIR%" mkdir %DEPENDENCIES_DIR%
+set DEPENDENCIES_HOME=%CMDHOME%\%DEPENDENCIES_DIR%
+powershell -f localmode\downloadtools.ps1 dependencies
+@echo Assemble dependencies
+xcopy /e /y "%DEPENDENCIES_HOME%"  "%SPARKCLR_HOME%\dependencies\"
+
 @echo Assemble SparkCLR examples
 pushd "%CMDHOME%\..\examples"
 call Clean.cmd

build/localmode/RunSamples.cmd

@@ -61,7 +61,12 @@ if defined ProjectVersion (
 )
 
 set SPARKCLR_HOME=%CMDHOME%\..\runtime
-set SPARKCSV_JARS=
+
+# spark-csv package and its depenedency are required for DataFrame operations in Mobius
+set SPARKCLR_EXT_PATH=%SPARKCLR_HOME%\dependencies
+set SPARKCSV_JAR1PATH=%SPARKCLR_EXT_PATH%\spark-csv_2.10-1.3.0.jar
+set SPARKCSV_JAR2PATH=%SPARKCLR_EXT_PATH%\commons-csv-1.1.jar
+set SPARKCLR_EXT_JARS=%SPARKCSV_JAR1PATH%,%SPARKCSV_JAR2PATH%
 
 @rem RunSamples.cmd is in local mode, should not load Hadoop or Yarn cluster config. Disable Hadoop/Yarn conf dir.
 set HADOOP_CONF_DIR=
@@ -75,7 +80,7 @@ set SAMPLES_DIR=%SPARKCLR_HOME%\samples
 @echo [RunSamples.cmd] JAVA_HOME=%JAVA_HOME%
 @echo [RunSamples.cmd] SPARK_HOME=%SPARK_HOME%
 @echo [RunSamples.cmd] SPARKCLR_HOME=%SPARKCLR_HOME%
-@echo [RunSamples.cmd] SPARKCSV_JARS=%SPARKCSV_JARS%
+@echo [RunSamples.cmd] SPARKCLR_EXT_JARS=%SPARKCLR_EXT_JARS%
 
 pushd "%SPARKCLR_HOME%\scripts"
 @echo [RunSamples.cmd] CWD=
@@ -83,8 +88,8 @@ pushd "%SPARKCLR_HOME%\scripts"
 @dir /s "%SPARKCLR_HOME%"
 
 if "!USER_EXE!"=="" (
-    @echo [RunSamples.cmd] call sparkclr-submit.cmd --exe SparkCLRSamples.exe %SAMPLES_DIR% spark.local.dir %TEMP_DIR% sparkclr.sampledata.loc %SPARKCLR_HOME%\data %*
-    call sparkclr-submit.cmd --exe SparkCLRSamples.exe %SAMPLES_DIR% spark.local.dir %TEMP_DIR% sparkclr.sampledata.loc %SPARKCLR_HOME%\data %*
+    @echo [RunSamples.cmd] call sparkclr-submit.cmd --jars %SPARKCLR_EXT_JARS% -exe SparkCLRSamples.exe %SAMPLES_DIR% spark.local.dir %TEMP_DIR% sparkclr.sampledata.loc %SPARKCLR_HOME%\data %*
+    call sparkclr-submit.cmd --jars %SPARKCLR_EXT_JARS% --exe SparkCLRSamples.exe %SAMPLES_DIR% spark.local.dir %TEMP_DIR% sparkclr.sampledata.loc %SPARKCLR_HOME%\data %*
 ) else (
     @echo [RunSamples.cmd] call sparkclr-submit.cmd %*
     call sparkclr-submit.cmd %*

build/localmode/downloadtools.ps1

@@ -280,6 +280,21 @@ function Download-BuildTools
     $envStream.close()
 }
 
+function Download-ExternalDependencies
+{
+    # Downloading spark-csv package and its depenency. These packages are required for DataFrame operations in Mobius
+	$url = "http://search.maven.org/remotecontent?filepath=com/databricks/spark-csv_2.10/1.3.0/spark-csv_2.10-1.3.0.jar"
+    $output="$scriptDir\..\dependencies\spark-csv_2.10-1.3.0.jar"
+    Download-File $url $output
+	Write-Output "[downloadtools.Download-ExternalDependencies] Downloading $url to $scriptDir\..\dependencies"
+	
+	$url = "http://search.maven.org/remotecontent?filepath=org/apache/commons/commons-csv/1.1/commons-csv-1.1.jar"
+	$output="$scriptDir\..\dependencies\commons-csv-1.1.jar"
+	Download-File $url $output
+    Write-Output "[downloadtools.Download-ExternalDependencies] Downloading $url to $scriptDir\..\dependencies"
+	return
+}
+
 function Download-RuntimeDependencies
 {
     # Create a cmd file to update environment variable
@@ -513,6 +528,10 @@ elseif ($stage.ToLower() -eq "run")
 {
     Download-RuntimeDependencies
 }
+elseif ($stage.ToLower() -eq "dependencies")
+{
+    Download-ExternalDependencies
+}
 else
 {
     Print-Usage