From 61b84579e647c47c6b87e0bba96f3eabee2cf52e Mon Sep 17 00:00:00 2001 From: tawan Date: Thu, 12 Nov 2015 17:36:04 +0800 Subject: [PATCH] Fix `SPARKCSV_JARS` is not supported issue --- README.md | 2 +- .../launcher/SparkCLRSubmitArguments.scala | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index febdc69..8b4054b 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ Setting `spark.local.dir` parameter is important. When local Spark instance dist ### Running in Standalone mode ``` -sparkclr-submit.cmd --verbose --exe SparkCLRSamples.exe %SPARKCLR_HOME%\samples sparkclr.sampledata.loc hdfs://path/to/sparkclr/sampledata +sparkclr-submit.cmd --verbose --master spark://host:port --exe SparkCLRSamples.exe %SPARKCLR_HOME%\samples sparkclr.sampledata.loc hdfs://path/to/sparkclr/sampledata ``` ### Running in YARN mode diff --git a/scala/src/main/org/apache/spark/launcher/SparkCLRSubmitArguments.scala b/scala/src/main/org/apache/spark/launcher/SparkCLRSubmitArguments.scala index e37432d..9b8798b 100644 --- a/scala/src/main/org/apache/spark/launcher/SparkCLRSubmitArguments.scala +++ b/scala/src/main/org/apache/spark/launcher/SparkCLRSubmitArguments.scala @@ -44,6 +44,8 @@ class SparkCLRSubmitArguments(args: Seq[String], env: Map[String, String], exitF var files: String = null + var jars: String = env.getOrElse("SPARKCSV_JARS", "").replace(";",",") + var primaryResource: String = null var propertiesFile: String = null @@ -133,6 +135,13 @@ class SparkCLRSubmitArguments(args: Seq[String], env: Map[String, String], exitF case FILES => files = Utils.resolveURIs(value) + case JARS => + if (jars != "") { + jars = s"$jars,$value" + } else { + jars = value + } + case HELP => printUsageAndExit() @@ -224,6 +233,14 @@ class SparkCLRSubmitArguments(args: Seq[String], env: Map[String, String], exitF case _ => } + if (jars != null && !jars.trim.isEmpty) { + if (cmd == "") { + cmd += s"--jars $jars" + } else { + cmd += s" --jars $jars" + } + } + master match { case m if m == null || m.startsWith("local") => concatLocalCmdOptions()