diff --git a/scala/pom.xml b/scala/pom.xml index bf27d48..8ee39a0 100644 --- a/scala/pom.xml +++ b/scala/pom.xml @@ -65,11 +65,15 @@ org.apache.spark spark-core_2.10 ${spark.version} + + org.apache.spark spark-streaming_2.10 ${spark.version} + + org.apache.spark @@ -80,6 +84,8 @@ org.apache.spark spark-sql_2.10 ${spark.version} + + com.databricks @@ -159,8 +165,8 @@ - - + + diff --git a/scripts/addotherplugin.ps1 b/scripts/addotherplugin.ps1 index 69f89bc..db9bac3 100644 --- a/scripts/addotherplugin.ps1 +++ b/scripts/addotherplugin.ps1 @@ -33,7 +33,7 @@ function Get-ScriptDirectory # function Replace-VariableInFile($variable, $value, $sourceFile, $targetFile) { - Write-Output "[addotherplugin.Replace-VariableInFile] variable=$variable, value=$value, sourceFile=$sourceFile, targetFile=$targetFile" + Write-Output "[addotherplugin.Replace-VariableInFile] variable=$variable, sourceFile=$sourceFile, targetFile=$targetFile" if (!(test-path $sourceFile)) { Write-Output "[addotherplugin.Replace-VariableInFile] [WARNING] $sourceFile does not exist. Abort." @@ -60,14 +60,14 @@ function Replace-VariableInFile($variable, $value, $sourceFile, $targetFile) # if (!(test-path $targetPom)) { - Write-Output "[addotherplugin.ps1] [WARNING] $targetPom does not exist. Abort." + Write-Output "[addotherplugin.ps1] [WARNING] $targetPom does not exist. Abort." Print-Usage return } if (!(test-path $sourcePom)) { - Write-Output "[addotherplugin.ps1] [WARNING] $sourcePom does not exist. Abort." + Write-Output "[addotherplugin.ps1] [WARNING] $sourcePom does not exist. Abort." Print-Usage return } @@ -84,7 +84,14 @@ $otherPlugin = [Io.File]::ReadAllText($sourcePom) $tempPom = "$temp\$targetPom" Replace-VariableInFile $variable $otherPlugin $targetPom $tempPom -Copy-Item $tempPom $targetPom -force + +# +# enable "provided" to exclude run-time spark packages from uber package +# +$content = (Get-Content $tempPom) +$content = $content -replace '', 'provided' + +$content | Out-File $targetPom -force Write-Output "[addotherplugin.ps1] updated $targetPom"