From 5f5215e6a4e011c4ece07a0f425ce01657ebcf6c Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 13 Jun 2021 16:14:06 -0700 Subject: [PATCH] Support for Spark 2.4.8 and Spark 3.1.2 (#953) --- azure-pipelines.yml | 20 +++++++++++++++++++ .../spark/deploy/dotnet/DotnetRunner.scala | 2 +- .../spark/deploy/dotnet/DotnetRunner.scala | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 72741d51..ba1242e4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -271,6 +271,16 @@ stages: testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) + - version: '2.4.8' + jobOptions: + - pool: 'Hosted VS2017' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_2_4) + forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Windows_2_4) + - pool: 'Hosted Ubuntu 1604' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_2_4) + forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_2_4) - version: '3.0.0' jobOptions: - pool: 'Hosted VS2017' @@ -311,3 +321,13 @@ stages: testOptions: "" backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1) forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1) + - version: '3.1.2' + jobOptions: + - pool: 'Hosted VS2017' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) + forwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Windows_3_1) + - pool: 'Hosted Ubuntu 1604' + testOptions: "" + backwardCompatibleTestOptions: $(backwardCompatibleTestOptions_Linux_3_1) + forwardCompatibleTestOptions: $(forwardCompatibleTestOptions_Linux_3_1) diff --git a/src/scala/microsoft-spark-2-4/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala b/src/scala/microsoft-spark-2-4/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala index d0f69e5f..8cf73ddd 100644 --- a/src/scala/microsoft-spark-2-4/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala +++ b/src/scala/microsoft-spark-2-4/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala @@ -37,7 +37,7 @@ object DotnetRunner extends Logging { private val DEBUG_PORT = 5567 private val supportedSparkMajorMinorVersionPrefix = "2.4" private val supportedSparkVersions = - Set[String]("2.4.0", "2.4.1", "2.4.3", "2.4.4", "2.4.5", "2.4.6", "2.4.7") + Set[String]("2.4.0", "2.4.1", "2.4.3", "2.4.4", "2.4.5", "2.4.6", "2.4.7", "2.4.8") val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION) diff --git a/src/scala/microsoft-spark-3-1/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala b/src/scala/microsoft-spark-3-1/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala index 17ec1f12..b7b2c988 100644 --- a/src/scala/microsoft-spark-3-1/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala +++ b/src/scala/microsoft-spark-3-1/src/main/scala/org/apache/spark/deploy/dotnet/DotnetRunner.scala @@ -36,7 +36,7 @@ import scala.util.Try object DotnetRunner extends Logging { private val DEBUG_PORT = 5567 private val supportedSparkMajorMinorVersionPrefix = "3.1" - private val supportedSparkVersions = Set[String]("3.1.1") + private val supportedSparkVersions = Set[String]("3.1.1", "3.1.2") val SPARK_VERSION = DotnetUtils.normalizeSparkVersion(spark.SPARK_VERSION)