remove hadoop-version in service config

cluster-configuration/services-configuration.yaml

@@ -47,7 +47,6 @@ hadoop:
   # More about hadoop-ai please follow the link: https://github.com/Microsoft/pai/tree/master/hadoop-ai.
   # Notice: the name should be hadoop-{hadoop-version}.tar.gz
   custom-hadoop-binary-path: /pathHadoop/hadoop-2.9.0.tar.gz
-  hadoop-version: 2.9.0
   # Step 1 of 4 to set up Hadoop queues.
   # Define all virtual clusters, equivalent concept of Hadoop queues:
   #   - Each VC will be assigned with (capacity / total_capacity * 100%) of the resources in the system.

pai-management/paiLibrary/clusterObjectModel/paiObjectModel.py

@@ -205,8 +205,6 @@ class paiObjectModel:
             self.rawData["serviceConfiguration"]["hadoop"]
         serviceDict["clusterinfo"]["hadoopinfo"]["custom_hadoop_binary_path"] = \
             serviceDict["clusterinfo"]["hadoopinfo"]["custom-hadoop-binary-path"]
-        serviceDict["clusterinfo"]["hadoopinfo"]["hadoopversion"] = \
-            serviceDict["clusterinfo"]["hadoopinfo"]["hadoop-version"]
         serviceDict["clusterinfo"]["hadoopinfo"]["configmapname"] = "hadoop-configuration"
         serviceDict["clusterinfo"]["hadoopinfo"]["hadoop_vip"] = \
             serviceDict["clusterinfo"]["hadoopinfo"]["hadoop-version"] = self.getMasterIP()

pai-management/paiLibrary/paiBuild/build_center.py

@@ -78,7 +78,6 @@ class build_center:
     def hadoop_binary_prepare(self):
 
         custom_hadoop_path = self.cluster_object_model['clusterinfo']['hadoopinfo']['custom_hadoop_binary_path']
-        hadoop_version = self.cluster_object_model['clusterinfo']['hadoopinfo']['hadoopversion']
 
         self.logger.info("Begin to prepare the hadoop binary for image building.")
 
@@ -89,11 +88,8 @@ class build_center:
             self.logger.info("Customized hadoop path is found.")
             directory_handler.directory_copy(custom_hadoop_path, "src/hadoop-run/hadoop")
         else:
-            self.logger.info("None customized hadoop path is found. An official hadoop will be downloaded.")
-            url = "http://archive.apache.org/dist/hadoop/common/{0}/{0}.tar.gz".format("hadoop-" + hadoop_version)
-            shell_cmd = "wget {0} -P src/hadoop-run/hadoop".format(url)
-            error_msg = "failed to wget hadoop binary"
-            linux_shell.execute_shell(shell_cmd, error_msg)
+            self.logger.info("None customized hadoop path is found.")
+            raise Exception("Hadoop-ai path not found")
 
         self.logger.info("Hadoop binary is prepared.")
 
@@ -148,10 +144,8 @@ class build_center:
 
 
     def hadoop_ai_build(self):
-
-        hadoop_version = self.cluster_object_model['clusterinfo']['hadoopinfo']['hadoopversion']
         hadoop_ai_path = self.cluster_object_model['clusterinfo']['hadoopinfo']['custom_hadoop_binary_path']
-        hadoop_ai_build_instance = hadoop_ai_build.hadoop_ai_build(self.os_type, hadoop_version, hadoop_ai_path)
+        hadoop_ai_build_instance = hadoop_ai_build.hadoop_ai_build(self.os_type, hadoop_ai_path)
         hadoop_ai_build_instance.build()
 
 

pai-management/paiLibrary/paiBuild/hadoop_ai_build.py

@@ -24,11 +24,10 @@ from ..common import linux_shell
 
 class hadoop_ai_build:
 
-    def __init__(self, os_type = "ubuntu16.04", hadoop_version = "2.9.0", hadoop_customize_path = None):
+    def __init__(self, os_type = "ubuntu16.04", hadoop_customize_path = None):
 
         self.logger = logging.getLogger(__name__)
 
-        self.hadoop_version = hadoop_version
         self.hadoop_customize_path = hadoop_customize_path
         self.os_type = os_type