support both python2/3

support both python2/3

VMExtension/HandlerManifest.json

@@ -2,11 +2,11 @@
   "name": "HPCNodeManager",
   "version": 1.0,
   "handlerManifest": {
-    "installCommand": "hpcnodemanager.py -install",
+    "installCommand": "./shim.sh -install",
-    "uninstallCommand": "hpcnodemanager.py -uninstall",
+    "uninstallCommand": "./shim.sh -uninstall",
-    "updateCommand": "hpcnodemanager.py -update",
+    "updateCommand": "./shim.sh -update",
-    "enableCommand": "hpcnodemanager.py -enable",
+    "enableCommand": "./shim.sh -enable",
-    "disableCommand": "hpcnodemanager.py -disable",
+    "disableCommand": "./shim.sh -disable",
     "rebootAfterInstall": false,
     "reportHeartbeat": false,
     "updateMode": "UpdateWithInstall"

VMExtension/Utils/HandlerUtil.py

@@ -1,4 +1,4 @@
-#
+#
 # Handler library for Linux IaaS
 #
 # Copyright 2014 Microsoft Corporation
@@ -14,8 +14,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Requires Python 2.7+
 
 
 """
@@ -53,7 +51,6 @@ Example Status Report:
 
 """
 
-
 import os
 import os.path
 import sys
@@ -61,15 +58,20 @@ import imp
 import base64
 import json
 import time
+import re
 
+from xml.etree import ElementTree
 from os.path import join
 from Utils.WAAgentUtil import waagent
 from waagent import LoggerInit
 
 DateTimeFormat = "%Y-%m-%dT%H:%M:%SZ"
 
+MANIFEST_XML = "manifest.xml"
+
+
 class HandlerContext:
-    def __init__(self,name):
+    def __init__(self, name):
         self._name = name
         self._version = '0.0'
         self._config_dir = None
@@ -83,14 +85,47 @@ class HandlerContext:
         self._config = None
         return
 
+
 class HandlerUtility:
-    def __init__(self, log, error, short_name):
+    def __init__(self, log, error, s_name=None, l_name=None, extension_version=None, logFileName='extension.log',
+                 console_logger=None, file_logger=None):
         self._log = log
+        self._log_to_con = console_logger
+        self._log_to_file = file_logger
         self._error = error
-        self._short_name = short_name
+        self._logFileName = logFileName
+        if s_name is None or l_name is None or extension_version is None:
+            (l_name, s_name, extension_version) = self._get_extension_info()
+
+        self._short_name = s_name
+        self._extension_version = extension_version
+        self._log_prefix = '[%s-%s] ' % (l_name, extension_version)
+
+    def get_extension_version(self):
+        return self._extension_version
 
     def _get_log_prefix(self):
-        return '[%s-%s]' %(self._context._name, self._context._version)
+        return self._log_prefix
+
+    def _get_extension_info(self):
+        if os.path.isfile(MANIFEST_XML):
+            return self._get_extension_info_manifest()
+
+        ext_dir = os.path.basename(os.getcwd())
+        (long_name, version) = ext_dir.split('-')
+        short_name = long_name.split('.')[-1]
+
+        return long_name, short_name, version
+
+    def _get_extension_info_manifest(self):
+        with open(MANIFEST_XML) as fh:
+            doc = ElementTree.parse(fh)
+            namespace = doc.find('{http://schemas.microsoft.com/windowsazure}ProviderNameSpace').text
+            short_name = doc.find('{http://schemas.microsoft.com/windowsazure}Type').text
+            version = doc.find('{http://schemas.microsoft.com/windowsazure}Version').text
+
+            long_name = "%s.%s" % (namespace, short_name)
+            return (long_name, short_name, version)
 
     def _get_current_seq_no(self, config_folder):
         seq_no = -1
@@ -100,13 +135,13 @@ class HandlerUtility:
             for file in files:
                 try:
                     cur_seq_no = int(os.path.basename(file).split('.')[0])
-                    if(freshest_time == None):
+                    if (freshest_time == None):
-                        freshest_time = os.path.getmtime(join(config_folder,file))
+                        freshest_time = os.path.getmtime(join(config_folder, file))
                         seq_no = cur_seq_no
                     else:
-                        current_file_m_time = os.path.getmtime(join(config_folder,file))
+                        current_file_m_time = os.path.getmtime(join(config_folder, file))
-                        if(current_file_m_time > freshest_time):
+                        if (current_file_m_time > freshest_time):
-                            freshest_time=current_file_m_time
+                            freshest_time = current_file_m_time
                             seq_no = cur_seq_no
                 except ValueError:
                     continue
@@ -115,69 +150,87 @@ class HandlerUtility:
     def log(self, message):
         self._log(self._get_log_prefix() + message)
 
+    def log_to_console(self, message):
+        if self._log_to_con is not None:
+            self._log_to_con(self._get_log_prefix() + message)
+        else:
+            self.error("Unable to log to console, console log method not set")
+
+    def log_to_file(self, message):
+        if self._log_to_file is not None:
+            self._log_to_file(self._get_log_prefix() + message)
+        else:
+            self.error("Unable to log to file, file log method not set")
+
     def error(self, message):
         self._error(self._get_log_prefix() + message)
 
+    @staticmethod
+    def redact_protected_settings(content):
+        redacted_tmp = re.sub('"protectedSettings":\s*"[^"]+=="', '"protectedSettings": "*** REDACTED ***"', content)
+        redacted = re.sub('"protectedSettingsCertThumbprint":\s*"[^"]+"', '"protectedSettingsCertThumbprint": "*** REDACTED ***"', redacted_tmp)
+        return redacted
+
     def _parse_config(self, ctxt):
         config = None
         try:
-            config=json.loads(ctxt)
+            config = json.loads(ctxt)
         except:
-            self.error('JSON exception decoding ' + ctxt)
+            self.error('JSON exception decoding ' + HandlerUtility.redact_protected_settings(ctxt))
 
-        if config == None:
+        if config is None:
-            self.error("JSON error processing settings file:" + ctxt)
+            self.error("JSON error processing settings file:" + HandlerUtility.redact_protected_settings(ctxt))
         else:
             handlerSettings = config['runtimeSettings'][0]['handlerSettings']
-            if handlerSettings.has_key('protectedSettings') and \
+            if 'protectedSettings' in handlerSettings and \
-                    handlerSettings.has_key("protectedSettingsCertThumbprint") and \
+                    'protectedSettingsCertThumbprint' in handlerSettings and \
                     handlerSettings['protectedSettings'] is not None and \
                     handlerSettings["protectedSettingsCertThumbprint"] is not None:
                 protectedSettings = handlerSettings['protectedSettings']
-                thumb=handlerSettings['protectedSettingsCertThumbprint']
+                thumb = handlerSettings['protectedSettingsCertThumbprint']
-                cert=waagent.LibDir+'/'+thumb+'.crt'
+                cert = waagent.LibDir + '/' + thumb + '.crt'
-                pkey=waagent.LibDir+'/'+thumb+'.prv'
+                pkey = waagent.LibDir + '/' + thumb + '.prv'
                 unencodedSettings = base64.standard_b64decode(protectedSettings)
                 openSSLcmd = "openssl smime -inform DER -decrypt -recip {0} -inkey {1}"
                 cleartxt = waagent.RunSendStdin(openSSLcmd.format(cert, pkey), unencodedSettings)[1]
-                if cleartxt == None:
+                if cleartxt is None:
-                    self.error("OpenSSh decode error using  thumbprint " + thumb )
+                    self.error("OpenSSL decode error using  thumbprint " + thumb)
-                    self.do_exit(1,operation,'error','1', 'Failed decrypting protectedSettings')
+                    self.do_exit(1, "Enable", 'error', '1', 'Failed to decrypt protectedSettings')
-                jctxt=''
+                jctxt = ''
                 try:
-                    jctxt=json.loads(cleartxt)
+                    jctxt = json.loads(cleartxt)
                 except:
-                    self.error('JSON exception decoding ' + cleartxt)
+                    self.error('JSON exception decoding ' + HandlerUtility.redact_protected_settings(cleartxt))
                 handlerSettings['protectedSettings']=jctxt
                 self.log('Config decoded correctly.')
         return config
 
-    def do_parse_context(self,operation):
+    def do_parse_context(self, operation):
         _context = self.try_parse_context()
         if not _context:
-            self.do_exit(1,operation,'error','1', operation + ' Failed')
+            self.do_exit(1, operation, 'error', '1', operation + ' Failed')
         return _context
 
     def try_parse_context(self):
         self._context = HandlerContext(self._short_name)
-        handler_env=None
+        handler_env = None
-        config=None
+        config = None
-        ctxt=None
+        ctxt = None
-        code=0
+        code = 0
         # get the HandlerEnvironment.json. According to the extension handler spec, it is always in the ./ directory
         self.log('cwd is ' + os.path.realpath(os.path.curdir))
-        handler_env_file='./HandlerEnvironment.json'
+        handler_env_file = './HandlerEnvironment.json'
         if not os.path.isfile(handler_env_file):
             self.error("Unable to locate " + handler_env_file)
             return None
         ctxt = waagent.GetFileContents(handler_env_file)
-        if ctxt == None :
+        if ctxt == None:
             self.error("Unable to read " + handler_env_file)
         try:
-            handler_env=json.loads(ctxt)
+            handler_env = json.loads(ctxt)
         except:
             pass
-        if handler_env == None :
+        if handler_env == None:
             self.log("JSON error processing " + handler_env_file)
             return None
         if type(handler_env) == list:
@@ -185,41 +238,41 @@ class HandlerUtility:
 
         self._context._name = handler_env['name']
         self._context._version = str(handler_env['version'])
-        self._context._config_dir=handler_env['handlerEnvironment']['configFolder']
+        self._context._config_dir = handler_env['handlerEnvironment']['configFolder']
-        self._context._log_dir= handler_env['handlerEnvironment']['logFolder']
+        self._context._log_dir = handler_env['handlerEnvironment']['logFolder']
-        self._context._log_file= os.path.join(handler_env['handlerEnvironment']['logFolder'],'extension.log')
+
+        self._context._log_file = os.path.join(handler_env['handlerEnvironment']['logFolder'], self._logFileName)
         self._change_log_file()
-        self._context._status_dir=handler_env['handlerEnvironment']['statusFolder']
+        self._context._status_dir = handler_env['handlerEnvironment']['statusFolder']
-        self._context._heartbeat_file=handler_env['handlerEnvironment']['heartbeatFile']
+        self._context._heartbeat_file = handler_env['handlerEnvironment']['heartbeatFile']
         self._context._seq_no = self._get_current_seq_no(self._context._config_dir)
         if self._context._seq_no < 0:
             self.error("Unable to locate a .settings file!")
             return None
         self._context._seq_no = str(self._context._seq_no)
         self.log('sequence number is ' + self._context._seq_no)
-        self._context._status_file= os.path.join(self._context._status_dir, self._context._seq_no +'.status')
+        self._context._status_file = os.path.join(self._context._status_dir, self._context._seq_no + '.status')
         self._context._settings_file = os.path.join(self._context._config_dir, self._context._seq_no + '.settings')
         self.log("setting file path is" + self._context._settings_file)
-        ctxt=None
+        ctxt = None
-        ctxt=waagent.GetFileContents(self._context._settings_file)
+        ctxt = waagent.GetFileContents(self._context._settings_file)
-        if ctxt == None :
+        if ctxt == None:
             error_msg = 'Unable to read ' + self._context._settings_file + '. '
             self.error(error_msg)
             return None
 
-        self.log("JSON config: " + ctxt)
+        self.log("JSON config: " + HandlerUtility.redact_protected_settings(ctxt))
         self._context._config = self._parse_config(ctxt)
         return self._context
 
-
     def _change_log_file(self):
         self.log("Change log file to " + self._context._log_file)
-        LoggerInit(self._context._log_file,'/dev/stdout')
+        LoggerInit(self._context._log_file, '/dev/stdout')
         self._log = waagent.Log
         self._error = waagent.Error
 
     def set_verbose_log(self, verbose):
-        if(verbose == "1" or verbose == 1):
+        if (verbose == "1" or verbose == 1):
             self.log("Enable verbose log")
             LoggerInit(self._context._log_file, '/dev/stdout', verbose=True)
         else:
@@ -233,19 +286,22 @@ class HandlerUtility:
         self._set_most_recent_seq(self._context._seq_no)
         self.log("set most recent sequence number to " + self._context._seq_no)
 
-    def exit_if_enabled(self):
+    def exit_if_enabled(self, remove_protected_settings=False):
-        self.exit_if_seq_smaller()
+        self.exit_if_seq_smaller(remove_protected_settings)
 
-    def exit_if_seq_smaller(self):
+    def exit_if_seq_smaller(self, remove_protected_settings):
         if(self.is_seq_smaller()):
-            self.log("Current sequence number, " + self._context._seq_no + ", is not greater than the sequnce number of the most recent executed configuration. Exiting...")
+            self.log("Current sequence number, " + self._context._seq_no + ", is not greater than the sequence number of the most recent executed configuration. Exiting...")
             sys.exit(0)
         self.save_seq()
 
+        if remove_protected_settings:
+            self.scrub_settings_file()
+
     def _get_most_recent_seq(self):
-        if(os.path.isfile('mrseq')):
+        if (os.path.isfile('mrseq')):
             seq = waagent.GetFileContents('mrseq')
-            if(seq):
+            if (seq):
                 return int(seq)
 
         return -1
@@ -256,47 +312,47 @@ class HandlerUtility:
     def get_inused_config_seq(self):
         return self._get_most_recent_seq()
 
-    def set_inused_config_seq(self,seq):
+    def set_inused_config_seq(self, seq):
         self._set_most_recent_seq(seq)
 
-    def _set_most_recent_seq(self,seq):
+    def _set_most_recent_seq(self, seq):
         waagent.SetFileContents('mrseq', str(seq))
 
     def do_status_report(self, operation, status, status_code, message):
         self.log("{0},{1},{2},{3}".format(operation, status, status_code, message))
-        tstamp=time.strftime(DateTimeFormat, time.gmtime())
+        tstamp = time.strftime(DateTimeFormat, time.gmtime())
         stat = [{
-            "version" : self._context._version,
+            "version": self._context._version,
-            "timestampUTC" : tstamp,
+            "timestampUTC": tstamp,
-            "status" : {
+            "status": {
-                "name" : self._context._name,
+                "name": self._context._name,
-                "operation" : operation,
+                "operation": operation,
-                "status" : status,
+                "status": status,
-                "code" : status_code,
+                "code": status_code,
-                "formattedMessage" : {
+                "formattedMessage": {
-                    "lang" : "en-US",
+                    "lang": "en-US",
-                    "message" : message
+                    "message": message
                 }
             }
         }]
         stat_rept = json.dumps(stat)
         if self._context._status_file:
-            tmp = "%s.tmp" %(self._context._status_file)
+            tmp = "%s.tmp" % (self._context._status_file)
-            with open(tmp,'w+') as f:
+            with open(tmp, 'w+') as f:
                 f.write(stat_rept)
             os.rename(tmp, self._context._status_file)
 
-    def do_heartbeat_report(self, heartbeat_file,status,code,message):
+    def do_heartbeat_report(self, heartbeat_file, status, code, message):
         # heartbeat
-        health_report='[{"version":"1.0","heartbeat":{"status":"' + status+ '","code":"'+ code + '","Message":"' + message + '"}}]'
+        health_report = '[{"version":"1.0","heartbeat":{"status":"' + status + '","code":"' + code + '","Message":"' + message + '"}}]'
-        if waagent.SetFileContents(heartbeat_file,health_report) == None :
+        if waagent.SetFileContents(heartbeat_file, health_report) == None:
             self.error('Unable to wite heartbeat info to ' + heartbeat_file)
 
-    def do_exit(self,exit_code,operation,status,code,message):
+    def do_exit(self, exit_code, operation, status, code, message):
         try:
-            self.do_status_report(operation, status,code,message)
+            self.do_status_report(operation, status, code, message)
         except Exception as e:
-            self.log("Can't update status: "+str(e))
+            self.log("Can't update status: " + str(e))
         sys.exit(exit_code)
 
     def get_name(self):
@@ -324,3 +380,8 @@ class HandlerUtility:
             return self.get_handler_settings().get('publicSettings')
         return None
 
+    def scrub_settings_file(self):
+        content = waagent.GetFileContents(self._context._settings_file)
+        redacted = HandlerUtility.redact_protected_settings(content)
+
+        waagent.SetFileContents(self._context._settings_file, redacted)

VMExtension/Utils/LogUtil.py

@@ -0,0 +1,50 @@
+# Logging utilities 
+#
+# Copyright 2014 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path
+import string
+import sys
+
+OutputSize = 4 * 1024
+
+
+def tail(log_file, output_size = OutputSize):
+    pos = min(output_size, os.path.getsize(log_file))
+    with open(log_file, "r") as log:
+        log.seek(0, os.SEEK_END)
+        log.seek(log.tell() - pos, os.SEEK_SET)
+        buf = log.read(output_size)
+        buf = filter(lambda x: x in string.printable, buf)
+
+        # encoding works different for between interpreter version, we are keeping separate implementation to ensure
+        # backward compatibility
+        if sys.version_info[0] == 3:
+            buf = ''.join(list(buf)).encode('ascii', 'ignore').decode("ascii", "ignore")
+        elif sys.version_info[0] == 2:
+            buf = buf.decode("ascii", "ignore")
+
+        return buf
+
+
+def get_formatted_log(summary, stdout, stderr):
+    msg_format = ("{0}\n"
+                  "---stdout---\n"
+                  "{1}\n"
+                  "---errout---\n"
+                  "{2}\n")
+    return msg_format.format(summary, stdout, stderr)

VMExtension/Utils/ScriptUtil.py

@@ -0,0 +1,140 @@
+# Script utilities
+#
+# Copyright 2014 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import os.path
+import time
+import subprocess
+import traceback
+import string
+import shlex
+import sys
+
+from Utils import LogUtil
+from Utils.WAAgentUtil import waagent
+
+DefaultStdoutFile = "stdout"
+DefaultErroutFile = "errout"
+
+
+def run_command(hutil, args, cwd, operation, extension_short_name, version, exit_after_run=True, interval=30,
+                std_out_file_name=DefaultStdoutFile, std_err_file_name=DefaultErroutFile):
+    std_out_file = os.path.join(cwd, std_out_file_name)
+    err_out_file = os.path.join(cwd, std_err_file_name)
+    std_out = None
+    err_out = None
+    try:
+        std_out = open(std_out_file, "w")
+        err_out = open(err_out_file, "w")
+        start_time = time.time()
+        child = subprocess.Popen(args,
+                                 cwd=cwd,
+                                 stdout=std_out,
+                                 stderr=err_out)
+        time.sleep(1)
+        while child.poll() is None:
+            msg = "Command is running..."
+            msg_with_cmd_output = LogUtil.get_formatted_log(msg, LogUtil.tail(std_out_file), LogUtil.tail(err_out_file))
+            msg_without_cmd_output = msg + " Stdout/Stderr omitted from output."
+
+            hutil.log_to_file(msg_with_cmd_output)
+            hutil.log_to_console(msg_without_cmd_output)
+            hutil.do_status_report(operation, 'transitioning', '0', msg_without_cmd_output)
+            time.sleep(interval)
+
+        exit_code = child.returncode
+        if child.returncode and child.returncode != 0:
+            msg = "Command returned an error."
+            msg_with_cmd_output = LogUtil.get_formatted_log(msg, LogUtil.tail(std_out_file), LogUtil.tail(err_out_file))
+            msg_without_cmd_output = msg + " Stdout/Stderr omitted from output."
+
+            hutil.error(msg_without_cmd_output)
+            waagent.AddExtensionEvent(name=extension_short_name,
+                                      op=operation,
+                                      isSuccess=False,
+                                      version=version,
+                                      message="(01302)" + msg_without_cmd_output)
+        else:
+            msg = "Command is finished."
+            msg_with_cmd_output = LogUtil.get_formatted_log(msg, LogUtil.tail(std_out_file), LogUtil.tail(err_out_file))
+            msg_without_cmd_output = msg + " Stdout/Stderr omitted from output."
+
+            hutil.log_to_file(msg_with_cmd_output)
+            hutil.log_to_console(msg_without_cmd_output)
+            waagent.AddExtensionEvent(name=extension_short_name,
+                                      op=operation,
+                                      isSuccess=True,
+                                      version=version,
+                                      message="(01302)" + msg_without_cmd_output)
+            end_time = time.time()
+            waagent.AddExtensionEvent(name=extension_short_name,
+                                      op=operation,
+                                      isSuccess=True,
+                                      version=version,
+                                      message=("(01304)Command execution time: "
+                                               "{0}s").format(str(end_time - start_time)))
+
+        log_or_exit(hutil, exit_after_run, exit_code, operation, msg_with_cmd_output)
+    except Exception as e:
+        error_msg = ("Failed to launch command with error: {0},"
+                     "stacktrace: {1}").format(e, traceback.format_exc())
+        hutil.error(error_msg)
+        waagent.AddExtensionEvent(name=extension_short_name,
+                                  op=operation,
+                                  isSuccess=False,
+                                  version=version,
+                                  message="(01101)" + error_msg)
+        exit_code = 1
+        msg = 'Launch command failed: {0}'.format(e)
+
+        log_or_exit(hutil, exit_after_run, exit_code, operation, msg)
+    finally:
+        if std_out:
+            std_out.close()
+        if err_out:
+            err_out.close()
+    return exit_code
+
+
+# do_exit calls sys.exit which raises an exception so we do not call it from the finally block
+def log_or_exit(hutil, exit_after_run, exit_code, operation, msg):
+    status = 'success' if exit_code == 0 else 'failed'
+    if exit_after_run:
+        hutil.do_exit(exit_code, operation, status, str(exit_code), msg)
+    else:
+        hutil.do_status_report(operation, status, str(exit_code), msg)
+
+
+def parse_args(cmd):
+    cmd = filter(lambda x: x in string.printable, cmd)
+
+    # encoding works different for between interpreter version, we are keeping separate implementation to ensure
+    # backward compatibility
+    if sys.version_info[0] == 3:
+        cmd = ''.join(list(cmd)).encode('ascii', 'ignore').decode("ascii", "ignore")
+    elif sys.version_info[0] == 2:
+        cmd = cmd.decode("ascii", "ignore")
+
+    args = shlex.split(cmd)
+    # From python 2.6 to python 2.7.2, shlex.split output UCS-4 result like
+    # '\x00\x00a'. Temp workaround is to replace \x00
+    for idx, val in enumerate(args):
+        if '\x00' in args[idx]:
+            args[idx] = args[idx].replace('\x00', '')
+    return args
+
+

VMExtension/Utils/WAAgentUtil.py

@@ -16,10 +16,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Requires Python 2.7+
-#
-
 
 import imp
 import os
@@ -74,11 +70,55 @@ if not hasattr(waagent, "WALAEventOperation"):
         Update = "Update"           
     waagent.WALAEventOperation = _WALAEventOperation
 
-__ExtensionName__=None
+# Better deal with the silly waagent typo, in anticipation of a proper fix of the typo later on waagent
+if not hasattr(waagent.WALAEventOperation, 'Uninstall'):
+    if hasattr(waagent.WALAEventOperation, 'UnIsntall'):
+        waagent.WALAEventOperation.Uninstall = waagent.WALAEventOperation.UnIsntall
+    else:  # This shouldn't happen, but just in case...
+        waagent.WALAEventOperation.Uninstall = 'Uninstall'
+
+
+def GetWaagentHttpProxyConfigString():
+    """
+    Get http_proxy and https_proxy from waagent config.
+    Username and password is not supported now.
+    This code is adopted from /usr/sbin/waagent
+    """
+    host = None
+    port = None
+    try:
+        waagent.Config = waagent.ConfigurationProvider(None)  # Use default waagent conf file (most likely /etc/waagent.conf)
+
+        host = waagent.Config.get("HttpProxy.Host")
+        port = waagent.Config.get("HttpProxy.Port")
+    except Exception as e:
+        # waagent.ConfigurationProvider(None) will throw an exception on an old waagent
+        # Has to silently swallow because logging is not yet available here
+        # and we don't want to bring that in here. Also if the call fails, then there's
+        # no proxy config in waagent.conf anyway, so it's safe to silently swallow.
+        pass
+
+    result = ''
+    if host is not None:
+        result = "http://" + host
+        if port is not None:
+            result += ":" + port
+
+    return result
+
+
+waagent.HttpProxyConfigString = GetWaagentHttpProxyConfigString()
+
+# end: waagent http proxy config stuff
+
+__ExtensionName__ = None
+
+
 def InitExtensionEventLog(name):
     global __ExtensionName__
     __ExtensionName__ = name
 
+
 def AddExtensionEvent(name=__ExtensionName__,
                       op=waagent.WALAEventOperation.Enable, 
                       isSuccess=False, 

VMExtension/Utils/__init__.py

@@ -12,8 +12,5 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Requires Python 2.7+
-#
 
 

VMExtension/hpcnodemanager.py

@@ -15,10 +15,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Requires Python 2.7+
-
-
 import os
 import sys
 import json
@@ -47,10 +43,10 @@ RestartIntervalInSeconds = 60
 
 def main():
     waagent.LoggerInit('/var/log/waagent.log','/dev/stdout')
-    waagent.Log("%s started to handle." %(ExtensionShortName))
+    waagent.Log('Microsoft.HpcPack Linux NodeAgent started to handle.')
     waagent.MyDistro = waagent.GetMyDistro()
     global DistroName, DistroVersion
-    distro = platform.dist()
+    distro = get_dist_info()
     DistroName = distro[0].lower()
     DistroVersion = distro[1]
     for a in sys.argv[1:]:        
@@ -78,7 +74,7 @@ def _is_nodemanager_daemon(pid):
     return False
 
 def install_package(package_name):
-    if DistroName == "centos" or DistroName == "redhat":
+    if DistroName in ["centos", "redhat", "alma", "rocky"]:
         cmd = "yum -y install " + package_name
     elif DistroName == "ubuntu":
         waagent.Log("Updating apt package lists with command: apt-get -y update")
@@ -271,7 +267,7 @@ def _update_dns_record(domain_fqdn):
         try:
             s.connect((domain_fqdn, 53))
             break
-        except Exception, e:
+        except Exception as e:
             waagent.Log('Failed to connect to {0}:53: {1}'.format(domain_fqdn, e))
     ipaddr = s.getsockname()[0]
     host_fqdn = "{0}.{1}".format(socket.gethostname().split('.')[0], domain_fqdn)
@@ -338,7 +334,7 @@ def install():
     try:
         cleanup_host_entries()
         _uninstall_nodemanager_files()
-        if DistroName == "centos" or DistroName == "redhat":
+        if DistroName in ["centos", "redhat", "alma", "rocky"]:
             waagent.Run("yum-config-manager --setopt=\\*.skip_if_unavailable=1 --save", chk_err=False)
         _install_cgroup_tool()
         _install_sysstat()
@@ -459,7 +455,7 @@ def install():
         shutil.copy2(configfile, backup_configfile)
         config_firewall_rules()
         hutil.do_exit(0, 'Install', 'success', '0', 'Install Succeeded.')
-    except Exception, e:
+    except Exception as e:
         hutil.do_exit(1, 'Install','error','1', '{0}'.format(e))
 
 def enable():
@@ -477,7 +473,7 @@ def enable():
                 os.killpg(int(pid), 9)
         os.remove(DaemonPidFilePath)
 
-    args = [os.path.join(os.getcwd(), __file__), "daemon"]
+    args = [get_python_executor(), os.path.join(os.getcwd(), __file__), "daemon"]
     devnull = open(os.devnull, 'w')
     child = subprocess.Popen(args, stdout=devnull, stderr=devnull, preexec_fn=os.setsid)
     if child.pid is None or child.pid < 1:
@@ -569,7 +565,7 @@ def daemon():
             waagent.Log("Restart HPC node manager process after {0} seconds".format(RestartIntervalInSeconds))
             time.sleep(RestartIntervalInSeconds)
 
-    except Exception, e:
+    except Exception as e:
         hutil.error("Failed to enable the extension with error: %s, stack trace: %s" %(str(e), traceback.format_exc()))
         hutil.do_exit(1, 'Enable','error','1', 'Enable failed.')
 
@@ -616,6 +612,57 @@ def update():
                     waagent.MyDistro.publishHostname(confighostname)
     hutil.do_exit(0,'Update','success','0', 'Update Succeeded')
 
+def get_python_executor():
+    cmd = ''
+    if sys.version_info.major == 2:
+        cmd = 'python2'
+    elif sys.version_info.major == 3:
+        cmd = 'python3'
+    if waagent.Run("command -v {0}".format(cmd), chk_err=False) != 0:
+        # If a user-installed python isn't available, check for a platform-python. This is typically only used in RHEL 8.0.
+        if waagent.Run("command -v /usr/libexec/platform-python", chk_err=False) == 0:
+            cmd = '/usr/libexec/platform-python'
+    return cmd
+
+def get_dist_info():
+    try:
+        return waagent.DistInfo()
+    except:
+        pass
+    errCode, info = waagent.RunGetOutput("cat /etc/*-release")
+    if errCode != 0:
+        raise Exception('Failed to get Linux Distro info by running command "cat /etc/*release", error code: {}'.format(errCode))
+    distroName = ''
+    distroVersion = ''
+    for line in info.splitlines():
+        if line.startswith('PRETTY_NAME='):
+            line = line.lower()
+            if 'ubuntu' in line:
+                distroName = 'ubuntu'
+            elif 'centos' in line:
+                distroName = 'centos'
+            elif 'red hat' in line:
+                distroName = 'redhat'
+            elif 'suse' in line:
+                distroName = 'suse'
+            elif 'alma' in line:
+                distroName = 'alma'
+            elif 'rocky' in line:
+                distroName = 'rocky'
+            elif 'fedora' in line:
+                distroName = 'fedora'
+            elif 'freebsd' in line:
+                distroName = 'freebsd'
+            else:
+                raise Exception('Unknown linux distribution with {}'.format(line))
+        if line.startswith('VERSION_ID='):
+            line = line.strip(' ')
+            quoteIndex = line.index('"')
+            if quoteIndex >= 0:
+                distroVersion = line[quoteIndex+1:-1]
+    return distroName, distroVersion, ""
+
+  
 if __name__ == '__main__' :
     main()
 

VMExtension/shim.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# This is the main driver file for HPC Pack Linux NodeAgent extension. This file first checks if Python 3 or 2 is available on the VM 
+# and if yes then uses that Python (if both are available then, default is set to python3) to run extension operations in hpcnodemanager.py
+# Control arguments passed to the shim are redirected to hpcnodemanager.py without validation.
+
+COMMAND="./hpcnodemanager.py"
+PYTHON=""
+ARG="$@"
+
+function find_python() {
+    local python_exec_command=$1
+
+    if command -v python3 >/dev/null 2>&1 ; then
+        eval ${python_exec_command}="python3"
+    elif command -v python2 >/dev/null 2>&1 ; then
+        eval ${python_exec_command}="python2"
+    elif command -v /usr/libexec/platform-python >/dev/null 2>&1 ; then
+        # If a user-installed python isn't available, check for a platform-python. This is typically only used in RHEL 8.0.
+        echo "User-installed python not found. Using /usr/libexec/platform-python as the python interpreter."
+        eval ${python_exec_command}="/usr/libexec/platform-python"
+    fi
+}
+
+find_python PYTHON
+
+if [ -z "$PYTHON" ] # If python is not installed, we will fail the install with the following error, requiring cx to have python pre-installed
+then
+    echo "No Python interpreter found. Please install Python 3, or Python 2 if the former is unavailable." >&2
+    exit 52 # Missing Dependency
+else
+    ${PYTHON} --version 2>&1
+fi
+
+${PYTHON} ${COMMAND} ${ARG}
+exit $?