From 8028efc9056ea2d2549a0837a48c26bb86940cbb Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Fri, 2 Sep 2022 15:19:40 +0000 Subject: [PATCH 1/7] [wip] add ability to use variables in config dict keys. needs tests --- pyazhpc/azconfig.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pyazhpc/azconfig.py b/pyazhpc/azconfig.py index f0dec735..b19b4746 100644 --- a/pyazhpc/azconfig.py +++ b/pyazhpc/azconfig.py @@ -39,8 +39,22 @@ class ConfigFile: def __evaluate_dict(self, x, extended): ret = {} + updated_keys = [] for k in x.keys(): - ret[k] = self.__evaluate(x[k], extended) + # Update key names if config variable (i.e. {{variables.key}} ) used + if "variables." in k: + log.debug(f"expanding key {k}") + new_key = self.__evaluate(k, extended) + ret[new_key] = self.__evaluate(x[k], extended) + updated_keys.append(k) + else: + ret[k] = self.__evaluate(x[k], extended) + + # Delete keys from dict that were updated + # - Not doing so can result in duplicate fields in deploy_*json + if updated_keys: + for old_key in updated_keys: + del x[old_key] return ret def __evaluate_list(self, x, extended): From 66e3e2f87e22143f086a93a1b3e3fff7647eabfb Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Fri, 2 Sep 2022 23:17:19 +0000 Subject: [PATCH 2/7] [wip] fix to not destroy config upon initial preprocess --- pyazhpc/azconfig.py | 4 +++- pyazhpc/azhpc.py | 10 +++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pyazhpc/azconfig.py b/pyazhpc/azconfig.py index b19b4746..e489824e 100644 --- a/pyazhpc/azconfig.py +++ b/pyazhpc/azconfig.py @@ -1,3 +1,4 @@ +import copy import json import os import re @@ -78,7 +79,8 @@ class ConfigFile: return input def preprocess(self, extended=True): - res = self.__evaluate(self.data, extended) + # copy of dict required + res = self.__evaluate(copy.deepcopy(self.data), extended) return res def read_keys(self, v): diff --git a/pyazhpc/azhpc.py b/pyazhpc/azhpc.py index 9432ae2d..86c1dd17 100644 --- a/pyazhpc/azhpc.py +++ b/pyazhpc/azhpc.py @@ -1,4 +1,5 @@ import argparse +import copy import datetime import json import os @@ -8,7 +9,10 @@ import socket import sys import textwrap import time -import copy + +from cryptography.hazmat.backends import default_backend as crypto_default_backend +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric import rsa import arm import azconfig @@ -16,10 +20,6 @@ import azinstall import azlog import azutil -from cryptography.hazmat.primitives import serialization as crypto_serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.hazmat.backends import default_backend as crypto_default_backend - log = azlog.getLogger(__name__) def do_preprocess(args): From 9bb50256dc1c0280cae6a1dea12d69266994cfc6 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Mon, 12 Sep 2022 18:51:51 +0000 Subject: [PATCH 3/7] localize deep-copy of config dict --- pyazhpc/azconfig.py | 23 ++++++++++++----------- pyazhpc/azhpc.py | 14 +++++++------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/pyazhpc/azconfig.py b/pyazhpc/azconfig.py index e489824e..afa7c10f 100644 --- a/pyazhpc/azconfig.py +++ b/pyazhpc/azconfig.py @@ -22,7 +22,7 @@ class ConfigFile: self.file_location = "." with open(fname) as f: self.data = json.load(f) - + def save(self, fname): with open(fname, "w") as f: json.dump(self.data, f, indent=4) @@ -37,8 +37,10 @@ class ConfigFile: dest = azutil.get_vm_private_ip(self.read_value("resource_group"), install_from) log.debug(f"install_from destination : {dest}") return dest - + def __evaluate_dict(self, x, extended): + # Create deep copy of dict to avoid unintentionally deleting items if variables are used in keys + x = copy.deepcopy(x) ret = {} updated_keys = [] for k in x.keys(): @@ -79,8 +81,7 @@ class ConfigFile: return input def preprocess(self, extended=True): - # copy of dict required - res = self.__evaluate(copy.deepcopy(self.data), extended) + res = self.__evaluate(self.data, extended) return res def read_keys(self, v): @@ -93,10 +94,10 @@ class ConfigFile: except KeyError: log.error("read_keys : "+v+" not in config") sys.exit(1) - + if type(it) is not dict: log.error("read_keys : "+v+" is not a dict") - + keys = list(it.keys()) log.debug("read_keys (exit): keys("+v+")="+",".join(keys)) return keys @@ -116,7 +117,7 @@ class ConfigFile: else: log.error("invalid path in config file ({v})") it = it[x] - + if type(it) is str: res = self.process_value(it) else: @@ -124,7 +125,7 @@ class ConfigFile: except KeyError: log.debug(f"using default value ({default})") res = default - + log.debug("read_value (exit): "+v+"="+str(res)) return res @@ -134,9 +135,9 @@ class ConfigFile: def repl(match): return str(self.process_value(match.group()[2:-2], extended)) - + v = self.regex.sub(lambda m: str(self.process_value(m.group()[2:-2], extended)), v) - + parts = v.split('.') prefix = parts[0] if len(parts) == 1: @@ -202,6 +203,6 @@ class ConfigFile: res = f.read() else: res = v - + log.debug("process_value (exit): "+str(v)+"="+str(res)) return res diff --git a/pyazhpc/azhpc.py b/pyazhpc/azhpc.py index 86c1dd17..38e8c4db 100644 --- a/pyazhpc/azhpc.py +++ b/pyazhpc/azhpc.py @@ -203,7 +203,7 @@ def do_connect(args): sshuser = adminuser else: sshuser = args.user - + sshport = c.read_value("ssh_port", 22) jumpbox = c.read_value("install_from") @@ -470,7 +470,7 @@ def _wait_for_deployment(resource_group, deploy_name): elif isinstance(value, list): wrapped_print(indent, str(key)) pretty_print(value, indent+4) - else: + else: wrapped_print(indent, f"{key}: {value}") else: wrapped_print(indent, str(d)) @@ -565,8 +565,8 @@ def do_slurm_resume(args): # first get the resource name resource_names, resource_list = _nodelist_expand(args.nodes) - # Create a copy of the configuration to use as template - # for the final deployment configuration + # Create a copy of the configuration to use as template + # for the final deployment configuration config = copy.deepcopy(config_orig) config["resources"] = {} @@ -716,7 +716,7 @@ def do_run_install(args): if os.path.isdir(tmpdir): log.debug("removing existing tmp directory") shutil.rmtree(tmpdir) - + adminuser = config["admin_user"] private_key_file = adminuser+"_id_rsa" public_key_file = adminuser+"_id_rsa.pub" @@ -897,7 +897,7 @@ if __name__ == "__main__": preprocess_parser.set_defaults(func=do_preprocess) run_parser = subparsers.add_parser( - "run", + "run", parents=[gopt_parser], add_help=False, description="run a command on the specified resources", @@ -923,7 +923,7 @@ if __name__ == "__main__": ) scp_parser = subparsers.add_parser( - "scp", + "scp", parents=[gopt_parser], add_help=False, description="secure copy", From 19996b3e068263f5691c99637bbf7804b20a0d7b Mon Sep 17 00:00:00 2001 From: Paul Edwards Date: Tue, 13 Sep 2022 19:50:38 +0100 Subject: [PATCH 4/7] test failing --- pyazhpc/test/test_azconfig.py | 3 +++ pyazhpc/test/test_config_file.json | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pyazhpc/test/test_azconfig.py b/pyazhpc/test/test_azconfig.py index 053b4998..6951d01d 100644 --- a/pyazhpc/test/test_azconfig.py +++ b/pyazhpc/test/test_azconfig.py @@ -20,5 +20,8 @@ class TestConfigFile(unittest.TestCase): def test_replace_double_curly_braces(self): self.assertEqual(self.config.read_value("double_curly_braces"), "simple_variable=42") + def test_replace_in_dict_key(self): + self.assertEqual(self.config.read_value("resources.foo"), "bar") + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/pyazhpc/test/test_config_file.json b/pyazhpc/test/test_config_file.json index 73fb258c..d0092597 100644 --- a/pyazhpc/test/test_config_file.json +++ b/pyazhpc/test/test_config_file.json @@ -4,6 +4,10 @@ "use_variable": "variables.simple_variable", "double_curly_braces": "simple_variable={{variables.simple_variable}}", "variables": { - "simple_variable": 42 + "simple_variable": 42, + "resource_name": "foo" + }, + "resources": { + "variables.resource_name": "bar" } } \ No newline at end of file From a8595e097acec5076b9e79b2443d779c99ae3334 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Tue, 13 Sep 2022 23:44:41 +0000 Subject: [PATCH 5/7] Changes to test to test variable expansion --- pyazhpc/test/test_azconfig.py | 27 +++++++++++++++++++++++---- pyazhpc/test/test_config_file.json | 6 ++++-- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pyazhpc/test/test_azconfig.py b/pyazhpc/test/test_azconfig.py index 6951d01d..180ab0bc 100644 --- a/pyazhpc/test/test_azconfig.py +++ b/pyazhpc/test/test_azconfig.py @@ -2,15 +2,17 @@ import unittest import azconfig + class TestConfigFile(unittest.TestCase): def setUp(self): self.config = azconfig.ConfigFile() self.config.open("test/test_config_file.json") + self.config_preprocessed = self.config.preprocess() def test_read_simple_value(self): self.assertEqual(self.config.read_value("int_value"), 42) - + def test_read_boolean_value(self): self.assertEqual(self.config.read_value("bool_value"), True) @@ -20,8 +22,25 @@ class TestConfigFile(unittest.TestCase): def test_replace_double_curly_braces(self): self.assertEqual(self.config.read_value("double_curly_braces"), "simple_variable=42") - def test_replace_in_dict_key(self): - self.assertEqual(self.config.read_value("resources.foo"), "bar") + # Before preprocessing, config fields using variable name definitions are not expanded + def test_replace_in_dict_key_before_preprocessing(self): + self.assertEqual( + self.config.read_value("resources")["variables.resource_name"], + "bar" + ) + + def test_replace_in_dict_value_before_preprocessing(self): + self.assertEqual( + self.config.read_value("resources")["resource-name"], + "variables.resource_type" + ) + + # After preprocessing, config fields using variable name definitions are expanded + def test_replace_in_dict_key_after_preprocessing(self): + self.assertEqual(self.config_preprocessed["resources"]["foo"], "bar") + + def test_replace_in_dict_value_after_preprocessing(self): + self.assertEqual(self.config_preprocessed["resources"]["resource-name"], "resource-type") if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/pyazhpc/test/test_config_file.json b/pyazhpc/test/test_config_file.json index d0092597..806ba520 100644 --- a/pyazhpc/test/test_config_file.json +++ b/pyazhpc/test/test_config_file.json @@ -5,9 +5,11 @@ "double_curly_braces": "simple_variable={{variables.simple_variable}}", "variables": { "simple_variable": 42, - "resource_name": "foo" + "resource_name": "foo", + "resource_type": "resource-type" }, "resources": { - "variables.resource_name": "bar" + "variables.resource_name": "bar", + "resource-name": "variables.resource_type" } } \ No newline at end of file From fc173aba4589fcec6761a5be861bdf76acb60ae3 Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Thu, 15 Sep 2022 15:50:23 +0000 Subject: [PATCH 6/7] Revert "Changes to test to test variable expansion" This reverts commit a8595e097acec5076b9e79b2443d779c99ae3334. --- pyazhpc/test/test_azconfig.py | 27 ++++----------------------- pyazhpc/test/test_config_file.json | 6 ++---- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/pyazhpc/test/test_azconfig.py b/pyazhpc/test/test_azconfig.py index 180ab0bc..6951d01d 100644 --- a/pyazhpc/test/test_azconfig.py +++ b/pyazhpc/test/test_azconfig.py @@ -2,17 +2,15 @@ import unittest import azconfig - class TestConfigFile(unittest.TestCase): def setUp(self): self.config = azconfig.ConfigFile() self.config.open("test/test_config_file.json") - self.config_preprocessed = self.config.preprocess() def test_read_simple_value(self): self.assertEqual(self.config.read_value("int_value"), 42) - + def test_read_boolean_value(self): self.assertEqual(self.config.read_value("bool_value"), True) @@ -22,25 +20,8 @@ class TestConfigFile(unittest.TestCase): def test_replace_double_curly_braces(self): self.assertEqual(self.config.read_value("double_curly_braces"), "simple_variable=42") - # Before preprocessing, config fields using variable name definitions are not expanded - def test_replace_in_dict_key_before_preprocessing(self): - self.assertEqual( - self.config.read_value("resources")["variables.resource_name"], - "bar" - ) - - def test_replace_in_dict_value_before_preprocessing(self): - self.assertEqual( - self.config.read_value("resources")["resource-name"], - "variables.resource_type" - ) - - # After preprocessing, config fields using variable name definitions are expanded - def test_replace_in_dict_key_after_preprocessing(self): - self.assertEqual(self.config_preprocessed["resources"]["foo"], "bar") - - def test_replace_in_dict_value_after_preprocessing(self): - self.assertEqual(self.config_preprocessed["resources"]["resource-name"], "resource-type") + def test_replace_in_dict_key(self): + self.assertEqual(self.config.read_value("resources.foo"), "bar") if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/pyazhpc/test/test_config_file.json b/pyazhpc/test/test_config_file.json index 806ba520..d0092597 100644 --- a/pyazhpc/test/test_config_file.json +++ b/pyazhpc/test/test_config_file.json @@ -5,11 +5,9 @@ "double_curly_braces": "simple_variable={{variables.simple_variable}}", "variables": { "simple_variable": 42, - "resource_name": "foo", - "resource_type": "resource-type" + "resource_name": "foo" }, "resources": { - "variables.resource_name": "bar", - "resource-name": "variables.resource_type" + "variables.resource_name": "bar" } } \ No newline at end of file From 83520aecbf72f3dc1b4c4eced413e290910809fd Mon Sep 17 00:00:00 2001 From: Jesse Lopez Date: Thu, 15 Sep 2022 15:58:32 +0000 Subject: [PATCH 7/7] Add preprocessing as default behavior for opening config to support key vars --- pyazhpc/azconfig.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyazhpc/azconfig.py b/pyazhpc/azconfig.py index afa7c10f..592b9240 100644 --- a/pyazhpc/azconfig.py +++ b/pyazhpc/azconfig.py @@ -15,7 +15,7 @@ class ConfigFile: self.data = {} self.regex = re.compile(r'({{([^{}]*)}})') - def open(self, fname): + def open(self, fname, preprocess=True, preprocess_extended=False): log.debug("opening "+fname) self.file_location = os.path.dirname(fname) if self.file_location == "": @@ -23,6 +23,9 @@ class ConfigFile: with open(fname) as f: self.data = json.load(f) + if preprocess: + self.data = self.preprocess(extended=preprocess_extended) + def save(self, fname): with open(fname, "w") as f: json.dump(self.data, f, indent=4)