Update collections of JS modules in one proposal (#1557)

* Update collections of JS modules in one proposal.
Fixes #1479.

* formatting

Co-authored-by: Amaury Chamayou <amchamay@microsoft.com>

python/ccf/proposal_generator.py

@@ -5,6 +5,7 @@ import argparse
 import collections
 import inspect
 import json
+import glob
 import os
 import sys
 from pathlib import PurePosixPath
@@ -294,6 +295,40 @@ def remove_module(module_name: str, **kwargs):
     return build_proposal("remove_module", module_name, **kwargs)
 
 
+@cli_proposal
+def update_modules(module_name_prefix: str, modules_path: Optional[str], **kwargs):
+    LOG.debug("Generating update_modules proposal")
+
+    # Validate module name prefix
+    module_name_prefix_ = PurePosixPath(module_name_prefix)
+    if not module_name_prefix_.is_absolute():
+        raise ValueError("module name prefix must be an absolute path")
+    if any(folder in [".", ".."] for folder in module_name_prefix_.parents):
+        raise ValueError("module name prefix must not contain . or .. components")
+    if not module_name_prefix.endswith("/"):
+        raise ValueError("module name prefix must end with /")
+
+    # Read module files and build relative module names
+    modules = []
+    if modules_path:
+        for path in glob.glob(f"{modules_path}/**/*.js", recursive=True):
+            rel_module_name = os.path.relpath(path, modules_path)
+            rel_module_name = rel_module_name.replace("\\", "/")  # Windows support
+            with open(path) as f:
+                js = f.read()
+                modules.append({"rel_name": rel_module_name, "module": {"js": js}})
+
+    proposal_args = {"prefix": module_name_prefix, "modules": modules}
+
+    return build_proposal("update_modules", proposal_args, **kwargs)
+
+
+@cli_proposal
+def remove_modules(module_name_prefix: str, **kwargs):
+    LOG.debug("Generating update_modules proposal (remove only)")
+    return update_modules(module_name_prefix, modules_path=None)
+
+
 @cli_proposal
 def trust_node(node_id: int, **kwargs):
     return build_proposal("trust_node", node_id, **kwargs)

src/runtime_config/gov.lua

@@ -97,4 +97,27 @@ return {
     end
   end
   return true]],
+
+  update_modules = [[
+  tables, args = ...
+  function starts_with(str, start)
+    return str:sub(1, #start) == start
+  end
+  function remove_modules_with_prefix(prefix)
+    tables["ccf.modules"]:foreach(function(module_name, _)
+      if starts_with(module_name, prefix) then
+        tables["ccf.modules"]:remove(module_name)
+      end
+    end)
+  end
+  function add_modules_with_prefix(prefix, modules)
+    for _, module in pairs(modules) do
+      module_name = prefix .. module.rel_name
+      tables["ccf.modules"]:put(module_name, module.module)
+    end
+  end
+  remove_modules_with_prefix(args.prefix)
+  add_modules_with_prefix(args.prefix, args.modules)
+  return true
+  ]]
 }

tests/modules.py

@@ -4,7 +4,6 @@ import tempfile
 import http
 import subprocess
 import os
-import glob
 import infra.network
 import infra.path
 import infra.proc
@@ -18,6 +17,7 @@ from loguru import logger as LOG
 
 THIS_DIR = os.path.dirname(__file__)
 
+MODULE_PREFIX_1 = "/app/"
 MODULE_PATH_1 = "/app/foo.js"
 MODULE_RETURN_1 = "Hello world!"
 MODULE_CONTENT_1 = f"""
@@ -85,7 +85,7 @@ def make_module_set_proposal(path, content, network):
 def test_module_set_and_remove(network, args):
     primary, _ = network.find_nodes()
 
-    LOG.info("Member makes a module update proposal")
+    LOG.info("Member makes a module set proposal")
     make_module_set_proposal(MODULE_PATH_1, MODULE_CONTENT_1, network)
 
     with primary.client(
@@ -110,6 +110,35 @@ def test_module_set_and_remove(network, args):
     return network
 
 
+@reqs.description("Test prefix-based modules remove")
+def test_modules_remove(network, args):
+    primary, _ = network.find_nodes()
+
+    LOG.info("Member makes a module set proposal")
+    make_module_set_proposal(MODULE_PATH_1, MODULE_CONTENT_1, network)
+
+    with primary.client(
+        f"member{network.consortium.get_any_active_member().member_id}"
+    ) as c:
+        r = c.post("/gov/read", {"table": "ccf.modules", "key": MODULE_PATH_1})
+        assert r.status_code == http.HTTPStatus.OK, r.status_code
+        assert r.body["js"] == MODULE_CONTENT_1, r.body
+
+    LOG.info("Member makes a prefix-based modules remove proposal")
+    proposal_body, _ = ccf.proposal_generator.remove_modules(MODULE_PREFIX_1)
+    proposal = network.consortium.get_any_active_member().propose(
+        primary, proposal_body
+    )
+    network.consortium.vote_using_majority(primary, proposal)
+
+    with primary.client(
+        f"member{network.consortium.get_any_active_member().member_id}"
+    ) as c:
+        r = c.post("/gov/read", {"table": "ccf.modules", "key": MODULE_PATH_1})
+        assert r.status_code == http.HTTPStatus.BAD_REQUEST, r.status_code
+    return network
+
+
 @reqs.description("Test module import")
 def test_module_import(network, args):
     primary, _ = network.find_nodes()
@@ -132,7 +161,7 @@ def test_module_import(network, args):
     return network
 
 
-@reqs.description("Test Node.js/npm app")
+@reqs.description("Test Node.js/npm app with prefix-based modules update")
 def test_npm_app(network, args):
     primary, _ = network.find_nodes()
 
@@ -142,15 +171,16 @@ def test_npm_app(network, args):
     subprocess.run(["npm", "run", "build"], cwd=app_dir, check=True)
 
     LOG.info("Deploying npm app modules")
-    kv_prefix = "/my-npm-app"
+    module_name_prefix = "/my-npm-app/"
     dist_dir = os.path.join(app_dir, "dist")
-    for module_path in glob.glob(os.path.join(dist_dir, "**", "*.js"), recursive=True):
-        module_name = os.path.join(kv_prefix, os.path.relpath(module_path, dist_dir))
-        proposal_body, _ = ccf.proposal_generator.set_module(module_name, module_path)
-        proposal = network.consortium.get_any_active_member().propose(
-            primary, proposal_body
-        )
-        network.consortium.vote_using_majority(primary, proposal)
+
+    proposal_body, _ = ccf.proposal_generator.update_modules(
+        module_name_prefix, dist_dir
+    )
+    proposal = network.consortium.get_any_active_member().propose(
+        primary, proposal_body
+    )
+    network.consortium.vote_using_majority(primary, proposal)
 
     LOG.info("Deploying endpoint script")
     with tempfile.NamedTemporaryFile("w") as f:
@@ -186,6 +216,7 @@ def run(args):
     ) as network:
         network.start_and_join(args)
         network = test_module_set_and_remove(network, args)
+        network = test_modules_remove(network, args)
         network = test_module_import(network, args)
         network = test_npm_app(network, args)