Add the epbft startup test (#399)

tests/addnode.py

@@ -36,7 +36,7 @@ def run(args):
         # TODO: For now, node is added straight away, without validation by
         # the consortium. See https://github.com/microsoft/CCF/issues/293
         LOG.debug("Add a valid node")
-        new_node = network.create_and_add_node(args.package, "localhost", args)
+        new_node = network.create_and_add_node(args.package, "localhost", args, True)
 
         with primary.management_client() as mc:
             check_commit = infra.ccf.Checker(mc)
@@ -49,7 +49,7 @@ def run(args):
         if args.enclave_type == "debug":
             LOG.debug("Add an invalid node (unknown code id)")
             assert (
-                network.create_and_add_node("libluagenericenc", "localhost", args)
+                network.create_and_add_node("libluagenericenc", "localhost", args, True)
                 == None
             ), "Adding node with unknown code id should fail"
         else:

tests/code_update.py

@@ -36,10 +36,9 @@ def add_new_code(network, new_code_id):
 
 def create_node_using_new_code(network, args):
     # add a node using unsupported code
-    assert network.create_and_add_node(args.patched_file_name, "localhost", args) == (
-        False,
-        infra.jsonrpc.ErrorCode.CODE_ID_NOT_FOUND,
-    )
+    assert network.create_and_add_node(
+        args.patched_file_name, "localhost", args, True
+    ) == (False, infra.jsonrpc.ErrorCode.CODE_ID_NOT_FOUND)
 
 
 def run(args):
@@ -51,14 +50,14 @@ def run(args):
         primary, others = network.start_and_join(args)
 
         LOG.debug("Adding a new node")
-        new_node = network.create_and_add_node(args.package, "localhost", args)
+        new_node = network.create_and_add_node(args.package, "localhost", args, True)
         assert new_node
 
         new_code_id = get_code_id(f"{args.patched_file_name}.so.signed")
 
         LOG.debug(f"Adding a node with unsupported code id {new_code_id}")
         assert (
-            network.create_and_add_node(args.patched_file_name, "localhost", args)
+            network.create_and_add_node(args.patched_file_name, "localhost", args, True)
             == None
         ), "Adding node with unsupported code id should fail"
 
@@ -71,7 +70,7 @@ def run(args):
         LOG.debug("Adding more new nodes than originally existed")
         for _ in range(0, old_nodes_count + 1):
             new_node = network.create_and_add_node(
-                args.patched_file_name, "localhost", args
+                args.patched_file_name, "localhost", args, True
             )
             assert new_node
             new_nodes.add(new_node)
@@ -93,7 +92,7 @@ def run(args):
         LOG.debug(f"Waited, new_primary is {new_primary.node_id}")
 
         new_node = network.create_and_add_node(
-            args.patched_file_name, "localhost", args
+            args.patched_file_name, "localhost", args, True
         )
         assert new_node
         network.wait_for_node_commit_sync()

tests/e2e_logging_pbft.py

@@ -23,7 +23,17 @@ def run(args):
     with infra.ccf.network(
         hosts, args.build_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
     ) as network:
-        primary, _ = network.start_and_join(args)
+        primary, _ = network.start_and_join(args, False)
+
+        for i in range(1, 4):
+            LOG.info(f"Adding node {i}")
+            network.create_and_add_node(args.package, "localhost", args, False)
+
+        network.add_users(primary, network.initial_users)
+        LOG.info("Initial set of users added")
+
+        network.open_network(primary)
+        LOG.info("***** Network is now open *****")
 
         with primary.management_client() as mc:
             check_commit = infra.ccf.Checker(mc)

tests/infra/ccf.py

@@ -136,7 +136,7 @@ class Network:
                     perf=str(local_node_id_) in (perf_nodes or []),
                 )
 
-    def start_and_join(self, args):
+    def start_and_join(self, args, open_network=True):
         # TODO: The node that starts should not necessarily be node 0
         cmd = ["rm", "-f"] + glob("member*.pem")
         infra.proc.ccall(*cmd)
@@ -149,8 +149,8 @@ class Network:
         LOG.info("Starting nodes on {}".format(hosts))
 
         self.create_members([1, 2, 3])
-        initial_users = [1, 2, 3]
-        self.create_users(initial_users)
+        self.initial_users = [1, 2, 3]
+        self.create_users(self.initial_users)
 
         if args.app_script:
             infra.proc.ccall("cp", args.app_script, args.build_dir).check_returncode()
@@ -184,10 +184,13 @@ class Network:
         if primary is None:
             primary = self.nodes[0]
 
+        if not open_network:
+            return primary, self.nodes[1:]
+
         self.wait_for_all_nodes_to_catch_up(primary)
         LOG.success("All nodes joined network")
 
-        self.add_users(primary, initial_users)
+        self.add_users(primary, self.initial_users)
         LOG.info("Initial set of users added")
 
         self.open_network(primary)
@@ -278,7 +281,7 @@ class Network:
     def remove_last_node(self):
         last_node = self.nodes.pop()
 
-    def add_node(self, node, lib_name, primary, args):
+    def add_node(self, node, lib_name, primary, args, should_wait=True):
         forwarded_args = {
             arg: getattr(args, arg) for arg in infra.ccf.Network.node_args_to_forward
         }
@@ -295,7 +298,8 @@ class Network:
         )
 
         try:
-            node.wait_for_node_to_join()
+            if should_wait:
+                node.wait_for_node_to_join()
         except TimeoutError:
             LOG.error(f"New node {node.local_node_id} failed to join the network")
             self.nodes.remove(node)
@@ -304,20 +308,21 @@ class Network:
         node.network_state = NodeNetworkState.joined
         return True
 
-    def create_and_add_node(self, lib_name, host, args):
+    def create_and_add_node(self, lib_name, host, args, should_wait):
         local_node_id = self.get_next_local_node_id()
 
         new_node = self.create_node(
             local_node_id, host, debug=str(local_node_id) in self.dbg_nodes
         )
 
-        if self.add_node(new_node, lib_name, None, args) is False:
+        if self.add_node(new_node, lib_name, None, args, should_wait) is False:
             return None
 
         primary, _ = self.find_primary()
-        self.wait_for_all_nodes_to_catch_up(primary)
-        LOG.success(f"New node {local_node_id} joined the network")
+        if should_wait:
+            self.wait_for_all_nodes_to_catch_up(primary)
 
+        LOG.success(f"New node {local_node_id} joined the network")
         return new_node
 
     def create_members(self, members):