2019-04-26 18:27:27 +03:00
|
|
|
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
|
|
# Licensed under the Apache 2.0 License.
|
|
|
|
import os
|
|
|
|
import getpass
|
|
|
|
import logging
|
|
|
|
import time
|
|
|
|
import math
|
|
|
|
import infra.ccf
|
|
|
|
import infra.proc
|
|
|
|
import infra.jsonrpc
|
|
|
|
import e2e_args
|
|
|
|
|
|
|
|
from loguru import logger as LOG
|
|
|
|
|
|
|
|
# This test starts from a given number of nodes (hosts), commits
|
2019-08-15 19:52:43 +03:00
|
|
|
# a transaction, stops the current primary, waits for an election and repeats
|
|
|
|
# this process until no progress can be made (i.e. no primary can be elected
|
2019-04-26 18:27:27 +03:00
|
|
|
# as F > N/2).
|
|
|
|
|
|
|
|
|
|
|
|
def wait_for_index_globally_committed(index, term, nodes):
|
|
|
|
"""
|
|
|
|
Wait for a specific version at a specific term to be committed on all nodes.
|
|
|
|
"""
|
|
|
|
for _ in range(infra.ccf.Network.replication_delay):
|
|
|
|
up_to_date_f = []
|
|
|
|
for f in nodes:
|
|
|
|
with f.management_client() as c:
|
Json schema: Take 3 (#113)
* Add a getApi method, listing all installed RPC method names
* Sketch RecordParams struct
* WIP
* Broken WIP
* Partial macro
* Basic examples working
* Partial file separation
* Move, rename, and fix FOR macro
* Use json get
* Build to_json from RequiredJsonFields too
* Remove unneeded pair specialisation
* Add comments, collide required and optional
* REformat
* Use new macros everywhere
* Remove unused template
* Rename getApi to listMethods
* Move frontend-specific calltypes to /rpc
* Specify GetTxHist return type
* Pretty-print client responses by default
* Add a GetSchema RPC
* Other tools demand ugly formatting by default
* mins and maxes for numerics, map of schemas
* Support _FOR_JSON_0
* Fix support for std::optional optional fields
* Test std optionals
* Define schemas for GetCommit
* More definitions for existing RPCs
* Tidy schema generation, including for vectors
* Add proper unit test
* Initial test of schema generation
* Fix failing tests
* Formatting
* Add (currently failing) test of nested structs
* Add misleadingly passing test
* Set correct expected pointers, test currently fails
* Oops - deexpand
* Correctly build pointer path for erroneous array elements
* Demonstrate invalid, not just missing, valeus
* Skeleton of json_bench
* Fix typo
* WIP
* Compare manual json parsers vs macro-defined
* mumble mumble
* Add valijson, +basic test
* Add benchmark of valijson validation
* Benchmark simple and complex structs
* Additional broken schema test
* Include pointer to parse errors
* Restore old basic translator macro
* Restore simpler macro for translators that don't need schema
* Add auto schema for private logging methods
* Add manual schema + validation for PUBLIC logging RPCs
* Match RPC format
* More RPC format fixes
* Correct scenario test target
* Add documentation entry on API schema
* Initial schema retrieval test
* Correct URLs in generated schema
* Send schema to a flat folder
* Remove unnecessary size_t max restriction
* Report non-matching schema
* Add current schemas
* Tidying
* clang-format
* Remove schema generation e2e test
* fmtlib, remove $id from schema
* Fix pointer paths
2019-06-05 12:36:50 +03:00
|
|
|
id = c.request("getCommit", {"commit": index})
|
2019-04-26 18:27:27 +03:00
|
|
|
res = c.response(id)
|
|
|
|
if res.result["term"] == term and res.global_commit > index:
|
|
|
|
up_to_date_f.append(f.node_id)
|
|
|
|
if len(up_to_date_f) == len(nodes):
|
|
|
|
break
|
|
|
|
time.sleep(1)
|
|
|
|
assert len(up_to_date_f) == len(
|
|
|
|
nodes
|
2019-08-15 19:52:43 +03:00
|
|
|
), "Only {} out of {} backups are up to date".format(len(up_to_date_f), len(nodes))
|
2019-04-26 18:27:27 +03:00
|
|
|
|
|
|
|
|
|
|
|
def run(args):
|
|
|
|
# Three nodes minimum to make sure that the raft network can still make progress
|
|
|
|
# if one node stops
|
|
|
|
hosts = ["localhost", "localhost", "localhost"]
|
|
|
|
|
|
|
|
with infra.ccf.network(
|
|
|
|
hosts, args.build_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
|
|
|
|
) as network:
|
|
|
|
|
2019-08-15 19:52:43 +03:00
|
|
|
initial_primary, _ = network.start_and_join(args)
|
2019-04-26 18:27:27 +03:00
|
|
|
current_term = None
|
|
|
|
|
|
|
|
# Time before an election completes
|
|
|
|
max_election_duration = args.election_timeout * 4 // 1000
|
|
|
|
|
|
|
|
# Number of nodes F to stop until network cannot make progress
|
|
|
|
nodes_to_stop = math.ceil(len(hosts) / 2)
|
|
|
|
|
|
|
|
for _ in range(nodes_to_stop):
|
2019-08-15 19:52:43 +03:00
|
|
|
# Note that for the first iteration, the primary is known in advance anyway
|
|
|
|
LOG.debug("Find freshly elected primary")
|
|
|
|
primary, current_term = network.find_primary()
|
2019-04-26 18:27:27 +03:00
|
|
|
|
|
|
|
LOG.debug("Commit new transactions")
|
|
|
|
commit_index = None
|
2019-08-15 19:52:43 +03:00
|
|
|
with primary.user_client() as c:
|
2019-04-26 18:27:27 +03:00
|
|
|
res = c.do(
|
|
|
|
"LOG_record",
|
|
|
|
{
|
|
|
|
"id": current_term,
|
|
|
|
"msg": "This log is committed in term {}".format(current_term),
|
|
|
|
},
|
2019-06-17 11:30:56 +03:00
|
|
|
True,
|
2019-04-26 18:27:27 +03:00
|
|
|
)
|
|
|
|
commit_index = res.commit
|
|
|
|
|
|
|
|
LOG.debug("Waiting for transaction to be committed by all nodes")
|
|
|
|
wait_for_index_globally_committed(
|
|
|
|
commit_index, current_term, network.get_running_nodes()
|
|
|
|
)
|
|
|
|
|
2019-08-15 19:52:43 +03:00
|
|
|
LOG.debug("Stopping primary")
|
|
|
|
primary.stop()
|
2019-04-26 18:27:27 +03:00
|
|
|
|
|
|
|
# Wait for next election to complete
|
|
|
|
time.sleep(max_election_duration)
|
|
|
|
|
|
|
|
# More than F nodes have been stopped, trying to commit any message
|
|
|
|
LOG.debug(
|
|
|
|
"No progress can be made as more than {} nodes have stopped".format(
|
|
|
|
nodes_to_stop
|
|
|
|
)
|
|
|
|
)
|
2019-05-20 13:45:53 +03:00
|
|
|
try:
|
2019-08-15 19:52:43 +03:00
|
|
|
primary, current_term = network.find_primary()
|
|
|
|
assert False, "Primary should not be found"
|
2019-05-20 13:45:53 +03:00
|
|
|
except AssertionError:
|
|
|
|
LOG.info(
|
2019-08-15 19:52:43 +03:00
|
|
|
"As expected, primary could not be found after election timeout. Test ended successfully."
|
2019-05-20 13:45:53 +03:00
|
|
|
)
|
2019-04-26 18:27:27 +03:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
|
|
|
args = e2e_args.cli_args()
|
|
|
|
args.package = "libloggingenc"
|
|
|
|
run(args)
|