This commit is contained in:
Eddy Ashton 2020-07-17 14:13:53 +01:00 коммит произвёл GitHub
Родитель 89ccf6624c
Коммит a9c13e45c1
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
8 изменённых файлов: 204 добавлений и 84 удалений

Просмотреть файл

@ -382,7 +382,7 @@ ignore-on-opaque-inference=yes
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
ignored-classes=optparse.Values,thread._local,_thread._local,ProposalGenerator
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime

Просмотреть файл

@ -260,14 +260,19 @@ class CurlClient:
]
if not request.params_in_query and request.params is not None:
if isinstance(request.params, bytes):
msg_bytes = request.params
if isinstance(request.params, str) and request.params.startswith("@"):
# Request is already a file path - pass it directly
cmd.extend(["--data-binary", request.params])
else:
msg_bytes = json.dumps(request.params).encode()
LOG.debug(f"Writing request body: {msg_bytes}")
nf.write(msg_bytes)
nf.flush()
cmd.extend(["--data-binary", f"@{nf.name}"])
# Write request to temp file
if isinstance(request.params, bytes):
msg_bytes = request.params
else:
msg_bytes = json.dumps(request.params).encode()
LOG.debug(f"Writing request body: {msg_bytes}")
nf.write(msg_bytes)
nf.flush()
cmd.extend(["--data-binary", f"@{nf.name}"])
if not "content-type" in request.headers:
request.headers["content-type"] = "application/json"
@ -368,10 +373,15 @@ class RequestClient:
}
if request.params is not None:
request_params = request.params
if isinstance(request.params, str) and request.params.startswith("@"):
# Request is a file path - read contents, assume json
request_params = json.load(open(request.params[1:]))
if request.params_in_query:
request_args["params"] = build_query_string(request.params)
request_args["params"] = build_query_string(request_params)
else:
request_args["json"] = request.params
request_args["json"] = request_params
try:
response = self.session.request(

Просмотреть файл

@ -7,6 +7,7 @@ import inspect
import json
import os
import sys
import functools
from loguru import logger as LOG
@ -24,11 +25,7 @@ def script_to_vote_object(script):
return {"ballot": {"text": script}}
TRIVIAL_YES_BALLOT = {"text": "return true"}
TRIVIAL_NO_BALLOT = {"text": "return false"}
LUA_FUNCTION_EQUAL_ARRAYS = """
function equal_arrays(a, b)
LUA_FUNCTION_EQUAL_ARRAYS = """function equal_arrays(a, b)
if #a ~= #b then
return false
else
@ -39,13 +36,43 @@ function equal_arrays(a, b)
end
return true
end
end
"""
end"""
DEFAULT_PROPOSAL_OUTPUT = "{proposal_name}_proposal.json"
DEFAULT_VOTE_OUTPUT = "{proposal_name}_vote_for.json"
def complete_proposal_output_path(
proposal_name, proposal_output_path=None, common_dir="."
):
if proposal_output_path is None:
proposal_output_path = DEFAULT_PROPOSAL_OUTPUT.format(
proposal_name=proposal_name
)
if not proposal_output_path.endswith(".json"):
proposal_output_path += ".json"
proposal_output_path = os.path.join(common_dir, proposal_output_path)
return proposal_output_path
def complete_vote_output_path(proposal_name, vote_output_path=None, common_dir="."):
if vote_output_path is None:
vote_output_path = DEFAULT_VOTE_OUTPUT.format(proposal_name=proposal_name)
if not vote_output_path.endswith(".json"):
vote_output_path += ".json"
vote_output_path = os.path.join(common_dir, vote_output_path)
return vote_output_path
def add_arg_construction(lines, arg, arg_name="args"):
if isinstance(arg, str):
lines.append(f'{arg_name} = "{arg}"')
lines.append(f"{arg_name} = [====[{arg}]====]")
elif isinstance(arg, collections.abc.Sequence):
lines.append(f"{arg_name} = {list_as_lua_literal(arg)}")
elif isinstance(arg, collections.abc.Mapping):
@ -56,11 +83,16 @@ def add_arg_construction(lines, arg, arg_name="args"):
lines.append(f"{arg_name} = {arg}")
def add_arg_checks(lines, arg, arg_name="args"):
lines.append(f"if {arg_name} == nil then return false")
def add_arg_checks(lines, arg, arg_name="args", added_equal_arrays_fn=False):
lines.append(f"if {arg_name} == nil then return false end")
if isinstance(arg, str):
lines.append(f'if not {arg_name} == "{arg}" then return false end')
lines.append(f"if not {arg_name} == [====[{arg}]====] then return false end")
elif isinstance(arg, collections.abc.Sequence):
if not added_equal_arrays_fn:
lines.extend(
line.strip() for line in LUA_FUNCTION_EQUAL_ARRAYS.splitlines()
)
added_equal_arrays_fn = True
expected_name = arg_name.replace(".", "_")
lines.append(f"{expected_name} = {list_as_lua_literal(arg)}")
lines.append(
@ -68,13 +100,18 @@ def add_arg_checks(lines, arg, arg_name="args"):
)
elif isinstance(arg, collections.abc.Mapping):
for k, v in arg.items():
add_arg_checks(lines, v, arg_name=f"{arg_name}.{k}")
add_arg_checks(
lines,
v,
arg_name=f"{arg_name}.{k}",
added_equal_arrays_fn=added_equal_arrays_fn,
)
else:
lines.append(f"if not {arg_name} == {arg} then return false end")
def build_proposal(proposed_call, args=None, inline_args=False):
LOG.debug(f"Generating {proposed_call} proposal")
def build_proposal(proposed_call, args=None, inline_args=False, vote_against=False):
LOG.trace(f"Generating {proposed_call} proposal")
proposal_script_lines = []
if args is None:
@ -92,15 +129,17 @@ def build_proposal(proposed_call, args=None, inline_args=False):
}
if args is not None and not inline_args:
proposal["parameter"] = args
if vote_against:
proposal["ballot"] = {"text": "return false"}
vote_lines = [
"tables, calls = ...",
"if not #calls == 1 then return false end",
"call = calls[1]",
f'if not call.func == "{proposed_call}" then return false end',
LUA_FUNCTION_EQUAL_ARRAYS,
]
if args is not None:
vote_lines.append("args = call.args")
add_arg_checks(vote_lines, args)
vote_lines.append("return true")
vote_text = "; ".join(vote_lines)
@ -121,7 +160,7 @@ def cli_proposal(func):
def new_member(member_cert_path, member_enc_pubk_path, **kwargs):
LOG.debug("Generating new_member proposal")
# Convert certs to byte arrays
# Read certs
member_cert = open(member_cert_path).read()
member_keyshare_encryptor = open(member_enc_pubk_path).read()
@ -137,6 +176,11 @@ def new_member(member_cert_path, member_enc_pubk_path, **kwargs):
"script": {"text": proposal_script_text},
}
vote_against = kwargs.pop("vote_against", False)
if vote_against:
proposal["ballot"] = {"text": "return false"}
# Sample vote script which checks the expected member is being added, and no other actions are being taken
verifying_vote_text = f"""
tables, calls = ...
@ -149,15 +193,13 @@ def new_member(member_cert_path, member_enc_pubk_path, **kwargs):
return false
end
{LUA_FUNCTION_EQUAL_ARRAYS}
expected_cert = {list_as_lua_literal(member_cert)}
if not equal_arrays(call.args.cert, expected_cert) then
expected_cert = [====[{member_cert}]====]
if not call.args.cert == expected_cert then
return false
end
expected_keyshare = {list_as_lua_literal(member_keyshare_encryptor)}
if not equal_arrays(call.args.keyshare, expected_keyshare) then
expected_keyshare = [====[{member_keyshare_encryptor}]====]
if not call.args.keyshare == expected_keyshare then
return false
end
@ -258,25 +300,66 @@ def set_recovery_threshold(threshold, **kwargs):
return build_proposal("set_recovery_threshold", threshold, **kwargs)
class ProposalGenerator:
def __init__(self, common_dir="."):
self.common_dir = common_dir
# Auto-generate methods wrapping inspected functions, dumping outputs to file
def wrapper(func):
@functools.wraps(func)
def wrapper_func(
*args, proposal_output_path=None, vote_output_path=None, **kwargs,
):
proposal_output_path = complete_proposal_output_path(
func.__name__,
proposal_output_path=proposal_output_path,
common_dir=self.common_dir,
)
vote_output_path = complete_vote_output_path(
func.__name__,
vote_output_path=vote_output_path,
common_dir=self.common_dir,
)
proposal_object, vote_object = func(*args, **kwargs)
dump_args = {"indent": 2}
LOG.debug(f"Writing proposal to {proposal_output_path}")
dump_to_file(proposal_output_path, proposal_object, dump_args)
LOG.debug(f"Writing vote to {vote_output_path}")
dump_to_file(vote_output_path, vote_object, dump_args)
return f"@{proposal_output_path}", f"@{vote_output_path}"
return wrapper_func
module = inspect.getmodule(inspect.currentframe())
proposal_generators = inspect.getmembers(module, predicate=inspect.isfunction)
for func_name, func in proposal_generators:
# Only wrap decorated functions
if hasattr(func, "is_cli_proposal"):
setattr(self, func_name, wrapper(func))
if __name__ == "__main__":
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
default_proposal_output = "{proposal_type}.json"
default_vote_output = "vote_for_{proposal_output}.json"
parser.add_argument(
"-po",
"--proposal-output-file",
type=str,
help=f"Path where proposal JSON object (request body for POST /gov/proposals) will be dumped. Default is {default_proposal_output}",
help=f"Path where proposal JSON object (request body for POST /gov/proposals) will be dumped. Default is {DEFAULT_PROPOSAL_OUTPUT}",
)
parser.add_argument(
"-vo",
"--vote-output-file",
type=str,
help=f"Path where vote JSON object (request body for POST /gov/proposals/{{proposal_id}}/votes) will be dumped. Default is {default_vote_output}",
help=f"Path where vote JSON object (request body for POST /gov/proposals/{{proposal_id}}/votes) will be dumped. Default is {DEFAULT_VOTE_OUTPUT}",
)
parser.add_argument(
"-pp",
@ -288,10 +371,16 @@ if __name__ == "__main__":
"-i",
"--inline-args",
action="store_true",
help="Create a fixed proposal script with the call arguments as literalsinside"
"the script. When not inlined, the parameters are passed separately and could"
help="Create a fixed proposal script with the call arguments as literals inside "
"the script. When not inlined, the parameters are passed separately and could "
"be replaced in the resulting object",
)
parser.add_argument(
"--vote-against",
action="store_true",
help="Include a negative initial vote when creating the proposal",
default=False,
)
parser.add_argument("-v", "--verbose", action="store_true")
# Auto-generate CLI args based on the inspected signatures of generator functions
@ -303,9 +392,7 @@ if __name__ == "__main__":
for func_name, func in proposal_generators:
# Only generate for decorated functions
try:
getattr(func, "is_cli_proposal")
except AttributeError:
if not hasattr(func, "is_cli_proposal"):
continue
subparser = subparsers.add_parser(func_name)
@ -329,6 +416,7 @@ if __name__ == "__main__":
proposal, vote = args.func(
**{name: getattr(args, name) for name in args.param_names},
vote_against=args.vote_against,
inline_args=args.inline_args,
)
@ -336,14 +424,14 @@ if __name__ == "__main__":
if args.pretty_print:
dump_args["indent"] = 2
proposal_output_path = args.proposal_output_file or default_proposal_output.format(
proposal_type=args.proposal_type
proposal_path = complete_proposal_output_path(
args.proposal_type, proposal_output_path=args.proposal_output_file
)
LOG.success(f"Writing proposal to {proposal_output_path}")
dump_to_file(proposal_output_path, proposal, dump_args)
LOG.success(f"Writing proposal to {proposal_path}")
dump_to_file(proposal_path, proposal, dump_args)
vote_output_path = args.vote_output_file or default_vote_output.format(
proposal_output=os.path.splitext(proposal_output_path)[0]
vote_path = complete_vote_output_path(
args.proposal_type, vote_output_path=args.vote_output_file
)
LOG.success(f"Writing vote to {vote_output_path}")
dump_to_file(vote_output_path, vote, dump_args)
LOG.success(f"Wrote vote to {vote_path}")
dump_to_file(vote_path, vote, dump_args)

Просмотреть файл

@ -78,13 +78,14 @@ def run(args):
transactions.append(json_tx)
# Manager is granted special privileges by members, which is later read by app to enforce access restrictions
proposal_body, _ = ccf.proposal_generator.set_user_data(
proposal_body, careful_vote = ccf.proposal_generator.set_user_data(
manager.ccf_id,
{"privileges": {"REGISTER_REGULATORS": True, "REGISTER_BANKS": True}},
)
proposal = network.consortium.get_any_active_member().propose(
primary, proposal_body
)
proposal.vote_for = careful_vote
network.consortium.vote_using_majority(primary, proposal)
# Check permissions are enforced

Просмотреть файл

@ -12,14 +12,11 @@ import ccf.checker
import infra.node
import infra.crypto
import infra.member
import ccf.proposal_generator
from ccf.proposal_generator import ProposalGenerator
from infra.proposal import ProposalState
from loguru import logger as LOG
# Votes are currently produced but unused, so temporarily disable this pylint warning throughout this file
# pylint: disable=unused-variable
class Consortium:
def __init__(
@ -37,6 +34,7 @@ class Consortium:
self.share_script = share_script
self.members = []
self.recovery_threshold = None
self.proposal_generator = ProposalGenerator(common_dir=self.common_dir)
# If a list of member IDs is passed in, generate fresh member identities.
# Otherwise, recover the state of the consortium from the state of CCF.
if member_ids is not None:
@ -96,13 +94,16 @@ class Consortium:
new_member_id, curve, self.common_dir, self.share_script, self.key_generator
)
proposal, vote = ccf.proposal_generator.new_member(
proposal_body, careful_vote = self.proposal_generator.new_member(
os.path.join(self.common_dir, f"member{new_member_id}_cert.pem"),
os.path.join(self.common_dir, f"member{new_member_id}_enc_pubk.pem"),
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return (
self.get_any_active_member().propose(remote_node, proposal),
proposal,
new_member,
)
@ -194,8 +195,11 @@ class Consortium:
return proposals
def retire_node(self, remote_node, node_to_retire):
proposal_body, vote = ccf.proposal_generator.retire_node(node_to_retire.node_id)
proposal_body, careful_vote = self.proposal_generator.retire_node(
node_to_retire.node_id
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
self.vote_using_majority(remote_node, proposal)
with remote_node.client(f"member{self.get_any_active_member().member_id}") as c:
@ -210,9 +214,9 @@ class Consortium:
):
raise ValueError(f"Node {node_id} does not exist in state PENDING")
proposal_body, vote = ccf.proposal_generator.trust_node(node_id)
proposal_body, careful_vote = self.proposal_generator.trust_node(node_id)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
self.vote_using_majority(remote_node, proposal)
if not self._check_node_exists(
@ -221,10 +225,11 @@ class Consortium:
raise ValueError(f"Node {node_id} does not exist in state TRUSTED")
def retire_member(self, remote_node, member_to_retire):
proposal_body, vote = ccf.proposal_generator.retire_member(
proposal_body, careful_vote = self.proposal_generator.retire_member(
member_to_retire.member_id
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
self.vote_using_majority(remote_node, proposal)
member_to_retire.status = infra.member.MemberStatus.RETIRED
@ -234,30 +239,33 @@ class Consortium:
proposal and make members vote to transition the network to state
OPEN.
"""
proposal_body, vote = ccf.proposal_generator.open_network()
proposal_body, careful_vote = self.proposal_generator.open_network()
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
self.vote_using_majority(
remote_node, proposal, wait_for_global_commit=(not pbft_open)
)
self.check_for_service(remote_node, infra.network.ServiceStatus.OPEN, pbft_open)
def rekey_ledger(self, remote_node):
proposal_body, vote = ccf.proposal_generator.rekey_ledger()
proposal_body, careful_vote = self.proposal_generator.rekey_ledger()
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def update_recovery_shares(self, remote_node):
proposal_body, vote = ccf.proposal_generator.update_recovery_shares()
proposal_body, careful_vote = self.proposal_generator.update_recovery_shares()
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def add_user(self, remote_node, user_id):
user_cert = []
proposal, vote = ccf.proposal_generator.new_user(
proposal, careful_vote = self.proposal_generator.new_user(
os.path.join(self.common_dir, f"user{user_id}_cert.pem")
)
proposal = self.get_any_active_member().propose(remote_node, proposal)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def add_users(self, remote_node, users):
@ -265,24 +273,32 @@ class Consortium:
self.add_user(remote_node, u)
def remove_user(self, remote_node, user_id):
proposal, vote = ccf.proposal_generator.remove_user(user_id)
proposal, careful_vote = self.proposal_generator.remove_user(user_id)
proposal = self.get_any_active_member().propose(remote_node, proposal)
proposal.vote_for = careful_vote
self.vote_using_majority(remote_node, proposal)
def set_lua_app(self, remote_node, app_script_path):
proposal_body, vote = ccf.proposal_generator.set_lua_app(app_script_path)
proposal_body, careful_vote = self.proposal_generator.set_lua_app(
app_script_path
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def set_js_app(self, remote_node, app_script_path):
proposal_body, vote = ccf.proposal_generator.set_js_app(app_script_path)
proposal_body, careful_vote = self.proposal_generator.set_js_app(
app_script_path
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def accept_recovery(self, remote_node):
proposal_body, vote = ccf.proposal_generator.accept_recovery()
proposal_body, careful_vote = self.proposal_generator.accept_recovery()
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def recover_with_shares(self, remote_node, defunct_network_enc_pubk):
@ -304,21 +320,24 @@ class Consortium:
assert "End of recovery procedure initiated" not in r.result
def set_recovery_threshold(self, remote_node, recovery_threshold):
proposal_body, vote = ccf.proposal_generator.set_recovery_threshold(
proposal_body, careful_vote = self.proposal_generator.set_recovery_threshold(
recovery_threshold
)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
self.recovery_threshold = recovery_threshold
return self.vote_using_majority(remote_node, proposal)
def add_new_code(self, remote_node, new_code_id):
proposal_body, vote = ccf.proposal_generator.new_node_code(new_code_id)
proposal_body, careful_vote = self.proposal_generator.new_node_code(new_code_id)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def add_new_user_code(self, remote_node, new_code_id):
proposal_body, vote = ccf.proposal_generator.new_user_code(new_code_id)
proposal_body, careful_vote = self.proposal_generator.new_user_code(new_code_id)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
proposal.vote_for = careful_vote
return self.vote_using_majority(remote_node, proposal)
def check_for_service(self, remote_node, status, pbft_open=False):

Просмотреть файл

@ -65,7 +65,7 @@ class Member:
# Use this with caution (i.e. only when the network is opening)
self.status = MemberStatus.ACTIVE
def propose(self, remote_node, proposal):
def propose(self, remote_node, proposal, has_proposer_voted_for=True):
with remote_node.client(f"member{self.member_id}") as mc:
r = mc.rpc("/gov/proposals", proposal, signed=True,)
if r.status != http.HTTPStatus.OK.value:
@ -75,20 +75,16 @@ class Member:
proposer_id=self.member_id,
proposal_id=r.result["proposal_id"],
state=infra.proposal.ProposalState(r.result["state"]),
has_proposer_voted_for=True,
has_proposer_voted_for=has_proposer_voted_for,
)
def vote(
self, remote_node, proposal, accept=True, wait_for_global_commit=True,
):
ballot = """
tables, changes = ...
return true
"""
with remote_node.client(f"member{self.member_id}") as mc:
r = mc.rpc(
f"/gov/proposals/{proposal.proposal_id}/votes",
{"ballot": {"text": ballot}},
params=proposal.vote_for,
signed=True,
)

Просмотреть файл

@ -34,6 +34,7 @@ class Proposal:
self.state = state
self.has_proposer_voted_for = has_proposer_voted_for
self.votes_for = 1 if self.has_proposer_voted_for else 0
self.vote_for = {"ballot": {"text": "return true"}}
def increment_votes_for(self):
self.votes_for += 1

Просмотреть файл

@ -216,9 +216,11 @@ def run(args):
assert response.status == params_error
LOG.info("New non-active member should get insufficient rights response")
proposal_trust_0, _ = ccf.proposal_generator.trust_node(0)
proposal_trust_0, careful_vote = ccf.proposal_generator.trust_node(
0, vote_against=True
)
try:
new_member.propose(primary, proposal_trust_0)
new_member.propose(primary, proposal_trust_0, has_proposer_voted_for=False)
assert (
False
), "New non-active member should get insufficient rights response"
@ -229,13 +231,16 @@ def run(args):
new_member.ack(primary)
LOG.info("New member is now active and send an accept node proposal")
trust_node_proposal_0 = new_member.propose(primary, proposal_trust_0)
trust_node_proposal_0 = new_member.propose(
primary, proposal_trust_0, has_proposer_voted_for=False
)
trust_node_proposal_0.vote_for = careful_vote
LOG.debug("Members vote to accept the accept node proposal")
network.consortium.vote_using_majority(primary, trust_node_proposal_0)
assert trust_node_proposal_0.state == infra.proposal.ProposalState.Accepted
LOG.info("New member makes a new proposal")
LOG.info("New member makes a new proposal, with initial no vote")
proposal_trust_1, _ = ccf.proposal_generator.trust_node(1)
trust_node_proposal = new_member.propose(primary, proposal_trust_1)