microsoft
/
CCF
зеркало из https://github.com/microsoft/CCF.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove `/gov/read` and `/gov/query` endpoints (#2442)

This commit is contained in:
Julien Maffre 2021-04-12 10:54:42 +01:00 коммит произвёл GitHub
Родитель 73bada24fe
Коммит 9e0e908012
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
22 изменённых файлов: 394 добавлений и 866 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
CHANGELOG.md
Просмотреть файл

@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
## [1.0]
### Added
- The service certificate is now returned as part of the `/node/network/` endpoint response.
### Removed
- `/gov/query` and `/gov/read` governance endpoints are removed.
## [0.99.0]
This is a bridging release to simplify the upgrade to 1.0. It includes the new JS constitution, but also supports the existing Lua governance so that users can upgrade in 2 steps - first implementing all of the changes below with their existing Lua governance, then upgrading to the JS governance. Lua governance will be removed in CCF 1.0. See [temporary docs](https://microsoft.github.io/CCF/ccf-0.99.0/governance/js_gov.html) for help with transitioning from Lua to JS.

77
doc/schemas/gov_openapi.json
Просмотреть файл

@ -161,21 +161,6 @@
],
"type": "object"
},
"KVRead__In": {
"properties": {
"key": {
"$ref": "#/components/schemas/json"
},
"table": {
"$ref": "#/components/schemas/string"
}
},
"required": [
"table",
"key"
],
"type": "object"
},
"Proposal": {
"properties": {
"actions": {
@ -912,68 +897,6 @@
]
}
},
"/query": {
"post": {
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/Script"
}
}
},
"description": "Auto-generated request body schema"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/json"
}
}
},
"description": "Default response description"
}
},
"security": [
{
"member_signature": []
}
]
}
},
"/read": {
"post": {
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/KVRead__In"
}
}
},
"description": "Auto-generated request body schema"
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/json"
}
}
},
"description": "Default response description"
}
},
"security": [
{
"member_signature": []
}
]
}
},
"/receipt": {
"get": {
"parameters": [

7
doc/schemas/node_openapi.json
Просмотреть файл

@ -111,12 +111,16 @@
"primary_id": {
"$ref": "#/components/schemas/EntityId"
},
"service_certificate": {
"$ref": "#/components/schemas/Pem"
},
"service_status": {
"$ref": "#/components/schemas/ServiceStatus"
}
},
"required": [
"service_status",
"service_certificate",
"current_view",
"primary_id"
],
@ -252,6 +256,9 @@
],
"type": "string"
},
"Pem": {
"type": "string"
},
"Quote": {
"properties": {
"endorsements": {

99
python/ccf/ledger.py
Просмотреть файл

@ -5,7 +5,7 @@ import io
import struct
import os
from typing import BinaryIO, NamedTuple, Optional
from typing import BinaryIO, NamedTuple, Optional, Tuple, Dict
import json
import base64
@ -31,6 +31,9 @@ LEDGER_HEADER_SIZE = 8
SIGNATURE_TX_TABLE_NAME = "public:ccf.internal.signatures"
NODES_TABLE_NAME = "public:ccf.gov.nodes.info"
# Key used by CCF to record single-key tables
WELL_KNOWN_SINGLETON_TABLE_KEY = bytes(bytearray(8))
def to_uint_32(buffer):
return struct.unpack("@I", buffer)[0]
@ -411,18 +414,15 @@ class Transaction:
def __next__(self):
if self._next_offset == self._file_size:
raise StopIteration()
try:
self._complete_read()
self._read_header()
# Adds every transaction to the ledger validator
# LedgerValidator does verification for every added transaction and throws when it finds any anomaly.
self._ledger_validator.add_transaction(self)
self._complete_read()
self._read_header()
return self
except Exception as exception:
LOG.exception(f"Encountered exception: {exception}")
raise
# Adds every transaction to the ledger validator
# LedgerValidator does verification for every added transaction and throws when it finds any anomaly.
self._ledger_validator.add_transaction(self)
return self
class LedgerChunk:
@ -466,10 +466,14 @@ class Ledger:
_current_chunk: LedgerChunk
_ledger_validator: LedgerValidator
def _reset_iterators(self):
self._fileindex = -1
# Initialize LedgerValidator instance which will be passed to LedgerChunks.
self._ledger_validator = LedgerValidator()
def __init__(self, directory: str):
self._filenames = []
self._fileindex = -1
ledgers = os.listdir(directory)
# Sorts the list based off the first number after ledger_ so that
@ -485,8 +489,7 @@ class Ledger:
if os.path.isfile(os.path.join(directory, chunk)):
self._filenames.append(os.path.join(directory, chunk))
# Initialize LedgerValidator instance which will be passed to LedgerChunks.
self._ledger_validator = LedgerValidator()
self._reset_iterators()
def __next__(self) -> LedgerChunk:
self._fileindex += 1
@ -513,6 +516,70 @@ class Ledger:
self._ledger_validator.last_verified_seqno,
)
def get_transaction(self, seqno: int) -> Transaction:
"""
Returns the :py:class:`ccf.Ledger.Transaction` recorded in the ledger at the given sequence number.
Note that the transaction returned may not yet be verified by a
signature transaction nor committed by the service.
:param int seqno: Sequence number of the transaction to fetch.
:return: :py:class:`ccf.Ledger.Transaction`
"""
if seqno < 1:
raise ValueError("Ledger first seqno is 1")
self._reset_iterators()
transaction = None
try:
# Note: This is slower than it really needs to as this will walk through
# all transactions from the start of the ledger.
for chunk in self:
for tx in chunk:
public_transaction = tx.get_public_domain()
if public_transaction.get_seqno() == seqno:
return tx
finally:
self._reset_iterators()
if transaction is None:
raise UnknownTransaction(
f"Transaction at seqno {seqno} does not exist in ledger"
)
return transaction
def get_latest_public_state(self) -> Tuple[dict, int]:
"""
Returns the current public state of the service.
Note that the public state returned may not yet be verified by a
signature transaction nor committed by the service.
:return: Tuple[Dict, int]: Tuple containing a dictionary of public tables and their values and the seqno of the state read from the ledger.
"""
self._reset_iterators()
public_tables: Dict[str, Dict] = {}
latest_seqno = 0
for chunk in self:
for tx in chunk:
latest_seqno = tx.get_public_domain().get_seqno()
for table_name, records in tx.get_public_domain().get_tables().items():
if table_name in public_tables:
public_tables[table_name].update(records)
# Remove deleted keys
public_tables[table_name] = {
k: v
for k, v in public_tables[table_name].items()
if v is not None
}
else:
public_tables[table_name] = records
return public_tables, latest_seqno
class InvalidRootException(Exception):
"""MerkleTree root doesn't match with the root reported in the signature's table"""
@ -528,3 +595,7 @@ class CommitIdRangeException(Exception):
class UntrustedNodeException(Exception):
"""The signing node wasn't part of the network"""
class UnknownTransaction(Exception):
"""The transaction at seqno does not exist in ledger"""

14
python/ledger_tutorial.py
Просмотреть файл

@ -4,6 +4,7 @@
import sys
from loguru import logger as LOG
import json
import random
# Note: It is safer to run the ledger tutorial when the service has stopped
# as all ledger files will have been written to.
@ -44,3 +45,16 @@ for chunk in ledger:
# In this case, the target table 'public:ccf.gov.nodes.info' is raw bytes to JSON.
LOG.info(f"{key.decode()} : {json.loads(value)}")
# SNIPPET_END: iterate_over_ledger
# Read state of ledger
latest_state, latest_seqno = ledger.get_latest_public_state()
seqnos = [1, 2, 3, latest_seqno // 2, latest_seqno]
random.shuffle(seqnos)
for seqno in seqnos:
transaction = ledger.get_transaction(seqno)
# Confirm latest state can still be accessed, and is unchanged
latest_state1, latest_seqno1 = ledger.get_latest_public_state()
assert latest_seqno == latest_seqno1
assert latest_state == latest_state1

10
src/crypto/pem.h
Просмотреть файл

@ -116,6 +116,16 @@ namespace crypto
fmt::format("Unable to parse pem from this JSON: {}", j.dump()));
}
}
inline std::string schema_name(const Pem&)
{
return "Pem";
}
inline void fill_json_schema(nlohmann::json& schema, const Pem&)
{
schema["type"] = "string";
}
}
namespace std

4
src/node/constitution.h
Просмотреть файл

@ -2,7 +2,9 @@
// Licensed under the Apache 2.0 License.
#pragma once
#include "service_map.h"
namespace ccf
{
using Constitution = kv::JsonSerialisedMap<size_t, std::string>;
using Constitution = ServiceMap<size_t, std::string>;
}

1
src/node/rpc/call_types.h
Просмотреть файл

@ -54,6 +54,7 @@ namespace ccf
struct Out
{
ServiceStatus service_status;
crypto::Pem service_certificate;
std::optional<ccf::View> current_view;
std::optional<NodeId> primary_id;
};

85
src/node/rpc/member_frontend.h
Просмотреть файл

@ -1010,12 +1010,6 @@ namespace ccf
return check_member_status(tx, id, {MemberStatus::ACTIVE});
}
bool check_member_accepted(kv::ReadOnlyTx& tx, const MemberId& id)
{
return check_member_status(
tx, id, {MemberStatus::ACTIVE, MemberStatus::ACCEPTED});
}
bool check_member_status(
kv::ReadOnlyTx& tx,
const MemberId& id,
@ -1114,85 +1108,6 @@ namespace ccf
const AuthnPolicies member_cert_or_sig = {member_cert_auth_policy,
member_signature_auth_policy};
auto read = [this](auto& ctx, nlohmann::json&& params) {
const auto member_id = get_caller_member_id(ctx);
if (!member_id.has_value())
{
return make_error(
HTTP_STATUS_FORBIDDEN,
ccf::errors::AuthorizationFailed,
"Member is unknown.");
}
if (!check_member_status(
ctx.tx,
member_id.value(),
{MemberStatus::ACTIVE, MemberStatus::ACCEPTED}))
{
return make_error(
HTTP_STATUS_FORBIDDEN,
ccf::errors::AuthorizationFailed,
"Member is not active or accepted.");
}
const auto in = params.get<KVRead::In>();
const ccf::Script read_script(R"xxx(
local tables, table_name, key = ...
return tables[table_name]:get(key) or {}
)xxx");
auto value = tsr.run<nlohmann::json>(
ctx.tx,
{read_script, {}, WlIds::MEMBER_CAN_READ, {}},
in.table,
in.key);
if (value.empty())
{
return make_error(
HTTP_STATUS_NOT_FOUND,
ccf::errors::KeyNotFound,
fmt::format(
"Key {} does not exist in table {}.", in.key.dump(), in.table));
}
return make_success(value);
};
make_endpoint("read", HTTP_POST, json_adapter(read), member_cert_or_sig)
// This can be executed locally, but can't currently take ReadOnlyTx due
// to restrictions in our lua wrappers
.set_forwarding_required(endpoints::ForwardingRequired::Sometimes)
.set_auto_schema<KVRead>()
.install();
auto query = [this](auto& ctx, nlohmann::json&& params) {
const auto member_id = get_caller_member_id(ctx);
if (!member_id.has_value())
{
return make_error(
HTTP_STATUS_FORBIDDEN,
ccf::errors::AuthorizationFailed,
"Member is unknown.");
}
if (!check_member_accepted(ctx.tx, member_id.value()))
{
return make_error(
HTTP_STATUS_FORBIDDEN,
ccf::errors::AuthorizationFailed,
"Member is not accepted.");
}
const auto script = params.get<ccf::Script>();
return make_success(tsr.run<nlohmann::json>(
ctx.tx, {script, {}, WlIds::MEMBER_CAN_READ, {}}));
};
make_endpoint("query", HTTP_POST, json_adapter(query), member_cert_or_sig)
// This can be executed locally, but can't currently take ReadOnlyTx due
// to restrictions in our lua wrappers
.set_forwarding_required(endpoints::ForwardingRequired::Sometimes)
.set_auto_schema<Script, nlohmann::json>()
.install();
auto propose = [this](auto& ctx, nlohmann::json&& params) {
const auto& caller_identity =
ctx.template get_caller<ccf::MemberSignatureAuthnIdentity>();

4
src/node/rpc/node_frontend.h
Просмотреть файл

@ -429,7 +429,9 @@ namespace ccf
auto service_state = service->get(0);
if (service_state.has_value())
{
out.service_status = service_state.value().status;
const auto& service_value = service_state.value();
out.service_status = service_value.status;
out.service_certificate = service_value.cert;
if (consensus != nullptr)
{
out.current_view = consensus->get_view();

6
src/node/rpc/serialization.h
Просмотреть файл

@ -70,7 +70,11 @@ namespace ccf
DECLARE_JSON_TYPE(GetNetworkInfo::Out)
DECLARE_JSON_REQUIRED_FIELDS(
GetNetworkInfo::Out, service_status, current_view, primary_id)
GetNetworkInfo::Out,
service_status,
service_certificate,
current_view,
primary_id)
DECLARE_JSON_TYPE(GetNode::NodeInfo)
DECLARE_JSON_REQUIRED_FIELDS(

20
src/node/rpc/test/frontend_test_infra.h
Просмотреть файл

@ -128,26 +128,6 @@ std::vector<uint8_t> create_signed_request(
return r.build_request();
}
template <typename T>
auto query_params(T script, bool compile)
{
json params;
if (compile)
params["bytecode"] = lua::compile(script);
else
params["text"] = script;
return params;
}
template <typename T>
auto read_params(const T& key, const string& table_name)
{
json params;
params["key"] = key;
params["table"] = table_name;
return params;
}
auto frontend_process(
MemberRpcFrontend& frontend,
const std::vector<uint8_t>& serialized_request,

446
src/node/rpc/test/member_voting_test.cpp
Просмотреть файл

@ -2,126 +2,6 @@
// Licensed under the Apache 2.0 License.
#include "node/rpc/test/frontend_test_infra.h"
DOCTEST_TEST_CASE("Member query/read")
{
// initialize the network state
NetworkState network;
auto gen_tx = network.tables->create_tx();
GenesisGenerator gen(network, gen_tx);
gen.init_values();
gen.create_service({});
ShareManager share_manager(network);
StubNodeContext context;
MemberRpcFrontend frontend(network, context, share_manager);
frontend.open();
const auto member_id = gen.add_member(member_cert);
DOCTEST_REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS);
const enclave::SessionContext member_session(
enclave::InvalidSessionId, member_cert.raw());
// put value to read
constexpr auto key = 123;
constexpr auto value = 456;
auto tx = network.tables->create_tx();
tx.rw(network.values)->put(key, value);
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
static constexpr auto query = R"xxx(
local tables = ...
return tables["public:ccf.internal.values"]:get(123)
)xxx";
DOCTEST_SUBCASE("Query: bytecode/script allowed access")
{
// set member ACL so that the VALUES table is accessible
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
bool compile = true;
do
{
const auto req = create_request(query_params(query, compile), "query");
const auto r = frontend_process(frontend, req, member_cert);
const auto result = parse_response_body<int>(r);
DOCTEST_CHECK(result == value);
compile = !compile;
} while (!compile);
}
DOCTEST_SUBCASE("Query: table not in ACL")
{
// set member ACL so that no table is accessible
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
auto req = create_request(query_params(query, true), "query");
const auto response = frontend_process(frontend, req, member_cert);
check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR);
}
DOCTEST_SUBCASE("Read: allowed access, key exists")
{
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
auto read_call =
create_request(read_params<int>(key, Tables::VALUES), "read");
const auto r = frontend_process(frontend, read_call, member_cert);
const auto result = parse_response_body<int>(r);
DOCTEST_CHECK(result == value);
}
DOCTEST_SUBCASE("Read: allowed access, key doesn't exist")
{
constexpr auto wrong_key = 321;
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
auto read_call =
create_request(read_params<int>(wrong_key, Tables::VALUES), "read");
const auto response = frontend_process(frontend, read_call, member_cert);
check_error(response, HTTP_STATUS_NOT_FOUND);
}
DOCTEST_SUBCASE("Read: access not allowed")
{
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
auto read_call =
create_request(read_params<int>(key, Tables::VALUES), "read");
const auto response = frontend_process(frontend, read_call, member_cert);
check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR);
}
DOCTEST_SUBCASE("Read: member is removed")
{
auto gen_tx = network.tables->create_tx();
GenesisGenerator gen(network, gen_tx);
gen.remove_member(member_id);
DOCTEST_REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS);
auto tx = network.tables->create_tx();
tx.rw(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES});
DOCTEST_CHECK(tx.commit() == kv::CommitResult::SUCCESS);
auto read_call =
create_request(read_params<int>(key, Tables::VALUES), "read");
const auto response = frontend_process(frontend, read_call, member_cert);
check_error(response, HTTP_STATUS_UNAUTHORIZED);
}
}
DOCTEST_TEST_CASE("Proposer ballot")
{
NetworkState network;
@ -312,238 +192,6 @@ struct TestNewMember
crypto::Pem cert;
};
DOCTEST_TEST_CASE("Add new members until there are 7 then reject")
{
logger::config::level() = logger::INFO;
constexpr auto initial_members = 3;
constexpr auto n_new_members = 7;
constexpr auto max_members = 8;
NetworkState network;
network.ledger_secrets = std::make_shared<LedgerSecrets>();
network.ledger_secrets->init();
init_network(network);
auto gen_tx = network.tables->create_tx();
GenesisGenerator gen(network, gen_tx);
gen.init_values();
gen.create_service({});
gen.init_configuration({1});
ShareManager share_manager(network);
StubNodeContext context;
// add three initial active members
// the proposer
auto proposer_id = gen.add_member({member_cert, dummy_enc_pubk});
gen.activate_member(proposer_id);
// the voters
const auto voter_a_cert = get_cert(1, kp);
auto voter_a = gen.add_member({voter_a_cert, dummy_enc_pubk});
gen.activate_member(voter_a);
const auto voter_b_cert = get_cert(2, kp);
auto voter_b = gen.add_member({voter_b_cert, dummy_enc_pubk});
gen.activate_member(voter_b);
set_whitelists(gen);
gen.set_gov_scripts(lua::Interpreter().invoke<json>(gov_script_file));
gen.open_service();
DOCTEST_REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS);
MemberRpcFrontend frontend(network, context, share_manager);
frontend.open();
vector<TestNewMember> new_members(n_new_members);
auto i = 0ul;
for (auto& new_member : new_members)
{
new_member.local_id = initial_members + i++;
// new member certificate
auto cert_pem = new_member.kp->self_sign(
fmt::format("CN=new member{}", new_member.local_id));
auto encryption_pub_key = dummy_enc_pubk;
auto cert_der = crypto::make_verifier(cert_pem)->cert_der();
new_member.service_id = crypto::Sha256Hash(cert_der).hex_str();
new_member.cert = cert_pem;
// check new_member id does not work before member is added
const auto read_next_req = create_request(
read_params(new_member.service_id, Tables::MEMBER_ACKS), "read");
const auto r = frontend_process(frontend, read_next_req, new_member.cert);
check_error(r, HTTP_STATUS_UNAUTHORIZED);
// propose new member, as proposer
Propose::In proposal;
proposal.script = std::string(R"xxx(
tables, member_info = ...
return Calls:call("new_member", member_info)
)xxx");
proposal.parameter["cert"] = cert_pem;
proposal.parameter["encryption_pub_key"] = dummy_enc_pubk;
const auto propose =
create_signed_request(proposal, "proposals", kp, member_cert);
ProposalId proposal_id;
{
const auto r = frontend_process(frontend, propose, member_cert);
const auto result = parse_response_body<Propose::Out>(r);
// the proposal should be accepted, but not succeed immediately
proposal_id = result.proposal_id;
DOCTEST_CHECK(result.state == ProposalState::OPEN);
}
{
// vote for own proposal
Script vote_yes("return true");
const auto vote = create_signed_request(
Vote{vote_yes},
fmt::format("proposals/{}/votes", proposal_id),
kp,
member_cert);
const auto r = frontend_process(frontend, vote, member_cert);
const auto result = parse_response_body<ProposalInfo>(r);
DOCTEST_CHECK(result.state == ProposalState::OPEN);
}
// read initial proposal, as second member
const Proposal initial_read =
get_proposal(frontend, proposal_id, voter_a_cert);
DOCTEST_CHECK(initial_read.proposer == proposer_id);
DOCTEST_CHECK(initial_read.script == proposal.script);
DOCTEST_CHECK(initial_read.parameter == proposal.parameter);
// vote as second member
Script vote_ballot(fmt::format(
R"xxx(
local tables, calls = ...
local n = 0
tables["public:ccf.gov.members.info"]:foreach( function(k, v) n = n + 1 end )
if n < {} then
return true
else
return false
end
)xxx",
max_members));
const auto vote = create_signed_request(
Vote{vote_ballot},
fmt::format("proposals/{}/votes", proposal_id),
kp,
voter_a_cert);
{
const auto r = frontend_process(frontend, vote, voter_a_cert);
const auto result = parse_response_body<ProposalInfo>(r);
if (new_member.local_id < max_members)
{
// vote should succeed
DOCTEST_CHECK(result.state == ProposalState::ACCEPTED);
// check that member with the new new_member cert can make RPCs now
auto r = frontend_process(frontend, read_next_req, new_member.cert);
DOCTEST_CHECK(r.status == HTTP_STATUS_OK);
// successful proposals are removed from the kv, so we can't confirm
// their final state
}
else
{
// vote should not succeed
DOCTEST_CHECK(result.state == ProposalState::OPEN);
// check that member with the new new_member cert can make RPCs now
check_error(
frontend_process(frontend, read_next_req, new_member.cert),
HTTP_STATUS_UNAUTHORIZED);
// re-read proposal, as second member
const Proposal final_read =
get_proposal(frontend, proposal_id, voter_a_cert);
DOCTEST_CHECK(final_read.proposer == proposer_id);
DOCTEST_CHECK(final_read.script == proposal.script);
DOCTEST_CHECK(final_read.parameter == proposal.parameter);
const auto my_vote = final_read.votes.find(voter_a);
DOCTEST_CHECK(my_vote != final_read.votes.end());
DOCTEST_CHECK(my_vote->second == vote_ballot);
}
}
}
DOCTEST_SUBCASE("ACK from newly added members")
{
// iterate over all new_members, except for the last one
for (auto new_member = new_members.cbegin(); new_member !=
new_members.cend() - (initial_members + n_new_members - max_members);
new_member++)
{
// (1) read ack entry
const auto read_state_digest_req = create_request(
read_params(new_member->service_id, Tables::MEMBER_ACKS), "read");
const auto ack0 = parse_response_body<StateDigest>(
frontend_process(frontend, read_state_digest_req, new_member->cert));
DOCTEST_REQUIRE(std::all_of(
ack0.state_digest.begin(), ack0.state_digest.end(), [](uint8_t i) {
return i == 0;
}));
{
// make sure that there is a signature in the signatures table since
// ack's depend on that
auto tx = network.tables->create_tx();
auto signatures = tx.rw(network.signatures);
PrimarySignature sig_value;
signatures->put(0, sig_value);
DOCTEST_REQUIRE(tx.commit() == kv::CommitResult::SUCCESS);
}
// (2) ask for a fresher digest of state
const auto freshen_state_digest_req =
create_request(nullptr, "ack/update_state_digest");
const auto freshen_state_digest = parse_response_body<StateDigest>(
frontend_process(frontend, freshen_state_digest_req, new_member->cert));
DOCTEST_CHECK(freshen_state_digest.state_digest != ack0.state_digest);
// (3) read ack entry again and check that the state digest has changed
const auto ack1 = parse_response_body<StateDigest>(
frontend_process(frontend, read_state_digest_req, new_member->cert));
DOCTEST_CHECK(ack0.state_digest != ack1.state_digest);
DOCTEST_CHECK(freshen_state_digest.state_digest == ack1.state_digest);
// (4) sign stale state and send it
StateDigest params;
params.state_digest = ack0.state_digest;
const auto send_stale_sig_req =
create_signed_request(params, "ack", new_member->kp, new_member->cert);
check_error(
frontend_process(frontend, send_stale_sig_req, new_member->cert),
HTTP_STATUS_BAD_REQUEST);
// (5) sign new state digest and send it
params.state_digest = ack1.state_digest;
const auto send_good_sig_req =
create_signed_request(params, "ack", new_member->kp, new_member->cert);
const auto good_response =
frontend_process(frontend, send_good_sig_req, new_member->cert);
DOCTEST_CHECK(good_response.status == HTTP_STATUS_NO_CONTENT);
// (6) read own member information
const auto read_cert_req = create_request(
read_params(new_member->service_id, Tables::MEMBER_CERTS), "read");
const auto cert = parse_response_body<crypto::Pem>(
frontend_process(frontend, read_cert_req, new_member->cert));
DOCTEST_CHECK(cert == new_member->cert);
const auto read_status_req = create_request(
read_params(new_member->service_id, Tables::MEMBER_INFO), "read");
const auto mi = parse_response_body<MemberDetails>(
frontend_process(frontend, read_status_req, new_member->cert));
DOCTEST_CHECK(mi.status == MemberStatus::ACTIVE);
}
}
}
DOCTEST_TEST_CASE("Accept node")
{
NetworkState network;
@ -581,12 +229,11 @@ DOCTEST_TEST_CASE("Accept node")
// check node exists with status pending
{
auto read_values =
create_request(read_params<NodeId>(node_id, Tables::NODES), "read");
const auto r = parse_response_body<NodeInfo>(
frontend_process(frontend, read_values, member_0_cert));
DOCTEST_CHECK(r.status == NodeStatus::PENDING);
auto tx = network.tables->create_tx();
auto nodes = tx.ro(network.nodes);
auto node = nodes->get(node_id);
DOCTEST_CHECK(node.has_value());
DOCTEST_CHECK(node->status == NodeStatus::PENDING);
}
// m0 proposes adding new node
@ -634,13 +281,13 @@ DOCTEST_TEST_CASE("Accept node")
frontend_process(frontend, vote, member_1_cert), ProposalState::ACCEPTED);
}
// check node exists with status pending
// check node exists with status trusted
{
const auto read_values =
create_request(read_params<NodeId>(node_id, Tables::NODES), "read");
const auto r = parse_response_body<NodeInfo>(
frontend_process(frontend, read_values, member_0_cert));
DOCTEST_CHECK(r.status == NodeStatus::TRUSTED);
auto tx = network.tables->create_tx();
auto nodes = tx.ro(network.nodes);
auto node = nodes->get(node_id);
DOCTEST_CHECK(node.has_value());
DOCTEST_CHECK(node->status == NodeStatus::TRUSTED);
}
// m0 proposes retire node
@ -686,11 +333,11 @@ DOCTEST_TEST_CASE("Accept node")
// check that node exists with status retired
{
auto read_values =
create_request(read_params<NodeId>(node_id, Tables::NODES), "read");
const auto r = parse_response_body<NodeInfo>(
frontend_process(frontend, read_values, member_0_cert));
DOCTEST_CHECK(r.status == NodeStatus::RETIRED);
auto tx = network.tables->create_tx();
auto nodes = tx.ro(network.nodes);
auto node = nodes->get(node_id);
DOCTEST_CHECK(node.has_value());
DOCTEST_CHECK(node->status == NodeStatus::RETIRED);
}
// check that retired node cannot be trusted
@ -1361,13 +1008,11 @@ DOCTEST_TEST_CASE("Passing operator change" * doctest::test_suite("operator"))
const ccf::Script vote_against("return false");
{
DOCTEST_INFO("Check node exists with status pending");
auto read_values =
create_request(read_params<NodeId>(node_id, Tables::NODES), "read");
const auto r = parse_response_body<NodeInfo>(
frontend_process(frontend, read_values, operator_cert));
DOCTEST_CHECK(r.status == NodeStatus::PENDING);
auto tx = network.tables->create_tx();
auto nodes = tx.ro(network.nodes);
auto node = nodes->get(node_id);
DOCTEST_CHECK(node.has_value());
DOCTEST_CHECK(node->status == NodeStatus::PENDING);
}
{
@ -1550,11 +1195,11 @@ DOCTEST_TEST_CASE(
{
DOCTEST_INFO("Check node exists with status pending");
const auto read_values =
create_request(read_params<NodeId>(node_id, Tables::NODES), "read");
const auto r = parse_response_body<NodeInfo>(
frontend_process(frontend, read_values, proposer_cert));
DOCTEST_CHECK(r.status == NodeStatus::PENDING);
auto tx = network.tables->create_tx();
auto nodes = tx.ro(network.nodes);
auto node = nodes->get(node_id);
DOCTEST_CHECK(node.has_value());
DOCTEST_CHECK(node->status == NodeStatus::PENDING);
}
{
@ -1656,21 +1301,18 @@ DOCTEST_TEST_CASE("User data")
frontend.open();
ccf::UserId user_id;
std::vector<uint8_t> read_user_info;
DOCTEST_SUBCASE("No initial user data")
{
user_id = gen.add_user({user_cert});
DOCTEST_REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS);
read_user_info =
create_request(read_params(user_id, Tables::USER_INFO), "read");
{
DOCTEST_INFO("user data is not initially set");
check_error(
frontend_process(frontend, read_user_info, member_cert),
HTTP_STATUS_NOT_FOUND);
auto tx = network.tables->create_tx();
auto users = tx.ro(network.user_info);
auto user = users->get(user_id);
DOCTEST_CHECK(!user.has_value());
}
}
@ -1680,14 +1322,13 @@ DOCTEST_TEST_CASE("User data")
user_id = gen.add_user({user_cert, user_data_string});
DOCTEST_REQUIRE(gen_tx.commit() == kv::CommitResult::SUCCESS);
read_user_info =
create_request(read_params(user_id, Tables::USER_INFO), "read");
{
DOCTEST_INFO("initial user data object can be read");
const auto read_response = parse_response_body<ccf::UserDetails>(
frontend_process(frontend, read_user_info, member_cert));
DOCTEST_CHECK(read_response.user_data == user_data_string);
auto tx = network.tables->create_tx();
auto users = tx.ro(network.user_info);
auto user = users->get(user_id);
DOCTEST_CHECK(user.has_value());
DOCTEST_CHECK(user->user_data == user_data_string);
}
}
@ -1726,9 +1367,12 @@ DOCTEST_TEST_CASE("User data")
}
DOCTEST_INFO("user data object can be read");
const auto read_response = parse_response_body<ccf::UserDetails>(
frontend_process(frontend, read_user_info, member_cert));
DOCTEST_CHECK(read_response.user_data == user_data_object);
auto tx = network.tables->create_tx();
auto users = tx.ro(network.user_info);
auto user = users->get(user_id);
DOCTEST_CHECK(user.has_value());
DOCTEST_CHECK(user->user_data == user_data_object);
}
{
@ -1762,9 +1406,11 @@ DOCTEST_TEST_CASE("User data")
}
DOCTEST_INFO("user data object can be read");
const auto response = parse_response_body<ccf::UserDetails>(
frontend_process(frontend, read_user_info, member_cert));
DOCTEST_CHECK(response.user_data == user_data_string);
auto tx = network.tables->create_tx();
auto users = tx.ro(network.user_info);
auto user = users->get(user_id);
DOCTEST_CHECK(user.has_value());
DOCTEST_CHECK(user->user_data == user_data_string);
}
}

37
tests/ca_certs.py
Просмотреть файл

@ -2,7 +2,6 @@
# Licensed under the Apache 2.0 License.
import os
import tempfile
import http
import infra.network
import infra.path
import infra.proc
@ -10,6 +9,7 @@ import infra.net
import infra.e2e_args
import suite.test_requirements as reqs
import ccf.proposal_generator
import json
from loguru import logger as LOG
@ -21,6 +21,7 @@ def test_cert_store(network, args):
primary, _ = network.find_nodes()
cert_name = "mycert"
raw_cert_name = cert_name.encode()
LOG.info("Member builds a ca cert update proposal with malformed cert")
with tempfile.NamedTemporaryFile("w") as f:
@ -55,29 +56,29 @@ def test_cert_store(network, args):
cert_pem_fp.write(cert_pem)
cert_pem_fp.write(cert2_pem)
cert_pem_fp.flush()
network.consortium.set_ca_cert_bundle(primary, cert_name, cert_pem_fp.name)
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.tls.ca_cert_bundles", "key": cert_name},
set_proposal = network.consortium.set_ca_cert_bundle(
primary, cert_name, cert_pem_fp.name
)
stored_cert = json.loads(
primary.get_ledger_public_state_at(set_proposal.completed_seqno)[
"public:ccf.gov.tls.ca_cert_bundles"
][raw_cert_name]
)
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
cert_ref = cert_pem + cert2_pem
cert_kv = r.body.json()
assert (
cert_ref == cert_kv
), f"stored cert not equal to input certs: {cert_ref} != {cert_kv}"
cert_ref == stored_cert
), f"input certs not equal to stored cert: {cert_ref} != {stored_cert}"
LOG.info("Member removes a ca cert")
network.consortium.remove_ca_cert_bundle(primary, cert_name)
remove_proposal = network.consortium.remove_ca_cert_bundle(primary, cert_name)
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.tls.ca_cert_bundles", "key": cert_name},
)
assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code
assert (
primary.get_ledger_public_state_at(remove_proposal.completed_seqno)[
"public:ccf.gov.tls.ca_cert_bundles"
][raw_cert_name]
== None
), "CA bundle was not removed"
return network

62
tests/governance.py
Просмотреть файл

@ -12,6 +12,7 @@ from infra.node import NodeStatus
import infra.e2e_args
import suite.test_requirements as reqs
import infra.logging_app as app
import json
from loguru import logger as LOG
@ -113,23 +114,22 @@ def test_no_quote(network, args):
def test_member_data(network, args):
assert args.initial_operator_count > 0
primary, _ = network.find_nodes()
with primary.client("member0") as mc:
def member_info(mid):
return mc.post(
"/gov/read", {"table": "public:ccf.gov.members.info", "key": mid}
).body.json()
latest_public_tables, _ = primary.get_latest_ledger_public_state()
members_info = latest_public_tables["public:ccf.gov.members.info"]
md_count = 0
for member in network.get_members():
if member.member_data:
assert (
member_info(member.service_id)["member_data"] == member.member_data
)
md_count += 1
else:
assert "member_data" not in member_info(member.service_id)
assert md_count == args.initial_operator_count
md_count = 0
for member in network.get_members():
stored_member_info = json.loads(members_info[member.service_id.encode()])
if member.member_data:
assert (
stored_member_info["member_data"] == member.member_data
), f'stored member data "{stored_member_info["member_data"]}" != expected "{member.member_data} "'
md_count += 1
else:
assert "member_data" not in stored_member_info
assert md_count == args.initial_operator_count
return network
@ -154,16 +154,9 @@ def test_service_principals(network, args):
principal_id = "0xdeadbeef"
def read_service_principal():
with node.client("member0") as mc:
return mc.post(
"/gov/read",
{"table": "public:ccf.gov.service_principals", "key": principal_id},
)
# Initially, there is nothing in this table
r = read_service_principal()
assert r.status_code == http.HTTPStatus.NOT_FOUND.value
latest_public_tables, _ = node.get_latest_ledger_public_state()
assert "public:ccf.gov.service_principals" not in latest_public_tables
# Create and accept a proposal which populates an entry in this table
principal_data = {"name": "Bob", "roles": ["Fireman", "Zookeeper"]}
@ -194,10 +187,15 @@ def test_service_principals(network, args):
network.consortium.vote_using_majority(node, proposal, ballot)
# Confirm it can be read
r = read_service_principal()
assert r.status_code == http.HTTPStatus.OK.value
j = r.body.json()
assert j == principal_data
latest_public_tables, _ = node.get_latest_ledger_public_state()
assert (
json.loads(
latest_public_tables["public:ccf.gov.service_principals"][
principal_id.encode()
]
)
== principal_data
)
# Create and accept a proposal which removes an entry from this table
if os.getenv("JS_GOVERNANCE"):
@ -219,9 +217,11 @@ def test_service_principals(network, args):
network.consortium.vote_using_majority(node, proposal, ballot)
# Confirm it is gone
r = read_service_principal()
assert r.status_code == http.HTTPStatus.NOT_FOUND.value
latest_public_tables, _ = node.get_latest_ledger_public_state()
assert (
principal_id.encode()
not in latest_public_tables["public:ccf.gov.service_principals"]
)
return network

147
tests/infra/consortium.py
Просмотреть файл

@ -14,6 +14,7 @@ import infra.node
import infra.crypto
import infra.member
import ccf.proposal_generator
import ccf.ledger
from infra.proposal import ProposalState
from loguru import logger as LOG
@ -75,46 +76,36 @@ class Consortium:
f"Successfully recovered member {local_id}: {new_member.service_id}"
)
# Retrieve state of service directly from ledger
latest_public_state, _ = remote_node.get_latest_ledger_public_state()
self.recovery_threshold = json.loads(
latest_public_state["public:ccf.gov.service.config"][
ccf.ledger.WELL_KNOWN_SINGLETON_TABLE_KEY
]
)["recovery_threshold"]
if not self.members:
LOG.warning("No consortium member to recover")
return
with remote_node.client(self.members[0].local_id) as c:
r = c.post(
"/gov/query",
{
"text": """tables = ...
members = {}
tables["public:ccf.gov.members.info"]:foreach(function(service_id, info)
table.insert(members, {service_id, info})
end)
return members
"""
},
)
for member_service_id, info in r.body.json():
status = info["status"]
member = self.get_member_by_service_id(member_service_id)
if member:
if (
infra.member.MemberStatus(status)
== infra.member.MemberStatus.ACTIVE
):
member.set_active()
else:
LOG.warning(
f"Keys and certificates for consortium member {member_service_id} do not exist locally"
)
for id_bytes, info_bytes in latest_public_state[
"public:ccf.gov.members.info"
].items():
member_id = id_bytes.decode()
member_info = json.loads(info_bytes)
r = c.post(
"/gov/query",
{
"text": """tables = ...
return tables["public:ccf.gov.service.config"]:get(0)
"""
},
)
self.recovery_threshold = r.body.json()["recovery_threshold"]
status = member_info["status"]
member = self.get_member_by_service_id(member_id)
if member:
if (
infra.member.MemberStatus(status)
== infra.member.MemberStatus.ACTIVE
):
member.set_active()
else:
LOG.warning(
f"Keys and certificates for consortium member {member_id} do not exist locally"
)
def set_authenticate_session(self, flag):
self.authenticate_session = flag
@ -271,35 +262,13 @@ class Consortium:
view = response.view
ccf.commit.wait_for_commit(c, seqno, view, timeout=timeout)
if proposal.state != ProposalState.ACCEPTED:
if proposal.state == ProposalState.ACCEPTED:
proposal.set_completed(seqno, view)
else:
raise infra.proposal.ProposalNotAccepted(proposal)
return proposal
def get_proposals(self, remote_node):
script = """
tables = ...
local proposals = {}
tables["public:ccf.gov.proposals"]:foreach( function(k, v)
proposals[tostring(k)] = v;
end )
return proposals;
"""
proposals = []
member = self.get_any_active_member()
with remote_node.client(*member.auth()) as c:
r = c.post("/gov/query", {"text": script})
assert r.status_code == http.HTTPStatus.OK.value
for proposal_id, attr in r.body.json().items():
proposals.append(
infra.proposal.Proposal(
proposal_id=proposal_id,
proposer_id=attr["proposer"],
state=infra.proposal.ProposalState(attr["state"]),
)
)
return proposals
def retire_node(self, remote_node, node_to_retire):
LOG.info(f"Retiring node {node_to_retire.local_node_id}")
if os.getenv("JS_GOVERNANCE"):
@ -313,12 +282,8 @@ class Consortium:
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
self.vote_using_majority(remote_node, proposal, careful_vote)
member = self.get_any_active_member()
with remote_node.client(*member.auth(write=True)) as c:
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.nodes.info", "key": node_to_retire.node_id},
)
with remote_node.client() as c:
r = c.get(f"/node/network/nodes/{node_to_retire.node_id}")
assert r.body.json()["status"] == infra.node.NodeStatus.RETIRED.value
def trust_node(self, remote_node, node_id, timeout=3):
@ -413,6 +378,11 @@ class Consortium:
# Large apps take a long time to process - wait longer than normal for commit
return self.vote_using_majority(remote_node, proposal, careful_vote, timeout=10)
def remove_js_app(self, remote_node):
proposal_body, careful_vote = ccf.proposal_generator.remove_js_app()
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
return self.vote_using_majority(remote_node, proposal, careful_vote)
def set_jwt_issuer(self, remote_node, json_path):
proposal_body, careful_vote = self.make_proposal("set_jwt_issuer", json_path)
proposal = self.get_any_active_member().propose(remote_node, proposal_body)
@ -521,39 +491,13 @@ class Consortium:
def check_for_service(self, remote_node, status):
"""
Check via the member frontend of the given node that the certificate
associated with current CCF service signing key has been recorded in
Check the certificate associated with current CCF service signing key has been recorded in
the KV store with the appropriate status.
"""
# When opening the service in BFT, the first transaction to be
# completed when f = 1 takes a significant amount of time
member = self.get_any_active_member()
with remote_node.client(*member.auth()) as c:
r = c.post(
"/gov/query",
{
"text": """tables = ...
service = tables["public:ccf.gov.service.info"]:get(0)
if service == nil then
LOG_DEBUG("Service is nil")
else
LOG_DEBUG("Service version: ", tostring(service.version))
LOG_DEBUG("Service status: ", tostring(service.status_code))
cert_len = #service.cert
LOG_DEBUG("Service cert len: ", tostring(cert_len))
LOG_DEBUG("Service cert bytes: " ..
tostring(service.cert[math.ceil(cert_len / 4)]) .. " " ..
tostring(service.cert[math.ceil(cert_len / 3)]) .. " " ..
tostring(service.cert[math.ceil(cert_len / 2)])
)
end
return service
"""
},
timeout=3,
)
current_status = r.body.json()["status"]
current_cert = r.body.json()["cert"]
with remote_node.client() as c:
r = c.get("/node/network")
current_status = r.body.json()["service_status"]
current_cert = r.body.json()["service_certificate"]
expected_cert = open(
os.path.join(self.common_dir, "networkcert.pem"), "rb"
@ -567,11 +511,8 @@ class Consortium:
), f"Service status {current_status} (expected {status.value})"
def _check_node_exists(self, remote_node, node_id, node_status=None):
member = self.get_any_active_member()
with remote_node.client(*member.auth()) as c:
r = c.post(
"/gov/read", {"table": "public:ccf.gov.nodes.info", "key": node_id}
)
with remote_node.client() as c:
r = c.get(f"/node/network/nodes/{node_id}")
if r.status_code != http.HTTPStatus.OK.value or (
node_status and r.body.json()["status"] != node_status.value

29
tests/infra/node.py
Просмотреть файл

@ -8,9 +8,11 @@ import infra.remote
import infra.net
import infra.path
import ccf.clients
import ccf.ledger
import os
import socket
import re
import time
from loguru import logger as LOG
@ -316,6 +318,33 @@ class Node:
f"Node {self.local_node_id} failed to join the network"
) from e
def get_ledger_public_state_at(self, seqno, timeout=3):
end_time = time.time() + timeout
while time.time() < end_time:
try:
ledger = ccf.ledger.Ledger(self.remote.ledger_path())
tx = ledger.get_transaction(seqno)
return tx.get_public_domain().get_tables()
except Exception:
time.sleep(0.1)
raise TimeoutError(
f"Could not read transaction at seqno {seqno} from ledger {self.remote.ledger_path()}"
)
def get_latest_ledger_public_state(self, timeout=3):
end_time = time.time() + timeout
while time.time() < end_time:
try:
ledger = ccf.ledger.Ledger(self.remote.ledger_path())
return ledger.get_latest_public_state()
except Exception:
time.sleep(0.1)
raise TimeoutError(
f"Could not read latest state from ledger {self.remote.ledger_path()}"
)
def get_ledger(self, include_read_only_dirs=False):
"""
Triage committed and un-committed (i.e. current) ledger files

7
tests/infra/proposal.py
Просмотреть файл

@ -41,5 +41,12 @@ class Proposal:
self.view = view
self.seqno = seqno
self.completed_view = view if state == ProposalState.ACCEPTED else None
self.completed_seqno = seqno if state == ProposalState.ACCEPTED else None
def set_completed(self, seqno, view):
self.completed_seqno = seqno
self.completed_view = view
def increment_votes_for(self):
self.votes_for += 1

31
tests/js-modules/modules.py
Просмотреть файл

@ -15,7 +15,6 @@ import infra.net
import infra.e2e_args
import infra.crypto
import suite.test_requirements as reqs
import ccf.proposal_generator
import openapi_spec_validator
from loguru import logger as LOG
@ -62,16 +61,21 @@ def test_app_bundle(network, args):
# Testing the bundle archive support of the Python client here.
# Plain bundle folders are tested in the npm-based app tests.
bundle_dir = os.path.join(PARENT_DIR, "js-app-bundle")
raw_module_name = "/math.js".encode()
with tempfile.TemporaryDirectory(prefix="ccf") as tmp_dir:
bundle_path = shutil.make_archive(
os.path.join(tmp_dir, "bundle"), "zip", bundle_dir
)
network.consortium.set_js_app(primary, bundle_path)
set_js_proposal = network.consortium.set_js_app(primary, bundle_path)
LOG.info("Verifying that modules and endpoints were added")
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post("/gov/read", {"table": "public:ccf.gov.modules", "key": "/math.js"})
assert r.status_code == http.HTTPStatus.OK, r.status_code
assert (
raw_module_name
in primary.get_ledger_public_state_at(set_js_proposal.completed_seqno)[
"public:ccf.gov.modules"
]
), "Module was not added"
LOG.info("Verifying that app was deployed")
with primary.client("user0") as c:
valid_body = {"op": "sub", "left": 82, "right": 40}
@ -89,20 +93,19 @@ def test_app_bundle(network, args):
validate_openapi(c)
LOG.info("Removing js app")
proposal_body, careful_vote = ccf.proposal_generator.remove_js_app()
proposal = network.consortium.get_any_active_member().propose(
primary, proposal_body
)
network.consortium.vote_using_majority(primary, proposal, careful_vote)
remove_js_proposal = network.consortium.remove_js_app(primary)
LOG.info("Verifying that modules and endpoints were removed")
with primary.client("user0") as c:
r = c.post("/app/compute", valid_body)
assert r.status_code == http.HTTPStatus.NOT_FOUND, r.status_code
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post("/gov/read", {"table": "public:ccf.gov.modules", "key": "/math.js"})
assert r.status_code == http.HTTPStatus.NOT_FOUND, r.status_code
assert (
primary.get_ledger_public_state_at(remove_js_proposal.completed_seqno)[
"public:ccf.gov.modules"
][raw_module_name]
is None
), "Module was not removed"
return network

122
tests/jwt_test.py
Просмотреть файл

@ -5,7 +5,6 @@ import tempfile
import json
import time
import base64
import http
from http.server import HTTPServer, BaseHTTPRequestHandler
from http import HTTPStatus
import ssl
@ -40,6 +39,7 @@ def test_jwt_without_key_policy(network, args):
cert_pem = infra.crypto.generate_cert(key_priv_pem)
kid = "my_kid"
issuer = "my_issuer"
raw_kid = kid.encode()
LOG.info("Try to add JWT signing key without matching issuer")
with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as jwks_fp:
@ -77,50 +77,43 @@ def test_jwt_without_key_policy(network, args):
with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as jwks_fp:
json.dump(create_jwks(kid, cert_pem), jwks_fp)
jwks_fp.flush()
network.consortium.set_jwt_public_signing_keys(primary, issuer, jwks_fp.name)
LOG.info("Check if JWT signing key was stored correctly")
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read", {"table": "public:ccf.gov.jwt.public_signing_keys", "key": kid}
set_jwt_proposal = network.consortium.set_jwt_public_signing_keys(
primary, issuer, jwks_fp.name
)
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
# Note that /gov/read returns all data as JSON.
# Here, the stored data is a uint8 array, therefore it
# is returned as an array of integers.
cert_kv_der = base64.b64decode(r.body.json())
cert_kv_pem = infra.crypto.cert_der_to_pem(cert_kv_der)
stored_jwt_signing_key = primary.get_ledger_public_state_at(
set_jwt_proposal.completed_seqno
)["public:ccf.gov.jwt.public_signing_keys"][raw_kid]
stored_cert = infra.crypto.cert_der_to_pem(stored_jwt_signing_key)
assert infra.crypto.are_certs_equal(
cert_pem, cert_kv_pem
), "stored cert not equal to input cert"
cert_pem, stored_cert
), "input cert is not equal to stored cert"
LOG.info("Remove JWT issuer")
network.consortium.remove_jwt_issuer(primary, issuer)
remove_jwt_proposal = network.consortium.remove_jwt_issuer(primary, issuer)
LOG.info("Check if JWT signing key was deleted")
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read", {"table": "public:ccf.gov.jwt.public_signing_keys", "key": kid}
)
assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code
assert (
primary.get_ledger_public_state_at(remove_jwt_proposal.completed_seqno)[
"public:ccf.gov.jwt.public_signing_keys"
][raw_kid]
is None
), "JWT issuer was not removed"
LOG.info("Add JWT issuer with initial keys")
with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as metadata_fp:
json.dump({"issuer": issuer, "jwks": create_jwks(kid, cert_pem)}, metadata_fp)
metadata_fp.flush()
network.consortium.set_jwt_issuer(primary, metadata_fp.name)
set_jwt_issuer = network.consortium.set_jwt_issuer(primary, metadata_fp.name)
LOG.info("Check if JWT signing key was stored correctly")
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read", {"table": "public:ccf.gov.jwt.public_signing_keys", "key": kid}
)
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
cert_kv_der = base64.b64decode(r.body.json())
cert_kv_pem = infra.crypto.cert_der_to_pem(cert_kv_der)
stored_jwt_signing_key = primary.get_ledger_public_state_at(
set_jwt_issuer.completed_seqno
)["public:ccf.gov.jwt.public_signing_keys"][raw_kid]
stored_cert = infra.crypto.cert_der_to_pem(stored_jwt_signing_key)
assert infra.crypto.are_certs_equal(
cert_pem, cert_kv_pem
), "stored cert not equal to input cert"
cert_pem, stored_cert
), "input cert is not equal to stored cert"
return network
@ -233,20 +226,16 @@ def test_jwt_with_sgx_key_filter(network, args):
jwks = {"keys": non_oe_jwks["keys"] + oe_jwks["keys"]}
json.dump(jwks, jwks_fp)
jwks_fp.flush()
network.consortium.set_jwt_public_signing_keys(primary, issuer, jwks_fp.name)
set_jwt_proposal = network.consortium.set_jwt_public_signing_keys(
primary, issuer, jwks_fp.name
)
LOG.info("Check that only SGX cert was added")
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.jwt.public_signing_keys", "key": non_oe_kid},
)
assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.jwt.public_signing_keys", "key": oe_kid},
)
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
stored_jwt_signing_keys = primary.get_ledger_public_state_at(
set_jwt_proposal.completed_seqno
)["public:ccf.gov.jwt.public_signing_keys"]
assert non_oe_kid.encode() not in stored_jwt_signing_keys
assert oe_kid.encode() in stored_jwt_signing_keys
return network
@ -308,23 +297,19 @@ class OpenIDProviderServer(AbstractContextManager):
def check_kv_jwt_key_matches(network, kid, cert_pem):
primary, _ = network.find_nodes()
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/read",
{"table": "public:ccf.gov.jwt.public_signing_keys", "key": kid},
)
if cert_pem is None:
assert r.status_code == http.HTTPStatus.NOT_FOUND.value, r.status_code
else:
assert r.status_code == http.HTTPStatus.OK.value, r.status_code
# Note that /gov/read returns all data as JSON.
# Here, the stored data is a uint8 array, therefore it
# is returned as an array of integers.
cert_kv_der = base64.b64decode(r.body.json())
cert_kv_pem = infra.crypto.cert_der_to_pem(cert_kv_der)
assert infra.crypto.are_certs_equal(
cert_pem, cert_kv_pem
), "stored cert not equal to input cert"
latest_public_state, _ = primary.get_latest_ledger_public_state()
latest_jwt_signing_key = latest_public_state[
"public:ccf.gov.jwt.public_signing_keys"
]
if cert_pem is None:
assert kid.encode() not in latest_jwt_signing_key
else:
stored_cert = infra.crypto.cert_der_to_pem(latest_jwt_signing_key[kid.encode()])
assert infra.crypto.are_certs_equal(
cert_pem, stored_cert
), "input cert is not equal to stored cert"
def get_jwt_refresh_endpoint_metrics(network) -> dict:
@ -378,11 +363,12 @@ def test_jwt_key_auto_refresh(network, args):
metadata_fp.flush()
network.consortium.set_jwt_issuer(primary, metadata_fp.name)
LOG.info("Check that keys got refreshed")
# Note: refresh interval is set to 1s, see network args below.
with_timeout(
lambda: check_kv_jwt_key_matches(network, kid, cert_pem), timeout=5
)
LOG.info("Check that keys got refreshed")
# Note: refresh interval is set to 1s, see network args below.
with_timeout(
lambda: check_kv_jwt_key_matches(network, kid, cert_pem),
timeout=5,
)
LOG.info("Check that JWT refresh endpoint has no failures")
m = get_jwt_refresh_endpoint_metrics(network)

17
tests/memberclient.py
Просмотреть файл

@ -154,15 +154,6 @@ def test_governance(network, args):
assert r.status_code == 200, r.body.text()
assert r.body.json()["state"] == infra.proposal.ProposalState.OPEN.value
else:
proposals = network.consortium.get_proposals(primary)
proposal_entry = next(
(p for p in proposals if p.proposal_id == new_member_proposal.proposal_id),
None,
)
assert proposal_entry
assert proposal_entry.state == ProposalState.OPEN
LOG.info("Rest of consortium accept the proposal")
network.consortium.vote_using_majority(node, new_member_proposal, careful_vote)
assert new_member_proposal.state == ProposalState.ACCEPTED
@ -239,14 +230,6 @@ def test_governance(network, args):
assert (
r.body.json()["state"] == infra.proposal.ProposalState.WITHDRAWN.value
)
else:
proposals = network.consortium.get_proposals(primary)
proposal_entry = next(
(p for p in proposals if p.proposal_id == proposal.proposal_id),
None,
)
assert proposal_entry
assert proposal_entry.state == ProposalState.WITHDRAWN
LOG.debug("Further withdraw proposals fail")
response = new_member.withdraw(node, proposal)

25
tests/suite/test_requirements.py
Просмотреть файл

@ -98,23 +98,16 @@ def sufficient_recovery_member_count():
def can_kill_n_nodes(nodes_to_kill_count):
def check(network, args, *nargs, **kwargs):
primary, _ = network.find_primary()
with primary.client(network.consortium.get_any_active_member().local_id) as c:
r = c.post(
"/gov/query",
{
"text": """tables = ...
trusted_nodes_count = 0
tables["public:ccf.gov.nodes.info"]:foreach(function(node_id, details)
if details["status"] == "Trusted" then
trusted_nodes_count = trusted_nodes_count + 1
end
end)
return trusted_nodes_count
"""
},
)
with primary.client() as c:
r = c.get("/node/network/nodes")
trusted_nodes_count = r.body.json()
trusted_nodes_count = len(
[
node
for node in r.body.json()["nodes"]
if node["status"] == infra.node.NodeStatus.TRUSTED.value
]
)
running_nodes_count = len(network.get_joined_nodes())
would_leave_nodes_count = running_nodes_count - nodes_to_kill_count
minimum_nodes_to_run_count = ceil((trusted_nodes_count + 1) / 2)