From 4987c0ad7245ea2c4a8aa0a2234e2b46d6ec6ca1 Mon Sep 17 00:00:00 2001 From: Julien Maffre <42961061+jumaffre@users.noreply.github.com> Date: Thu, 4 Feb 2021 15:35:33 +0000 Subject: [PATCH] Split shares KV table (#2140) --- CMakeLists.txt | 2 +- src/crypto/symmetric_key.h | 3 +- src/node/entities.h | 2 + src/node/genesis_gen.h | 6 - src/node/ledger_secrets.h | 14 +- src/node/network_tables.h | 4 +- src/node/node_state.h | 110 ++++++-------- src/node/rpc/member_frontend.h | 9 +- src/node/rpc/test/member_voting_test.cpp | 2 +- src/node/rpc/test/node_stub.h | 8 +- src/node/secret_broadcast.h | 2 +- src/node/share_manager.h | 181 +++++++++++++++-------- src/node/shares.h | 136 ++++++++++++----- tests/e2e_suite.py | 3 +- tests/suite/test_suite.py | 10 ++ 15 files changed, 295 insertions(+), 197 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dfb26edf88..9db7f66a54 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -526,7 +526,7 @@ if(BUILD_TESTS) NAME recovery_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py CONSENSUS cft - ADDITIONAL_ARGS --recovery 2 + ADDITIONAL_ARGS --recovery 3 ) add_e2e_test( diff --git a/src/crypto/symmetric_key.h b/src/crypto/symmetric_key.h index 01827f373e..d117ccb0eb 100644 --- a/src/crypto/symmetric_key.h +++ b/src/crypto/symmetric_key.h @@ -124,10 +124,9 @@ namespace crypto return serial; } - void apply(const std::vector& serial) + void deserialise(const std::vector& serial) { auto size = serial.size(); - auto data_ = serial.data(); hdr = serialized::read(data_, size, GcmHeader<>::RAW_DATA_SIZE); cipher = serialized::read(data_, size, size); diff --git a/src/node/entities.h b/src/node/entities.h index 31015c0872..80aa3bcbfe 100644 --- a/src/node/entities.h +++ b/src/node/entities.h @@ -83,6 +83,8 @@ namespace ccf static constexpr auto GOV_HISTORY = "public:ccf.gov.governance.history"; static constexpr auto SERVICE = "public:ccf.gov.service"; static constexpr auto SHARES = "public:ccf.gov.shares"; + static constexpr auto ENCRYPTED_LEDGER_SECRETS = + "public:ccf.internal.encrypted_ledger_secrets"; static constexpr auto CONFIGURATION = "public:ccf.gov.config"; static constexpr auto SUBMITTED_SHARES = "public:ccf.gov.submitted_shares"; static constexpr auto SNAPSHOT_EVIDENCE = diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index 8c49f96d8d..72a5fc125e 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -443,12 +443,6 @@ namespace ccf codeid->put(node_code_id, CodeStatus::ALLOWED_TO_JOIN); } - void add_key_share_info(const RecoverySharesInfo& key_share_info) - { - auto shares = tx.rw(tables.shares); - shares->put(0, key_share_info); - } - bool set_recovery_threshold(size_t threshold, bool allow_zero = false) { auto config = tx.rw(tables.config); diff --git a/src/node/ledger_secrets.h b/src/node/ledger_secrets.h index ea20c93119..924e46654b 100644 --- a/src/node/ledger_secrets.h +++ b/src/node/ledger_secrets.h @@ -46,6 +46,7 @@ namespace ccf } using LedgerSecretsMap = std::map; + using VersionedLedgerSecret = LedgerSecretsMap::value_type; class LedgerSecrets { @@ -160,7 +161,7 @@ namespace ccf self = id; } - LedgerSecretsMap::value_type get_latest(kv::Tx& tx) + VersionedLedgerSecret get_latest(kv::Tx& tx) { std::lock_guard guard(lock); @@ -172,12 +173,10 @@ namespace ccf "Could not retrieve latest ledger secret: no secret set"); } - const auto& latest_ledger_secret = ledger_secrets.rbegin(); - return std::make_pair( - latest_ledger_secret->first, latest_ledger_secret->second); + return *ledger_secrets.rbegin(); } - std::pair> + std::pair> get_latest_and_penultimate(kv::Tx& tx) { std::lock_guard guard(lock); @@ -193,11 +192,10 @@ namespace ccf const auto& latest_ledger_secret = ledger_secrets.rbegin(); if (ledger_secrets.size() < 2) { - return std::make_pair(latest_ledger_secret->second, std::nullopt); + return std::make_pair(*latest_ledger_secret, std::nullopt); } return std::make_pair( - latest_ledger_secret->second, - std::next(ledger_secrets.rbegin())->second); + *latest_ledger_secret, *std::next(latest_ledger_secret)); } LedgerSecretsMap get( diff --git a/src/node/network_tables.h b/src/node/network_tables.h index c85a5fb8fe..d9521a59ba 100644 --- a/src/node/network_tables.h +++ b/src/node/network_tables.h @@ -55,7 +55,8 @@ namespace ccf CodeIDs node_code_ids; MemberAcks member_acks; GovernanceHistory governance_history; - Shares shares; + RecoveryShares shares; + EncryptedLedgerSecretsInfo encrypted_ledger_secrets; SubmittedShares submitted_shares; Configuration config; @@ -121,6 +122,7 @@ namespace ccf member_acks(Tables::MEMBER_ACKS), governance_history(Tables::GOV_HISTORY), shares(Tables::SHARES), + encrypted_ledger_secrets(Tables::ENCRYPTED_LEDGER_SECRETS), submitted_shares(Tables::SUBMITTED_SHARES), config(Tables::CONFIGURATION), ca_certs(Tables::CA_CERT_DERS), diff --git a/src/node/node_state.h b/src/node/node_state.h index 6d33bc4977..80177b0018 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -161,7 +161,7 @@ namespace ccf crypto::Sha256Hash recovery_root; std::vector view_history; consensus::Index last_recovered_signed_idx = 1; - std::list recovery_ledger_secrets; + RecoveredEncryptedLedgerSecrets recovery_ledger_secrets; consensus::Index ledger_idx = 0; struct StartupSnapshotInfo @@ -380,7 +380,7 @@ namespace ccf setup_snapshotter(config.snapshot_tx_interval); bool from_snapshot = !config.startup_snapshot.empty(); - setup_recovery_hook(from_snapshot); + setup_recovery_hook(); if (from_snapshot) { @@ -480,6 +480,14 @@ namespace ccf setup_progress_tracker(); setup_history(); + if (resp.network_info.public_only) + { + last_recovered_signed_idx = + resp.network_info.last_recovered_signed_idx; + setup_recovery_hook(); + snapshotter->set_snapshot_generation(false); + } + if (startup_snapshot_info) { // It is only possible to deserialise the entire snapshot then, @@ -538,11 +546,6 @@ namespace ccf if (resp.network_info.public_only) { - last_recovered_signed_idx = - resp.network_info.last_recovered_signed_idx; - setup_recovery_hook(startup_snapshot_info != nullptr); - snapshotter->set_snapshot_generation(false); - sm.advance(State::partOfPublicNetwork); } else @@ -1004,8 +1007,7 @@ namespace ccf // Shares for the new ledger secret can only be issued now, once the // previous ledger secrets have been recovered - share_manager.issue_shares_on_recovery( - tx, last_recovered_signed_idx + 1); + share_manager.issue_recovery_shares(tx); GenesisGenerator g(network, tx); if (!g.open_service()) { @@ -1116,8 +1118,8 @@ namespace ccf std::lock_guard guard(lock); sm.expect(State::partOfPublicNetwork); - auto restored_ledger_secrets = - share_manager.restore_recovery_shares_info(tx, recovery_ledger_secrets); + auto restored_ledger_secrets = share_manager.restore_recovery_shares_info( + tx, std::move(recovery_ledger_secrets)); // Broadcast decrypted ledger secrets to other nodes for them to initiate // private recovery too @@ -1139,7 +1141,6 @@ namespace ccf ledger_idx = recovery_store->current_version(); read_ledger_idx(++ledger_idx); - recovery_ledger_secrets.clear(); sm.advance(State::readingPrivateLedger); } @@ -1280,7 +1281,7 @@ namespace ccf // once the local hook on the secrets table has been triggered. auto new_ledger_secret = make_ledger_secret(); - share_manager.issue_shares_on_rekey(tx, new_ledger_secret); + share_manager.issue_recovery_shares(tx, new_ledger_secret); LedgerSecretsBroadcast::broadcast_new( network, node_encrypt_kp, tx, std::move(new_ledger_secret)); @@ -1595,68 +1596,51 @@ namespace ccf return last_recovered_signed_idx; } - void setup_recovery_hook(bool from_snapshot) + void setup_recovery_hook() { - // When recoverying from a snapshot, the first secret is valid from the - // version at which it was recorded - static bool is_first_secret = !from_snapshot; - network.tables->set_map_hook( - network.shares.get_name(), - network.shares.wrap_map_hook( - [this](kv::Version version, const Shares::Write& w) + network.encrypted_ledger_secrets.get_name(), + network.encrypted_ledger_secrets.wrap_map_hook( + [this]( + kv::Version version, const EncryptedLedgerSecretsInfo::Write& w) -> kv::ConsensusHookPtr { - for (const auto& [k, opt_v] : w) + if (w.size() > 1) { - if (!opt_v.has_value()) - { - throw std::logic_error( - fmt::format("Unexpected: removal from shares table ({})", k)); - } - - const auto& v = opt_v.value(); - - kv::Version ledger_secret_version; - if (is_first_secret) - { - // Special case for the first recovery share issuing (at network - // open), which is applicable from the very first transaction. - ledger_secret_version = 1; - is_first_secret = false; - } - else - { - // If the version is not set (rekeying), use the version - // from the hook plus one. Otherwise (recovery), use the - // version specified. - ledger_secret_version = - v.wrapped_latest_ledger_secret.version == kv::NoVersion ? - (version + 1) : - v.wrapped_latest_ledger_secret.version; - } - - // No encrypted ledger secret are stored in the case of a pure - // re-share (i.e. no ledger rekey). - if ( - !v.encrypted_previous_ledger_secret.empty() || - ledger_secret_version == 1) - { - LOG_TRACE_FMT( - "Adding one encrypted recovery ledger secret at {}", - ledger_secret_version); - - recovery_ledger_secrets.push_back( - {ledger_secret_version, v.encrypted_previous_ledger_secret}); - } + throw std::logic_error(fmt::format( + "Transaction contains {} writes to map {}, expected one", + w.size(), + network.encrypted_ledger_secrets.get_name())); } + auto encrypted_ledger_secret_info = w.at(0); + if (!encrypted_ledger_secret_info.has_value()) + { + throw std::logic_error(fmt::format( + "Removal from {} table", + network.encrypted_ledger_secrets.get_name())); + } + + // If the version of the next ledger secret is not set, deduce it + // from the hook version (i.e. ledger rekey) + if (!encrypted_ledger_secret_info->next_version.has_value()) + { + encrypted_ledger_secret_info->next_version = version + 1; + } + + LOG_DEBUG_FMT( + "Recovering encrypted ledger secret valid at seqno {}", version); + + recovery_ledger_secrets.emplace_back( + std::move(encrypted_ledger_secret_info.value())); + return kv::ConsensusHookPtr(nullptr); })); } void reset_recovery_hook() { - network.tables->unset_map_hook(network.shares.get_name()); + network.tables->unset_map_hook( + network.encrypted_ledger_secrets.get_name()); } void setup_n2n_channels() diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 59f8868ee0..bfdffa5c84 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -936,7 +936,7 @@ namespace ccf // allocated to each recovery member try { - share_manager.issue_shares(tx); + share_manager.issue_recovery_shares(tx); } catch (const std::logic_error& e) { @@ -975,7 +975,7 @@ namespace ccf const ProposalId& proposal_id, kv::Tx& tx, const nlohmann::json&) { try { - share_manager.issue_shares(tx); + share_manager.shuffle_recovery_shares(tx); } catch (const std::logic_error& e) { @@ -1008,10 +1008,9 @@ namespace ccf return false; } - // Update recovery shares (same number of shares) try { - share_manager.issue_shares(tx); + share_manager.shuffle_recovery_shares(tx); } catch (const std::logic_error& e) { @@ -1715,7 +1714,7 @@ namespace ccf // member, all recovery members are allocated new recovery shares try { - share_manager.issue_shares(ctx.tx); + share_manager.shuffle_recovery_shares(ctx.tx); } catch (const std::logic_error& e) { diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 3e2fdb7a30..d258333d62 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -1784,7 +1784,7 @@ DOCTEST_TEST_CASE("Submit recovery shares") members[id] = {cert, enc_kp}; } gen.set_recovery_threshold(recovery_threshold); - share_manager.issue_shares(gen_tx); + share_manager.issue_recovery_shares(gen_tx); gen.finalize(); frontend.open(); } diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index b847664122..06f6a9d743 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -91,7 +91,13 @@ namespace ccf void initiate_private_recovery(kv::Tx& tx) override { - share_manager.restore_recovery_shares_info(tx, {}); + kv::Version current_ledger_secret_version = 1; + RecoveredEncryptedLedgerSecrets recovered_secrets; + recovered_secrets.push_back( + EncryptedLedgerSecretInfo{std::nullopt, current_ledger_secret_version}); + + share_manager.restore_recovery_shares_info( + tx, std::move(recovered_secrets)); } }; } \ No newline at end of file diff --git a/src/node/secret_broadcast.h b/src/node/secret_broadcast.h index 17f0dea8df..a99ecc45c0 100644 --- a/src/node/secret_broadcast.h +++ b/src/node/secret_broadcast.h @@ -102,7 +102,7 @@ namespace ccf const std::vector& cipher) { crypto::GcmCipher gcmcipher; - gcmcipher.apply(cipher); + gcmcipher.deserialise(cipher); std::vector plain(gcmcipher.cipher.size()); crypto::KeyAesGcm primary_shared_key( diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 10f44c510c..3b023a3f65 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -69,7 +69,7 @@ namespace ccf const std::vector& wrapped_latest_ledger_secret) { crypto::GcmCipher encrypted_ls; - encrypted_ls.apply(wrapped_latest_ledger_secret); + encrypted_ls.deserialise(wrapped_latest_ledger_secret); std::vector decrypted_ls(encrypted_ls.cipher.size()); if (!crypto::KeyAesGcm(data).decrypt( @@ -86,20 +86,13 @@ namespace ccf } }; - // During recovery, a list of RecoveredLedgerSecret is constructed from a - // local hook. - struct RecoveredLedgerSecret - { - // Version at which the next ledger secret is applicable from - kv::Version next_version; + // During recovery, a list of EncryptedLedgerSecretInfo is constructed + // from the local hook on the encrypted ledger secrets table. + using RecoveredEncryptedLedgerSecrets = std::list; - // Previous ledger secret, encrypted with the current ledger secret - std::vector encrypted_ledger_secret; - }; - - // The ShareManager class provides the interface between the ledger secrets, - // the ccf.shares and ccf.submitted_shares KV tables and the rest of the - // service. In particular, it is used to: + // The ShareManager class provides the interface between the ledger secrets + // object and the shares, ledger secrets and submitted shares KV tables. In + // particular, it is used to: // - Issue new recovery shares whenever required (e.g. on startup, rekey and // membership updates) // - Re-assemble the ledger secrets on recovery, once a threshold of members @@ -153,44 +146,65 @@ namespace ccf return encrypted_shares; } + void shuffle_recovery_shares( + kv::Tx& tx, const LedgerSecret& latest_ledger_secret) + { + auto ls_wrapping_key = LedgerSecretWrappingKey(); + auto wrapped_latest_ls = ls_wrapping_key.wrap(latest_ledger_secret); + auto recovery_shares = tx.rw(network.shares); + recovery_shares->put( + 0, {wrapped_latest_ls, compute_encrypted_shares(tx, ls_wrapping_key)}); + } + void set_recovery_shares_info( kv::Tx& tx, const LedgerSecret& latest_ledger_secret, - const std::optional& previous_ledger_secret = std::nullopt, - kv::Version latest_ls_version = kv::NoVersion) + const std::optional& previous_ledger_secret = + std::nullopt, + std::optional latest_ls_version = std::nullopt) { // First, generate a fresh ledger secrets wrapping key and wrap the // latest ledger secret with it. Then, encrypt the penultimate ledger // secret with the latest ledger secret and split the ledger secret - // wrapping key, allocating a new share for each active member. Finally, - // encrypt each share with the public key of each member and record it in - // the shares table. + // wrapping key, allocating a new share for each active recovery member. + // Finally, encrypt each share with the public key of each member and + // record it in the shares table. - auto ls_wrapping_key = LedgerSecretWrappingKey(); - auto wrapped_latest_ls = ls_wrapping_key.wrap(latest_ledger_secret); + shuffle_recovery_shares(tx, latest_ledger_secret); + + auto encrypted_ls = tx.rw(network.encrypted_ledger_secrets); std::vector encrypted_previous_secret = {}; + kv::Version version_previous_secret = kv::NoVersion; if (previous_ledger_secret.has_value()) { + version_previous_secret = previous_ledger_secret->first; + crypto::GcmCipher encrypted_previous_ls( - previous_ledger_secret->raw_key.size()); + previous_ledger_secret->second.raw_key.size()); auto iv = tls::create_entropy()->random(crypto::GCM_SIZE_IV); encrypted_previous_ls.hdr.set_iv(iv.data(), iv.size()); latest_ledger_secret.key->encrypt( encrypted_previous_ls.hdr.get_iv(), - previous_ledger_secret->raw_key, + previous_ledger_secret->second.raw_key, nullb, encrypted_previous_ls.cipher.data(), encrypted_previous_ls.hdr.tag); encrypted_previous_secret = encrypted_previous_ls.serialise(); + encrypted_ls->put( + 0, + {PreviousLedgerSecretInfo( + std::move(encrypted_previous_secret), + version_previous_secret, + std::nullopt), + latest_ls_version}); + } + else + { + encrypted_ls->put(0, {std::nullopt, latest_ls_version}); } - - GenesisGenerator g(network, tx); - g.add_key_share_info({{latest_ls_version, wrapped_latest_ls}, - encrypted_previous_secret, - compute_encrypted_shares(tx, ls_wrapping_key)}); } std::vector encrypt_submitted_share( @@ -218,7 +232,7 @@ namespace ccf LedgerSecret&& current_ledger_secret) { crypto::GcmCipher encrypted_share; - encrypted_share.apply(encrypted_submitted_share); + encrypted_share.deserialise(encrypted_submitted_share); std::vector decrypted_share(encrypted_share.cipher.size()); current_ledger_secret.key->decrypt( @@ -266,33 +280,53 @@ namespace ccf public: ShareManager(NetworkState& network_) : network(network_) {} - void issue_shares(kv::Tx& tx) - { - // Assumes that the ledger secrets have not been updated since the - // last time shares have been issued (i.e. genesis or re-sharing only) - set_recovery_shares_info( - tx, network.ledger_secrets->get_latest(tx).second); - } - - void issue_shares_on_recovery(kv::Tx& tx, kv::Version latest_ls_version) + /** Issue new recovery shares for the current ledger secret, recording the + * wrapped new ledger secret and encrypted previous ledger secret in the + * store. + * + * @param tx Store transaction object + */ + void issue_recovery_shares(kv::Tx& tx) { auto [latest, penultimate] = network.ledger_secrets->get_latest_and_penultimate(tx); - set_recovery_shares_info(tx, latest, penultimate, latest_ls_version); + set_recovery_shares_info(tx, latest.second, penultimate, latest.first); } - void issue_shares_on_rekey( + /** Issue new recovery shares of the new ledger secret, recording the + * wrapped new ledger secret and encrypted current (now previous) ledger + * secret in the store. + * + * @param tx Store transaction object + * @param new_ledger_secret New ledger secret + * + * Note: The version at which the new ledger secret is applicable from is + * derived from the hook at which the ledger secret is applied to the + * store. + */ + void issue_recovery_shares( kv::Tx& tx, const LedgerSecret& new_ledger_secret) { set_recovery_shares_info( - tx, new_ledger_secret, network.ledger_secrets->get_latest(tx).second); + tx, new_ledger_secret, network.ledger_secrets->get_latest(tx)); + } + + /** Issue new recovery shares of the same current ledger secret to all + * active recovery members. The encrypted ledger secrets recorded in the + * store are not updated. + * + * @param tx Store transaction object + */ + void shuffle_recovery_shares(kv::Tx& tx) + { + shuffle_recovery_shares( + tx, network.ledger_secrets->get_latest(tx).second); } std::optional get_encrypted_share( kv::Tx& tx, MemberId member_id) { - std::optional encrypted_share = std::nullopt; auto recovery_shares_info = tx.rw(network.shares)->get(0); if (!recovery_shares_info.has_value()) { @@ -300,27 +334,29 @@ namespace ccf "Failed to retrieve current recovery shares info"); } - for (auto const& s : recovery_shares_info->encrypted_shares) + auto search = recovery_shares_info->encrypted_shares.find(member_id); + if (search == recovery_shares_info->encrypted_shares.end()) { - if (s.first == member_id) - { - encrypted_share = s.second; - } + return std::nullopt; } - return encrypted_share; + + return search->second; } LedgerSecretsMap restore_recovery_shares_info( - kv::Tx& tx, - const std::list& encrypted_recovery_secrets) + kv::Tx& tx, RecoveredEncryptedLedgerSecrets&& recovery_ledger_secrets) { // First, re-assemble the ledger secret wrapping key from the submitted // encrypted shares. Then, unwrap the latest ledger secret and use it to - // decrypt the previous ledger secret and so on. + // decrypt the sequence of recovered ledger secrets, from the last one. - auto ls_wrapping_key = combine_from_submitted_shares(tx); + if (recovery_ledger_secrets.empty()) + { + throw std::logic_error( + "Could not find any ledger secrets in the ledger"); + } - auto recovery_shares_info = tx.rw(network.shares)->get(0); + auto recovery_shares_info = tx.ro(network.shares)->get(0); if (!recovery_shares_info.has_value()) { throw std::logic_error( @@ -329,25 +365,38 @@ namespace ccf LedgerSecretsMap restored_ledger_secrets; - auto restored_ls = ls_wrapping_key.unwrap( - recovery_shares_info->wrapped_latest_ledger_secret.encrypted_data); + auto restored_ls = combine_from_submitted_shares(tx).unwrap( + recovery_shares_info->wrapped_latest_ledger_secret); auto decryption_key = restored_ls.raw_key; - restored_ledger_secrets.emplace( - encrypted_recovery_secrets.back().next_version, std::move(restored_ls)); + LOG_DEBUG_FMT( + "Recovering {} encrypted ledger secrets", + recovery_ledger_secrets.size()); - for (auto i = encrypted_recovery_secrets.rbegin(); - i != encrypted_recovery_secrets.rend(); - i++) + auto& current_ledger_secret_version = + recovery_ledger_secrets.back().next_version; + if (!current_ledger_secret_version.has_value()) { - if (i->encrypted_ledger_secret.empty()) + // This should always be set by the recovery hook, which sets this to + // the version at which it is called if unset in the store + throw std::logic_error("Current ledger secret version should be set"); + } + + restored_ledger_secrets.emplace( + current_ledger_secret_version.value(), std::move(restored_ls)); + + for (auto it = recovery_ledger_secrets.rbegin(); + it != recovery_ledger_secrets.rend(); + it++) + { + if (!it->previous_ledger_secret.has_value()) { - // First entry does not encrypt any other ledger secret (i.e. genesis) + // Very first entry does not encrypt any other ledger secret break; } crypto::GcmCipher encrypted_ls; - encrypted_ls.apply(i->encrypted_ledger_secret); + encrypted_ls.deserialise(it->previous_ledger_secret->encrypted_data); std::vector decrypted_ls(encrypted_ls.cipher.size()); if (!crypto::KeyAesGcm(decryption_key) @@ -360,14 +409,16 @@ namespace ccf { throw std::logic_error(fmt::format( "Decryption of ledger secret at {} failed", - std::next(i)->next_version)); + it->previous_ledger_secret->version)); } decryption_key = decrypted_ls; restored_ledger_secrets.emplace( - std::next(i)->next_version, std::move(decrypted_ls)); + it->previous_ledger_secret->version, std::move(decrypted_ls)); } + recovery_ledger_secrets.clear(); + return restored_ledger_secrets; } diff --git a/src/node/shares.h b/src/node/shares.h index 39a8f1bd8d..f9cd7063ee 100644 --- a/src/node/shares.h +++ b/src/node/shares.h @@ -7,6 +7,7 @@ #include #include +#include #include namespace ccf @@ -14,56 +15,109 @@ namespace ccf using EncryptedShare = std::vector; using EncryptedSharesMap = std::map; - struct LatestLedgerSecret - { - // In most cases (e.g. re-key, member retirement), this is unset - // (kv::NoVersion), and the version at which the ledger secret is applicable - // from is derived from the local hook on recovery. - // In one case (i.e. after recovery of the public ledger), a new ledger - // secret is created to protect the integrity on the public-only - // transactions. However, the corresponding shares are only written at a - // later version, once the previous ledger secrets have been restored. - kv::Version version; - - std::vector encrypted_data; - - MSGPACK_DEFINE(version, encrypted_data) - }; - - DECLARE_JSON_TYPE(LatestLedgerSecret) - DECLARE_JSON_REQUIRED_FIELDS(LatestLedgerSecret, version, encrypted_data) - struct RecoverySharesInfo { - // Keeping track of the latest and penultimate ledger secret allows the - // value of this table to remain at a constant size through the lifetime of - // the service. On recovery, a local hook on this table allows the service - // to reconstruct the history of encrypted ledger secrets which are - // decrypted in sequence once the ledger secret wrapping key is - // re-assembled. - // Latest ledger secret wrapped with the ledger secret wrapping key - LatestLedgerSecret wrapped_latest_ledger_secret; - - // Previous ledger secret encrypted with the latest ledger secret - std::vector encrypted_previous_ledger_secret; + std::vector wrapped_latest_ledger_secret; + // Recovery shares encrypted with each active recovery member's public + // encryption key EncryptedSharesMap encrypted_shares; - MSGPACK_DEFINE( - wrapped_latest_ledger_secret, - encrypted_previous_ledger_secret, - encrypted_shares); + MSGPACK_DEFINE(wrapped_latest_ledger_secret, encrypted_shares); }; DECLARE_JSON_TYPE(RecoverySharesInfo) DECLARE_JSON_REQUIRED_FIELDS( - RecoverySharesInfo, - wrapped_latest_ledger_secret, - encrypted_previous_ledger_secret, - encrypted_shares) + RecoverySharesInfo, wrapped_latest_ledger_secret, encrypted_shares) - // The key for this table will always be 0 since a live service never needs to - // access historical recovery shares info. - using Shares = kv::Map; + struct PreviousLedgerSecretInfo + { + // Past ledger secret encrypted with the latest ledger secret + std::vector encrypted_data = {}; + + // Version at which the ledger secret is applicable from + kv::Version version = kv::NoVersion; + + // Version at which the ledger secret was written to the store (unused for + // now) + std::optional stored_version = std::nullopt; + + PreviousLedgerSecretInfo() = default; + + PreviousLedgerSecretInfo( + std::vector&& encrypted_data_, + kv::Version version_, + std::optional stored_version_) : + encrypted_data(std::move(encrypted_data_)), + version(version_), + stored_version(stored_version_) + {} + + bool operator==(const PreviousLedgerSecretInfo& other) const + { + return encrypted_data == other.encrypted_data && + version == other.version && stored_version == other.stored_version; + } + + bool operator!=(const PreviousLedgerSecretInfo& other) const + { + return !(*this == other); + } + + MSGPACK_DEFINE(encrypted_data, version, stored_version) + }; + + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(PreviousLedgerSecretInfo) + DECLARE_JSON_REQUIRED_FIELDS( + PreviousLedgerSecretInfo, encrypted_data, version) + DECLARE_JSON_OPTIONAL_FIELDS(PreviousLedgerSecretInfo, stored_version) + + struct EncryptedLedgerSecretInfo + { + // Previous ledger secret info, encrypted with the current ledger secret. + // Unset on service opening. + std::optional previous_ledger_secret = + std::nullopt; + + // Version at which the _next_ ledger secret is applicable from + // Note: In most cases (e.g. re-key, member retirement), this is unset and + // the version at which the next ledger secret is applicable from is + // derived from the local hook on recovery. In one case (i.e. after recovery + // of the public ledger), a new ledger secret is created to protect the + // integrity on the public-only transactions. However, the corresponding + // shares are only written at a later version, once the previous ledger + // secrets have been restored. + std::optional next_version = std::nullopt; + + MSGPACK_DEFINE(previous_ledger_secret, next_version) + }; + + // Note: Both fields are never empty at the same time + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EncryptedLedgerSecretInfo) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-parameter" +#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" + DECLARE_JSON_REQUIRED_FIELDS(EncryptedLedgerSecretInfo) +#pragma clang diagnostic pop + DECLARE_JSON_OPTIONAL_FIELDS( + EncryptedLedgerSecretInfo, previous_ledger_secret, next_version) + + // The following two tables are distinct because some operations trigger a + // re-share without requiring the ledger secrets to be updated (e.g. updating + // the recovery threshold), and vice versa (e.g. ledger rekey). For historical + // queries, when recovering ledger secrets from the ledger, the version at + // which the previous ledger secret was _written_ to the store must be known + // and can be deduced to the version at which the + // EncryptedPastLedgerSecret map was updated. + + // The key for this table is always 0. It is updated every time the member + // recovery shares are updated, e.g. when the recovery threshold is modified + // and when the ledger secret is updated + using RecoveryShares = kv::Map; + + // The key for this table is always 0. It is updated every time the ledger + // secret is updated, e.g. at startup or on ledger rekey. It is not updated on + // a pure re-share. + using EncryptedLedgerSecretsInfo = kv::Map; } \ No newline at end of file diff --git a/tests/e2e_suite.py b/tests/e2e_suite.py index 4de024d7aa..594a3a2621 100644 --- a/tests/e2e_suite.py +++ b/tests/e2e_suite.py @@ -117,8 +117,7 @@ def run(args): if new_network is None: raise ValueError(f"Network returned by {s.test_name(test)} is None") - # If the network was changed (e.g. recovery test), stop the previous network - # and use the new network from now on + # If the network was changed (e.g. recovery test), use the new network from now on if new_network != network: network = new_network diff --git a/tests/suite/test_suite.py b/tests/suite/test_suite.py index 87742e047b..2c73868fbb 100644 --- a/tests/suite/test_suite.py +++ b/tests/suite/test_suite.py @@ -15,6 +15,16 @@ from inspect import signature, Parameter suites = dict() +# This test suite currently fails and is not yet run by the CI +# https://github.com/microsoft/CCF/issues/1648 +historical_recovery_snapshot_failure = [ + e2e_logging.test_historical_query, + rekey.test, + rekey.test, + recovery.test, +] + + # This suite tests that rekeying, network configuration changes # and recoveries can be interleaved suite_rekey_recovery = [