From b6e42b2db61f932d31ce2b131632ce9ed1a8a7e8 Mon Sep 17 00:00:00 2001 From: olgavrou Date: Tue, 12 May 2020 08:48:36 +0100 Subject: [PATCH] Store new views into the ledger (#1167) --- src/consensus/pbft/libbyz/ledger_writer.cpp | 31 ++++++------- src/consensus/pbft/libbyz/ledger_writer.h | 9 ++-- src/consensus/pbft/libbyz/libbyz.cpp | 2 + src/consensus/pbft/libbyz/libbyz.h | 2 + src/consensus/pbft/libbyz/replica.cpp | 43 ++++++------------- src/consensus/pbft/libbyz/replica.h | 5 ++- .../pbft/libbyz/test/replica_unit_tests.cpp | 12 +++++- src/consensus/pbft/libbyz/view_info.cpp | 5 +++ src/consensus/pbft/libbyz/view_info.h | 4 ++ src/consensus/pbft/pbft.h | 2 + src/consensus/pbft/pbft_new_views.h | 30 +++++++++++++ src/consensus/pbft/pbft_tables.h | 3 +- src/consensus/pbft/pbft_types.h | 33 ++++++++++++++ src/kv/kv.h | 19 ++++---- src/kv/kv_types.h | 3 +- src/node/network_tables.h | 6 ++- src/node/node_state.h | 1 + 17 files changed, 143 insertions(+), 67 deletions(-) create mode 100644 src/consensus/pbft/pbft_new_views.h diff --git a/src/consensus/pbft/libbyz/ledger_writer.cpp b/src/consensus/pbft/libbyz/ledger_writer.cpp index dc45a2a2c..042f7a7f5 100644 --- a/src/consensus/pbft/libbyz/ledger_writer.cpp +++ b/src/consensus/pbft/libbyz/ledger_writer.cpp @@ -7,10 +7,12 @@ LedgerWriter::LedgerWriter( pbft::PbftStore& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_, - ccf::Signatures& signatures_) : + ccf::Signatures& signatures_, + pbft::NewViewsMap& pbft_new_views_map_) : store(store_), pbft_pre_prepares_map(pbft_pre_prepares_map_), - signatures(signatures_) + signatures(signatures_), + pbft_new_views_map(pbft_new_views_map_) {} kv::Version LedgerWriter::write_pre_prepare(ccf::Store::Tx& tx, Pre_prepare* pp) @@ -51,21 +53,14 @@ kv::Version LedgerWriter::write_pre_prepare(Pre_prepare* pp) signatures); } -void LedgerWriter::write_view_change(View_change* vc) +void LedgerWriter::write_new_view(New_view* nv) { - View_change_ledger_header header( - vc->id(), - vc->max_seqno(), - vc->view() -#ifdef SIGN_BATCH - , - vc->digest(), - vc->signature() -#endif - ); - - std::vector entry(sizeof(View_change_ledger_header)); - uint8_t* tdata = entry.data(); - auto size = sizeof(View_change_ledger_header); - serialized::write(tdata, size, (uint8_t*)&header, size); + LOG_TRACE_FMT( + "Writing new view with view {} for node {}", nv->view(), nv->id()); + store.commit_new_view( + {nv->view(), + nv->id(), + {(const uint8_t*)nv->contents(), + (const uint8_t*)nv->contents() + nv->size()}}, + pbft_new_views_map); } \ No newline at end of file diff --git a/src/consensus/pbft/libbyz/ledger_writer.h b/src/consensus/pbft/libbyz/ledger_writer.h index fceec92bf..57f134732 100644 --- a/src/consensus/pbft/libbyz/ledger_writer.h +++ b/src/consensus/pbft/libbyz/ledger_writer.h @@ -2,15 +2,16 @@ // Licensed under the MIT license. #pragma once +#include "consensus/pbft/pbft_new_views.h" #include "consensus/pbft/pbft_pre_prepares.h" #include "consensus/pbft/pbft_requests.h" #include "consensus/pbft/pbft_types.h" #include "kv/kv.h" #include "ledger.h" +#include "new_view.h" #include "node/signatures.h" #include "prepared_cert.h" #include "types.h" -#include "view_change.h" class LedgerWriter { @@ -18,15 +19,17 @@ private: pbft::PbftStore& store; pbft::PrePreparesMap& pbft_pre_prepares_map; ccf::Signatures& signatures; + pbft::NewViewsMap& pbft_new_views_map; public: LedgerWriter( pbft::PbftStore& store_, pbft::PrePreparesMap& pbft_pre_prepares_map_, - ccf::Signatures& signatures_); + ccf::Signatures& signatures_, + pbft::NewViewsMap& pbft_new_views_map_); virtual ~LedgerWriter() = default; kv::Version write_pre_prepare(Pre_prepare* pp); kv::Version write_pre_prepare(Pre_prepare* pp, View view); kv::Version write_pre_prepare(ccf::Store::Tx& tx, Pre_prepare* pp); - void write_view_change(View_change* vc); + void write_new_view(New_view* nv); }; diff --git a/src/consensus/pbft/libbyz/libbyz.cpp b/src/consensus/pbft/libbyz/libbyz.cpp index acfdfe113..73748d9a4 100644 --- a/src/consensus/pbft/libbyz/libbyz.cpp +++ b/src/consensus/pbft/libbyz/libbyz.cpp @@ -59,6 +59,7 @@ int Byz_init_replica( pbft::RequestsMap& pbft_requests_map, pbft::PrePreparesMap& pbft_pre_prepares_map, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map, pbft::PbftStore& store, IMessageReceiveBase** message_receiver) { @@ -71,6 +72,7 @@ int Byz_init_replica( pbft_requests_map, pbft_pre_prepares_map, signatures, + pbft_new_views_map, store)); if (message_receiver != nullptr) diff --git a/src/consensus/pbft/libbyz/libbyz.h b/src/consensus/pbft/libbyz/libbyz.h index ff427fe76..fcc4eccc8 100644 --- a/src/consensus/pbft/libbyz/libbyz.h +++ b/src/consensus/pbft/libbyz/libbyz.h @@ -5,6 +5,7 @@ #pragma once +#include "consensus/pbft/pbft_new_views.h" #include "consensus/pbft/pbft_pre_prepares.h" #include "consensus/pbft/pbft_requests.h" #include "consensus/pbft/pbft_types.h" @@ -54,6 +55,7 @@ int Byz_init_replica( pbft::RequestsMap& pbft_requests_map, pbft::PrePreparesMap& pbft_pre_prepares_map, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map, pbft::PbftStore& store_, IMessageReceiveBase** message_receiver = nullptr); /* Requires: "mem" is vm page aligned and "size" is a multiple of the vm page diff --git a/src/consensus/pbft/libbyz/replica.cpp b/src/consensus/pbft/libbyz/replica.cpp index b2a8fc21c..8d0d6523a 100644 --- a/src/consensus/pbft/libbyz/replica.cpp +++ b/src/consensus/pbft/libbyz/replica.cpp @@ -56,6 +56,7 @@ Replica::Replica( pbft::RequestsMap& pbft_requests_map_, pbft::PrePreparesMap& pbft_pre_prepares_map_, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map_, pbft::PbftStore& store) : Node(node_info), rqueue(), @@ -66,6 +67,7 @@ Replica::Replica( brt(node_info.general_info.num_replicas), pbft_requests_map(pbft_requests_map_), pbft_pre_prepares_map(pbft_pre_prepares_map_), + pbft_new_views_map(pbft_new_views_map_), replies(mem, nbytes), rep_cb(nullptr), global_commit_cb(nullptr), @@ -163,8 +165,8 @@ Replica::Replica( exec_command = nullptr; - ledger_writer = - std::make_unique(store, pbft_pre_prepares_map, signatures); + ledger_writer = std::make_unique( + store, pbft_pre_prepares_map, signatures, pbft_new_views_map); encryptor = store.get_encryptor(); } @@ -1897,41 +1899,20 @@ void Replica::send_view_change() // Create and send view-change message. vi.view_change(v, last_executed, &state); - - // Write the view change proof to the ledger -#ifdef SIGN_BATCH - write_view_change_to_ledger(); -#endif } -void Replica::write_view_change_to_ledger() +void Replica::write_new_view_to_ledger() { if (!ledger_writer) { return; } - auto principals = get_principals(); - for (const auto& it : *principals) - { - const std::shared_ptr& p = it.second; - if (p == nullptr || !p->is_replica()) - { - continue; - } - View_change* vc = vi.view_change(p->pid()); - if (vc == nullptr) - { - continue; - } - - LOG_TRACE_FMT( - "Writing view for {} with digest {} to ledger", - vc->view(), - vc->digest().hash()); - - ledger_writer->write_view_change(vc); - } + auto nv = vi.new_view(); + PBFT_ASSERT(nv != nullptr, "Invalid state"); + LOG_TRACE_FMT( + "Writing new view: {} from node: {} to ledger", nv->view(), nv->id()); + ledger_writer->write_new_view(nv); } void Replica::handle(New_principal* m) @@ -2094,6 +2075,8 @@ void Replica::process_new_view(Seqno min, Digest d, Seqno max, Seqno ms) ntimer->start(); } + write_new_view_to_ledger(); + if (!has_nv_state) { #ifdef DEBUG_SLOW @@ -2143,7 +2126,6 @@ void Replica::rollback_to_globally_comitted() { // Rollback to last checkpoint PBFT_ASSERT(!state.in_fetch_state(), "Invalid state"); - auto rv = last_gb_version + 1; if (rollback_cb != nullptr) @@ -2491,6 +2473,7 @@ void Replica::execute_committed(bool was_f_0) pp->seqno()); return; } + last_te_version = ledger_writer->write_pre_prepare(pp); PBFT_ASSERT( last_executed + 1 == last_tentative_execute, diff --git a/src/consensus/pbft/libbyz/replica.h b/src/consensus/pbft/libbyz/replica.h index 01b8878ea..57aa2f99d 100644 --- a/src/consensus/pbft/libbyz/replica.h +++ b/src/consensus/pbft/libbyz/replica.h @@ -59,6 +59,7 @@ public: pbft::RequestsMap& pbft_requests_map_, pbft::PrePreparesMap& pbft_pre_prepares_map_, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map_, pbft::PbftStore& store_); // Requires: "mem" is vm page aligned and nbytes is a multiple of the // vm page size. @@ -93,7 +94,8 @@ public: // of a propagated request +1; and "ms" is the maximum sequence // number known to be stable. - void write_view_change_to_ledger(); + void write_new_view_to_ledger(); + // Effects: writes new view to ledger void send_view_change(); // Effects: Send view-change message. @@ -522,6 +524,7 @@ private: pbft::RequestsMap& pbft_requests_map; pbft::PrePreparesMap& pbft_pre_prepares_map; + pbft::NewViewsMap& pbft_new_views_map; // used to callback when we have committed a batch global_commit_handler_cb global_commit_cb; diff --git a/src/consensus/pbft/libbyz/test/replica_unit_tests.cpp b/src/consensus/pbft/libbyz/test/replica_unit_tests.cpp index 07ed61b29..ef21e1e19 100644 --- a/src/consensus/pbft/libbyz/test/replica_unit_tests.cpp +++ b/src/consensus/pbft/libbyz/test/replica_unit_tests.cpp @@ -117,6 +117,7 @@ NodeInfo create_replica( pbft::RequestsMap& pbft_requests_map, pbft::PrePreparesMap& pbft_pre_prepares_map, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map, NodeId node_id = 0) { auto node_info = get_node_info(node_id); @@ -129,6 +130,7 @@ NodeInfo create_replica( pbft_requests_map, pbft_pre_prepares_map, signatures, + pbft_new_views_map, store)); pbft::GlobalState::get_replica().init_state(); @@ -203,6 +205,7 @@ struct PbftState pbft::RequestsMap& pbft_requests_map; ccf::Signatures& signatures; pbft::PrePreparesMap& pbft_pre_prepares_map; + pbft::NewViewsMap& pbft_new_views_map; std::vector service_mem; ExecutionMock exec_mock; @@ -215,6 +218,8 @@ struct PbftState signatures(store->create(ccf::Tables::SIGNATURES)), pbft_pre_prepares_map(store->create( pbft::Tables::PBFT_PRE_PREPARES, kv::SecurityDomain::PUBLIC)), + pbft_new_views_map(store->create( + pbft::Tables::PBFT_NEW_VIEWS, kv::SecurityDomain::PUBLIC)), service_mem(mem_size, 0), exec_mock(0) {} @@ -228,6 +233,7 @@ NodeInfo init_test_state(PbftState& pbft_state, NodeId node_id = 0) pbft_state.pbft_requests_map, pbft_state.pbft_pre_prepares_map, pbft_state.signatures, + pbft_state.pbft_new_views_map, node_id); pbft::GlobalState::get_replica().register_exec( pbft_state.exec_mock.exec_command); @@ -293,7 +299,8 @@ TEST_CASE("Test Ledger Replay") LedgerWriter ledger_writer( *pbft_state.pbft_store, pbft_state.pbft_pre_prepares_map, - pbft_state.signatures); + pbft_state.signatures, + pbft_state.pbft_new_views_map); Req_queue rqueue; for (size_t i = 1; i < total_requests; i++) @@ -562,7 +569,8 @@ TEST_CASE("Verify prepare proof") LedgerWriter ledger_writer( *pbft_state.pbft_store, pbft_state.pbft_pre_prepares_map, - pbft_state.signatures); + pbft_state.signatures, + pbft_state.pbft_new_views_map); // let this node know about the node that signed the prepare // otherwise we can't add its prepare to the prepared cert diff --git a/src/consensus/pbft/libbyz/view_info.cpp b/src/consensus/pbft/libbyz/view_info.cpp index 35b77dc69..365598157 100644 --- a/src/consensus/pbft/libbyz/view_info.cpp +++ b/src/consensus/pbft/libbyz/view_info.cpp @@ -615,6 +615,11 @@ New_view* View_info::my_new_view() return last_nvs[id].new_view(t); } +New_view* View_info::new_view() +{ + return last_nvs[pbft::GlobalState::get_node().primary(view())].new_view(); +} + void View_info::mark_stable(Seqno ls) { last_stable = ls; diff --git a/src/consensus/pbft/libbyz/view_info.h b/src/consensus/pbft/libbyz/view_info.h index 816ba9763..02b375e61 100644 --- a/src/consensus/pbft/libbyz/view_info.h +++ b/src/consensus/pbft/libbyz/view_info.h @@ -130,6 +130,10 @@ public: // Effects: Returns any new-view message sent by the calling replica // in view "view()" or 0 if there is no such message. If t is supplied, // it is set to the tiem when the new view was first sent. + New_view* new_view(); + // Effects: Returns the new-view message either sent by the calling replica + // if calling replica is primary, or the new-view message that caused + // a new view to be processed View_change_ack* my_vc_ack(int id); // Effects: Returns any view-change ack produced by the calling diff --git a/src/consensus/pbft/pbft.h b/src/consensus/pbft/pbft.h index 20927ae9b..5d46072a7 100644 --- a/src/consensus/pbft/pbft.h +++ b/src/consensus/pbft/pbft.h @@ -364,6 +364,7 @@ namespace pbft pbft::RequestsMap& pbft_requests_map, pbft::PrePreparesMap& pbft_pre_prepares_map, ccf::Signatures& signatures, + pbft::NewViewsMap& pbft_new_views_map, const std::string& privk_pem, const std::vector& cert, const consensus::Config& consensus_config) : @@ -418,6 +419,7 @@ namespace pbft pbft_requests_map, pbft_pre_prepares_map, signatures, + pbft_new_views_map, *store, &message_receiver_base); LOG_INFO_FMT("PBFT setup for local_id: {}", local_id); diff --git a/src/consensus/pbft/pbft_new_views.h b/src/consensus/pbft/pbft_new_views.h new file mode 100644 index 000000000..928196134 --- /dev/null +++ b/src/consensus/pbft/pbft_new_views.h @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "consensus/pbft/libbyz/parameters.h" +#include "node/entities.h" + +#include +#include +#include + +namespace pbft +{ + struct NewView + { + int64_t view; + int node_id; + std::vector contents; + + MSGPACK_DEFINE(view, node_id, contents); + }; + + DECLARE_JSON_TYPE(NewView); + DECLARE_JSON_REQUIRED_FIELDS(NewView, view, node_id, contents); + + // size_t is used as the key of the table. This key will always be 0 since we + // don't want to store the new views in the kv over time, we just want to + // get them into the ledger + using NewViewsMap = ccf::Store::Map; +} \ No newline at end of file diff --git a/src/consensus/pbft/pbft_tables.h b/src/consensus/pbft/pbft_tables.h index 0591ebada..40434ae71 100644 --- a/src/consensus/pbft/pbft_tables.h +++ b/src/consensus/pbft/pbft_tables.h @@ -12,9 +12,10 @@ namespace pbft { static constexpr auto PBFT_REQUESTS = "ccf.pbft.requests"; static constexpr auto PBFT_PRE_PREPARES = "ccf.pbft.preprepares"; + static constexpr auto PBFT_NEW_VIEWS = "ccf.pbft.newviews"; }; static constexpr auto replicate_type_pbft = kv::ReplicateType::SOME; static const std::unordered_set replicated_tables_pbft = { - Tables::PBFT_REQUESTS, Tables::PBFT_PRE_PREPARES}; + Tables::PBFT_REQUESTS, Tables::PBFT_PRE_PREPARES, Tables::PBFT_NEW_VIEWS}; } diff --git a/src/consensus/pbft/pbft_types.h b/src/consensus/pbft/pbft_types.h index 44c794077..ca6a5e7a1 100644 --- a/src/consensus/pbft/pbft_types.h +++ b/src/consensus/pbft/pbft_types.h @@ -2,6 +2,7 @@ // Licensed under the Apache 2.0 License. #pragma once +#include "consensus/pbft/pbft_new_views.h" #include "consensus/pbft/pbft_pre_prepares.h" #include "ds/ring_buffer_types.h" #include "kv/kv_types.h" @@ -55,6 +56,8 @@ namespace pbft ccf::Signatures& signatures) = 0; virtual kv::Version commit_tx( ccf::Store::Tx& tx, CBuffer root, ccf::Signatures& signatures) = 0; + virtual void commit_new_view( + const pbft::NewView& new_view, pbft::NewViewsMap& pbft_new_views_map) = 0; virtual std::shared_ptr get_encryptor() = 0; }; @@ -133,6 +136,36 @@ namespace pbft } } + void commit_new_view( + const pbft::NewView& new_view, pbft::NewViewsMap& pbft_new_views_map) + { + while (true) + { + auto p = x.lock(); + if (p) + { + auto version = p->next_version(); + LOG_TRACE_FMT( + "Storing new view message at view {} for node {}", + new_view.view, + new_view.node_id); + auto success = p->commit( + version, + [&]() { + ccf::Store::Tx tx(version); + auto vc_view = tx.get_view(pbft_new_views_map); + vc_view->put(0, new_view); + return tx.commit_reserved(); + }, + false); + if (success == kv::CommitSuccess::OK) + { + return; + } + } + } + } + void compact(Index v) { auto p = x.lock(); diff --git a/src/kv/kv.h b/src/kv/kv.h index b24816b40..94a41cbd0 100644 --- a/src/kv/kv.h +++ b/src/kv/kv.h @@ -1713,20 +1713,19 @@ namespace kv return DeserialiseSuccess::FAILED; } - auto search = views.find("ccf.pbft.preprepares"); - if (search != views.end()) + if (views.find("ccf.pbft.preprepares") != views.end()) { success = DeserialiseSuccess::PASS_PRE_PREPARE; } - else + else if (views.find("ccf.pbft.newviews") != views.end()) { - auto search = views.find("ccf.pbft.requests"); - if (search == views.end()) - { - // we have deserialised an entry that didn't belong to the pbft - // requests nor the pbft pre prepares table - return DeserialiseSuccess::FAILED; - } + success = DeserialiseSuccess::PASS_NEW_VIEW; + } + else if (views.find("ccf.pbft.requests") == views.end()) + { + // we have deserialised an entry that didn't belong to the pbft + // requests, nor the pbft new views, nor the pbft pre prepares table + return DeserialiseSuccess::FAILED; } } diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index 7def62259..5ed8e6053 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -53,7 +53,8 @@ namespace kv FAILED = 0, PASS = 1, PASS_SIGNATURE = 2, - PASS_PRE_PREPARE = 3 + PASS_PRE_PREPARE = 3, + PASS_NEW_VIEW = 4 }; enum ReplicateType diff --git a/src/node/network_tables.h b/src/node/network_tables.h index e591f7f60..b1f15dbdf 100644 --- a/src/node/network_tables.h +++ b/src/node/network_tables.h @@ -7,6 +7,7 @@ #include "code_id.h" #include "config.h" #include "consensus.h" +#include "consensus/pbft/pbft_new_views.h" #include "consensus/pbft/pbft_pre_prepares.h" #include "consensus/pbft/pbft_requests.h" #include "consensus/pbft/pbft_tables.h" @@ -85,6 +86,7 @@ namespace ccf // pbft::RequestsMap& pbft_requests_map; pbft::PrePreparesMap& pbft_pre_prepares_map; + pbft::NewViewsMap& pbft_new_views_map; NetworkTables(const ConsensusType& consensus_type = ConsensusType::RAFT) : tables( @@ -137,7 +139,9 @@ namespace ccf pbft_requests_map( tables->create(pbft::Tables::PBFT_REQUESTS)), pbft_pre_prepares_map( - tables->create(pbft::Tables::PBFT_PRE_PREPARES)) + tables->create(pbft::Tables::PBFT_PRE_PREPARES)), + pbft_new_views_map( + tables->create(pbft::Tables::PBFT_NEW_VIEWS)) {} /** Returns a tuple of all tables that are possibly accessible from scripts diff --git a/src/node/node_state.h b/src/node/node_state.h index 28f790db9..9e8f8a282 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -1660,6 +1660,7 @@ namespace ccf network.pbft_requests_map, network.pbft_pre_prepares_map, network.signatures, + network.pbft_new_views_map, node_sign_kp->private_key_pem().str(), node_cert, consensus_config);