Store new views into the ledger (#1167)

src/consensus/pbft/libbyz/ledger_writer.cpp

@@ -7,10 +7,12 @@
 LedgerWriter::LedgerWriter(
   pbft::PbftStore& store_,
   pbft::PrePreparesMap& pbft_pre_prepares_map_,
-  ccf::Signatures& signatures_) :
+  ccf::Signatures& signatures_,
+  pbft::NewViewsMap& pbft_new_views_map_) :
   store(store_),
   pbft_pre_prepares_map(pbft_pre_prepares_map_),
-  signatures(signatures_)
+  signatures(signatures_),
+  pbft_new_views_map(pbft_new_views_map_)
 {}
 
 kv::Version LedgerWriter::write_pre_prepare(ccf::Store::Tx& tx, Pre_prepare* pp)
@@ -51,21 +53,14 @@ kv::Version LedgerWriter::write_pre_prepare(Pre_prepare* pp)
     signatures);
 }
 
-void LedgerWriter::write_view_change(View_change* vc)
+void LedgerWriter::write_new_view(New_view* nv)
 {
-  View_change_ledger_header header(
-    vc->id(),
-    vc->max_seqno(),
-    vc->view()
-#ifdef SIGN_BATCH
-      ,
-    vc->digest(),
-    vc->signature()
-#endif
-  );
-
-  std::vector<uint8_t> entry(sizeof(View_change_ledger_header));
-  uint8_t* tdata = entry.data();
-  auto size = sizeof(View_change_ledger_header);
-  serialized::write(tdata, size, (uint8_t*)&header, size);
+  LOG_TRACE_FMT(
+    "Writing new view with view {} for node {}", nv->view(), nv->id());
+  store.commit_new_view(
+    {nv->view(),
+     nv->id(),
+     {(const uint8_t*)nv->contents(),
+      (const uint8_t*)nv->contents() + nv->size()}},
+    pbft_new_views_map);
 }

src/consensus/pbft/libbyz/ledger_writer.h

@@ -2,15 +2,16 @@
 // Licensed under the MIT license.
 #pragma once
 
+#include "consensus/pbft/pbft_new_views.h"
 #include "consensus/pbft/pbft_pre_prepares.h"
 #include "consensus/pbft/pbft_requests.h"
 #include "consensus/pbft/pbft_types.h"
 #include "kv/kv.h"
 #include "ledger.h"
+#include "new_view.h"
 #include "node/signatures.h"
 #include "prepared_cert.h"
 #include "types.h"
-#include "view_change.h"
 
 class LedgerWriter
 {
@@ -18,15 +19,17 @@ private:
   pbft::PbftStore& store;
   pbft::PrePreparesMap& pbft_pre_prepares_map;
   ccf::Signatures& signatures;
+  pbft::NewViewsMap& pbft_new_views_map;
 
 public:
   LedgerWriter(
     pbft::PbftStore& store_,
     pbft::PrePreparesMap& pbft_pre_prepares_map_,
-    ccf::Signatures& signatures_);
+    ccf::Signatures& signatures_,
+    pbft::NewViewsMap& pbft_new_views_map_);
   virtual ~LedgerWriter() = default;
   kv::Version write_pre_prepare(Pre_prepare* pp);
   kv::Version write_pre_prepare(Pre_prepare* pp, View view);
   kv::Version write_pre_prepare(ccf::Store::Tx& tx, Pre_prepare* pp);
-  void write_view_change(View_change* vc);
+  void write_new_view(New_view* nv);
 };

src/consensus/pbft/libbyz/libbyz.cpp

@@ -59,6 +59,7 @@ int Byz_init_replica(
   pbft::RequestsMap& pbft_requests_map,
   pbft::PrePreparesMap& pbft_pre_prepares_map,
   ccf::Signatures& signatures,
+  pbft::NewViewsMap& pbft_new_views_map,
   pbft::PbftStore& store,
   IMessageReceiveBase** message_receiver)
 {
@@ -71,6 +72,7 @@ int Byz_init_replica(
     pbft_requests_map,
     pbft_pre_prepares_map,
     signatures,
+    pbft_new_views_map,
     store));
 
   if (message_receiver != nullptr)

src/consensus/pbft/libbyz/libbyz.h

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include "consensus/pbft/pbft_new_views.h"
 #include "consensus/pbft/pbft_pre_prepares.h"
 #include "consensus/pbft/pbft_requests.h"
 #include "consensus/pbft/pbft_types.h"
@@ -54,6 +55,7 @@ int Byz_init_replica(
   pbft::RequestsMap& pbft_requests_map,
   pbft::PrePreparesMap& pbft_pre_prepares_map,
   ccf::Signatures& signatures,
+  pbft::NewViewsMap& pbft_new_views_map,
   pbft::PbftStore& store_,
   IMessageReceiveBase** message_receiver = nullptr);
 /* Requires: "mem" is vm page aligned and "size" is a multiple of the vm page

src/consensus/pbft/libbyz/replica.cpp

@@ -56,6 +56,7 @@ Replica::Replica(
   pbft::RequestsMap& pbft_requests_map_,
   pbft::PrePreparesMap& pbft_pre_prepares_map_,
   ccf::Signatures& signatures,
+  pbft::NewViewsMap& pbft_new_views_map_,
   pbft::PbftStore& store) :
   Node(node_info),
   rqueue(),
@@ -66,6 +67,7 @@ Replica::Replica(
   brt(node_info.general_info.num_replicas),
   pbft_requests_map(pbft_requests_map_),
   pbft_pre_prepares_map(pbft_pre_prepares_map_),
+  pbft_new_views_map(pbft_new_views_map_),
   replies(mem, nbytes),
   rep_cb(nullptr),
   global_commit_cb(nullptr),
@@ -163,8 +165,8 @@ Replica::Replica(
 
   exec_command = nullptr;
 
-  ledger_writer =
-    std::make_unique<LedgerWriter>(store, pbft_pre_prepares_map, signatures);
+  ledger_writer = std::make_unique<LedgerWriter>(
+    store, pbft_pre_prepares_map, signatures, pbft_new_views_map);
   encryptor = store.get_encryptor();
 }
 
@@ -1897,41 +1899,20 @@ void Replica::send_view_change()
 
   // Create and send view-change message.
   vi.view_change(v, last_executed, &state);
-
-  // Write the view change proof to the ledger
-#ifdef SIGN_BATCH
-  write_view_change_to_ledger();
-#endif
 }
 
-void Replica::write_view_change_to_ledger()
+void Replica::write_new_view_to_ledger()
 {
   if (!ledger_writer)
   {
     return;
   }
 
-  auto principals = get_principals();
-  for (const auto& it : *principals)
-  {
-    const std::shared_ptr<Principal>& p = it.second;
-    if (p == nullptr || !p->is_replica())
-    {
-      continue;
-    }
-    View_change* vc = vi.view_change(p->pid());
-    if (vc == nullptr)
-    {
-      continue;
-    }
-
-    LOG_TRACE_FMT(
-      "Writing view for {} with digest {} to ledger",
-      vc->view(),
-      vc->digest().hash());
-
-    ledger_writer->write_view_change(vc);
-  }
+  auto nv = vi.new_view();
+  PBFT_ASSERT(nv != nullptr, "Invalid state");
+  LOG_TRACE_FMT(
+    "Writing new view: {} from node: {} to ledger", nv->view(), nv->id());
+  ledger_writer->write_new_view(nv);
 }
 
 void Replica::handle(New_principal* m)
@@ -2094,6 +2075,8 @@ void Replica::process_new_view(Seqno min, Digest d, Seqno max, Seqno ms)
     ntimer->start();
   }
 
+  write_new_view_to_ledger();
+
   if (!has_nv_state)
   {
 #ifdef DEBUG_SLOW
@@ -2143,7 +2126,6 @@ void Replica::rollback_to_globally_comitted()
   {
     // Rollback to last checkpoint
     PBFT_ASSERT(!state.in_fetch_state(), "Invalid state");
-
     auto rv = last_gb_version + 1;
 
     if (rollback_cb != nullptr)
@@ -2491,6 +2473,7 @@ void Replica::execute_committed(bool was_f_0)
               pp->seqno());
             return;
           }
+
           last_te_version = ledger_writer->write_pre_prepare(pp);
           PBFT_ASSERT(
             last_executed + 1 == last_tentative_execute,

src/consensus/pbft/libbyz/replica.h

@@ -59,6 +59,7 @@ public:
     pbft::RequestsMap& pbft_requests_map_,
     pbft::PrePreparesMap& pbft_pre_prepares_map_,
     ccf::Signatures& signatures,
+    pbft::NewViewsMap& pbft_new_views_map_,
     pbft::PbftStore& store_);
   // Requires: "mem" is vm page aligned and nbytes is a multiple of the
   // vm page size.
@@ -93,7 +94,8 @@ public:
   // of a propagated request +1; and "ms" is the maximum sequence
   // number known to be stable.
 
-  void write_view_change_to_ledger();
+  void write_new_view_to_ledger();
+  // Effects: writes new view to ledger
 
   void send_view_change();
   // Effects: Send view-change message.
@@ -522,6 +524,7 @@ private:
 
   pbft::RequestsMap& pbft_requests_map;
   pbft::PrePreparesMap& pbft_pre_prepares_map;
+  pbft::NewViewsMap& pbft_new_views_map;
 
   // used to callback when we have committed a batch
   global_commit_handler_cb global_commit_cb;

src/consensus/pbft/libbyz/test/replica_unit_tests.cpp

@@ -117,6 +117,7 @@ NodeInfo create_replica(
   pbft::RequestsMap& pbft_requests_map,
   pbft::PrePreparesMap& pbft_pre_prepares_map,
   ccf::Signatures& signatures,
+  pbft::NewViewsMap& pbft_new_views_map,
   NodeId node_id = 0)
 {
   auto node_info = get_node_info(node_id);
@@ -129,6 +130,7 @@ NodeInfo create_replica(
     pbft_requests_map,
     pbft_pre_prepares_map,
     signatures,
+    pbft_new_views_map,
     store));
 
   pbft::GlobalState::get_replica().init_state();
@@ -203,6 +205,7 @@ struct PbftState
   pbft::RequestsMap& pbft_requests_map;
   ccf::Signatures& signatures;
   pbft::PrePreparesMap& pbft_pre_prepares_map;
+  pbft::NewViewsMap& pbft_new_views_map;
   std::vector<char> service_mem;
   ExecutionMock exec_mock;
 
@@ -215,6 +218,8 @@ struct PbftState
     signatures(store->create<ccf::Signatures>(ccf::Tables::SIGNATURES)),
     pbft_pre_prepares_map(store->create<pbft::PrePreparesMap>(
       pbft::Tables::PBFT_PRE_PREPARES, kv::SecurityDomain::PUBLIC)),
+    pbft_new_views_map(store->create<pbft::NewViewsMap>(
+      pbft::Tables::PBFT_NEW_VIEWS, kv::SecurityDomain::PUBLIC)),
     service_mem(mem_size, 0),
     exec_mock(0)
   {}
@@ -228,6 +233,7 @@ NodeInfo init_test_state(PbftState& pbft_state, NodeId node_id = 0)
     pbft_state.pbft_requests_map,
     pbft_state.pbft_pre_prepares_map,
     pbft_state.signatures,
+    pbft_state.pbft_new_views_map,
     node_id);
   pbft::GlobalState::get_replica().register_exec(
     pbft_state.exec_mock.exec_command);
@@ -293,7 +299,8 @@ TEST_CASE("Test Ledger Replay")
     LedgerWriter ledger_writer(
       *pbft_state.pbft_store,
       pbft_state.pbft_pre_prepares_map,
-      pbft_state.signatures);
+      pbft_state.signatures,
+      pbft_state.pbft_new_views_map);
 
     Req_queue rqueue;
     for (size_t i = 1; i < total_requests; i++)
@@ -562,7 +569,8 @@ TEST_CASE("Verify prepare proof")
     LedgerWriter ledger_writer(
       *pbft_state.pbft_store,
       pbft_state.pbft_pre_prepares_map,
-      pbft_state.signatures);
+      pbft_state.signatures,
+      pbft_state.pbft_new_views_map);
 
     // let this node know about the node that signed the prepare
     // otherwise we can't add its prepare to the prepared cert

src/consensus/pbft/libbyz/view_info.cpp

@@ -615,6 +615,11 @@ New_view* View_info::my_new_view()
   return last_nvs[id].new_view(t);
 }
 
+New_view* View_info::new_view()
+{
+  return last_nvs[pbft::GlobalState::get_node().primary(view())].new_view();
+}
+
 void View_info::mark_stable(Seqno ls)
 {
   last_stable = ls;

src/consensus/pbft/libbyz/view_info.h

@@ -130,6 +130,10 @@ public:
   // Effects: Returns any new-view message sent by the calling replica
   // in view "view()" or 0 if there is no such message. If t is supplied,
   // it is set to the tiem when the new view was first sent.
+  New_view* new_view();
+  // Effects: Returns the new-view message either sent by the calling replica
+  // if calling replica is primary, or the new-view message that caused
+  // a new view to be processed
 
   View_change_ack* my_vc_ack(int id);
   // Effects: Returns any view-change ack produced by the calling

src/consensus/pbft/pbft.h

@@ -364,6 +364,7 @@ namespace pbft
       pbft::RequestsMap& pbft_requests_map,
       pbft::PrePreparesMap& pbft_pre_prepares_map,
       ccf::Signatures& signatures,
+      pbft::NewViewsMap& pbft_new_views_map,
       const std::string& privk_pem,
       const std::vector<uint8_t>& cert,
       const consensus::Config& consensus_config) :
@@ -418,6 +419,7 @@ namespace pbft
         pbft_requests_map,
         pbft_pre_prepares_map,
         signatures,
+        pbft_new_views_map,
         *store,
         &message_receiver_base);
       LOG_INFO_FMT("PBFT setup for local_id: {}", local_id);

src/consensus/pbft/pbft_new_views.h

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "consensus/pbft/libbyz/parameters.h"
+#include "node/entities.h"
+
+#include <array>
+#include <msgpack/msgpack.hpp>
+#include <vector>
+
+namespace pbft
+{
+  struct NewView
+  {
+    int64_t view;
+    int node_id;
+    std::vector<uint8_t> contents;
+
+    MSGPACK_DEFINE(view, node_id, contents);
+  };
+
+  DECLARE_JSON_TYPE(NewView);
+  DECLARE_JSON_REQUIRED_FIELDS(NewView, view, node_id, contents);
+
+  // size_t is used as the key of the table. This key will always be 0 since we
+  // don't want to store the new views in the kv over time, we just want to
+  // get them into the ledger
+  using NewViewsMap = ccf::Store::Map<size_t, NewView>;
+}

src/consensus/pbft/pbft_tables.h

@@ -12,9 +12,10 @@ namespace pbft
   {
     static constexpr auto PBFT_REQUESTS = "ccf.pbft.requests";
     static constexpr auto PBFT_PRE_PREPARES = "ccf.pbft.preprepares";
+    static constexpr auto PBFT_NEW_VIEWS = "ccf.pbft.newviews";
   };
 
   static constexpr auto replicate_type_pbft = kv::ReplicateType::SOME;
   static const std::unordered_set<std::string> replicated_tables_pbft = {
-    Tables::PBFT_REQUESTS, Tables::PBFT_PRE_PREPARES};
+    Tables::PBFT_REQUESTS, Tables::PBFT_PRE_PREPARES, Tables::PBFT_NEW_VIEWS};
 }

src/consensus/pbft/pbft_types.h

@@ -2,6 +2,7 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
+#include "consensus/pbft/pbft_new_views.h"
 #include "consensus/pbft/pbft_pre_prepares.h"
 #include "ds/ring_buffer_types.h"
 #include "kv/kv_types.h"
@@ -55,6 +56,8 @@ namespace pbft
       ccf::Signatures& signatures) = 0;
     virtual kv::Version commit_tx(
       ccf::Store::Tx& tx, CBuffer root, ccf::Signatures& signatures) = 0;
+    virtual void commit_new_view(
+      const pbft::NewView& new_view, pbft::NewViewsMap& pbft_new_views_map) = 0;
     virtual std::shared_ptr<kv::AbstractTxEncryptor> get_encryptor() = 0;
   };
 
@@ -133,6 +136,36 @@ namespace pbft
       }
     }
 
+    void commit_new_view(
+      const pbft::NewView& new_view, pbft::NewViewsMap& pbft_new_views_map)
+    {
+      while (true)
+      {
+        auto p = x.lock();
+        if (p)
+        {
+          auto version = p->next_version();
+          LOG_TRACE_FMT(
+            "Storing new view message at view {} for node {}",
+            new_view.view,
+            new_view.node_id);
+          auto success = p->commit(
+            version,
+            [&]() {
+              ccf::Store::Tx tx(version);
+              auto vc_view = tx.get_view(pbft_new_views_map);
+              vc_view->put(0, new_view);
+              return tx.commit_reserved();
+            },
+            false);
+          if (success == kv::CommitSuccess::OK)
+          {
+            return;
+          }
+        }
+      }
+    }
+
     void compact(Index v)
     {
       auto p = x.lock();

src/kv/kv.h

@@ -1713,20 +1713,19 @@ namespace kv
           return DeserialiseSuccess::FAILED;
         }
 
-        auto search = views.find("ccf.pbft.preprepares");
-        if (search != views.end())
+        if (views.find("ccf.pbft.preprepares") != views.end())
         {
           success = DeserialiseSuccess::PASS_PRE_PREPARE;
         }
-        else
+        else if (views.find("ccf.pbft.newviews") != views.end())
         {
-          auto search = views.find("ccf.pbft.requests");
-          if (search == views.end())
-          {
-            // we have deserialised an entry that didn't belong to the pbft
-            // requests nor the pbft pre prepares table
-            return DeserialiseSuccess::FAILED;
-          }
+          success = DeserialiseSuccess::PASS_NEW_VIEW;
+        }
+        else if (views.find("ccf.pbft.requests") == views.end())
+        {
+          // we have deserialised an entry that didn't belong to the pbft
+          // requests, nor the pbft new views, nor the pbft pre prepares table
+          return DeserialiseSuccess::FAILED;
         }
       }
 

src/kv/kv_types.h

@@ -53,7 +53,8 @@ namespace kv
     FAILED = 0,
     PASS = 1,
     PASS_SIGNATURE = 2,
-    PASS_PRE_PREPARE = 3
+    PASS_PRE_PREPARE = 3,
+    PASS_NEW_VIEW = 4
   };
 
   enum ReplicateType

src/node/network_tables.h

@@ -7,6 +7,7 @@
 #include "code_id.h"
 #include "config.h"
 #include "consensus.h"
+#include "consensus/pbft/pbft_new_views.h"
 #include "consensus/pbft/pbft_pre_prepares.h"
 #include "consensus/pbft/pbft_requests.h"
 #include "consensus/pbft/pbft_tables.h"
@@ -85,6 +86,7 @@ namespace ccf
     //
     pbft::RequestsMap& pbft_requests_map;
     pbft::PrePreparesMap& pbft_pre_prepares_map;
+    pbft::NewViewsMap& pbft_new_views_map;
 
     NetworkTables(const ConsensusType& consensus_type = ConsensusType::RAFT) :
       tables(
@@ -137,7 +139,9 @@ namespace ccf
       pbft_requests_map(
         tables->create<pbft::RequestsMap>(pbft::Tables::PBFT_REQUESTS)),
       pbft_pre_prepares_map(
-        tables->create<pbft::PrePreparesMap>(pbft::Tables::PBFT_PRE_PREPARES))
+        tables->create<pbft::PrePreparesMap>(pbft::Tables::PBFT_PRE_PREPARES)),
+      pbft_new_views_map(
+        tables->create<pbft::NewViewsMap>(pbft::Tables::PBFT_NEW_VIEWS))
     {}
 
     /** Returns a tuple of all tables that are possibly accessible from scripts

src/node/node_state.h

@@ -1660,6 +1660,7 @@ namespace ccf
         network.pbft_requests_map,
         network.pbft_pre_prepares_map,
         network.signatures,
+        network.pbft_new_views_map,
         node_sign_kp->private_key_pem().str(),
         node_cert,
         consensus_config);