PBFT - late joining node 1/???? (#394)

src/consensus/pbft/pbft.h

@@ -209,7 +209,8 @@ namespace pbft
 
     bool on_request(const kv::TxHistory::RequestCallbackArgs& args) override
     {
-      auto total_req_size = pbft_config->message_size() + args.request.size();
+      auto total_req_size = pbft_config->message_size() + args.request.size() +
+        args.caller_cert.rawSize();
 
       uint8_t request_buffer[total_req_size];
       pbft_config->fill_request(
@@ -217,7 +218,8 @@ namespace pbft
         total_req_size,
         args.request,
         args.actor,
-        args.caller_id);
+        args.caller_id,
+        args.caller_cert);
 
       auto rep_cb = [&](
                       void* owner,
@@ -280,7 +282,6 @@ namespace pbft
       {
         return;
       }
-
       PrincipalInfo info;
       info.id = node_conf.node_id;
       info.port = short(atoi(node_conf.port.c_str()));

src/consensus/pbft/pbftconfig.h

@@ -21,7 +21,8 @@ namespace pbft
       size_t total_req_size,
       const std::vector<uint8_t>& data,
       uint64_t actor,
-      uint64_t caller_id) = 0;
+      uint64_t caller_id,
+      const CBuffer& caller_cert) = 0;
   };
 
   class PbftConfigCcf : public AbstractPbftConfig
@@ -52,35 +53,51 @@ namespace pbft
       size_t total_req_size,
       const std::vector<uint8_t>& data,
       uint64_t actor,
-      uint64_t caller_id) override
+      uint64_t caller_id,
+      const CBuffer& caller_cert) override
     {
       serialized::write(buffer, total_req_size, actor);
       serialized::write(buffer, total_req_size, caller_id);
+      uint32_t cert_size = caller_cert.rawSize();
+      serialized::write(buffer, total_req_size, cert_size);
+      if (caller_cert.p != nullptr)
+      {
+        serialized::write(
+          buffer, total_req_size, caller_cert.p, caller_cert.rawSize());
+      }
       serialized::write(buffer, total_req_size, data.data(), data.size());
     }
 
   private:
     std::shared_ptr<enclave::RpcMap> rpc_map;
 
+#pragma pack(push, 1)
     struct ccf_req
     {
       ccf::ActorsType actor;
       uint64_t caller_id;
+      uint32_t cert_size;
 
-      uint8_t* get_data()
+      uint8_t* cert()
       {
         return (uint8_t*)((uintptr_t)this + sizeof(ccf_req));
       }
 
+      uint8_t* get_data()
+      {
+        return (uint8_t*)((uintptr_t)this + sizeof(ccf_req) + cert_size);
+      }
+
       size_t get_size(size_t total_size)
       {
-        if (total_size < sizeof(ccf_req))
+        if (total_size < (sizeof(ccf_req) + cert_size))
         {
           return 0;
         }
-        return total_size - sizeof(ccf_req);
+        return total_size - (sizeof(ccf_req) + cert_size);
       }
     };
+#pragma pack(pop)
 
     ExecCommand exec_command = [this](
                                  Byz_req* inb,
@@ -90,7 +107,7 @@ namespace pbft
                                  bool ro,
                                  Seqno total_requests_executed,
                                  ByzInfo& info) {
-      auto request = new (inb->contents) ccf_req;
+      auto request = reinterpret_cast<ccf_req*>(inb->contents);
 
       LOG_DEBUG_FMT("PBFT exec_command() for frontend {}", request->actor);
 
@@ -104,7 +121,12 @@ namespace pbft
 
       // TODO: For now, re-use the RPCContext for forwarded commands.
       // Eventually, the two process_() commands will be refactored accordingly.
-      enclave::RPCContext ctx(0, 0, request->caller_id);
+      enclave::RPCContext ctx(
+        0,
+        0,
+        request->caller_id,
+        CBuffer(
+          request->cert(), request->cert_size)); // add the pointer and size
 
       auto rep = frontend->process_pbft(
         ctx,

src/enclave/enclavetypes.h

@@ -71,6 +71,19 @@ namespace enclave
       actor(actor_)
     {}
 
+    // Constructor used for forwarded RPC and the the caller_cert
+    RPCContext(
+      size_t fwd_session_id_,
+      ccf::NodeId from_,
+      ccf::CallerId caller_id_,
+      CBuffer caller_cert_,
+      ccf::ActorsType actor_ = ccf::ActorsType::unknown) :
+      fwd(std::make_optional<struct forwarded>(
+        fwd_session_id_, from_, caller_id_)),
+      caller_cert(caller_cert_),
+      actor(actor_)
+    {}
+
     // Constructor used for forwarded RPC
     RPCContext(
       size_t fwd_session_id_,

src/kv/kvtypes.h

@@ -64,6 +64,7 @@ namespace kv
       std::vector<uint8_t> request;
       uint64_t actor;
       uint64_t caller_id;
+      CBuffer caller_cert;
     };
 
     struct ResultCallbackArgs
@@ -92,6 +93,7 @@ namespace kv
       kv::TxHistory::RequestID id,
       uint64_t actor,
       uint64_t caller_id,
+      CBuffer& caller_cert,
       const std::vector<uint8_t>& request) = 0;
     virtual void add_result(
       RequestID id, kv::Version version, const std::vector<uint8_t>& data) = 0;

src/node/history.h

@@ -146,6 +146,7 @@ namespace ccf
       kv::TxHistory::RequestID id,
       uint64_t actor,
       CallerId caller_id,
+      CBuffer& caller_cert,
       const std::vector<uint8_t>& request) override
     {
       return true;
@@ -380,6 +381,7 @@ namespace ccf
       kv::TxHistory::RequestID id,
       uint64_t actor,
       CallerId caller_id,
+      CBuffer& caller_cert,
       const std::vector<uint8_t>& request) override
     {
       LOG_DEBUG << fmt::format("HISTORY: add_request {0}", id) << std::endl;
@@ -389,7 +391,8 @@ namespace ccf
       if (!consensus)
         return false;
 
-      return consensus->on_request({id, request, actor, caller_id});
+      return consensus->on_request(
+        {id, request, actor, caller_id, caller_cert});
     }
 
     void add_result(

src/node/nodestate.h

@@ -360,10 +360,12 @@ namespace ccf
             !public_only);
 
           self = res.id;
-#ifndef PBFT
+#ifdef PBFT
+          setup_pbft();
+#else
           setup_raft(public_only);
-          setup_history();
 #endif
+          setup_history();
           setup_encryptor();
 
           accept_node_connections();

src/node/rpc/frontend.h

@@ -525,7 +525,8 @@ namespace ccf
       reqid = {caller_id.value(), ctx.client_session_id, jsonrpc_id};
       if (history)
       {
-        if (!history->add_request(reqid, ctx.actor, caller_id.value(), input))
+        if (!history->add_request(
+              reqid, ctx.actor, caller_id.value(), ctx.caller_cert, input))
         {
           LOG_FAIL_FMT("Adding request {} failed", jsonrpc_id);
           return jsonrpc::pack(