microsoft
/
CCF
зеркало из https://github.com/microsoft/CCF.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove msgpack from framework - Part I (#2343)

This commit is contained in:
Julien Maffre 2021-03-29 12:40:20 +01:00 коммит произвёл GitHub
Родитель 68ee1ea54b
Коммит 7af1179111
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
56 изменённых файлов: 352 добавлений и 622 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

1
CHANGELOG.md
Просмотреть файл

@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- `UserRpcFrontend` has been removed, and the return value of `get_rpc_handler` which apps should construct is now simply a `ccf::RpcFrontend`.
- There is now a distinction between public and private headers. The public headers under `include/ccf` are those we expect apps to use, and others are implementation details which may change/be deprecated/be hidden in future. Most apps should now be including `"ccf/app_interface.h"` and `"ccf/common_endpoint_registry.h"`.
- Various endpoint-related types have moved under the `ccf::endpoints` namespace. Apps will need to rename these types where they are not using `auto`, for instance to `ccf::endpoints::EndpointContext` and `ccf::endpoints::ForwardingRequired`.
- Ledger entry frames are no longer serialised with `msgpack` (#2343).
## [0.19.3]

3
doc/build_apps/kv/kv_serialisation.rst
Просмотреть файл

@ -30,8 +30,7 @@ The following table describes the structure of a serialised KV Store transaction
+ +------------------------------------------+-------------------------------------------------------------------------+
| | **Repeating [0..n]** | With ``n`` the number of maps in the transaction |
+ +-----+------------------------------------+-------------------------------------------------------------------------+
| | | | ``KOT_MAP_START_INDICATOR`` | | Indicates the start of a new serialised :cpp:type:`kv::Map` |
| | | | char[] | | Name of the serialised :cpp:type:`kv::Map` |
| | | std::string | Name of the serialised :cpp:type:`kv::Map` |
| +-----+------------------------------------+-------------------------------------------------------------------------+
| | | | :cpp:type:`kv::Version` | | Read version |
| +-----+------------------------------------+-------------------------------------------------------------------------+

16
include/ccf/endpoint.h
Просмотреть файл

@ -19,8 +19,6 @@ namespace ccf::endpoints
{
URI uri_path;
RESTVerb verb = HTTP_POST;
MSGPACK_DEFINE(uri_path, verb);
};
DECLARE_JSON_TYPE(EndpointKey);
@ -81,14 +79,6 @@ namespace ccf::endpoints
nlohmann::json openapi;
bool openapi_hidden = false;
MSGPACK_DEFINE(
forwarding_required,
execute_outside_consensus,
authn_policies,
openapi,
openapi_hidden,
mode);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EndpointProperties);
@ -397,8 +387,4 @@ namespace kv::serialisers
return {uri_path, verb};
}
};
}
MSGPACK_ADD_ENUM(ccf::endpoints::ForwardingRequired);
MSGPACK_ADD_ENUM(ccf::endpoints::ExecuteOutsideConsensus);
MSGPACK_ADD_ENUM(ccf::endpoints::Mode);
}

3
include/ccf/entity_id.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "ds/json.h"
#include "ds/serialized.h"
#include <msgpack/msgpack.hpp>
#include <string>
namespace ccf
@ -84,8 +83,6 @@ namespace ccf
{
return id.size();
}
MSGPACK_DEFINE(id);
};
inline void to_json(nlohmann::json& j, const EntityId& entity_id)

64
python/ccf/ledger.py
Просмотреть файл

@ -7,8 +7,6 @@ import os
from typing import BinaryIO, NamedTuple, Optional
# Default implementation has buggy interaction between read_bytes and tell, so use fallback
import msgpack.fallback as msgpack # type: ignore
import json
import base64
@ -27,8 +25,6 @@ LEDGER_TRANSACTION_SIZE = 4
LEDGER_DOMAIN_SIZE = 8
LEDGER_HEADER_SIZE = 8
UNPACK_ARGS = {"raw": True, "strict_map_key": False}
# Public table names as defined in CCF
# https://github.com/microsoft/CCF/blob/main/src/node/entities.h
SIGNATURE_TX_TABLE_NAME = "public:ccf.internal.signatures"
@ -47,6 +43,14 @@ def is_ledger_chunk_committed(file_name):
return file_name.endswith(".committed")
def unpack(stream, fmt):
size = struct.calcsize(fmt)
buf = stream.read(size)
if not buf:
raise EOFError # Reached end of stream
return struct.unpack(fmt, buf)[0]
class GcmHeader:
_gcm_tag = ["\0"] * GCM_SIZE_TAG
_gcm_iv = ["\0"] * GCM_SIZE_IV
@ -69,7 +73,6 @@ class PublicDomain:
_buffer: io.BytesIO
_buffer_size: int
_unpacker: msgpack.Unpacker
_is_snapshot: bool
_version: int
_max_conflict_version: int
@ -77,50 +80,57 @@ class PublicDomain:
def __init__(self, buffer: io.BytesIO):
self._buffer = buffer
self._buffer_size = buffer.getbuffer().nbytes
self._unpacker = msgpack.Unpacker(self._buffer, **UNPACK_ARGS)
self._is_snapshot = self._read_next()
self._version = self._read_next()
self._max_conflict_version = self._read_next()
self._buffer_size = self._buffer.getbuffer().nbytes
self._is_snapshot = self._read_is_snapshot()
self._version = self._read_version()
self._max_conflict_version = self._read_version()
self._tables = {}
self._read()
def _read_next(self):
return self._unpacker.unpack()
def _read_is_snapshot(self):
return unpack(self._buffer, "<?")
def _read_next_string(self):
return self._unpacker.unpack().decode()
def _read_version(self):
return unpack(self._buffer, "<q")
def _read_size(self):
return unpack(self._buffer, "<q")
def _read_string(self):
size = self._read_size()
return self._buffer.read(size).decode()
def _read_next_entry(self):
size_bytes = self._unpacker.read_bytes(8)
(size,) = struct.unpack("<Q", size_bytes)
entry_bytes = bytes(self._unpacker.read_bytes(size))
return entry_bytes
size = self._read_size()
return self._buffer.read(size)
def _read(self):
while self._buffer_size > self._unpacker.tell():
# map_start_indicator
self._read_next()
map_name = self._read_next_string()
LOG.trace(f"Reading map {map_name}")
while True:
try:
map_name = self._read_string()
LOG.trace(f"Reading map {map_name}")
except EOFError:
break
records = {}
self._tables[map_name] = records
# read_version
self._read_next()
self._read_version()
# read_count
read_count = self._read_next()
# Note: Read keys are not currently included in ledger transactions
read_count = self._read_size()
assert read_count == 0, f"Unexpected read count: {read_count}"
write_count = self._read_next()
write_count = self._read_size()
if write_count:
for _ in range(write_count):
k = self._read_next_entry()
val = self._read_next_entry()
records[k] = val
remove_count = self._read_next()
remove_count = self._read_size()
if remove_count:
for _ in range(remove_count):
k = self._read_next_entry()

1
python/requirements.txt
Просмотреть файл

@ -1,4 +1,3 @@
msgpack >= 1.0, == 1.*
loguru >= 0.5, == 0.*
requests >= 2.24, == 2.*
requests-http-signature >= 0.2, == 0.*

3
src/consensus/aft/request.h
Просмотреть файл

@ -6,7 +6,6 @@
#include "kv/map.h"
#include "node/entities.h"
#include <msgpack/msgpack.hpp>
#include <vector>
namespace aft
@ -18,8 +17,6 @@ namespace aft
std::vector<uint8_t> raw;
uint8_t frame_format = enclave::FrameFormat::http;
MSGPACK_DEFINE(rid, caller_cert, raw, frame_format);
size_t serialised_size() const
{
size_t size = sizeof(rid) + sizeof(bool) + sizeof(size_t) + raw.size() +

5
src/consensus/aft/revealed_nonces.h
Просмотреть файл

@ -6,7 +6,6 @@
#include "crypto/hash.h"
#include "kv/map.h"
#include <msgpack/msgpack.hpp>
#include <string>
#include <vector>
@ -18,8 +17,6 @@ namespace aft
ccf::NodeId node_id;
Nonce nonce;
MSGPACK_DEFINE(node_id, nonce);
RevealedNonce(const ccf::NodeId& node_id_, Nonce nonce_) :
node_id(node_id_),
nonce(nonce_)
@ -35,8 +32,6 @@ namespace aft
kv::TxID tx_id;
std::vector<RevealedNonce> nonces;
MSGPACK_DEFINE(tx_id, nonces);
RevealedNonces() = default;
RevealedNonces(kv::TxID tx_id_) : tx_id(tx_id_) {}

13
src/consensus/consensus_types.h
Просмотреть файл

@ -4,7 +4,6 @@
#include "node/nodes.h"
#include <msgpack/msgpack.hpp>
#include <stdint.h>
namespace consensus
@ -15,12 +14,14 @@ namespace consensus
size_t raft_election_timeout;
size_t bft_view_change_timeout;
size_t bft_status_interval;
MSGPACK_DEFINE(
raft_request_timeout,
raft_election_timeout,
bft_view_change_timeout,
bft_status_interval);
};
DECLARE_JSON_TYPE(Configuration);
DECLARE_JSON_REQUIRED_FIELDS(
Configuration,
raft_request_timeout,
raft_election_timeout,
bft_view_change_timeout,
bft_status_interval);
#pragma pack(push, 1)
template <typename T>

7
src/crypto/curve.h
Просмотреть файл

@ -3,6 +3,7 @@
#pragma once
#include "crypto/hash.h"
#include "ds/json.h"
#include "ds/logger.h"
#include <mbedtls/ecp.h>
@ -23,6 +24,12 @@ namespace crypto
SECP256R1
};
DECLARE_JSON_ENUM(
CurveID,
{{CurveID::NONE, "None"},
{CurveID::SECP384R1, "secp384r1"},
{CurveID::SECP256R1, "secp256r1"}});
static constexpr CurveID service_identity_curve_choice = CurveID::SECP384R1;
// SNIPPET_END: supported_curves

5
src/crypto/hash_provider.h
Просмотреть файл

@ -8,7 +8,6 @@
#include <cstdint>
#include <iostream>
#include <msgpack/msgpack.hpp>
#include <vector>
namespace crypto
@ -31,8 +30,6 @@ namespace crypto
{MDType::SHA512, "SHA512"}});
}
MSGPACK_ADD_ENUM(crypto::MDType);
namespace crypto
{
extern void default_sha256(const CBuffer& data, uint8_t* h);
@ -80,8 +77,6 @@ namespace crypto
{
return ds::to_hex(h);
};
MSGPACK_DEFINE(h);
};
inline void to_json(nlohmann::json& j, const Sha256Hash& hash)

1
src/crypto/mbedtls/hash.h
Просмотреть файл

@ -10,7 +10,6 @@
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include <msgpack/msgpack.hpp>
namespace crypto
{

1
src/crypto/openssl/hash.h
Просмотреть файл

@ -11,7 +11,6 @@
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include <msgpack/msgpack.hpp>
namespace crypto
{

3
src/crypto/pem.h
Просмотреть файл

@ -8,7 +8,6 @@
#include <cstring>
#include <exception>
#include <memory>
#include <msgpack/msgpack.hpp>
#include <vector>
namespace crypto
@ -94,8 +93,6 @@ namespace crypto
{
return {data(), data() + s.size()};
}
MSGPACK_DEFINE(s);
};
inline void to_json(nlohmann::json& j, const Pem& p)

7
src/crypto/san.h
Просмотреть файл

@ -2,7 +2,8 @@
// Licensed under the Apache 2.0 License.
#pragma once
#include <msgpack/msgpack.hpp>
#include "ds/json.h"
#include <string>
namespace crypto
@ -11,7 +12,7 @@ namespace crypto
{
std::string san;
bool is_ip;
MSGPACK_DEFINE(san, is_ip);
};
DECLARE_JSON_TYPE(SubjectAltName);
DECLARE_JSON_REQUIRED_FIELDS(SubjectAltName, san, is_ip);
}

4
src/ds/champ_map_serializers.h
Просмотреть файл

@ -5,10 +5,6 @@
#include "ccf_assert.h"
#include "serialized.h"
#include <msgpack/msgpack.hpp>
#include <nlohmann/json.hpp>
#include <small_vector/SmallVector.h>
namespace champ
{
using Version = int64_t;

3
src/ds/hex.h
Просмотреть файл

@ -3,7 +3,8 @@
#pragma once
#include "ds/buffer.h"
#include "ds/logger.h"
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include <string>

3
src/ds/logger_formatters.h
Просмотреть файл

@ -2,9 +2,10 @@
// Licensed under the Apache 2.0 License.
#pragma once
#include "ds/hex.h"
#define FMT_HEADER_ONLY
#include <fmt/format.h>
#include <msgpack/msgpack.hpp>
#include <sstream>
namespace fmt

10
src/ds/nonstd.h
Просмотреть файл

@ -38,6 +38,16 @@ namespace nonstd
struct is_std_array<std::array<T, N>> : public std::true_type
{};
/** Similar to is_specialization, but for detecting std::vector specifically
*/
template <typename T>
struct is_std_vector : std::false_type
{};
template <typename T>
struct is_std_vector<std::vector<T>> : public std::true_type
{};
/** dependent_false produces a static, compile-time false, dependent on a
* specific type or value instantiation. This is useful for producing a
* static_assert which will fail only when invalid paths are called, but

4
src/enclave/consensus_type.h
Просмотреть файл

@ -4,8 +4,6 @@
#include "ds/json.h"
#include <msgpack/msgpack.hpp>
enum ConsensusType
{
CFT = 0,
@ -14,5 +12,3 @@ enum ConsensusType
DECLARE_JSON_ENUM(
ConsensusType, {{ConsensusType::CFT, "CFT"}, {ConsensusType::BFT, "BFT"}})
MSGPACK_ADD_ENUM(ConsensusType);

54
src/enclave/interface.h
Просмотреть файл

@ -9,6 +9,7 @@
#include "crypto/curve.h"
#include "crypto/san.h"
#include "ds/buffer.h"
#include "ds/json.h"
#include "ds/logger.h"
#include "ds/oversized.h"
#include "ds/ring_buffer_types.h"
@ -41,8 +42,6 @@ struct EnclaveConfig
#endif
};
MSGPACK_ADD_ENUM(crypto::CurveID);
struct CCFConfig
{
consensus::Configuration consensus_config = {};
@ -58,7 +57,6 @@ struct CCFConfig
{
size_t sig_tx_interval;
size_t sig_ms_interval;
MSGPACK_DEFINE(sig_tx_interval, sig_ms_interval);
};
SignatureIntervals signature_intervals = {};
@ -68,7 +66,6 @@ struct CCFConfig
std::string gov_script;
std::string constitution;
size_t recovery_threshold;
MSGPACK_DEFINE(members_info, gov_script, constitution, recovery_threshold);
};
Genesis genesis = {};
@ -78,7 +75,6 @@ struct CCFConfig
std::string target_port;
std::vector<uint8_t> network_cert;
size_t join_timer;
MSGPACK_DEFINE(target_host, target_port, network_cert, join_timer);
};
Joining joining = {};
@ -88,23 +84,41 @@ struct CCFConfig
size_t jwt_key_refresh_interval_s;
crypto::CurveID curve_id;
MSGPACK_DEFINE(
consensus_config,
node_info_network,
domain,
snapshot_tx_interval,
startup_snapshot,
startup_snapshot_evidence_seqno,
signature_intervals,
genesis,
joining,
subject_name,
subject_alternative_names,
jwt_key_refresh_interval_s,
curve_id);
};
DECLARE_JSON_TYPE(CCFConfig::SignatureIntervals);
DECLARE_JSON_REQUIRED_FIELDS(
CCFConfig::SignatureIntervals, sig_tx_interval, sig_ms_interval);
DECLARE_JSON_TYPE(CCFConfig::Genesis);
DECLARE_JSON_REQUIRED_FIELDS(
CCFConfig::Genesis,
members_info,
gov_script,
constitution,
recovery_threshold);
DECLARE_JSON_TYPE(CCFConfig::Joining);
DECLARE_JSON_REQUIRED_FIELDS(
CCFConfig::Joining, target_host, target_port, network_cert, join_timer);
DECLARE_JSON_TYPE(CCFConfig);
DECLARE_JSON_REQUIRED_FIELDS(
CCFConfig,
consensus_config,
node_info_network,
domain,
snapshot_tx_interval,
startup_snapshot,
startup_snapshot_evidence_seqno,
signature_intervals,
genesis,
joining,
subject_name,
subject_alternative_names,
jwt_key_refresh_interval_s,
curve_id);
/// General administrative messages
enum AdminMessage : ringbuffer::Message
{

8
src/enclave/main.cpp
Просмотреть файл

@ -1,5 +1,6 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#include "ds/json.h"
#include "ds/logger.h"
#include "ds/spin_lock.h"
#include "ds/stacktrace_utils.h"
@ -8,7 +9,6 @@
#include "oe_shim.h"
#include <chrono>
#include <msgpack/msgpack.hpp>
#include <thread>
// the central enclave object
@ -127,10 +127,8 @@ extern "C"
oe_lfence();
msgpack::object_handle oh = msgpack::unpack(ccf_config, ccf_config_size);
msgpack::object obj = oh.get();
CCFConfig cc;
obj.convert(cc);
CCFConfig cc =
nlohmann::json::parse(ccf_config, ccf_config + ccf_config_size);
#ifdef DEBUG_CONFIG
reserved_memory = new uint8_t[ec->debug_config.memory_reserve_startup];

2
src/enclave/rpc_context.h
Просмотреть файл

@ -74,8 +74,6 @@ namespace ccf
{
return !(*this == o);
}
MSGPACK_DEFINE(verb);
};
// Custom to_json and from_json specializations which encode RESTVerb in a

8
src/host/enclave.h
Просмотреть файл

@ -8,7 +8,6 @@
#include "enclave/interface.h"
#include <dlfcn.h>
#include <msgpack/msgpack.hpp>
#ifdef VIRTUAL_ENCLAVE
# include "enclave/ccf_v.h"
#else
@ -82,15 +81,14 @@ namespace host
size_t node_cert_len = 0;
size_t network_cert_len = 0;
msgpack::sbuffer sbuf;
msgpack::pack(sbuf, ccf_config);
auto config = nlohmann::json(ccf_config).dump();
auto err = enclave_create_node(
e,
&ret,
(void*)&enclave_config,
sbuf.data(),
sbuf.size(),
config.data(),
config.size(),
node_cert.data(),
node_cert.size(),
&node_cert_len,

136
src/kv/generic_serialise_wrapper.h
Просмотреть файл

@ -13,35 +13,6 @@ namespace kv
using SerialisedKey = kv::serialisers::SerialisedEntry;
using SerialisedValue = kv::serialisers::SerialisedEntry;
enum class KvOperationType : uint32_t
{
KOT_NOT_SUPPORTED = 0,
KOT_SET_VERSION = (1 << 0),
KOT_MAP_START_INDICATOR = (1 << 1),
KOT_ENTRY_VERSION = (1 << 2),
KOT_READ = (1 << 3),
KOT_WRITE_VERSION = (1 << 4),
KOT_WRITE = (1 << 5),
KOT_REMOVE_VERSION = (1 << 6),
KOT_REMOVE = (1 << 7),
};
typedef std::underlying_type<KvOperationType>::type KotBase;
inline KvOperationType operator&(
const KvOperationType& a, const KvOperationType& b)
{
return static_cast<KvOperationType>(
static_cast<KotBase>(a) & static_cast<KotBase>(b));
}
inline KvOperationType operator|(
const KvOperationType& a, const KvOperationType& b)
{
return static_cast<KvOperationType>(
static_cast<KotBase>(a) | static_cast<KotBase>(b));
}
template <typename W>
class GenericSerialiseWrapper
{
@ -59,15 +30,9 @@ namespace kv
SecurityDomain current_domain;
template <typename T>
void serialise_internal(T&& t)
void serialise_internal(const T& t)
{
current_writer->append(std::forward<T>(t));
}
template <typename T>
void serialise_internal_pre_serialised(const T& raw)
{
current_writer->template append_pre_serialised<T>(raw);
current_writer->append(t);
}
void set_current_domain(SecurityDomain domain)
@ -113,15 +78,16 @@ namespace kv
}
if (domain != current_domain)
{
set_current_domain(domain);
}
serialise_internal(KvOperationType::KOT_MAP_START_INDICATOR);
serialise_internal(name);
}
void serialise_raw(const std::vector<uint8_t>& raw)
{
serialise_internal_pre_serialised(raw);
serialise_internal(raw);
}
void serialise_view_history(const std::vector<Version>& view_history)
@ -142,34 +108,19 @@ namespace kv
void serialise_read(const SerialisedKey& k, const Version& version)
{
serialise_internal_pre_serialised(k);
serialise_internal(k);
serialise_internal(version);
}
void serialise_write(const SerialisedKey& k, const SerialisedValue& v)
{
serialise_internal_pre_serialised(k);
serialise_internal_pre_serialised(v);
}
void serialise_write_version(
const SerialisedKey& k, const SerialisedValue& v, const Version& version)
{
serialise_internal(KvOperationType::KOT_WRITE_VERSION);
serialise_internal_pre_serialised(k);
serialise_internal_pre_serialised(v);
serialise_internal(version);
}
void serialise_remove_version(const SerialisedKey& k)
{
serialise_internal(KvOperationType::KOT_REMOVE_VERSION);
serialise_internal_pre_serialised(k);
serialise_internal(k);
serialise_internal(v);
}
void serialise_remove(const SerialisedKey& k)
{
serialise_internal_pre_serialised(k);
serialise_internal(k);
}
std::vector<uint8_t> get_raw_data()
@ -253,51 +204,12 @@ namespace kv
R private_reader;
R* current_reader;
std::vector<uint8_t> decrypted_buffer;
KvOperationType unhandled_op;
bool is_snapshot;
Version version;
Version max_conflict_version;
std::shared_ptr<AbstractTxEncryptor> crypto_util;
std::optional<SecurityDomain> domain_restriction;
bool try_read_op(KvOperationType type)
{
return try_read_op(type, *current_reader);
}
bool try_read_op(KvOperationType type, R& reader)
{
return try_read_op_flag(type, reader) == type;
}
KvOperationType try_read_op_flag(KvOperationType type)
{
return try_read_op_flag(type, *current_reader);
}
KvOperationType try_read_op_flag(KvOperationType type, R& reader)
{
if (unhandled_op != KvOperationType::KOT_NOT_SUPPORTED)
{
auto curr_type = (type & unhandled_op);
if (curr_type != KvOperationType::KOT_NOT_SUPPORTED)
{
// clear cached op header
unhandled_op = KvOperationType::KOT_NOT_SUPPORTED;
}
return curr_type;
}
auto next_op = reader.template read_next<KvOperationType>();
if ((type & next_op) == next_op)
{
return next_op;
}
unhandled_op = next_op;
return KvOperationType::KOT_NOT_SUPPORTED;
}
// Should only be called once, once the GCM header and length of public
// domain have been read
void read_public_header()
@ -311,7 +223,6 @@ namespace kv
GenericDeserialiseWrapper(
std::shared_ptr<AbstractTxEncryptor> e,
std::optional<SecurityDomain> domain_restriction = std::nullopt) :
unhandled_op(KvOperationType::KOT_NOT_SUPPORTED),
crypto_util(e),
domain_restriction(domain_restriction)
{}
@ -376,18 +287,16 @@ namespace kv
if (current_reader->is_eos())
{
if (current_reader == &public_reader && !private_reader.is_eos())
{
current_reader = &private_reader;
}
else
return {};
{
return std::nullopt;
}
}
if (!try_read_op(KvOperationType::KOT_MAP_START_INDICATOR))
{
return {};
}
return std::optional<std::string>{
current_reader->template read_next<std::string>()};
return current_reader->template read_next<std::string>();
}
Version deserialise_entry_version()
@ -402,9 +311,8 @@ namespace kv
std::tuple<SerialisedKey, Version> deserialise_read()
{
return {
current_reader->template read_next_pre_serialised<SerialisedKey>(),
current_reader->template read_next<Version>()};
return {current_reader->template read_next<SerialisedKey>(),
current_reader->template read_next<Version>()};
}
uint64_t deserialise_write_header()
@ -414,15 +322,13 @@ namespace kv
std::tuple<SerialisedKey, SerialisedValue> deserialise_write()
{
return {
current_reader->template read_next_pre_serialised<SerialisedKey>(),
current_reader->template read_next_pre_serialised<SerialisedValue>()};
return {current_reader->template read_next<SerialisedKey>(),
current_reader->template read_next<SerialisedValue>()};
}
std::vector<uint8_t> deserialise_raw()
{
return current_reader
->template read_next_pre_serialised<std::vector<uint8_t>>();
return current_reader->template read_next<std::vector<uint8_t>>();
}
std::vector<Version> deserialise_view_history()
@ -437,7 +343,7 @@ namespace kv
SerialisedKey deserialise_remove()
{
return current_reader->template read_next_pre_serialised<SerialisedKey>();
return current_reader->template read_next<SerialisedKey>();
}
bool end()

2
src/kv/kv_serialiser.h
Просмотреть файл

@ -6,5 +6,5 @@
#ifdef USE_NLJSON_KV_SERIALISER
# include "kv/nljson_serialise.h"
#else
# include "kv/msgpack_serialise.h"
# include "kv/raw_serialise.h"
#endif

2
src/kv/kv_types.h
Просмотреть файл

@ -14,6 +14,7 @@
#include <functional>
#include <limits>
#include <memory>
#include <set>
#include <string>
#include <unordered_set>
#include <vector>
@ -51,7 +52,6 @@ namespace kv
{
Term term = 0;
Version version = 0;
MSGPACK_DEFINE(term, version);
};
DECLARE_JSON_TYPE(TxID);
DECLARE_JSON_REQUIRED_FIELDS(TxID, term, version)

127
src/kv/msgpack_serialise.h
Просмотреть файл

@ -1,127 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#pragma once
#include "ds/msgpack_adaptor_nlohmann.h"
#include "ds/serialized.h"
#include "generic_serialise_wrapper.h"
#include "serialised_entry.h"
#include <iterator>
#include <msgpack/msgpack.hpp>
#include <nlohmann/json.hpp>
#include <sstream>
#include <type_traits>
MSGPACK_ADD_ENUM(kv::KvOperationType);
MSGPACK_ADD_ENUM(kv::SecurityDomain);
namespace kv
{
class MsgPackWriter
{
private:
msgpack::sbuffer sb;
public:
template <typename T>
void append(T&& t)
{
msgpack::pack(sb, std::forward<T>(t));
}
// Where we have pre-serialised data, we dump it length-prefixed into the
// output buffer. If we call append, then pack will prefix the data with
// some type information, potentially redundantly repacking already-packed
// data. This means the output is no longer a stream of msgpack objects!
// Parsers are expected to know the type of the Ks and Vs for the tables
// they care about, and skip over any others
template <typename T>
void append_pre_serialised(const T& entry)
{
const uint64_t size = entry.size();
sb.write(reinterpret_cast<char const*>(&size), sizeof(size));
if (entry.size() > 0)
{
sb.write(reinterpret_cast<char const*>(entry.data()), entry.size());
}
}
void clear()
{
sb.clear();
}
bool is_empty()
{
return sb.size() == 0;
}
std::vector<uint8_t> get_raw_data()
{
return {reinterpret_cast<uint8_t*>(sb.data()),
reinterpret_cast<uint8_t*>(sb.data()) + sb.size()};
}
};
class MsgPackReader
{
public:
const char* data_ptr;
size_t data_offset;
size_t data_size;
msgpack::object_handle msg;
public:
MsgPackReader(const MsgPackReader& other) = delete;
MsgPackReader& operator=(const MsgPackReader& other) = delete;
MsgPackReader(const uint8_t* data_in_ptr = nullptr, size_t data_in_size = 0)
{
init(data_in_ptr, data_in_size);
}
void init(const uint8_t* data_in_ptr, size_t data_in_size)
{
data_offset = 0;
data_ptr = (const char*)data_in_ptr;
data_size = data_in_size;
}
template <typename T>
T read_next()
{
msgpack::unpack(msg, data_ptr, data_size, data_offset);
return msg->as<T>();
}
template <typename T>
T read_next_pre_serialised()
{
auto remainder = data_size - data_offset;
auto data = reinterpret_cast<const uint8_t*>(data_ptr + data_offset);
const auto entry_size = serialized::read<uint64_t>(data, remainder);
if (remainder < entry_size)
{
throw std::runtime_error(fmt::format(
"Expected {} byte entry, found only {}", entry_size, remainder));
}
const auto bytes_read = data_size - data_offset - remainder;
data_offset += bytes_read;
const auto before_offset = data_offset;
data_offset += entry_size;
return {data_ptr + before_offset, data_ptr + data_offset};
}
bool is_eos()
{
return data_offset >= data_size;
}
};
using KvStoreSerialiser = GenericSerialiseWrapper<MsgPackWriter>;
using KvStoreDeserialiser = GenericDeserialiseWrapper<MsgPackReader>;
}

200
src/kv/raw_serialise.h Normal file
Просмотреть файл

@ -0,0 +1,200 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#pragma once
#include "ds/serialized.h"
#include "generic_serialise_wrapper.h"
#include <small_vector/SmallVector.h>
#include <type_traits>
namespace kv
{
class RawWriter
{
private:
// Avoid heap allocations for transactions which only touch a limited number
// keys of keys in a few maps
using WriterData = llvm_vecsmall::SmallVector<uint8_t, 64>;
WriterData buf;
template <typename T>
void serialise_entry(const T& t)
{
size_t size_before = buf.size();
buf.resize(buf.size() + sizeof(T));
auto data_ = buf.data() + size_before;
auto size_ = buf.size() - size_before;
serialized::write(data_, size_, t);
}
template <typename T>
void serialise_vector(const T& entry)
{
size_t entry_size_bytes = sizeof(typename T::value_type) * entry.size();
size_t size_before = buf.size();
buf.resize(buf.size() + entry_size_bytes);
auto data_ = buf.data() + size_before;
auto size_ = buf.size() - size_before;
serialized::write(
data_,
size_,
reinterpret_cast<const uint8_t*>(entry.data()),
entry_size_bytes);
}
void serialise_string(const std::string& str)
{
size_t size_before = buf.size();
buf.resize(buf.size() + sizeof(size_t) + str.size());
auto data_ = buf.data() + size_before;
auto size_ = buf.size() - size_before;
serialized::write(data_, size_, str);
}
public:
RawWriter() = default;
template <typename T>
void append(const T& entry)
{
if constexpr (
nonstd::is_std_vector<T>::value ||
std::is_same_v<T, kv::serialisers::SerialisedEntry>)
{
serialise_entry(entry.size() * sizeof(typename T::value_type));
if (entry.size() > 0)
{
serialise_vector(entry);
}
}
else if constexpr (std::is_same_v<T, std::string>)
{
serialise_string(entry);
}
else if constexpr (std::is_integral_v<T>)
{
serialise_entry(entry);
}
else
{
static_assert(
nonstd::dependent_false<T>::value, "Can't serialise this type");
}
}
void clear()
{
buf.clear();
}
std::vector<uint8_t> get_raw_data()
{
return {buf.data(), buf.data() + buf.size()};
}
};
class RawReader
{
public:
const uint8_t* data_ptr;
size_t data_offset;
size_t data_size;
/** Reads the next entry, advancing data_offset
*/
template <typename T>
T read_entry()
{
auto remainder = data_size - data_offset;
auto data = data_ptr + data_offset;
const auto entry = serialized::read<T>(data, remainder);
const auto bytes_read = data_size - data_offset - remainder;
data_offset += bytes_read;
return entry;
}
/** Reads the next size-prefixed entry
*/
size_t read_size_prefixed_entry(size_t& start_offset)
{
auto remainder = data_size - data_offset;
auto entry_size = read_entry<size_t>();
if (remainder < entry_size)
{
throw std::runtime_error(fmt::format(
"Expected {} byte entry, found only {}", entry_size, remainder));
}
start_offset = data_offset;
data_offset += entry_size;
return entry_size;
}
public:
RawReader(const RawReader& other) = delete;
RawReader& operator=(const RawReader& other) = delete;
RawReader(const uint8_t* data_in_ptr = nullptr, size_t data_in_size = 0)
{
init(data_in_ptr, data_in_size);
}
void init(const uint8_t* data_in_ptr, size_t data_in_size)
{
data_offset = 0;
data_ptr = data_in_ptr;
data_size = data_in_size;
}
template <typename T>
T read_next()
{
if constexpr (
nonstd::is_std_vector<T>::value ||
std::is_same_v<T, kv::serialisers::SerialisedEntry>)
{
size_t entry_offset = 0;
size_t entry_size = read_size_prefixed_entry(entry_offset);
T ret(entry_size / sizeof(typename T::value_type));
auto data_ = reinterpret_cast<uint8_t*>(ret.data());
auto size_ = entry_size;
serialized::write(data_, size_, data_ptr + entry_offset, entry_size);
return ret;
}
else if constexpr (std::is_same_v<T, std::string>)
{
size_t entry_offset = 0;
read_size_prefixed_entry(entry_offset);
return {data_ptr + entry_offset, data_ptr + data_offset};
}
else if constexpr (std::is_integral_v<T>)
{
return read_entry<T>();
}
else
{
static_assert(
nonstd::dependent_false<T>::value, "Can't deserialise this type");
}
}
bool is_eos()
{
return data_offset >= data_size;
}
};
using KvStoreSerialiser = GenericSerialiseWrapper<RawWriter>;
using KvStoreDeserialiser = GenericDeserialiseWrapper<RawReader>;
}

1
src/kv/serialised_entry.h
Просмотреть файл

@ -2,7 +2,6 @@
// Licensed under the Apache 2.0 License.
#pragma once
#include <msgpack/msgpack.hpp>
#include <nlohmann/json.hpp>
#include <small_vector/SmallVector.h>

8
src/kv/serialiser_declare.h
Просмотреть файл

@ -21,10 +21,10 @@ namespace kv
class JsonReader;
using KvStoreDeserialiser = GenericDeserialiseWrapper<JsonReader>;
#else
class MsgPackWriter;
using KvStoreSerialiser = GenericSerialiseWrapper<MsgPackWriter>;
class RawWriter;
using KvStoreSerialiser = GenericSerialiseWrapper<RawWriter>;
class MsgPackReader;
using KvStoreDeserialiser = GenericDeserialiseWrapper<MsgPackReader>;
class RawReader;
using KvStoreDeserialiser = GenericDeserialiseWrapper<RawReader>;
#endif
}

1
src/kv/store.h
Просмотреть файл

@ -14,6 +14,7 @@
#include "snapshot.h"
#include "tx.h"
#define FMT_HEADER_ONLY
#include <fmt/format.h>
namespace kv

8
src/kv/test/kv_bench.cpp
Просмотреть файл

@ -90,7 +90,7 @@ static void serialise(picobench::state& s)
}
template <kv::SecurityDomain SD>
static void apply(picobench::state& s)
static void deserialise(picobench::state& s)
{
logger::config::level() = logger::INFO;
@ -255,12 +255,12 @@ PICOBENCH(serialise<SD::PUBLIC>)
.baseline();
PICOBENCH(serialise<SD::PRIVATE>).iterations(tx_count).samples(sample_size);
PICOBENCH_SUITE("apply");
PICOBENCH(apply<SD::PUBLIC>)
PICOBENCH_SUITE("deserialise");
PICOBENCH(deserialise<SD::PUBLIC>)
.iterations(tx_count)
.samples(sample_size)
.baseline();
PICOBENCH(apply<SD::PRIVATE>).iterations(tx_count).samples(sample_size);
PICOBENCH(deserialise<SD::PRIVATE>).iterations(tx_count).samples(sample_size);
const uint32_t snapshot_sample_size = 10;
const std::vector<int> map_count = {20, 100};

2
src/kv/test/kv_contention.cpp
Просмотреть файл

@ -8,6 +8,8 @@
#include <atomic>
#include <chrono>
#define DOCTEST_CONFIG_NO_SHORT_MACRO_NAMES
#include "ds/msgpack_adaptor_nlohmann.h"
#include <doctest/doctest.h>
#include <string>
#include <thread>

1
src/kv/test/kv_serialisation.cpp
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#include "ds/logger.h"
#include "ds/msgpack_adaptor_nlohmann.h"
#include "kv/kv_serialiser.h"
#include "kv/store.h"
#include "kv/test/null_encryptor.h"

1
src/kv/untyped_map.h
Просмотреть файл

@ -10,6 +10,7 @@
#include "kv/untyped_map_handle.h"
#include <functional>
#include <list>
#include <optional>
#include <unordered_set>

5
src/node/backup_signatures.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "node_signature.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <string>
#include <vector>
@ -18,8 +17,6 @@ namespace ccf
crypto::Sha256Hash root;
std::vector<NodeSignature> signatures;
MSGPACK_DEFINE(view, seqno, root, signatures);
BackupSignatures() = default;
BackupSignatures(
@ -32,7 +29,7 @@ namespace ccf
{}
};
DECLARE_JSON_TYPE(BackupSignatures);
DECLARE_JSON_REQUIRED_FIELDS(BackupSignatures, view, seqno, root, signatures)
DECLARE_JSON_REQUIRED_FIELDS(BackupSignatures, view, seqno, root, signatures);
// Always recorded at key 0
using BackupSignaturesMap = ServiceMap<size_t, BackupSignatures>;

3
src/node/client_signatures.h
Просмотреть файл

@ -8,7 +8,6 @@
#include "service_map.h"
#include <mbedtls/md.h>
#include <msgpack/msgpack.hpp>
#include <vector>
namespace ccf
@ -34,8 +33,6 @@ namespace ccf
return (sig == other.sig) && (req == other.req) && (md == other.md) &&
(request_body == other.request_body) && (key_id == other.key_id);
}
MSGPACK_DEFINE(sig, req, request_body, md);
};
DECLARE_JSON_TYPE(SignedReq)
DECLARE_JSON_REQUIRED_FIELDS(SignedReq, sig, req, request_body, md, key_id)

7
src/node/code_id.h
Просмотреть файл

@ -8,8 +8,6 @@
#include "entities.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
namespace ccf
{
struct CodeDigest
@ -45,11 +43,6 @@ namespace ccf
};
DECLARE_JSON_ENUM(
CodeStatus, {{CodeStatus::ALLOWED_TO_JOIN, "AllowedToJoin"}});
}
MSGPACK_ADD_ENUM(ccf::CodeStatus);
namespace ccf
{
using CodeIDs = ServiceMap<CodeDigest, CodeStatus>;
}

4
src/node/config.h
Просмотреть файл

@ -6,8 +6,6 @@
#include "enclave/consensus_type.h"
#include "entities.h"
#include <msgpack/msgpack.hpp>
namespace ccf
{
struct ServiceConfiguration
@ -16,8 +14,6 @@ namespace ccf
size_t recovery_threshold = 0;
ConsensusType consensus = ConsensusType::CFT;
MSGPACK_DEFINE(recovery_threshold, consensus)
};
DECLARE_JSON_TYPE(ServiceConfiguration)
DECLARE_JSON_REQUIRED_FIELDS(

9
src/node/jwt.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "entities.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <optional>
namespace ccf
@ -20,8 +19,6 @@ namespace ccf
{
return rhs.sgx_claims != sgx_claims;
}
MSGPACK_DEFINE(sgx_claims);
};
DECLARE_JSON_TYPE(JwtIssuerKeyPolicy);
@ -43,8 +40,6 @@ namespace ccf
std::optional<JwtIssuerKeyPolicy> key_policy;
std::optional<std::string> ca_cert_bundle_name;
bool auto_refresh = false;
MSGPACK_DEFINE(key_filter, key_policy, ca_cert_bundle_name, auto_refresh);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(JwtIssuerMetadata);
@ -59,6 +54,4 @@ namespace ccf
using JwtPublicSigningKeys = kv::RawCopySerialisedMap<JwtKeyId, Cert>;
using JwtPublicSigningKeyIssuer =
kv::RawCopySerialisedMap<JwtKeyId, JwtIssuer>;
}
MSGPACK_ADD_ENUM(ccf::JwtIssuerKeyFilter);
}

9
src/node/members.h
Просмотреть файл

@ -7,7 +7,6 @@
#include "ds/hash.h"
#include "node_signature.h"
#include <msgpack/msgpack.hpp>
#include <vector>
namespace ccf
@ -22,8 +21,6 @@ namespace ccf
{{MemberStatus::ACCEPTED, "Accepted"}, {MemberStatus::ACTIVE, "Active"}});
}
MSGPACK_ADD_ENUM(ccf::MemberStatus);
namespace ccf
{
// Current limitations of secret sharing library (sss).
@ -53,8 +50,6 @@ namespace ccf
return cert == rhs.cert && encryption_pub_key == rhs.encryption_pub_key &&
member_data == rhs.member_data;
}
MSGPACK_DEFINE(cert, encryption_pub_key, member_data);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(NewMember)
DECLARE_JSON_REQUIRED_FIELDS(NewMember, cert)
@ -92,8 +87,6 @@ namespace ccf
StateDigest(const crypto::Sha256Hash& root) : state_digest(root.hex_str())
{}
MSGPACK_DEFINE(state_digest);
};
DECLARE_JSON_TYPE(StateDigest)
DECLARE_JSON_REQUIRED_FIELDS(StateDigest, state_digest)
@ -111,8 +104,6 @@ namespace ccf
StateDigest(root),
signed_req(signed_req_)
{}
MSGPACK_DEFINE(MSGPACK_BASE(StateDigest), signed_req);
};
DECLARE_JSON_TYPE_WITH_BASE(MemberAck, StateDigest)
DECLARE_JSON_REQUIRED_FIELDS(MemberAck, signed_req)

1
src/node/modules.h
Просмотреть файл

@ -4,7 +4,6 @@
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <optional>
#include <stdint.h>
#include <string>

3
src/node/node_info_network.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "ds/json.h"
#include <msgpack/msgpack.hpp>
#include <string>
namespace ccf
@ -18,8 +17,6 @@ namespace ccf
std::string nodeport;
std::string rpcport;
std::string pubport;
MSGPACK_DEFINE(rpchost, pubhost, nodehost, nodeport, rpcport, pubport);
};
DECLARE_JSON_TYPE(NodeInfoNetwork);
DECLARE_JSON_REQUIRED_FIELDS(

2
src/node/node_signature.h
Просмотреть файл

@ -74,8 +74,6 @@ namespace ccf
return n;
}
MSGPACK_DEFINE(sig, node, hashed_nonce);
};
DECLARE_JSON_TYPE(NodeSignature);
DECLARE_JSON_REQUIRED_FIELDS(NodeSignature, sig, node, hashed_nonce);

11
src/node/nodes.h
Просмотреть файл

@ -9,7 +9,6 @@
#include "quote_info.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <string>
#include <vector>
@ -28,8 +27,6 @@ namespace ccf
{NodeStatus::RETIRED, "Retired"}});
}
MSGPACK_ADD_ENUM(ccf::NodeStatus);
namespace ccf
{
struct NodeInfo : NodeInfoNetwork
@ -42,14 +39,6 @@ namespace ccf
// Set to the seqno of the latest ledger secret at the time the node is
// trusted
std::optional<kv::Version> ledger_secret_seqno = std::nullopt;
MSGPACK_DEFINE(
MSGPACK_BASE(NodeInfoNetwork),
cert,
quote_info,
encryption_pub_key,
status,
ledger_secret_seqno);
};
DECLARE_JSON_TYPE_WITH_BASE_AND_OPTIONAL_FIELDS(NodeInfo, NodeInfoNetwork);
DECLARE_JSON_REQUIRED_FIELDS(

4
src/node/proposals.h
Просмотреть файл

@ -3,12 +3,10 @@
#pragma once
#include "ds/json.h"
#include "ds/msgpack_adaptor_nlohmann.h"
#include "entities.h"
#include "script.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <unordered_map>
#include <vector>
@ -86,8 +84,6 @@ namespace ccf
return script == o.script && parameter == o.parameter &&
proposer == o.proposer && state == o.state && votes == o.votes;
}
MSGPACK_DEFINE(script, parameter, proposer, state, votes);
};
DECLARE_JSON_TYPE(Proposal)
DECLARE_JSON_REQUIRED_FIELDS(

8
src/node/quote_info.h
Просмотреть файл

@ -4,7 +4,6 @@
#pragma once
#include "ds/json.h"
#include <msgpack/msgpack.hpp>
#include <vector>
namespace ccf
@ -15,20 +14,13 @@ namespace ccf
};
DECLARE_JSON_ENUM(QuoteFormat, {{QuoteFormat::oe_sgx_v1, "OE_SGX_v1"}})
}
MSGPACK_ADD_ENUM(ccf::QuoteFormat);
namespace ccf
{
struct QuoteInfo
{
QuoteFormat format = QuoteFormat::oe_sgx_v1;
std::vector<uint8_t> quote;
std::vector<uint8_t> endorsements;
MSGPACK_DEFINE(format, quote, endorsements);
};
DECLARE_JSON_TYPE(QuoteInfo);

3
src/node/script.h
Просмотреть файл

@ -2,7 +2,6 @@
// Licensed under the Apache 2.0 License.
#pragma once
#include <msgpack/msgpack.hpp>
#include <optional>
#include <stdint.h>
#include <string>
@ -38,8 +37,6 @@ namespace ccf
{
return !operator==(other);
}
MSGPACK_DEFINE(bytecode, text);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(Script);

5
src/node/secrets.h
Просмотреть файл

@ -4,7 +4,6 @@
#include "entities.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <vector>
namespace ccf
@ -20,8 +19,6 @@ namespace ccf
// Version at which the previous secret is stored at
std::optional<kv::Version> previous_secret_stored_version = std::nullopt;
MSGPACK_DEFINE(version, encrypted_secret, previous_secret_stored_version);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EncryptedLedgerSecret)
@ -37,8 +34,6 @@ namespace ccf
std::vector<uint8_t> primary_public_encryption_key = {};
SecretsForNodes secrets_for_nodes = {};
MSGPACK_DEFINE(primary_public_encryption_key, secrets_for_nodes)
};
DECLARE_JSON_TYPE(EncryptedLedgerSecretsNodesInfo)
DECLARE_JSON_REQUIRED_FIELDS(

9
src/node/service.h
Просмотреть файл

@ -6,8 +6,6 @@
#include "entities.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
namespace ccf
{
enum class ServiceStatus
@ -24,18 +22,11 @@ namespace ccf
{ServiceStatus::OPEN, "Open"},
{ServiceStatus::WAITING_FOR_RECOVERY_SHARES, "WaitingForRecoveryShares"},
{ServiceStatus::CLOSED, "Closed"}});
}
MSGPACK_ADD_ENUM(ccf::ServiceStatus);
namespace ccf
{
struct ServiceInfo
{
crypto::Pem cert;
ServiceStatus status;
MSGPACK_DEFINE(cert, status);
};
DECLARE_JSON_TYPE(ServiceInfo);
DECLARE_JSON_REQUIRED_FIELDS(ServiceInfo, cert, status);

10
src/node/shares.h
Просмотреть файл

@ -6,7 +6,6 @@
#include "service_map.h"
#include <map>
#include <msgpack/msgpack.hpp>
#include <optional>
#include <vector>
@ -26,11 +25,6 @@ namespace ccf
// Version at which the previous ledger secret was written to the store
std::optional<kv::Version> previous_secret_stored_version = std::nullopt;
MSGPACK_DEFINE(
wrapped_latest_ledger_secret,
encrypted_shares,
previous_secret_stored_version);
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(RecoverySharesInfo)
@ -73,8 +67,6 @@ namespace ccf
{
return !(*this == other);
}
MSGPACK_DEFINE(encrypted_data, version, previous_secret_stored_version)
};
DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(PreviousLedgerSecretInfo)
@ -99,8 +91,6 @@ namespace ccf
// shares are only written at a later version, once the previous ledger
// secrets have been restored.
std::optional<kv::Version> next_version = std::nullopt;
MSGPACK_DEFINE(previous_ledger_secret, next_version)
};
// Note: Both fields are never empty at the same time

9
src/node/signatures.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "node_signature.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <string>
#include <vector>
@ -19,14 +18,6 @@ namespace ccf
kv::Consensus::View commit_view = 0;
crypto::Sha256Hash root;
MSGPACK_DEFINE(
MSGPACK_BASE(NodeSignature),
seqno,
view,
commit_seqno,
commit_view,
root);
PrimarySignature() {}
PrimarySignature(const ccf::NodeId& node_, kv::Consensus::SeqNo seqno_) :

4
src/node/snapshot_evidence.h
Просмотреть файл

@ -8,16 +8,12 @@
#include "kv/kv_types.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
namespace ccf
{
struct SnapshotHash
{
crypto::Sha256Hash hash;
kv::Version version;
MSGPACK_DEFINE(hash, version);
};
DECLARE_JSON_TYPE(SnapshotHash)

146
src/node/test/msgpack_serialization.cpp
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the Apache 2.0 License.
#include "node/members.h"
#include "node/proposals.h"
#include "node/signatures.h"
#include "ds/msgpack_adaptor_nlohmann.h"
#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
#include <doctest/doctest.h>
@ -66,144 +64,4 @@ TEST_CASE("nlohmann::json")
const auto converted = msgpack_roundtrip(j_object);
CHECK(j_object == converted);
}
}
TEST_CASE("Proposal")
{
using namespace ccf;
{
INFO("Empty proposal");
Proposal proposal;
const auto converted = msgpack_roundtrip(proposal);
CHECK(proposal == converted);
}
{
INFO("Initial proposal");
Script s("return true");
nlohmann::json p("hello world");
auto m = MemberId("member");
Proposal proposal(s, p, m);
const auto converted = msgpack_roundtrip(proposal);
CHECK(proposal == converted);
}
{
INFO("Voted proposal");
Script s("return true");
nlohmann::json p("hello world");
auto m = MemberId("member");
Proposal proposal(s, p, m);
proposal.votes[MemberId("member1")] = Script("return true");
proposal.votes[MemberId("member2")] = Script("return false");
proposal.votes[MemberId("member3")] = Script("return RoN");
proposal.votes[MemberId("member4")] =
Script("Robert'); DROP TABLE Students;--");
const auto converted = msgpack_roundtrip(proposal);
CHECK(proposal == converted);
}
}
void fill_rand(std::vector<uint8_t>& v, size_t n)
{
v.resize(n);
for (size_t i = 0; i < n; ++i)
{
v[i] = rand();
}
}
TEST_CASE("NodeSignature")
{
using namespace ccf;
{
INFO("Empty signature");
NodeSignature rs;
const auto converted = msgpack_roundtrip(rs);
CHECK(rs == converted);
}
{
INFO("Byte signature");
NodeSignature rs;
rs.sig.push_back(42);
const auto converted = msgpack_roundtrip(rs);
CHECK(rs == converted);
}
{
INFO("Large signature");
NodeSignature rs;
fill_rand(rs.sig, 256);
const auto converted = msgpack_roundtrip(rs);
CHECK(rs == converted);
}
}
TEST_CASE("MemberAck")
{
using namespace ccf;
{
INFO("Empty ack");
MemberAck ma;
const auto converted = msgpack_roundtrip(ma);
CHECK(ma.state_digest == converted.state_digest);
}
{
INFO("Implausible ack");
MemberAck ma;
ma.state_digest = "string";
const auto converted = msgpack_roundtrip(ma);
CHECK(ma.state_digest == converted.state_digest);
}
{
INFO("Plausible ack");
MemberAck ma;
std::vector<uint8_t> data;
fill_rand(data, 1024);
ma.state_digest = crypto::Sha256Hash(data).hex_str();
const auto converted = msgpack_roundtrip(ma);
CHECK(ma.state_digest == converted.state_digest);
}
}
TEST_CASE("Signature")
{
using namespace ccf;
{
INFO("Empty sig");
PrimarySignature sig;
const auto converted = msgpack_roundtrip(sig);
CHECK(sig == converted);
}
{
INFO("Simple sig");
PrimarySignature sig;
sig.sig.push_back(0);
sig.node = NodeId("node");
sig.seqno = 1;
sig.view = 2;
sig.commit_seqno = 3;
const auto converted = msgpack_roundtrip(sig);
CHECK(sig == converted);
}
{
INFO("Rand sig");
PrimarySignature sig;
fill_rand(sig.sig, 256);
sig.node = NodeId("node");
sig.seqno = rand();
sig.view = rand();
sig.commit_seqno = rand();
const auto converted = msgpack_roundtrip(sig);
CHECK(sig == converted);
}
}
}

4
src/node/view_change.h
Просмотреть файл

@ -5,7 +5,6 @@
#include "node_signature.h"
#include "service_map.h"
#include <msgpack/msgpack.hpp>
#include <string>
#include <vector>
@ -60,7 +59,6 @@ namespace ccf
return v;
}
MSGPACK_DEFINE(signatures, signature);
};
DECLARE_JSON_TYPE(ViewChangeRequest);
DECLARE_JSON_REQUIRED_FIELDS(ViewChangeRequest, signatures, signature);
@ -79,8 +77,6 @@ namespace ccf
view(view_),
seqno(seqno_)
{}
MSGPACK_DEFINE(view, seqno, signature, view_change_messages);
};
DECLARE_JSON_TYPE(ViewChangeConfirmation);
DECLARE_JSON_REQUIRED_FIELDS(