From 3b0492cfd0f6f0ee6f7885736e378c0e7b28d0b8 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Thu, 4 Jun 2020 13:34:27 +0100 Subject: [PATCH] Use early-serialising KV implementation by default (take 2) (#1234) --- 3rdparty/small_vector/LICENSE.TXT | 43 ++ 3rdparty/small_vector/README.md | 3 + 3rdparty/small_vector/SmallVector.h | 983 ++++++++++++++++++++++++++++ CMakeLists.txt | 3 + cgmanifest.json | 11 +- src/apps/js_generic/js_generic.cpp | 9 +- src/ds/champ_map.h | 10 +- src/ds/hash.h | 66 +- src/ds/siphash.h | 11 +- src/ds/test/hash_bench.cpp | 37 ++ src/kv/change_set.h | 61 ++ src/kv/experimental.h | 321 --------- src/kv/generic_serialise_wrapper.h | 98 +-- src/kv/kv_types.h | 5 + src/kv/map.h | 799 ++++------------------ src/kv/msgpack_serialise.h | 35 + src/kv/serialise_entry_blit.h | 73 +++ src/kv/serialise_entry_json.h | 27 + src/kv/serialise_entry_msgpack.h | 43 ++ src/kv/serialised_entry.h | 12 + src/kv/store.h | 14 +- src/kv/test/kv_bench.cpp | 66 +- src/kv/test/kv_serialisation.cpp | 312 ++------- src/kv/test/kv_test.cpp | 143 ++-- src/kv/tx_view.h | 260 ++------ src/kv/untyped_map.h | 723 ++++++++++++++++++++ src/kv/untyped_tx_view.h | 216 ++++++ src/node/scripts.h | 7 +- src/node/shares.h | 10 + tests/infra/ledger.py | 4 +- 30 files changed, 2732 insertions(+), 1673 deletions(-) create mode 100644 3rdparty/small_vector/LICENSE.TXT create mode 100644 3rdparty/small_vector/README.md create mode 100644 3rdparty/small_vector/SmallVector.h create mode 100644 src/ds/test/hash_bench.cpp create mode 100644 src/kv/change_set.h delete mode 100644 src/kv/experimental.h create mode 100644 src/kv/serialise_entry_blit.h create mode 100644 src/kv/serialise_entry_json.h create mode 100644 src/kv/serialise_entry_msgpack.h create mode 100644 src/kv/serialised_entry.h create mode 100644 src/kv/untyped_map.h create mode 100644 src/kv/untyped_tx_view.h diff --git a/3rdparty/small_vector/LICENSE.TXT b/3rdparty/small_vector/LICENSE.TXT new file mode 100644 index 0000000000..896d8f3e0c --- /dev/null +++ b/3rdparty/small_vector/LICENSE.TXT @@ -0,0 +1,43 @@ +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2016 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. diff --git a/3rdparty/small_vector/README.md b/3rdparty/small_vector/README.md new file mode 100644 index 0000000000..5a36d78c53 --- /dev/null +++ b/3rdparty/small_vector/README.md @@ -0,0 +1,3 @@ +# SmallVector + +This is [llvm::SmallVector](http://llvm.org/docs/doxygen/html/classllvm_1_1SmallVector.html) stripped from any LLVM dependency. diff --git a/3rdparty/small_vector/SmallVector.h b/3rdparty/small_vector/SmallVector.h new file mode 100644 index 0000000000..ad2b676ff9 --- /dev/null +++ b/3rdparty/small_vector/SmallVector.h @@ -0,0 +1,983 @@ +//===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the SmallVector class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_VECSMALL_ADT_SMALLVECTOR_H +#define LLVM_VECSMALL_ADT_SMALLVECTOR_H + +#include +#include +#include +#include +#include +#include +#include +#include + +// LLVM Macros +#define LLVM_VECSMALL_NODISCARD +#define LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE inline +#define LLVM_VECSMALL_UNLIKELY(x) (x) + +// LLVM External Functions +namespace llvm_vecsmall { + namespace detail { + /// NextPowerOf2 - Returns the next power of two (in 64-bits) + /// that is strictly greater than A. Returns zero on overflow. + inline uint64_t NextPowerOf2(uint64_t A) { + A |= (A >> 1); + A |= (A >> 2); + A |= (A >> 4); + A |= (A >> 8); + A |= (A >> 16); + A |= (A >> 32); + return A + 1; + } + } +} + + +namespace llvm_vecsmall { + +// std::is_pod has been deprecated in C++20. +template +struct IsPod : std::integral_constant::value && + std::is_trivial::value> {}; + +/// This is all the non-templated stuff common to all SmallVectors. +class SmallVectorBase { +protected: + void *BeginX, *EndX, *CapacityX; + +protected: + SmallVectorBase(void *FirstEl, size_t Size) + : BeginX(FirstEl), EndX(FirstEl), CapacityX((char*)FirstEl+Size) {} + + /// This is an implementation of the grow() method which only works + /// on POD-like data types and is out of line to reduce code duplication. + void grow_pod(void *FirstEl, size_t MinSizeInBytes, size_t TSize); + +public: + /// This returns size()*sizeof(T). + size_t size_in_bytes() const { + return size_t((char*)EndX - (char*)BeginX); + } + + /// capacity_in_bytes - This returns capacity()*sizeof(T). + size_t capacity_in_bytes() const { + return size_t((char*)CapacityX - (char*)BeginX); + } + + LLVM_VECSMALL_NODISCARD bool empty() const { return BeginX == EndX; } +}; + +template struct SmallVectorStorage; + +/// This is the part of SmallVectorTemplateBase which does not depend on whether +/// the type T is a POD. The extra dummy template argument is used by ArrayRef +/// to avoid unnecessarily requiring T to be complete. +template +class SmallVectorTemplateCommon : public SmallVectorBase { +private: + template friend struct SmallVectorStorage; + + // Allocate raw space for N elements of type T. If T has a ctor or dtor, we + // don't want it to be automatically run, so we need to represent the space as + // something else. Use an array of char of sufficient alignment. + ////////////typedef llvm_vecsmall::AlignedCharArrayUnion U; + typedef typename std::aligned_union<1, T>::type U; + U FirstEl; + // Space after 'FirstEl' is clobbered, do not add any instance vars after it. + +protected: + SmallVectorTemplateCommon(size_t Size) : SmallVectorBase(&FirstEl, Size) {} + + void grow_pod(size_t MinSizeInBytes, size_t TSize) { + SmallVectorBase::grow_pod(&FirstEl, MinSizeInBytes, TSize); + } + + /// Return true if this is a smallvector which has not had dynamic + /// memory allocated for it. + bool isSmall() const { + return BeginX == static_cast(&FirstEl); + } + + /// Put this vector in a state of being small. + void resetToSmall() { + BeginX = EndX = CapacityX = &FirstEl; + } + + void setEnd(T *P) { this->EndX = P; } +public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T value_type; + typedef T *iterator; + typedef const T *const_iterator; + + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + typedef T &reference; + typedef const T &const_reference; + typedef T *pointer; + typedef const T *const_pointer; + + // forward iterator creation methods. + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + iterator begin() { return (iterator)this->BeginX; } + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + const_iterator begin() const { return (const_iterator)this->BeginX; } + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + iterator end() { return (iterator)this->EndX; } + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + const_iterator end() const { return (const_iterator)this->EndX; } +protected: + iterator capacity_ptr() { return (iterator)this->CapacityX; } + const_iterator capacity_ptr() const { return (const_iterator)this->CapacityX;} +public: + + // reverse iterator creation methods. + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const{ return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin());} + + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + size_type size() const { return end()-begin(); } + size_type max_size() const { return size_type(-1) / sizeof(T); } + + /// Return the total number of elements in the currently allocated buffer. + size_t capacity() const { return capacity_ptr() - begin(); } + + /// Return a pointer to the vector's buffer, even if empty(). + pointer data() { return pointer(begin()); } + /// Return a pointer to the vector's buffer, even if empty(). + const_pointer data() const { return const_pointer(begin()); } + + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + reference operator[](size_type idx) { + assert(idx < size()); + return begin()[idx]; + } + LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE + const_reference operator[](size_type idx) const { + assert(idx < size()); + return begin()[idx]; + } + + reference front() { + assert(!empty()); + return begin()[0]; + } + const_reference front() const { + assert(!empty()); + return begin()[0]; + } + + reference back() { + assert(!empty()); + return end()[-1]; + } + const_reference back() const { + assert(!empty()); + return end()[-1]; + } +}; + +/// SmallVectorTemplateBase - This is where we put method +/// implementations that are designed to work with non-POD-like T's. +template +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { +protected: + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + static void destroy_range(T *S, T *E) { + while (S != E) { + --E; + E->~T(); + } + } + + /// Move the range [I, E) into the uninitialized memory starting with "Dest", + /// constructing elements as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(std::make_move_iterator(I), + std::make_move_iterator(E), Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory starting with "Dest", + /// constructing elements as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + std::uninitialized_copy(I, E, Dest); + } + + /// Grow the allocated memory (without initializing new elements), doubling + /// the size of the allocated memory. Guarantees space for at least one more + /// element, or MinSize more elements if specified. + void grow(size_t MinSize = 0); + +public: + void push_back(const T &Elt) { + if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + ::new ((void*) this->end()) T(Elt); + this->setEnd(this->end()+1); + } + + void push_back(T &&Elt) { + if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + ::new ((void*) this->end()) T(::std::move(Elt)); + this->setEnd(this->end()+1); + } + + void pop_back() { + this->setEnd(this->end()-1); + this->end()->~T(); + } +}; + +// Define this out-of-line to dissuade the C++ compiler from inlining it. +template +void SmallVectorTemplateBase::grow(size_t MinSize) { + size_t CurCapacity = this->capacity(); + size_t CurSize = this->size(); + // Always grow, even from zero. + size_t NewCapacity = size_t(llvm_vecsmall::detail::NextPowerOf2(CurCapacity+2)); + if (NewCapacity < MinSize) + NewCapacity = MinSize; + T *NewElts = static_cast(malloc(NewCapacity*sizeof(T))); + + // Move the elements over. + this->uninitialized_move(this->begin(), this->end(), NewElts); + + // Destroy the original elements. + destroy_range(this->begin(), this->end()); + + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + free(this->begin()); + + this->setEnd(NewElts+CurSize); + this->BeginX = NewElts; + this->CapacityX = this->begin()+NewCapacity; +} + + +/// SmallVectorTemplateBase - This is where we put method +/// implementations that are designed to work with POD-like T's. +template +class SmallVectorTemplateBase : public SmallVectorTemplateCommon { +protected: + SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} + + // No need to do a destroy loop for POD's. + static void destroy_range(T *, T *) {} + + /// Move the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_move(It1 I, It1 E, It2 Dest) { + // Just do a copy. + uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy(It1 I, It1 E, It2 Dest) { + // Arbitrary iterator types; just use the basic implementation. + std::uninitialized_copy(I, E, Dest); + } + + /// Copy the range [I, E) onto the uninitialized memory + /// starting with "Dest", constructing elements into it as needed. + template + static void uninitialized_copy( + T1 *I, T1 *E, T2 *Dest, + typename std::enable_if::type, + T2>::value>::type * = nullptr) { + // Use memcpy for PODs iterated by pointers (which includes SmallVector + // iterators): std::uninitialized_copy optimizes to memmove, but we can + // use memcpy here. Note that I and E are iterators and thus might be + // invalid for memcpy if they are equal. + if (I != E) + memcpy(Dest, I, (E - I) * sizeof(T)); + } + + /// Double the size of the allocated memory, guaranteeing space for at + /// least one more element or MinSize if specified. + void grow(size_t MinSize = 0) { + this->grow_pod(MinSize*sizeof(T), sizeof(T)); + } +public: + void push_back(const T &Elt) { + if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + memcpy(this->end(), &Elt, sizeof(T)); + this->setEnd(this->end()+1); + } + + void pop_back() { + this->setEnd(this->end()-1); + } +}; + + +/// This class consists of common code factored out of the SmallVector class to +/// reduce code duplication based on the SmallVector 'N' template parameter. +template +class SmallVectorImpl : public SmallVectorTemplateBase::value> { + typedef SmallVectorTemplateBase::value> SuperClass; + + SmallVectorImpl(const SmallVectorImpl&) = delete; + +public: + typedef typename SuperClass::iterator iterator; + typedef typename SuperClass::const_iterator const_iterator; + typedef typename SuperClass::size_type size_type; + +protected: + // Default ctor - Initialize to empty. + explicit SmallVectorImpl(unsigned N) + : SmallVectorTemplateBase::value>(N*sizeof(T)) { + } + +public: + ~SmallVectorImpl() { + // Destroy the constructed elements in the vector. + this->destroy_range(this->begin(), this->end()); + + // If this wasn't grown from the inline copy, deallocate the old space. + if (!this->isSmall()) + free(this->begin()); + } + + + void clear() { + this->destroy_range(this->begin(), this->end()); + this->EndX = this->BeginX; + } + + void resize(size_type N) { + if (N < this->size()) { + this->destroy_range(this->begin()+N, this->end()); + this->setEnd(this->begin()+N); + } else if (N > this->size()) { + if (this->capacity() < N) + this->grow(N); + for (auto I = this->end(), E = this->begin() + N; I != E; ++I) + new (&*I) T(); + this->setEnd(this->begin()+N); + } + } + + void resize(size_type N, const T &NV) { + if (N < this->size()) { + this->destroy_range(this->begin()+N, this->end()); + this->setEnd(this->begin()+N); + } else if (N > this->size()) { + if (this->capacity() < N) + this->grow(N); + std::uninitialized_fill(this->end(), this->begin()+N, NV); + this->setEnd(this->begin()+N); + } + } + + void reserve(size_type N) { + if (this->capacity() < N) + this->grow(N); + } + + LLVM_VECSMALL_NODISCARD T pop_back_val() { + T Result = ::std::move(this->back()); + this->pop_back(); + return Result; + } + + void swap(SmallVectorImpl &RHS); + + /// Add the specified range to the end of the SmallVector. + template + void append(in_iter in_start, in_iter in_end) { + size_type NumInputs = std::distance(in_start, in_end); + // Grow allocated space if needed. + if (NumInputs > size_type(this->capacity_ptr()-this->end())) + this->grow(this->size()+NumInputs); + + // Copy the new elements over. + this->uninitialized_copy(in_start, in_end, this->end()); + this->setEnd(this->end() + NumInputs); + } + + /// Add the specified range to the end of the SmallVector. + void append(size_type NumInputs, const T &Elt) { + // Grow allocated space if needed. + if (NumInputs > size_type(this->capacity_ptr()-this->end())) + this->grow(this->size()+NumInputs); + + // Copy the new elements over. + std::uninitialized_fill_n(this->end(), NumInputs, Elt); + this->setEnd(this->end() + NumInputs); + } + + void append(std::initializer_list IL) { + append(IL.begin(), IL.end()); + } + + void assign(size_type NumElts, const T &Elt) { + clear(); + if (this->capacity() < NumElts) + this->grow(NumElts); + this->setEnd(this->begin()+NumElts); + std::uninitialized_fill(this->begin(), this->end(), Elt); + } + + void assign(std::initializer_list IL) { + clear(); + append(IL); + } + + iterator erase(const_iterator CI) { + // Just cast away constness because this is a non-const member function. + iterator I = const_cast(CI); + + assert(I >= this->begin() && "Iterator to erase is out of bounds."); + assert(I < this->end() && "Erasing at past-the-end iterator."); + + iterator N = I; + // Shift all elts down one. + std::move(I+1, this->end(), I); + // Drop the last elt. + this->pop_back(); + return(N); + } + + iterator erase(const_iterator CS, const_iterator CE) { + // Just cast away constness because this is a non-const member function. + iterator S = const_cast(CS); + iterator E = const_cast(CE); + + assert(S >= this->begin() && "Range to erase is out of bounds."); + assert(S <= E && "Trying to erase invalid range."); + assert(E <= this->end() && "Trying to erase past the end."); + + iterator N = S; + // Shift all elts down. + iterator I = std::move(E, this->end(), S); + // Drop the last elts. + this->destroy_range(I, this->end()); + this->setEnd(I); + return(N); + } + + iterator insert(iterator I, T &&Elt) { + if (I == this->end()) { // Important special case for empty vector. + this->push_back(::std::move(Elt)); + return this->end()-1; + } + + assert(I >= this->begin() && "Insertion iterator is out of bounds."); + assert(I <= this->end() && "Inserting past the end of the vector."); + + if (this->EndX >= this->CapacityX) { + size_t EltNo = I-this->begin(); + this->grow(); + I = this->begin()+EltNo; + } + + ::new ((void*) this->end()) T(::std::move(this->back())); + // Push everything else over. + std::move_backward(I, this->end()-1, this->end()); + this->setEnd(this->end()+1); + + // If we just moved the element we're inserting, be sure to update + // the reference. + T *EltPtr = &Elt; + if (I <= EltPtr && EltPtr < this->EndX) + ++EltPtr; + + *I = ::std::move(*EltPtr); + return I; + } + + iterator insert(iterator I, const T &Elt) { + if (I == this->end()) { // Important special case for empty vector. + this->push_back(Elt); + return this->end()-1; + } + + assert(I >= this->begin() && "Insertion iterator is out of bounds."); + assert(I <= this->end() && "Inserting past the end of the vector."); + + if (this->EndX >= this->CapacityX) { + size_t EltNo = I-this->begin(); + this->grow(); + I = this->begin()+EltNo; + } + ::new ((void*) this->end()) T(std::move(this->back())); + // Push everything else over. + std::move_backward(I, this->end()-1, this->end()); + this->setEnd(this->end()+1); + + // If we just moved the element we're inserting, be sure to update + // the reference. + const T *EltPtr = &Elt; + if (I <= EltPtr && EltPtr < this->EndX) + ++EltPtr; + + *I = *EltPtr; + return I; + } + + iterator insert(iterator I, size_type NumToInsert, const T &Elt) { + // Convert iterator to elt# to avoid invalidating iterator when we reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(NumToInsert, Elt); + return this->begin()+InsertElt; + } + + assert(I >= this->begin() && "Insertion iterator is out of bounds."); + assert(I <= this->end() && "Inserting past the end of the vector."); + + // Ensure there is enough space. + reserve(this->size() + NumToInsert); + + // Uninvalidate the iterator. + I = this->begin()+InsertElt; + + // If there are more elements between the insertion point and the end of the + // range than there are being inserted, we can use a simple approach to + // insertion. Since we already reserved space, we know that this won't + // reallocate the vector. + if (size_t(this->end()-I) >= NumToInsert) { + T *OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd-NumToInsert, OldEnd); + + std::fill_n(I, NumToInsert, Elt); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and we're + // not inserting at the end. + + // Move over the elements that we're about to overwrite. + T *OldEnd = this->end(); + this->setEnd(this->end() + NumToInsert); + size_t NumOverwritten = OldEnd-I; + this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); + + // Replace the overwritten part. + std::fill_n(I, NumOverwritten, Elt); + + // Insert the non-overwritten middle part. + std::uninitialized_fill_n(OldEnd, NumToInsert-NumOverwritten, Elt); + return I; + } + + template + iterator insert(iterator I, ItTy From, ItTy To) { + // Convert iterator to elt# to avoid invalidating iterator when we reserve() + size_t InsertElt = I - this->begin(); + + if (I == this->end()) { // Important special case for empty vector. + append(From, To); + return this->begin()+InsertElt; + } + + assert(I >= this->begin() && "Insertion iterator is out of bounds."); + assert(I <= this->end() && "Inserting past the end of the vector."); + + size_t NumToInsert = std::distance(From, To); + + // Ensure there is enough space. + reserve(this->size() + NumToInsert); + + // Uninvalidate the iterator. + I = this->begin()+InsertElt; + + // If there are more elements between the insertion point and the end of the + // range than there are being inserted, we can use a simple approach to + // insertion. Since we already reserved space, we know that this won't + // reallocate the vector. + if (size_t(this->end()-I) >= NumToInsert) { + T *OldEnd = this->end(); + append(std::move_iterator(this->end() - NumToInsert), + std::move_iterator(this->end())); + + // Copy the existing elements that get replaced. + std::move_backward(I, OldEnd-NumToInsert, OldEnd); + + std::copy(From, To, I); + return I; + } + + // Otherwise, we're inserting more elements than exist already, and we're + // not inserting at the end. + + // Move over the elements that we're about to overwrite. + T *OldEnd = this->end(); + this->setEnd(this->end() + NumToInsert); + size_t NumOverwritten = OldEnd-I; + this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten); + + // Replace the overwritten part. + for (T *J = I; NumOverwritten > 0; --NumOverwritten) { + *J = *From; + ++J; ++From; + } + + // Insert the non-overwritten middle part. + this->uninitialized_copy(From, To, OldEnd); + return I; + } + + void insert(iterator I, std::initializer_list IL) { + insert(I, IL.begin(), IL.end()); + } + + template void emplace_back(ArgTypes &&... Args) { + if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX)) + this->grow(); + ::new ((void *)this->end()) T(std::forward(Args)...); + this->setEnd(this->end() + 1); + } + + SmallVectorImpl &operator=(const SmallVectorImpl &RHS); + + SmallVectorImpl &operator=(SmallVectorImpl &&RHS); + + bool operator==(const SmallVectorImpl &RHS) const { + if (this->size() != RHS.size()) return false; + return std::equal(this->begin(), this->end(), RHS.begin()); + } + bool operator!=(const SmallVectorImpl &RHS) const { + return !(*this == RHS); + } + + bool operator<(const SmallVectorImpl &RHS) const { + return std::lexicographical_compare(this->begin(), this->end(), + RHS.begin(), RHS.end()); + } + + /// Set the array size to \p N, which the current array must have enough + /// capacity for. + /// + /// This does not construct or destroy any elements in the vector. + /// + /// Clients can use this in conjunction with capacity() to write past the end + /// of the buffer when they know that more elements are available, and only + /// update the size later. This avoids the cost of value initializing elements + /// which will only be overwritten. + void set_size(size_type N) { + assert(N <= this->capacity()); + this->setEnd(this->begin() + N); + } +}; + + +template +void SmallVectorImpl::swap(SmallVectorImpl &RHS) { + if (this == &RHS) return; + + // We can only avoid copying elements if neither vector is small. + if (!this->isSmall() && !RHS.isSmall()) { + std::swap(this->BeginX, RHS.BeginX); + std::swap(this->EndX, RHS.EndX); + std::swap(this->CapacityX, RHS.CapacityX); + return; + } + if (RHS.size() > this->capacity()) + this->grow(RHS.size()); + if (this->size() > RHS.capacity()) + RHS.grow(this->size()); + + // Swap the shared elements. + size_t NumShared = this->size(); + if (NumShared > RHS.size()) NumShared = RHS.size(); + for (size_type i = 0; i != NumShared; ++i) + std::swap((*this)[i], RHS[i]); + + // Copy over the extra elts. + if (this->size() > RHS.size()) { + size_t EltDiff = this->size() - RHS.size(); + this->uninitialized_copy(this->begin()+NumShared, this->end(), RHS.end()); + RHS.setEnd(RHS.end()+EltDiff); + this->destroy_range(this->begin()+NumShared, this->end()); + this->setEnd(this->begin()+NumShared); + } else if (RHS.size() > this->size()) { + size_t EltDiff = RHS.size() - this->size(); + this->uninitialized_copy(RHS.begin()+NumShared, RHS.end(), this->end()); + this->setEnd(this->end() + EltDiff); + this->destroy_range(RHS.begin()+NumShared, RHS.end()); + RHS.setEnd(RHS.begin()+NumShared); + } +} + +template +SmallVectorImpl &SmallVectorImpl:: + operator=(const SmallVectorImpl &RHS) { + // Avoid self-assignment. + if (this == &RHS) return *this; + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd; + if (RHSSize) + NewEnd = std::copy(RHS.begin(), RHS.begin()+RHSSize, this->begin()); + else + NewEnd = this->begin(); + + // Destroy excess elements. + this->destroy_range(NewEnd, this->end()); + + // Trim. + this->setEnd(NewEnd); + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: don't do this if they're efficiently moveable. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->destroy_range(this->begin(), this->end()); + this->setEnd(this->begin()); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::copy(RHS.begin(), RHS.begin()+CurSize, this->begin()); + } + + // Copy construct the new elements in place. + this->uninitialized_copy(RHS.begin()+CurSize, RHS.end(), + this->begin()+CurSize); + + // Set end. + this->setEnd(this->begin()+RHSSize); + return *this; +} + +template +SmallVectorImpl &SmallVectorImpl::operator=(SmallVectorImpl &&RHS) { + // Avoid self-assignment. + if (this == &RHS) return *this; + + // If the RHS isn't small, clear this vector and then steal its buffer. + if (!RHS.isSmall()) { + this->destroy_range(this->begin(), this->end()); + if (!this->isSmall()) free(this->begin()); + this->BeginX = RHS.BeginX; + this->EndX = RHS.EndX; + this->CapacityX = RHS.CapacityX; + RHS.resetToSmall(); + return *this; + } + + // If we already have sufficient space, assign the common elements, then + // destroy any excess. + size_t RHSSize = RHS.size(); + size_t CurSize = this->size(); + if (CurSize >= RHSSize) { + // Assign common elements. + iterator NewEnd = this->begin(); + if (RHSSize) + NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd); + + // Destroy excess elements and trim the bounds. + this->destroy_range(NewEnd, this->end()); + this->setEnd(NewEnd); + + // Clear the RHS. + RHS.clear(); + + return *this; + } + + // If we have to grow to have enough elements, destroy the current elements. + // This allows us to avoid copying them during the grow. + // FIXME: this may not actually make any sense if we can efficiently move + // elements. + if (this->capacity() < RHSSize) { + // Destroy current elements. + this->destroy_range(this->begin(), this->end()); + this->setEnd(this->begin()); + CurSize = 0; + this->grow(RHSSize); + } else if (CurSize) { + // Otherwise, use assignment for the already-constructed elements. + std::move(RHS.begin(), RHS.begin()+CurSize, this->begin()); + } + + // Move-construct the new elements in place. + this->uninitialized_move(RHS.begin()+CurSize, RHS.end(), + this->begin()+CurSize); + + // Set end. + this->setEnd(this->begin()+RHSSize); + + RHS.clear(); + return *this; +} + +/// Storage for the SmallVector elements which aren't contained in +/// SmallVectorTemplateCommon. There are 'N-1' elements here. The remaining '1' +/// element is in the base class. This is specialized for the N=1 and N=0 cases +/// to avoid allocating unnecessary storage. +template +struct SmallVectorStorage { + typename SmallVectorTemplateCommon::U InlineElts[N - 1]; +}; +template struct SmallVectorStorage {}; +template struct SmallVectorStorage {}; + +/// This is a 'vector' (really, a variable-sized array), optimized +/// for the case when the array is small. It contains some number of elements +/// in-place, which allows it to avoid heap allocation when the actual number of +/// elements is below that threshold. This allows normal "small" cases to be +/// fast without losing generality for large inputs. +/// +/// Note that this does not attempt to be exception safe. +/// +template +class SmallVector : public SmallVectorImpl { + /// Inline space for elements which aren't stored in the base class. + SmallVectorStorage Storage; +public: + SmallVector() : SmallVectorImpl(N) { + } + + explicit SmallVector(size_t Size, const T &Value = T()) + : SmallVectorImpl(N) { + this->assign(Size, Value); + } + + template + SmallVector(ItTy S, ItTy E) : SmallVectorImpl(N) { + this->append(S, E); + } + +/* + template + explicit SmallVector(const llvm_vecsmall::iterator_range &R) + : SmallVectorImpl(N) { + this->append(R.begin(), R.end()); + } +*/ + + SmallVector(std::initializer_list IL) : SmallVectorImpl(N) { + this->assign(IL); + } + + SmallVector(const SmallVector &RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(RHS); + } + + const SmallVector &operator=(const SmallVector &RHS) { + SmallVectorImpl::operator=(RHS); + return *this; + } + + SmallVector(SmallVector &&RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + const SmallVector &operator=(SmallVector &&RHS) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + + SmallVector(SmallVectorImpl &&RHS) : SmallVectorImpl(N) { + if (!RHS.empty()) + SmallVectorImpl::operator=(::std::move(RHS)); + } + + const SmallVector &operator=(SmallVectorImpl &&RHS) { + SmallVectorImpl::operator=(::std::move(RHS)); + return *this; + } + + const SmallVector &operator=(std::initializer_list IL) { + this->assign(IL); + return *this; + } +}; + +template +static inline size_t capacity_in_bytes(const SmallVector &X) { + return X.capacity_in_bytes(); +} + +} // End llvm_vecsmall namespace + +namespace std { + /// Implement std::swap in terms of SmallVector swap. + template + inline void + swap(llvm_vecsmall::SmallVectorImpl &LHS, llvm_vecsmall::SmallVectorImpl &RHS) { + LHS.swap(RHS); + } + + /// Implement std::swap in terms of SmallVector swap. + template + inline void + swap(llvm_vecsmall::SmallVector &LHS, llvm_vecsmall::SmallVector &RHS) { + LHS.swap(RHS); + } +} + +namespace llvm_vecsmall { +/// grow_pod - This is an implementation of the grow() method which only works +/// on POD-like datatypes and is out of line to reduce code duplication. +inline +void SmallVectorBase::grow_pod(void *FirstEl, size_t MinSizeInBytes, + size_t TSize) { + size_t CurSizeBytes = size_in_bytes(); + size_t NewCapacityInBytes = 2 * capacity_in_bytes() + TSize; // Always grow. + if (NewCapacityInBytes < MinSizeInBytes) + NewCapacityInBytes = MinSizeInBytes; + + void *NewElts; + if (BeginX == FirstEl) { + NewElts = malloc(NewCapacityInBytes); + + // Copy the elements over. No need to run dtors on PODs. + memcpy(NewElts, this->BeginX, CurSizeBytes); + } else { + // If this wasn't grown from the inline copy, grow the allocated space. + NewElts = realloc(this->BeginX, NewCapacityInBytes); + } + assert(NewElts && "Out of memory"); + + this->EndX = (char*)NewElts+CurSizeBytes; + this->BeginX = NewElts; + this->CapacityX = (char*)this->BeginX + NewCapacityInBytes; +} +} + +#endif diff --git a/CMakeLists.txt b/CMakeLists.txt index 35bb21656a..d6c28338d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -446,6 +446,9 @@ if(BUILD_TESTS) kv_bench SRCS src/kv/test/kv_bench.cpp src/crypto/symmetric_key.cpp src/enclave/thread_local.cpp ) + add_picobench( + hash_bench SRCS src/ds/test/hash_bench.cpp + ) # Storing signed governance operations add_e2e_test( diff --git a/cgmanifest.json b/cgmanifest.json index fdc58c3405..3428e1a117 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -145,7 +145,16 @@ "commitHash": "dc8c3a9a1089e962b32ecdcc940ae11bd2b69e4b" } } - }, + },, + { + "component": { + "type": "git", + "git": { + "repositoryUrl": "https://github.com/thelink2012/SmallVector", + "commitHash": "febc8cb7b1a83d902b86dd1612feb7c86c690186" + } + } + } ], "Version": 1 } \ No newline at end of file diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 51148a5979..ac3b846237 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the Apache 2.0 License. #include "enclave/app_interface.h" +#include "kv/untyped_map.h" #include "node/rpc/user_frontend.h" #include "quickjs.h" @@ -77,7 +78,7 @@ namespace ccfapp size_t sz = 0; auto k = JS_ToCStringLen(ctx, &sz, argv[0]); - auto v = table_view->get(std::vector(k, k + sz)); + auto v = table_view->get({k, k + sz}); JS_FreeCString(ctx, k); if (v.has_value()) @@ -100,7 +101,7 @@ namespace ccfapp size_t sz = 0; auto k = JS_ToCStringLen(ctx, &sz, argv[0]); - auto v = table_view->remove(std::vector(k, k + sz)); + auto v = table_view->remove({k, k + sz}); JS_FreeCString(ctx, k); if (v) @@ -124,13 +125,11 @@ namespace ccfapp size_t k_sz = 0; auto k = JS_ToCStringLen(ctx, &k_sz, argv[0]); - std::vector k_(k, k + k_sz); size_t v_sz = 0; auto v = JS_ToCStringLen(ctx, &v_sz, argv[1]); - std::vector v_(v, v + v_sz); - if (!table_view->put(k_, v_)) + if (!table_view->put({k, k + k_sz}, {v, v + v_sz})) { r = JS_ThrowRangeError(ctx, "Could not insert at key"); } diff --git a/src/ds/champ_map.h b/src/ds/champ_map.h index 0dc62fc028..77d319ced2 100644 --- a/src/ds/champ_map.h +++ b/src/ds/champ_map.h @@ -148,10 +148,10 @@ namespace champ SubNodes() {} - SubNodes(std::vector> ns) : nodes(ns) {} + SubNodes(std::vector>&& ns) : nodes(std::move(ns)) {} - SubNodes(std::vector> ns, Bitmap nm, Bitmap dm) : - nodes(ns), + SubNodes(std::vector>&& ns, Bitmap nm, Bitmap dm) : + nodes(std::move(ns)), node_map(nm), data_map(dm) {} @@ -311,8 +311,8 @@ namespace champ std::shared_ptr> root; size_t _size = 0; - Map(std::shared_ptr> root_, size_t size_) : - root(root_), + Map(std::shared_ptr>&& root_, size_t size_) : + root(std::move(root_)), _size(size_) {} diff --git a/src/ds/hash.h b/src/ds/hash.h index deb86926ed..1f4bfb76b1 100644 --- a/src/ds/hash.h +++ b/src/ds/hash.h @@ -6,8 +6,32 @@ #include #include +#include #include +namespace ds::hashutils +{ + template + inline void hash_combine(size_t& n, const T& v, std::hash& h) + { + n ^= h(v) + (n << 6) + (n >> 2); + } + + template + inline size_t hash_container(const T& v) + { + size_t n = 0x444e414c544f4353; + std::hash h{}; + + for (const auto& e : v) + { + hash_combine(n, e, h); + } + + return n; + } +} + namespace std { template <> @@ -23,35 +47,12 @@ namespace std } }; - namespace - { - template - inline void hash_combine(size_t& n, const T& v, std::hash& h) - { - n ^= h(v) + (n << 6) + (n >> 2); - } - - template - inline size_t hash_container(const T& v) - { - size_t n = 0x444e414c544f4353; - std::hash h{}; - - for (const auto& e : v) - { - hash_combine(n, e, h); - } - - return n; - } - } - template struct hash> { size_t operator()(const std::vector& v) const { - return hash_container(v); + return ds::hashutils::hash_container(v); } }; @@ -60,7 +61,7 @@ namespace std { size_t operator()(const std::array& v) const { - return hash_container(v); + return ds::hashutils::hash_container(v); } }; @@ -72,14 +73,25 @@ namespace std size_t n = 0x444e414c544f4353; std::hash h_a{}; - hash_combine(n, v.first, h_a); + ds::hashutils::hash_combine(n, v.first, h_a); std::hash h_b{}; - hash_combine(n, v.second, h_b); + ds::hashutils::hash_combine(n, v.second, h_b); return n; } }; + + template + struct hash> + { + size_t operator()(const llvm_vecsmall::SmallVector& v) const + { + static constexpr siphash::SipKey k{0x7720796f726c694b, + 0x2165726568207361}; + return siphash::siphash<2, 4>(v.data(), v.size(), k); + } + }; } namespace ds diff --git a/src/ds/siphash.h b/src/ds/siphash.h index 856436033a..22dd264c43 100644 --- a/src/ds/siphash.h +++ b/src/ds/siphash.h @@ -164,7 +164,7 @@ namespace siphash } template - uint64_t siphash(const std::vector& in, const SipKey& key) + uint64_t siphash(const uint8_t* data, size_t size, const SipKey& key) { uint64_t out; @@ -172,8 +172,15 @@ namespace siphash CompressionRounds, FinalizationRounds, OutputLength::EightBytes>( - in.data(), in.size(), key, reinterpret_cast(&out)); + data, size, key, reinterpret_cast(&out)); return out; } + + template + uint64_t siphash(const std::vector& in, const SipKey& key) + { + return siphash( + in.data(), in.size(), key); + } } diff --git a/src/ds/test/hash_bench.cpp b/src/ds/test/hash_bench.cpp new file mode 100644 index 0000000000..cce04c756d --- /dev/null +++ b/src/ds/test/hash_bench.cpp @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ds/hash.h" + +#define PICOBENCH_IMPLEMENT_WITH_MAIN +#include + +template +static void hash(picobench::state& s) +{ + T v(s.iterations()); + auto* d = v.data(); + for (size_t i = 0; i < v.size(); ++i) + { + d[i] = rand(); + } + + std::hash hasher; + + s.start_timer(); + for (size_t i = 0; i < 1000; ++i) + { + volatile auto n = hasher(v); + s.stop_timer(); + } +} + +const std::vector hash_sizes = {1, 8, 64, 1024, 16536}; + +PICOBENCH_SUITE("hash"); +auto hash_vec = hash>; +PICOBENCH(hash_vec).iterations(hash_sizes).baseline(); +auto hash_small_vec_16 = hash>; +PICOBENCH(hash_small_vec_16).iterations(hash_sizes).baseline(); +auto hash_small_vec_128 = hash>; +PICOBENCH(hash_small_vec_128).iterations(hash_sizes).baseline(); diff --git a/src/kv/change_set.h b/src/kv/change_set.h new file mode 100644 index 0000000000..d1333cde5b --- /dev/null +++ b/src/kv/change_set.h @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once +#include "ds/champ_map.h" +#include "ds/hash.h" +#include "kv/kv_types.h" + +#include + +namespace kv +{ + template + struct VersionV + { + Version version; + V value; + + VersionV() = default; + VersionV(Version ver, V val) : version(ver), value(val) {} + }; + + template + using State = champ::Map, H>; + + template + using Read = std::map; + + // nullopt values represent deletions + template + using Write = std::map>; + + // This is a container for a write-set + dependencies. It can be applied to a + // given state, or used to track a set of operations on a state + template + struct ChangeSet + { + public: + State state; + State committed; + Version start_version; + + Version read_version = NoVersion; + Read reads = {}; + Write writes = {}; + + ChangeSet( + State& current_state, + State& committed_state, + Version current_version) : + state(current_state), + committed(committed_state), + start_version(current_version) + {} + + ChangeSet(ChangeSet&) = delete; + }; + + /// Signature for transaction commit handlers + template + using CommitHook = std::function; +} \ No newline at end of file diff --git a/src/kv/experimental.h b/src/kv/experimental.h deleted file mode 100644 index b84c55f117..0000000000 --- a/src/kv/experimental.h +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. -#pragma once - -#include "ds/hash.h" -#include "kv_types.h" -#include "map.h" -#include "tx_view.h" - -#include - -namespace kv -{ - namespace experimental - { - using SerialisedRep = std::vector; - - using RepHasher = std::hash; - - using UntypedMap = kv::Map; - - using UntypedOperationsView = - kv::TxView; - - using UntypedCommitter = - kv::TxViewCommitter; - - using UntypedState = kv::State; - - template - struct MsgPackSerialiser - { - static SerialisedRep to_serialised(const T& t) - { - msgpack::sbuffer sb; - msgpack::pack(sb, t); - auto sb_data = reinterpret_cast(sb.data()); - return SerialisedRep(sb_data, sb_data + sb.size()); - } - - static T from_serialised(const SerialisedRep& rep) - { - msgpack::object_handle oh = msgpack::unpack( - reinterpret_cast(rep.data()), rep.size()); - auto object = oh.get(); - return object.as(); - } - }; - - template - struct JsonSerialiser - { - static SerialisedRep to_serialised(const T& t) - { - const nlohmann::json j = t; - const auto dumped = j.dump(); - return SerialisedRep(dumped.begin(), dumped.end()); - } - - static T from_serialised(const SerialisedRep& rep) - { - const auto j = nlohmann::json::parse(rep); - return j.get(); - } - }; - - template < - typename K, - typename V, - typename KSerialiser = MsgPackSerialiser, - typename VSerialiser = MsgPackSerialiser> - class TxView : public UntypedCommitter - { - protected: - // This _has_ a (non-visible, untyped) view, whereas the standard impl - // _is_ a typed view - UntypedOperationsView untyped_view; - - public: - using KeyType = K; - using ValueType = V; - - TxView( - UntypedMap& m, - size_t rollbacks, - UntypedState& current_state, - UntypedState& committed_state, - Version v) : - UntypedCommitter(m, rollbacks, current_state, committed_state, v), - untyped_view(UntypedCommitter::change_set) - {} - - std::optional get(const K& key) - { - const auto k_rep = KSerialiser::to_serialised(key); - const auto opt_v_rep = untyped_view.get(k_rep); - - if (opt_v_rep.has_value()) - { - return VSerialiser::from_serialised(*opt_v_rep); - } - - return std::nullopt; - } - - std::optional get_globally_committed(const K& key) - { - const auto k_rep = KSerialiser::to_serialised(key); - const auto opt_v_rep = untyped_view.get_globally_committed(k_rep); - - if (opt_v_rep.has_value()) - { - return VSerialiser::from_serialised(*opt_v_rep); - } - - return std::nullopt; - } - - bool put(const K& key, const V& value) - { - const auto k_rep = KSerialiser::to_serialised(key); - const auto v_rep = VSerialiser::to_serialised(value); - - return untyped_view.put(k_rep, v_rep); - } - - bool remove(const K& key) - { - const auto k_rep = KSerialiser::to_serialised(key); - - return untyped_view.remove(k_rep); - } - - template - void foreach(F&& f) - { - auto g = [&](const SerialisedRep& k_rep, const SerialisedRep& v_rep) { - return f( - KSerialiser::from_serialised(k_rep), - VSerialiser::from_serialised(v_rep)); - }; - untyped_view.foreach(g); - } - }; - - template < - typename K, - typename V, - typename KSerialiser = MsgPackSerialiser, - typename VSerialiser = MsgPackSerialiser> - class Map : public AbstractMap - { - protected: - using This = Map; - - UntypedMap untyped_map; - - public: - // Expose correct public aliases of types - using VersionV = VersionV; - - using Write = Write; - - using CommitHook = CommitHook; - - using TxView = kv::experimental::TxView; - - template - Map(Ts&&... ts) : untyped_map(std::forward(ts)...) - {} - - bool operator==(const AbstractMap& that) const override - { - auto p = dynamic_cast(&that); - if (p == nullptr) - { - return false; - } - - return untyped_map == p->untyped_map; - } - - bool operator!=(const AbstractMap& that) const override - { - return !(*this == that); - } - - AbstractStore* get_store() override - { - return untyped_map.get_store(); - } - - void serialise( - const AbstractTxView* view, - KvStoreSerialiser& s, - bool include_reads) override - { - untyped_map.serialise(view, s, include_reads); - } - - AbstractTxView* deserialise( - KvStoreDeserialiser& d, Version version) override - { - return untyped_map.deserialise(d, version); - } - - const std::string& get_name() const override - { - return untyped_map.get_name(); - } - - void compact(Version v) override - { - return untyped_map.compact(v); - } - - void post_compact() override - { - return untyped_map.post_compact(); - } - - void rollback(Version v) override - { - untyped_map.rollback(v); - } - - void lock() override - { - untyped_map.lock(); - } - - void unlock() override - { - untyped_map.unlock(); - } - - SecurityDomain get_security_domain() override - { - return untyped_map.get_security_domain(); - } - - bool is_replicated() override - { - return untyped_map.is_replicated(); - } - - void clear() override - { - untyped_map.clear(); - } - - AbstractMap* clone(AbstractStore* store) override - { - return new Map( - store, - untyped_map.get_name(), - untyped_map.get_security_domain(), - untyped_map.is_replicated()); - } - - void swap(AbstractMap* map) override - { - auto p = dynamic_cast(map); - if (p == nullptr) - throw std::logic_error( - "Attempted to swap maps with incompatible types"); - - untyped_map.swap(&p->untyped_map); - } - - template - TView* create_view(Version v) - { - return untyped_map.create_view(v); - } - - static UntypedMap::CommitHook wrap_commit_hook(const CommitHook& hook) - { - return [hook](Version v, const UntypedMap::Write& w) { - Write typed_w; - for (const auto& [uk, opt_uv] : w) - { - if (!opt_uv.has_value()) - { - // Deletions are indicated by nullopt. We cannot deserialise them, - // they are deletions here as well - typed_w[KSerialiser::from_serialised(uk)] = std::nullopt; - } - else - { - typed_w[KSerialiser::from_serialised(uk)] = - VSerialiser::from_serialised(opt_uv.value()); - } - } - - hook(v, typed_w); - }; - } - - void set_local_hook(const CommitHook& hook) - { - untyped_map.set_local_hook(wrap_commit_hook(hook)); - } - - void unset_local_hook() - { - untyped_map.unset_local_hook(); - } - - void set_global_hook(const CommitHook& hook) - { - untyped_map.set_global_hook(wrap_commit_hook(hook)); - } - - void unset_global_hook() - { - untyped_map.unset_global_hook(); - } - }; - } -} \ No newline at end of file diff --git a/src/kv/generic_serialise_wrapper.h b/src/kv/generic_serialise_wrapper.h index 0d91d5018a..0744ad7593 100644 --- a/src/kv/generic_serialise_wrapper.h +++ b/src/kv/generic_serialise_wrapper.h @@ -4,11 +4,15 @@ #include "ds/buffer.h" #include "kv_types.h" +#include "serialised_entry.h" #include namespace kv { + using SerialisedKey = kv::serialisers::SerialisedEntry; + using SerialisedValue = kv::serialisers::SerialisedEntry; + enum class KvOperationType : uint32_t { KOT_NOT_SUPPORTED = 0, @@ -38,16 +42,6 @@ namespace kv static_cast(a) | static_cast(b)); } - template - struct KeyValVersion - { - K key; - V value; - Version version; - - KeyValVersion(K k, V v, Version ver) : key(k), value(v), version(ver) {} - }; - template class GenericSerialiseWrapper { @@ -68,10 +62,10 @@ namespace kv current_writer->append(std::forward(t)); } - template - void serialise_internal_public(T&& t) + void serialise_internal_pre_serialised( + const kv::serialisers::SerialisedEntry& raw) { - public_writer.append(std::forward(t)); + current_writer->append_pre_serialised(raw); } void set_current_domain(SecurityDomain domain) @@ -127,40 +121,36 @@ namespace kv serialise_internal(ctr); } - template - void serialise_read(const K& k, const Version& version) + void serialise_read(const SerialisedKey& k, const Version& version) { - serialise_internal(k); + serialise_internal_pre_serialised(k); serialise_internal(version); } - template - void serialise_write(const K& k, const V& v) + void serialise_write(const SerialisedKey& k, const SerialisedValue& v) { - serialise_internal(k); - serialise_internal(v); + serialise_internal_pre_serialised(k); + serialise_internal_pre_serialised(v); } - template - void serialise_write_version(const K& k, const V& v, const Version& version) + void serialise_write_version( + const SerialisedKey& k, const SerialisedValue& v, const Version& version) { serialise_internal(KvOperationType::KOT_WRITE_VERSION); - serialise_internal(k); - serialise_internal(v); + serialise_internal_pre_serialised(k); + serialise_internal_pre_serialised(v); serialise_internal(version); } - template - void serialise_remove_version(const K& k) + void serialise_remove_version(const SerialisedKey& k) { serialise_internal(KvOperationType::KOT_REMOVE_VERSION); - serialise_internal(k); + serialise_internal_pre_serialised(k); } - template - void serialise_remove(const K& k) + void serialise_remove(const SerialisedKey& k) { - serialise_internal(k); + serialise_internal_pre_serialised(k); } std::vector get_raw_data() @@ -342,7 +332,6 @@ namespace kv return true; } - template Version deserialise_version() { version = current_reader->template read_next(); @@ -368,7 +357,6 @@ namespace kv current_reader->template read_next()}; } - template Version deserialise_read_version() { return current_reader->template read_next(); @@ -379,10 +367,9 @@ namespace kv return current_reader->template read_next(); } - template - std::tuple deserialise_read() + std::tuple deserialise_read() { - return {current_reader->template read_next(), + return {current_reader->read_next_pre_serialised(), current_reader->template read_next()}; } @@ -391,11 +378,10 @@ namespace kv return current_reader->template read_next(); } - template - std::tuple deserialise_write() + std::tuple deserialise_write() { - return {current_reader->template read_next(), - current_reader->template read_next()}; + return {current_reader->read_next_pre_serialised(), + current_reader->read_next_pre_serialised()}; } uint64_t deserialise_remove_header() @@ -403,39 +389,9 @@ namespace kv return current_reader->template read_next(); } - template - K deserialise_remove() + SerialisedKey deserialise_remove() { - return current_reader->template read_next(); - } - - template - std::optional> deserialise_write_version() - { - if (end()) - return {}; - - auto curr_op = try_read_op_flag( - KvOperationType::KOT_WRITE_VERSION | - KvOperationType::KOT_REMOVE_VERSION); - - switch (curr_op) - { - case KvOperationType::KOT_WRITE_VERSION: - { - K key = current_reader->template read_next(); - V value = current_reader->template read_next(); - Version version = current_reader->template read_next(); - return {{key, value, version, false}}; - } - case KvOperationType::KOT_REMOVE_VERSION: - { - K key = current_reader->template read_next(); - return {{key, V(), Version(), true}}; - } - default: - return {}; - } + return current_reader->read_next_pre_serialised(); } bool end() diff --git a/src/kv/kv_types.h b/src/kv/kv_types.h index 10a2755f74..f9f6572c6a 100644 --- a/src/kv/kv_types.h +++ b/src/kv/kv_types.h @@ -21,6 +21,11 @@ namespace kv using Version = int64_t; static const Version NoVersion = std::numeric_limits::min(); + static bool is_deleted(Version version) + { + return version < 0; + } + // Term describes an epoch of Versions. It is incremented when global kv's // writer(s) changes. Term and Version combined give a unique identifier for // all accepted kv modifications. Terms are handled by Raft via the diff --git a/src/kv/map.h b/src/kv/map.h index ff9ec1eefc..49c6f21037 100644 --- a/src/kv/map.h +++ b/src/kv/map.h @@ -2,567 +2,45 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ds/dl_list.h" -#include "ds/logger.h" -#include "ds/spin_lock.h" -#include "kv_serialiser.h" #include "kv_types.h" +#include "serialise_entry_blit.h" +#include "serialise_entry_json.h" +#include "serialise_entry_msgpack.h" #include "tx_view.h" -#include -#include -#include - namespace kv { - namespace Check - { - struct No - {}; - - template - No operator!=(const T&, const Arg&) - { - return No(); - } - - template - struct Ne - { - enum - { - value = !std::is_same::value - }; - }; - - template - bool ne(std::enable_if_t::value, const T&> a, const T& b) - { - return a != b; - } - - template - bool ne(std::enable_if_t::value, const T&> a, const T& b) - { - return false; - } - } - - template > - class Map; - - template - class TxViewCommitter : public AbstractTxView + template + class TypedMap : public AbstractMap { protected: - using MyMap = Map; + using This = TypedMap; - ChangeSet change_set; - - MyMap& map; - size_t rollback_counter; - - Version commit_version = NoVersion; - - bool changes = false; - bool committed_writes = false; + kv::untyped::Map untyped_map; public: - template - TxViewCommitter(MyMap& m, size_t rollbacks, Ts&&... ts) : - map(m), - rollback_counter(rollbacks), - change_set(std::forward(ts)...) - {} - - // Commit-related methods - bool has_writes() override - { - return committed_writes || !change_set.writes.empty(); - } - - bool has_changes() override - { - return changes; - } - - bool prepare() override - { - if (change_set.writes.empty()) - return true; - - // If the parent map has rolled back since this transaction began, this - // transaction must fail. - if (rollback_counter != map.rollback_counter) - return false; - - // If we have iterated over the map, check for a global version match. - auto current = map.roll->get_tail(); - - if ( - (change_set.read_version != NoVersion) && - (change_set.read_version != current->version)) - { - LOG_DEBUG_FMT("Read version {} is invalid", change_set.read_version); - return false; - } - - // Check each key in our read set. - for (auto it = change_set.reads.begin(); it != change_set.reads.end(); - ++it) - { - // Get the value from the current state. - auto search = current->state.get(it->first); - - if (it->second == NoVersion) - { - // If we depend on the key not existing, it must be absent. - if (search.has_value()) - { - LOG_DEBUG_FMT("Read depends on non-existing entry"); - return false; - } - } - else - { - // If we depend on the key existing, it must be present and have the - // version that we expect. - if (!search.has_value() || (it->second != search.value().version)) - { - LOG_DEBUG_FMT("Read depends on invalid version of entry"); - return false; - } - } - } - - return true; - } - - void commit(Version v) override - { - if (change_set.writes.empty()) - { - commit_version = change_set.start_version; - return; - } - - // Record our commit time. - commit_version = v; - committed_writes = true; - - if (!change_set.writes.empty()) - { - auto state = map.roll->get_tail()->state; - - for (auto it = change_set.writes.begin(); it != change_set.writes.end(); - ++it) - { - if (it->second.has_value()) - { - // Write the new value with the global version. - changes = true; - state = state.put(it->first, VersionV{v, it->second.value()}); - } - else - { - // Write an empty value with the deleted global version only if - // the key exists. - auto search = state.get(it->first); - if (search.has_value()) - { - changes = true; - state = state.put(it->first, VersionV{-v, V()}); - } - } - } - - if (changes) - { - map.roll->insert_back( - map.create_new_local_commit(v, state, change_set.writes)); - } - } - } - - void post_commit() override - { - // This is run separately from commit so that all commits in the Tx - // have been applied before local hooks are run. The maps in the Tx - // are still locked when post_commit is run. - if (change_set.writes.empty()) - return; - - if (map.local_hook) - { - auto roll = map.roll->get_tail(); - map.local_hook(roll->version, roll->writes); - } - } - - // Used by owning map during serialise and deserialise - ChangeSet& get_change_set() - { - return change_set; - } - - const ChangeSet& get_change_set() const - { - return change_set; - } - - void set_commit_version(Version v) - { - commit_version = v; - } - }; - - template - struct ConcreteTxView : public TxViewCommitter, - public TxView - { - public: - ConcreteTxView( - Map& m, - size_t rollbacks, - State& current_state, - State& committed_state, - Version v) : - TxViewCommitter(m, rollbacks, current_state, committed_state, v), - TxView(TxViewCommitter::change_set) - {} - }; - - /// Signature for transaction commit handlers - template - using CommitHook = std::function; - - template - class Map : public AbstractMap - { - public: + // Expose correct public aliases of types using VersionV = VersionV; - using State = State; - using Write = Write; + + using Write = std::map>; + using CommitHook = CommitHook; - private: - using This = Map; + using TxView = kv::TxView; - struct LocalCommit - { - LocalCommit() = default; - LocalCommit(Version v, State s, Write w) : - version(std::move(v)), - state(std::move(s)), - writes(std::move(w)), - next(nullptr), - prev(nullptr) - {} - - Version version; - State state; - Write writes; - LocalCommit* next; - LocalCommit* prev; - }; - using LocalCommits = snmalloc::DLList; - - AbstractStore* store; - std::string name; - size_t rollback_counter; - std::unique_ptr roll; - CommitHook local_hook = nullptr; - CommitHook global_hook = nullptr; - LocalCommits commit_deltas; - SpinLock sl; - const SecurityDomain security_domain; - const bool replicated; - - LocalCommits empty_commits; - - template - LocalCommit* create_new_local_commit(Args&&... args) - { - LocalCommit* c = empty_commits.pop(); - if (c == nullptr) - { - c = new LocalCommit(std::forward(args)...); - } - else - { - c->~LocalCommit(); - new (c) LocalCommit(std::forward(args)...); - } - return c; - } - - public: - // Public typedef for external consumption - using TxView = ConcreteTxView; - - // Provide access to hidden rollback_counter, roll, create_new_local_commit - friend TxViewCommitter; - - Map( - AbstractStore* store_, - std::string name_, - SecurityDomain security_domain_, - bool replicated_) : - store(store_), - name(name_), - roll(std::make_unique()), - rollback_counter(0), - security_domain(security_domain_), - replicated(replicated_) - { - roll->insert_back(create_new_local_commit(0, State(), Write())); - } - - Map(const Map& that) = delete; - - virtual AbstractMap* clone(AbstractStore* other) override - { - return new Map(other, name, security_domain, replicated); - } - - void serialise( - const AbstractTxView* view, - KvStoreSerialiser& s, - bool include_reads) override - { - const auto committer = - dynamic_cast*>(view); - if (committer == nullptr) - { - LOG_FAIL_FMT("Unable to serialise map due to type mismatch"); - return; - } - - const auto& change_set = committer->get_change_set(); - - s.start_map(name, security_domain); - - if (include_reads) - { - s.serialise_read_version(change_set.read_version); - - s.serialise_count_header(change_set.reads.size()); - for (auto it = change_set.reads.begin(); it != change_set.reads.end(); - ++it) - { - s.serialise_read(it->first, it->second); - } - } - else - { - s.serialise_read_version(NoVersion); - s.serialise_count_header(0); - } - - uint64_t write_ctr = 0; - uint64_t remove_ctr = 0; - for (auto it = change_set.writes.begin(); it != change_set.writes.end(); - ++it) - { - if (it->second.has_value()) - { - ++write_ctr; - } - else - { - auto search = roll->get_tail()->state.get(it->first); - if (search.has_value()) - { - ++remove_ctr; - } - } - } - - s.serialise_count_header(write_ctr); - for (auto it = change_set.writes.begin(); it != change_set.writes.end(); - ++it) - { - if (it->second.has_value()) - { - s.serialise_write(it->first, it->second.value()); - } - } - - s.serialise_count_header(remove_ctr); - for (auto it = change_set.writes.begin(); it != change_set.writes.end(); - ++it) - { - if (!it->second.has_value()) - { - s.serialise_remove(it->first); - } - } - } - - AbstractTxView* deserialise( - KvStoreDeserialiser& d, Version version) override - { - // Create a new change set, and deserialise d's contents into it. - auto view = create_view(version); - view->set_commit_version(version); - - auto& change_set = view->get_change_set(); - - uint64_t ctr; - - auto rv = d.template deserialise_read_version(); - if (rv != NoVersion) - { - change_set.read_version = rv; - } - - ctr = d.deserialise_read_header(); - for (size_t i = 0; i < ctr; ++i) - { - auto r = d.template deserialise_read(); - change_set.reads[std::get<0>(r)] = std::get<1>(r); - } - - ctr = d.deserialise_write_header(); - for (size_t i = 0; i < ctr; ++i) - { - auto w = d.template deserialise_write(); - change_set.writes[std::get<0>(w)] = std::get<1>(w); - } - - ctr = d.deserialise_remove_header(); - for (size_t i = 0; i < ctr; ++i) - { - auto r = d.template deserialise_remove(); - change_set.writes[r] = std::nullopt; - } - - return view; - } - - /** Get the name of the map - * - * @return const std::string& - */ - const std::string& get_name() const override - { - return name; - } - - /** Get store that the map belongs to - * - * @return Pointer to `kv::AbstractStore` - */ - AbstractStore* get_store() override - { - return store; - } - - /** Set handler to be called on local transaction commit - * - * @param hook function to be called on local transaction commit - */ - void set_local_hook(const CommitHook& hook) - { - std::lock_guard guard(sl); - local_hook = hook; - } - - /** Reset local transaction commit handler - */ - void unset_local_hook() - { - std::lock_guard guard(sl); - local_hook = nullptr; - } - - /** Set handler to be called on global transaction commit - * - * @param hook function to be called on global transaction commit - */ - void set_global_hook(const CommitHook& hook) - { - std::lock_guard guard(sl); - global_hook = hook; - } - - /** Reset global transaction commit handler - */ - void unset_global_hook() - { - std::lock_guard guard(sl); - global_hook = nullptr; - } - - /** Get security domain of a Map - * - * @return Security domain of the map (affects serialisation) - */ - virtual SecurityDomain get_security_domain() override - { - return security_domain; - } - - /** Get Map replicability - * - * @return true if the map is to be replicated, false if it is to be derived - */ - virtual bool is_replicated() override - { - return replicated; - } + template + TypedMap(Ts&&... ts) : untyped_map(std::forward(ts)...) + {} bool operator==(const AbstractMap& that) const override { auto p = dynamic_cast(&that); if (p == nullptr) + { return false; + } - if (name != p->name) - return false; - - auto state1 = roll->get_tail(); - auto state2 = p->roll->get_tail(); - - if (state1->version != state2->version) - return false; - - size_t count = 0; - state2->state.foreach([&count](const K& k, const VersionV& v) { - count++; - return true; - }); - - size_t i = 0; - bool ok = - state1->state.foreach([&state2, &i](const K& k, const VersionV& v) { - auto search = state2->state.get(k); - - if (search.has_value()) - { - auto& found = search.value(); - if (found.version != v.version) - { - return false; - } - else if (Check::ne(found.value, v.value)) - { - return false; - } - } - else - { - return false; - } - - i++; - return true; - }); - - if (i != count) - ok = false; - - return ok; + return untyped_map == p->untyped_map; } bool operator!=(const AbstractMap& that) const override @@ -570,155 +48,164 @@ namespace kv return !(*this == that); } + AbstractStore* get_store() override + { + return untyped_map.get_store(); + } + + void serialise( + const AbstractTxView* view, + KvStoreSerialiser& s, + bool include_reads) override + { + untyped_map.serialise(view, s, include_reads); + } + + AbstractTxView* deserialise( + KvStoreDeserialiser& d, Version version) override + { + return untyped_map.deserialise_internal(d, version); + } + + const std::string& get_name() const override + { + return untyped_map.get_name(); + } + void compact(Version v) override { - // This discards available rollback state before version v, and populates - // the commit_deltas to be passed to the global commit hook, if there is - // one, up to version v. The Map expects to be locked during compaction. - while (roll->get_head() != roll->get_tail()) - { - auto r = roll->get_head(); - - // Globally committed but not discardable. - if (r->version == v) - { - // We know that write set is not empty. - if (global_hook) - { - commit_deltas.insert_back( - create_new_local_commit(r->version, r->state, move(r->writes))); - } - return; - } - - // Discardable, so move to commit_deltas. - if (global_hook && !r->writes.empty()) - { - commit_deltas.insert_back( - create_new_local_commit(r->version, r->state, move(r->writes))); - } - - // Stop if the next state may be rolled back or is the only state. - // This ensures there is always a state present. - if (r->next->version > v) - return; - - auto c = roll->pop(); - empty_commits.insert(c); - } - - // There is only one roll. We may need to call the commit hook. - auto r = roll->get_head(); - - if (global_hook && !r->writes.empty()) - { - commit_deltas.insert_back( - create_new_local_commit(r->version, r->state, move(r->writes))); - } + return untyped_map.compact(v); } void post_compact() override { - if (global_hook) - { - for (auto r = commit_deltas.get_head(); r != nullptr; r = r->next) - { - global_hook(r->version, r->writes); - } - } - - commit_deltas.clear(); + return untyped_map.post_compact(); } void rollback(Version v) override { - // This rolls the current state back to version v. - // The Map expects to be locked during rollback. - bool advance = false; - - while (roll->get_head() != roll->get_tail()) - { - auto r = roll->get_tail(); - - // The initial empty state has v = 0, so will not be discarded if it - // is present. - if (r->version <= v) - break; - - advance = true; - auto c = roll->pop_tail(); - empty_commits.insert(c); - } - - if (advance) - rollback_counter++; - } - - void clear() override - { - // This discards all entries in the roll and resets the compacted value - // and rollback counter. The Map expects to be locked before clearing it. - roll->clear(); - roll->insert_back(create_new_local_commit(0, State(), Write())); - rollback_counter = 0; + untyped_map.rollback(v); } void lock() override { - sl.lock(); + untyped_map.lock(); } void unlock() override { - sl.unlock(); + untyped_map.unlock(); } - void swap(AbstractMap* map_) override + SecurityDomain get_security_domain() override { - This* map = dynamic_cast(map_); - if (map == nullptr) + return untyped_map.get_security_domain(); + } + + bool is_replicated() override + { + return untyped_map.is_replicated(); + } + + void clear() override + { + untyped_map.clear(); + } + + AbstractMap* clone(AbstractStore* store) override + { + return new TypedMap( + store, + untyped_map.get_name(), + untyped_map.get_security_domain(), + untyped_map.is_replicated()); + } + + void swap(AbstractMap* map) override + { + auto p = dynamic_cast(map); + if (p == nullptr) throw std::logic_error( "Attempted to swap maps with incompatible types"); - std::swap(rollback_counter, map->rollback_counter); - std::swap(roll, map->roll); + untyped_map.swap(&p->untyped_map); } template - TView* create_view(Version version) + TView* create_view(Version v) { - lock(); + return untyped_map.create_view(v); + } - // Find the last entry committed at or before this version. - TView* view = nullptr; - - for (auto current = roll->get_tail(); current != nullptr; - current = current->prev) - { - if (current->version <= version) + static kv::untyped::Map::CommitHook wrap_commit_hook(const CommitHook& hook) + { + return [hook](Version v, const kv::untyped::Write& w) { + Write typed_writes; + for (const auto& [uk, opt_uv] : w) { - view = new TView( - *this, - rollback_counter, - current->state, - roll->get_head()->state, - current->version); - break; + if (!opt_uv.has_value()) + { + // Deletions are indicated by nullopt. We cannot deserialise them, + // they are deletions here as well + typed_writes[KSerialiser::from_serialised(uk)] = std::nullopt; + } + else + { + typed_writes[KSerialiser::from_serialised(uk)] = + VSerialiser::from_serialised(opt_uv.value()); + } } - } - if (view == nullptr) - { - view = new TView( - *this, - rollback_counter, - roll->get_head()->state, - roll->get_head()->state, - roll->get_head()->version); - } + hook(v, typed_writes); + }; + } - unlock(); - return view; + void set_local_hook(const CommitHook& hook) + { + untyped_map.set_local_hook(wrap_commit_hook(hook)); + } + + void unset_local_hook() + { + untyped_map.unset_local_hook(); + } + + void set_global_hook(const CommitHook& hook) + { + untyped_map.set_global_hook(wrap_commit_hook(hook)); + } + + void unset_global_hook() + { + untyped_map.unset_global_hook(); } }; + + template < + typename K, + typename V, + template + typename KSerialiser, + template typename VSerialiser = KSerialiser> + using MapSerialisedWith = TypedMap, VSerialiser>; + + template + using JsonSerialisedMap = + MapSerialisedWith; + + template + using RawCopySerialisedMap = TypedMap< + K, + V, + kv::serialisers::BlitSerialiser, + kv::serialisers::BlitSerialiser>; + + template + using MsgPackSerialisedMap = + MapSerialisedWith; + + // The default kv::Map will use msgpack serialisers. Custom types are + // supported through the MSGPACK_DEFINE macro + template + using Map = MsgPackSerialisedMap; } \ No newline at end of file diff --git a/src/kv/msgpack_serialise.h b/src/kv/msgpack_serialise.h index 6ff63b45bd..45de10d607 100644 --- a/src/kv/msgpack_serialise.h +++ b/src/kv/msgpack_serialise.h @@ -5,6 +5,7 @@ #include "ds/msgpack_adaptor_nlohmann.h" #include "ds/serialized.h" #include "generic_serialise_wrapper.h" +#include "serialised_entry.h" #include #include @@ -15,6 +16,13 @@ MSGPACK_ADD_ENUM(kv::KvOperationType); MSGPACK_ADD_ENUM(kv::SecurityDomain); +// Currently we have pre-serialised keys and values, which are often msgpack, +// which are then re-packed into msgpack to go into the ledger. This is +// wasteful. But without this we can't _unpack_ custom types at this level. We +// should replace this with a custom serialisation format for the ledger. This +// macro gates the intended code path. +#define MSGPACK_DONT_REPACK (1) + namespace kv { class MsgPackWriter @@ -29,6 +37,21 @@ namespace kv msgpack::pack(sb, std::forward(t)); } + // Where we have pre-serialised data, we dump it directly into the output + // buffer. If we call append, then pack will prefix the data with some type + // information, potentially redundantly repacking already-packed data. We + // assume the serialised entry is already msgpack so we retain a consistent + // msgpack stream. If it is in some other format, every parser will need to + // be able to distinguish it from ths valid stream + void append_pre_serialised(const kv::serialisers::SerialisedEntry& entry) + { +#if MSGPACK_DONT_REPACK + sb.write(reinterpret_cast(entry.data()), entry.size()); +#else + append(entry); +#endif + } + void clear() { sb.clear(); @@ -77,6 +100,18 @@ namespace kv return msg->as(); } + kv::serialisers::SerialisedEntry read_next_pre_serialised() + { +#if MSGPACK_DONT_REPACK + const auto before_offset = data_offset; + msgpack::unpack(msg, data_ptr, data_size, data_offset); + return kv::serialisers::SerialisedEntry( + data_ptr + before_offset, data_ptr + data_offset); +#else + return read_next(); +#endif + } + template T peek_next() { diff --git a/src/kv/serialise_entry_blit.h b/src/kv/serialise_entry_blit.h new file mode 100644 index 0000000000..81cdeea773 --- /dev/null +++ b/src/kv/serialise_entry_blit.h @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "serialised_entry.h" + +namespace kv::serialisers +{ + namespace + { + template + struct is_std_array : std::false_type + {}; + + template + struct is_std_array> : public std::true_type + {}; + + template + struct dependent_false : public std::false_type + {}; + } + + template + struct BlitSerialiser + { + static SerialisedEntry to_serialised(const T& t) + { + if constexpr (std::is_same_v>) + { + return SerialisedEntry(t.begin(), t.end()); + } + else if constexpr (is_std_array::value) + { + return SerialisedEntry(t.begin(), t.end()); + } + else if constexpr (std::is_integral_v) + { + SerialisedEntry s(sizeof(t)); + std::memcpy(s.data(), (uint8_t*)&t, sizeof(t)); + return s; + } + else + { + static_assert(dependent_false::value, "Can't serialise this type"); + } + } + + static T from_serialised(const SerialisedEntry& rep) + { + if constexpr (std::is_same_v>) + { + return T(rep.begin(), rep.end()); + } + else if constexpr (is_std_array::value) + { + return T(rep.begin(), rep.end()); + } + else if constexpr (std::is_integral_v) + { + if (rep.size() != sizeof(T)) + { + throw std::logic_error("Wrong size for deserialising"); + } + return *(T*)rep.data(); + } + else + { + static_assert(dependent_false::value, "Can't deserialise this type"); + } + } + }; +} \ No newline at end of file diff --git a/src/kv/serialise_entry_json.h b/src/kv/serialise_entry_json.h new file mode 100644 index 0000000000..6bebb35a5b --- /dev/null +++ b/src/kv/serialise_entry_json.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "serialised_entry.h" + +#include + +namespace kv::serialisers +{ + template + struct JsonSerialiser + { + static SerialisedEntry to_serialised(const T& t) + { + const nlohmann::json j = t; + const auto dumped = j.dump(); + return SerialisedEntry(dumped.begin(), dumped.end()); + } + + static T from_serialised(const SerialisedEntry& rep) + { + const auto j = nlohmann::json::parse(rep); + return j.get(); + } + }; +} \ No newline at end of file diff --git a/src/kv/serialise_entry_msgpack.h b/src/kv/serialise_entry_msgpack.h new file mode 100644 index 0000000000..0087383569 --- /dev/null +++ b/src/kv/serialise_entry_msgpack.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "serialised_entry.h" + +#include + +namespace kv::serialisers +{ + namespace detail + { + struct SerialisedEntryWriter + { + SerialisedEntry& entry; + + void write(const char* d, size_t n) + { + entry.insert(entry.end(), d, d + n); + } + }; + } + + template + struct MsgPackSerialiser + { + static SerialisedEntry to_serialised(const T& t) + { + SerialisedEntry e; + detail::SerialisedEntryWriter w{e}; + msgpack::pack(w, t); + return e; + } + + static T from_serialised(const SerialisedEntry& rep) + { + msgpack::object_handle oh = + msgpack::unpack(reinterpret_cast(rep.data()), rep.size()); + auto object = oh.get(); + return object.as(); + } + }; +} \ No newline at end of file diff --git a/src/kv/serialised_entry.h b/src/kv/serialised_entry.h new file mode 100644 index 0000000000..1d4165d0a8 --- /dev/null +++ b/src/kv/serialised_entry.h @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include +#include +#include + +namespace kv::serialisers +{ + using SerialisedEntry = llvm_vecsmall::SmallVector; +} diff --git a/src/kv/store.h b/src/kv/store.h index be53966921..7545fef302 100644 --- a/src/kv/store.h +++ b/src/kv/store.h @@ -106,10 +106,10 @@ namespace kv return encryptor; } - template > - Map* get(std::string name) + template + Map* get(std::string name) { - return get>(name); + return get>(name); } /** Get Map by name @@ -147,12 +147,12 @@ namespace kv * * @return Newly created Map */ - template > - Map& create( + template + Map& create( std::string name, SecurityDomain security_domain = kv::SecurityDomain::PRIVATE) { - return create>(name, security_domain); + return create>(name, security_domain); } /** Create a Map @@ -297,7 +297,7 @@ namespace kv return DeserialiseSuccess::FAILED; } - Version v = d->template deserialise_version(); + Version v = d->deserialise_version(); // Throw away any local commits that have not propagated via the // consensus. rollback(v - 1); diff --git a/src/kv/test/kv_bench.cpp b/src/kv/test/kv_bench.cpp index d8008ee9e0..47e5bca830 100644 --- a/src/kv/test/kv_bench.cpp +++ b/src/kv/test/kv_bench.cpp @@ -7,14 +7,37 @@ #include "kv/tx.h" #include "node/encryptor.h" +#include #include #include +using KeyType = kv::serialisers::SerialisedEntry; +using ValueType = kv::serialisers::SerialisedEntry; +using MapType = kv::untyped::Map; + inline void clobber_memory() { asm volatile("" : : : "memory"); } +KeyType gen_key(size_t i, const std::string& suf = "") +{ + const auto s = "key" + std::to_string(i) + suf; + msgpack::sbuffer buf; + msgpack::pack(buf, s); + const auto raw = reinterpret_cast(buf.data()); + return KeyType(raw, raw + buf.size()); +} + +ValueType gen_value(size_t i) +{ + const auto s = "value" + std::to_string(i); + msgpack::sbuffer buf; + msgpack::pack(buf, s); + const auto raw = reinterpret_cast(buf.data()); + return ValueType(raw, raw + buf.size()); +} + // Helper functions to use a dummy encryption key std::shared_ptr create_ledger_secrets() { @@ -37,16 +60,17 @@ static void serialise(picobench::state& s) encryptor->set_iv_id(1); kv_store.set_encryptor(encryptor); - auto& map0 = kv_store.create("map0", SD); - auto& map1 = kv_store.create("map1", SD); + auto& map0 = kv_store.create("map0", SD); + auto& map1 = kv_store.create("map1", SD); kv::Tx tx; auto [tx0, tx1] = tx.get_view(map0, map1); for (int i = 0; i < s.iterations(); i++) { - auto key = "key" + std::to_string(i); - tx0->put(key, "value"); - tx1->put(key, "value"); + const auto key = gen_key(i); + const auto value = gen_value(i); + tx0->put(key, value); + tx1->put(key, value); } s.start_timer(); @@ -71,18 +95,19 @@ static void deserialise(picobench::state& s) kv_store.set_encryptor(encryptor); kv_store2.set_encryptor(encryptor); - auto& map0 = kv_store.create("map0", SD); - auto& map1 = kv_store.create("map1", SD); - auto& map0_ = kv_store2.create("map0", SD); - auto& map1_ = kv_store2.create("map1", SD); + auto& map0 = kv_store.create("map0", SD); + auto& map1 = kv_store.create("map1", SD); + auto& map0_ = kv_store2.create("map0", SD); + auto& map1_ = kv_store2.create("map1", SD); kv::Tx tx; auto [tx0, tx1] = tx.get_view(map0, map1); for (int i = 0; i < s.iterations(); i++) { - auto key = "key" + std::to_string(i); - tx0->put(key, "value"); - tx1->put(key, "value"); + const auto key = gen_key(i); + const auto value = gen_value(i); + tx0->put(key, value); + tx1->put(key, value); } tx.commit(); @@ -105,8 +130,8 @@ static void commit_latency(picobench::state& s) encryptor->set_iv_id(1); kv_store.set_encryptor(encryptor); - auto& map0 = kv_store.create("map0"); - auto& map1 = kv_store.create("map1"); + auto& map0 = kv_store.create("map0"); + auto& map1 = kv_store.create("map1"); for (int i = 0; i < s.iterations(); i++) { @@ -114,9 +139,10 @@ static void commit_latency(picobench::state& s) auto [tx0, tx1] = tx.get_view(map0, map1); for (int iTx = 0; iTx < S; iTx++) { - auto key = "key" + std::to_string(i) + " - " + std::to_string(iTx); - tx0->put(key, "value"); - tx1->put(key, "value"); + const auto key = gen_key(i, std::to_string(iTx)); + const auto value = gen_value(i); + tx0->put(key, value); + tx1->put(key, value); } auto rc = tx.commit(); @@ -153,3 +179,9 @@ PICOBENCH(deserialise) .samples(sample_size) .baseline(); PICOBENCH(deserialise).iterations(tx_count).samples(sample_size); + +// int main(int argc, char* argv[]) +// { +// picobench::state s(1'000'000); +// serialise(s); +// } \ No newline at end of file diff --git a/src/kv/test/kv_serialisation.cpp b/src/kv/test/kv_serialisation.cpp index 6ffbc30469..c7481ab160 100644 --- a/src/kv/test/kv_serialisation.cpp +++ b/src/kv/test/kv_serialisation.cpp @@ -2,7 +2,6 @@ // Licensed under the Apache 2.0 License. #include "ds/logger.h" #include "kv/encryptor.h" -#include "kv/experimental.h" #include "kv/kv_serialiser.h" #include "kv/store.h" #include "kv/test/null_encryptor.h" @@ -14,7 +13,7 @@ #include #include -struct RawMapTypes +struct MapTypes { using StringString = kv::Map; using NumNum = kv::Map; @@ -22,20 +21,9 @@ struct RawMapTypes using StringNum = kv::Map; }; -struct ExperimentalMapTypes -{ - using StringString = kv::experimental::Map; - using NumNum = kv::experimental::Map; - using NumString = kv::experimental::Map; - using StringNum = kv::experimental::Map; -}; - -TEST_CASE_TEMPLATE( +TEST_CASE( "Serialise/deserialise public map only" * - doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) + doctest::test_suite("serialisation")) { // No need for an encryptor here as all maps are public. Both serialisation // and deserialisation should succeed. @@ -43,12 +31,12 @@ TEST_CASE_TEMPLATE( kv::Store kv_store(consensus); - auto& pub_map = kv_store.create( + auto& pub_map = kv_store.create( "pub_map", kv::SecurityDomain::PUBLIC); kv::Store kv_store_target; kv_store_target.clone_schema(kv_store); - auto* target_map = kv_store.get("pub_map"); + auto* target_map = kv_store.get("pub_map"); REQUIRE(target_map != nullptr); INFO("Commit to public map in source store"); @@ -73,23 +61,20 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( +TEST_CASE( "Serialise/deserialise private map only" * - doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) + doctest::test_suite("serialisation")) { auto consensus = std::make_shared(); auto encryptor = std::make_shared(); kv::Store kv_store(consensus); - auto& priv_map = kv_store.create("priv_map"); + auto& priv_map = kv_store.create("priv_map"); kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); kv_store_target.clone_schema(kv_store); - auto* target_map = kv_store.get("priv_map"); + auto* target_map = kv_store.get("priv_map"); REQUIRE(target_map != nullptr); INFO("Commit a private transaction without an encryptor throws an exception"); @@ -126,29 +111,24 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( +TEST_CASE( "Serialise/deserialise private map and public maps" * - doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) + doctest::test_suite("serialisation")) { auto consensus = std::make_shared(); auto encryptor = std::make_shared(); kv::Store kv_store(consensus); kv_store.set_encryptor(encryptor); - auto& priv_map = kv_store.create("priv_map"); - auto& pub_map = kv_store.create( + auto& priv_map = kv_store.create("priv_map"); + auto& pub_map = kv_store.create( "pub_map", kv::SecurityDomain::PUBLIC); kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); kv_store_target.clone_schema(kv_store); - auto* target_priv_map = - kv_store.get("priv_map"); - auto* target_pub_map = - kv_store.get("pub_map"); + auto* target_priv_map = kv_store.get("priv_map"); + auto* target_pub_map = kv_store.get("pub_map"); REQUIRE(target_priv_map != nullptr); REQUIRE(target_pub_map != nullptr); @@ -177,24 +157,20 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Serialise/deserialise removed keys" * doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) +TEST_CASE( + "Serialise/deserialise removed keys" * doctest::test_suite("serialisation")) { auto consensus = std::make_shared(); auto encryptor = std::make_shared(); kv::Store kv_store(consensus); kv_store.set_encryptor(encryptor); - auto& priv_map = kv_store.create("priv_map"); + auto& priv_map = kv_store.create("priv_map"); kv::Store kv_store_target; kv_store_target.set_encryptor(encryptor); kv_store_target.clone_schema(kv_store); - auto* target_priv_map = - kv_store.get("priv_map"); + auto* target_priv_map = kv_store.get("priv_map"); REQUIRE(target_priv_map != nullptr); INFO("Commit a new key in source store and deserialise in target store"); @@ -236,112 +212,19 @@ TEST_CASE_TEMPLATE( } struct CustomClass -{ - int m_i; - - CustomClass() : CustomClass(-1) {} - CustomClass(int i) : m_i(i) {} - - int get() const - { - return m_i; - } - void set(std::string val) - { - m_i = std::stoi(val); - } - - CustomClass operator()() - { - CustomClass ret; - return ret; - } - - bool operator<(const CustomClass& other) const - { - return m_i < other.m_i; - } - - bool operator==(const CustomClass& other) const - { - return !(other < *this) && !(*this < other); - } - - MSGPACK_DEFINE(m_i); -}; - -namespace std -{ - template <> - struct hash - { - std::size_t operator()(const CustomClass& inst) const - { - return inst.get(); - } - }; -} - -DECLARE_JSON_TYPE(CustomClass) -DECLARE_JSON_REQUIRED_FIELDS(CustomClass, m_i) - -TEST_CASE( - "Custom type serialisation test (original scheme)" * - doctest::test_suite("serialisation")) -{ - kv::Store kv_store; - - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); - - CustomClass k(3); - CustomClass v1(33); - - CustomClass k2(2); - CustomClass v2(22); - - INFO("Serialise/Deserialise 2 kv stores"); - { - kv::Store kv_store2; - auto& map2 = kv_store2.create( - "map", kv::SecurityDomain::PUBLIC); - - kv::Tx tx(kv_store.next_version()); - auto view = tx.get_view(map); - view->put(k, v1); - view->put(k2, v2); - - auto [success, reqid, data] = tx.commit_reserved(); - REQUIRE(success == kv::CommitSuccess::OK); - kv_store.compact(kv_store.current_version()); - - REQUIRE(kv_store2.deserialise(data) == kv::DeserialiseSuccess::PASS); - kv::Tx tx2; - auto view2 = tx2.get_view(map2); - auto va = view2->get(k); - - REQUIRE(va.has_value()); - REQUIRE(va.value() == v1); - auto vb = view2->get(k2); - REQUIRE(vb.has_value()); - REQUIRE(vb.value() == v2); - // we only require operator==() to be implemented, so for consistency - - // this is the operator we use for comparison, and not operator!=() - REQUIRE(!(vb.value() == v1)); - } -} - -struct CustomClass2 { std::string s; size_t n; + + // This macro allows the default serialiser to be used + MSGPACK_DEFINE(s, n); }; struct CustomJsonSerialiser { - using Bytes = kv::experimental::SerialisedRep; + using Bytes = kv::serialisers::SerialisedEntry; - static Bytes to_serialised(const CustomClass2& c) + static Bytes to_serialised(const CustomClass& c) { nlohmann::json j = nlohmann::json::object(); j["s"] = c.s; @@ -350,10 +233,10 @@ struct CustomJsonSerialiser return Bytes(s.begin(), s.end()); } - static CustomClass2 from_serialised(const Bytes& b) + static CustomClass from_serialised(const Bytes& b) { const auto j = nlohmann::json::parse(b); - CustomClass2 c; + CustomClass c; c.s = j["s"]; c.n = j["n"]; return c; @@ -373,15 +256,15 @@ struct VPrefix template struct CustomVerboseDumbSerialiser { - using Bytes = kv::experimental::SerialisedRep; + using Bytes = kv::serialisers::SerialisedEntry; - static Bytes to_serialised(const CustomClass2& c) + static Bytes to_serialised(const CustomClass& c) { const auto verbose = fmt::format("{}\ns={}\nn={}", T::prefix, c.s, c.n); return Bytes(verbose.begin(), verbose.end()); } - static CustomClass2 from_serialised(const Bytes& b) + static CustomClass from_serialised(const Bytes& b) { std::string s(b.begin(), b.end()); const auto prefix_start = s.find(T::prefix); @@ -390,7 +273,7 @@ struct CustomVerboseDumbSerialiser throw std::logic_error("Missing expected prefix"); } - CustomClass2 c; + CustomClass c; const auto first_linebreak = s.find('\n'); const auto last_linebreak = s.rfind('\n'); const auto seg_a = s.substr(0, first_linebreak); @@ -405,30 +288,37 @@ struct CustomVerboseDumbSerialiser } }; -using MapA = kv::experimental:: - Map; -using MapB = kv::experimental::Map< - CustomClass2, - CustomClass2, +using DefaultSerialisedMap = kv::Map; +using CustomJsonMap = kv::TypedMap< + CustomClass, + CustomClass, + CustomJsonSerialiser, + CustomJsonSerialiser>; +using VerboseSerialisedMap = kv::TypedMap< + CustomClass, + CustomClass, CustomVerboseDumbSerialiser, CustomVerboseDumbSerialiser>; +// While the ledger expects everything to be msgpack, we don't actually support +// custom types +#if !MSGPACK_DONT_REPACK TEST_CASE_TEMPLATE( - "Custom type serialisation test (experimental scheme)" * - doctest::test_suite("serialisation"), + "Custom type serialisation test" * doctest::test_suite("serialisation"), MapType, - MapA, - MapB) + DefaultSerialisedMap, + CustomJsonMap, + VerboseSerialisedMap) { kv::Store kv_store; auto& map = kv_store.create("map", kv::SecurityDomain::PUBLIC); - CustomClass2 k1{"hello", 42}; - CustomClass2 v1{"world", 43}; + CustomClass k1{"hello", 42}; + CustomClass v1{"world", 43}; - CustomClass2 k2{"saluton", 100}; - CustomClass2 v2{"mondo", 1024}; + CustomClass k2{"saluton", 100}; + CustomClass v2{"mondo", 1024}; INFO("Serialise/Deserialise 2 kv stores"); { @@ -461,6 +351,7 @@ TEST_CASE_TEMPLATE( REQUIRE(vb->n == v2.n); } } +#endif bool corrupt_serialised_tx( std::vector& serialised_tx, std::vector& value_to_corrupt) @@ -488,11 +379,7 @@ bool corrupt_serialised_tx( return false; } -TEST_CASE_TEMPLATE( - "Integrity" * doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) +TEST_CASE("Integrity" * doctest::test_suite("serialisation")) { SUBCASE("Public and Private") { @@ -515,10 +402,9 @@ TEST_CASE_TEMPLATE( kv_store.set_encryptor(encryptor); kv_store_target.set_encryptor(encryptor); - auto& public_map = kv_store.create( + auto& public_map = kv_store.create( "public_map", kv::SecurityDomain::PUBLIC); - auto& private_map = - kv_store.create("private_map"); + auto& private_map = kv_store.create("private_map"); kv_store_target.clone_schema(kv_store); @@ -584,14 +470,11 @@ TEST_CASE("nlohmann (de)serialisation" * doctest::test_suite("serialisation")) } } -TEST_CASE_TEMPLATE( +TEST_CASE( "Replicated and derived table serialisation" * - doctest::test_suite("serialisation"), - MapImpl, - RawMapTypes, - ExperimentalMapTypes) + doctest::test_suite("serialisation")) { - using T = typename MapImpl::NumNum; + using T = MapTypes::NumNum; auto encryptor = std::make_shared(); std::unordered_set replicated_tables = { @@ -668,76 +551,9 @@ TEST_CASE_TEMPLATE( struct NonSerialisable {}; -namespace msgpack -{ - MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS) - { - namespace adaptor - { - // msgpack conversion for uint256_t - template <> - struct convert - { - msgpack::object const& operator()( - msgpack::object const& o, NonSerialisable& ns) const - { - throw std::runtime_error("Deserialise failure"); - } - }; - - template <> - struct pack - { - template - packer& operator()( - msgpack::packer& o, NonSerialisable const& ns) const - { - throw std::runtime_error("Serialise failure"); - } - }; - } - } -} - -TEST_CASE( - "Exceptional serdes (old scheme)" * doctest::test_suite("serialisation")) -{ - auto encryptor = std::make_shared(); - auto consensus = std::make_shared(); - - kv::Store store(consensus); - store.set_encryptor(encryptor); - - auto& good_map = store.create("good_map"); - auto& bad_map = store.create("bad_map"); - - { - kv::Tx tx; - auto good_view = tx.get_view(good_map); - good_view->put(1, 2); - REQUIRE(tx.commit() == kv::CommitSuccess::OK); - } - - { - kv::Tx tx; - auto bad_view = tx.get_view(bad_map); - bad_view->put(0, {}); - REQUIRE_THROWS_AS(tx.commit(), kv::KvSerialiserException); - } - - { - kv::Tx tx; - auto good_view = tx.get_view(good_map); - good_view->put(1, 2); - auto bad_view = tx.get_view(bad_map); - bad_view->put(0, {}); - REQUIRE_THROWS_AS(tx.commit(), kv::KvSerialiserException); - } -} - struct NonSerialiser { - using Bytes = kv::experimental::SerialisedRep; + using Bytes = kv::serialisers::SerialisedEntry; static Bytes to_serialised(const NonSerialisable& ns) { @@ -750,9 +566,7 @@ struct NonSerialiser } }; -TEST_CASE( - "Exceptional serdes (experimental scheme)" * - doctest::test_suite("serialisation")) +TEST_CASE("Exceptional serdes" * doctest::test_suite("serialisation")) { auto encryptor = std::make_shared(); auto consensus = std::make_shared(); @@ -760,15 +574,15 @@ TEST_CASE( kv::Store store(consensus); store.set_encryptor(encryptor); - auto& bad_map_k = store.create>>("bad_map_k"); - auto& bad_map_v = store.create>>("bad_map_k"); + auto& bad_map_v = store.create, + kv::serialisers::MsgPackSerialiser, NonSerialiser>>("bad_map_v"); { diff --git a/src/kv/test/kv_test.cpp b/src/kv/test/kv_test.cpp index 9787091bf8..cdc8535994 100644 --- a/src/kv/test/kv_test.cpp +++ b/src/kv/test/kv_test.cpp @@ -2,7 +2,6 @@ // Licensed under the Apache 2.0 License. #include "ds/logger.h" #include "enclave/app_interface.h" -#include "kv/experimental.h" #include "kv/kv_serialiser.h" #include "kv/store.h" #include "kv/test/null_encryptor.h" @@ -15,7 +14,7 @@ #include #include -struct RawMapTypes +struct MapTypes { using StringString = kv::Map; using NumNum = kv::Map; @@ -23,65 +22,54 @@ struct RawMapTypes using StringNum = kv::Map; }; -struct ExperimentalMapTypes -{ - using StringString = kv::experimental::Map; - using NumNum = kv::experimental::Map; - using NumString = kv::experimental::Map; - using StringNum = kv::experimental::Map; -}; - -TEST_CASE_TEMPLATE("Map creation", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Map creation") { kv::Store kv_store; const auto map_name = "map"; - auto& map = kv_store.create(map_name); + auto& map = kv_store.create(map_name); INFO("Get a map that does not exist"); { - REQUIRE( - kv_store.get("invalid_map") == nullptr); + REQUIRE(kv_store.get("invalid_map") == nullptr); } INFO("Get a map that does exist"); { - auto* p_map = kv_store.get(map_name); + auto* p_map = kv_store.get(map_name); REQUIRE(*p_map == map); REQUIRE(p_map == &map); // They're the _same instance_, not just equal } INFO("Compare different maps"); { - auto& map2 = kv_store.create("map2"); + auto& map2 = kv_store.create("map2"); REQUIRE(map != map2); } INFO("Can't create map that already exists"); { REQUIRE_THROWS_AS( - kv_store.create(map_name), - std::logic_error); + kv_store.create(map_name), std::logic_error); } INFO("Can't get a map with the wrong type"); { - REQUIRE(kv_store.get(map_name) == nullptr); - REQUIRE(kv_store.get(map_name) == nullptr); - REQUIRE(kv_store.get(map_name) == nullptr); + REQUIRE(kv_store.get(map_name) == nullptr); + REQUIRE(kv_store.get(map_name) == nullptr); + REQUIRE(kv_store.get(map_name) == nullptr); } INFO("Can create a map with a previously invalid name"); { - CHECK_NOTHROW(kv_store.create("version")); + CHECK_NOTHROW(kv_store.create("version")); } } -TEST_CASE_TEMPLATE( - "Reads/writes and deletions", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Reads/writes and deletions") { kv::Store kv_store; - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); + auto& map = + kv_store.create("map", kv::SecurityDomain::PUBLIC); constexpr auto k = "key"; constexpr auto invalid_key = "invalid_key"; @@ -155,11 +143,11 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE("foreach", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("foreach") { kv::Store kv_store; - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); + auto& map = + kv_store.create("map", kv::SecurityDomain::PUBLIC); std::map iterated_entries; @@ -333,12 +321,11 @@ TEST_CASE_TEMPLATE("foreach", MapImpl, RawMapTypes, ExperimentalMapTypes) } } -TEST_CASE_TEMPLATE( - "Rollback and compact", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Rollback and compact") { kv::Store kv_store; - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); + auto& map = + kv_store.create("map", kv::SecurityDomain::PUBLIC); constexpr auto k = "key"; constexpr auto v1 = "value1"; @@ -388,14 +375,13 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Clear entire store", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Clear entire store") { kv::Store kv_store; - auto& map1 = kv_store.create( - "map1", kv::SecurityDomain::PUBLIC); - auto& map2 = kv_store.create( - "map2", kv::SecurityDomain::PUBLIC); + auto& map1 = + kv_store.create("map1", kv::SecurityDomain::PUBLIC); + auto& map2 = + kv_store.create("map2", kv::SecurityDomain::PUBLIC); INFO("Commit a transaction over two maps"); { @@ -428,10 +414,9 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Local commit hooks", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Local commit hooks") { - using Write = typename MapImpl::StringString::Write; + using Write = MapTypes::StringString::Write; std::vector local_writes; std::vector global_writes; @@ -443,8 +428,8 @@ TEST_CASE_TEMPLATE( }; kv::Store kv_store; - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); + auto& map = + kv_store.create("map", kv::SecurityDomain::PUBLIC); map.set_local_hook(local_hook); map.set_global_hook(global_hook); @@ -513,10 +498,9 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Global commit hooks", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Global commit hooks") { - using Write = typename MapImpl::StringString::Write; + using Write = MapTypes::StringString::Write; struct GlobalHookInput { @@ -666,15 +650,15 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE("Clone schema", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Clone schema") { auto encryptor = std::make_shared(); kv::Store store; store.set_encryptor(encryptor); - auto& public_map = store.create( - "public", kv::SecurityDomain::PUBLIC); - auto& private_map = store.create("private"); + auto& public_map = + store.create("public", kv::SecurityDomain::PUBLIC); + auto& private_map = store.create("private"); kv::Tx tx1(store.next_version()); auto [view1, view2] = tx1.get_view(public_map, private_map); view1->put(42, "aardvark"); @@ -689,8 +673,7 @@ TEST_CASE_TEMPLATE("Clone schema", MapImpl, RawMapTypes, ExperimentalMapTypes) REQUIRE(clone.deserialise(data) == kv::DeserialiseSuccess::PASS); } -TEST_CASE_TEMPLATE( - "Deserialise return status", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Deserialise return status") { kv::Store store; @@ -699,7 +682,7 @@ TEST_CASE_TEMPLATE( auto& nodes = store.create(ccf::Tables::NODES, kv::SecurityDomain::PUBLIC); auto& data = - store.create("data", kv::SecurityDomain::PUBLIC); + store.create("data", kv::SecurityDomain::PUBLIC); auto kp = tls::make_key_pair(); @@ -742,22 +725,21 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Map swap between stores", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Map swap between stores") { auto encryptor = std::make_shared(); kv::Store s1; s1.set_encryptor(encryptor); - auto& d1 = s1.create("data"); - auto& pd1 = s1.create( - "public_data", kv::SecurityDomain::PUBLIC); + auto& d1 = s1.create("data"); + auto& pd1 = + s1.create("public_data", kv::SecurityDomain::PUBLIC); kv::Store s2; s2.set_encryptor(encryptor); - auto& d2 = s2.create("data"); - auto& pd2 = s2.create( - "public_data", kv::SecurityDomain::PUBLIC); + auto& d2 = s2.create("data"); + auto& pd2 = + s2.create("public_data", kv::SecurityDomain::PUBLIC); { kv::Tx tx; @@ -815,16 +797,15 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Invalid map swaps", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Invalid map swaps") { { kv::Store s1; - s1.create("one"); + s1.create("one"); kv::Store s2; - s2.create("one"); - s2.create("two"); + s2.create("one"); + s2.create("two"); REQUIRE_THROWS_WITH( s2.swap_private_maps(s1), @@ -833,11 +814,11 @@ TEST_CASE_TEMPLATE( { kv::Store s1; - s1.create("one"); - s1.create("two"); + s1.create("one"); + s1.create("two"); kv::Store s2; - s2.create("one"); + s2.create("one"); REQUIRE_THROWS_WITH( s2.swap_private_maps(s1), @@ -845,21 +826,20 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Private recovery map swap", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Private recovery map swap") { auto encryptor = std::make_shared(); kv::Store s1; s1.set_encryptor(encryptor); - auto& priv1 = s1.create("private"); - auto& pub1 = s1.create( - "public", kv::SecurityDomain::PUBLIC); + auto& priv1 = s1.create("private"); + auto& pub1 = + s1.create("public", kv::SecurityDomain::PUBLIC); kv::Store s2; s2.set_encryptor(encryptor); - auto& priv2 = s2.create("private"); - auto& pub2 = s2.create( - "public", kv::SecurityDomain::PUBLIC); + auto& priv2 = s2.create("private"); + auto& pub2 = + s2.create("public", kv::SecurityDomain::PUBLIC); INFO("Populate s1 with public entries"); // We compact twice, deliberately. A public KV during recovery @@ -976,12 +956,11 @@ TEST_CASE_TEMPLATE( } } -TEST_CASE_TEMPLATE( - "Conflict resolution", MapImpl, RawMapTypes, ExperimentalMapTypes) +TEST_CASE("Conflict resolution") { kv::Store kv_store; - auto& map = kv_store.create( - "map", kv::SecurityDomain::PUBLIC); + auto& map = + kv_store.create("map", kv::SecurityDomain::PUBLIC); auto try_write = [&](kv::Tx& tx, const std::string& s) { auto view = tx.get_view(map); diff --git a/src/kv/tx_view.h b/src/kv/tx_view.h index 2b85453410..8cccdd158d 100644 --- a/src/kv/tx_view.h +++ b/src/kv/tx_view.h @@ -2,266 +2,80 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "ds/champ_map.h" +#include "kv/untyped_map.h" +#include "kv/untyped_tx_view.h" #include "kv_types.h" -#include - namespace kv { - static bool is_deleted(Version version) - { - return version < 0; - } - - template - struct VersionV - { - Version version; - V value; - - VersionV() = default; - VersionV(Version ver, V val) : version(ver), value(val) {} - }; - - template - using State = champ::Map, H>; - - template - using Read = std::unordered_map; - - template > - using Write = - std::unordered_map, H>; //< nullopt indicates a deletion - - // This is a container for a write-set + dependencies. It can be applied to a - // given state, or used to track a set of operations on a state - template > - struct ChangeSet - { - public: - using State = State; - using Read = Read; - using Write = Write; - - State state; - State committed; - Version start_version; - - Version read_version = NoVersion; - Read reads = {}; - Write writes = {}; - - ChangeSet( - State& current_state, State& committed_state, Version current_version) : - state(current_state), - committed(committed_state), - start_version(current_version) - {} - - ChangeSet(ChangeSet&) = delete; - }; - - template > - class TxView + template + class TxView : public kv::untyped::Map::TxViewCommitter { protected: - using State = State; - - using ChangeSet = ChangeSet; - ChangeSet& tx_changes; + kv::untyped::TxView untyped_view; public: - // Expose these types so that other code can use them as MyTx::KeyType or - // MyMap::TxView::KeyType, templated on the TxView or Map type rather than - // explicitly on K and V using KeyType = K; using ValueType = V; - TxView(ChangeSet& cs) : tx_changes(cs) {} + TxView( + kv::untyped::Map& m, + size_t rollbacks, + kv::untyped::State& current_state, + kv::untyped::State& committed_state, + Version v) : + kv::untyped::Map::TxViewCommitter( + m, rollbacks, current_state, committed_state, v), + untyped_view(kv::untyped::Map::TxViewCommitter::change_set) + {} - /** Get value for key - * - * This returns the value for the key inside the transaction. If the key - * has been updated in the current transaction, that update will be - * reflected in the return of this call. - * - * @param key Key - * - * @return optional containing value, empty if the key doesn't exist - */ std::optional get(const K& key) { - // A write followed by a read doesn't introduce a read dependency. - // If we have written, return the value without updating the read set. - auto write = tx_changes.writes.find(key); - if (write != tx_changes.writes.end()) + const auto opt_v_rep = untyped_view.get(KSerialiser::to_serialised(key)); + + if (opt_v_rep.has_value()) { - // May be empty, for a key that has been removed. This matches the - // return semantics - return write->second; + return VSerialiser::from_serialised(*opt_v_rep); } - // If the key doesn't exist, return empty and record that we depend on - // the key not existing. - auto search = tx_changes.state.get(key); - if (!search.has_value()) - { - tx_changes.reads.insert(std::make_pair(key, NoVersion)); - return std::nullopt; - } - - // Record the version that we depend on. - auto& found = search.value(); - tx_changes.reads.insert(std::make_pair(key, found.version)); - - // If the key has been deleted, return empty. - if (is_deleted(found.version)) - { - return std::nullopt; - } - - // Return the value. - return found.value; + return std::nullopt; } - /** Get globally committed value for key - * - * This reads a globally replicated value for the specified key. - * The value will have been the replicated value when the transaction - * began, but the map may be compacted while the transaction is in - * flight. If that happens, there may be a more recent committed - * version. This is undetectable to the transaction. - * - * @param key Key - * - * @return optional containing value, empty if the key doesn't exist in - * globally committed state - */ std::optional get_globally_committed(const K& key) { - // If there is no committed value, return empty. - auto search = tx_changes.committed.get(key); - if (!search.has_value()) + const auto opt_v_rep = + untyped_view.get_globally_committed(KSerialiser::to_serialised(key)); + + if (opt_v_rep.has_value()) { - return std::nullopt; + return VSerialiser::from_serialised(*opt_v_rep); } - // If the key has been deleted, return empty. - auto& found = search.value(); - if (is_deleted(found.version)) - { - return std::nullopt; - } - - // Return the value. - return found.value; + return std::nullopt; } - /** Write value at key - * - * If the key already exists, the value will be replaced. - * This will fail if the transaction is already committed. - * - * @param key Key - * @param value Value - * - * @return true if successful, false otherwise - */ bool put(const K& key, const V& value) { - // Record in the write set. - tx_changes.writes[key] = value; - return true; + return untyped_view.put( + KSerialiser::to_serialised(key), VSerialiser::to_serialised(value)); } - /** Remove key - * - * This will fail if the key does not exist, or if the transaction - * is already committed. - * - * @param key Key - * - * @return true if successful, false otherwise - */ bool remove(const K& key) { - auto write = tx_changes.writes.find(key); - auto search = tx_changes.state.get(key).has_value(); - - if (write != tx_changes.writes.end()) - { - if (!search) - { - // this key only exists locally, there is no reason to maintain and - // serialise it - tx_changes.writes.erase(key); - } - else - { - // If we have written, change the write set to indicate a remove. - write->second = std::nullopt; - } - - return true; - } - - // If the key doesn't exist, return false. - if (!search) - { - return false; - } - - // Record in the write set. - tx_changes.writes.emplace( - std::piecewise_construct, - std::forward_as_tuple(key), - std::forward_as_tuple(std::nullopt)); - return true; + return untyped_view.remove(KSerialiser::to_serialised(key)); } - /** Iterate over all entries in the map - * - * @param F functor, taking a key and a value, return value determines - * whether the iteration should continue (true) or stop (false) - */ template void foreach(F&& f) { - // Record a global read dependency. - tx_changes.read_version = tx_changes.start_version; - auto& w = tx_changes.writes; - bool should_continue = true; - - tx_changes.state.foreach( - [&w, &f, &should_continue](const K& k, const VersionV& v) { - auto write = w.find(k); - - if ((write == w.end()) && !is_deleted(v.version)) - { - should_continue = f(k, v.value); - } - - return should_continue; - }); - - if (should_continue) - { - for (auto write = tx_changes.writes.begin(); - write != tx_changes.writes.end(); - ++write) - { - if (write->second.has_value()) - { - should_continue = f(write->first, write->second.value()); - } - - if (!should_continue) - { - break; - } - } - } + auto g = [&]( + const kv::serialisers::SerialisedEntry& k_rep, + const kv::serialisers::SerialisedEntry& v_rep) { + return f( + KSerialiser::from_serialised(k_rep), + VSerialiser::from_serialised(v_rep)); + }; + untyped_view.foreach(g); } }; } \ No newline at end of file diff --git a/src/kv/untyped_map.h b/src/kv/untyped_map.h new file mode 100644 index 0000000000..5d89decc68 --- /dev/null +++ b/src/kv/untyped_map.h @@ -0,0 +1,723 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "ds/dl_list.h" +#include "ds/logger.h" +#include "ds/spin_lock.h" +#include "kv/kv_serialiser.h" +#include "kv/kv_types.h" +#include "kv/untyped_tx_view.h" + +#include +#include +#include + +namespace kv::untyped +{ + namespace Check + { + struct No + {}; + + template + No operator!=(const T&, const Arg&) + { + return No(); + } + + template + struct Ne + { + enum + { + value = !std::is_same::value + }; + }; + + template + bool ne(std::enable_if_t::value, const T&> a, const T& b) + { + return a != b; + } + + template + bool ne(std::enable_if_t::value, const T&> a, const T& b) + { + return false; + } + } + + struct LocalCommit + { + LocalCommit() = default; + LocalCommit(Version v, State&& s, const Write& w) : + version(v), + state(std::move(s)), + writes(w) + {} + + Version version; + State state; + Write writes; + LocalCommit* next = nullptr; + LocalCommit* prev = nullptr; + }; + using LocalCommits = snmalloc::DLList; + + struct Roll + { + std::unique_ptr commits; + size_t rollback_counter; + }; + + class Map : public AbstractMap + { + public: + using K = kv::serialisers::SerialisedEntry; + using V = kv::serialisers::SerialisedEntry; + using H = SerialisedKeyHasher; + + using CommitHook = CommitHook; + + private: + AbstractStore* store; + std::string name; + Roll roll; + CommitHook local_hook = nullptr; + CommitHook global_hook = nullptr; + std::list> commit_deltas; + SpinLock sl; + const SecurityDomain security_domain; + const bool replicated; + + LocalCommits empty_commits; + + template + LocalCommit* create_new_local_commit(Args&&... args) + { + LocalCommit* c = empty_commits.pop(); + if (c == nullptr) + { + c = new LocalCommit(std::forward(args)...); + } + else + { + c->~LocalCommit(); + new (c) LocalCommit(std::forward(args)...); + } + return c; + } + + public: + class TxViewCommitter : public AbstractTxView + { + protected: + ChangeSet change_set; + + Map& map; + size_t rollback_counter; + + Version commit_version = NoVersion; + + bool changes = false; + bool committed_writes = false; + + public: + template + TxViewCommitter(Map& m, size_t rollbacks, Ts&&... ts) : + map(m), + rollback_counter(rollbacks), + change_set(std::forward(ts)...) + {} + + // Commit-related methods + bool has_writes() override + { + return committed_writes || !change_set.writes.empty(); + } + + bool has_changes() override + { + return changes; + } + + bool prepare() override + { + if (change_set.writes.empty()) + return true; + + auto& roll = map.get_roll(); + + // If the parent map has rolled back since this transaction began, this + // transaction must fail. + if (rollback_counter != roll.rollback_counter) + return false; + + // If we have iterated over the map, check for a global version match. + auto current = roll.commits->get_tail(); + + if ( + (change_set.read_version != NoVersion) && + (change_set.read_version != current->version)) + { + LOG_DEBUG_FMT("Read version {} is invalid", change_set.read_version); + return false; + } + + // Check each key in our read set. + for (auto it = change_set.reads.begin(); it != change_set.reads.end(); + ++it) + { + // Get the value from the current state. + auto search = current->state.get(it->first); + + if (it->second == NoVersion) + { + // If we depend on the key not existing, it must be absent. + if (search.has_value()) + { + LOG_DEBUG_FMT("Read depends on non-existing entry"); + return false; + } + } + else + { + // If we depend on the key existing, it must be present and have the + // version that we expect. + if (!search.has_value() || (it->second != search.value().version)) + { + LOG_DEBUG_FMT("Read depends on invalid version of entry"); + return false; + } + } + } + + return true; + } + + void commit(Version v) override + { + if (change_set.writes.empty()) + { + commit_version = change_set.start_version; + return; + } + + // Record our commit time. + commit_version = v; + committed_writes = true; + + if (!change_set.writes.empty()) + { + auto& roll = map.get_roll(); + auto state = roll.commits->get_tail()->state; + + for (auto it = change_set.writes.begin(); + it != change_set.writes.end(); + ++it) + { + if (it->second.has_value()) + { + // Write the new value with the global version. + changes = true; + state = state.put(it->first, VersionV{v, it->second.value()}); + } + else + { + // Write an empty value with the deleted global version only if + // the key exists. + auto search = state.get(it->first); + if (search.has_value()) + { + changes = true; + state = state.put(it->first, VersionV{-v, {}}); + } + } + } + + if (changes) + { + map.roll.commits->insert_back(map.create_new_local_commit( + v, std::move(state), change_set.writes)); + } + } + } + + void post_commit() override + { + // This is run separately from commit so that all commits in the Tx + // have been applied before local hooks are run. The maps in the Tx + // are still locked when post_commit is run. + if (change_set.writes.empty()) + return; + + map.trigger_local_hook(); + } + + // Used by owning map during serialise and deserialise + ChangeSet& get_change_set() + { + return change_set; + } + + const ChangeSet& get_change_set() const + { + return change_set; + } + + void set_commit_version(Version v) + { + commit_version = v; + } + }; + + struct ConcreteTxView : public TxViewCommitter, public TxView + { + ConcreteTxView( + Map& m, + size_t rollbacks, + State& current_state, + State& committed_state, + Version v) : + TxViewCommitter(m, rollbacks, current_state, committed_state, v), + TxView(TxViewCommitter::change_set) + {} + }; + + // Public typedef for external consumption + using TxView = ConcreteTxView; + + Map( + AbstractStore* store_, + std::string name_, + SecurityDomain security_domain_, + bool replicated_) : + store(store_), + name(name_), + roll{std::make_unique(), 0}, + security_domain(security_domain_), + replicated(replicated_) + { + roll.commits->insert_back(create_new_local_commit(0, State(), Write())); + } + + Map(const Map& that) = delete; + + virtual AbstractMap* clone(AbstractStore* other) override + { + return new Map(other, name, security_domain, replicated); + } + + void serialise( + const AbstractTxView* view, + KvStoreSerialiser& s, + bool include_reads) override + { + const auto committer = dynamic_cast(view); + if (committer == nullptr) + { + LOG_FAIL_FMT("Unable to serialise map due to type mismatch"); + return; + } + + const auto& change_set = committer->get_change_set(); + + s.start_map(name, security_domain); + + if (include_reads) + { + s.serialise_read_version(change_set.read_version); + + s.serialise_count_header(change_set.reads.size()); + for (auto it = change_set.reads.begin(); it != change_set.reads.end(); + ++it) + { + s.serialise_read(it->first, it->second); + } + } + else + { + s.serialise_read_version(NoVersion); + s.serialise_count_header(0); + } + + uint64_t write_ctr = 0; + uint64_t remove_ctr = 0; + for (auto it = change_set.writes.begin(); it != change_set.writes.end(); + ++it) + { + if (it->second.has_value()) + { + ++write_ctr; + } + else + { + auto search = roll.commits->get_tail()->state.get(it->first); + if (search.has_value()) + { + ++remove_ctr; + } + } + } + + s.serialise_count_header(write_ctr); + for (auto it = change_set.writes.begin(); it != change_set.writes.end(); + ++it) + { + if (it->second.has_value()) + { + s.serialise_write(it->first, it->second.value()); + } + } + + s.serialise_count_header(remove_ctr); + for (auto it = change_set.writes.begin(); it != change_set.writes.end(); + ++it) + { + if (!it->second.has_value()) + { + s.serialise_remove(it->first); + } + } + } + + AbstractTxView* deserialise( + KvStoreDeserialiser& d, Version version) override + { + return deserialise_internal(d, version); + } + + template + TView* deserialise_internal(KvStoreDeserialiser& d, Version version) + { + // Create a new change set, and deserialise d's contents into it. + auto view = create_view(version); + view->set_commit_version(version); + + auto& change_set = view->get_change_set(); + + uint64_t ctr; + + auto rv = d.deserialise_read_version(); + if (rv != NoVersion) + { + change_set.read_version = rv; + } + + ctr = d.deserialise_read_header(); + for (size_t i = 0; i < ctr; ++i) + { + auto r = d.deserialise_read(); + change_set.reads[std::get<0>(r)] = std::get<1>(r); + } + + ctr = d.deserialise_write_header(); + for (size_t i = 0; i < ctr; ++i) + { + auto w = d.deserialise_write(); + change_set.writes[std::get<0>(w)] = std::get<1>(w); + } + + ctr = d.deserialise_remove_header(); + for (size_t i = 0; i < ctr; ++i) + { + auto r = d.deserialise_remove(); + change_set.writes[r] = std::nullopt; + } + + return view; + } + + /** Get the name of the map + * + * @return const std::string& + */ + const std::string& get_name() const override + { + return name; + } + + /** Get store that the map belongs to + * + * @return Pointer to `kv::AbstractStore` + */ + AbstractStore* get_store() override + { + return store; + } + + /** Set handler to be called on local transaction commit + * + * @param hook function to be called on local transaction commit + */ + void set_local_hook(const CommitHook& hook) + { + std::lock_guard guard(sl); + local_hook = hook; + } + + /** Reset local transaction commit handler + */ + void unset_local_hook() + { + std::lock_guard guard(sl); + local_hook = nullptr; + } + + /** Set handler to be called on global transaction commit + * + * @param hook function to be called on global transaction commit + */ + void set_global_hook(const CommitHook& hook) + { + std::lock_guard guard(sl); + global_hook = hook; + } + + /** Reset global transaction commit handler + */ + void unset_global_hook() + { + std::lock_guard guard(sl); + global_hook = nullptr; + } + + /** Get security domain of a Map + * + * @return Security domain of the map (affects serialisation) + */ + virtual SecurityDomain get_security_domain() override + { + return security_domain; + } + + /** Get Map replicability + * + * @return true if the map is to be replicated, false if it is to be derived + */ + virtual bool is_replicated() override + { + return replicated; + } + + bool operator==(const AbstractMap& that) const override + { + auto p = dynamic_cast(&that); + if (p == nullptr) + return false; + + if (name != p->name) + return false; + + auto state1 = roll.commits->get_tail(); + auto state2 = p->roll.commits->get_tail(); + + if (state1->version != state2->version) + return false; + + size_t count = 0; + state2->state.foreach([&count](const K& k, const VersionV& v) { + count++; + return true; + }); + + size_t i = 0; + bool ok = + state1->state.foreach([&state2, &i](const K& k, const VersionV& v) { + auto search = state2->state.get(k); + + if (search.has_value()) + { + auto& found = search.value(); + if (found.version != v.version) + { + return false; + } + else if (Check::ne(found.value, v.value)) + { + return false; + } + } + else + { + return false; + } + + i++; + return true; + }); + + if (i != count) + ok = false; + + return ok; + } + + bool operator!=(const AbstractMap& that) const override + { + return !(*this == that); + } + + void compact(Version v) override + { + // This discards available rollback state before version v, and populates + // the commit_deltas to be passed to the global commit hook, if there is + // one, up to version v. The Map expects to be locked during compaction. + while (roll.commits->get_head() != roll.commits->get_tail()) + { + auto r = roll.commits->get_head(); + + // Globally committed but not discardable. + if (r->version == v) + { + // We know that write set is not empty. + if (global_hook) + { + commit_deltas.emplace_back(r->version, std::move(r->writes)); + } + return; + } + + // Discardable, so move to commit_deltas. + if (global_hook && !r->writes.empty()) + { + commit_deltas.emplace_back(r->version, std::move(r->writes)); + } + + // Stop if the next state may be rolled back or is the only state. + // This ensures there is always a state present. + if (r->next->version > v) + return; + + auto c = roll.commits->pop(); + empty_commits.insert(c); + } + + // There is only one roll. We may need to call the commit hook. + auto r = roll.commits->get_head(); + + if (global_hook && !r->writes.empty()) + { + commit_deltas.emplace_back(r->version, std::move(r->writes)); + } + } + + void post_compact() override + { + if (global_hook) + { + for (auto& [version, writes] : commit_deltas) + { + global_hook(version, writes); + } + } + + commit_deltas.clear(); + } + + void rollback(Version v) override + { + // This rolls the current state back to version v. + // The Map expects to be locked during rollback. + bool advance = false; + + while (roll.commits->get_head() != roll.commits->get_tail()) + { + auto r = roll.commits->get_tail(); + + // The initial empty state has v = 0, so will not be discarded if it + // is present. + if (r->version <= v) + break; + + advance = true; + auto c = roll.commits->pop_tail(); + empty_commits.insert(c); + } + + if (advance) + roll.rollback_counter++; + } + + void clear() override + { + // This discards all entries in the roll and resets the compacted value + // and rollback counter. The Map expects to be locked before clearing it. + roll.commits->clear(); + roll.commits->insert_back(create_new_local_commit(0, State(), Write())); + roll.rollback_counter = 0; + } + + void lock() override + { + sl.lock(); + } + + void unlock() override + { + sl.unlock(); + } + + void swap(AbstractMap* map_) override + { + Map* map = dynamic_cast(map_); + if (map == nullptr) + throw std::logic_error( + "Attempted to swap maps with incompatible types"); + + std::swap(roll, map->roll); + } + + template + TView* create_view(Version version) + { + lock(); + + // Find the last entry committed at or before this version. + TView* view = nullptr; + + for (auto current = roll.commits->get_tail(); current != nullptr; + current = current->prev) + { + if (current->version <= version) + { + view = new TView( + *this, + roll.rollback_counter, + current->state, + roll.commits->get_head()->state, + current->version); + break; + } + } + + if (view == nullptr) + { + view = new TView( + *this, + roll.rollback_counter, + roll.commits->get_head()->state, + roll.commits->get_head()->state, + roll.commits->get_head()->version); + } + + unlock(); + return view; + } + + Roll& get_roll() + { + return roll; + } + + void trigger_local_hook() + { + if (local_hook) + { + auto last_commit = roll.commits->get_tail(); + local_hook(last_commit->version, last_commit->writes); + } + } + }; +} \ No newline at end of file diff --git a/src/kv/untyped_tx_view.h b/src/kv/untyped_tx_view.h new file mode 100644 index 0000000000..a9e7368db0 --- /dev/null +++ b/src/kv/untyped_tx_view.h @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "kv/change_set.h" +#include "kv/kv_types.h" +#include "kv/serialised_entry.h" + +namespace kv::untyped +{ + using SerialisedEntry = kv::serialisers::SerialisedEntry; + using SerialisedKeyHasher = std::hash; + + using VersionV = kv::VersionV; + using State = + kv::State; + using Read = kv::Read; + using Write = kv::Write; + using ChangeSet = + kv::ChangeSet; + + class TxView + { + protected: + ChangeSet& tx_changes; + + public: + // Expose these types so that other code can use them as MyTx::KeyType or + // MyMap::TxView::KeyType, templated on the TxView or Map type + using KeyType = SerialisedEntry; + using ValueType = SerialisedEntry; + + TxView(ChangeSet& cs) : tx_changes(cs) {} + + /** Get value for key + * + * This returns the value for the key inside the transaction. If the key + * has been updated in the current transaction, that update will be + * reflected in the return of this call. + * + * @param key Key + * + * @return optional containing value, empty if the key doesn't exist + */ + std::optional get(const KeyType& key) + { + // A write followed by a read doesn't introduce a read dependency. + // If we have written, return the value without updating the read set. + auto write = tx_changes.writes.find(key); + if (write != tx_changes.writes.end()) + { + // May be empty, for a key that has been removed. This matches the + // return semantics + return write->second; + } + + // If the key doesn't exist, return empty and record that we depend on + // the key not existing. + auto search = tx_changes.state.get(key); + if (!search.has_value()) + { + tx_changes.reads.insert(std::make_pair(key, NoVersion)); + return std::nullopt; + } + + // Record the version that we depend on. + auto& found = search.value(); + tx_changes.reads.insert(std::make_pair(key, found.version)); + + // If the key has been deleted, return empty. + if (is_deleted(found.version)) + { + return std::nullopt; + } + + // Return the value. + return found.value; + } + + /** Get globally committed value for key + * + * This reads a globally replicated value for the specified key. + * The value will have been the replicated value when the transaction + * began, but the map may be compacted while the transaction is in + * flight. If that happens, there may be a more recent committed + * version. This is undetectable to the transaction. + * + * @param key Key + * + * @return optional containing value, empty if the key doesn't exist in + * globally committed state + */ + std::optional get_globally_committed(const KeyType& key) + { + // If there is no committed value, return empty. + auto search = tx_changes.committed.get(key); + if (!search.has_value()) + { + return std::nullopt; + } + + // If the key has been deleted, return empty. + auto& found = search.value(); + if (is_deleted(found.version)) + { + return std::nullopt; + } + + // Return the value. + return found.value; + } + + /** Write value at key + * + * If the key already exists, the value will be replaced. + * This will fail if the transaction is already committed. + * + * @param key Key + * @param value Value + * + * @return true if successful, false otherwise + */ + bool put(const KeyType& key, const ValueType& value) + { + // Record in the write set. + tx_changes.writes[key] = value; + return true; + } + + /** Remove key + * + * This will fail if the key does not exist, or if the transaction + * is already committed. + * + * @param key Key + * + * @return true if successful, false otherwise + */ + bool remove(const KeyType& key) + { + auto write = tx_changes.writes.find(key); + auto search = tx_changes.state.get(key).has_value(); + + if (write != tx_changes.writes.end()) + { + if (!search) + { + // this key only exists locally, there is no reason to maintain and + // serialise it + tx_changes.writes.erase(key); + } + else + { + // If we have written, change the write set to indicate a remove. + write->second = std::nullopt; + } + + return true; + } + + // If the key doesn't exist, return false. + if (!search) + { + return false; + } + + // Record in the write set. + tx_changes.writes[key] = std::nullopt; + return true; + } + + /** Iterate over all entries in the map + * + * @param F functor, taking a key and a value, return value determines + * whether the iteration should continue (true) or stop (false) + */ + template + void foreach(F&& f) + { + // Record a global read dependency. + tx_changes.read_version = tx_changes.start_version; + auto& w = tx_changes.writes; + bool should_continue = true; + + tx_changes.state.foreach( + [&w, &f, &should_continue](const KeyType& k, const VersionV& v) { + auto write = w.find(k); + + if ((write == w.end()) && !is_deleted(v.version)) + { + should_continue = f(k, v.value); + } + + return should_continue; + }); + + if (should_continue) + { + for (auto write = tx_changes.writes.begin(); + write != tx_changes.writes.end(); + ++write) + { + if (write->second.has_value()) + { + should_continue = f(write->first, write->second.value()); + } + + if (!should_continue) + { + break; + } + } + } + } + }; +} \ No newline at end of file diff --git a/src/node/scripts.h b/src/node/scripts.h index 8efbfbb79b..8a659ff917 100644 --- a/src/node/scripts.h +++ b/src/node/scripts.h @@ -2,16 +2,11 @@ // Licensed under the Apache 2.0 License. #pragma once -#include "kv/experimental.h" #include "script.h" namespace ccf { - using Scripts = kv::experimental::Map< - std::string, - Script, - kv::experimental::JsonSerialiser, - kv::experimental::JsonSerialiser