Use early-serialising KV implementation by default (take 2) (#1234)

3rdparty/small_vector/LICENSE.TXT

@@ -0,0 +1,43 @@
+==============================================================================
+LLVM Release License
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2003-2016 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.

3rdparty/small_vector/README.md

@@ -0,0 +1,3 @@
+# SmallVector
+
+This is [llvm::SmallVector](http://llvm.org/docs/doxygen/html/classllvm_1_1SmallVector.html) stripped from any LLVM dependency.

3rdparty/small_vector/SmallVector.h

@@ -0,0 +1,983 @@
+//===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the SmallVector class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_VECSMALL_ADT_SMALLVECTOR_H
+#define LLVM_VECSMALL_ADT_SMALLVECTOR_H
+
+#include <algorithm>
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <initializer_list>
+#include <iterator>
+#include <memory>
+
+// LLVM Macros
+#define LLVM_VECSMALL_NODISCARD
+#define LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE inline
+#define LLVM_VECSMALL_UNLIKELY(x) (x)
+
+// LLVM External Functions
+namespace llvm_vecsmall {
+    namespace detail {
+        /// NextPowerOf2 - Returns the next power of two (in 64-bits)
+        /// that is strictly greater than A.  Returns zero on overflow.
+        inline uint64_t NextPowerOf2(uint64_t A) {
+            A |= (A >> 1);
+            A |= (A >> 2);
+            A |= (A >> 4);
+            A |= (A >> 8);
+            A |= (A >> 16);
+            A |= (A >> 32);
+            return A + 1;
+        }
+    }
+}
+
+
+namespace llvm_vecsmall {
+
+// std::is_pod has been deprecated in C++20.
+template <typename T>
+struct IsPod : std::integral_constant<bool, std::is_standard_layout<T>::value &&
+                                                std::is_trivial<T>::value> {};
+
+/// This is all the non-templated stuff common to all SmallVectors.
+class SmallVectorBase {
+protected:
+  void *BeginX, *EndX, *CapacityX;
+
+protected:
+  SmallVectorBase(void *FirstEl, size_t Size)
+    : BeginX(FirstEl), EndX(FirstEl), CapacityX((char*)FirstEl+Size) {}
+
+  /// This is an implementation of the grow() method which only works
+  /// on POD-like data types and is out of line to reduce code duplication.
+  void grow_pod(void *FirstEl, size_t MinSizeInBytes, size_t TSize);
+
+public:
+  /// This returns size()*sizeof(T).
+  size_t size_in_bytes() const {
+    return size_t((char*)EndX - (char*)BeginX);
+  }
+
+  /// capacity_in_bytes - This returns capacity()*sizeof(T).
+  size_t capacity_in_bytes() const {
+    return size_t((char*)CapacityX - (char*)BeginX);
+  }
+
+  LLVM_VECSMALL_NODISCARD bool empty() const { return BeginX == EndX; }
+};
+
+template <typename T, unsigned N> struct SmallVectorStorage;
+
+/// This is the part of SmallVectorTemplateBase which does not depend on whether
+/// the type T is a POD. The extra dummy template argument is used by ArrayRef
+/// to avoid unnecessarily requiring T to be complete.
+template <typename T, typename = void>
+class SmallVectorTemplateCommon : public SmallVectorBase {
+private:
+  template <typename, unsigned> friend struct SmallVectorStorage;
+
+  // Allocate raw space for N elements of type T.  If T has a ctor or dtor, we
+  // don't want it to be automatically run, so we need to represent the space as
+  // something else.  Use an array of char of sufficient alignment.
+  ////////////typedef llvm_vecsmall::AlignedCharArrayUnion<T> U;
+  typedef typename std::aligned_union<1, T>::type U;
+  U FirstEl;
+  // Space after 'FirstEl' is clobbered, do not add any instance vars after it.
+
+protected:
+  SmallVectorTemplateCommon(size_t Size) : SmallVectorBase(&FirstEl, Size) {}
+
+  void grow_pod(size_t MinSizeInBytes, size_t TSize) {
+    SmallVectorBase::grow_pod(&FirstEl, MinSizeInBytes, TSize);
+  }
+
+  /// Return true if this is a smallvector which has not had dynamic
+  /// memory allocated for it.
+  bool isSmall() const {
+    return BeginX == static_cast<const void*>(&FirstEl);
+  }
+
+  /// Put this vector in a state of being small.
+  void resetToSmall() {
+    BeginX = EndX = CapacityX = &FirstEl;
+  }
+
+  void setEnd(T *P) { this->EndX = P; }
+public:
+  typedef size_t size_type;
+  typedef ptrdiff_t difference_type;
+  typedef T value_type;
+  typedef T *iterator;
+  typedef const T *const_iterator;
+
+  typedef std::reverse_iterator<const_iterator> const_reverse_iterator;
+  typedef std::reverse_iterator<iterator> reverse_iterator;
+
+  typedef T &reference;
+  typedef const T &const_reference;
+  typedef T *pointer;
+  typedef const T *const_pointer;
+
+  // forward iterator creation methods.
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  iterator begin() { return (iterator)this->BeginX; }
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  const_iterator begin() const { return (const_iterator)this->BeginX; }
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  iterator end() { return (iterator)this->EndX; }
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  const_iterator end() const { return (const_iterator)this->EndX; }
+protected:
+  iterator capacity_ptr() { return (iterator)this->CapacityX; }
+  const_iterator capacity_ptr() const { return (const_iterator)this->CapacityX;}
+public:
+
+  // reverse iterator creation methods.
+  reverse_iterator rbegin()            { return reverse_iterator(end()); }
+  const_reverse_iterator rbegin() const{ return const_reverse_iterator(end()); }
+  reverse_iterator rend()              { return reverse_iterator(begin()); }
+  const_reverse_iterator rend() const { return const_reverse_iterator(begin());}
+
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  size_type size() const { return end()-begin(); }
+  size_type max_size() const { return size_type(-1) / sizeof(T); }
+
+  /// Return the total number of elements in the currently allocated buffer.
+  size_t capacity() const { return capacity_ptr() - begin(); }
+
+  /// Return a pointer to the vector's buffer, even if empty().
+  pointer data() { return pointer(begin()); }
+  /// Return a pointer to the vector's buffer, even if empty().
+  const_pointer data() const { return const_pointer(begin()); }
+
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  reference operator[](size_type idx) {
+    assert(idx < size());
+    return begin()[idx];
+  }
+  LLVM_VECSMALL_ATTRIBUTE_ALWAYS_INLINE
+  const_reference operator[](size_type idx) const {
+    assert(idx < size());
+    return begin()[idx];
+  }
+
+  reference front() {
+    assert(!empty());
+    return begin()[0];
+  }
+  const_reference front() const {
+    assert(!empty());
+    return begin()[0];
+  }
+
+  reference back() {
+    assert(!empty());
+    return end()[-1];
+  }
+  const_reference back() const {
+    assert(!empty());
+    return end()[-1];
+  }
+};
+
+/// SmallVectorTemplateBase<isPodLike = false> - This is where we put method
+/// implementations that are designed to work with non-POD-like T's.
+template <typename T, bool isPodLike>
+class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> {
+protected:
+  SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {}
+
+  static void destroy_range(T *S, T *E) {
+    while (S != E) {
+      --E;
+      E->~T();
+    }
+  }
+
+  /// Move the range [I, E) into the uninitialized memory starting with "Dest",
+  /// constructing elements as needed.
+  template<typename It1, typename It2>
+  static void uninitialized_move(It1 I, It1 E, It2 Dest) {
+    std::uninitialized_copy(std::make_move_iterator(I),
+                            std::make_move_iterator(E), Dest);
+  }
+
+  /// Copy the range [I, E) onto the uninitialized memory starting with "Dest",
+  /// constructing elements as needed.
+  template<typename It1, typename It2>
+  static void uninitialized_copy(It1 I, It1 E, It2 Dest) {
+    std::uninitialized_copy(I, E, Dest);
+  }
+
+  /// Grow the allocated memory (without initializing new elements), doubling
+  /// the size of the allocated memory. Guarantees space for at least one more
+  /// element, or MinSize more elements if specified.
+  void grow(size_t MinSize = 0);
+
+public:
+  void push_back(const T &Elt) {
+    if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX))
+      this->grow();
+    ::new ((void*) this->end()) T(Elt);
+    this->setEnd(this->end()+1);
+  }
+
+  void push_back(T &&Elt) {
+    if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX))
+      this->grow();
+    ::new ((void*) this->end()) T(::std::move(Elt));
+    this->setEnd(this->end()+1);
+  }
+
+  void pop_back() {
+    this->setEnd(this->end()-1);
+    this->end()->~T();
+  }
+};
+
+// Define this out-of-line to dissuade the C++ compiler from inlining it.
+template <typename T, bool isPodLike>
+void SmallVectorTemplateBase<T, isPodLike>::grow(size_t MinSize) {
+  size_t CurCapacity = this->capacity();
+  size_t CurSize = this->size();
+  // Always grow, even from zero.
+  size_t NewCapacity = size_t(llvm_vecsmall::detail::NextPowerOf2(CurCapacity+2));
+  if (NewCapacity < MinSize)
+    NewCapacity = MinSize;
+  T *NewElts = static_cast<T*>(malloc(NewCapacity*sizeof(T)));
+
+  // Move the elements over.
+  this->uninitialized_move(this->begin(), this->end(), NewElts);
+
+  // Destroy the original elements.
+  destroy_range(this->begin(), this->end());
+
+  // If this wasn't grown from the inline copy, deallocate the old space.
+  if (!this->isSmall())
+    free(this->begin());
+
+  this->setEnd(NewElts+CurSize);
+  this->BeginX = NewElts;
+  this->CapacityX = this->begin()+NewCapacity;
+}
+
+
+/// SmallVectorTemplateBase<isPodLike = true> - This is where we put method
+/// implementations that are designed to work with POD-like T's.
+template <typename T>
+class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> {
+protected:
+  SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {}
+
+  // No need to do a destroy loop for POD's.
+  static void destroy_range(T *, T *) {}
+
+  /// Move the range [I, E) onto the uninitialized memory
+  /// starting with "Dest", constructing elements into it as needed.
+  template<typename It1, typename It2>
+  static void uninitialized_move(It1 I, It1 E, It2 Dest) {
+    // Just do a copy.
+    uninitialized_copy(I, E, Dest);
+  }
+
+  /// Copy the range [I, E) onto the uninitialized memory
+  /// starting with "Dest", constructing elements into it as needed.
+  template<typename It1, typename It2>
+  static void uninitialized_copy(It1 I, It1 E, It2 Dest) {
+    // Arbitrary iterator types; just use the basic implementation.
+    std::uninitialized_copy(I, E, Dest);
+  }
+
+  /// Copy the range [I, E) onto the uninitialized memory
+  /// starting with "Dest", constructing elements into it as needed.
+  template <typename T1, typename T2>
+  static void uninitialized_copy(
+      T1 *I, T1 *E, T2 *Dest,
+      typename std::enable_if<std::is_same<typename std::remove_const<T1>::type,
+                                           T2>::value>::type * = nullptr) {
+    // Use memcpy for PODs iterated by pointers (which includes SmallVector
+    // iterators): std::uninitialized_copy optimizes to memmove, but we can
+    // use memcpy here. Note that I and E are iterators and thus might be
+    // invalid for memcpy if they are equal.
+    if (I != E)
+      memcpy(Dest, I, (E - I) * sizeof(T));
+  }
+
+  /// Double the size of the allocated memory, guaranteeing space for at
+  /// least one more element or MinSize if specified.
+  void grow(size_t MinSize = 0) {
+    this->grow_pod(MinSize*sizeof(T), sizeof(T));
+  }
+public:
+  void push_back(const T &Elt) {
+    if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX))
+      this->grow();
+    memcpy(this->end(), &Elt, sizeof(T));
+    this->setEnd(this->end()+1);
+  }
+
+  void pop_back() {
+    this->setEnd(this->end()-1);
+  }
+};
+
+
+/// This class consists of common code factored out of the SmallVector class to
+/// reduce code duplication based on the SmallVector 'N' template parameter.
+template <typename T>
+class SmallVectorImpl : public SmallVectorTemplateBase<T, IsPod<T>::value> {
+  typedef SmallVectorTemplateBase<T, IsPod<T>::value> SuperClass;
+
+  SmallVectorImpl(const SmallVectorImpl&) = delete;
+
+public:
+  typedef typename SuperClass::iterator iterator;
+  typedef typename SuperClass::const_iterator const_iterator;
+  typedef typename SuperClass::size_type size_type;
+
+protected:
+  // Default ctor - Initialize to empty.
+  explicit SmallVectorImpl(unsigned N)
+    : SmallVectorTemplateBase<T, IsPod<T>::value>(N*sizeof(T)) {
+  }
+
+public:
+  ~SmallVectorImpl() {
+    // Destroy the constructed elements in the vector.
+    this->destroy_range(this->begin(), this->end());
+
+    // If this wasn't grown from the inline copy, deallocate the old space.
+    if (!this->isSmall())
+      free(this->begin());
+  }
+
+
+  void clear() {
+    this->destroy_range(this->begin(), this->end());
+    this->EndX = this->BeginX;
+  }
+
+  void resize(size_type N) {
+    if (N < this->size()) {
+      this->destroy_range(this->begin()+N, this->end());
+      this->setEnd(this->begin()+N);
+    } else if (N > this->size()) {
+      if (this->capacity() < N)
+        this->grow(N);
+      for (auto I = this->end(), E = this->begin() + N; I != E; ++I)
+        new (&*I) T();
+      this->setEnd(this->begin()+N);
+    }
+  }
+
+  void resize(size_type N, const T &NV) {
+    if (N < this->size()) {
+      this->destroy_range(this->begin()+N, this->end());
+      this->setEnd(this->begin()+N);
+    } else if (N > this->size()) {
+      if (this->capacity() < N)
+        this->grow(N);
+      std::uninitialized_fill(this->end(), this->begin()+N, NV);
+      this->setEnd(this->begin()+N);
+    }
+  }
+
+  void reserve(size_type N) {
+    if (this->capacity() < N)
+      this->grow(N);
+  }
+
+  LLVM_VECSMALL_NODISCARD T pop_back_val() {
+    T Result = ::std::move(this->back());
+    this->pop_back();
+    return Result;
+  }
+
+  void swap(SmallVectorImpl &RHS);
+
+  /// Add the specified range to the end of the SmallVector.
+  template<typename in_iter>
+  void append(in_iter in_start, in_iter in_end) {
+    size_type NumInputs = std::distance(in_start, in_end);
+    // Grow allocated space if needed.
+    if (NumInputs > size_type(this->capacity_ptr()-this->end()))
+      this->grow(this->size()+NumInputs);
+
+    // Copy the new elements over.
+    this->uninitialized_copy(in_start, in_end, this->end());
+    this->setEnd(this->end() + NumInputs);
+  }
+
+  /// Add the specified range to the end of the SmallVector.
+  void append(size_type NumInputs, const T &Elt) {
+    // Grow allocated space if needed.
+    if (NumInputs > size_type(this->capacity_ptr()-this->end()))
+      this->grow(this->size()+NumInputs);
+
+    // Copy the new elements over.
+    std::uninitialized_fill_n(this->end(), NumInputs, Elt);
+    this->setEnd(this->end() + NumInputs);
+  }
+
+  void append(std::initializer_list<T> IL) {
+    append(IL.begin(), IL.end());
+  }
+
+  void assign(size_type NumElts, const T &Elt) {
+    clear();
+    if (this->capacity() < NumElts)
+      this->grow(NumElts);
+    this->setEnd(this->begin()+NumElts);
+    std::uninitialized_fill(this->begin(), this->end(), Elt);
+  }
+
+  void assign(std::initializer_list<T> IL) {
+    clear();
+    append(IL);
+  }
+
+  iterator erase(const_iterator CI) {
+    // Just cast away constness because this is a non-const member function.
+    iterator I = const_cast<iterator>(CI);
+
+    assert(I >= this->begin() && "Iterator to erase is out of bounds.");
+    assert(I < this->end() && "Erasing at past-the-end iterator.");
+
+    iterator N = I;
+    // Shift all elts down one.
+    std::move(I+1, this->end(), I);
+    // Drop the last elt.
+    this->pop_back();
+    return(N);
+  }
+
+  iterator erase(const_iterator CS, const_iterator CE) {
+    // Just cast away constness because this is a non-const member function.
+    iterator S = const_cast<iterator>(CS);
+    iterator E = const_cast<iterator>(CE);
+
+    assert(S >= this->begin() && "Range to erase is out of bounds.");
+    assert(S <= E && "Trying to erase invalid range.");
+    assert(E <= this->end() && "Trying to erase past the end.");
+
+    iterator N = S;
+    // Shift all elts down.
+    iterator I = std::move(E, this->end(), S);
+    // Drop the last elts.
+    this->destroy_range(I, this->end());
+    this->setEnd(I);
+    return(N);
+  }
+
+  iterator insert(iterator I, T &&Elt) {
+    if (I == this->end()) {  // Important special case for empty vector.
+      this->push_back(::std::move(Elt));
+      return this->end()-1;
+    }
+
+    assert(I >= this->begin() && "Insertion iterator is out of bounds.");
+    assert(I <= this->end() && "Inserting past the end of the vector.");
+
+    if (this->EndX >= this->CapacityX) {
+      size_t EltNo = I-this->begin();
+      this->grow();
+      I = this->begin()+EltNo;
+    }
+
+    ::new ((void*) this->end()) T(::std::move(this->back()));
+    // Push everything else over.
+    std::move_backward(I, this->end()-1, this->end());
+    this->setEnd(this->end()+1);
+
+    // If we just moved the element we're inserting, be sure to update
+    // the reference.
+    T *EltPtr = &Elt;
+    if (I <= EltPtr && EltPtr < this->EndX)
+      ++EltPtr;
+
+    *I = ::std::move(*EltPtr);
+    return I;
+  }
+
+  iterator insert(iterator I, const T &Elt) {
+    if (I == this->end()) {  // Important special case for empty vector.
+      this->push_back(Elt);
+      return this->end()-1;
+    }
+
+    assert(I >= this->begin() && "Insertion iterator is out of bounds.");
+    assert(I <= this->end() && "Inserting past the end of the vector.");
+
+    if (this->EndX >= this->CapacityX) {
+      size_t EltNo = I-this->begin();
+      this->grow();
+      I = this->begin()+EltNo;
+    }
+    ::new ((void*) this->end()) T(std::move(this->back()));
+    // Push everything else over.
+    std::move_backward(I, this->end()-1, this->end());
+    this->setEnd(this->end()+1);
+
+    // If we just moved the element we're inserting, be sure to update
+    // the reference.
+    const T *EltPtr = &Elt;
+    if (I <= EltPtr && EltPtr < this->EndX)
+      ++EltPtr;
+
+    *I = *EltPtr;
+    return I;
+  }
+
+  iterator insert(iterator I, size_type NumToInsert, const T &Elt) {
+    // Convert iterator to elt# to avoid invalidating iterator when we reserve()
+    size_t InsertElt = I - this->begin();
+
+    if (I == this->end()) {  // Important special case for empty vector.
+      append(NumToInsert, Elt);
+      return this->begin()+InsertElt;
+    }
+
+    assert(I >= this->begin() && "Insertion iterator is out of bounds.");
+    assert(I <= this->end() && "Inserting past the end of the vector.");
+
+    // Ensure there is enough space.
+    reserve(this->size() + NumToInsert);
+
+    // Uninvalidate the iterator.
+    I = this->begin()+InsertElt;
+
+    // If there are more elements between the insertion point and the end of the
+    // range than there are being inserted, we can use a simple approach to
+    // insertion.  Since we already reserved space, we know that this won't
+    // reallocate the vector.
+    if (size_t(this->end()-I) >= NumToInsert) {
+      T *OldEnd = this->end();
+      append(std::move_iterator<iterator>(this->end() - NumToInsert),
+             std::move_iterator<iterator>(this->end()));
+
+      // Copy the existing elements that get replaced.
+      std::move_backward(I, OldEnd-NumToInsert, OldEnd);
+
+      std::fill_n(I, NumToInsert, Elt);
+      return I;
+    }
+
+    // Otherwise, we're inserting more elements than exist already, and we're
+    // not inserting at the end.
+
+    // Move over the elements that we're about to overwrite.
+    T *OldEnd = this->end();
+    this->setEnd(this->end() + NumToInsert);
+    size_t NumOverwritten = OldEnd-I;
+    this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten);
+
+    // Replace the overwritten part.
+    std::fill_n(I, NumOverwritten, Elt);
+
+    // Insert the non-overwritten middle part.
+    std::uninitialized_fill_n(OldEnd, NumToInsert-NumOverwritten, Elt);
+    return I;
+  }
+
+  template<typename ItTy>
+  iterator insert(iterator I, ItTy From, ItTy To) {
+    // Convert iterator to elt# to avoid invalidating iterator when we reserve()
+    size_t InsertElt = I - this->begin();
+
+    if (I == this->end()) {  // Important special case for empty vector.
+      append(From, To);
+      return this->begin()+InsertElt;
+    }
+
+    assert(I >= this->begin() && "Insertion iterator is out of bounds.");
+    assert(I <= this->end() && "Inserting past the end of the vector.");
+
+    size_t NumToInsert = std::distance(From, To);
+
+    // Ensure there is enough space.
+    reserve(this->size() + NumToInsert);
+
+    // Uninvalidate the iterator.
+    I = this->begin()+InsertElt;
+
+    // If there are more elements between the insertion point and the end of the
+    // range than there are being inserted, we can use a simple approach to
+    // insertion.  Since we already reserved space, we know that this won't
+    // reallocate the vector.
+    if (size_t(this->end()-I) >= NumToInsert) {
+      T *OldEnd = this->end();
+      append(std::move_iterator<iterator>(this->end() - NumToInsert),
+             std::move_iterator<iterator>(this->end()));
+
+      // Copy the existing elements that get replaced.
+      std::move_backward(I, OldEnd-NumToInsert, OldEnd);
+
+      std::copy(From, To, I);
+      return I;
+    }
+
+    // Otherwise, we're inserting more elements than exist already, and we're
+    // not inserting at the end.
+
+    // Move over the elements that we're about to overwrite.
+    T *OldEnd = this->end();
+    this->setEnd(this->end() + NumToInsert);
+    size_t NumOverwritten = OldEnd-I;
+    this->uninitialized_move(I, OldEnd, this->end()-NumOverwritten);
+
+    // Replace the overwritten part.
+    for (T *J = I; NumOverwritten > 0; --NumOverwritten) {
+      *J = *From;
+      ++J; ++From;
+    }
+
+    // Insert the non-overwritten middle part.
+    this->uninitialized_copy(From, To, OldEnd);
+    return I;
+  }
+
+  void insert(iterator I, std::initializer_list<T> IL) {
+    insert(I, IL.begin(), IL.end());
+  }
+
+  template <typename... ArgTypes> void emplace_back(ArgTypes &&... Args) {
+    if (LLVM_VECSMALL_UNLIKELY(this->EndX >= this->CapacityX))
+      this->grow();
+    ::new ((void *)this->end()) T(std::forward<ArgTypes>(Args)...);
+    this->setEnd(this->end() + 1);
+  }
+
+  SmallVectorImpl &operator=(const SmallVectorImpl &RHS);
+
+  SmallVectorImpl &operator=(SmallVectorImpl &&RHS);
+
+  bool operator==(const SmallVectorImpl &RHS) const {
+    if (this->size() != RHS.size()) return false;
+    return std::equal(this->begin(), this->end(), RHS.begin());
+  }
+  bool operator!=(const SmallVectorImpl &RHS) const {
+    return !(*this == RHS);
+  }
+
+  bool operator<(const SmallVectorImpl &RHS) const {
+    return std::lexicographical_compare(this->begin(), this->end(),
+                                        RHS.begin(), RHS.end());
+  }
+
+  /// Set the array size to \p N, which the current array must have enough
+  /// capacity for.
+  ///
+  /// This does not construct or destroy any elements in the vector.
+  ///
+  /// Clients can use this in conjunction with capacity() to write past the end
+  /// of the buffer when they know that more elements are available, and only
+  /// update the size later. This avoids the cost of value initializing elements
+  /// which will only be overwritten.
+  void set_size(size_type N) {
+    assert(N <= this->capacity());
+    this->setEnd(this->begin() + N);
+  }
+};
+
+
+template <typename T>
+void SmallVectorImpl<T>::swap(SmallVectorImpl<T> &RHS) {
+  if (this == &RHS) return;
+
+  // We can only avoid copying elements if neither vector is small.
+  if (!this->isSmall() && !RHS.isSmall()) {
+    std::swap(this->BeginX, RHS.BeginX);
+    std::swap(this->EndX, RHS.EndX);
+    std::swap(this->CapacityX, RHS.CapacityX);
+    return;
+  }
+  if (RHS.size() > this->capacity())
+    this->grow(RHS.size());
+  if (this->size() > RHS.capacity())
+    RHS.grow(this->size());
+
+  // Swap the shared elements.
+  size_t NumShared = this->size();
+  if (NumShared > RHS.size()) NumShared = RHS.size();
+  for (size_type i = 0; i != NumShared; ++i)
+    std::swap((*this)[i], RHS[i]);
+
+  // Copy over the extra elts.
+  if (this->size() > RHS.size()) {
+    size_t EltDiff = this->size() - RHS.size();
+    this->uninitialized_copy(this->begin()+NumShared, this->end(), RHS.end());
+    RHS.setEnd(RHS.end()+EltDiff);
+    this->destroy_range(this->begin()+NumShared, this->end());
+    this->setEnd(this->begin()+NumShared);
+  } else if (RHS.size() > this->size()) {
+    size_t EltDiff = RHS.size() - this->size();
+    this->uninitialized_copy(RHS.begin()+NumShared, RHS.end(), this->end());
+    this->setEnd(this->end() + EltDiff);
+    this->destroy_range(RHS.begin()+NumShared, RHS.end());
+    RHS.setEnd(RHS.begin()+NumShared);
+  }
+}
+
+template <typename T>
+SmallVectorImpl<T> &SmallVectorImpl<T>::
+  operator=(const SmallVectorImpl<T> &RHS) {
+  // Avoid self-assignment.
+  if (this == &RHS) return *this;
+
+  // If we already have sufficient space, assign the common elements, then
+  // destroy any excess.
+  size_t RHSSize = RHS.size();
+  size_t CurSize = this->size();
+  if (CurSize >= RHSSize) {
+    // Assign common elements.
+    iterator NewEnd;
+    if (RHSSize)
+      NewEnd = std::copy(RHS.begin(), RHS.begin()+RHSSize, this->begin());
+    else
+      NewEnd = this->begin();
+
+    // Destroy excess elements.
+    this->destroy_range(NewEnd, this->end());
+
+    // Trim.
+    this->setEnd(NewEnd);
+    return *this;
+  }
+
+  // If we have to grow to have enough elements, destroy the current elements.
+  // This allows us to avoid copying them during the grow.
+  // FIXME: don't do this if they're efficiently moveable.
+  if (this->capacity() < RHSSize) {
+    // Destroy current elements.
+    this->destroy_range(this->begin(), this->end());
+    this->setEnd(this->begin());
+    CurSize = 0;
+    this->grow(RHSSize);
+  } else if (CurSize) {
+    // Otherwise, use assignment for the already-constructed elements.
+    std::copy(RHS.begin(), RHS.begin()+CurSize, this->begin());
+  }
+
+  // Copy construct the new elements in place.
+  this->uninitialized_copy(RHS.begin()+CurSize, RHS.end(),
+                           this->begin()+CurSize);
+
+  // Set end.
+  this->setEnd(this->begin()+RHSSize);
+  return *this;
+}
+
+template <typename T>
+SmallVectorImpl<T> &SmallVectorImpl<T>::operator=(SmallVectorImpl<T> &&RHS) {
+  // Avoid self-assignment.
+  if (this == &RHS) return *this;
+
+  // If the RHS isn't small, clear this vector and then steal its buffer.
+  if (!RHS.isSmall()) {
+    this->destroy_range(this->begin(), this->end());
+    if (!this->isSmall()) free(this->begin());
+    this->BeginX = RHS.BeginX;
+    this->EndX = RHS.EndX;
+    this->CapacityX = RHS.CapacityX;
+    RHS.resetToSmall();
+    return *this;
+  }
+
+  // If we already have sufficient space, assign the common elements, then
+  // destroy any excess.
+  size_t RHSSize = RHS.size();
+  size_t CurSize = this->size();
+  if (CurSize >= RHSSize) {
+    // Assign common elements.
+    iterator NewEnd = this->begin();
+    if (RHSSize)
+      NewEnd = std::move(RHS.begin(), RHS.end(), NewEnd);
+
+    // Destroy excess elements and trim the bounds.
+    this->destroy_range(NewEnd, this->end());
+    this->setEnd(NewEnd);
+
+    // Clear the RHS.
+    RHS.clear();
+
+    return *this;
+  }
+
+  // If we have to grow to have enough elements, destroy the current elements.
+  // This allows us to avoid copying them during the grow.
+  // FIXME: this may not actually make any sense if we can efficiently move
+  // elements.
+  if (this->capacity() < RHSSize) {
+    // Destroy current elements.
+    this->destroy_range(this->begin(), this->end());
+    this->setEnd(this->begin());
+    CurSize = 0;
+    this->grow(RHSSize);
+  } else if (CurSize) {
+    // Otherwise, use assignment for the already-constructed elements.
+    std::move(RHS.begin(), RHS.begin()+CurSize, this->begin());
+  }
+
+  // Move-construct the new elements in place.
+  this->uninitialized_move(RHS.begin()+CurSize, RHS.end(),
+                           this->begin()+CurSize);
+
+  // Set end.
+  this->setEnd(this->begin()+RHSSize);
+
+  RHS.clear();
+  return *this;
+}
+
+/// Storage for the SmallVector elements which aren't contained in
+/// SmallVectorTemplateCommon. There are 'N-1' elements here. The remaining '1'
+/// element is in the base class. This is specialized for the N=1 and N=0 cases
+/// to avoid allocating unnecessary storage.
+template <typename T, unsigned N>
+struct SmallVectorStorage {
+  typename SmallVectorTemplateCommon<T>::U InlineElts[N - 1];
+};
+template <typename T> struct SmallVectorStorage<T, 1> {};
+template <typename T> struct SmallVectorStorage<T, 0> {};
+
+/// This is a 'vector' (really, a variable-sized array), optimized
+/// for the case when the array is small.  It contains some number of elements
+/// in-place, which allows it to avoid heap allocation when the actual number of
+/// elements is below that threshold.  This allows normal "small" cases to be
+/// fast without losing generality for large inputs.
+///
+/// Note that this does not attempt to be exception safe.
+///
+template <typename T, unsigned N>
+class SmallVector : public SmallVectorImpl<T> {
+  /// Inline space for elements which aren't stored in the base class.
+  SmallVectorStorage<T, N> Storage;
+public:
+  SmallVector() : SmallVectorImpl<T>(N) {
+  }
+
+  explicit SmallVector(size_t Size, const T &Value = T())
+    : SmallVectorImpl<T>(N) {
+    this->assign(Size, Value);
+  }
+
+  template<typename ItTy>
+  SmallVector(ItTy S, ItTy E) : SmallVectorImpl<T>(N) {
+    this->append(S, E);
+  }
+
+/*
+  template <typename RangeTy>
+  explicit SmallVector(const llvm_vecsmall::iterator_range<RangeTy> &R)
+      : SmallVectorImpl<T>(N) {
+    this->append(R.begin(), R.end());
+  }
+*/
+
+  SmallVector(std::initializer_list<T> IL) : SmallVectorImpl<T>(N) {
+    this->assign(IL);
+  }
+
+  SmallVector(const SmallVector &RHS) : SmallVectorImpl<T>(N) {
+    if (!RHS.empty())
+      SmallVectorImpl<T>::operator=(RHS);
+  }
+
+  const SmallVector &operator=(const SmallVector &RHS) {
+    SmallVectorImpl<T>::operator=(RHS);
+    return *this;
+  }
+
+  SmallVector(SmallVector &&RHS) : SmallVectorImpl<T>(N) {
+    if (!RHS.empty())
+      SmallVectorImpl<T>::operator=(::std::move(RHS));
+  }
+
+  const SmallVector &operator=(SmallVector &&RHS) {
+    SmallVectorImpl<T>::operator=(::std::move(RHS));
+    return *this;
+  }
+
+  SmallVector(SmallVectorImpl<T> &&RHS) : SmallVectorImpl<T>(N) {
+    if (!RHS.empty())
+      SmallVectorImpl<T>::operator=(::std::move(RHS));
+  }
+
+  const SmallVector &operator=(SmallVectorImpl<T> &&RHS) {
+    SmallVectorImpl<T>::operator=(::std::move(RHS));
+    return *this;
+  }
+
+  const SmallVector &operator=(std::initializer_list<T> IL) {
+    this->assign(IL);
+    return *this;
+  }
+};
+
+template<typename T, unsigned N>
+static inline size_t capacity_in_bytes(const SmallVector<T, N> &X) {
+  return X.capacity_in_bytes();
+}
+
+} // End llvm_vecsmall namespace
+
+namespace std {
+  /// Implement std::swap in terms of SmallVector swap.
+  template<typename T>
+  inline void
+  swap(llvm_vecsmall::SmallVectorImpl<T> &LHS, llvm_vecsmall::SmallVectorImpl<T> &RHS) {
+    LHS.swap(RHS);
+  }
+
+  /// Implement std::swap in terms of SmallVector swap.
+  template<typename T, unsigned N>
+  inline void
+  swap(llvm_vecsmall::SmallVector<T, N> &LHS, llvm_vecsmall::SmallVector<T, N> &RHS) {
+    LHS.swap(RHS);
+  }
+}
+
+namespace llvm_vecsmall {
+/// grow_pod - This is an implementation of the grow() method which only works
+/// on POD-like datatypes and is out of line to reduce code duplication.
+inline
+void SmallVectorBase::grow_pod(void *FirstEl, size_t MinSizeInBytes,
+                               size_t TSize) {
+  size_t CurSizeBytes = size_in_bytes();
+  size_t NewCapacityInBytes = 2 * capacity_in_bytes() + TSize; // Always grow.
+  if (NewCapacityInBytes < MinSizeInBytes)
+    NewCapacityInBytes = MinSizeInBytes;
+
+  void *NewElts;
+  if (BeginX == FirstEl) {
+    NewElts = malloc(NewCapacityInBytes);
+
+    // Copy the elements over.  No need to run dtors on PODs.
+    memcpy(NewElts, this->BeginX, CurSizeBytes);
+  } else {
+    // If this wasn't grown from the inline copy, grow the allocated space.
+    NewElts = realloc(this->BeginX, NewCapacityInBytes);
+  }
+  assert(NewElts && "Out of memory");
+
+  this->EndX = (char*)NewElts+CurSizeBytes;
+  this->BeginX = NewElts;
+  this->CapacityX = (char*)this->BeginX + NewCapacityInBytes;
+}
+}
+
+#endif

CMakeLists.txt

@@ -446,6 +446,9 @@ if(BUILD_TESTS)
     kv_bench SRCS src/kv/test/kv_bench.cpp src/crypto/symmetric_key.cpp
                   src/enclave/thread_local.cpp
   )
+  add_picobench(
+    hash_bench SRCS src/ds/test/hash_bench.cpp
+  )
 
   # Storing signed governance operations
   add_e2e_test(

cgmanifest.json

@@ -145,7 +145,16 @@
           "commitHash": "dc8c3a9a1089e962b32ecdcc940ae11bd2b69e4b"
         }
       }
-    },
+    },,
+    {
+      "component": {
+        "type": "git",
+        "git": {
+          "repositoryUrl": "https://github.com/thelink2012/SmallVector",
+          "commitHash": "febc8cb7b1a83d902b86dd1612feb7c86c690186"
+        }
+      }
+    }
   ],
   "Version": 1
 }

src/apps/js_generic/js_generic.cpp

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the Apache 2.0 License.
 #include "enclave/app_interface.h"
+#include "kv/untyped_map.h"
 #include "node/rpc/user_frontend.h"
 #include "quickjs.h"
 
@@ -77,7 +78,7 @@ namespace ccfapp
 
     size_t sz = 0;
     auto k = JS_ToCStringLen(ctx, &sz, argv[0]);
-    auto v = table_view->get(std::vector<uint8_t>(k, k + sz));
+    auto v = table_view->get({k, k + sz});
     JS_FreeCString(ctx, k);
 
     if (v.has_value())
@@ -100,7 +101,7 @@ namespace ccfapp
 
     size_t sz = 0;
     auto k = JS_ToCStringLen(ctx, &sz, argv[0]);
-    auto v = table_view->remove(std::vector<uint8_t>(k, k + sz));
+    auto v = table_view->remove({k, k + sz});
     JS_FreeCString(ctx, k);
 
     if (v)
@@ -124,13 +125,11 @@ namespace ccfapp
 
     size_t k_sz = 0;
     auto k = JS_ToCStringLen(ctx, &k_sz, argv[0]);
-    std::vector<uint8_t> k_(k, k + k_sz);
 
     size_t v_sz = 0;
     auto v = JS_ToCStringLen(ctx, &v_sz, argv[1]);
-    std::vector<uint8_t> v_(v, v + v_sz);
 
-    if (!table_view->put(k_, v_))
+    if (!table_view->put({k, k + k_sz}, {v, v + v_sz}))
     {
       r = JS_ThrowRangeError(ctx, "Could not insert at key");
     }

src/ds/champ_map.h

@@ -148,10 +148,10 @@ namespace champ
 
     SubNodes() {}
 
-    SubNodes(std::vector<Node<K, V, H>> ns) : nodes(ns) {}
+    SubNodes(std::vector<Node<K, V, H>>&& ns) : nodes(std::move(ns)) {}
 
-    SubNodes(std::vector<Node<K, V, H>> ns, Bitmap nm, Bitmap dm) :
+    SubNodes(std::vector<Node<K, V, H>>&& ns, Bitmap nm, Bitmap dm) :
-      nodes(ns),
+      nodes(std::move(ns)),
       node_map(nm),
       data_map(dm)
     {}
@@ -311,8 +311,8 @@ namespace champ
     std::shared_ptr<SubNodes<K, V, H>> root;
     size_t _size = 0;
 
-    Map(std::shared_ptr<SubNodes<K, V, H>> root_, size_t size_) :
+    Map(std::shared_ptr<SubNodes<K, V, H>>&& root_, size_t size_) :
-      root(root_),
+      root(std::move(root_)),
       _size(size_)
     {}
 

src/ds/hash.h

@@ -6,8 +6,32 @@
 
 #include <array>
 #include <cstdint>
+#include <small_vector/SmallVector.h>
 #include <vector>
 
+namespace ds::hashutils
+{
+  template <typename T>
+  inline void hash_combine(size_t& n, const T& v, std::hash<T>& h)
+  {
+    n ^= h(v) + (n << 6) + (n >> 2);
+  }
+
+  template <typename T>
+  inline size_t hash_container(const T& v)
+  {
+    size_t n = 0x444e414c544f4353;
+    std::hash<typename T::value_type> h{};
+
+    for (const auto& e : v)
+    {
+      hash_combine(n, e, h);
+    }
+
+    return n;
+  }
+}
+
 namespace std
 {
   template <>
@@ -23,35 +47,12 @@ namespace std
     }
   };
 
-  namespace
-  {
-    template <typename T>
-    inline void hash_combine(size_t& n, const T& v, std::hash<T>& h)
-    {
-      n ^= h(v) + (n << 6) + (n >> 2);
-    }
-
-    template <typename T>
-    inline size_t hash_container(const T& v)
-    {
-      size_t n = 0x444e414c544f4353;
-      std::hash<typename T::value_type> h{};
-
-      for (const auto& e : v)
-      {
-        hash_combine(n, e, h);
-      }
-
-      return n;
-    }
-  }
-
   template <typename T>
   struct hash<std::vector<T>>
   {
     size_t operator()(const std::vector<T>& v) const
     {
-      return hash_container(v);
+      return ds::hashutils::hash_container(v);
     }
   };
 
@@ -60,7 +61,7 @@ namespace std
   {
     size_t operator()(const std::array<T, N>& v) const
     {
-      return hash_container(v);
+      return ds::hashutils::hash_container(v);
     }
   };
 
@@ -72,14 +73,25 @@ namespace std
       size_t n = 0x444e414c544f4353;
 
       std::hash<A> h_a{};
-      hash_combine(n, v.first, h_a);
+      ds::hashutils::hash_combine(n, v.first, h_a);
 
       std::hash<B> h_b{};
-      hash_combine(n, v.second, h_b);
+      ds::hashutils::hash_combine(n, v.second, h_b);
 
       return n;
     }
   };
+
+  template <typename T, unsigned N>
+  struct hash<llvm_vecsmall::SmallVector<T, N>>
+  {
+    size_t operator()(const llvm_vecsmall::SmallVector<T, N>& v) const
+    {
+      static constexpr siphash::SipKey k{0x7720796f726c694b,
+                                         0x2165726568207361};
+      return siphash::siphash<2, 4>(v.data(), v.size(), k);
+    }
+  };
 }
 
 namespace ds

src/ds/siphash.h

@@ -164,7 +164,7 @@ namespace siphash
   }
 
   template <size_t CompressionRounds, size_t FinalizationRounds>
-  uint64_t siphash(const std::vector<uint8_t>& in, const SipKey& key)
+  uint64_t siphash(const uint8_t* data, size_t size, const SipKey& key)
   {
     uint64_t out;
 
@@ -172,8 +172,15 @@ namespace siphash
       CompressionRounds,
       FinalizationRounds,
       OutputLength::EightBytes>(
-      in.data(), in.size(), key, reinterpret_cast<uint8_t*>(&out));
+      data, size, key, reinterpret_cast<uint8_t*>(&out));
 
     return out;
   }
+
+  template <size_t CompressionRounds, size_t FinalizationRounds>
+  uint64_t siphash(const std::vector<uint8_t>& in, const SipKey& key)
+  {
+    return siphash<CompressionRounds, FinalizationRounds>(
+      in.data(), in.size(), key);
+  }
 }

src/ds/test/hash_bench.cpp

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+
+#include "ds/hash.h"
+
+#define PICOBENCH_IMPLEMENT_WITH_MAIN
+#include <picobench/picobench.hpp>
+
+template <typename T>
+static void hash(picobench::state& s)
+{
+  T v(s.iterations());
+  auto* d = v.data();
+  for (size_t i = 0; i < v.size(); ++i)
+  {
+    d[i] = rand();
+  }
+
+  std::hash<T> hasher;
+
+  s.start_timer();
+  for (size_t i = 0; i < 1000; ++i)
+  {
+    volatile auto n = hasher(v);
+    s.stop_timer();
+  }
+}
+
+const std::vector<int> hash_sizes = {1, 8, 64, 1024, 16536};
+
+PICOBENCH_SUITE("hash");
+auto hash_vec = hash<std::vector<uint8_t>>;
+PICOBENCH(hash_vec).iterations(hash_sizes).baseline();
+auto hash_small_vec_16 = hash<llvm_vecsmall::SmallVector<uint8_t, 16>>;
+PICOBENCH(hash_small_vec_16).iterations(hash_sizes).baseline();
+auto hash_small_vec_128 = hash<llvm_vecsmall::SmallVector<uint8_t, 128>>;
+PICOBENCH(hash_small_vec_128).iterations(hash_sizes).baseline();

src/kv/change_set.h

@@ -0,0 +1,61 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+#include "ds/champ_map.h"
+#include "ds/hash.h"
+#include "kv/kv_types.h"
+
+#include <map>
+
+namespace kv
+{
+  template <typename V>
+  struct VersionV
+  {
+    Version version;
+    V value;
+
+    VersionV() = default;
+    VersionV(Version ver, V val) : version(ver), value(val) {}
+  };
+
+  template <typename K, typename V, typename H>
+  using State = champ::Map<K, VersionV<V>, H>;
+
+  template <typename K>
+  using Read = std::map<K, Version>;
+
+  // nullopt values represent deletions
+  template <typename K, typename V>
+  using Write = std::map<K, std::optional<V>>;
+
+  // This is a container for a write-set + dependencies. It can be applied to a
+  // given state, or used to track a set of operations on a state
+  template <typename K, typename V, typename H>
+  struct ChangeSet
+  {
+  public:
+    State<K, V, H> state;
+    State<K, V, H> committed;
+    Version start_version;
+
+    Version read_version = NoVersion;
+    Read<K> reads = {};
+    Write<K, V> writes = {};
+
+    ChangeSet(
+      State<K, V, H>& current_state,
+      State<K, V, H>& committed_state,
+      Version current_version) :
+      state(current_state),
+      committed(committed_state),
+      start_version(current_version)
+    {}
+
+    ChangeSet(ChangeSet&) = delete;
+  };
+
+  /// Signature for transaction commit handlers
+  template <typename W>
+  using CommitHook = std::function<void(Version, const W&)>;
+}

src/kv/experimental.h

@@ -1,321 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the Apache 2.0 License.
-#pragma once
-
-#include "ds/hash.h"
-#include "kv_types.h"
-#include "map.h"
-#include "tx_view.h"
-
-#include <vector>
-
-namespace kv
-{
-  namespace experimental
-  {
-    using SerialisedRep = std::vector<uint8_t>;
-
-    using RepHasher = std::hash<SerialisedRep>;
-
-    using UntypedMap = kv::Map<SerialisedRep, SerialisedRep, RepHasher>;
-
-    using UntypedOperationsView =
-      kv::TxView<SerialisedRep, SerialisedRep, RepHasher>;
-
-    using UntypedCommitter =
-      kv::TxViewCommitter<SerialisedRep, SerialisedRep, RepHasher>;
-
-    using UntypedState = kv::State<SerialisedRep, SerialisedRep, RepHasher>;
-
-    template <typename T>
-    struct MsgPackSerialiser
-    {
-      static SerialisedRep to_serialised(const T& t)
-      {
-        msgpack::sbuffer sb;
-        msgpack::pack(sb, t);
-        auto sb_data = reinterpret_cast<const uint8_t*>(sb.data());
-        return SerialisedRep(sb_data, sb_data + sb.size());
-      }
-
-      static T from_serialised(const SerialisedRep& rep)
-      {
-        msgpack::object_handle oh = msgpack::unpack(
-          reinterpret_cast<const char*>(rep.data()), rep.size());
-        auto object = oh.get();
-        return object.as<T>();
-      }
-    };
-
-    template <typename T>
-    struct JsonSerialiser
-    {
-      static SerialisedRep to_serialised(const T& t)
-      {
-        const nlohmann::json j = t;
-        const auto dumped = j.dump();
-        return SerialisedRep(dumped.begin(), dumped.end());
-      }
-
-      static T from_serialised(const SerialisedRep& rep)
-      {
-        const auto j = nlohmann::json::parse(rep);
-        return j.get<T>();
-      }
-    };
-
-    template <
-      typename K,
-      typename V,
-      typename KSerialiser = MsgPackSerialiser<K>,
-      typename VSerialiser = MsgPackSerialiser<V>>
-    class TxView : public UntypedCommitter
-    {
-    protected:
-      // This _has_ a (non-visible, untyped) view, whereas the standard impl
-      // _is_ a typed view
-      UntypedOperationsView untyped_view;
-
-    public:
-      using KeyType = K;
-      using ValueType = V;
-
-      TxView(
-        UntypedMap& m,
-        size_t rollbacks,
-        UntypedState& current_state,
-        UntypedState& committed_state,
-        Version v) :
-        UntypedCommitter(m, rollbacks, current_state, committed_state, v),
-        untyped_view(UntypedCommitter::change_set)
-      {}
-
-      std::optional<V> get(const K& key)
-      {
-        const auto k_rep = KSerialiser::to_serialised(key);
-        const auto opt_v_rep = untyped_view.get(k_rep);
-
-        if (opt_v_rep.has_value())
-        {
-          return VSerialiser::from_serialised(*opt_v_rep);
-        }
-
-        return std::nullopt;
-      }
-
-      std::optional<V> get_globally_committed(const K& key)
-      {
-        const auto k_rep = KSerialiser::to_serialised(key);
-        const auto opt_v_rep = untyped_view.get_globally_committed(k_rep);
-
-        if (opt_v_rep.has_value())
-        {
-          return VSerialiser::from_serialised(*opt_v_rep);
-        }
-
-        return std::nullopt;
-      }
-
-      bool put(const K& key, const V& value)
-      {
-        const auto k_rep = KSerialiser::to_serialised(key);
-        const auto v_rep = VSerialiser::to_serialised(value);
-
-        return untyped_view.put(k_rep, v_rep);
-      }
-
-      bool remove(const K& key)
-      {
-        const auto k_rep = KSerialiser::to_serialised(key);
-
-        return untyped_view.remove(k_rep);
-      }
-
-      template <class F>
-      void foreach(F&& f)
-      {
-        auto g = [&](const SerialisedRep& k_rep, const SerialisedRep& v_rep) {
-          return f(
-            KSerialiser::from_serialised(k_rep),
-            VSerialiser::from_serialised(v_rep));
-        };
-        untyped_view.foreach(g);
-      }
-    };
-
-    template <
-      typename K,
-      typename V,
-      typename KSerialiser = MsgPackSerialiser<K>,
-      typename VSerialiser = MsgPackSerialiser<V>>
-    class Map : public AbstractMap
-    {
-    protected:
-      using This = Map<K, V, KSerialiser, VSerialiser>;
-
-      UntypedMap untyped_map;
-
-    public:
-      // Expose correct public aliases of types
-      using VersionV = VersionV<V>;
-
-      using Write = Write<K, V>;
-
-      using CommitHook = CommitHook<Write>;
-
-      using TxView = kv::experimental::TxView<K, V, KSerialiser, VSerialiser>;
-
-      template <typename... Ts>
-      Map(Ts&&... ts) : untyped_map(std::forward<Ts>(ts)...)
-      {}
-
-      bool operator==(const AbstractMap& that) const override
-      {
-        auto p = dynamic_cast<const This*>(&that);
-        if (p == nullptr)
-        {
-          return false;
-        }
-
-        return untyped_map == p->untyped_map;
-      }
-
-      bool operator!=(const AbstractMap& that) const override
-      {
-        return !(*this == that);
-      }
-
-      AbstractStore* get_store() override
-      {
-        return untyped_map.get_store();
-      }
-
-      void serialise(
-        const AbstractTxView* view,
-        KvStoreSerialiser& s,
-        bool include_reads) override
-      {
-        untyped_map.serialise(view, s, include_reads);
-      }
-
-      AbstractTxView* deserialise(
-        KvStoreDeserialiser& d, Version version) override
-      {
-        return untyped_map.deserialise(d, version);
-      }
-
-      const std::string& get_name() const override
-      {
-        return untyped_map.get_name();
-      }
-
-      void compact(Version v) override
-      {
-        return untyped_map.compact(v);
-      }
-
-      void post_compact() override
-      {
-        return untyped_map.post_compact();
-      }
-
-      void rollback(Version v) override
-      {
-        untyped_map.rollback(v);
-      }
-
-      void lock() override
-      {
-        untyped_map.lock();
-      }
-
-      void unlock() override
-      {
-        untyped_map.unlock();
-      }
-
-      SecurityDomain get_security_domain() override
-      {
-        return untyped_map.get_security_domain();
-      }
-
-      bool is_replicated() override
-      {
-        return untyped_map.is_replicated();
-      }
-
-      void clear() override
-      {
-        untyped_map.clear();
-      }
-
-      AbstractMap* clone(AbstractStore* store) override
-      {
-        return new Map(
-          store,
-          untyped_map.get_name(),
-          untyped_map.get_security_domain(),
-          untyped_map.is_replicated());
-      }
-
-      void swap(AbstractMap* map) override
-      {
-        auto p = dynamic_cast<This*>(map);
-        if (p == nullptr)
-          throw std::logic_error(
-            "Attempted to swap maps with incompatible types");
-
-        untyped_map.swap(&p->untyped_map);
-      }
-
-      template <typename TView>
-      TView* create_view(Version v)
-      {
-        return untyped_map.create_view<TView>(v);
-      }
-
-      static UntypedMap::CommitHook wrap_commit_hook(const CommitHook& hook)
-      {
-        return [hook](Version v, const UntypedMap::Write& w) {
-          Write typed_w;
-          for (const auto& [uk, opt_uv] : w)
-          {
-            if (!opt_uv.has_value())
-            {
-              // Deletions are indicated by nullopt. We cannot deserialise them,
-              // they are deletions here as well
-              typed_w[KSerialiser::from_serialised(uk)] = std::nullopt;
-            }
-            else
-            {
-              typed_w[KSerialiser::from_serialised(uk)] =
-                VSerialiser::from_serialised(opt_uv.value());
-            }
-          }
-
-          hook(v, typed_w);
-        };
-      }
-
-      void set_local_hook(const CommitHook& hook)
-      {
-        untyped_map.set_local_hook(wrap_commit_hook(hook));
-      }
-
-      void unset_local_hook()
-      {
-        untyped_map.unset_local_hook();
-      }
-
-      void set_global_hook(const CommitHook& hook)
-      {
-        untyped_map.set_global_hook(wrap_commit_hook(hook));
-      }
-
-      void unset_global_hook()
-      {
-        untyped_map.unset_global_hook();
-      }
-    };
-  }
-}

src/kv/generic_serialise_wrapper.h

@@ -4,11 +4,15 @@
 
 #include "ds/buffer.h"
 #include "kv_types.h"
+#include "serialised_entry.h"
 
 #include <optional>
 
 namespace kv
 {
+  using SerialisedKey = kv::serialisers::SerialisedEntry;
+  using SerialisedValue = kv::serialisers::SerialisedEntry;
+
   enum class KvOperationType : uint32_t
   {
     KOT_NOT_SUPPORTED = 0,
@@ -38,16 +42,6 @@ namespace kv
       static_cast<KotBase>(a) | static_cast<KotBase>(b));
   }
 
-  template <typename K, typename V, typename Version>
-  struct KeyValVersion
-  {
-    K key;
-    V value;
-    Version version;
-
-    KeyValVersion(K k, V v, Version ver) : key(k), value(v), version(ver) {}
-  };
-
   template <typename W>
   class GenericSerialiseWrapper
   {
@@ -68,10 +62,10 @@ namespace kv
       current_writer->append(std::forward<T>(t));
     }
 
-    template <typename T>
+    void serialise_internal_pre_serialised(
-    void serialise_internal_public(T&& t)
+      const kv::serialisers::SerialisedEntry& raw)
     {
-      public_writer.append(std::forward<T>(t));
+      current_writer->append_pre_serialised(raw);
     }
 
     void set_current_domain(SecurityDomain domain)
@@ -127,40 +121,36 @@ namespace kv
       serialise_internal(ctr);
     }
 
-    template <class K>
+    void serialise_read(const SerialisedKey& k, const Version& version)
-    void serialise_read(const K& k, const Version& version)
     {
-      serialise_internal(k);
+      serialise_internal_pre_serialised(k);
       serialise_internal(version);
     }
 
-    template <class K, class V>
+    void serialise_write(const SerialisedKey& k, const SerialisedValue& v)
-    void serialise_write(const K& k, const V& v)
     {
-      serialise_internal(k);
+      serialise_internal_pre_serialised(k);
-      serialise_internal(v);
+      serialise_internal_pre_serialised(v);
     }
 
-    template <class K, class V, class Version>
+    void serialise_write_version(
-    void serialise_write_version(const K& k, const V& v, const Version& version)
+      const SerialisedKey& k, const SerialisedValue& v, const Version& version)
     {
       serialise_internal(KvOperationType::KOT_WRITE_VERSION);
-      serialise_internal(k);
+      serialise_internal_pre_serialised(k);
-      serialise_internal(v);
+      serialise_internal_pre_serialised(v);
       serialise_internal(version);
     }
 
-    template <class K>
+    void serialise_remove_version(const SerialisedKey& k)
-    void serialise_remove_version(const K& k)
     {
       serialise_internal(KvOperationType::KOT_REMOVE_VERSION);
-      serialise_internal(k);
+      serialise_internal_pre_serialised(k);
     }
 
-    template <class K>
+    void serialise_remove(const SerialisedKey& k)
-    void serialise_remove(const K& k)
     {
-      serialise_internal(k);
+      serialise_internal_pre_serialised(k);
     }
 
     std::vector<uint8_t> get_raw_data()
@@ -342,7 +332,6 @@ namespace kv
       return true;
     }
 
-    template <class Version>
     Version deserialise_version()
     {
       version = current_reader->template read_next<Version>();
@@ -368,7 +357,6 @@ namespace kv
         current_reader->template read_next<std::string>()};
     }
 
-    template <class Version>
     Version deserialise_read_version()
     {
       return current_reader->template read_next<Version>();
@@ -379,10 +367,9 @@ namespace kv
       return current_reader->template read_next<uint64_t>();
     }
 
-    template <class K>
+    std::tuple<SerialisedKey, Version> deserialise_read()
-    std::tuple<K, Version> deserialise_read()
     {
-      return {current_reader->template read_next<K>(),
+      return {current_reader->read_next_pre_serialised(),
               current_reader->template read_next<Version>()};
     }
 
@@ -391,11 +378,10 @@ namespace kv
       return current_reader->template read_next<uint64_t>();
     }
 
-    template <class K, class V>
+    std::tuple<SerialisedKey, SerialisedValue> deserialise_write()
-    std::tuple<K, V> deserialise_write()
     {
-      return {current_reader->template read_next<K>(),
+      return {current_reader->read_next_pre_serialised(),
-              current_reader->template read_next<V>()};
+              current_reader->read_next_pre_serialised()};
     }
 
     uint64_t deserialise_remove_header()
@@ -403,39 +389,9 @@ namespace kv
       return current_reader->template read_next<uint64_t>();
     }
 
-    template <class K>
+    SerialisedKey deserialise_remove()
-    K deserialise_remove()
     {
-      return current_reader->template read_next<K>();
+      return current_reader->read_next_pre_serialised();
-    }
-
-    template <class K, class V, class Version>
-    std::optional<KeyValVersion<K, V, Version>> deserialise_write_version()
-    {
-      if (end())
-        return {};
-
-      auto curr_op = try_read_op_flag(
-        KvOperationType::KOT_WRITE_VERSION |
-        KvOperationType::KOT_REMOVE_VERSION);
-
-      switch (curr_op)
-      {
-        case KvOperationType::KOT_WRITE_VERSION:
-        {
-          K key = current_reader->template read_next<K>();
-          V value = current_reader->template read_next<V>();
-          Version version = current_reader->template read_next<Version>();
-          return {{key, value, version, false}};
-        }
-        case KvOperationType::KOT_REMOVE_VERSION:
-        {
-          K key = current_reader->template read_next<K>();
-          return {{key, V(), Version(), true}};
-        }
-        default:
-          return {};
-      }
     }
 
     bool end()

src/kv/kv_types.h

@@ -21,6 +21,11 @@ namespace kv
   using Version = int64_t;
   static const Version NoVersion = std::numeric_limits<Version>::min();
 
+  static bool is_deleted(Version version)
+  {
+    return version < 0;
+  }
+
   // Term describes an epoch of Versions. It is incremented when global kv's
   // writer(s) changes. Term and Version combined give a unique identifier for
   // all accepted kv modifications. Terms are handled by Raft via the

src/kv/map.h

@@ -2,567 +2,45 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
-#include "ds/dl_list.h"
-#include "ds/logger.h"
-#include "ds/spin_lock.h"
-#include "kv_serialiser.h"
 #include "kv_types.h"
+#include "serialise_entry_blit.h"
+#include "serialise_entry_json.h"
+#include "serialise_entry_msgpack.h"
 #include "tx_view.h"
 
-#include <functional>
-#include <optional>
-#include <unordered_set>
-
 namespace kv
 {
-  namespace Check
+  template <typename K, typename V, typename KSerialiser, typename VSerialiser>
-  {
+  class TypedMap : public AbstractMap
-    struct No
-    {};
-
-    template <typename T, typename Arg>
-    No operator!=(const T&, const Arg&)
-    {
-      return No();
-    }
-
-    template <typename T, typename Arg = T>
-    struct Ne
-    {
-      enum
-      {
-        value = !std::is_same<decltype(*(T*)(0) != *(Arg*)(0)), No>::value
-      };
-    };
-
-    template <class T>
-    bool ne(std::enable_if_t<Ne<T>::value, const T&> a, const T& b)
-    {
-      return a != b;
-    }
-
-    template <class T>
-    bool ne(std::enable_if_t<!Ne<T>::value, const T&> a, const T& b)
-    {
-      return false;
-    }
-  }
-
-  template <class K, class V, class H = std::hash<K>>
-  class Map;
-
-  template <class K, class V, class H>
-  class TxViewCommitter : public AbstractTxView
   {
   protected:
-    using MyMap = Map<K, V, H>;
+    using This = TypedMap<K, V, KSerialiser, VSerialiser>;
 
-    ChangeSet<K, V, H> change_set;
+    kv::untyped::Map untyped_map;
-
-    MyMap& map;
-    size_t rollback_counter;
-
-    Version commit_version = NoVersion;
-
-    bool changes = false;
-    bool committed_writes = false;
 
   public:
-    template <typename... Ts>
+    // Expose correct public aliases of types
-    TxViewCommitter(MyMap& m, size_t rollbacks, Ts&&... ts) :
-      map(m),
-      rollback_counter(rollbacks),
-      change_set(std::forward<Ts>(ts)...)
-    {}
-
-    // Commit-related methods
-    bool has_writes() override
-    {
-      return committed_writes || !change_set.writes.empty();
-    }
-
-    bool has_changes() override
-    {
-      return changes;
-    }
-
-    bool prepare() override
-    {
-      if (change_set.writes.empty())
-        return true;
-
-      // If the parent map has rolled back since this transaction began, this
-      // transaction must fail.
-      if (rollback_counter != map.rollback_counter)
-        return false;
-
-      // If we have iterated over the map, check for a global version match.
-      auto current = map.roll->get_tail();
-
-      if (
-        (change_set.read_version != NoVersion) &&
-        (change_set.read_version != current->version))
-      {
-        LOG_DEBUG_FMT("Read version {} is invalid", change_set.read_version);
-        return false;
-      }
-
-      // Check each key in our read set.
-      for (auto it = change_set.reads.begin(); it != change_set.reads.end();
-           ++it)
-      {
-        // Get the value from the current state.
-        auto search = current->state.get(it->first);
-
-        if (it->second == NoVersion)
-        {
-          // If we depend on the key not existing, it must be absent.
-          if (search.has_value())
-          {
-            LOG_DEBUG_FMT("Read depends on non-existing entry");
-            return false;
-          }
-        }
-        else
-        {
-          // If we depend on the key existing, it must be present and have the
-          // version that we expect.
-          if (!search.has_value() || (it->second != search.value().version))
-          {
-            LOG_DEBUG_FMT("Read depends on invalid version of entry");
-            return false;
-          }
-        }
-      }
-
-      return true;
-    }
-
-    void commit(Version v) override
-    {
-      if (change_set.writes.empty())
-      {
-        commit_version = change_set.start_version;
-        return;
-      }
-
-      // Record our commit time.
-      commit_version = v;
-      committed_writes = true;
-
-      if (!change_set.writes.empty())
-      {
-        auto state = map.roll->get_tail()->state;
-
-        for (auto it = change_set.writes.begin(); it != change_set.writes.end();
-             ++it)
-        {
-          if (it->second.has_value())
-          {
-            // Write the new value with the global version.
-            changes = true;
-            state = state.put(it->first, VersionV{v, it->second.value()});
-          }
-          else
-          {
-            // Write an empty value with the deleted global version only if
-            // the key exists.
-            auto search = state.get(it->first);
-            if (search.has_value())
-            {
-              changes = true;
-              state = state.put(it->first, VersionV{-v, V()});
-            }
-          }
-        }
-
-        if (changes)
-        {
-          map.roll->insert_back(
-            map.create_new_local_commit(v, state, change_set.writes));
-        }
-      }
-    }
-
-    void post_commit() override
-    {
-      // This is run separately from commit so that all commits in the Tx
-      // have been applied before local hooks are run. The maps in the Tx
-      // are still locked when post_commit is run.
-      if (change_set.writes.empty())
-        return;
-
-      if (map.local_hook)
-      {
-        auto roll = map.roll->get_tail();
-        map.local_hook(roll->version, roll->writes);
-      }
-    }
-
-    // Used by owning map during serialise and deserialise
-    ChangeSet<K, V, H>& get_change_set()
-    {
-      return change_set;
-    }
-
-    const ChangeSet<K, V, H>& get_change_set() const
-    {
-      return change_set;
-    }
-
-    void set_commit_version(Version v)
-    {
-      commit_version = v;
-    }
-  };
-
-  template <typename K, typename V, typename H>
-  struct ConcreteTxView : public TxViewCommitter<K, V, H>,
-                          public TxView<K, V, H>
-  {
-  public:
-    ConcreteTxView(
-      Map<K, V, H>& m,
-      size_t rollbacks,
-      State<K, V, H>& current_state,
-      State<K, V, H>& committed_state,
-      Version v) :
-      TxViewCommitter<K, V, H>(m, rollbacks, current_state, committed_state, v),
-      TxView<K, V, H>(TxViewCommitter<K, V, H>::change_set)
-    {}
-  };
-
-  /// Signature for transaction commit handlers
-  template <typename W>
-  using CommitHook = std::function<void(Version, const W&)>;
-
-  template <class K, class V, class H>
-  class Map : public AbstractMap
-  {
-  public:
     using VersionV = VersionV<V>;
-    using State = State<K, V, H>;
+
-    using Write = Write<K, V, H>;
+    using Write = std::map<K, std::optional<V>>;
+
     using CommitHook = CommitHook<Write>;
 
-  private:
+    using TxView = kv::TxView<K, V, KSerialiser, VSerialiser>;
-    using This = Map<K, V, H>;
 
-    struct LocalCommit
+    template <typename... Ts>
-    {
+    TypedMap(Ts&&... ts) : untyped_map(std::forward<Ts>(ts)...)
-      LocalCommit() = default;
+    {}
-      LocalCommit(Version v, State s, Write w) :
-        version(std::move(v)),
-        state(std::move(s)),
-        writes(std::move(w)),
-        next(nullptr),
-        prev(nullptr)
-      {}
-
-      Version version;
-      State state;
-      Write writes;
-      LocalCommit* next;
-      LocalCommit* prev;
-    };
-    using LocalCommits = snmalloc::DLList<LocalCommit, std::nullptr_t, true>;
-
-    AbstractStore* store;
-    std::string name;
-    size_t rollback_counter;
-    std::unique_ptr<LocalCommits> roll;
-    CommitHook local_hook = nullptr;
-    CommitHook global_hook = nullptr;
-    LocalCommits commit_deltas;
-    SpinLock sl;
-    const SecurityDomain security_domain;
-    const bool replicated;
-
-    LocalCommits empty_commits;
-
-    template <typename... Args>
-    LocalCommit* create_new_local_commit(Args&&... args)
-    {
-      LocalCommit* c = empty_commits.pop();
-      if (c == nullptr)
-      {
-        c = new LocalCommit(std::forward<Args>(args)...);
-      }
-      else
-      {
-        c->~LocalCommit();
-        new (c) LocalCommit(std::forward<Args>(args)...);
-      }
-      return c;
-    }
-
-  public:
-    // Public typedef for external consumption
-    using TxView = ConcreteTxView<K, V, H>;
-
-    // Provide access to hidden rollback_counter, roll, create_new_local_commit
-    friend TxViewCommitter<K, V, H>;
-
-    Map(
-      AbstractStore* store_,
-      std::string name_,
-      SecurityDomain security_domain_,
-      bool replicated_) :
-      store(store_),
-      name(name_),
-      roll(std::make_unique<LocalCommits>()),
-      rollback_counter(0),
-      security_domain(security_domain_),
-      replicated(replicated_)
-    {
-      roll->insert_back(create_new_local_commit(0, State(), Write()));
-    }
-
-    Map(const Map& that) = delete;
-
-    virtual AbstractMap* clone(AbstractStore* other) override
-    {
-      return new Map(other, name, security_domain, replicated);
-    }
-
-    void serialise(
-      const AbstractTxView* view,
-      KvStoreSerialiser& s,
-      bool include_reads) override
-    {
-      const auto committer =
-        dynamic_cast<const TxViewCommitter<K, V, H>*>(view);
-      if (committer == nullptr)
-      {
-        LOG_FAIL_FMT("Unable to serialise map due to type mismatch");
-        return;
-      }
-
-      const auto& change_set = committer->get_change_set();
-
-      s.start_map(name, security_domain);
-
-      if (include_reads)
-      {
-        s.serialise_read_version(change_set.read_version);
-
-        s.serialise_count_header(change_set.reads.size());
-        for (auto it = change_set.reads.begin(); it != change_set.reads.end();
-             ++it)
-        {
-          s.serialise_read(it->first, it->second);
-        }
-      }
-      else
-      {
-        s.serialise_read_version(NoVersion);
-        s.serialise_count_header(0);
-      }
-
-      uint64_t write_ctr = 0;
-      uint64_t remove_ctr = 0;
-      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
-           ++it)
-      {
-        if (it->second.has_value())
-        {
-          ++write_ctr;
-        }
-        else
-        {
-          auto search = roll->get_tail()->state.get(it->first);
-          if (search.has_value())
-          {
-            ++remove_ctr;
-          }
-        }
-      }
-
-      s.serialise_count_header(write_ctr);
-      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
-           ++it)
-      {
-        if (it->second.has_value())
-        {
-          s.serialise_write(it->first, it->second.value());
-        }
-      }
-
-      s.serialise_count_header(remove_ctr);
-      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
-           ++it)
-      {
-        if (!it->second.has_value())
-        {
-          s.serialise_remove(it->first);
-        }
-      }
-    }
-
-    AbstractTxView* deserialise(
-      KvStoreDeserialiser& d, Version version) override
-    {
-      // Create a new change set, and deserialise d's contents into it.
-      auto view = create_view<TxView>(version);
-      view->set_commit_version(version);
-
-      auto& change_set = view->get_change_set();
-
-      uint64_t ctr;
-
-      auto rv = d.template deserialise_read_version<Version>();
-      if (rv != NoVersion)
-      {
-        change_set.read_version = rv;
-      }
-
-      ctr = d.deserialise_read_header();
-      for (size_t i = 0; i < ctr; ++i)
-      {
-        auto r = d.template deserialise_read<K>();
-        change_set.reads[std::get<0>(r)] = std::get<1>(r);
-      }
-
-      ctr = d.deserialise_write_header();
-      for (size_t i = 0; i < ctr; ++i)
-      {
-        auto w = d.template deserialise_write<K, V>();
-        change_set.writes[std::get<0>(w)] = std::get<1>(w);
-      }
-
-      ctr = d.deserialise_remove_header();
-      for (size_t i = 0; i < ctr; ++i)
-      {
-        auto r = d.template deserialise_remove<K>();
-        change_set.writes[r] = std::nullopt;
-      }
-
-      return view;
-    }
-
-    /** Get the name of the map
-     *
-     * @return const std::string&
-     */
-    const std::string& get_name() const override
-    {
-      return name;
-    }
-
-    /** Get store that the map belongs to
-     *
-     * @return Pointer to `kv::AbstractStore`
-     */
-    AbstractStore* get_store() override
-    {
-      return store;
-    }
-
-    /** Set handler to be called on local transaction commit
-     *
-     * @param hook function to be called on local transaction commit
-     */
-    void set_local_hook(const CommitHook& hook)
-    {
-      std::lock_guard<SpinLock> guard(sl);
-      local_hook = hook;
-    }
-
-    /** Reset local transaction commit handler
-     */
-    void unset_local_hook()
-    {
-      std::lock_guard<SpinLock> guard(sl);
-      local_hook = nullptr;
-    }
-
-    /** Set handler to be called on global transaction commit
-     *
-     * @param hook function to be called on global transaction commit
-     */
-    void set_global_hook(const CommitHook& hook)
-    {
-      std::lock_guard<SpinLock> guard(sl);
-      global_hook = hook;
-    }
-
-    /** Reset global transaction commit handler
-     */
-    void unset_global_hook()
-    {
-      std::lock_guard<SpinLock> guard(sl);
-      global_hook = nullptr;
-    }
-
-    /** Get security domain of a Map
-     *
-     * @return Security domain of the map (affects serialisation)
-     */
-    virtual SecurityDomain get_security_domain() override
-    {
-      return security_domain;
-    }
-
-    /** Get Map replicability
-     *
-     * @return true if the map is to be replicated, false if it is to be derived
-     */
-    virtual bool is_replicated() override
-    {
-      return replicated;
-    }
 
     bool operator==(const AbstractMap& that) const override
     {
       auto p = dynamic_cast<const This*>(&that);
       if (p == nullptr)
+      {
         return false;
+      }
 
-      if (name != p->name)
+      return untyped_map == p->untyped_map;
-        return false;
-
-      auto state1 = roll->get_tail();
-      auto state2 = p->roll->get_tail();
-
-      if (state1->version != state2->version)
-        return false;
-
-      size_t count = 0;
-      state2->state.foreach([&count](const K& k, const VersionV& v) {
-        count++;
-        return true;
-      });
-
-      size_t i = 0;
-      bool ok =
-        state1->state.foreach([&state2, &i](const K& k, const VersionV& v) {
-          auto search = state2->state.get(k);
-
-          if (search.has_value())
-          {
-            auto& found = search.value();
-            if (found.version != v.version)
-            {
-              return false;
-            }
-            else if (Check::ne(found.value, v.value))
-            {
-              return false;
-            }
-          }
-          else
-          {
-            return false;
-          }
-
-          i++;
-          return true;
-        });
-
-      if (i != count)
-        ok = false;
-
-      return ok;
     }
 
     bool operator!=(const AbstractMap& that) const override
@@ -570,155 +48,164 @@ namespace kv
       return !(*this == that);
     }
 
+    AbstractStore* get_store() override
+    {
+      return untyped_map.get_store();
+    }
+
+    void serialise(
+      const AbstractTxView* view,
+      KvStoreSerialiser& s,
+      bool include_reads) override
+    {
+      untyped_map.serialise(view, s, include_reads);
+    }
+
+    AbstractTxView* deserialise(
+      KvStoreDeserialiser& d, Version version) override
+    {
+      return untyped_map.deserialise_internal<TxView>(d, version);
+    }
+
+    const std::string& get_name() const override
+    {
+      return untyped_map.get_name();
+    }
+
     void compact(Version v) override
     {
-      // This discards available rollback state before version v, and populates
+      return untyped_map.compact(v);
-      // the commit_deltas to be passed to the global commit hook, if there is
-      // one, up to version v. The Map expects to be locked during compaction.
-      while (roll->get_head() != roll->get_tail())
-      {
-        auto r = roll->get_head();
-
-        // Globally committed but not discardable.
-        if (r->version == v)
-        {
-          // We know that write set is not empty.
-          if (global_hook)
-          {
-            commit_deltas.insert_back(
-              create_new_local_commit(r->version, r->state, move(r->writes)));
-          }
-          return;
-        }
-
-        // Discardable, so move to commit_deltas.
-        if (global_hook && !r->writes.empty())
-        {
-          commit_deltas.insert_back(
-            create_new_local_commit(r->version, r->state, move(r->writes)));
-        }
-
-        // Stop if the next state may be rolled back or is the only state.
-        // This ensures there is always a state present.
-        if (r->next->version > v)
-          return;
-
-        auto c = roll->pop();
-        empty_commits.insert(c);
-      }
-
-      // There is only one roll. We may need to call the commit hook.
-      auto r = roll->get_head();
-
-      if (global_hook && !r->writes.empty())
-      {
-        commit_deltas.insert_back(
-          create_new_local_commit(r->version, r->state, move(r->writes)));
-      }
     }
 
     void post_compact() override
     {
-      if (global_hook)
+      return untyped_map.post_compact();
-      {
-        for (auto r = commit_deltas.get_head(); r != nullptr; r = r->next)
-        {
-          global_hook(r->version, r->writes);
-        }
-      }
-
-      commit_deltas.clear();
     }
 
     void rollback(Version v) override
     {
-      // This rolls the current state back to version v.
+      untyped_map.rollback(v);
-      // The Map expects to be locked during rollback.
-      bool advance = false;
-
-      while (roll->get_head() != roll->get_tail())
-      {
-        auto r = roll->get_tail();
-
-        // The initial empty state has v = 0, so will not be discarded if it
-        // is present.
-        if (r->version <= v)
-          break;
-
-        advance = true;
-        auto c = roll->pop_tail();
-        empty_commits.insert(c);
-      }
-
-      if (advance)
-        rollback_counter++;
-    }
-
-    void clear() override
-    {
-      // This discards all entries in the roll and resets the compacted value
-      // and rollback counter. The Map expects to be locked before clearing it.
-      roll->clear();
-      roll->insert_back(create_new_local_commit(0, State(), Write()));
-      rollback_counter = 0;
     }
 
     void lock() override
     {
-      sl.lock();
+      untyped_map.lock();
     }
 
     void unlock() override
     {
-      sl.unlock();
+      untyped_map.unlock();
     }
 
-    void swap(AbstractMap* map_) override
+    SecurityDomain get_security_domain() override
     {
-      This* map = dynamic_cast<This*>(map_);
+      return untyped_map.get_security_domain();
-      if (map == nullptr)
+    }
+
+    bool is_replicated() override
+    {
+      return untyped_map.is_replicated();
+    }
+
+    void clear() override
+    {
+      untyped_map.clear();
+    }
+
+    AbstractMap* clone(AbstractStore* store) override
+    {
+      return new TypedMap(
+        store,
+        untyped_map.get_name(),
+        untyped_map.get_security_domain(),
+        untyped_map.is_replicated());
+    }
+
+    void swap(AbstractMap* map) override
+    {
+      auto p = dynamic_cast<This*>(map);
+      if (p == nullptr)
         throw std::logic_error(
           "Attempted to swap maps with incompatible types");
 
-      std::swap(rollback_counter, map->rollback_counter);
+      untyped_map.swap(&p->untyped_map);
-      std::swap(roll, map->roll);
     }
 
     template <typename TView>
-    TView* create_view(Version version)
+    TView* create_view(Version v)
     {
-      lock();
+      return untyped_map.create_view<TView>(v);
+    }
 
-      // Find the last entry committed at or before this version.
+    static kv::untyped::Map::CommitHook wrap_commit_hook(const CommitHook& hook)
-      TView* view = nullptr;
+    {
-
+      return [hook](Version v, const kv::untyped::Write& w) {
-      for (auto current = roll->get_tail(); current != nullptr;
+        Write typed_writes;
-           current = current->prev)
+        for (const auto& [uk, opt_uv] : w)
-      {
-        if (current->version <= version)
         {
-          view = new TView(
+          if (!opt_uv.has_value())
-            *this,
+          {
-            rollback_counter,
+            // Deletions are indicated by nullopt. We cannot deserialise them,
-            current->state,
+            // they are deletions here as well
-            roll->get_head()->state,
+            typed_writes[KSerialiser::from_serialised(uk)] = std::nullopt;
-            current->version);
+          }
-          break;
+          else
+          {
+            typed_writes[KSerialiser::from_serialised(uk)] =
+              VSerialiser::from_serialised(opt_uv.value());
+          }
         }
-      }
 
-      if (view == nullptr)
+        hook(v, typed_writes);
-      {
+      };
-        view = new TView(
+    }
-          *this,
-          rollback_counter,
-          roll->get_head()->state,
-          roll->get_head()->state,
-          roll->get_head()->version);
-      }
 
-      unlock();
+    void set_local_hook(const CommitHook& hook)
-      return view;
+    {
+      untyped_map.set_local_hook(wrap_commit_hook(hook));
+    }
+
+    void unset_local_hook()
+    {
+      untyped_map.unset_local_hook();
+    }
+
+    void set_global_hook(const CommitHook& hook)
+    {
+      untyped_map.set_global_hook(wrap_commit_hook(hook));
+    }
+
+    void unset_global_hook()
+    {
+      untyped_map.unset_global_hook();
     }
   };
+
+  template <
+    typename K,
+    typename V,
+    template <typename>
+    typename KSerialiser,
+    template <typename> typename VSerialiser = KSerialiser>
+  using MapSerialisedWith = TypedMap<K, V, KSerialiser<K>, VSerialiser<V>>;
+
+  template <typename K, typename V>
+  using JsonSerialisedMap =
+    MapSerialisedWith<K, V, kv::serialisers::JsonSerialiser>;
+
+  template <typename K, typename V>
+  using RawCopySerialisedMap = TypedMap<
+    K,
+    V,
+    kv::serialisers::BlitSerialiser<K>,
+    kv::serialisers::BlitSerialiser<V>>;
+
+  template <typename K, typename V>
+  using MsgPackSerialisedMap =
+    MapSerialisedWith<K, V, kv::serialisers::MsgPackSerialiser>;
+
+  // The default kv::Map will use msgpack serialisers. Custom types are
+  // supported through the MSGPACK_DEFINE macro
+  template <typename K, typename V>
+  using Map = MsgPackSerialisedMap<K, V>;
 }

src/kv/msgpack_serialise.h

@@ -5,6 +5,7 @@
 #include "ds/msgpack_adaptor_nlohmann.h"
 #include "ds/serialized.h"
 #include "generic_serialise_wrapper.h"
+#include "serialised_entry.h"
 
 #include <iterator>
 #include <msgpack/msgpack.hpp>
@@ -15,6 +16,13 @@
 MSGPACK_ADD_ENUM(kv::KvOperationType);
 MSGPACK_ADD_ENUM(kv::SecurityDomain);
 
+// Currently we have pre-serialised keys and values, which are often msgpack,
+// which are then re-packed into msgpack to go into the ledger. This is
+// wasteful. But without this we can't _unpack_ custom types at this level. We
+// should replace this with a custom serialisation format for the ledger. This
+// macro gates the intended code path.
+#define MSGPACK_DONT_REPACK (1)
+
 namespace kv
 {
   class MsgPackWriter
@@ -29,6 +37,21 @@ namespace kv
       msgpack::pack(sb, std::forward<T>(t));
     }
 
+    // Where we have pre-serialised data, we dump it directly into the output
+    // buffer. If we call append, then pack will prefix the data with some type
+    // information, potentially redundantly repacking already-packed data. We
+    // assume the serialised entry is already msgpack so we retain a consistent
+    // msgpack stream. If it is in some other format, every parser will need to
+    // be able to distinguish it from ths valid stream
+    void append_pre_serialised(const kv::serialisers::SerialisedEntry& entry)
+    {
+#if MSGPACK_DONT_REPACK
+      sb.write(reinterpret_cast<char const*>(entry.data()), entry.size());
+#else
+      append(entry);
+#endif
+    }
+
     void clear()
     {
       sb.clear();
@@ -77,6 +100,18 @@ namespace kv
       return msg->as<T>();
     }
 
+    kv::serialisers::SerialisedEntry read_next_pre_serialised()
+    {
+#if MSGPACK_DONT_REPACK
+      const auto before_offset = data_offset;
+      msgpack::unpack(msg, data_ptr, data_size, data_offset);
+      return kv::serialisers::SerialisedEntry(
+        data_ptr + before_offset, data_ptr + data_offset);
+#else
+      return read_next<kv::serialisers::SerialisedEntry>();
+#endif
+    }
+
     template <typename T>
     T peek_next()
     {

src/kv/serialise_entry_blit.h

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "serialised_entry.h"
+
+namespace kv::serialisers
+{
+  namespace
+  {
+    template <typename T>
+    struct is_std_array : std::false_type
+    {};
+
+    template <typename T, size_t N>
+    struct is_std_array<std::array<T, N>> : public std::true_type
+    {};
+
+    template <typename T>
+    struct dependent_false : public std::false_type
+    {};
+  }
+
+  template <typename T>
+  struct BlitSerialiser
+  {
+    static SerialisedEntry to_serialised(const T& t)
+    {
+      if constexpr (std::is_same_v<T, std::vector<uint8_t>>)
+      {
+        return SerialisedEntry(t.begin(), t.end());
+      }
+      else if constexpr (is_std_array<T>::value)
+      {
+        return SerialisedEntry(t.begin(), t.end());
+      }
+      else if constexpr (std::is_integral_v<T>)
+      {
+        SerialisedEntry s(sizeof(t));
+        std::memcpy(s.data(), (uint8_t*)&t, sizeof(t));
+        return s;
+      }
+      else
+      {
+        static_assert(dependent_false<T>::value, "Can't serialise this type");
+      }
+    }
+
+    static T from_serialised(const SerialisedEntry& rep)
+    {
+      if constexpr (std::is_same_v<T, std::vector<uint8_t>>)
+      {
+        return T(rep.begin(), rep.end());
+      }
+      else if constexpr (is_std_array<T>::value)
+      {
+        return T(rep.begin(), rep.end());
+      }
+      else if constexpr (std::is_integral_v<T>)
+      {
+        if (rep.size() != sizeof(T))
+        {
+          throw std::logic_error("Wrong size for deserialising");
+        }
+        return *(T*)rep.data();
+      }
+      else
+      {
+        static_assert(dependent_false<T>::value, "Can't deserialise this type");
+      }
+    }
+  };
+}

src/kv/serialise_entry_json.h

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "serialised_entry.h"
+
+#include <nlohmann/json.hpp>
+
+namespace kv::serialisers
+{
+  template <typename T>
+  struct JsonSerialiser
+  {
+    static SerialisedEntry to_serialised(const T& t)
+    {
+      const nlohmann::json j = t;
+      const auto dumped = j.dump();
+      return SerialisedEntry(dumped.begin(), dumped.end());
+    }
+
+    static T from_serialised(const SerialisedEntry& rep)
+    {
+      const auto j = nlohmann::json::parse(rep);
+      return j.get<T>();
+    }
+  };
+}

src/kv/serialise_entry_msgpack.h

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "serialised_entry.h"
+
+#include <msgpack/msgpack.hpp>
+
+namespace kv::serialisers
+{
+  namespace detail
+  {
+    struct SerialisedEntryWriter
+    {
+      SerialisedEntry& entry;
+
+      void write(const char* d, size_t n)
+      {
+        entry.insert(entry.end(), d, d + n);
+      }
+    };
+  }
+
+  template <typename T>
+  struct MsgPackSerialiser
+  {
+    static SerialisedEntry to_serialised(const T& t)
+    {
+      SerialisedEntry e;
+      detail::SerialisedEntryWriter w{e};
+      msgpack::pack(w, t);
+      return e;
+    }
+
+    static T from_serialised(const SerialisedEntry& rep)
+    {
+      msgpack::object_handle oh =
+        msgpack::unpack(reinterpret_cast<const char*>(rep.data()), rep.size());
+      auto object = oh.get();
+      return object.as<T>();
+    }
+  };
+}

src/kv/serialised_entry.h

@@ -0,0 +1,12 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include <msgpack/msgpack.hpp>
+#include <nlohmann/json.hpp>
+#include <small_vector/SmallVector.h>
+
+namespace kv::serialisers
+{
+  using SerialisedEntry = llvm_vecsmall::SmallVector<uint8_t, 8>;
+}

src/kv/store.h

@@ -106,10 +106,10 @@ namespace kv
       return encryptor;
     }
 
-    template <class K, class V, class H = std::hash<K>>
+    template <class K, class V>
-    Map<K, V, H>* get(std::string name)
+    Map<K, V>* get(std::string name)
     {
-      return get<Map<K, V, H>>(name);
+      return get<Map<K, V>>(name);
     }
 
     /** Get Map by name
@@ -147,12 +147,12 @@ namespace kv
      *
      * @return Newly created Map
      */
-    template <class K, class V, class H = std::hash<K>>
+    template <class K, class V>
-    Map<K, V, H>& create(
+    Map<K, V>& create(
       std::string name,
       SecurityDomain security_domain = kv::SecurityDomain::PRIVATE)
     {
-      return create<Map<K, V, H>>(name, security_domain);
+      return create<Map<K, V>>(name, security_domain);
     }
 
     /** Create a Map
@@ -297,7 +297,7 @@ namespace kv
         return DeserialiseSuccess::FAILED;
       }
 
-      Version v = d->template deserialise_version<Version>();
+      Version v = d->deserialise_version();
       // Throw away any local commits that have not propagated via the
       // consensus.
       rollback(v - 1);

src/kv/test/kv_bench.cpp

@@ -7,14 +7,37 @@
 #include "kv/tx.h"
 #include "node/encryptor.h"
 
+#include <msgpack/msgpack.hpp>
 #include <picobench/picobench.hpp>
 #include <string>
 
+using KeyType = kv::serialisers::SerialisedEntry;
+using ValueType = kv::serialisers::SerialisedEntry;
+using MapType = kv::untyped::Map;
+
 inline void clobber_memory()
 {
   asm volatile("" : : : "memory");
 }
 
+KeyType gen_key(size_t i, const std::string& suf = "")
+{
+  const auto s = "key" + std::to_string(i) + suf;
+  msgpack::sbuffer buf;
+  msgpack::pack(buf, s);
+  const auto raw = reinterpret_cast<const uint8_t*>(buf.data());
+  return KeyType(raw, raw + buf.size());
+}
+
+ValueType gen_value(size_t i)
+{
+  const auto s = "value" + std::to_string(i);
+  msgpack::sbuffer buf;
+  msgpack::pack(buf, s);
+  const auto raw = reinterpret_cast<const uint8_t*>(buf.data());
+  return ValueType(raw, raw + buf.size());
+}
+
 // Helper functions to use a dummy encryption key
 std::shared_ptr<ccf::LedgerSecrets> create_ledger_secrets()
 {
@@ -37,16 +60,17 @@ static void serialise(picobench::state& s)
   encryptor->set_iv_id(1);
   kv_store.set_encryptor(encryptor);
 
-  auto& map0 = kv_store.create<std::string, std::string>("map0", SD);
+  auto& map0 = kv_store.create<MapType>("map0", SD);
-  auto& map1 = kv_store.create<std::string, std::string>("map1", SD);
+  auto& map1 = kv_store.create<MapType>("map1", SD);
   kv::Tx tx;
   auto [tx0, tx1] = tx.get_view(map0, map1);
 
   for (int i = 0; i < s.iterations(); i++)
   {
-    auto key = "key" + std::to_string(i);
+    const auto key = gen_key(i);
-    tx0->put(key, "value");
+    const auto value = gen_value(i);
-    tx1->put(key, "value");
+    tx0->put(key, value);
+    tx1->put(key, value);
   }
 
   s.start_timer();
@@ -71,18 +95,19 @@ static void deserialise(picobench::state& s)
   kv_store.set_encryptor(encryptor);
   kv_store2.set_encryptor(encryptor);
 
-  auto& map0 = kv_store.create<std::string, std::string>("map0", SD);
+  auto& map0 = kv_store.create<MapType>("map0", SD);
-  auto& map1 = kv_store.create<std::string, std::string>("map1", SD);
+  auto& map1 = kv_store.create<MapType>("map1", SD);
-  auto& map0_ = kv_store2.create<std::string, std::string>("map0", SD);
+  auto& map0_ = kv_store2.create<MapType>("map0", SD);
-  auto& map1_ = kv_store2.create<std::string, std::string>("map1", SD);
+  auto& map1_ = kv_store2.create<MapType>("map1", SD);
   kv::Tx tx;
   auto [tx0, tx1] = tx.get_view(map0, map1);
 
   for (int i = 0; i < s.iterations(); i++)
   {
-    auto key = "key" + std::to_string(i);
+    const auto key = gen_key(i);
-    tx0->put(key, "value");
+    const auto value = gen_value(i);
-    tx1->put(key, "value");
+    tx0->put(key, value);
+    tx1->put(key, value);
   }
   tx.commit();
 
@@ -105,8 +130,8 @@ static void commit_latency(picobench::state& s)
   encryptor->set_iv_id(1);
   kv_store.set_encryptor(encryptor);
 
-  auto& map0 = kv_store.create<std::string, std::string>("map0");
+  auto& map0 = kv_store.create<MapType>("map0");
-  auto& map1 = kv_store.create<std::string, std::string>("map1");
+  auto& map1 = kv_store.create<MapType>("map1");
 
   for (int i = 0; i < s.iterations(); i++)
   {
@@ -114,9 +139,10 @@ static void commit_latency(picobench::state& s)
     auto [tx0, tx1] = tx.get_view(map0, map1);
     for (int iTx = 0; iTx < S; iTx++)
     {
-      auto key = "key" + std::to_string(i) + " - " + std::to_string(iTx);
+      const auto key = gen_key(i, std::to_string(iTx));
-      tx0->put(key, "value");
+      const auto value = gen_value(i);
-      tx1->put(key, "value");
+      tx0->put(key, value);
+      tx1->put(key, value);
     }
 
     auto rc = tx.commit();
@@ -153,3 +179,9 @@ PICOBENCH(deserialise<SD::PUBLIC>)
   .samples(sample_size)
   .baseline();
 PICOBENCH(deserialise<SD::PRIVATE>).iterations(tx_count).samples(sample_size);
+
+// int main(int argc, char* argv[])
+// {
+//   picobench::state s(1'000'000);
+//   serialise<SD::PUBLIC>(s);
+// }

src/kv/test/kv_serialisation.cpp

@@ -2,7 +2,6 @@
 // Licensed under the Apache 2.0 License.
 #include "ds/logger.h"
 #include "kv/encryptor.h"
-#include "kv/experimental.h"
 #include "kv/kv_serialiser.h"
 #include "kv/store.h"
 #include "kv/test/null_encryptor.h"
@@ -14,7 +13,7 @@
 #include <string>
 #include <vector>
 
-struct RawMapTypes
+struct MapTypes
 {
   using StringString = kv::Map<std::string, std::string>;
   using NumNum = kv::Map<size_t, size_t>;
@@ -22,20 +21,9 @@ struct RawMapTypes
   using StringNum = kv::Map<std::string, size_t>;
 };
 
-struct ExperimentalMapTypes
+TEST_CASE(
-{
-  using StringString = kv::experimental::Map<std::string, std::string>;
-  using NumNum = kv::experimental::Map<size_t, size_t>;
-  using NumString = kv::experimental::Map<size_t, std::string>;
-  using StringNum = kv::experimental::Map<std::string, size_t>;
-};
-
-TEST_CASE_TEMPLATE(
   "Serialise/deserialise public map only" *
-    doctest::test_suite("serialisation"),
+  doctest::test_suite("serialisation"))
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
   // No need for an encryptor here as all maps are public. Both serialisation
   // and deserialisation should succeed.
@@ -43,12 +31,12 @@ TEST_CASE_TEMPLATE(
 
   kv::Store kv_store(consensus);
 
-  auto& pub_map = kv_store.create<typename MapImpl::StringString>(
+  auto& pub_map = kv_store.create<MapTypes::StringString>(
     "pub_map", kv::SecurityDomain::PUBLIC);
 
   kv::Store kv_store_target;
   kv_store_target.clone_schema(kv_store);
-  auto* target_map = kv_store.get<typename MapImpl::StringString>("pub_map");
+  auto* target_map = kv_store.get<MapTypes::StringString>("pub_map");
   REQUIRE(target_map != nullptr);
 
   INFO("Commit to public map in source store");
@@ -73,23 +61,20 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE(
   "Serialise/deserialise private map only" *
-    doctest::test_suite("serialisation"),
+  doctest::test_suite("serialisation"))
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
   auto consensus = std::make_shared<kv::StubConsensus>();
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
 
   kv::Store kv_store(consensus);
-  auto& priv_map = kv_store.create<typename MapImpl::StringString>("priv_map");
+  auto& priv_map = kv_store.create<MapTypes::StringString>("priv_map");
 
   kv::Store kv_store_target;
   kv_store_target.set_encryptor(encryptor);
   kv_store_target.clone_schema(kv_store);
-  auto* target_map = kv_store.get<typename MapImpl::StringString>("priv_map");
+  auto* target_map = kv_store.get<MapTypes::StringString>("priv_map");
   REQUIRE(target_map != nullptr);
 
   INFO("Commit a private transaction without an encryptor throws an exception");
@@ -126,29 +111,24 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE(
   "Serialise/deserialise private map and public maps" *
-    doctest::test_suite("serialisation"),
+  doctest::test_suite("serialisation"))
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
   auto consensus = std::make_shared<kv::StubConsensus>();
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
 
   kv::Store kv_store(consensus);
   kv_store.set_encryptor(encryptor);
-  auto& priv_map = kv_store.create<typename MapImpl::StringString>("priv_map");
+  auto& priv_map = kv_store.create<MapTypes::StringString>("priv_map");
-  auto& pub_map = kv_store.create<typename MapImpl::StringString>(
+  auto& pub_map = kv_store.create<MapTypes::StringString>(
     "pub_map", kv::SecurityDomain::PUBLIC);
 
   kv::Store kv_store_target;
   kv_store_target.set_encryptor(encryptor);
   kv_store_target.clone_schema(kv_store);
-  auto* target_priv_map =
+  auto* target_priv_map = kv_store.get<MapTypes::StringString>("priv_map");
-    kv_store.get<typename MapImpl::StringString>("priv_map");
+  auto* target_pub_map = kv_store.get<MapTypes::StringString>("pub_map");
-  auto* target_pub_map =
-    kv_store.get<typename MapImpl::StringString>("pub_map");
   REQUIRE(target_priv_map != nullptr);
   REQUIRE(target_pub_map != nullptr);
 
@@ -177,24 +157,20 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE(
-  "Serialise/deserialise removed keys" * doctest::test_suite("serialisation"),
+  "Serialise/deserialise removed keys" * doctest::test_suite("serialisation"))
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
   auto consensus = std::make_shared<kv::StubConsensus>();
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
 
   kv::Store kv_store(consensus);
   kv_store.set_encryptor(encryptor);
-  auto& priv_map = kv_store.create<typename MapImpl::StringString>("priv_map");
+  auto& priv_map = kv_store.create<MapTypes::StringString>("priv_map");
 
   kv::Store kv_store_target;
   kv_store_target.set_encryptor(encryptor);
   kv_store_target.clone_schema(kv_store);
-  auto* target_priv_map =
+  auto* target_priv_map = kv_store.get<MapTypes::StringString>("priv_map");
-    kv_store.get<typename MapImpl::StringString>("priv_map");
   REQUIRE(target_priv_map != nullptr);
 
   INFO("Commit a new key in source store and deserialise in target store");
@@ -236,112 +212,19 @@ TEST_CASE_TEMPLATE(
 }
 
 struct CustomClass
-{
-  int m_i;
-
-  CustomClass() : CustomClass(-1) {}
-  CustomClass(int i) : m_i(i) {}
-
-  int get() const
-  {
-    return m_i;
-  }
-  void set(std::string val)
-  {
-    m_i = std::stoi(val);
-  }
-
-  CustomClass operator()()
-  {
-    CustomClass ret;
-    return ret;
-  }
-
-  bool operator<(const CustomClass& other) const
-  {
-    return m_i < other.m_i;
-  }
-
-  bool operator==(const CustomClass& other) const
-  {
-    return !(other < *this) && !(*this < other);
-  }
-
-  MSGPACK_DEFINE(m_i);
-};
-
-namespace std
-{
-  template <>
-  struct hash<CustomClass>
-  {
-    std::size_t operator()(const CustomClass& inst) const
-    {
-      return inst.get();
-    }
-  };
-}
-
-DECLARE_JSON_TYPE(CustomClass)
-DECLARE_JSON_REQUIRED_FIELDS(CustomClass, m_i)
-
-TEST_CASE(
-  "Custom type serialisation test (original scheme)" *
-  doctest::test_suite("serialisation"))
-{
-  kv::Store kv_store;
-
-  auto& map = kv_store.create<CustomClass, CustomClass>(
-    "map", kv::SecurityDomain::PUBLIC);
-
-  CustomClass k(3);
-  CustomClass v1(33);
-
-  CustomClass k2(2);
-  CustomClass v2(22);
-
-  INFO("Serialise/Deserialise 2 kv stores");
-  {
-    kv::Store kv_store2;
-    auto& map2 = kv_store2.create<CustomClass, CustomClass>(
-      "map", kv::SecurityDomain::PUBLIC);
-
-    kv::Tx tx(kv_store.next_version());
-    auto view = tx.get_view(map);
-    view->put(k, v1);
-    view->put(k2, v2);
-
-    auto [success, reqid, data] = tx.commit_reserved();
-    REQUIRE(success == kv::CommitSuccess::OK);
-    kv_store.compact(kv_store.current_version());
-
-    REQUIRE(kv_store2.deserialise(data) == kv::DeserialiseSuccess::PASS);
-    kv::Tx tx2;
-    auto view2 = tx2.get_view(map2);
-    auto va = view2->get(k);
-
-    REQUIRE(va.has_value());
-    REQUIRE(va.value() == v1);
-    auto vb = view2->get(k2);
-    REQUIRE(vb.has_value());
-    REQUIRE(vb.value() == v2);
-    // we only require operator==() to be implemented, so for consistency -
-    // this is the operator we use for comparison, and not operator!=()
-    REQUIRE(!(vb.value() == v1));
-  }
-}
-
-struct CustomClass2
 {
   std::string s;
   size_t n;
+
+  // This macro allows the default serialiser to be used
+  MSGPACK_DEFINE(s, n);
 };
 
 struct CustomJsonSerialiser
 {
-  using Bytes = kv::experimental::SerialisedRep;
+  using Bytes = kv::serialisers::SerialisedEntry;
 
-  static Bytes to_serialised(const CustomClass2& c)
+  static Bytes to_serialised(const CustomClass& c)
   {
     nlohmann::json j = nlohmann::json::object();
     j["s"] = c.s;
@@ -350,10 +233,10 @@ struct CustomJsonSerialiser
     return Bytes(s.begin(), s.end());
   }
 
-  static CustomClass2 from_serialised(const Bytes& b)
+  static CustomClass from_serialised(const Bytes& b)
   {
     const auto j = nlohmann::json::parse(b);
-    CustomClass2 c;
+    CustomClass c;
     c.s = j["s"];
     c.n = j["n"];
     return c;
@@ -373,15 +256,15 @@ struct VPrefix
 template <typename T>
 struct CustomVerboseDumbSerialiser
 {
-  using Bytes = kv::experimental::SerialisedRep;
+  using Bytes = kv::serialisers::SerialisedEntry;
 
-  static Bytes to_serialised(const CustomClass2& c)
+  static Bytes to_serialised(const CustomClass& c)
   {
     const auto verbose = fmt::format("{}\ns={}\nn={}", T::prefix, c.s, c.n);
     return Bytes(verbose.begin(), verbose.end());
   }
 
-  static CustomClass2 from_serialised(const Bytes& b)
+  static CustomClass from_serialised(const Bytes& b)
   {
     std::string s(b.begin(), b.end());
     const auto prefix_start = s.find(T::prefix);
@@ -390,7 +273,7 @@ struct CustomVerboseDumbSerialiser
       throw std::logic_error("Missing expected prefix");
     }
 
-    CustomClass2 c;
+    CustomClass c;
     const auto first_linebreak = s.find('\n');
     const auto last_linebreak = s.rfind('\n');
     const auto seg_a = s.substr(0, first_linebreak);
@@ -405,30 +288,37 @@ struct CustomVerboseDumbSerialiser
   }
 };
 
-using MapA = kv::experimental::
+using DefaultSerialisedMap = kv::Map<CustomClass, CustomClass>;
-  Map<CustomClass2, CustomClass2, CustomJsonSerialiser, CustomJsonSerialiser>;
+using CustomJsonMap = kv::TypedMap<
-using MapB = kv::experimental::Map<
+  CustomClass,
-  CustomClass2,
+  CustomClass,
-  CustomClass2,
+  CustomJsonSerialiser,
+  CustomJsonSerialiser>;
+using VerboseSerialisedMap = kv::TypedMap<
+  CustomClass,
+  CustomClass,
   CustomVerboseDumbSerialiser<KPrefix>,
   CustomVerboseDumbSerialiser<VPrefix>>;
 
+// While the ledger expects everything to be msgpack, we don't actually support
+// custom types
+#if !MSGPACK_DONT_REPACK
 TEST_CASE_TEMPLATE(
-  "Custom type serialisation test (experimental scheme)" *
+  "Custom type serialisation test" * doctest::test_suite("serialisation"),
-    doctest::test_suite("serialisation"),
   MapType,
-  MapA,
+  DefaultSerialisedMap,
-  MapB)
+  CustomJsonMap,
+  VerboseSerialisedMap)
 {
   kv::Store kv_store;
 
   auto& map = kv_store.create<MapType>("map", kv::SecurityDomain::PUBLIC);
 
-  CustomClass2 k1{"hello", 42};
+  CustomClass k1{"hello", 42};
-  CustomClass2 v1{"world", 43};
+  CustomClass v1{"world", 43};
 
-  CustomClass2 k2{"saluton", 100};
+  CustomClass k2{"saluton", 100};
-  CustomClass2 v2{"mondo", 1024};
+  CustomClass v2{"mondo", 1024};
 
   INFO("Serialise/Deserialise 2 kv stores");
   {
@@ -461,6 +351,7 @@ TEST_CASE_TEMPLATE(
     REQUIRE(vb->n == v2.n);
   }
 }
+#endif
 
 bool corrupt_serialised_tx(
   std::vector<uint8_t>& serialised_tx, std::vector<uint8_t>& value_to_corrupt)
@@ -488,11 +379,7 @@ bool corrupt_serialised_tx(
   return false;
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Integrity" * doctest::test_suite("serialisation"))
-  "Integrity" * doctest::test_suite("serialisation"),
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
   SUBCASE("Public and Private")
   {
@@ -515,10 +402,9 @@ TEST_CASE_TEMPLATE(
     kv_store.set_encryptor(encryptor);
     kv_store_target.set_encryptor(encryptor);
 
-    auto& public_map = kv_store.create<typename MapImpl::StringString>(
+    auto& public_map = kv_store.create<MapTypes::StringString>(
       "public_map", kv::SecurityDomain::PUBLIC);
-    auto& private_map =
+    auto& private_map = kv_store.create<MapTypes::StringString>("private_map");
-      kv_store.create<typename MapImpl::StringString>("private_map");
 
     kv_store_target.clone_schema(kv_store);
 
@@ -584,14 +470,11 @@ TEST_CASE("nlohmann (de)serialisation" * doctest::test_suite("serialisation"))
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE(
   "Replicated and derived table serialisation" *
-    doctest::test_suite("serialisation"),
+  doctest::test_suite("serialisation"))
-  MapImpl,
-  RawMapTypes,
-  ExperimentalMapTypes)
 {
-  using T = typename MapImpl::NumNum;
+  using T = MapTypes::NumNum;
 
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
   std::unordered_set<std::string> replicated_tables = {
@@ -668,76 +551,9 @@ TEST_CASE_TEMPLATE(
 struct NonSerialisable
 {};
 
-namespace msgpack
-{
-  MSGPACK_API_VERSION_NAMESPACE(MSGPACK_DEFAULT_API_NS)
-  {
-    namespace adaptor
-    {
-      // msgpack conversion for uint256_t
-      template <>
-      struct convert<NonSerialisable>
-      {
-        msgpack::object const& operator()(
-          msgpack::object const& o, NonSerialisable& ns) const
-        {
-          throw std::runtime_error("Deserialise failure");
-        }
-      };
-
-      template <>
-      struct pack<NonSerialisable>
-      {
-        template <typename Stream>
-        packer<Stream>& operator()(
-          msgpack::packer<Stream>& o, NonSerialisable const& ns) const
-        {
-          throw std::runtime_error("Serialise failure");
-        }
-      };
-    }
-  }
-}
-
-TEST_CASE(
-  "Exceptional serdes (old scheme)" * doctest::test_suite("serialisation"))
-{
-  auto encryptor = std::make_shared<kv::NullTxEncryptor>();
-  auto consensus = std::make_shared<kv::StubConsensus>();
-
-  kv::Store store(consensus);
-  store.set_encryptor(encryptor);
-
-  auto& good_map = store.create<size_t, size_t>("good_map");
-  auto& bad_map = store.create<size_t, NonSerialisable>("bad_map");
-
-  {
-    kv::Tx tx;
-    auto good_view = tx.get_view(good_map);
-    good_view->put(1, 2);
-    REQUIRE(tx.commit() == kv::CommitSuccess::OK);
-  }
-
-  {
-    kv::Tx tx;
-    auto bad_view = tx.get_view(bad_map);
-    bad_view->put(0, {});
-    REQUIRE_THROWS_AS(tx.commit(), kv::KvSerialiserException);
-  }
-
-  {
-    kv::Tx tx;
-    auto good_view = tx.get_view(good_map);
-    good_view->put(1, 2);
-    auto bad_view = tx.get_view(bad_map);
-    bad_view->put(0, {});
-    REQUIRE_THROWS_AS(tx.commit(), kv::KvSerialiserException);
-  }
-}
-
 struct NonSerialiser
 {
-  using Bytes = kv::experimental::SerialisedRep;
+  using Bytes = kv::serialisers::SerialisedEntry;
 
   static Bytes to_serialised(const NonSerialisable& ns)
   {
@@ -750,9 +566,7 @@ struct NonSerialiser
   }
 };
 
-TEST_CASE(
+TEST_CASE("Exceptional serdes" * doctest::test_suite("serialisation"))
-  "Exceptional serdes (experimental scheme)" *
-  doctest::test_suite("serialisation"))
 {
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
   auto consensus = std::make_shared<kv::StubConsensus>();
@@ -760,15 +574,15 @@ TEST_CASE(
   kv::Store store(consensus);
   store.set_encryptor(encryptor);
 
-  auto& bad_map_k = store.create<kv::experimental::Map<
+  auto& bad_map_k = store.create<kv::TypedMap<
     NonSerialisable,
     size_t,
     NonSerialiser,
-    kv::experimental::MsgPackSerialiser<size_t>>>("bad_map_k");
+    kv::serialisers::MsgPackSerialiser<size_t>>>("bad_map_k");
-  auto& bad_map_v = store.create<kv::experimental::Map<
+  auto& bad_map_v = store.create<kv::TypedMap<
     size_t,
     NonSerialisable,
-    kv::experimental::MsgPackSerialiser<size_t>,
+    kv::serialisers::MsgPackSerialiser<size_t>,
     NonSerialiser>>("bad_map_v");
 
   {

src/kv/test/kv_test.cpp

@@ -2,7 +2,6 @@
 // Licensed under the Apache 2.0 License.
 #include "ds/logger.h"
 #include "enclave/app_interface.h"
-#include "kv/experimental.h"
 #include "kv/kv_serialiser.h"
 #include "kv/store.h"
 #include "kv/test/null_encryptor.h"
@@ -15,7 +14,7 @@
 #include <string>
 #include <vector>
 
-struct RawMapTypes
+struct MapTypes
 {
   using StringString = kv::Map<std::string, std::string>;
   using NumNum = kv::Map<size_t, size_t>;
@@ -23,65 +22,54 @@ struct RawMapTypes
   using StringNum = kv::Map<std::string, size_t>;
 };
 
-struct ExperimentalMapTypes
+TEST_CASE("Map creation")
-{
-  using StringString = kv::experimental::Map<std::string, std::string>;
-  using NumNum = kv::experimental::Map<size_t, size_t>;
-  using NumString = kv::experimental::Map<size_t, std::string>;
-  using StringNum = kv::experimental::Map<std::string, size_t>;
-};
-
-TEST_CASE_TEMPLATE("Map creation", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store kv_store;
   const auto map_name = "map";
-  auto& map = kv_store.create<typename MapImpl::StringString>(map_name);
+  auto& map = kv_store.create<MapTypes::StringString>(map_name);
 
   INFO("Get a map that does not exist");
   {
-    REQUIRE(
+    REQUIRE(kv_store.get<MapTypes::StringString>("invalid_map") == nullptr);
-      kv_store.get<typename MapImpl::StringString>("invalid_map") == nullptr);
   }
 
   INFO("Get a map that does exist");
   {
-    auto* p_map = kv_store.get<typename MapImpl::StringString>(map_name);
+    auto* p_map = kv_store.get<MapTypes::StringString>(map_name);
     REQUIRE(*p_map == map);
     REQUIRE(p_map == &map); // They're the _same instance_, not just equal
   }
 
   INFO("Compare different maps");
   {
-    auto& map2 = kv_store.create<typename MapImpl::StringString>("map2");
+    auto& map2 = kv_store.create<MapTypes::StringString>("map2");
     REQUIRE(map != map2);
   }
 
   INFO("Can't create map that already exists");
   {
     REQUIRE_THROWS_AS(
-      kv_store.create<typename MapImpl::StringString>(map_name),
+      kv_store.create<MapTypes::StringString>(map_name), std::logic_error);
-      std::logic_error);
   }
 
   INFO("Can't get a map with the wrong type");
   {
-    REQUIRE(kv_store.get<typename MapImpl::NumNum>(map_name) == nullptr);
+    REQUIRE(kv_store.get<MapTypes::NumNum>(map_name) == nullptr);
-    REQUIRE(kv_store.get<typename MapImpl::NumString>(map_name) == nullptr);
+    REQUIRE(kv_store.get<MapTypes::NumString>(map_name) == nullptr);
-    REQUIRE(kv_store.get<typename MapImpl::StringNum>(map_name) == nullptr);
+    REQUIRE(kv_store.get<MapTypes::StringNum>(map_name) == nullptr);
   }
 
   INFO("Can create a map with a previously invalid name");
   {
-    CHECK_NOTHROW(kv_store.create<typename MapImpl::StringString>("version"));
+    CHECK_NOTHROW(kv_store.create<MapTypes::StringString>("version"));
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Reads/writes and deletions")
-  "Reads/writes and deletions", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store kv_store;
-  auto& map = kv_store.create<typename MapImpl::StringString>(
+  auto& map =
-    "map", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map", kv::SecurityDomain::PUBLIC);
 
   constexpr auto k = "key";
   constexpr auto invalid_key = "invalid_key";
@@ -155,11 +143,11 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE("foreach", MapImpl, RawMapTypes, ExperimentalMapTypes)
+TEST_CASE("foreach")
 {
   kv::Store kv_store;
-  auto& map = kv_store.create<typename MapImpl::StringString>(
+  auto& map =
-    "map", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map", kv::SecurityDomain::PUBLIC);
 
   std::map<std::string, std::string> iterated_entries;
 
@@ -333,12 +321,11 @@ TEST_CASE_TEMPLATE("foreach", MapImpl, RawMapTypes, ExperimentalMapTypes)
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Rollback and compact")
-  "Rollback and compact", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store kv_store;
-  auto& map = kv_store.create<typename MapImpl::StringString>(
+  auto& map =
-    "map", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map", kv::SecurityDomain::PUBLIC);
 
   constexpr auto k = "key";
   constexpr auto v1 = "value1";
@@ -388,14 +375,13 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Clear entire store")
-  "Clear entire store", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store kv_store;
-  auto& map1 = kv_store.create<typename MapImpl::StringString>(
+  auto& map1 =
-    "map1", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map1", kv::SecurityDomain::PUBLIC);
-  auto& map2 = kv_store.create<typename MapImpl::StringString>(
+  auto& map2 =
-    "map2", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map2", kv::SecurityDomain::PUBLIC);
 
   INFO("Commit a transaction over two maps");
   {
@@ -428,10 +414,9 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Local commit hooks")
-  "Local commit hooks", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
-  using Write = typename MapImpl::StringString::Write;
+  using Write = MapTypes::StringString::Write;
   std::vector<Write> local_writes;
   std::vector<Write> global_writes;
 
@@ -443,8 +428,8 @@ TEST_CASE_TEMPLATE(
   };
 
   kv::Store kv_store;
-  auto& map = kv_store.create<typename MapImpl::StringString>(
+  auto& map =
-    "map", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map", kv::SecurityDomain::PUBLIC);
   map.set_local_hook(local_hook);
   map.set_global_hook(global_hook);
 
@@ -513,10 +498,9 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Global commit hooks")
-  "Global commit hooks", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
-  using Write = typename MapImpl::StringString::Write;
+  using Write = MapTypes::StringString::Write;
 
   struct GlobalHookInput
   {
@@ -666,15 +650,15 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE("Clone schema", MapImpl, RawMapTypes, ExperimentalMapTypes)
+TEST_CASE("Clone schema")
 {
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
   kv::Store store;
   store.set_encryptor(encryptor);
 
-  auto& public_map = store.create<typename MapImpl::NumString>(
+  auto& public_map =
-    "public", kv::SecurityDomain::PUBLIC);
+    store.create<MapTypes::NumString>("public", kv::SecurityDomain::PUBLIC);
-  auto& private_map = store.create<typename MapImpl::NumString>("private");
+  auto& private_map = store.create<MapTypes::NumString>("private");
   kv::Tx tx1(store.next_version());
   auto [view1, view2] = tx1.get_view(public_map, private_map);
   view1->put(42, "aardvark");
@@ -689,8 +673,7 @@ TEST_CASE_TEMPLATE("Clone schema", MapImpl, RawMapTypes, ExperimentalMapTypes)
   REQUIRE(clone.deserialise(data) == kv::DeserialiseSuccess::PASS);
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Deserialise return status")
-  "Deserialise return status", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store store;
 
@@ -699,7 +682,7 @@ TEST_CASE_TEMPLATE(
   auto& nodes =
     store.create<ccf::Nodes>(ccf::Tables::NODES, kv::SecurityDomain::PUBLIC);
   auto& data =
-    store.create<typename MapImpl::NumNum>("data", kv::SecurityDomain::PUBLIC);
+    store.create<MapTypes::NumNum>("data", kv::SecurityDomain::PUBLIC);
 
   auto kp = tls::make_key_pair();
 
@@ -742,22 +725,21 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Map swap between stores")
-  "Map swap between stores", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
   kv::Store s1;
   s1.set_encryptor(encryptor);
 
-  auto& d1 = s1.create<typename MapImpl::NumNum>("data");
+  auto& d1 = s1.create<MapTypes::NumNum>("data");
-  auto& pd1 = s1.create<typename MapImpl::NumNum>(
+  auto& pd1 =
-    "public_data", kv::SecurityDomain::PUBLIC);
+    s1.create<MapTypes::NumNum>("public_data", kv::SecurityDomain::PUBLIC);
 
   kv::Store s2;
   s2.set_encryptor(encryptor);
-  auto& d2 = s2.create<typename MapImpl::NumNum>("data");
+  auto& d2 = s2.create<MapTypes::NumNum>("data");
-  auto& pd2 = s2.create<typename MapImpl::NumNum>(
+  auto& pd2 =
-    "public_data", kv::SecurityDomain::PUBLIC);
+    s2.create<MapTypes::NumNum>("public_data", kv::SecurityDomain::PUBLIC);
 
   {
     kv::Tx tx;
@@ -815,16 +797,15 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Invalid map swaps")
-  "Invalid map swaps", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   {
     kv::Store s1;
-    s1.create<typename MapImpl::NumNum>("one");
+    s1.create<MapTypes::NumNum>("one");
 
     kv::Store s2;
-    s2.create<typename MapImpl::NumNum>("one");
+    s2.create<MapTypes::NumNum>("one");
-    s2.create<typename MapImpl::NumNum>("two");
+    s2.create<MapTypes::NumNum>("two");
 
     REQUIRE_THROWS_WITH(
       s2.swap_private_maps(s1),
@@ -833,11 +814,11 @@ TEST_CASE_TEMPLATE(
 
   {
     kv::Store s1;
-    s1.create<typename MapImpl::NumNum>("one");
+    s1.create<MapTypes::NumNum>("one");
-    s1.create<typename MapImpl::NumNum>("two");
+    s1.create<MapTypes::NumNum>("two");
 
     kv::Store s2;
-    s2.create<typename MapImpl::NumNum>("one");
+    s2.create<MapTypes::NumNum>("one");
 
     REQUIRE_THROWS_WITH(
       s2.swap_private_maps(s1),
@@ -845,21 +826,20 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Private recovery map swap")
-  "Private recovery map swap", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   auto encryptor = std::make_shared<kv::NullTxEncryptor>();
   kv::Store s1;
   s1.set_encryptor(encryptor);
-  auto& priv1 = s1.create<typename MapImpl::NumNum>("private");
+  auto& priv1 = s1.create<MapTypes::NumNum>("private");
-  auto& pub1 = s1.create<typename MapImpl::NumString>(
+  auto& pub1 =
-    "public", kv::SecurityDomain::PUBLIC);
+    s1.create<MapTypes::NumString>("public", kv::SecurityDomain::PUBLIC);
 
   kv::Store s2;
   s2.set_encryptor(encryptor);
-  auto& priv2 = s2.create<typename MapImpl::NumNum>("private");
+  auto& priv2 = s2.create<MapTypes::NumNum>("private");
-  auto& pub2 = s2.create<typename MapImpl::NumString>(
+  auto& pub2 =
-    "public", kv::SecurityDomain::PUBLIC);
+    s2.create<MapTypes::NumString>("public", kv::SecurityDomain::PUBLIC);
 
   INFO("Populate s1 with public entries");
   // We compact twice, deliberately. A public KV during recovery
@@ -976,12 +956,11 @@ TEST_CASE_TEMPLATE(
   }
 }
 
-TEST_CASE_TEMPLATE(
+TEST_CASE("Conflict resolution")
-  "Conflict resolution", MapImpl, RawMapTypes, ExperimentalMapTypes)
 {
   kv::Store kv_store;
-  auto& map = kv_store.create<typename MapImpl::StringString>(
+  auto& map =
-    "map", kv::SecurityDomain::PUBLIC);
+    kv_store.create<MapTypes::StringString>("map", kv::SecurityDomain::PUBLIC);
 
   auto try_write = [&](kv::Tx& tx, const std::string& s) {
     auto view = tx.get_view(map);

src/kv/tx_view.h

@@ -2,266 +2,80 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
-#include "ds/champ_map.h"
+#include "kv/untyped_map.h"
+#include "kv/untyped_tx_view.h"
 #include "kv_types.h"
 
-#include <unordered_map>
-
 namespace kv
 {
-  static bool is_deleted(Version version)
+  template <typename K, typename V, typename KSerialiser, typename VSerialiser>
-  {
+  class TxView : public kv::untyped::Map::TxViewCommitter
-    return version < 0;
-  }
-
-  template <typename V>
-  struct VersionV
-  {
-    Version version;
-    V value;
-
-    VersionV() = default;
-    VersionV(Version ver, V val) : version(ver), value(val) {}
-  };
-
-  template <typename K, typename V, typename H>
-  using State = champ::Map<K, VersionV<V>, H>;
-
-  template <typename K, typename V, typename H>
-  using Read = std::unordered_map<K, Version, H>;
-
-  template <typename K, typename V, typename H = std::hash<K>>
-  using Write =
-    std::unordered_map<K, std::optional<V>, H>; //< nullopt indicates a deletion
-
-  // This is a container for a write-set + dependencies. It can be applied to a
-  // given state, or used to track a set of operations on a state
-  template <typename K, typename V, typename H = std::hash<K>>
-  struct ChangeSet
-  {
-  public:
-    using State = State<K, V, H>;
-    using Read = Read<K, V, H>;
-    using Write = Write<K, V, H>;
-
-    State state;
-    State committed;
-    Version start_version;
-
-    Version read_version = NoVersion;
-    Read reads = {};
-    Write writes = {};
-
-    ChangeSet(
-      State& current_state, State& committed_state, Version current_version) :
-      state(current_state),
-      committed(committed_state),
-      start_version(current_version)
-    {}
-
-    ChangeSet(ChangeSet&) = delete;
-  };
-
-  template <typename K, typename V, typename H = std::hash<K>>
-  class TxView
   {
   protected:
-    using State = State<K, V, H>;
+    kv::untyped::TxView untyped_view;
-
-    using ChangeSet = ChangeSet<K, V, H>;
-    ChangeSet& tx_changes;
 
   public:
-    // Expose these types so that other code can use them as MyTx::KeyType or
-    // MyMap::TxView::KeyType, templated on the TxView or Map type rather than
-    // explicitly on K and V
     using KeyType = K;
     using ValueType = V;
 
-    TxView(ChangeSet& cs) : tx_changes(cs) {}
+    TxView(
+      kv::untyped::Map& m,
+      size_t rollbacks,
+      kv::untyped::State& current_state,
+      kv::untyped::State& committed_state,
+      Version v) :
+      kv::untyped::Map::TxViewCommitter(
+        m, rollbacks, current_state, committed_state, v),
+      untyped_view(kv::untyped::Map::TxViewCommitter::change_set)
+    {}
 
-    /** Get value for key
-     *
-     * This returns the value for the key inside the transaction. If the key
-     * has been updated in the current transaction, that update will be
-     * reflected in the return of this call.
-     *
-     * @param key Key
-     *
-     * @return optional containing value, empty if the key doesn't exist
-     */
     std::optional<V> get(const K& key)
     {
-      // A write followed by a read doesn't introduce a read dependency.
+      const auto opt_v_rep = untyped_view.get(KSerialiser::to_serialised(key));
-      // If we have written, return the value without updating the read set.
+
-      auto write = tx_changes.writes.find(key);
+      if (opt_v_rep.has_value())
-      if (write != tx_changes.writes.end())
       {
-        // May be empty, for a key that has been removed. This matches the
+        return VSerialiser::from_serialised(*opt_v_rep);
-        // return semantics
-        return write->second;
       }
 
-      // If the key doesn't exist, return empty and record that we depend on
+      return std::nullopt;
-      // the key not existing.
-      auto search = tx_changes.state.get(key);
-      if (!search.has_value())
-      {
-        tx_changes.reads.insert(std::make_pair(key, NoVersion));
-        return std::nullopt;
-      }
-
-      // Record the version that we depend on.
-      auto& found = search.value();
-      tx_changes.reads.insert(std::make_pair(key, found.version));
-
-      // If the key has been deleted, return empty.
-      if (is_deleted(found.version))
-      {
-        return std::nullopt;
-      }
-
-      // Return the value.
-      return found.value;
     }
 
-    /** Get globally committed value for key
-     *
-     * This reads a globally replicated value for the specified key.
-     * The value will have been the replicated value when the transaction
-     * began, but the map may be compacted while the transaction is in
-     * flight. If that happens, there may be a more recent committed
-     * version. This is undetectable to the transaction.
-     *
-     * @param key Key
-     *
-     * @return optional containing value, empty if the key doesn't exist in
-     * globally committed state
-     */
     std::optional<V> get_globally_committed(const K& key)
     {
-      // If there is no committed value, return empty.
+      const auto opt_v_rep =
-      auto search = tx_changes.committed.get(key);
+        untyped_view.get_globally_committed(KSerialiser::to_serialised(key));
-      if (!search.has_value())
+
+      if (opt_v_rep.has_value())
       {
-        return std::nullopt;
+        return VSerialiser::from_serialised(*opt_v_rep);
       }
 
-      // If the key has been deleted, return empty.
+      return std::nullopt;
-      auto& found = search.value();
-      if (is_deleted(found.version))
-      {
-        return std::nullopt;
-      }
-
-      // Return the value.
-      return found.value;
     }
 
-    /** Write value at key
-     *
-     * If the key already exists, the value will be replaced.
-     * This will fail if the transaction is already committed.
-     *
-     * @param key Key
-     * @param value Value
-     *
-     * @return true if successful, false otherwise
-     */
     bool put(const K& key, const V& value)
     {
-      // Record in the write set.
+      return untyped_view.put(
-      tx_changes.writes[key] = value;
+        KSerialiser::to_serialised(key), VSerialiser::to_serialised(value));
-      return true;
     }
 
-    /** Remove key
-     *
-     * This will fail if the key does not exist, or if the transaction
-     * is already committed.
-     *
-     * @param key Key
-     *
-     * @return true if successful, false otherwise
-     */
     bool remove(const K& key)
     {
-      auto write = tx_changes.writes.find(key);
+      return untyped_view.remove(KSerialiser::to_serialised(key));
-      auto search = tx_changes.state.get(key).has_value();
-
-      if (write != tx_changes.writes.end())
-      {
-        if (!search)
-        {
-          // this key only exists locally, there is no reason to maintain and
-          // serialise it
-          tx_changes.writes.erase(key);
-        }
-        else
-        {
-          // If we have written, change the write set to indicate a remove.
-          write->second = std::nullopt;
-        }
-
-        return true;
-      }
-
-      // If the key doesn't exist, return false.
-      if (!search)
-      {
-        return false;
-      }
-
-      // Record in the write set.
-      tx_changes.writes.emplace(
-        std::piecewise_construct,
-        std::forward_as_tuple(key),
-        std::forward_as_tuple(std::nullopt));
-      return true;
     }
 
-    /** Iterate over all entries in the map
-     *
-     * @param F functor, taking a key and a value, return value determines
-     * whether the iteration should continue (true) or stop (false)
-     */
     template <class F>
     void foreach(F&& f)
     {
-      // Record a global read dependency.
+      auto g = [&](
-      tx_changes.read_version = tx_changes.start_version;
+                 const kv::serialisers::SerialisedEntry& k_rep,
-      auto& w = tx_changes.writes;
+                 const kv::serialisers::SerialisedEntry& v_rep) {
-      bool should_continue = true;
+        return f(
-
+          KSerialiser::from_serialised(k_rep),
-      tx_changes.state.foreach(
+          VSerialiser::from_serialised(v_rep));
-        [&w, &f, &should_continue](const K& k, const VersionV<V>& v) {
+      };
-          auto write = w.find(k);
+      untyped_view.foreach(g);
-
-          if ((write == w.end()) && !is_deleted(v.version))
-          {
-            should_continue = f(k, v.value);
-          }
-
-          return should_continue;
-        });
-
-      if (should_continue)
-      {
-        for (auto write = tx_changes.writes.begin();
-             write != tx_changes.writes.end();
-             ++write)
-        {
-          if (write->second.has_value())
-          {
-            should_continue = f(write->first, write->second.value());
-          }
-
-          if (!should_continue)
-          {
-            break;
-          }
-        }
-      }
     }
   };
 }

src/kv/untyped_map.h

@@ -0,0 +1,723 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "ds/dl_list.h"
+#include "ds/logger.h"
+#include "ds/spin_lock.h"
+#include "kv/kv_serialiser.h"
+#include "kv/kv_types.h"
+#include "kv/untyped_tx_view.h"
+
+#include <functional>
+#include <optional>
+#include <unordered_set>
+
+namespace kv::untyped
+{
+  namespace Check
+  {
+    struct No
+    {};
+
+    template <typename T, typename Arg>
+    No operator!=(const T&, const Arg&)
+    {
+      return No();
+    }
+
+    template <typename T, typename Arg = T>
+    struct Ne
+    {
+      enum
+      {
+        value = !std::is_same<decltype(*(T*)(0) != *(Arg*)(0)), No>::value
+      };
+    };
+
+    template <class T>
+    bool ne(std::enable_if_t<Ne<T>::value, const T&> a, const T& b)
+    {
+      return a != b;
+    }
+
+    template <class T>
+    bool ne(std::enable_if_t<!Ne<T>::value, const T&> a, const T& b)
+    {
+      return false;
+    }
+  }
+
+  struct LocalCommit
+  {
+    LocalCommit() = default;
+    LocalCommit(Version v, State&& s, const Write& w) :
+      version(v),
+      state(std::move(s)),
+      writes(w)
+    {}
+
+    Version version;
+    State state;
+    Write writes;
+    LocalCommit* next = nullptr;
+    LocalCommit* prev = nullptr;
+  };
+  using LocalCommits = snmalloc::DLList<LocalCommit, std::nullptr_t, true>;
+
+  struct Roll
+  {
+    std::unique_ptr<LocalCommits> commits;
+    size_t rollback_counter;
+  };
+
+  class Map : public AbstractMap
+  {
+  public:
+    using K = kv::serialisers::SerialisedEntry;
+    using V = kv::serialisers::SerialisedEntry;
+    using H = SerialisedKeyHasher;
+
+    using CommitHook = CommitHook<Write>;
+
+  private:
+    AbstractStore* store;
+    std::string name;
+    Roll roll;
+    CommitHook local_hook = nullptr;
+    CommitHook global_hook = nullptr;
+    std::list<std::pair<Version, Write>> commit_deltas;
+    SpinLock sl;
+    const SecurityDomain security_domain;
+    const bool replicated;
+
+    LocalCommits empty_commits;
+
+    template <typename... Args>
+    LocalCommit* create_new_local_commit(Args&&... args)
+    {
+      LocalCommit* c = empty_commits.pop();
+      if (c == nullptr)
+      {
+        c = new LocalCommit(std::forward<Args>(args)...);
+      }
+      else
+      {
+        c->~LocalCommit();
+        new (c) LocalCommit(std::forward<Args>(args)...);
+      }
+      return c;
+    }
+
+  public:
+    class TxViewCommitter : public AbstractTxView
+    {
+    protected:
+      ChangeSet change_set;
+
+      Map& map;
+      size_t rollback_counter;
+
+      Version commit_version = NoVersion;
+
+      bool changes = false;
+      bool committed_writes = false;
+
+    public:
+      template <typename... Ts>
+      TxViewCommitter(Map& m, size_t rollbacks, Ts&&... ts) :
+        map(m),
+        rollback_counter(rollbacks),
+        change_set(std::forward<Ts>(ts)...)
+      {}
+
+      // Commit-related methods
+      bool has_writes() override
+      {
+        return committed_writes || !change_set.writes.empty();
+      }
+
+      bool has_changes() override
+      {
+        return changes;
+      }
+
+      bool prepare() override
+      {
+        if (change_set.writes.empty())
+          return true;
+
+        auto& roll = map.get_roll();
+
+        // If the parent map has rolled back since this transaction began, this
+        // transaction must fail.
+        if (rollback_counter != roll.rollback_counter)
+          return false;
+
+        // If we have iterated over the map, check for a global version match.
+        auto current = roll.commits->get_tail();
+
+        if (
+          (change_set.read_version != NoVersion) &&
+          (change_set.read_version != current->version))
+        {
+          LOG_DEBUG_FMT("Read version {} is invalid", change_set.read_version);
+          return false;
+        }
+
+        // Check each key in our read set.
+        for (auto it = change_set.reads.begin(); it != change_set.reads.end();
+             ++it)
+        {
+          // Get the value from the current state.
+          auto search = current->state.get(it->first);
+
+          if (it->second == NoVersion)
+          {
+            // If we depend on the key not existing, it must be absent.
+            if (search.has_value())
+            {
+              LOG_DEBUG_FMT("Read depends on non-existing entry");
+              return false;
+            }
+          }
+          else
+          {
+            // If we depend on the key existing, it must be present and have the
+            // version that we expect.
+            if (!search.has_value() || (it->second != search.value().version))
+            {
+              LOG_DEBUG_FMT("Read depends on invalid version of entry");
+              return false;
+            }
+          }
+        }
+
+        return true;
+      }
+
+      void commit(Version v) override
+      {
+        if (change_set.writes.empty())
+        {
+          commit_version = change_set.start_version;
+          return;
+        }
+
+        // Record our commit time.
+        commit_version = v;
+        committed_writes = true;
+
+        if (!change_set.writes.empty())
+        {
+          auto& roll = map.get_roll();
+          auto state = roll.commits->get_tail()->state;
+
+          for (auto it = change_set.writes.begin();
+               it != change_set.writes.end();
+               ++it)
+          {
+            if (it->second.has_value())
+            {
+              // Write the new value with the global version.
+              changes = true;
+              state = state.put(it->first, VersionV{v, it->second.value()});
+            }
+            else
+            {
+              // Write an empty value with the deleted global version only if
+              // the key exists.
+              auto search = state.get(it->first);
+              if (search.has_value())
+              {
+                changes = true;
+                state = state.put(it->first, VersionV{-v, {}});
+              }
+            }
+          }
+
+          if (changes)
+          {
+            map.roll.commits->insert_back(map.create_new_local_commit(
+              v, std::move(state), change_set.writes));
+          }
+        }
+      }
+
+      void post_commit() override
+      {
+        // This is run separately from commit so that all commits in the Tx
+        // have been applied before local hooks are run. The maps in the Tx
+        // are still locked when post_commit is run.
+        if (change_set.writes.empty())
+          return;
+
+        map.trigger_local_hook();
+      }
+
+      // Used by owning map during serialise and deserialise
+      ChangeSet& get_change_set()
+      {
+        return change_set;
+      }
+
+      const ChangeSet& get_change_set() const
+      {
+        return change_set;
+      }
+
+      void set_commit_version(Version v)
+      {
+        commit_version = v;
+      }
+    };
+
+    struct ConcreteTxView : public TxViewCommitter, public TxView
+    {
+      ConcreteTxView(
+        Map& m,
+        size_t rollbacks,
+        State& current_state,
+        State& committed_state,
+        Version v) :
+        TxViewCommitter(m, rollbacks, current_state, committed_state, v),
+        TxView(TxViewCommitter::change_set)
+      {}
+    };
+
+    // Public typedef for external consumption
+    using TxView = ConcreteTxView;
+
+    Map(
+      AbstractStore* store_,
+      std::string name_,
+      SecurityDomain security_domain_,
+      bool replicated_) :
+      store(store_),
+      name(name_),
+      roll{std::make_unique<LocalCommits>(), 0},
+      security_domain(security_domain_),
+      replicated(replicated_)
+    {
+      roll.commits->insert_back(create_new_local_commit(0, State(), Write()));
+    }
+
+    Map(const Map& that) = delete;
+
+    virtual AbstractMap* clone(AbstractStore* other) override
+    {
+      return new Map(other, name, security_domain, replicated);
+    }
+
+    void serialise(
+      const AbstractTxView* view,
+      KvStoreSerialiser& s,
+      bool include_reads) override
+    {
+      const auto committer = dynamic_cast<const TxViewCommitter*>(view);
+      if (committer == nullptr)
+      {
+        LOG_FAIL_FMT("Unable to serialise map due to type mismatch");
+        return;
+      }
+
+      const auto& change_set = committer->get_change_set();
+
+      s.start_map(name, security_domain);
+
+      if (include_reads)
+      {
+        s.serialise_read_version(change_set.read_version);
+
+        s.serialise_count_header(change_set.reads.size());
+        for (auto it = change_set.reads.begin(); it != change_set.reads.end();
+             ++it)
+        {
+          s.serialise_read(it->first, it->second);
+        }
+      }
+      else
+      {
+        s.serialise_read_version(NoVersion);
+        s.serialise_count_header(0);
+      }
+
+      uint64_t write_ctr = 0;
+      uint64_t remove_ctr = 0;
+      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
+           ++it)
+      {
+        if (it->second.has_value())
+        {
+          ++write_ctr;
+        }
+        else
+        {
+          auto search = roll.commits->get_tail()->state.get(it->first);
+          if (search.has_value())
+          {
+            ++remove_ctr;
+          }
+        }
+      }
+
+      s.serialise_count_header(write_ctr);
+      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
+           ++it)
+      {
+        if (it->second.has_value())
+        {
+          s.serialise_write(it->first, it->second.value());
+        }
+      }
+
+      s.serialise_count_header(remove_ctr);
+      for (auto it = change_set.writes.begin(); it != change_set.writes.end();
+           ++it)
+      {
+        if (!it->second.has_value())
+        {
+          s.serialise_remove(it->first);
+        }
+      }
+    }
+
+    AbstractTxView* deserialise(
+      KvStoreDeserialiser& d, Version version) override
+    {
+      return deserialise_internal<TxView>(d, version);
+    }
+
+    template <typename TView>
+    TView* deserialise_internal(KvStoreDeserialiser& d, Version version)
+    {
+      // Create a new change set, and deserialise d's contents into it.
+      auto view = create_view<TView>(version);
+      view->set_commit_version(version);
+
+      auto& change_set = view->get_change_set();
+
+      uint64_t ctr;
+
+      auto rv = d.deserialise_read_version();
+      if (rv != NoVersion)
+      {
+        change_set.read_version = rv;
+      }
+
+      ctr = d.deserialise_read_header();
+      for (size_t i = 0; i < ctr; ++i)
+      {
+        auto r = d.deserialise_read();
+        change_set.reads[std::get<0>(r)] = std::get<1>(r);
+      }
+
+      ctr = d.deserialise_write_header();
+      for (size_t i = 0; i < ctr; ++i)
+      {
+        auto w = d.deserialise_write();
+        change_set.writes[std::get<0>(w)] = std::get<1>(w);
+      }
+
+      ctr = d.deserialise_remove_header();
+      for (size_t i = 0; i < ctr; ++i)
+      {
+        auto r = d.deserialise_remove();
+        change_set.writes[r] = std::nullopt;
+      }
+
+      return view;
+    }
+
+    /** Get the name of the map
+     *
+     * @return const std::string&
+     */
+    const std::string& get_name() const override
+    {
+      return name;
+    }
+
+    /** Get store that the map belongs to
+     *
+     * @return Pointer to `kv::AbstractStore`
+     */
+    AbstractStore* get_store() override
+    {
+      return store;
+    }
+
+    /** Set handler to be called on local transaction commit
+     *
+     * @param hook function to be called on local transaction commit
+     */
+    void set_local_hook(const CommitHook& hook)
+    {
+      std::lock_guard<SpinLock> guard(sl);
+      local_hook = hook;
+    }
+
+    /** Reset local transaction commit handler
+     */
+    void unset_local_hook()
+    {
+      std::lock_guard<SpinLock> guard(sl);
+      local_hook = nullptr;
+    }
+
+    /** Set handler to be called on global transaction commit
+     *
+     * @param hook function to be called on global transaction commit
+     */
+    void set_global_hook(const CommitHook& hook)
+    {
+      std::lock_guard<SpinLock> guard(sl);
+      global_hook = hook;
+    }
+
+    /** Reset global transaction commit handler
+     */
+    void unset_global_hook()
+    {
+      std::lock_guard<SpinLock> guard(sl);
+      global_hook = nullptr;
+    }
+
+    /** Get security domain of a Map
+     *
+     * @return Security domain of the map (affects serialisation)
+     */
+    virtual SecurityDomain get_security_domain() override
+    {
+      return security_domain;
+    }
+
+    /** Get Map replicability
+     *
+     * @return true if the map is to be replicated, false if it is to be derived
+     */
+    virtual bool is_replicated() override
+    {
+      return replicated;
+    }
+
+    bool operator==(const AbstractMap& that) const override
+    {
+      auto p = dynamic_cast<const Map*>(&that);
+      if (p == nullptr)
+        return false;
+
+      if (name != p->name)
+        return false;
+
+      auto state1 = roll.commits->get_tail();
+      auto state2 = p->roll.commits->get_tail();
+
+      if (state1->version != state2->version)
+        return false;
+
+      size_t count = 0;
+      state2->state.foreach([&count](const K& k, const VersionV& v) {
+        count++;
+        return true;
+      });
+
+      size_t i = 0;
+      bool ok =
+        state1->state.foreach([&state2, &i](const K& k, const VersionV& v) {
+          auto search = state2->state.get(k);
+
+          if (search.has_value())
+          {
+            auto& found = search.value();
+            if (found.version != v.version)
+            {
+              return false;
+            }
+            else if (Check::ne(found.value, v.value))
+            {
+              return false;
+            }
+          }
+          else
+          {
+            return false;
+          }
+
+          i++;
+          return true;
+        });
+
+      if (i != count)
+        ok = false;
+
+      return ok;
+    }
+
+    bool operator!=(const AbstractMap& that) const override
+    {
+      return !(*this == that);
+    }
+
+    void compact(Version v) override
+    {
+      // This discards available rollback state before version v, and populates
+      // the commit_deltas to be passed to the global commit hook, if there is
+      // one, up to version v. The Map expects to be locked during compaction.
+      while (roll.commits->get_head() != roll.commits->get_tail())
+      {
+        auto r = roll.commits->get_head();
+
+        // Globally committed but not discardable.
+        if (r->version == v)
+        {
+          // We know that write set is not empty.
+          if (global_hook)
+          {
+            commit_deltas.emplace_back(r->version, std::move(r->writes));
+          }
+          return;
+        }
+
+        // Discardable, so move to commit_deltas.
+        if (global_hook && !r->writes.empty())
+        {
+          commit_deltas.emplace_back(r->version, std::move(r->writes));
+        }
+
+        // Stop if the next state may be rolled back or is the only state.
+        // This ensures there is always a state present.
+        if (r->next->version > v)
+          return;
+
+        auto c = roll.commits->pop();
+        empty_commits.insert(c);
+      }
+
+      // There is only one roll. We may need to call the commit hook.
+      auto r = roll.commits->get_head();
+
+      if (global_hook && !r->writes.empty())
+      {
+        commit_deltas.emplace_back(r->version, std::move(r->writes));
+      }
+    }
+
+    void post_compact() override
+    {
+      if (global_hook)
+      {
+        for (auto& [version, writes] : commit_deltas)
+        {
+          global_hook(version, writes);
+        }
+      }
+
+      commit_deltas.clear();
+    }
+
+    void rollback(Version v) override
+    {
+      // This rolls the current state back to version v.
+      // The Map expects to be locked during rollback.
+      bool advance = false;
+
+      while (roll.commits->get_head() != roll.commits->get_tail())
+      {
+        auto r = roll.commits->get_tail();
+
+        // The initial empty state has v = 0, so will not be discarded if it
+        // is present.
+        if (r->version <= v)
+          break;
+
+        advance = true;
+        auto c = roll.commits->pop_tail();
+        empty_commits.insert(c);
+      }
+
+      if (advance)
+        roll.rollback_counter++;
+    }
+
+    void clear() override
+    {
+      // This discards all entries in the roll and resets the compacted value
+      // and rollback counter. The Map expects to be locked before clearing it.
+      roll.commits->clear();
+      roll.commits->insert_back(create_new_local_commit(0, State(), Write()));
+      roll.rollback_counter = 0;
+    }
+
+    void lock() override
+    {
+      sl.lock();
+    }
+
+    void unlock() override
+    {
+      sl.unlock();
+    }
+
+    void swap(AbstractMap* map_) override
+    {
+      Map* map = dynamic_cast<Map*>(map_);
+      if (map == nullptr)
+        throw std::logic_error(
+          "Attempted to swap maps with incompatible types");
+
+      std::swap(roll, map->roll);
+    }
+
+    template <typename TView>
+    TView* create_view(Version version)
+    {
+      lock();
+
+      // Find the last entry committed at or before this version.
+      TView* view = nullptr;
+
+      for (auto current = roll.commits->get_tail(); current != nullptr;
+           current = current->prev)
+      {
+        if (current->version <= version)
+        {
+          view = new TView(
+            *this,
+            roll.rollback_counter,
+            current->state,
+            roll.commits->get_head()->state,
+            current->version);
+          break;
+        }
+      }
+
+      if (view == nullptr)
+      {
+        view = new TView(
+          *this,
+          roll.rollback_counter,
+          roll.commits->get_head()->state,
+          roll.commits->get_head()->state,
+          roll.commits->get_head()->version);
+      }
+
+      unlock();
+      return view;
+    }
+
+    Roll& get_roll()
+    {
+      return roll;
+    }
+
+    void trigger_local_hook()
+    {
+      if (local_hook)
+      {
+        auto last_commit = roll.commits->get_tail();
+        local_hook(last_commit->version, last_commit->writes);
+      }
+    }
+  };
+}

src/kv/untyped_tx_view.h

@@ -0,0 +1,216 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "kv/change_set.h"
+#include "kv/kv_types.h"
+#include "kv/serialised_entry.h"
+
+namespace kv::untyped
+{
+  using SerialisedEntry = kv::serialisers::SerialisedEntry;
+  using SerialisedKeyHasher = std::hash<SerialisedEntry>;
+
+  using VersionV = kv::VersionV<SerialisedEntry>;
+  using State =
+    kv::State<SerialisedEntry, SerialisedEntry, SerialisedKeyHasher>;
+  using Read = kv::Read<SerialisedEntry>;
+  using Write = kv::Write<SerialisedEntry, SerialisedEntry>;
+  using ChangeSet =
+    kv::ChangeSet<SerialisedEntry, SerialisedEntry, SerialisedKeyHasher>;
+
+  class TxView
+  {
+  protected:
+    ChangeSet& tx_changes;
+
+  public:
+    // Expose these types so that other code can use them as MyTx::KeyType or
+    // MyMap::TxView::KeyType, templated on the TxView or Map type
+    using KeyType = SerialisedEntry;
+    using ValueType = SerialisedEntry;
+
+    TxView(ChangeSet& cs) : tx_changes(cs) {}
+
+    /** Get value for key
+     *
+     * This returns the value for the key inside the transaction. If the key
+     * has been updated in the current transaction, that update will be
+     * reflected in the return of this call.
+     *
+     * @param key Key
+     *
+     * @return optional containing value, empty if the key doesn't exist
+     */
+    std::optional<ValueType> get(const KeyType& key)
+    {
+      // A write followed by a read doesn't introduce a read dependency.
+      // If we have written, return the value without updating the read set.
+      auto write = tx_changes.writes.find(key);
+      if (write != tx_changes.writes.end())
+      {
+        // May be empty, for a key that has been removed. This matches the
+        // return semantics
+        return write->second;
+      }
+
+      // If the key doesn't exist, return empty and record that we depend on
+      // the key not existing.
+      auto search = tx_changes.state.get(key);
+      if (!search.has_value())
+      {
+        tx_changes.reads.insert(std::make_pair(key, NoVersion));
+        return std::nullopt;
+      }
+
+      // Record the version that we depend on.
+      auto& found = search.value();
+      tx_changes.reads.insert(std::make_pair(key, found.version));
+
+      // If the key has been deleted, return empty.
+      if (is_deleted(found.version))
+      {
+        return std::nullopt;
+      }
+
+      // Return the value.
+      return found.value;
+    }
+
+    /** Get globally committed value for key
+     *
+     * This reads a globally replicated value for the specified key.
+     * The value will have been the replicated value when the transaction
+     * began, but the map may be compacted while the transaction is in
+     * flight. If that happens, there may be a more recent committed
+     * version. This is undetectable to the transaction.
+     *
+     * @param key Key
+     *
+     * @return optional containing value, empty if the key doesn't exist in
+     * globally committed state
+     */
+    std::optional<ValueType> get_globally_committed(const KeyType& key)
+    {
+      // If there is no committed value, return empty.
+      auto search = tx_changes.committed.get(key);
+      if (!search.has_value())
+      {
+        return std::nullopt;
+      }
+
+      // If the key has been deleted, return empty.
+      auto& found = search.value();
+      if (is_deleted(found.version))
+      {
+        return std::nullopt;
+      }
+
+      // Return the value.
+      return found.value;
+    }
+
+    /** Write value at key
+     *
+     * If the key already exists, the value will be replaced.
+     * This will fail if the transaction is already committed.
+     *
+     * @param key Key
+     * @param value Value
+     *
+     * @return true if successful, false otherwise
+     */
+    bool put(const KeyType& key, const ValueType& value)
+    {
+      // Record in the write set.
+      tx_changes.writes[key] = value;
+      return true;
+    }
+
+    /** Remove key
+     *
+     * This will fail if the key does not exist, or if the transaction
+     * is already committed.
+     *
+     * @param key Key
+     *
+     * @return true if successful, false otherwise
+     */
+    bool remove(const KeyType& key)
+    {
+      auto write = tx_changes.writes.find(key);
+      auto search = tx_changes.state.get(key).has_value();
+
+      if (write != tx_changes.writes.end())
+      {
+        if (!search)
+        {
+          // this key only exists locally, there is no reason to maintain and
+          // serialise it
+          tx_changes.writes.erase(key);
+        }
+        else
+        {
+          // If we have written, change the write set to indicate a remove.
+          write->second = std::nullopt;
+        }
+
+        return true;
+      }
+
+      // If the key doesn't exist, return false.
+      if (!search)
+      {
+        return false;
+      }
+
+      // Record in the write set.
+      tx_changes.writes[key] = std::nullopt;
+      return true;
+    }
+
+    /** Iterate over all entries in the map
+     *
+     * @param F functor, taking a key and a value, return value determines
+     * whether the iteration should continue (true) or stop (false)
+     */
+    template <class F>
+    void foreach(F&& f)
+    {
+      // Record a global read dependency.
+      tx_changes.read_version = tx_changes.start_version;
+      auto& w = tx_changes.writes;
+      bool should_continue = true;
+
+      tx_changes.state.foreach(
+        [&w, &f, &should_continue](const KeyType& k, const VersionV& v) {
+          auto write = w.find(k);
+
+          if ((write == w.end()) && !is_deleted(v.version))
+          {
+            should_continue = f(k, v.value);
+          }
+
+          return should_continue;
+        });
+
+      if (should_continue)
+      {
+        for (auto write = tx_changes.writes.begin();
+             write != tx_changes.writes.end();
+             ++write)
+        {
+          if (write->second.has_value())
+          {
+            should_continue = f(write->first, write->second.value());
+          }
+
+          if (!should_continue)
+          {
+            break;
+          }
+        }
+      }
+    }
+  };
+}

src/node/scripts.h

@@ -2,16 +2,11 @@
 // Licensed under the Apache 2.0 License.
 #pragma once
 
-#include "kv/experimental.h"
 #include "script.h"
 
 namespace ccf
 {
-  using Scripts = kv::experimental::Map<
+  using Scripts = kv::Map<std::string, Script>;
-    std::string,
-    Script,
-    kv::experimental::JsonSerialiser<std::string>,
-    kv::experimental::JsonSerialiser<Script>>;
 
   struct GovScriptIds
   {

src/node/shares.h

@@ -40,6 +40,9 @@ namespace ccf
     MSGPACK_DEFINE(version, encrypted_data)
   };
 
+  DECLARE_JSON_TYPE(LatestLedgerSecret)
+  DECLARE_JSON_REQUIRED_FIELDS(LatestLedgerSecret, version, encrypted_data)
+
   struct RecoverySharesInfo
   {
     // Keeping track of the latest and penultimate ledger secret allows the
@@ -63,6 +66,13 @@ namespace ccf
       encrypted_shares);
   };
 
+  DECLARE_JSON_TYPE(RecoverySharesInfo)
+  DECLARE_JSON_REQUIRED_FIELDS(
+    RecoverySharesInfo,
+    wrapped_latest_ledger_secret,
+    encrypted_previous_ledger_secret,
+    encrypted_shares)
+
   // The key for this table will always be 0 since a live service never needs to
   // access historical recovery shares info.
   using Shares = kv::Map<size_t, RecoverySharesInfo>;

tests/infra/ledger.py

@@ -9,6 +9,8 @@ GCM_SIZE_IV = 12
 LEDGER_TRANSACTION_SIZE = 4
 LEDGER_DOMAIN_SIZE = 8
 
+UNPACK_ARGS = {"raw": True, "strict_map_key": False}
+
 
 def to_uint_32(buffer):
     return struct.unpack("@I", buffer)[0]
@@ -38,7 +40,7 @@ class LedgerDomain:
     def __init__(self, buffer):
         self._buffer = buffer
         self._buffer_size = buffer.getbuffer().nbytes
-        self._unpacker = msgpack.Unpacker(self._buffer, raw=True, strict_map_key=False)
+        self._unpacker = msgpack.Unpacker(self._buffer, **UNPACK_ARGS)
         self._version = self._read_next()
         self._tables = {}
         self._read()