COSE back-endorsements for previous service identities (#6510)

Co-authored-by: Amaury Chamayou <amchamay@microsoft.com>
Co-authored-by: Amaury Chamayou <amaury@xargs.fr>

CHANGELOG.md

@@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.0-dev2]
+
+[6.0.0-dev2]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev2
+
+### Added
+
+- Introduced `ccf::describe_cose_endorsements_v1(receipt)` for COSE-endorsements chain of previous service identities (#6500).
+
 ## [6.0.0-dev1]
 
 [6.0.0-dev1]: https://github.com/microsoft/CCF/releases/tag/6.0.0-dev1

doc/audit/builtin_maps.rst

@@ -509,4 +509,12 @@ Evidence inserted in the ledger by a primary producing a snapshot to establish p
 
 Used to persist submitted shares during a recovery.
 
-While the contents themselves are encrypted, the table is public so as to be accessible by nodes bootstrapping a recovery service.
+While the contents themselves are encrypted, the table is public so as to be accessible by nodes bootstrapping a recovery service.
+
+
+``previous_service_identity_endorsement``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+**Key** Sentinel value 0, represented as a little-endian 64-bit unsigned integer.
+
+**Value** Endorsed COSE sign1 for the interface, represented as a DER-encoded string.

doc/schemas/app_openapi.json

@@ -98,6 +98,17 @@
         ],
         "type": "object"
       },
+      "LoggingGetCoseEndorsements__Out": {
+        "properties": {
+          "endorsements": {
+            "$ref": "#/components/schemas/base64string_array"
+          }
+        },
+        "required": [
+          "endorsements"
+        ],
+        "type": "object"
+      },
       "LoggingGetHistoricalRange__Entry": {
         "properties": {
           "id": {
@@ -215,6 +226,16 @@
         ],
         "type": "string"
       },
+      "base64string": {
+        "format": "base64",
+        "type": "string"
+      },
+      "base64string_array": {
+        "items": {
+          "$ref": "#/components/schemas/base64string"
+        },
+        "type": "array"
+      },
       "boolean": {
         "type": "boolean"
       },
@@ -1210,6 +1231,37 @@
         }
       }
     },
+    "/app/log/public/cose_endorsements": {
+      "get": {
+        "operationId": "GetAppLogPublicCoseEndorsements",
+        "responses": {
+          "200": {
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/LoggingGetCoseEndorsements__Out"
+                }
+              }
+            },
+            "description": "Default response description"
+          },
+          "default": {
+            "$ref": "#/components/responses/default"
+          }
+        },
+        "security": [
+          {
+            "jwt": []
+          },
+          {
+            "user_cose_sign1": []
+          }
+        ],
+        "x-ccf-forwarding": {
+          "$ref": "#/components/x-ccf-forwarding/never"
+        }
+      }
+    },
     "/app/log/public/count": {
       "get": {
         "operationId": "GetAppLogPublicCount",

include/ccf/crypto/cose_verifier.h

@@ -27,4 +27,12 @@ namespace ccf::crypto
   COSEVerifierUniquePtr make_cose_verifier_from_cert(
     const std::vector<uint8_t>& cert);
   COSEVerifierUniquePtr make_cose_verifier_from_key(const Pem& public_key);
+
+  struct COSEEndorsementValidity
+  {
+    std::string from_txid{};
+    std::string to_txid{};
+  };
+  COSEEndorsementValidity extract_cose_endorsement_validity(
+    std::span<const uint8_t> cose_msg);
 }

include/ccf/historical_queries_utils.h

@@ -25,4 +25,19 @@ namespace ccf::historical
     AbstractStateCache& state_cache,
     std::shared_ptr<NetworkIdentitySubsystemInterface>
       network_identity_subsystem);
+
+  // Modifies the receipt stored in state to include historical service
+  // endorsements, where required. If the state talks about a different service
+  // identity, which is known to be a predecessor of this service (via disaster
+  // recoveries), then an endorsement chain of all service identities preceding
+  // the current one will be created. This may need to use the state_cache to
+  // request additional historical entries to construct this endorsement, and
+  // may read from the current/latest state via tx. Returns true if the
+  // operation is complete, though it may still have failed to produce an
+  // endorsement. Returns false if additional entries have been requested, in
+  // which case the caller should retry later.
+  bool populate_cose_service_endorsements(
+    ccf::kv::ReadOnlyTx& tx,
+    ccf::historical::StatePtr& state,
+    AbstractStateCache& state_cache);
 }

include/ccf/receipt.h

@@ -147,6 +147,11 @@ namespace ccf
   std::optional<std::vector<uint8_t>> describe_merkle_proof_v1(
     const TxReceiptImpl& in);
 
+  using SerialisedCoseEndorsement = std::vector<uint8_t>;
+  using SerialisedCoseEndorsements = std::vector<SerialisedCoseEndorsement>;
+  std::optional<SerialisedCoseEndorsements> describe_cose_endorsements_v1(
+    const TxReceiptImpl& in);
+
   // Manual JSON serializers are specified for these types as they are not
   // trivial POD structs
 

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "6.0.0-dev1"
+version = "6.0.0-dev2"
 authors = [
   { name="CCF Team", email="CCF-Sec@microsoft.com" },
 ]

python/src/ccf/cose.py

@@ -174,16 +174,14 @@ def create_cose_sign1_finish(
     return msg.encode(sign=False)
 
 
-def validate_cose_sign1(payload: bytes, cert_pem: Pem, cose_sign1: bytes):
-    cert = load_pem_x509_certificate(cert_pem.encode("ascii"), default_backend())
-    if not isinstance(cert.public_key(), EllipticCurvePublicKey):
-        raise NotImplementedError("unsupported key type")
-
-    key = cert.public_key()
-    cose_key = from_cryptography_eckey_obj(key)
+def validate_cose_sign1(pubkey, cose_sign1, payload=None):
+    cose_key = from_cryptography_eckey_obj(pubkey)
     msg = Sign1Message.decode(cose_sign1)
     msg.key = cose_key
-    msg.payload = payload
+
+    if payload:
+        # Detached payload
+        msg.payload = payload
 
     if not msg.verify_signature():
         raise ValueError("signature is invalid")

python/src/ccf/ledger.py

@@ -571,8 +571,11 @@ class LedgerValidator:
 
     def _verify_root_cose_signature(self, root, cose_sign1):
         try:
+            cert = load_pem_x509_certificate(
+                self.service_cert.encode("ascii"), default_backend()
+            )
             validate_cose_sign1(
-                payload=root, cert_pem=self.service_cert, cose_sign1=cose_sign1
+                cose_sign1=cose_sign1, pubkey=cert.public_key(), payload=root
             )
         except Exception as exc:
             raise InvalidRootCoseSignatureException(

samples/apps/logging/logging.cpp

@@ -1961,6 +1961,47 @@ namespace loggingapp
         .set_auto_schema<void, void>()
         .set_forwarding_required(ccf::endpoints::ForwardingRequired::Never)
         .install();
+
+      auto get_cose_endorsements =
+        [this](
+          ccf::endpoints::ReadOnlyEndpointContext& ctx,
+          ccf::historical::StatePtr historical_state) {
+          auto historical_tx = historical_state->store->create_read_only_tx();
+
+          assert(historical_state->receipt);
+          auto endorsements =
+            describe_cose_endorsements_v1(*historical_state->receipt);
+          if (!endorsements.has_value())
+          {
+            ctx.rpc_ctx->set_error(
+              HTTP_STATUS_NOT_FOUND,
+              ccf::errors::ResourceNotFound,
+              "No COSE endorsements available for this transaction");
+            return;
+          }
+          LoggingGetCoseEndorsements::Out response{
+            .endorsements = ccf::SerialisedCoseEndorsements{}};
+          for (const auto& endorsement : *endorsements)
+          {
+            response.endorsements->push_back(endorsement);
+          }
+
+          nlohmann::json j_response = response;
+          ctx.rpc_ctx->set_response_status(HTTP_STATUS_OK);
+          ctx.rpc_ctx->set_response_header(
+            ccf::http::headers::CONTENT_TYPE,
+            ccf::http::headervalues::contenttype::JSON);
+          ctx.rpc_ctx->set_response_body(j_response.dump());
+        };
+      make_read_only_endpoint(
+        "/log/public/cose_endorsements",
+        HTTP_GET,
+        ccf::historical::read_only_adapter_v4(
+          get_cose_endorsements, context, is_tx_committed),
+        auth_policies)
+        .set_auto_schema<void, LoggingGetCoseEndorsements::Out>()
+        .set_forwarding_required(ccf::endpoints::ForwardingRequired::Never)
+        .install();
     }
   };
 }

samples/apps/logging/logging_schema.h

@@ -98,6 +98,16 @@ namespace loggingapp
   DECLARE_JSON_OPTIONAL_FIELDS_WITH_RENAMES(
     LoggingGetHistoricalRange::Out, next_link, "@nextLink");
 
+  struct LoggingGetCoseEndorsements
+  {
+    struct Out
+    {
+      std::optional<ccf::SerialisedCoseEndorsements> endorsements;
+    };
+  };
+  DECLARE_JSON_TYPE(LoggingGetCoseEndorsements::Out);
+  DECLARE_JSON_REQUIRED_FIELDS(LoggingGetCoseEndorsements::Out, endorsements);
+
   // Public record/get
   // Manual schemas, verified then parsed in handler
   static const std::string j_record_public_in = R"!!!(

src/crypto/openssl/cose_sign.cpp

@@ -79,7 +79,8 @@ namespace
 
 namespace ccf::crypto
 {
-  std::optional<int> key_to_cose_alg_id(ccf::crypto::PublicKey_OpenSSL& key)
+  std::optional<int> key_to_cose_alg_id(
+    const ccf::crypto::PublicKey_OpenSSL& key)
   {
     const auto cid = key.get_curve_id();
     switch (cid)
@@ -155,10 +156,25 @@ namespace ccf::crypto
       args_size);
   }
 
+  COSEParametersFactory cose_params_string_bytes(
+    const std::string& key, const std::vector<uint8_t>& value)
+  {
+    const size_t args_size = key.size() + value.size() +
+      extra_size_for_seq_tag + extra_size_for_seq_tag;
+    q_useful_buf_c buf{value.data(), value.size()};
+    return COSEParametersFactory(
+      [=](QCBOREncodeContext* ctx) {
+        QCBOREncode_AddSZString(ctx, key.data());
+        QCBOREncode_AddBytes(ctx, buf);
+      },
+      args_size);
+  }
+
   std::vector<uint8_t> cose_sign1(
-    KeyPair_OpenSSL& key,
+    const KeyPair_OpenSSL& key,
     const std::vector<COSEParametersFactory>& protected_headers,
-    std::span<const uint8_t> payload)
+    std::span<const uint8_t> payload,
+    bool detached_payload)
   {
     const auto buf_size = estimate_buffer_size(protected_headers, payload);
     std::vector<uint8_t> underlying_buffer(buf_size);
@@ -188,8 +204,16 @@ namespace ccf::crypto
 
     encode_parameters_custom(&sign_ctx, &cbor_encode, protected_headers);
 
-    // Mark empty payload manually.
-    QCBOREncode_AddNULL(&cbor_encode);
+    if (detached_payload)
+    {
+      // Mark empty payload explicitly.
+      QCBOREncode_AddNULL(&cbor_encode);
+    }
+    else
+    {
+      UsefulBufC payload_buffer{payload.data(), payload.size()};
+      QCBOREncode_AddBytes(&cbor_encode, payload_buffer);
+    }
 
     // If payload is empty - we still want to sign. Putting NULL_Q_USEFUL_BUF_C,
     // however, makes t_cose think that the payload is included into the

src/crypto/openssl/cose_sign.h

@@ -19,7 +19,11 @@ namespace ccf::crypto
   // Standardised: verifiable data structure
   static constexpr int64_t COSE_PHEADER_KEY_VDS = 395;
   // CCF-specific: last signed TxID
-  static constexpr const char* COSE_PHEADER_KEY_TXID = "ccf.txid";
+  static const std::string COSE_PHEADER_KEY_TXID = "ccf.txid";
+  // CCF-specific: first TX in the range.
+  static const std::string COSE_PHEADER_KEY_RANGE_BEGIN = "ccf.epoch.begin";
+  // CCF-specific: last TX included in the range.
+  static const std::string COSE_PHEADER_KEY_RANGE_END = "ccf.epoch.end";
 
   class COSEParametersFactory
   {
@@ -59,12 +63,16 @@ namespace ccf::crypto
   COSEParametersFactory cose_params_int_bytes(
     int64_t key, const std::vector<uint8_t>& value);
 
+  COSEParametersFactory cose_params_string_bytes(
+    const std::string& key, const std::vector<uint8_t>& value);
+
   struct COSESignError : public std::runtime_error
   {
     COSESignError(const std::string& msg) : std::runtime_error(msg) {}
   };
 
-  std::optional<int> key_to_cose_alg_id(ccf::crypto::PublicKey_OpenSSL& key);
+  std::optional<int> key_to_cose_alg_id(
+    const ccf::crypto::PublicKey_OpenSSL& key);
 
   /* Sign a cose_sign1 payload with custom protected headers as strings, where
        - key: integer label to be assigned in a COSE value
@@ -74,7 +82,8 @@ namespace ccf::crypto
     https://www.iana.org/assignments/cose/cose.xhtml#header-parameters.
    */
   std::vector<uint8_t> cose_sign1(
-    KeyPair_OpenSSL& key,
+    const KeyPair_OpenSSL& key,
     const std::vector<COSEParametersFactory>& protected_headers,
-    std::span<const uint8_t> payload);
+    std::span<const uint8_t> payload,
+    bool detached_payload = true);
 }

src/crypto/openssl/cose_verifier.cpp

@@ -19,6 +19,11 @@
 
 namespace
 {
+  static std::string qcbor_buf_to_string(const UsefulBufC& buf)
+  {
+    return std::string(reinterpret_cast<const char*>(buf.ptr), buf.len);
+  }
+
   static std::optional<int> extract_algorithm_from_header(
     std::span<const uint8_t> cose_msg)
   {
@@ -223,4 +228,106 @@ namespace ccf::crypto
   {
     return std::make_unique<COSEKeyVerifier_OpenSSL>(public_key);
   }
+
+  COSEEndorsementValidity extract_cose_endorsement_validity(
+    std::span<const uint8_t> cose_msg)
+  {
+    UsefulBufC msg{cose_msg.data(), cose_msg.size()};
+    QCBORError qcbor_result;
+
+    QCBORDecodeContext ctx;
+    QCBORDecode_Init(&ctx, msg, QCBOR_DECODE_MODE_NORMAL);
+
+    QCBORDecode_EnterArray(&ctx, nullptr);
+    qcbor_result = QCBORDecode_GetError(&ctx);
+    if (qcbor_result != QCBOR_SUCCESS)
+    {
+      throw std::logic_error("Failed to parse COSE_Sign1 outer array");
+    }
+
+    const uint64_t tag = QCBORDecode_GetNthTagOfLast(&ctx, 0);
+    if (tag != CBOR_TAG_COSE_SIGN1)
+    {
+      throw std::logic_error("Failed to parse COSE_Sign1 tag");
+    }
+
+    struct q_useful_buf_c protected_parameters;
+
+    QCBORDecode_EnterBstrWrapped(
+      &ctx, QCBOR_TAG_REQUIREMENT_NOT_A_TAG, &protected_parameters);
+
+    qcbor_result = QCBORDecode_GetError(&ctx);
+    if (qcbor_result != QCBOR_SUCCESS)
+    {
+      throw std::logic_error("Failed to parse COSE_Sign1 as bstr");
+    }
+
+    QCBORDecode_EnterMap(&ctx, NULL);
+
+    qcbor_result = QCBORDecode_GetError(&ctx);
+    if (qcbor_result != QCBOR_SUCCESS)
+    {
+      throw std::logic_error("Failed to parse COSE_Sign1 wrapped map");
+    }
+
+    enum
+    {
+      FROM_INDEX,
+      TO_INDEX,
+      END_INDEX
+    };
+    QCBORItem header_items[END_INDEX + 1];
+
+    header_items[FROM_INDEX].label.string = UsefulBufC{
+      ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN.data(),
+      ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN.size()};
+    header_items[FROM_INDEX].uLabelType = QCBOR_TYPE_TEXT_STRING;
+    header_items[FROM_INDEX].uDataType = QCBOR_TYPE_TEXT_STRING;
+
+    header_items[TO_INDEX].label.string = UsefulBufC{
+      ccf::crypto::COSE_PHEADER_KEY_RANGE_END.data(),
+      ccf::crypto::COSE_PHEADER_KEY_RANGE_END.size()};
+    header_items[TO_INDEX].uLabelType = QCBOR_TYPE_TEXT_STRING;
+    header_items[TO_INDEX].uDataType = QCBOR_TYPE_TEXT_STRING;
+
+    header_items[END_INDEX].uLabelType = QCBOR_TYPE_NONE;
+
+    QCBORDecode_GetItemsInMap(&ctx, header_items);
+    qcbor_result = QCBORDecode_GetError(&ctx);
+    if (qcbor_result != QCBOR_SUCCESS)
+    {
+      throw std::logic_error("Failed to decode protected header");
+    }
+
+    if (header_items[FROM_INDEX].uDataType == QCBOR_TYPE_NONE)
+    {
+      throw std::logic_error(fmt::format(
+        "Failed to retrieve (missing) {} parameter",
+        ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN));
+    }
+
+    if (header_items[TO_INDEX].uDataType == QCBOR_TYPE_NONE)
+    {
+      throw std::logic_error(fmt::format(
+        "Failed to retrieve (missing) {} parameter",
+        ccf::crypto::COSE_PHEADER_KEY_RANGE_END));
+    }
+
+    const auto from = qcbor_buf_to_string(header_items[FROM_INDEX].val.string);
+    const auto to = qcbor_buf_to_string(header_items[TO_INDEX].val.string);
+
+    // Complete decode to ensure well-formed CBOR.
+
+    QCBORDecode_ExitMap(&ctx);
+    QCBORDecode_ExitBstrWrapped(&ctx);
+
+    qcbor_result = QCBORDecode_GetError(&ctx);
+    if (qcbor_result != QCBOR_SUCCESS)
+    {
+      throw std::logic_error(fmt::format(
+        "Failed to decode protected header with error code: {}", qcbor_result));
+    }
+
+    return COSEEndorsementValidity{.from_txid = from, .to_txid = to};
+  }
 }

src/node/historical_queries_adapter.cpp

@@ -248,6 +248,12 @@ namespace ccf
     underlying_buffer.shrink_to_fit();
     return underlying_buffer;
   }
+
+  std::optional<SerialisedCoseEndorsements> describe_cose_endorsements_v1(
+    const TxReceiptImpl& receipt)
+  {
+    return receipt.cose_endorsements;
+  }
 }
 
 namespace ccf::historical
@@ -539,18 +545,36 @@ namespace ccf::historical
       // Get a state at the target version from the cache, if it is present
       auto historical_state =
         state_cache.get_state_at(historic_request_handle, target_tx_id.seqno);
-      if (
-        historical_state == nullptr ||
-        (!populate_service_endorsements(
-          args.tx, historical_state, state_cache, network_identity_subsystem)))
+      try
+      {
+        if (
+          historical_state == nullptr ||
+          (!populate_service_endorsements(
+            args.tx,
+            historical_state,
+            state_cache,
+            network_identity_subsystem)) ||
+          !populate_cose_service_endorsements(
+            args.tx, historical_state, state_cache))
+        {
+          auto reason = fmt::format(
+            "Historical transaction {} is not currently available.",
+            target_tx_id.to_str());
+          ehandler(
+            HistoricalQueryErrorCode::TransactionPartiallyReady,
+            std::move(reason),
+            args);
+          return;
+        }
+      }
+      catch (const std::exception& e)
       {
         auto reason = fmt::format(
-          "Historical transaction {} is not currently available.",
-          target_tx_id.to_str());
+          "Historical transaction {} failed with error: {}",
+          target_tx_id.to_str(),
+          e.what());
         ehandler(
-          HistoricalQueryErrorCode::TransactionPartiallyReady,
-          std::move(reason),
-          args);
+          HistoricalQueryErrorCode::InternalError, std::move(reason), args);
         return;
       }
 

src/node/historical_queries_utils.cpp

@@ -3,11 +3,201 @@
 
 #include "ccf/historical_queries_utils.h"
 
+#include "ccf/crypto/cose_verifier.h"
 #include "ccf/rpc_context.h"
 #include "ccf/service/tables/service.h"
+#include "consensus/aft/raft_types.h"
 #include "kv/kv_types.h"
 #include "node/identity.h"
 #include "node/tx_receipt_impl.h"
+#include "service/tables/previous_service_identity.h"
+
+namespace
+{
+  using Endorsements = std::vector<std::vector<uint8_t>>;
+  struct FetchResult
+  {
+    std::optional<Endorsements> endorsements{std::nullopt};
+    bool retry{false};
+  };
+
+  static std::vector<ccf::CoseEndorsement> cose_endorsements_cache = {};
+
+  bool is_self_endorsement(const ccf::CoseEndorsement& endorsement)
+  {
+    return !endorsement.previous_version.has_value();
+  }
+
+  void validate_fetched_endorsement(
+    const std::optional<ccf::CoseEndorsement>& endorsement)
+  {
+    if (!endorsement)
+    {
+      throw std::logic_error("Fetched COSE endorsement is invalid");
+    }
+
+    if (!is_self_endorsement(*endorsement))
+    {
+      const auto [from, to] = ccf::crypto::extract_cose_endorsement_validity(
+        endorsement->endorsement);
+
+      const auto from_txid = ccf::TxID::from_str(from);
+      if (!from_txid)
+      {
+        throw std::logic_error(fmt::format(
+          "Cannot parse COSE endorsement header: {}",
+          ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN));
+      }
+
+      const auto to_txid = ccf::TxID::from_str(to);
+      if (!to_txid)
+      {
+        throw std::logic_error(fmt::format(
+          "Cannot parse COSE endorsement header: {}",
+          ccf::crypto::COSE_PHEADER_KEY_RANGE_END));
+      }
+
+      if (!endorsement->endorsement_epoch_end)
+      {
+        throw std::logic_error(
+          "COSE endorsement doesn't contain epoch end in the table entry");
+      }
+      if (
+        endorsement->endorsement_epoch_begin != *from_txid ||
+        *endorsement->endorsement_epoch_end != *to_txid)
+      {
+        throw std::logic_error(fmt ::format(
+          "COSE endorsement fetched but range is invalid, epoch begin {}, "
+          "epoch end {}, header epoch begin: {}, header epoch end: {}",
+          endorsement->endorsement_epoch_begin.to_str(),
+          endorsement->endorsement_epoch_end->to_str(),
+          from,
+          to));
+      }
+    }
+  }
+
+  void validate_chain_integrity(
+    const ccf::CoseEndorsement& newer, const ccf::CoseEndorsement& older)
+  {
+    if (
+      !is_self_endorsement(older) &&
+      (newer.endorsement_epoch_begin.view - aft::starting_view_change !=
+         older.endorsement_epoch_end->view ||
+       newer.endorsement_epoch_begin.seqno - 1 !=
+         older.endorsement_epoch_end->seqno))
+    {
+      throw std::logic_error(fmt::format(
+        "COSE endorsement chain integrity is violated, previous endorsement "
+        "epoch end {} is not chained with newer endorsement epoch begin {}",
+        older.endorsement_epoch_end->to_str(),
+        newer.endorsement_epoch_begin.to_str()));
+    }
+  }
+
+  void ensure_first_fetch(ccf::kv::ReadOnlyTx& tx)
+  {
+    if (cose_endorsements_cache.empty()) [[unlikely]]
+    {
+      const auto endorsement =
+        tx.template ro<ccf::PreviousServiceIdentityEndorsement>(
+            ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
+          ->get();
+      validate_fetched_endorsement(endorsement);
+      cose_endorsements_cache.push_back(*endorsement);
+    }
+  }
+
+  bool keep_fetching(ccf::SeqNo target_seq)
+  {
+    return !is_self_endorsement(cose_endorsements_cache.back()) &&
+      cose_endorsements_cache.back().endorsement_epoch_begin.seqno > target_seq;
+  }
+
+  FetchResult fetch_endorsements_for(
+    ccf::kv::ReadOnlyTx& tx,
+    ccf::historical::AbstractStateCache& state_cache,
+    ccf::SeqNo target_seq)
+  {
+    ensure_first_fetch(tx);
+
+    while (keep_fetching(target_seq))
+    {
+      const auto prev_endorsement_seqno =
+        cose_endorsements_cache.back().previous_version.value();
+      const auto hstate = state_cache.get_state_at(
+        prev_endorsement_seqno, prev_endorsement_seqno);
+
+      if (!hstate)
+      {
+        return {.endorsements = std::nullopt, .retry = true};
+      }
+
+      auto htx = hstate->store->create_read_only_tx();
+      const auto endorsement =
+        htx
+          .template ro<ccf::PreviousServiceIdentityEndorsement>(
+            ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT)
+          ->get();
+
+      validate_fetched_endorsement(endorsement);
+      validate_chain_integrity(
+        cose_endorsements_cache.back(), endorsement.value());
+      cose_endorsements_cache.push_back(*endorsement);
+    }
+
+    if (cose_endorsements_cache.size() == 1)
+    {
+      LOG_TRACE_FMT(
+        "Only current service self-endorsement was found, no historical TXs "
+        "for previous epochs were COSE-endorsed.");
+      return {.endorsements = std::nullopt, .retry = false};
+    }
+
+    auto last_valid_endorsement = cose_endorsements_cache.end() - 1;
+    if (is_self_endorsement(*last_valid_endorsement))
+    {
+      --last_valid_endorsement;
+    }
+
+    const auto search_to = last_valid_endorsement + 1;
+
+    if (last_valid_endorsement->endorsement_epoch_begin.seqno > target_seq)
+    {
+      LOG_TRACE_FMT(
+        "COSE-endorsements are fetched for newer epochs, but target_seq {} is "
+        "far behind and was never endorsed.",
+        target_seq);
+
+      return {.endorsements = std::nullopt, .retry = false};
+    }
+
+    const auto final_endorsement = std::upper_bound(
+      cose_endorsements_cache.begin(),
+      search_to,
+      target_seq,
+      [](const auto& seq, const auto& endorsement) {
+        return endorsement.endorsement_epoch_begin.seqno <= seq;
+      });
+
+    if (final_endorsement == search_to)
+    {
+      throw std::logic_error(fmt::format(
+        "Error during COSE endorsement chain reconstruction for seqno {}",
+        target_seq));
+    }
+
+    Endorsements endorsements;
+
+    std::transform(
+      cose_endorsements_cache.begin(),
+      final_endorsement + 1, // Inclusive
+      std::back_inserter(endorsements),
+      [](const auto& e) { return e.endorsement; });
+
+    return {.endorsements = std::move(endorsements), .retry = false};
+  }
+}
 
 namespace ccf
 {
@@ -96,9 +286,9 @@ namespace ccf
                 {&network_identity->cert}, {}, /* ignore_time */ true))
           {
             // The current service identity does not endorse the node
-            // certificate in the receipt, so we search for the the most recent
-            // write to the service info table before the historical transaction
-            // ID to get the historical service identity.
+            // certificate in the receipt, so we search for the the most
+            // recent write to the service info table before the historical
+            // transaction ID to get the historical service identity.
 
             auto opt_psi =
               find_previous_service_identity(tx, state, state_cache);
@@ -145,5 +335,40 @@ namespace ccf
 
       return true;
     }
+
+    bool populate_cose_service_endorsements(
+      ccf::kv::ReadOnlyTx& tx,
+      ccf::historical::StatePtr& state,
+      AbstractStateCache& state_cache)
+    {
+      const auto service_info = tx.template ro<Service>(Tables::SERVICE)->get();
+      const auto service_start = service_info->current_service_create_txid;
+      if (!service_start)
+      {
+        throw std::logic_error(
+          "COSE endorsements fetch: current service create_txid not available");
+      }
+
+      const auto target_seq = state->transaction_id.seqno;
+      if (service_start->seqno <= target_seq)
+      {
+        LOG_TRACE_FMT(
+          "Target seqno {} belongs to current service started at {}",
+          target_seq,
+          service_start->seqno);
+        return true;
+      }
+
+      const auto result =
+        fetch_endorsements_for(tx, state_cache, state->transaction_id.seqno);
+      if (!result.endorsements)
+      {
+        const bool final_result = !result.retry;
+        return final_result;
+      }
+
+      state->receipt->cose_endorsements = result.endorsements.value();
+      return true;
+    }
   }
 }

src/node/node_state.h

@@ -1176,7 +1176,10 @@ namespace ccf
         // previous ledger secrets have been recovered
         share_manager.issue_recovery_shares(tx);
 
-        if (!InternalTablesAccess::open_service(tx))
+        if (
+          !InternalTablesAccess::open_service(tx) ||
+          !InternalTablesAccess::endorse_previous_identity(
+            tx, *network.identity->get_key_pair()))
         {
           throw std::logic_error("Service could not be opened");
         }
@@ -1501,6 +1504,8 @@ namespace ccf
         }
 
         InternalTablesAccess::open_service(tx);
+        InternalTablesAccess::endorse_previous_identity(
+          tx, *network.identity->get_key_pair());
         trigger_snapshot(tx);
         return;
       }

src/node/rpc/test/node_frontend_test.cpp

@@ -207,6 +207,8 @@ TEST_CASE("Add a node to an open service")
       gen_tx,
       {member_cert, ccf::crypto::make_rsa_key_pair()->public_key_pem()}));
   REQUIRE(InternalTablesAccess::open_service(gen_tx));
+  REQUIRE(InternalTablesAccess::endorse_previous_identity(
+    gen_tx, *network.identity->get_key_pair()));
   REQUIRE(gen_tx.commit() == ccf::kv::CommitResult::SUCCESS);
 
   // Node certificate

src/node/tx_receipt_impl.h

@@ -22,6 +22,8 @@ namespace ccf
     ccf::ClaimsDigest claims_digest = {};
     std::optional<std::vector<ccf::crypto::Pem>> service_endorsements =
       std::nullopt;
+    std::optional<std::vector<std::vector<uint8_t>>> cose_endorsements =
+      std::nullopt;
 
     TxReceiptImpl(
       const std::vector<uint8_t>& signature_,
@@ -37,7 +39,9 @@ namespace ccf
       // May not be set on historical transactions
       const ccf::ClaimsDigest& claims_digest_ = ccf::no_claims(),
       const std::optional<std::vector<ccf::crypto::Pem>>&
-        service_endorsements_ = std::nullopt) :
+        service_endorsements_ = std::nullopt,
+      const std::optional<std::vector<std::vector<uint8_t>>>&
+        cose_endorsements_ = std::nullopt) :
       signature(signature_),
       cose_signature(cose_signature),
       root(root_),
@@ -47,7 +51,8 @@ namespace ccf
       write_set_digest(write_set_digest_),
       commit_evidence(commit_evidence_),
       claims_digest(claims_digest_),
-      service_endorsements(service_endorsements_)
+      service_endorsements(service_endorsements_),
+      cose_endorsements(cose_endorsements_)
     {}
   };
 

src/service/internal_tables_access.h

@@ -11,6 +11,8 @@
 #include "ccf/service/tables/snp_measurements.h"
 #include "ccf/service/tables/users.h"
 #include "ccf/tx.h"
+#include "consensus/aft/raft_types.h"
+#include "crypto/openssl/cose_sign.h"
 #include "node/ledger_secrets.h"
 #include "node/uvm_endorsements.h"
 #include "service/tables/governance_history.h"
@@ -21,6 +23,19 @@
 
 namespace ccf
 {
+  /* We can't query the past epochs' TXs if the service hasn't been opened
+   * yet. We do guess values based on epoch value and seqno changing rules. */
+  ccf::TxID previous_tx_if_recovery(ccf::TxID txid)
+  {
+    return ccf::TxID{
+      .view = txid.view - aft::starting_view_change, .seqno = txid.seqno - 1};
+  }
+  ccf::TxID next_tx_if_recovery(ccf::TxID txid)
+  {
+    return ccf::TxID{
+      .view = txid.view + aft::starting_view_change, .seqno = txid.seqno + 1};
+  }
+
   // This class provides functions for interacting with various internal
   // service-governance tables. Specifically, it aims to maintain some
   // invariants amongst these tables (eg - keys being present in multiple
@@ -357,6 +372,79 @@ namespace ccf
       return service.has_value() && service->cert == expected_service_cert;
     }
 
+    static bool endorse_previous_identity(
+      ccf::kv::Tx& tx, const ccf::crypto::KeyPair_OpenSSL& service_key)
+    {
+      auto service = tx.ro<ccf::Service>(Tables::SERVICE);
+      auto active_service = service->get();
+
+      auto previous_identity_endorsement =
+        tx.rw<ccf::PreviousServiceIdentityEndorsement>(
+          ccf::Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT);
+
+      ccf::CoseEndorsement endorsement{};
+      std::vector<ccf::crypto::COSEParametersFactory> pheaders{};
+      std::vector<uint8_t> key_to_endorse{};
+
+      endorsement.endorsing_key = service_key.public_key_der();
+
+      if (previous_identity_endorsement->has())
+      {
+        const auto prev_endorsement = previous_identity_endorsement->get();
+
+        endorsement.endorsement_epoch_begin =
+          prev_endorsement->endorsement_epoch_end.has_value() ?
+          next_tx_if_recovery(prev_endorsement->endorsement_epoch_end.value()) :
+          prev_endorsement->endorsement_epoch_begin;
+
+        endorsement.endorsement_epoch_end = previous_tx_if_recovery(
+          active_service->current_service_create_txid.value());
+
+        endorsement.previous_version =
+          previous_identity_endorsement->get_version_of_previous_write();
+
+        key_to_endorse = prev_endorsement->endorsing_key;
+      }
+      else
+      {
+        // There's no `epoch_end` for the a self-endorsement, leave it
+        // open-ranged and sign the current service key.
+
+        endorsement.endorsement_epoch_begin =
+          active_service->current_service_create_txid.value();
+
+        key_to_endorse = endorsement.endorsing_key;
+      }
+
+      pheaders.push_back(ccf::crypto::cose_params_string_string(
+        ccf::crypto::COSE_PHEADER_KEY_RANGE_BEGIN,
+        endorsement.endorsement_epoch_begin.to_str()));
+      if (endorsement.endorsement_epoch_end)
+      {
+        pheaders.push_back(ccf::crypto::cose_params_string_string(
+          ccf::crypto::COSE_PHEADER_KEY_RANGE_END,
+          endorsement.endorsement_epoch_end->to_str()));
+      }
+
+      try
+      {
+        endorsement.endorsement = cose_sign1(
+          service_key,
+          pheaders,
+          key_to_endorse,
+          false // detached payload
+        );
+      }
+      catch (const ccf::crypto::COSESignError& e)
+      {
+        LOG_FAIL_FMT("Failed to sign previous service identity: {}", e.what());
+        return false;
+      }
+
+      previous_identity_endorsement->put(endorsement);
+      return true;
+    }
+
     static bool open_service(ccf::kv::Tx& tx)
     {
       auto service = tx.rw<ccf::Service>(Tables::SERVICE);

src/service/network_tables.h

@@ -178,6 +178,9 @@ namespace ccf
     const Service service = {Tables::SERVICE};
     const PreviousServiceIdentity previous_service_identity = {
       Tables::PREVIOUS_SERVICE_IDENTITY};
+    const PreviousServiceIdentityEndorsement
+      previous_service_identity_endorsement = {
+        Tables::PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT};
     const Configuration config = {Tables::CONFIGURATION};
     const Constitution constitution = {Tables::CONSTITUTION};
 

src/service/tables/previous_service_identity.h

@@ -12,9 +12,40 @@ namespace ccf
 {
   using PreviousServiceIdentity = ServiceValue<ccf::crypto::Pem>;
 
+  struct CoseEndorsement
+  {
+    /// COSE-sign of the a previous service identity's public key.
+    std::vector<uint8_t> endorsement{};
+
+    /// Service key at the moment of endorsing.
+    std::vector<uint8_t> endorsing_key{};
+
+    /// The transaction ID when the *endorsed* service was created.
+    ccf::TxID endorsement_epoch_begin{};
+
+    /// Last transaction ID that the endorsement is valid for. Only present for
+    /// previouse services endorsements, self-endorsed services must not have
+    /// this set.
+    std::optional<ccf::TxID> endorsement_epoch_end{};
+
+    /// Pointer to the previous CoseEndorsement entry. Only present for previous
+    /// service endorsements, self-endorsed services must not have this set.
+    std::optional<ccf::kv::Version> previous_version{};
+  };
+
+  DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(CoseEndorsement);
+  DECLARE_JSON_REQUIRED_FIELDS(
+    CoseEndorsement, endorsement, endorsement_epoch_begin, endorsing_key);
+  DECLARE_JSON_OPTIONAL_FIELDS(
+    CoseEndorsement, previous_version, endorsement_epoch_end);
+
+  using PreviousServiceIdentityEndorsement = ServiceValue<CoseEndorsement>;
+
   namespace Tables
   {
     static constexpr auto PREVIOUS_SERVICE_IDENTITY =
       "public:ccf.gov.service.previous_service_identity";
+    static constexpr auto PREVIOUS_SERVICE_IDENTITY_ENDORSEMENT =
+      "public:ccf.internal.previous_service_identity_endorsement";
   }
 }

tests/recovery.py

@@ -20,7 +20,13 @@ import infra.service_load
 import ccf.tx_id
 import tempfile
 import http
+import base64
 import shutil
+from cryptography.x509 import load_pem_x509_certificate
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from ccf.cose import validate_cose_sign1
+from pycose.messages import Sign1Message  # type: ignore
 
 from loguru import logger as LOG
 
@@ -47,6 +53,26 @@ def get_and_verify_historical_receipt(network, ref_msg):
     return ref_msg
 
 
+def query_endorsements_chain(node, txid):
+    for _ in range(0, 10):
+        with node.client("user0") as cli:
+            response = cli.get(
+                "/log/public/cose_endorsements",
+                headers={infra.clients.CCF_TX_ID_HEADER: str(txid)},
+            )
+            if response.status_code != http.HTTPStatus.ACCEPTED:
+                return response
+        time.sleep(0.1)
+    return response
+
+
+def verify_endorsements_chain(endorsements, pubkey):
+    for endorsement in endorsements:
+        validate_cose_sign1(cose_sign1=endorsement, pubkey=pubkey)
+        next_key_bytes = Sign1Message.decode(endorsement).payload
+        pubkey = serialization.load_der_public_key(next_key_bytes, default_backend())
+
+
 @reqs.description("Recover a service")
 @reqs.recover(number_txs=2)
 def test_recover_service(network, args, from_snapshot=True, no_ledger=False):
@@ -299,7 +325,9 @@ def test_recover_service_with_wrong_identity(network, args):
             cli.get("/node/commit").body.json()["transaction_id"]
         )
 
-    # Check receipts for transactions after multiple recoveries
+    # Check receipts for transactions after multiple recoveries. This test
+    # relies on previous recoveries and is therefore prone to failures if
+    # surrounding test calls change.
     txids = [
         # Last TX before previous recovery
         shifted_tx(previous_service_created_tx_id, -2, -1),
@@ -325,6 +353,34 @@ def test_recover_service_with_wrong_identity(network, args):
             # try again with a flag to force skip leaf components verification.
             verify_receipt(receipt, recovered_network.cert, is_signature_tx=True)
 
+    with primary.client() as cli:
+        service_cert = cli.get("/node/network").body.json()["service_certificate"]
+        cert = load_pem_x509_certificate(
+            service_cert.encode("ascii"), default_backend()
+        )
+
+    for tx in txids[0:1]:
+        response = query_endorsements_chain(primary, tx)
+        assert response.status_code == http.HTTPStatus.OK, response
+        endorsements = [
+            base64.b64decode(x) for x in response.body.json()["endorsements"]
+        ]
+        assert len(endorsements) == 2  # 2 recoveries behind
+        verify_endorsements_chain(endorsements, cert.public_key())
+
+    for tx in txids[1:4]:
+        response = query_endorsements_chain(primary, tx)
+        assert response.status_code == http.HTTPStatus.OK, response
+        endorsements = [
+            base64.b64decode(x) for x in response.body.json()["endorsements"]
+        ]
+        assert len(endorsements) == 1  # 1 recovery behind
+        verify_endorsements_chain(endorsements, cert.public_key())
+
+    for tx in txids[4:]:
+        response = query_endorsements_chain(primary, tx)
+        assert response.status_code == http.HTTPStatus.NOT_FOUND, response
+
     return recovered_network