Add JWT public signing key auto-refresh (#1908)

python/ccf/proposal_generator.py

@@ -18,9 +18,6 @@ import cryptography.hazmat.backends as crypto_backends
 from loguru import logger as LOG  # type: ignore
 
 
-CERT_OID_SGX_QUOTE = "1.2.840.113556.10.1.1"
-
-
 def dump_to_file(output_path: str, obj: dict, dump_args: dict):
     with open(output_path, "w") as f:
         json.dump(obj, f, **dump_args)
@@ -428,20 +425,12 @@ def update_ca_cert(cert_name, cert_path, skip_checks=False, **kwargs):
 
     if not skip_checks:
         try:
-            cert = x509.load_pem_x509_certificate(
+            x509.load_pem_x509_certificate(
                 cert_pem.encode(), crypto_backends.default_backend()
             )
         except Exception as exc:
             raise ValueError("Cannot parse PEM certificate") from exc
 
-        try:
-            oid = x509.ObjectIdentifier(CERT_OID_SGX_QUOTE)
-            _ = cert.extensions.get_extension_for_oid(oid)
-        except x509.ExtensionNotFound as exc:
-            raise ValueError(
-                "X.509 extension with SGX quote not found in certificate"
-            ) from exc
-
     args = {"name": cert_name, "cert": cert_pem}
     return build_proposal("update_ca_cert", args, **kwargs)
 
@@ -454,6 +443,8 @@ def set_jwt_issuer(json_path: str, **kwargs):
         "issuer": obj["issuer"],
         "key_filter": obj.get("key_filter", "all"),
         "key_policy": obj.get("key_policy"),
+        "ca_cert_name": obj.get("ca_cert_name"),
+        "auto_refresh": obj.get("auto_refresh", False),
         "jwks": obj.get("jwks"),
     }
     return build_proposal("set_jwt_issuer", args, **kwargs)

src/enclave/interface.h

@@ -80,6 +80,8 @@ struct CCFConfig
   std::string subject_name;
   std::vector<tls::SubjectAltName> subject_alternative_names;
 
+  size_t jwt_key_refresh_interval_s;
+
   MSGPACK_DEFINE(
     consensus_config,
     node_info_network,
@@ -90,7 +92,8 @@ struct CCFConfig
     genesis,
     joining,
     subject_name,
-    subject_alternative_names);
+    subject_alternative_names,
+    jwt_key_refresh_interval_s);
 };
 
 /// General administrative messages

src/host/main.cpp

@@ -318,6 +318,14 @@ int main(int argc, char** argv)
     "Subject Alternative Name in node certificate. Can be either "
     "iPAddress:xxx.xxx.xxx.xxx, or dNSName:sub.domain.tld");
 
+  size_t jwt_key_refresh_interval_s = 1800;
+  app
+    .add_option(
+      "--jwt-key-refresh-interval-s",
+      jwt_key_refresh_interval_s,
+      "Interval in seconds for JWT public signing key refresh.")
+    ->capture_default_str();
+
   size_t memory_reserve_startup = 0;
   app
     .add_option(
@@ -653,6 +661,8 @@ int main(int argc, char** argv)
     ccf_config.subject_name = subject_name;
     ccf_config.subject_alternative_names = subject_alternative_names;
 
+    ccf_config.jwt_key_refresh_interval_s = jwt_key_refresh_interval_s;
+
     if (*start)
     {
       start_type = StartType::New;

src/http/http_parser.h

@@ -156,7 +156,7 @@ namespace http
       http_parser_parse_url(url.data(), url.size(), 0, &parser_url);
     if (err != 0)
     {
-      throw std::runtime_error(fmt::format("Error parsing url: {}", err));
+      throw std::invalid_argument(fmt::format("Error parsing url: {}", err));
     }
 
     return std::make_pair(
@@ -164,6 +164,38 @@ namespace http
       extract_url_field(parser_url, UF_QUERY, url));
   }
 
+  struct URL
+  {
+    std::string_view schema;
+    std::string_view host;
+    std::string_view port;
+    std::string_view path;
+    std::string_view query;
+    std::string_view fragment;
+  };
+
+  inline URL parse_url_full(const std::string& url)
+  {
+    LOG_TRACE_FMT("Received url to parse: {}", url);
+
+    http_parser_url parser_url;
+    http_parser_url_init(&parser_url);
+
+    const auto err =
+      http_parser_parse_url(url.data(), url.size(), 0, &parser_url);
+    if (err != 0)
+    {
+      throw std::invalid_argument(fmt::format("Error parsing url: {}", err));
+    }
+
+    return {extract_url_field(parser_url, UF_SCHEMA, url),
+            extract_url_field(parser_url, UF_HOST, url),
+            extract_url_field(parser_url, UF_PORT, url),
+            extract_url_field(parser_url, UF_PATH, url),
+            extract_url_field(parser_url, UF_QUERY, url),
+            extract_url_field(parser_url, UF_FRAGMENT, url)};
+  }
+
   class Parser
   {
   protected:

src/node/jwt.h

@@ -41,13 +41,16 @@ namespace ccf
   {
     JwtIssuerKeyFilter key_filter;
     std::optional<JwtIssuerKeyPolicy> key_policy;
+    std::optional<std::string> ca_cert_name;
+    bool auto_refresh = false;
 
-    MSGPACK_DEFINE(key_filter, key_policy);
+    MSGPACK_DEFINE(key_filter, key_policy, ca_cert_name, auto_refresh);
   };
 
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(JwtIssuerMetadata);
   DECLARE_JSON_REQUIRED_FIELDS(JwtIssuerMetadata, key_filter);
-  DECLARE_JSON_OPTIONAL_FIELDS(JwtIssuerMetadata, key_policy);
+  DECLARE_JSON_OPTIONAL_FIELDS(
+    JwtIssuerMetadata, key_policy, ca_cert_name, auto_refresh);
 
   using JwtIssuer = std::string;
   using JwtKeyId = std::string;

src/node/jwt_key_auto_refresh.h

@@ -0,0 +1,309 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the Apache 2.0 License.
+#pragma once
+
+#include "http/http_builder.h"
+#include "http/http_rpc_context.h"
+#include "kv/tx.h"
+#include "node/jwt.h"
+#include "node/rpc/member_frontend.h"
+#include "node/rpc/serdes.h"
+
+#define FMT_HEADER_ONLY
+#include <fmt/format.h>
+#include <mutex>
+
+namespace ccf
+{
+  class JwtKeyAutoRefresh
+  {
+  private:
+    size_t refresh_interval_s;
+    NetworkState& network;
+    std::shared_ptr<kv::Consensus> consensus;
+    std::shared_ptr<enclave::RPCSessions> rpcsessions;
+    std::shared_ptr<enclave::RPCMap> rpc_map;
+    tls::Pem node_cert;
+
+  public:
+    JwtKeyAutoRefresh(
+      size_t refresh_interval_s,
+      NetworkState& network,
+      std::shared_ptr<kv::Consensus> consensus,
+      std::shared_ptr<enclave::RPCSessions> rpcsessions,
+      std::shared_ptr<enclave::RPCMap> rpc_map,
+      tls::Pem node_cert) :
+      refresh_interval_s(refresh_interval_s),
+      network(network),
+      consensus(consensus),
+      rpcsessions(rpcsessions),
+      rpc_map(rpc_map),
+      node_cert(node_cert)
+    {}
+
+    void start()
+    {
+      struct RefreshTimeMsg
+      {
+        RefreshTimeMsg(JwtKeyAutoRefresh& self_) : self(self_) {}
+
+        JwtKeyAutoRefresh& self;
+      };
+
+      auto refresh_msg = std::make_unique<threading::Tmsg<RefreshTimeMsg>>(
+        [](std::unique_ptr<threading::Tmsg<RefreshTimeMsg>> msg) {
+          if (!msg->data.self.consensus->is_primary())
+          {
+            LOG_DEBUG_FMT(
+              "JWT key auto-refresh: Node is not primary, skipping");
+          }
+          else
+          {
+            msg->data.self.refresh_jwt_keys();
+          }
+          LOG_DEBUG_FMT(
+            "JWT key auto-refresh: Scheduling in {}s",
+            msg->data.self.refresh_interval_s);
+          auto delay = std::chrono::seconds(msg->data.self.refresh_interval_s);
+          threading::ThreadMessaging::thread_messaging.add_task_after(
+            std::move(msg), delay);
+        },
+        *this);
+
+      LOG_DEBUG_FMT(
+        "JWT key auto-refresh: Scheduling in {}s", refresh_interval_s);
+      auto delay = std::chrono::seconds(refresh_interval_s);
+      threading::ThreadMessaging::thread_messaging.add_task_after(
+        std::move(refresh_msg), delay);
+    }
+
+    template <typename T>
+    void send_refresh_jwt_keys(T msg)
+    {
+      auto body = serdes::pack(msg, serdes::Pack::Text);
+
+      http::Request request(fmt::format(
+        "/{}/{}",
+        ccf::get_actor_prefix(ccf::ActorsType::members),
+        "jwt_keys/refresh"));
+      request.set_header(
+        http::headers::CONTENT_TYPE, http::headervalues::contenttype::JSON);
+      request.set_body(&body);
+      // Need a custom authentication policy that accepts only node certs.
+      // See https://github.com/microsoft/CCF/issues/1904
+      // http::sign_request(request, node_sign_kp);
+      auto packed = request.build_request();
+
+      auto node_session = std::make_shared<enclave::SessionContext>(
+        enclave::InvalidSessionId, node_cert.raw());
+      auto ctx = enclave::make_rpc_context(node_session, packed);
+
+      const auto actor_opt = http::extract_actor(*ctx);
+      if (!actor_opt.has_value())
+      {
+        throw std::logic_error("Unable to get actor");
+      }
+
+      const auto actor = rpc_map->resolve(actor_opt.value());
+      auto frontend_opt = this->rpc_map->find(actor);
+      if (!frontend_opt.has_value())
+      {
+        throw std::logic_error(
+          "RpcMap::find returned invalid (empty) frontend");
+      }
+      auto frontend = frontend_opt.value();
+      frontend->process(ctx);
+    }
+
+    void send_refresh_jwt_keys_error()
+    {
+      // A message that the endpoint fails to parse, leading to 500.
+      // This is done purely for exposing errors as endpoint metrics.
+      auto msg = false;
+      send_refresh_jwt_keys(msg);
+    }
+
+    void handle_jwt_jwks_response(
+      const std::string& issuer,
+      http_status status,
+      std::vector<uint8_t>&& data)
+    {
+      if (status != HTTP_STATUS_OK)
+      {
+        LOG_FAIL_FMT(
+          "JWT key auto-refresh: Error while requesting JWKS: {} {}{}",
+          status,
+          http_status_str(status),
+          data.empty() ?
+            "" :
+            fmt::format("  '{}'", std::string(data.begin(), data.end())));
+        send_refresh_jwt_keys_error();
+        return;
+      }
+
+      LOG_DEBUG_FMT(
+        "JWT key auto-refresh: Received JWKS for issuer '{}'", issuer);
+
+      JsonWebKeySet jwks;
+      try
+      {
+        jwks = nlohmann::json::parse(data).get<JsonWebKeySet>();
+      }
+      catch (const std::exception& e)
+      {
+        LOG_FAIL_FMT(
+          "JWT key auto-refresh: Cannot parse JWKS for issuer '{}': {}",
+          issuer,
+          e.what());
+        send_refresh_jwt_keys_error();
+        return;
+      }
+
+      // call internal endpoint to update keys
+      auto msg = SetJwtPublicSigningKeys{issuer, jwks};
+      send_refresh_jwt_keys(msg);
+    }
+
+    void handle_jwt_metadata_response(
+      const std::string& issuer,
+      std::shared_ptr<tls::Cert> ca_cert,
+      http_status status,
+      std::vector<uint8_t>&& data)
+    {
+      if (status != HTTP_STATUS_OK)
+      {
+        LOG_FAIL_FMT(
+          "JWT key auto-refresh: Error while requesting OpenID metadata: {} "
+          "{}{}",
+          status,
+          http_status_str(status),
+          data.empty() ?
+            "" :
+            fmt::format("  '{}'", std::string(data.begin(), data.end())));
+        send_refresh_jwt_keys_error();
+        return;
+      }
+
+      LOG_DEBUG_FMT(
+        "JWT key auto-refresh: Received OpenID metadata for issuer '{}'",
+        issuer);
+
+      std::string jwks_url_str;
+      try
+      {
+        auto metadata = nlohmann::json::parse(data);
+        jwks_url_str = metadata.at("jwks_uri").get<std::string>();
+      }
+      catch (const std::exception& e)
+      {
+        LOG_FAIL_FMT(
+          "JWT key auto-refresh: Cannot parse OpenID metadata for issuer '{}': "
+          "{}",
+          issuer,
+          e.what());
+        send_refresh_jwt_keys_error();
+        return;
+      }
+      http::URL jwks_url;
+      try
+      {
+        jwks_url = http::parse_url_full(jwks_url_str);
+      }
+      catch (const std::invalid_argument& e)
+      {
+        LOG_FAIL_FMT(
+          "JWT key auto-refresh: Cannot parse jwks_uri for issuer '{}': {}",
+          issuer,
+          jwks_url_str);
+        send_refresh_jwt_keys_error();
+        return;
+      }
+      auto jwks_url_port = !jwks_url.port.empty() ? jwks_url.port : "443";
+
+      LOG_DEBUG_FMT(
+        "JWT key auto-refresh: Requesting JWKS at https://{}:{}{}",
+        jwks_url.host,
+        jwks_url_port,
+        jwks_url.path);
+      auto http_client = rpcsessions->create_client(ca_cert);
+      // Note: Connection errors are not signalled and hence not tracked in
+      // endpoint metrics currently.
+      http_client->connect(
+        std::string(jwks_url.host),
+        std::string(jwks_url_port),
+        [this, issuer](
+          http_status status, http::HeaderMap&&, std::vector<uint8_t>&& data) {
+          handle_jwt_jwks_response(issuer, status, std::move(data));
+          return true;
+        });
+      http::Request r(jwks_url.path, HTTP_GET);
+      r.set_header(http::headers::HOST, std::string(jwks_url.host));
+      http_client->send_request(r.build_request());
+    }
+
+    void refresh_jwt_keys()
+    {
+      auto tx = network.tables->create_read_only_tx();
+      auto jwt_issuers_view = tx.get_read_only_view(network.jwt_issuers);
+      auto ca_certs_view = tx.get_read_only_view(network.ca_certs);
+      jwt_issuers_view->foreach([this, &ca_certs_view](
+                                  const JwtIssuer& issuer,
+                                  const JwtIssuerMetadata& metadata) {
+        if (!metadata.auto_refresh)
+        {
+          LOG_DEBUG_FMT(
+            "JWT key auto-refresh: Skipping issuer '{}', auto-refresh is "
+            "disabled",
+            issuer);
+          return true;
+        }
+        LOG_DEBUG_FMT(
+          "JWT key auto-refresh: Refreshing keys for issuer '{}'", issuer);
+        auto& ca_cert_name = metadata.ca_cert_name.value();
+        auto ca_cert_der = ca_certs_view->get(ca_cert_name);
+        if (!ca_cert_der.has_value())
+        {
+          LOG_FAIL_FMT(
+            "JWT key auto-refresh: CA cert with name '{}' for issuer '{}' not "
+            "found",
+            ca_cert_name,
+            issuer);
+          send_refresh_jwt_keys_error();
+          return true;
+        }
+        auto ca = std::make_shared<tls::CA>(ca_cert_der.value());
+        auto ca_cert = std::make_shared<tls::Cert>(ca);
+
+        auto metadata_url_str = issuer + "/.well-known/openid-configuration";
+        auto metadata_url = http::parse_url_full(metadata_url_str);
+        auto metadata_url_port =
+          !metadata_url.port.empty() ? metadata_url.port : "443";
+
+        LOG_DEBUG_FMT(
+          "JWT key auto-refresh: Requesting OpenID metadata at https://{}:{}{}",
+          metadata_url.host,
+          metadata_url_port,
+          metadata_url.path);
+        auto http_client = rpcsessions->create_client(ca_cert);
+        // Note: Connection errors are not signalled and hence not tracked in
+        // endpoint metrics currently.
+        http_client->connect(
+          std::string(metadata_url.host),
+          std::string(metadata_url_port),
+          [this, issuer, ca_cert](
+            http_status status,
+            http::HeaderMap&&,
+            std::vector<uint8_t>&& data) {
+            handle_jwt_metadata_response(
+              issuer, ca_cert, status, std::move(data));
+            return true;
+          });
+        http::Request r(metadata_url.path, HTTP_GET);
+        r.set_header(http::headers::HOST, std::string(metadata_url.host));
+        http_client->send_request(r.build_request());
+        return true;
+      });
+    }
+  };
+
+}

src/node/node_state.h

@@ -12,6 +12,7 @@
 #include "genesis_gen.h"
 #include "history.h"
 #include "network_state.h"
+#include "node/jwt_key_auto_refresh.h"
 #include "node/progress_tracker.h"
 #include "node/rpc/serdes.h"
 #include "node_to_node.h"
@@ -182,6 +183,11 @@ namespace ccf
 
     consensus::Index ledger_idx = 0;
 
+    //
+    // JWT key auto-refresh
+    //
+    std::shared_ptr<JwtKeyAutoRefresh> jwt_key_auto_refresh;
+
   public:
     NodeState(
       ringbuffer::AbstractWriterFactory& writer_factory,
@@ -289,6 +295,7 @@ namespace ccf
           }
 
           accept_network_tls_connections(args.config);
+          auto_refresh_jwt_keys(args.config);
 
           reset_data(quote);
           sm.advance(State::partOfNetwork);
@@ -300,6 +307,7 @@ namespace ccf
           // TLS connections are not endorsed by the network until the node
           // has joined
           accept_node_tls_connections();
+          auto_refresh_jwt_keys(args.config);
 
           sm.advance(State::pending);
 
@@ -355,6 +363,7 @@ namespace ccf
           }
 
           accept_network_tls_connections(args.config);
+          auto_refresh_jwt_keys(args.config);
 
           sm.advance(State::readingPublicLedger);
 
@@ -596,6 +605,25 @@ namespace ccf
         std::chrono::milliseconds(config.joining.join_timer));
     }
 
+    void auto_refresh_jwt_keys(const CCFConfig& config)
+    {
+      if (!consensus)
+      {
+        LOG_INFO_FMT(
+          "JWT key auto-refresh: consensus not initialized, not starting "
+          "auto-refresh");
+        return;
+      }
+      jwt_key_auto_refresh = std::make_shared<JwtKeyAutoRefresh>(
+        config.jwt_key_refresh_interval_s,
+        network,
+        consensus,
+        rpcsessions,
+        rpc_map,
+        node_cert);
+      jwt_key_auto_refresh->start();
+    }
+
     //
     // funcs in state "readingPublicLedger"
     //

src/node/rpc/endpoint_registry.h

@@ -626,6 +626,9 @@ namespace ccf
 
       for (const auto& [path, verb_endpoints] : fully_qualified_endpoints)
       {
+        // Special endpoint, can only be called from the node.
+        if (path == "jwt_keys/refresh")
+          continue;
         for (const auto& [verb, endpoint] : verb_endpoints)
         {
           add_endpoint_to_api_document(document, endpoint);

src/node/rpc/member_frontend.h

@@ -410,10 +410,14 @@ namespace ccf
       auto key_issuer =
         tx.get_view(this->network.jwt_public_signing_key_issuer);
 
+      auto log_prefix = proposal_id != INVALID_ID ?
+        fmt::format("Proposal {}", proposal_id) :
+        "JWT key auto-refresh";
+
       // add keys
       if (jwks.keys.empty())
       {
-        LOG_FAIL_FMT("Proposal {}: JWKS has no keys", proposal_id);
+        LOG_FAIL_FMT("{}: JWKS has no keys", log_prefix, proposal_id);
         return false;
       }
       std::map<std::string, std::vector<uint8_t>> new_keys;
@@ -422,19 +426,32 @@ namespace ccf
         if (keys->has(jwk.kid) && key_issuer->get(jwk.kid).value() != issuer)
         {
           LOG_FAIL_FMT(
-            "Proposal {}: key id {} already added for different issuer",
-            proposal_id,
+            "{}: key id {} already added for different issuer",
+            log_prefix,
             jwk.kid);
           return false;
         }
         if (jwk.x5c.empty())
         {
-          LOG_FAIL_FMT("Proposal {}: JWKS is invalid (empty x5c)", proposal_id);
+          LOG_FAIL_FMT("{}: JWKS is invalid (empty x5c)", log_prefix);
           return false;
         }
 
         auto& der_base64 = jwk.x5c[0];
-        auto der = tls::raw_from_b64(der_base64);
+        ccf::Cert der;
+        try
+        {
+          der = tls::raw_from_b64(der_base64);
+        }
+        catch (const std::invalid_argument& e)
+        {
+          LOG_FAIL_FMT(
+            "{}: Could not parse x5c of key id {}: {}",
+            log_prefix,
+            jwk.kid,
+            e.what());
+          return false;
+        }
 
         std::map<std::string, std::vector<uint8_t>> claims;
         bool has_key_policy_sgx_claims =
@@ -458,9 +475,9 @@ namespace ccf
           claims.empty())
         {
           LOG_INFO_FMT(
-            "Proposal {}: Skipping JWT signing key with kid {} (not OE "
+            "{}: Skipping JWT signing key with kid {} (not OE "
             "attested)",
-            proposal_id,
+            log_prefix,
             jwk.kid);
           continue;
         }
@@ -473,8 +490,8 @@ namespace ccf
             if (claims.find(claim_name) == claims.end())
             {
               LOG_FAIL_FMT(
-                "Proposal {}: JWKS kid {} is missing the {} SGX claim",
-                proposal_id,
+                "{}: JWKS kid {} is missing the {} SGX claim",
+                log_prefix,
                 jwk.kid,
                 claim_name);
               return false;
@@ -485,8 +502,8 @@ namespace ccf
             if (expected_claim_val_hex != actual_claim_val_hex)
             {
               LOG_FAIL_FMT(
-                "Proposal {}: JWKS kid {} has a mismatching {} SGX claim",
-                proposal_id,
+                "{}: JWKS kid {} has a mismatching {} SGX claim",
+                log_prefix,
                 jwk.kid,
                 claim_name);
               return false;
@@ -499,27 +516,23 @@ namespace ccf
           {
             tls::check_is_cert(der);
           }
-          catch (std::exception& exc)
+          catch (std::invalid_argument& exc)
           {
             LOG_FAIL_FMT(
-              "Proposal {}: JWKS kid {} has an invalid X.509 certificate: "
-              "{}",
-              proposal_id,
+              "{}: JWKS kid {} has an invalid X.509 certificate: {}",
+              log_prefix,
               jwk.kid,
               exc.what());
             return false;
           }
         }
         LOG_INFO_FMT(
-          "Proposal {}: Storing JWT signing key with kid {}",
-          proposal_id,
-          jwk.kid);
+          "{}: Storing JWT signing key with kid {}", log_prefix, jwk.kid);
         new_keys.emplace(jwk.kid, der);
       }
       if (new_keys.empty())
       {
-        LOG_FAIL_FMT(
-          "Proposal {}: no keys left after applying filter", proposal_id);
+        LOG_FAIL_FMT("{}: no keys left after applying filter", log_prefix);
         return false;
       }
 
@@ -723,6 +736,55 @@ namespace ccf
          [this](ObjectId proposal_id, kv::Tx& tx, const nlohmann::json& args) {
            const auto parsed = args.get<SetJwtIssuer>();
            auto issuers = tx.get_view(this->network.jwt_issuers);
+           auto ca_certs = tx.get_read_only_view(this->network.ca_certs);
+
+           if (parsed.auto_refresh)
+           {
+             if (!parsed.ca_cert_name.has_value())
+             {
+               LOG_FAIL_FMT(
+                 "Proposal {}: ca_cert_name is missing but required if "
+                 "auto_refresh is true",
+                 proposal_id);
+               return false;
+             }
+             if (!ca_certs->has(parsed.ca_cert_name.value()))
+             {
+               LOG_FAIL_FMT(
+                 "Proposal {}: No CA cert found with name '{}'",
+                 proposal_id,
+                 parsed.ca_cert_name.value());
+               return false;
+             }
+             http::URL issuer_url;
+             try
+             {
+               issuer_url = http::parse_url_full(parsed.issuer);
+             }
+             catch (const std::runtime_error&)
+             {
+               LOG_FAIL_FMT(
+                 "Proposal {}: issuer must be a URL if auto_refresh is true",
+                 proposal_id);
+               return false;
+             }
+             if (issuer_url.schema != "https")
+             {
+               LOG_FAIL_FMT(
+                 "Proposal {}: issuer must be a URL starting with https:// if "
+                 "auto_refresh is true",
+                 proposal_id);
+               return false;
+             }
+             if (!issuer_url.query.empty() || !issuer_url.fragment.empty())
+             {
+               LOG_FAIL_FMT(
+                 "Proposal {}: issuer must be a URL without query/fragment if "
+                 "auto_refresh is true",
+                 proposal_id);
+               return false;
+             }
+           }
 
            bool success = true;
            if (parsed.jwks.has_value())
@@ -1796,6 +1858,96 @@ namespace ccf
       make_endpoint("create", HTTP_POST, json_adapter(create))
         .set_require_client_identity(false)
         .install();
+
+      // Only called from node. See node_state.h.
+      auto refresh_jwt_keys = [this](
+                                EndpointContext& args, nlohmann::json&& body) {
+        // All errors are server errors since the client is the server.
+
+        if (!consensus)
+        {
+          LOG_FAIL_FMT("JWT key auto-refresh: no consensus available");
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR, "no consensus available");
+        }
+
+        auto primary_id = consensus->primary();
+        auto nodes_view = args.tx.get_read_only_view(this->network.nodes);
+        auto info = nodes_view->get(primary_id);
+        if (!info.has_value())
+        {
+          LOG_FAIL_FMT(
+            "JWT key auto-refresh: could not find node info of primary");
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR,
+            "could not find node info of primary");
+        }
+
+        auto primary_cert_pem = info.value().cert;
+        auto cert_der = args.rpc_ctx->session->caller_cert;
+        auto caller_cert_pem = tls::cert_der_to_pem(cert_der);
+        if (caller_cert_pem != primary_cert_pem)
+        {
+          LOG_FAIL_FMT(
+            "JWT key auto-refresh: request does not originate from primary");
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR,
+            "request does not originate from primary");
+        }
+
+        SetJwtPublicSigningKeys parsed;
+        try
+        {
+          parsed = body.get<SetJwtPublicSigningKeys>();
+        }
+        catch (const JsonParseError& e)
+        {
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR, "unable to parse body");
+        }
+
+        auto issuers = args.tx.get_view(this->network.jwt_issuers);
+        auto issuer_metadata_ = issuers->get(parsed.issuer);
+        if (!issuer_metadata_.has_value())
+        {
+          LOG_FAIL_FMT(fmt::format(
+            "JWT key auto-refresh: {} is not a valid issuer", parsed.issuer));
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR,
+            fmt::format("{} is not a valid issuer", parsed.issuer));
+        }
+        auto& issuer_metadata = issuer_metadata_.value();
+
+        if (!issuer_metadata.auto_refresh)
+        {
+          LOG_FAIL_FMT(fmt::format(
+            "JWT key auto-refresh: {} does not have auto_refresh enabled",
+            parsed.issuer));
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR,
+            fmt::format(
+              "{} does not have auto_refresh enabled", parsed.issuer));
+        }
+
+        if (!set_jwt_public_signing_keys(
+              args.tx, INVALID_ID, parsed.issuer, issuer_metadata, parsed.jwks))
+        {
+          LOG_FAIL_FMT(fmt::format(
+            "JWT key auto-refresh: error while storing signing keys for issuer "
+            "{}",
+            parsed.issuer));
+          return make_error(
+            HTTP_STATUS_INTERNAL_SERVER_ERROR,
+            fmt::format(
+              "error while storing signing keys for issuer {}", parsed.issuer));
+        }
+
+        return make_success(true);
+      };
+      make_endpoint(
+        "jwt_keys/refresh", HTTP_POST, json_adapter(refresh_jwt_keys))
+        .set_require_client_identity(false)
+        .install();
     }
   };
 

src/runtime_config/gov.lua

@@ -106,4 +106,11 @@ return {
   end
   return true]],
 
+  update_ca_cert = [[
+  tables, args = ...
+  t = tables["public:ccf.gov.ca_cert_ders"]
+  cert_der = pem_to_der(args.cert)
+  t:put(args.name, cert_der)
+  return true
+  ]],
 }

src/tls/base64.h

@@ -32,7 +32,7 @@ namespace tls
       decoded.data(), decoded.size(), &len_written, data, size);
     if (rc != 0)
     {
-      throw std::logic_error(
+      throw std::invalid_argument(
         fmt::format("Could not decode base64 string: {}", error_string(rc)));
     }
 

src/tls/key_pair.h

@@ -890,7 +890,7 @@ namespace tls
     mbedtls_x509_crt_free(&cert);
     if (rc != 0)
     {
-      throw std::runtime_error(fmt::format(
+      throw std::invalid_argument(fmt::format(
         "Failed to parse certificate, mbedtls_x509_crt_parse: {}",
         tls::error_string(rc)));
     }

tests/infra/consortium.py

@@ -387,6 +387,13 @@ class Consortium:
         proposal = self.get_any_active_member().propose(remote_node, proposal_body)
         return self.vote_using_majority(remote_node, proposal, careful_vote)
 
+    def update_ca_cert(self, remote_node, cert_name, cert_pem_path):
+        proposal_body, careful_vote = self.make_proposal(
+            "update_ca_cert", cert_name, cert_pem_path
+        )
+        proposal = self.get_any_active_member().propose(remote_node, proposal_body)
+        return self.vote_using_majority(remote_node, proposal, careful_vote)
+
     def accept_recovery(self, remote_node):
         proposal_body, careful_vote = self.make_proposal("accept_recovery")
         proposal = self.get_any_active_member().propose(remote_node, proposal_body)

tests/infra/crypto.py

@@ -147,12 +147,12 @@ def generate_rsa_keypair(key_size: int) -> Tuple[str, str]:
     return priv_pem, pub_pem
 
 
-def generate_cert(priv_key_pem: str) -> str:
+def generate_cert(priv_key_pem: str, cn="dummy") -> str:
     priv = load_pem_private_key(priv_key_pem.encode("ascii"), None, default_backend())
     pub = priv.public_key()
     subject = issuer = x509.Name(
         [
-            x509.NameAttribute(NameOID.COMMON_NAME, "dummy"),
+            x509.NameAttribute(NameOID.COMMON_NAME, cn),
         ]
     )
     cert = (

tests/infra/e2e_args.py

@@ -248,6 +248,11 @@ def cli_args(add=lambda x: None, parser=None, accept_unknown=False):
         help="Number of transactions between two snapshots",
         default=None,
     )
+    parser.add_argument(
+        "--jwt-key-refresh-interval-s",
+        help="JWT key refresh interval in seconds",
+        default=None,
+    )
 
     add(parser)
 

tests/infra/network.py

@@ -78,6 +78,7 @@ class Network:
         "ledger_chunk_bytes",
         "domain",
         "snapshot_tx_interval",
+        "jwt_key_refresh_interval_s",
     ]
 
     # Maximum delay (seconds) for updates to propagate from the primary to backups

tests/infra/remote.py

@@ -590,6 +590,7 @@ class CCFRemote(object):
         ledger_chunk_bytes=(5 * 1000 * 1000),
         domain=None,
         snapshot_tx_interval=None,
+        jwt_key_refresh_interval_s=None,
     ):
         """
         Run a ccf binary on a remote host.
@@ -670,6 +671,9 @@ class CCFRemote(object):
         if snapshot_tx_interval:
             cmd += [f"--snapshot-tx-interval={snapshot_tx_interval}"]
 
+        if jwt_key_refresh_interval_s:
+            cmd += [f"--jwt-key-refresh-interval-s={jwt_key_refresh_interval_s}"]
+
         for read_only_ledger_dir in self.read_only_ledger_dirs:
             cmd += [f"--read-only-ledger-dir={os.path.basename(read_only_ledger_dir)}"]
             data_files += [os.path.join(self.common_dir, read_only_ledger_dir)]

tests/jwt_test.py

@@ -3,7 +3,13 @@
 import os
 import tempfile
 import json
+import time
 import base64
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from http import HTTPStatus
+import ssl
+import threading
+from contextlib import AbstractContextManager
 import infra.network
 import infra.path
 import infra.proc
@@ -252,6 +258,163 @@ def test_jwt_with_sgx_key_filter(network, args):
     return network
 
 
+class OpenIDProviderServer(AbstractContextManager):
+    def __init__(self, port: int, tls_key_pem: str, tls_cert_pem: str, jwks: dict):
+        host = "localhost"
+        metadata = {"jwks_uri": f"https://{host}:{port}/keys"}
+        self.jwks = jwks
+        self_ = self
+
+        class MyHTTPRequestHandler(BaseHTTPRequestHandler):
+            def do_GET(self):
+                routes = {
+                    "/.well-known/openid-configuration": metadata,
+                    "/keys": self_.jwks,
+                }
+                body = routes.get(self.path)
+                if body is None:
+                    self.send_error(HTTPStatus.NOT_FOUND)
+                    return
+                body = json.dumps(body).encode()
+                self.send_response(HTTPStatus.OK)
+                self.send_header("Content-Length", str(len(body)))
+                self.end_headers()
+                self.wfile.write(body)
+
+            def log_message(self, fmt, *args):  # pylint: disable=arguments-differ
+                LOG.debug(f"OpenIDProviderServer: {fmt % args}")
+
+        with tempfile.NamedTemporaryFile(
+            prefix="ccf", mode="w+"
+        ) as keyfile_fp, tempfile.NamedTemporaryFile(
+            prefix="ccf", mode="w+"
+        ) as certfile_fp:
+            keyfile_fp.write(tls_key_pem)
+            keyfile_fp.flush()
+            certfile_fp.write(tls_cert_pem)
+            certfile_fp.flush()
+
+            self.httpd = HTTPServer((host, port), MyHTTPRequestHandler)
+            self.httpd.socket = ssl.wrap_socket(
+                self.httpd.socket,
+                keyfile=keyfile_fp.name,
+                certfile=certfile_fp.name,
+                server_side=True,
+            )
+            self.thread = threading.Thread(None, self.httpd.serve_forever)
+            self.thread.start()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+        self.thread.join()
+
+
+@reqs.description("JWT with auto_refresh enabled")
+def test_jwt_key_auto_refresh(network, args):
+    primary, _ = network.find_nodes()
+
+    ca_cert_name = "jwt"
+    kid = "my_kid"
+    issuer_host = "localhost"
+    issuer_port = 12345
+    issuer = f"https://{issuer_host}:{issuer_port}"
+
+    key_priv_pem, _ = infra.crypto.generate_rsa_keypair(2048)
+    cert_pem = infra.crypto.generate_cert(key_priv_pem, cn=issuer_host)
+
+    LOG.info("Add CA cert for JWT issuer")
+    with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as ca_cert_fp:
+        ca_cert_fp.write(cert_pem)
+        ca_cert_fp.flush()
+        network.consortium.update_ca_cert(primary, ca_cert_name, ca_cert_fp.name)
+
+    def check_kv_jwt_key_matches(kid, cert_pem):
+        with primary.client(
+            f"member{network.consortium.get_any_active_member().member_id}"
+        ) as c:
+            r = c.post(
+                "/gov/read",
+                {"table": "public:ccf.gov.jwt_public_signing_keys", "key": kid},
+            )
+            if cert_pem is None:
+                assert r.status_code == 400, r.status_code
+            else:
+                assert r.status_code == 200, r.status_code
+                # Note that /gov/read returns all data as JSON.
+                # Here, the stored data is a uint8 array, therefore it
+                # is returned as an array of integers.
+                cert_kv_der = bytes(r.body.json())
+                cert_kv_pem = infra.crypto.cert_der_to_pem(cert_kv_der)
+                assert infra.crypto.are_certs_equal(
+                    cert_pem, cert_kv_pem
+                ), "stored cert not equal to input cert"
+
+    def get_jwt_refresh_endpoint_metrics() -> dict:
+        with primary.client(
+            f"member{network.consortium.get_any_active_member().member_id}"
+        ) as c:
+            r = c.get("/gov/endpoint_metrics")
+            m = r.body.json()["metrics"]["jwt_keys/refresh"]["POST"]
+            assert m["errors"] == 0, m["errors"]  # not used in jwt refresh endpoint
+            m["successes"] = m["calls"] - m["failures"]
+            return m
+
+    LOG.info("Start OpenID endpoint server")
+    jwks = create_jwks(kid, cert_pem)
+    with OpenIDProviderServer(issuer_port, key_priv_pem, cert_pem, jwks) as server:
+        LOG.info("Add JWT issuer with auto-refresh")
+        with tempfile.NamedTemporaryFile(prefix="ccf", mode="w+") as metadata_fp:
+            json.dump(
+                {"issuer": issuer, "auto_refresh": True, "ca_cert_name": ca_cert_name},
+                metadata_fp,
+            )
+            metadata_fp.flush()
+            network.consortium.set_jwt_issuer(primary, metadata_fp.name)
+
+        LOG.info("Check that keys got refreshed")
+        # Note: refresh interval is set to 1s, see network args below.
+        with_timeout(lambda: check_kv_jwt_key_matches(kid, cert_pem), timeout=5)
+
+        LOG.info("Check that JWT refresh endpoint has no failures")
+        m = get_jwt_refresh_endpoint_metrics()
+        assert m["failures"] == 0, m["failures"]
+        assert m["successes"] > 0, m["successes"]
+
+        LOG.info("Serve invalid JWKS")
+        server.jwks = {"foo": "bar"}
+
+        LOG.info("Check that JWT refresh endpoint has some failures")
+
+        def check_has_failures():
+            m = get_jwt_refresh_endpoint_metrics()
+            assert m["failures"] > 0, m["failures"]
+
+        with_timeout(check_has_failures, timeout=5)
+
+    LOG.info("Restart OpenID endpoint server with new keys")
+    kid2 = "my_kid_2"
+    key2_priv_pem, _ = infra.crypto.generate_rsa_keypair(2048)
+    cert2_pem = infra.crypto.generate_cert(key2_priv_pem, cn=issuer_host)
+    jwks = create_jwks(kid2, cert2_pem)
+    with OpenIDProviderServer(issuer_port, key_priv_pem, cert_pem, jwks):
+        LOG.info("Check that keys got refreshed")
+        with_timeout(lambda: check_kv_jwt_key_matches(kid, None), timeout=5)
+        check_kv_jwt_key_matches(kid2, cert2_pem)
+
+
+def with_timeout(fn, timeout):
+    t0 = time.time()
+    while True:
+        try:
+            return fn()
+        except Exception:
+            if time.time() - t0 < timeout:
+                time.sleep(0.1)
+            else:
+                raise
+
+
 def run(args):
     with infra.network.network(
         args.nodes, args.binary_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
@@ -260,6 +423,7 @@ def run(args):
         network = test_jwt_without_key_policy(network, args)
         network = test_jwt_with_sgx_key_policy(network, args)
         network = test_jwt_with_sgx_key_filter(network, args)
+        network = test_jwt_key_auto_refresh(network, args)
 
 
 if __name__ == "__main__":
@@ -267,4 +431,5 @@ if __name__ == "__main__":
     args = infra.e2e_args.cli_args()
     args.package = "liblogging"
     args.nodes = infra.e2e_args.max_nodes(args, f=0)
+    args.jwt_key_refresh_interval_s = 1
     run(args)