Support for HTTP in the CI (#550)

2019-11-13 09:54:32 +00:00 · 2019-11-13 09:54:32 +00:00 · 2dcf7354b8
--- a/.azure-pipelines-templates/matrix.yml
+++ b/.azure-pipelines-templates/matrix.yml
@ -30,6 +30,8 @@ parameters:
      cmake_args: '-DSERVICE_IDENTITY_CURVE_CHOICE=secp256k1_bitcoin'
    san:
      cmake_args: '-DSAN=ON'
+    http:
+      cmake_args: '-DHTTP=ON'

  test:
    NoSGX:
@ -88,3 +90,15 @@ jobs:
        ctest_filter: '${{ parameters.test.perf.ctest_args }}'
        suffix: 'Perf'
        depends_on: ${{ parameters.static_check_job_name }}
+
+  # HTTP build (behind a toggle for now), only on NoSGX and CFT
+  # TODO: Once large messages are supported with HTTP (e.g. quotes over join request), SGX should be run
+  - template: common.yml
+    parameters:
+      target: NoSGX
+      consensus: CFT
+      env: ${{ parameters.env.NoSGX }}
+      cmake_args: '${{ parameters.build.common.cmake_args }} ${{ parameters.build.debug.cmake_args }} ${{ parameters.build.NoSGX.cmake_args }} ${{ parameters.build.CFT.cmake_args }} ${{ parameters.build.http.cmake_args }}'
+      ctest_filter: '${{ parameters.test.NoSGX.ctest_args }}'
+      suffix: 'HTTP'
+      depends_on: ${{ parameters.static_check_job_name }}
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -273,6 +273,7 @@ if(BUILD_TESTS)
          ${CMAKE_SOURCE_DIR}/tests/raft_scenarios ${CMAKE_SOURCE_DIR})
    set_property(TEST raft_scenario_test PROPERTY LABELS raft_scenario)

+    # TODO: All end-to-end tests should be supported once large messages work with HTTP
    if (NOT HTTP)
      # Member client end to end tests
      add_e2e_test(
@ -286,13 +287,6 @@ if(BUILD_TESTS)
        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/loggingclient.py
      )

-      if (NOT SAN)
-        add_e2e_test(
-          NAME connections
-          PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py
-        )
-      endif()
-
      ## Storing signed votes test
      add_e2e_test(
        NAME voting_history_test
@ -313,52 +307,8 @@ if(BUILD_TESTS)
          ADDITIONAL_ARGS
            --oesign ${OESIGN}
        )
-
-        add_e2e_test(
-          NAME reconfiguration_test
-          PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/reconfiguration.py
-        )
-
-        add_e2e_test(
-         NAME code_update_test
-         PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/code_update.py
-         ADDITIONAL_ARGS
-           --oesign ${OESIGN}
-           --oeconfpath ${CMAKE_CURRENT_SOURCE_DIR}/src/apps/logging/oe_sign.conf
-           --oesignkeypath ${CMAKE_CURRENT_SOURCE_DIR}/src/apps/sample_key.pem
-           # TODO: This test spins up many nodes that go through the join protocol
-           # Since oe_verify_report can take quite a long time to execute (~2s)
-           # and trigger Raft elections, the election timeout should stay high
-           # until https://github.com/microsoft/CCF/issues/480 is fixed
-           --election-timeout 5000
-        )
      endif()

-      add_e2e_test(
-        NAME end_to_end_logging
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
-      )
-
-      add_e2e_test(
-        NAME lua_end_to_end_logging
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
-        ADDITIONAL_ARGS
-          --app-script ${CMAKE_SOURCE_DIR}/src/apps/logging/logging.lua)
-
-      add_e2e_test(
-        NAME end_to_end_scenario
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_scenarios.py
-        ADDITIONAL_ARGS
-          --scenario ${CMAKE_SOURCE_DIR}/tests/simple_logging_scenario.json
-      )
-
-      add_e2e_test(
-        NAME election_tests
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py
-        ADDITIONAL_ARGS
-          --election-timeout 2000
-      )
-
      add_e2e_test(
        NAME recovery_tests
        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
@ -366,14 +316,6 @@ if(BUILD_TESTS)
          ${RECOVERY_ARGS}
      )

-      add_e2e_test(
-        NAME schema_tests
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/schema.py
-        ADDITIONAL_ARGS
-        -p libloggingenc
-        --schema-dir ${CMAKE_SOURCE_DIR}/sphinx/source/schemas
-        )
-
      add_e2e_test(
        NAME test_suite
        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_suite.py
@ -386,11 +328,67 @@ if(BUILD_TESTS)
      if (BUILD_SMALLBANK)
        include(${CMAKE_CURRENT_SOURCE_DIR}/samples/apps/smallbank/smallbank.cmake)
      endif()
+    endif()

-    else()
+    add_e2e_test(
+      NAME end_to_end_logging
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
+    )
+
+    add_e2e_test(
+      NAME lua_end_to_end_logging
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
+      ADDITIONAL_ARGS
+        --app-script ${CMAKE_SOURCE_DIR}/src/apps/logging/logging.lua
+    )
+
+    add_e2e_test(
+      NAME end_to_end_scenario
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_scenarios.py
+      ADDITIONAL_ARGS
+        --scenario ${CMAKE_SOURCE_DIR}/tests/simple_logging_scenario.json
+    )
+
+    add_e2e_test(
+      NAME election_tests
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py
+      ADDITIONAL_ARGS
+        --election-timeout 2000
+    )
+
+    if (NOT SAN)
      add_e2e_test(
-        NAME end_to_end_http
-        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_http.py
+        NAME connections
+        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py
+      )
+    endif()
+
+    add_e2e_test(
+      NAME schema_tests
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/schema.py
+      ADDITIONAL_ARGS
+      -p libloggingenc
+      --schema-dir ${CMAKE_SOURCE_DIR}/sphinx/source/schemas
+    )
+
+    if(QUOTES_ENABLED)
+      add_e2e_test(
+        NAME reconfiguration_test
+        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/reconfiguration.py
+      )
+
+      add_e2e_test(
+        NAME code_update_test
+        PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/code_update.py
+        ADDITIONAL_ARGS
+          --oesign ${OESIGN}
+          --oeconfpath ${CMAKE_CURRENT_SOURCE_DIR}/src/apps/logging/oe_sign.conf
+          --oesignkeypath ${CMAKE_CURRENT_SOURCE_DIR}/src/apps/sample_key.pem
+          # TODO: This test spins up many nodes that go through the join protocol
+          # Since oe_verify_report can take quite a long time to execute (~2s)
+          # and trigger Raft elections, the election timeout should stay high
+          # until https://github.com/microsoft/CCF/issues/480 is fixed
+          --election-timeout 5000
      )
    endif()

--- a/src/clients/memberclient.cpp
+++ b/src/clients/memberclient.cpp
@ -134,8 +134,7 @@ void add_new(
  RpcTlsClient& tls_connection, const string& cert_file, const string& proposal)
 {
  const auto cert = slurp(cert_file);
-  auto verifier = tls::make_verifier(cert);
-  const auto params = proposal_params(proposal, verifier->raw_cert_data());
+  const auto params = proposal_params(proposal, cert);
  const auto response =
    json::from_msgpack(tls_connection.call("propose", params));
  cout << response.dump() << endl;
@ -255,7 +254,7 @@ void submit_ack(
  // member using its own certificate reads its member id
  auto verifier = tls::make_verifier(raw_cert);
  Response<ObjectId> read_id = json::from_msgpack(tls_connection.call(
-    "read", read_params(verifier->raw_cert_data(), Tables::MEMBER_CERTS)));
+    "read", read_params(verifier->der_cert_data(), Tables::MEMBER_CERTS)));
  const auto member_id = read_id.result;

  // member reads nonce
--- a/src/enclave/http.h
+++ b/src/enclave/http.h
@ -11,23 +11,12 @@ namespace enclave
  namespace http
  {
    // TODO: Split into a request formatter class
-    std::vector<uint8_t> post(const std::string& body)
-    {
-      auto req = fmt::format(
-        "POST / HTTP/1.1\r\n"
-        "Content-Type: application/json\r\n"
-        "Content-Length: {}\r\n\r\n{}",
-        body.size(),
-        body);
-      return std::vector<uint8_t>(req.begin(), req.end());
-    }
-
    std::vector<uint8_t> post_header(const std::vector<uint8_t>& body)
    {
      auto req = fmt::format(
        "POST / HTTP/1.1\r\n"
        "Content-Type: application/json\r\n"
-        "Content-Length: {}\r\n\r\n{}",
+        "Content-Length: {}\r\n\r\n",
        body.size());
      return std::vector<uint8_t>(req.begin(), req.end());
    }
@ -128,7 +117,7 @@ namespace enclave
    class ResponseHeaderEmitter
    {
    public:
-      static std::vector<uint8_t> emit(const std::vector<uint8_t> data)
+      static std::vector<uint8_t> emit(const std::vector<uint8_t>& data)
      {
        if (data.size() == 0)
        {
@ -149,7 +138,7 @@ namespace enclave
    class RequestHeaderEmitter
    {
    public:
-      static std::vector<uint8_t> emit(const std::vector<uint8_t> data)
+      static std::vector<uint8_t> emit(const std::vector<uint8_t>& data)
      {
        return http::post_header(data);
      }
--- a/src/host/main.cpp
+++ b/src/host/main.cpp
@ -321,7 +321,7 @@ int main(int argc, char** argv)
    for (auto const& cert_file : member_cert_files)
    {
      ccf_config.genesis.member_certs.emplace_back(
-        tls::make_verifier(files::slurp(cert_file))->raw_cert_data());
+        tls::make_verifier(files::slurp(cert_file))->der_cert_data());
    }
    ccf_config.genesis.gov_script = files::slurp_string(gov_script);
    LOG_INFO_FMT(
--- a/src/node/genesisgen.h
+++ b/src/node/genesisgen.h
@ -110,7 +110,7 @@ namespace ccf
      auto node_id =
        get_next_id(tx.get_view(tables.values), ValueIds::NEXT_NODE_ID);

-      auto raw_cert = tls::make_verifier(node_info.cert)->raw_cert_data();
+      auto raw_cert = tls::make_verifier(node_info.cert)->der_cert_data();

      auto node_view = tx.get_view(tables.nodes);
      node_view->put(node_id, node_info);
--- a/src/node/nodestate.h
+++ b/src/node/nodestate.h
@ -465,6 +465,9 @@ namespace ccf
            setup_history();
            setup_encryptor();

+            // TODO: accept_node_connections() should be moved earlier once
+            // certificate endorsement is changed so that it is possible for
+            // operators to query a node before it has joined
            accept_node_connections();
            accept_member_connections();
            if (public_only)
--- a/src/node/rpc/memberfrontend.h
+++ b/src/node/rpc/memberfrontend.h
@ -72,22 +72,52 @@ namespace ccf
        // add a new member
        {"new_member",
         [this](Store::Tx& tx, const nlohmann::json& args) {
-           const Cert cert = args;
-           auto mc = tx.get_view(this->network.member_certs);
+           const Cert pem_cert = args;
+           auto [mc, m, ack, v] = tx.get_view(
+             this->network.member_certs,
+             this->network.members,
+             this->network.member_acks,
+             this->network.values);
           // the cert needs to be unique
-           if (mc->get(cert))
-             throw std::logic_error("Certificate already exists");
+           auto cert = tls::make_verifier(pem_cert)->der_cert_data();
+           auto member_id = mc->get(cert);
+           if (member_id.has_value())
+           {
+             throw std::logic_error(fmt::format(
+               "Member certificate already exists (member {})",
+               member_id.value()));
+           }

-           const auto id = get_next_id(
-             tx.get_view(this->network.values), ValueIds::NEXT_MEMBER_ID);
+           const auto id = get_next_id(v, ValueIds::NEXT_MEMBER_ID);
           // store cert
           mc->put(cert, id);
           // set state to ACCEPTED
-           tx.get_view(this->network.members)
-             ->put(id, {cert, MemberStatus::ACCEPTED});
+           m->put(id, {cert, MemberStatus::ACCEPTED});
           // create nonce for ACK
-           tx.get_view(this->network.member_acks)
-             ->put(id, {rng->random(SIZE_NONCE)});
+           ack->put(id, {rng->random(SIZE_NONCE)});
+           return true;
+         }},
+        // add a new user
+        {"new_user",
+         [this](Store::Tx& tx, const nlohmann::json& args) {
+           const Cert pem_cert = args;
+           auto [uc, u, v] = tx.get_view(
+             this->network.user_certs,
+             this->network.users,
+             this->network.values);
+           // the cert needs to be unique
+           auto cert = tls::make_verifier(pem_cert)->der_cert_data();
+           auto user_id = uc->get(cert);
+           if (user_id.has_value())
+           {
+             throw std::logic_error(fmt::format(
+               "User certificate already exists (user {})", user_id.value()));
+           }
+
+           const auto id = get_next_id(v, ValueIds::NEXT_USER_ID);
+           // store cert (bi-directional)
+           uc->put(cert, id);
+           u->put(id, {cert});
           return true;
         }},
        // accept a node
@ -404,8 +434,11 @@ namespace ccf
        proposals->put(vote.id, *proposal);

        auto voting_history = args.tx.get_view(this->network.voting_history);
+        // TODO: https://github.com/microsoft/CCF/issues/546
+#ifndef HTTP
        voting_history->put(
          args.caller_id, {args.rpc_ctx.signed_request.value()});
+#endif

        return jsonrpc::success(complete_proposal(args.tx, vote.id));
      };
--- a/src/node/rpc/test/frontend_test.cpp
+++ b/src/node/rpc/test/frontend_test.cpp
@ -264,24 +264,24 @@ std::optional<SignedReq> get_signed_req(CallerId caller_id)
 // caller used throughout
 auto ca = kp -> self_sign("CN=name");
 auto verifier = tls::make_verifier(ca);
-auto user_caller = verifier -> raw_cert_data();
+auto user_caller = verifier -> der_cert_data();

 auto ca_mem = kp -> self_sign("CN=name_member");
 auto verifier_mem = tls::make_verifier(ca_mem);
-auto member_caller = verifier_mem -> raw_cert_data();
+auto member_caller = verifier_mem -> der_cert_data();

 auto ca_node = kp -> self_sign("CN=node");
 auto verifier_node = tls::make_verifier(ca_node);
-auto node_caller = verifier_node -> raw_cert_data();
+auto node_caller = verifier_node -> der_cert_data();

 auto ca_nos = kp -> self_sign("CN=nostore_user");
 auto verifier_nos = tls::make_verifier(ca_nos);
-auto nos_caller = verifier_nos -> raw_cert_data();
+auto nos_caller = verifier_nos -> der_cert_data();

 auto kp_other = tls::make_key_pair();
 auto ca_inv = kp_other -> self_sign("CN=name");
 auto verifier_inv = tls::make_verifier(ca_inv);
-auto invalid_caller = verifier_inv -> raw_cert_data();
+auto invalid_caller = verifier_inv -> der_cert_data();

 const enclave::SessionContext user_session(
  enclave::InvalidSessionId, user_caller);
@ -416,7 +416,8 @@ TEST_CASE("process")
    CHECK(value.req.empty());
    CHECK(value.sig == signed_call[jsonrpc::SIG]);
  }
-
+  // TODO: verify_client_signature
+#ifndef HTTP
  SUBCASE("signature not verified")
  {
    const auto serialized_call = jsonrpc::pack(signed_call, default_pack);
@ -433,6 +434,7 @@ TEST_CASE("process")
    const auto signed_resp = get_signed_req(invalid_user_id);
    CHECK(!signed_resp.has_value());
  }
+#endif
 }

 TEST_CASE("MinimalHandleFuction")
--- a/src/node/rpc/test/membervoting_test.cpp
+++ b/src/node/rpc/test/membervoting_test.cpp
@ -32,7 +32,7 @@ using namespace nlohmann;
 auto kp = tls::make_key_pair();
 auto ca_mem = kp -> self_sign("CN=name_member");
 auto verifier_mem = tls::make_verifier(ca_mem);
-auto member_caller = verifier_mem -> raw_cert_data();
+auto member_caller = verifier_mem -> der_cert_data();
 auto encryptor = std::make_shared<ccf::NullTxEncryptor>();

 constexpr auto default_pack = jsonrpc::Pack::MsgPack;
@ -150,7 +150,7 @@ std::vector<uint8_t> get_cert_data(uint64_t member_id, tls::KeyPairPtr& kp_mem)
  std::vector<uint8_t> ca_mem =
    kp_mem->self_sign("CN=new member" + to_string(member_id));
  auto v_mem = tls::make_verifier(ca_mem);
-  std::vector<uint8_t> cert_data = v_mem->raw_cert_data();
+  std::vector<uint8_t> cert_data = v_mem->der_cert_data();
  return cert_data;
 }

@ -790,7 +790,7 @@ TEST_CASE("Remove proposal")
 {
  NewMember caller;
  auto v = tls::make_verifier(caller.kp->self_sign("CN=new member"));
-  caller.cert = v->raw_cert_data();
+  caller.cert = v->der_cert_data();

  NetworkTables network;
  network.tables->set_encryptor(encryptor);
@ -955,7 +955,7 @@ TEST_CASE("Add user via proposed call")
      return Calls:call("new_user", user_cert)
    )xxx");

-  const vector<uint8_t> user_cert = {1, 2, 3};
+  const vector<uint8_t> user_cert = kp->self_sign("CN=new user");
  json proposej = create_json_req(Propose::In{proposal, user_cert}, "propose");
  ccf::SignedReq sr(proposej);

@ -967,7 +967,8 @@ TEST_CASE("Add user via proposed call")
  const auto uid = tx1.get_view(network.values)->get(ValueIds::NEXT_USER_ID);
  REQUIRE(uid);
  CHECK(*uid == 1);
-  const auto uid1 = tx1.get_view(network.user_certs)->get(user_cert);
+  const auto uid1 = tx1.get_view(network.user_certs)
+                      ->get(tls::make_verifier(user_cert)->der_cert_data());
  REQUIRE(uid1);
  CHECK(*uid1 == 0);
 }
--- a/src/node/rpc/test/nodefrontend_test.cpp
+++ b/src/node/rpc/test/nodefrontend_test.cpp
@ -75,7 +75,7 @@ TEST_CASE("Add a node to an opening service")
  // Node certificate
  tls::KeyPairPtr kp = tls::make_key_pair();
  auto v = tls::make_verifier(kp->self_sign(fmt::format("CN=nodes")));
-  Cert caller = v->raw_cert_data();
+  Cert caller = v->der_cert_data();

  INFO("Add first node before a service exists");
  {
@ -134,7 +134,7 @@ TEST_CASE("Add a node to an opening service")
  {
    tls::KeyPairPtr kp = tls::make_key_pair();
    auto v = tls::make_verifier(kp->self_sign(fmt::format("CN=nodes")));
-    Cert caller = v->raw_cert_data();
+    Cert caller = v->der_cert_data();

    // Network node info is empty (same as before)
    JoinNetworkNodeToNode::In join_input;
@ -166,7 +166,7 @@ TEST_CASE("Add a node to an open service")
  // Node certificate
  tls::KeyPairPtr kp = tls::make_key_pair();
  auto v = tls::make_verifier(kp->self_sign(fmt::format("CN=nodes")));
-  Cert caller = v->raw_cert_data();
+  Cert caller = v->der_cert_data();

  std::optional<NodeInfo> node_info;
  Store::Tx tx;
@ -198,7 +198,7 @@ TEST_CASE("Add a node to an open service")
  {
    tls::KeyPairPtr kp = tls::make_key_pair();
    auto v = tls::make_verifier(kp->self_sign(fmt::format("CN=nodes")));
-    Cert caller = v->raw_cert_data();
+    Cert caller = v->der_cert_data();

    // Network node info is empty (same as before)
    JoinNetworkNodeToNode::In join_input;
--- a/src/node/rpc/userfrontend.h
+++ b/src/node/rpc/userfrontend.h
@ -15,5 +15,11 @@ namespace ccf
        tables_.get<Certs>(Tables::USER_CERTS),
        tables_.get<Users>(Tables::USERS))
    {}
+
+  protected:
+    virtual std::string invalid_caller_error_message() const
+    {
+      return "Could not find matching user certificate";
+    }
  };
 }
--- a/src/runtime_config/gov.lua
+++ b/src/runtime_config/gov.lua
@ -87,14 +87,4 @@ return {
    end
  end
  return true]],
-
-  new_user = [[
-  tables, cert = ...
-  if tables["ccf.user_certs"]:get(cert) then return end
-  NEXT_USER_ID = 1
-  user_id = tables["ccf.values"]:get(NEXT_USER_ID)
-  tables["ccf.values"]:put(NEXT_USER_ID, user_id + 1)
-  tables["ccf.users"]:put(user_id, {cert=cert})
-  tables["ccf.user_certs"]:put(cert, user_id)
-  ]]
 }
--- a/src/runtime_config/operator_gov.lua
+++ b/src/runtime_config/operator_gov.lua
@ -118,14 +118,4 @@ return {
    end
  end
  return true]],
-
-  new_user = [[
-  tables, cert = ...
-  if tables["ccf.user_certs"]:get(cert) then return end
-  NEXT_USER_ID = 1
-  user_id = tables["ccf.values"]:get(NEXT_USER_ID)
-  tables["ccf.values"]:put(NEXT_USER_ID, user_id + 1)
-  tables["ccf.users"]:put(user_id, {cert=cert})
-  tables["ccf.user_certs"]:put(cert, user_id)
-  ]]
 }
--- a/src/tls/keypair.h
+++ b/src/tls/keypair.h
@ -1143,7 +1143,7 @@ namespace tls
      return &cert;
    }

-    std::vector<uint8_t> raw_cert_data()
+    std::vector<uint8_t> der_cert_data()
    {
      const auto crt = raw();
      return {crt->raw.p, crt->raw.p + crt->raw.len};
--- a/tests/code_update.py
+++ b/tests/code_update.py
@ -25,17 +25,6 @@ def get_code_id(lib_path):
    return lines[0].split("=")[1]


-def add_new_code(network, new_code_id):
-    LOG.debug(f"Adding new code id: {new_code_id}")
-
-    primary, term = network.find_primary()
-    result = network.consortium.propose(
-        1, primary, None, None, "add_code", f"--new-code-id={new_code_id}"
-    )
-
-    network.consortium.vote_using_majority(primary, result[1]["id"])
-
-
 def run(args):
    hosts = ["localhost", "localhost"]

@ -57,7 +46,7 @@ def run(args):
            == None
        ), "Adding node with unsupported code id should fail"

-        add_new_code(network, new_code_id)
+        network.consortium.add_new_code(1, primary, new_code_id)

        new_nodes = set()
        old_nodes_count = len(network.nodes)
--- a/tests/e2e_http.py
+++ b/tests/e2e_http.py
@ -1,47 +0,0 @@
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the Apache 2.0 License.
-import os
-import getpass
-import time
-import logging
-import multiprocessing
-import shutil
-from random import seed
-import infra.ccf
-import infra.proc
-import infra.jsonrpc
-import infra.notification
-import infra.net
-import e2e_args
-
-from loguru import logger as LOG
-
-
-def run(args):
-    hosts = ["localhost"]
-
-    with infra.notification.notification_server(args.notify_server) as notifications:
-
-        with infra.ccf.network(
-            hosts, args.build_dir, args.debug_nodes, args.perf_nodes, pdb=args.pdb
-        ) as network:
-            network.start_and_join(args)
-            primary, term = network.find_primary()
-
-            time.sleep(3)
-
-            with primary.user_client(format="json") as c:
-                c.rpc("LOG_record", {"id": 42, "msg": "Hello"})
-
-
-if __name__ == "__main__":
-
-    args = e2e_args.cli_args()
-    args.package = "libloggingenc"
-    notify_server_host = "localhost"
-    args.notify_server = (
-        notify_server_host
-        + ":"
-        + str(infra.net.probably_free_local_port(notify_server_host))
-    )
-    run(args)
--- a/tests/e2e_logging.py
+++ b/tests/e2e_logging.py
@ -100,16 +100,18 @@ def test(network, args, notifications_queue=None):
            check(c.rpc("LOG_get", {"id": 100}), result={"msg": backup_msg})
            check(c.rpc("LOG_get", {"id": 42}), result={"msg": msg})

-        LOG.info("Write/Read large messages on primary")
-        with primary.user_client(format="json") as c:
-            id = 44
-            for p in range(14, 20):
-                long_msg = "X" * (2 ** p)
-                check_commit(
-                    c.rpc("LOG_record", {"id": id, "msg": long_msg}), result=True,
-                )
-                check(c.rpc("LOG_get", {"id": id}), result={"msg": long_msg})
-            id += 1
+        # TODO: Remove when HTTP supports large messages
+        if not os.getenv("HTTP"):
+            LOG.info("Write/Read large messages on primary")
+            with primary.user_client(format="json") as c:
+                id = 44
+                for p in range(14, 20):
+                    long_msg = "X" * (2 ** p)
+                    check_commit(
+                        c.rpc("LOG_record", {"id": id, "msg": long_msg}), result=True,
+                    )
+                    check(c.rpc("LOG_get", {"id": id}), result={"msg": long_msg})
+                id += 1

    return network

--- a/tests/e2e_scenarios.py
+++ b/tests/e2e_scenarios.py
@ -40,7 +40,7 @@ def run(args):

            check = infra.checker.Checker()
            check_commit = infra.checker.Checker(mc)
-            with primary.user_client() as uc:
+            with primary.user_client(format="json") as uc:
                check_commit(uc.do("mkSign", params={}), result=True)

            for connection in scenario["connections"]:
--- a/tests/election.py
+++ b/tests/election.py
@ -63,7 +63,7 @@ def run(args):

            LOG.debug("Commit new transactions")
            commit_index = None
-            with primary.user_client() as c:
+            with primary.user_client(format="json") as c:
                res = c.do(
                    "LOG_record",
                    {
--- a/tests/infra/ccf.py
+++ b/tests/infra/ccf.py
@ -199,6 +199,10 @@ class Network:
        self.status = ServiceStatus.OPEN
        LOG.success("***** Network is now open *****")

+        if os.getenv("HTTP"):
+            LOG.warning("Sleeping 3 seconds before continuing (HTTP)...")
+            time.sleep(3)
+
    def start_in_recovery(self, args, ledger_file, sealed_secrets):
        primary = self._start_all_nodes(
            args, recovery=True, ledger_file=ledger_file, sealed_secrets=sealed_secrets
@ -252,6 +256,9 @@ class Network:
        try:
            if self.status is ServiceStatus.OPEN:
                self.consortium.trust_node(1, primary, new_node.node_id)
+                if os.getenv("HTTP"):
+                    LOG.warning("Sleeping 3 seconds before continuing (HTTP)...")
+                    time.sleep(3)
            if not args.pbft:
                new_node.wait_for_node_to_join()
        except (ValueError, TimeoutError):
--- a/tests/infra/consortium.py
+++ b/tests/infra/consortium.py
@ -11,8 +11,6 @@ import infra.proc
 import infra.checker
 import infra.node

-from loguru import logger as LOG
-

 class Consortium:
    def __init__(self, members):
@ -38,19 +36,12 @@ class Consortium:
        j_result = json.loads(result.stdout)
        return j_result

-    def propose(self, member_id, remote_node, script=None, params=None, *args):
-        if os.getenv("HTTP"):
-            with remote_node.member_client() as mc:
-                r = mc.rpc("propose", {"parameter": params, "script": {"text": script}})
-                return (True, r.result)
-        else:
-            j_result = self._member_client_rpc_as_json(member_id, remote_node, *args)
-
-            if j_result.get("error") is not None:
-                return (False, j_result["error"])
-
-            return (True, j_result["result"])
+    def propose(self, member_id, remote_node, script=None, params=None):
+        with remote_node.member_client(format="json", member_id=member_id) as mc:
+            r = mc.rpc("propose", {"parameter": params, "script": {"text": script}})
+            return r.result, r.error

+    # TODO: Remove use of memberclient when client signatures are supported from JSON-RPC/HTTP
    def vote(
        self,
        member_id,
@ -65,7 +56,7 @@ class Consortium:
            tables, changes = ...
            return true
            """
-            with remote_node.member_client(member_id) as mc:
+            with remote_node.member_client(format="json", member_id=member_id) as mc:
                r = mc.rpc(
                    "vote", {"ballot": {"text": script}, "id": proposal_id}, signed=True
                )
@ -127,45 +118,45 @@ class Consortium:
        return self._member_client_rpc_as_json(member_id, remote_node, "ack")

    def get_proposals(self, member_id, remote_node):
-        return self._member_client_rpc_as_json(
-            member_id, remote_node, "proposal_display"
-        )
+        script = """
+        tables = ...
+        local proposals = {}
+        tables["ccf.proposals"]:foreach( function(k, v)
+            proposals[tostring(k)] = v;
+        end )
+        return proposals;
+        """

-    def raw_puts(self, member_id, remote_node, script, params):
-        return self._member_client_rpc_as_json(
-            member_id,
-            remote_node,
-            "raw_puts",
-            "raw_puts",
-            f"--script={script}",
-            f"--param={params}",
-        )
+        with remote_node.member_client(format="json", member_id=member_id) as c:
+            rep = c.do("query", {"text": script})
+            return rep.result

    def propose_retire_node(self, member_id, remote_node, node_id):
-        return self.propose(
-            member_id, remote_node, None, None, "retire_node", f"--node-id={node_id}"
-        )
+        script = """
+        tables, node_id = ...
+        return Calls:call("retire_node", node_id)
+        """
+        return self.propose(member_id, remote_node, script, node_id)

    def retire_node(self, remote_node, node_to_retire):
        member_id = 1
-        result = self.propose_retire_node(
+        result, error = self.propose_retire_node(
            member_id, remote_node, node_to_retire.node_id
        )
-        self.vote_using_majority(remote_node, result[1]["id"])
+        self.vote_using_majority(remote_node, result["id"])

-        with remote_node.member_client() as c:
+        with remote_node.member_client(format="json") as c:
            id = c.request(
                "read", {"table": "ccf.nodes", "key": node_to_retire.node_id}
            )
-            assert (
-                c.response(id).result["status"].decode()
-                == infra.node.NodeStatus.RETIRED.name
-            )
+            assert c.response(id).result["status"] == infra.node.NodeStatus.RETIRED.name

    def propose_trust_node(self, member_id, remote_node, node_id):
-        return self.propose(
-            member_id, remote_node, None, None, "trust_node", f"--node-id={node_id}"
-        )
+        script = """
+        tables, node_id = ...
+        return Calls:call("trust_node", node_id)
+        """
+        return self.propose(member_id, remote_node, script, node_id)

    def trust_node(self, member_id, remote_node, node_id):
        if not self._check_node_exists(
@ -173,8 +164,8 @@ class Consortium:
        ):
            raise ValueError(f"Node {node_id} does not exist in state PENDING")

-        result = self.propose_trust_node(member_id, remote_node, node_id)
-        self.vote_using_majority(remote_node, result[1]["id"])
+        result, error = self.propose_trust_node(member_id, remote_node, node_id)
+        self.vote_using_majority(remote_node, result["id"])

        if not self._check_node_exists(
            remote_node, node_id, infra.node.NodeStatus.TRUSTED
@ -182,14 +173,13 @@ class Consortium:
            raise ValueError(f"Node {node_id} does not exist in state TRUSTED")

    def propose_add_member(self, member_id, remote_node, new_member_cert):
-        return self.propose(
-            member_id,
-            remote_node,
-            None,
-            None,
-            "add_member",
-            f"--member-cert={new_member_cert}",
-        )
+        script = """
+        tables, member_cert = ...
+        return Calls:call("new_member", member_cert)
+        """
+        with open(new_member_cert) as cert:
+            new_member_cert_pem = [ord(c) for c in cert.read()]
+        return self.propose(member_id, remote_node, script, new_member_cert_pem)

    def open_network(self, member_id, remote_node, pbft_open=False):
        """
@ -197,73 +187,56 @@ class Consortium:
        proposal and make members vote to transition the network to state
        OPEN.
        """
-        script = None
-        if os.getenv("HTTP"):
-            script = """
-            tables = ...
-            return Calls:call("open_network")
-            """
-        result = self.propose(member_id, remote_node, script, None, "open_network")
-        self.vote_using_majority(remote_node, result[1]["id"], pbft_open)
+        script = """
+        tables = ...
+        return Calls:call("open_network")
+        """
+        result, error = self.propose(member_id, remote_node, script)
+        self.vote_using_majority(remote_node, result["id"], pbft_open)
        self.check_for_service(remote_node, infra.ccf.ServiceStatus.OPEN)

    def add_users(self, remote_node, users):
-        if os.getenv("HTTP"):
-            with remote_node.member_client() as mc:
-                for u in users:
-                    user_cert = []
-                    with open(f"user{u}_cert.pem") as cert:
-                        user_cert = [ord(c) for c in cert.read()]
-                    script = """
-                    tables, user_cert = ...
-                    return Calls:call("new_user", user_cert)
-                    """
-                    r = mc.rpc(
-                        "propose", {"parameter": user_cert, "script": {"text": script}}
-                    )
-                    with remote_node.member_client(2) as mc2:
-                        script = """
-                        tables, changes = ...
-                        return true
-                        """
-                        r = mc2.rpc(
-                            "vote",
-                            {"ballot": {"text": script}, "id": r.result["id"]},
-                            signed=True,
-                        )
-        else:
-            for u in users:
-                result = self.propose(
-                    1,
-                    remote_node,
-                    None,
-                    None,
-                    "add_user",
-                    f"--user-cert=user{u}_cert.pem",
-                )
-                self.vote_using_majority(remote_node, result[1]["id"])
+        for u in users:
+            user_cert = []
+            with open(f"user{u}_cert.pem") as cert:
+                user_cert = [ord(c) for c in cert.read()]
+            script = """
+            tables, user_cert = ...
+            return Calls:call("new_user", user_cert)
+            """
+            result, error = self.propose(1, remote_node, script, user_cert)
+            self.vote_using_majority(remote_node, result["id"])

    def set_lua_app(self, member_id, remote_node, app_script):
-        result = self.propose(
-            member_id,
-            remote_node,
-            None,
-            None,
-            "set_lua_app",
-            f"--lua-app-file={app_script}",
-        )
-        self.vote_using_majority(remote_node, result[1]["id"])
+        script = """
+        tables, app = ...
+        return Calls:call("set_lua_app", app)
+        """
+        with open(app_script) as app:
+            new_lua_app = app.read()
+        result, error = self.propose(member_id, remote_node, script, new_lua_app)
+        self.vote_using_majority(remote_node, result["id"])

    def accept_recovery(self, member_id, remote_node, sealed_secrets):
-        result = self.propose(
-            member_id,
-            remote_node,
-            None,
-            None,
-            "accept_recovery",
-            f"--sealed-secrets={sealed_secrets}",
+        script = """
+        tables, sealed_secrets = ...
+        return Calls:call("accept_recovery", sealed_secrets)
+        """
+        with open(sealed_secrets) as s:
+            sealed_json = json.load(s)
+
+        result, error = self.propose(member_id, remote_node, script, sealed_json)
+        self.vote_using_majority(remote_node, result["id"])
+
+    def add_new_code(self, member_id, remote_node, new_code_id):
+        script = """
+        tables, code_digest = ...
+        return Calls:call("new_code", code_digest)
+        """
+        result = self._member_client_rpc_as_json(
+            member_id, remote_node, "add_code", f"--new-code-id={new_code_id}",
        )
-        self.vote_using_majority(remote_node, result[1]["id"])
+        self.vote_using_majority(remote_node, result["result"]["id"])

    def check_for_service(self, remote_node, status):
        """
@ -291,11 +264,11 @@ class Consortium:
            ), f"Service status {current_status} (expected {status.name})"

    def _check_node_exists(self, remote_node, node_id, node_status=None):
-        with remote_node.member_client() as c:
+        with remote_node.member_client(format="json") as c:
            rep = c.do("read", {"table": "ccf.nodes", "key": node_id})

            if rep.error is not None or (
-                node_status and rep.result["status"].decode() != node_status.name
+                node_status and rep.result["status"] != node_status.name
            ):
                return False

--- a/tests/infra/jsonrpc.py
+++ b/tests/infra/jsonrpc.py
@ -374,7 +374,7 @@ class CurlClient:
            nf.write(msg)
            nf.flush()
            dgst = subprocess.run(
-                ["openssl", "dgst", "-sha256", "-sign", "member1_privk.pem", nf.name],
+                ["openssl", "dgst", "-sha256", "-sign", self.key, nf.name],
                check=True,
                capture_output=True,
            )
@ -407,8 +407,8 @@ class CurlClient:
            self.stream.update(rc.stdout)
        return r.id

-    def request(self, method, params):
-        r = self.stream.request(f"{self.prefix}/{method}", params)
+    def request(self, method, params, *args, **kwargs):
+        r = self.stream.request(f"{self.prefix}/{method}", params, *args, **kwargs)
        with tempfile.NamedTemporaryFile() as nf:
            msg = getattr(r, "to_{}".format(self.format))()
            LOG.debug("Going to send {}".format(msg))
--- a/tests/infra/node.py
+++ b/tests/infra/node.py
@ -209,7 +209,7 @@ class Node:
            **kwargs,
        )

-    def member_client(self, member_id=1, **kwargs):
+    def member_client(self, format="msgpack", member_id=1, **kwargs):
        return infra.jsonrpc.client(
            self.host,
            self.rpc_port,
@ -218,6 +218,7 @@ class Node:
            key="member{}_privk.pem".format(member_id),
            cafile="networkcert.pem",
            description="node {} (member)".format(self.node_id),
+            format=format,
            prefix="members",
            **kwargs,
        )
--- a/tests/memberclient.py
+++ b/tests/memberclient.py
@ -29,35 +29,38 @@ def run(args):
        primary, term = network.find_primary()

        LOG.debug("Network should not be able to be opened twice")
-        result = network.consortium.propose(1, primary, None, None, "open_network")
+        script = """
+        tables = ...
+        return Calls:call("open_network")
+        """
+        result, _ = network.consortium.propose(1, primary, script)
        assert not network.consortium.vote_using_majority(
-            primary, result[1]["id"]
+            primary, result["id"]
        ), "Network should not be opened twice"

        # Create a lua query file to change a member state to accepted
-        with open("query.lua", "w") as qfile:
-            qfile.write(
-                """local tables, param = ...
-               local member_id = param
-               local STATE_ACCEPTED = 0
-               local member_info = {cert = {}, status = STATE_ACCEPTED}
-               local p = Puts:new()
-               p:put("ccf.members", member_id, member_info)
-               return Calls:call("raw_puts", p)"""
-            )
-
-        # Create json file to be passed as the argument to the query.lua file
-        # It is passing a member id
-        with open("param.json", "w") as pfile:
-            pfile.write("""{"p": 0}""")
+        query = """local tables, param = ...
+        local member_id = param
+        local STATE_ACCEPTED = 0
+        local member_info = {cert = {}, status = STATE_ACCEPTED}
+        local p = Puts:new()
+        p:put("ccf.members", member_id, member_info)
+        return Calls:call("raw_puts", p)
+        """

        LOG.info("Proposal to add a new member")
        infra.proc.ccall("./keygenerator", "--name=member4")
-        result = network.consortium.propose_add_member(1, primary, "member4_cert.pem")
+        script = """
+        tables, member_cert = ...
+        return Calls:call("new_member", member_cert)
+        """
+        result, _ = network.consortium.propose_add_member(
+            1, primary, "member4_cert.pem"
+        )

        # When proposal is added the proposal id and the result of running complete proposal are returned
-        proposal_id = result[1]["id"]
-        assert not result[1]["completed"]
+        proposal_id = result["id"]
+        assert not result["completed"]

        # Display all proposals
        proposals = network.consortium.get_proposals(1, primary)
@ -112,21 +115,21 @@ def run(args):
            result["error"]["code"] == infra.jsonrpc.ErrorCode.INVALID_CALLER_ID.value
        )

-        LOG.info("New non-accepted member should get insufficient rights response")
-        result = network.consortium.propose(
-            4, primary, None, None, "trust_node", "--node-id=0"
-        )
-        assert result[1]["code"] == infra.jsonrpc.ErrorCode.INSUFFICIENT_RIGHTS.value
+        LOG.info("New non-active member should get insufficient rights response")
+        script = """
+        tables, node_id = ...
+        return Calls:call("trust_node", node_id)
+        """
+        result, error = network.consortium.propose(4, primary, script, 0)
+        assert error["code"] == infra.jsonrpc.ErrorCode.INSUFFICIENT_RIGHTS.value

        LOG.debug("New member ACK")
        result = network.consortium.ack(4, primary)

        LOG.info("New member is now active and send an accept node proposal")
-        result = network.consortium.propose(
-            4, primary, None, None, "trust_node", "--node-id=0"
-        )
-        assert not result[1]["completed"]
-        proposal_id = result[1]["id"]
+        result, _ = network.consortium.propose(4, primary, script, 0)
+        assert not result["completed"]
+        proposal_id = result["id"]

        LOG.debug("Members vote to accept the accept node proposal")
        result = network.consortium.vote(1, primary, proposal_id, True)
@ -137,11 +140,9 @@ def run(args):
        assert result[0] and result[1]

        LOG.info("New member makes a new proposal")
-        result = network.consortium.propose(
-            4, primary, None, None, "trust_node", "--node-id=1"
-        )
-        proposal_id = result[1]["id"]
-        assert not result[1]["completed"]
+        result, _ = network.consortium.propose(4, primary, script, 1)
+        proposal_id = result["id"]
+        assert not result["completed"]

        LOG.debug("Other members (non proposer) are unable to withdraw new proposal")
        result = network.consortium.withdraw(2, primary, proposal_id)
@ -172,9 +173,9 @@ def run(args):
        assert result[1]["code"] == params_error

        LOG.debug("New member proposes to deactivate member 1")
-        result = network.consortium.raw_puts(4, primary, "query.lua", "param.json")
-        assert not result["result"]["completed"]
-        proposal_id = result["result"]["id"]
+        result, _ = network.consortium.propose(4, primary, query, 0)
+        assert not result["completed"]
+        proposal_id = result["id"]

        LOG.debug("Other members accept the proposal")
        result = network.consortium.vote(3, primary, proposal_id, True)
@ -184,16 +185,12 @@ def run(args):
        assert result[0] and result[1]

        LOG.debug("Deactivated member cannot make a new proposal")
-        result = network.consortium.propose(
-            1, primary, None, None, "trust_node", "--node-id=0"
-        )
-        assert result[1]["code"] == infra.jsonrpc.ErrorCode.INSUFFICIENT_RIGHTS.value
+        result, error = network.consortium.propose(1, primary, script, 0)
+        assert error["code"] == infra.jsonrpc.ErrorCode.INSUFFICIENT_RIGHTS.value

        LOG.debug("New member should still be able to make a new proposal")
-        result = network.consortium.propose(
-            4, primary, None, None, "add_user", "--user-cert=member3_cert.pem"
-        )
-        assert not result[1]["completed"]
+        result, _ = network.consortium.propose(4, primary, script, 0)
+        assert not result["completed"]


 if __name__ == "__main__":
--- a/tests/recovery.py
+++ b/tests/recovery.py
@ -38,20 +38,20 @@ def check_nodes_have_msgs(nodes, txs):
    makes sure nodes have recovered state.
    """
    for node in nodes:
-        with node.user_client() as c:
+        with node.user_client(format="json") as c:
            for n, msg in txs.priv.items():
                c.do(
                    "LOG_get",
                    {"id": n},
                    readonly_hint=None,
-                    expected_result={"msg": msg.encode()},
+                    expected_result={"msg": msg},
                )
            for n, msg in txs.pub.items():
                c.do(
                    "LOG_get_pub",
                    {"id": n},
                    readonly_hint=None,
-                    expected_result={"msg": msg.encode()},
+                    expected_result={"msg": msg},
                )


@ -61,7 +61,7 @@ def log_msgs(primary, txs):
    """
    LOG.debug("Applying new transactions")
    responses = []
-    with primary.user_client() as c:
+    with primary.user_client(format="json") as c:
        for n, msg in txs.priv.items():
            responses.append(c.rpc("LOG_record", {"id": n, "msg": msg}))
        for n, msg in txs.pub.items():
--- a/tests/votinghistory.py
+++ b/tests/votinghistory.py
@ -95,12 +95,14 @@ def run(args):

        LOG.debug("Propose to add a new member")
        infra.proc.ccall("./keygenerator", "--name=member4")
-        result = network.consortium.propose_add_member(1, primary, "member4_cert.pem")
+        result, error = network.consortium.propose_add_member(
+            1, primary, "member4_cert.pem"
+        )

        # When proposal is added the proposal id and the result of running
        # complete proposal are returned
-        assert not result[1]["completed"]
-        proposal_id = result[1]["id"]
+        assert not result["completed"]
+        proposal_id = result["id"]

        # 2 out of 3 members vote to accept the new member so that
        # that member can send its own proposals