Switch from pylint to ruff (#5270)

.ruff.toml

@@ -0,0 +1,2 @@
+extend-exclude = ["*_pb2*.py"]
+line-length = 2000

python/ccf/ledger.py

@@ -507,7 +507,7 @@ class LedgerValidator:
             elif self.service_status == "Open":
                 assert updated_status in ["Recovering"], updated_status
             else:
-                assert self.service_status == None, self.service_status
+                assert self.service_status is None, self.service_status
             self.service_status = updated_status
 
         # Checks complete, add this transaction to tree

python/ccf/ledger_viz.py

@@ -151,8 +151,8 @@ def main():
         else None,
     )
 
-    l = DefaultLiner(args.write_views, args.split_views, args.split_services)
-    l.help()
+    liner = DefaultLiner(args.write_views, args.split_views, args.split_services)
+    liner.help()
     current_service_identity = None
     for chunk in ledger:
         for tx in chunk:
@@ -163,33 +163,33 @@ def main():
             seqno = tx.gcm_header.seqno
             if not has_private:
                 if ccf.ledger.SIGNATURE_TX_TABLE_NAME in public:
-                    l.entry("Signature", view, seqno)
+                    liner.entry("Signature", view, seqno)
                 else:
                     if all(
                         table.startswith("public:ccf.internal.") for table in public
                     ):
-                        l.entry("Internal", view, seqno)
+                        liner.entry("Internal", view, seqno)
                     elif any(table.startswith("public:ccf.gov.") for table in public):
                         service_info = try_get_service_info(public)
                         if service_info is None:
-                            l.entry("Governance", view, seqno)
+                            liner.entry("Governance", view, seqno)
                         elif service_info["status"] == "Opening":
-                            l.entry("New Service", view, seqno)
+                            liner.entry("New Service", view, seqno)
                             current_service_identity = service_info["cert"]
                         elif service_info["status"] == "Recovering":
-                            l.entry("Recovering Service", view, seqno)
+                            liner.entry("Recovering Service", view, seqno)
                             current_service_identity = service_info["cert"]
                         elif (
                             service_info["cert"] == current_service_identity
                             and service_info["status"] == "Open"
                         ):
-                            l.entry("Service Open", view, seqno)
+                            liner.entry("Service Open", view, seqno)
                     else:
-                        l.entry("User Public", view, seqno)
+                        liner.entry("User Public", view, seqno)
             else:
-                l.entry("User Private", view, seqno)
+                liner.entry("User Private", view, seqno)
 
-    l.flush()
+    liner.flush()
 
 
 if __name__ == "__main__":

python/ccf/read_ledger.py

@@ -76,8 +76,8 @@ def print_key(key, table_name, tables_format_rules, indent_s, is_removed=False):
         LOG.info(f"{indent_s}{k}:")
 
 
-def counted_string(l, name):
-    return f"{len(l)} {name}{'s' * bool(len(l) != 1)}"
+def counted_string(string, name):
+    return f"{len(string)} {name}{'s' * bool(len(string) != 1)}"
 
 
 def dump_entry(entry, table_filter, tables_format_rules):

python/ledger_tutorial.py

@@ -5,6 +5,7 @@ import sys
 from loguru import logger as LOG
 import json
 import random
+import ccf.ledger
 
 # Change default log format
 LOG.remove()
@@ -19,12 +20,10 @@ if len(sys.argv) < 2:
 
 ledger_dirs = sys.argv[1:]
 
-
 # Because all ledger files are closed and are no longer being
 # written to, it is safe to read all of them, even those that may
 # contain uncommitted transactions.
 # SNIPPET_START: create_ledger
-import ccf.ledger
 
 ledger = ccf.ledger.Ledger(ledger_dirs, committed_only=False)
 # SNIPPET_END: create_ledger

scripts/ci-checks.sh

@@ -101,7 +101,7 @@ fi
 
 source scripts/env/bin/activate
 pip install -U pip
-pip install -U wheel black pylint mypy 1>/dev/null
+pip install -U wheel black pylint mypy ruff 1>/dev/null
 endgroup
 
 group "Python format"
@@ -119,7 +119,7 @@ pip install -U -r python/requirements.txt 1>/dev/null
 endgroup
 
 group "Python lint"
-PYTHONPATH=./tests git ls-files tests/ python/ | grep -e '\.py$' | xargs python -m pylint --ignored-modules "*_pb2"
+ruff check python/ tests/
 endgroup
 
 group "Python types"

tests/ca_certs.py

@@ -75,7 +75,7 @@ def test_cert_store(network, args):
         network.get_ledger_public_state_at(remove_proposal.completed_seqno)[
             "public:ccf.gov.tls.ca_cert_bundles"
         ][raw_cert_name]
-        == None
+        is None
     ), "CA bundle was not removed"
 
     return network

tests/e2e_logging.py

@@ -1126,7 +1126,7 @@ def test_forwarding_frontends(network, args):
             ack = network.consortium.get_any_active_member().ack(backup)
             check_commit(ack)
     except AckException as e:
-        assert args.http2 == True
+        assert args.http2 is True
         assert e.response.status_code == http.HTTPStatus.NOT_IMPLEMENTED
         r = e.response.body.json()
         assert (
@@ -1134,7 +1134,7 @@ def test_forwarding_frontends(network, args):
             == "Request cannot be forwarded to primary on HTTP/2 interface."
         ), r
     else:
-        assert args.http2 == False
+        assert args.http2 is False
 
     try:
         msg = "forwarded_msg"
@@ -1148,7 +1148,7 @@ def test_forwarding_frontends(network, args):
             msg=msg,
         )
     except infra.logging_app.LoggingTxsIssueException as e:
-        assert args.http2 == True
+        assert args.http2 is True
         assert e.response.status_code == http.HTTPStatus.NOT_IMPLEMENTED
         r = e.response.body.json()
         assert (
@@ -1156,7 +1156,7 @@ def test_forwarding_frontends(network, args):
             == "Request cannot be forwarded to primary on HTTP/2 interface."
         ), r
     else:
-        assert args.http2 == False
+        assert args.http2 is False
 
     if args.package == "samples/apps/logging/liblogging" and not args.http2:
         with backup.client("user0") as c:
@@ -1617,7 +1617,7 @@ def test_post_local_commit_failure(network, args):
             "/app/log/private/anonymous/v2?fail=false", {"id": 100, "msg": "hello"}
         )
         assert r.status_code == http.HTTPStatus.OK.value, r.status_code
-        assert r.body.json()["success"] == True
+        assert r.body.json()["success"] is True
         TxID.from_str(r.body.json()["tx_id"])
 
         r = c.post(
@@ -1720,8 +1720,8 @@ def test_basic_constraints(network, args):
     basic_constraints = ca_cert.extensions.get_extension_for_oid(
         ObjectIdentifier("2.5.29.19")
     )
-    assert basic_constraints.critical == True
-    assert basic_constraints.value.ca == True
+    assert basic_constraints.critical is True
+    assert basic_constraints.value.ca is True
     assert basic_constraints.value.path_length == 0
 
     node_pem = primary.get_tls_certificate_pem()
@@ -1729,8 +1729,8 @@ def test_basic_constraints(network, args):
     basic_constraints = node_cert.extensions.get_extension_for_oid(
         ObjectIdentifier("2.5.29.19")
     )
-    assert basic_constraints.critical == True
-    assert basic_constraints.value.ca == False
+    assert basic_constraints.critical is True
+    assert basic_constraints.value.ca is False
 
 
 def run_udp_tests(args):

tests/e2e_operations.py

@@ -31,11 +31,11 @@ def test_save_committed_ledger_files(network, args):
     LOG.info(f"Moving committed ledger files to {args.common_read_only_ledger_dir}")
     primary, _ = network.find_primary()
     for ledger_dir in primary.remote.ledger_paths():
-        for l in os.listdir(ledger_dir):
-            if infra.node.is_file_committed(l):
+        for ledger_file_path in os.listdir(ledger_dir):
+            if infra.node.is_file_committed(ledger_file_path):
                 shutil.move(
-                    os.path.join(ledger_dir, l),
-                    os.path.join(args.common_read_only_ledger_dir, l),
+                    os.path.join(ledger_dir, ledger_file_path),
+                    os.path.join(args.common_read_only_ledger_dir, ledger_file_path),
                 )
 
     network.txs.verify(network)

tests/external_executor/executors/logging_app/logging_app.py

@@ -113,7 +113,7 @@ class LoggingExecutor:
                 response.body = e.details().encode()
                 return
 
-            if result.retry == True:
+            if result.retry is True:
                 response.status_code = HTTP.HttpStatusCode.ACCEPTED
                 response.body = "Historical transaction is not currently available. Please retry.".encode()
                 return

tests/external_executor/executors/util.py

@@ -13,7 +13,7 @@ class ExecutorThread:
         self.activated_event = None
 
     def start(self):
-        assert self.thread == None, "Already started"
+        assert self.thread is None, "Already started"
         LOG.info("Starting executor")
         self.activated_event = threading.Event()
         self.thread = threading.Thread(
@@ -25,7 +25,7 @@ class ExecutorThread:
         ), "Executor failed to activate after 3 seconds"
 
     def terminate(self):
-        assert self.thread != None, "Already terminated"
+        assert self.thread is not None, "Already terminated"
         LOG.info("Terminating executor")
         self.executor.terminate()
         self.thread.join()

tests/external_executor/executors/wiki_cacher/wiki_cacher.py

@@ -74,7 +74,7 @@ class WikiCacherExecutor:
         prefix = "/update_cache/"
         title = request.uri[len(prefix) :]
         description = self._get_description(title)
-        if description == None:
+        if description is None:
             response.status_code = HTTP.HttpStatusCode.BAD_GATEWAY
             response.body = f"Error when fetching article with title '{title}'".encode(
                 "utf-8"

tests/governance.py

@@ -633,9 +633,10 @@ def single_node(args):
 
             test_desc("Logging levels of governance operations")
             consortium = network.consortium
-            rand_hex = lambda: md5(
-                random.getrandbits(32).to_bytes(4, "big")
-            ).hexdigest()
+
+            def rand_hex():
+                return md5(random.getrandbits(32).to_bytes(4, "big")).hexdigest()
+
             validate_info = f"Logged at info during validate: {rand_hex()}"
             validate_warn = f"Logged at warn during validate: {rand_hex()}"
             validate_js = (

tests/governance_history.py

@@ -121,7 +121,7 @@ def check_signatures(ledger):
                 sig_txid = TxID(tr.gcm_header.view, tr.gcm_header.seqno)
 
                 # Adjacent signatures only occur on a view change
-                if prev_sig_txid != None:
+                if prev_sig_txid is not None:
                     if prev_sig_txid.seqno + 1 == sig_txid.seqno:
                         # Reduced from assert while investigating cause
                         # https://github.com/microsoft/CCF/issues/5078

tests/infra/clients.py

@@ -446,7 +446,7 @@ class CurlClient:
                     nf.write(msg_bytes)
                     nf.flush()
                     content_path = f"@{nf.name}"
-                if not "content-type" in headers and request.body:
+                if "content-type" not in headers and request.body:
                     headers["content-type"] = content_type
 
             cmd = ["curl"]
@@ -636,7 +636,7 @@ class HttpxClient:
                 request_body = json.dumps(request.body).encode()
                 content_type = CONTENT_TYPE_JSON
 
-            if not "content-type" in request.headers and len(request.body) > 0:
+            if "content-type" not in request.headers and len(request.body) > 0:
                 extra_headers["content-type"] = content_type
 
         if self.cose_signing_auth is not None and request.http_verb != "GET":
@@ -837,10 +837,10 @@ class RawSocketClient:
                 content_type = CONTENT_TYPE_JSON
             content_length = len(request_body)
 
-            if not "content-type" in request.headers and len(request.body) > 0:
+            if "content-type" not in request.headers and len(request.body) > 0:
                 extra_headers["content-type"] = content_type
 
-        if not "content-length" in extra_headers:
+        if "content-length" not in extra_headers:
             extra_headers["content-length"] = content_length
 
         if self.signing_details is not None:

tests/infra/consortium.py

@@ -250,9 +250,9 @@ class Consortium:
 
     def get_any_active_member(self, recovery_member=None):
         if recovery_member is not None:
-            if recovery_member == True:
+            if recovery_member is True:
                 return random.choice(self.get_active_recovery_members())
-            elif recovery_member == False:
+            elif recovery_member is False:
                 return random.choice(self.get_active_non_recovery_members())
         else:
             return random.choice(self.get_active_members())

tests/infra/logging_app.py

@@ -35,21 +35,21 @@ class LoggingTxsVerifyException(Exception):
     """
 
 
-def sample_list(l, n):
-    if n > len(l):
+def sample_list(list_, n):
+    if n > len(list_):
         # Return all elements
-        return l
+        return list_
     elif n == 0:
         return []
     elif n == 1:
         # Return last element only
-        return l[-1:]
+        return list_[-1:]
     elif n == 2:
         # Return first and last elements
-        return l[:1] + l[-1:]
+        return list_[:1] + list_[-1:]
     else:
         # Return first, last, and random sample of values in-between
-        return l[:1] + random.sample(l[1:-1], n - 2) + l[-1:]
+        return list_[:1] + random.sample(list_[1:-1], n - 2) + list_[-1:]
 
 
 class LoggingTxs:

tests/infra/network.py

@@ -1122,7 +1122,7 @@ class Network:
         end_time = time.time() + timeout
 
         # If no TxID is specified, retrieve latest readable one
-        if tx_id == None:
+        if tx_id is None:
             while time.time() < end_time:
                 with primary.client() as c:
                     resp = c.get("/node/state")  # Well-known read-only endpoint

tests/infra/piccolo_driver.py

@@ -17,10 +17,10 @@ import hashlib
 import json
 
 sys.path.insert(0, "../tests/perf-system/generator")
-import generator
+import generator  # noqa: E402
 
 sys.path.insert(0, "../tests/perf-system/analyzer")
-import analyzer
+import analyzer  # noqa: E402
 
 
 def get_command_args(args, network, get_command):

tests/infra/remote_client.py

@@ -87,13 +87,13 @@ class CCFRemoteClient(object):
             remote_file_dst = f"{self.name}_{csv}"
             self.remote.get(csv, self.common_dir, 1, remote_file_dst)
             if csv == "perf_summary.csv":
-                with open("perf_summary.csv", "a", encoding="utf-8") as l:
+                with open("perf_summary.csv", "a", encoding="utf-8") as csvfd:
                     with open(
                         os.path.join(self.common_dir, remote_file_dst),
                         "r",
                         encoding="utf-8",
                     ) as r:
-                        l.write(r.read())
+                        csvfd.write(r.read())
 
     def check_done(self):
         return self.remote.check_done()

tests/js-modules/modules.py

@@ -617,7 +617,7 @@ def test_npm_app(network, args):
             },
         )
         assert r.status_code == http.HTTPStatus.OK, r.status_code
-        assert r.body.json() == True, r.body
+        assert r.body.json() is True, r.body
 
         try:
             infra.crypto.verify_signature(
@@ -656,7 +656,7 @@ def test_npm_app(network, args):
                 },
             )
             assert r.status_code == http.HTTPStatus.OK, r.status_code
-            assert r.body.json() == True, r.body
+            assert r.body.json() is True, r.body
 
             try:
                 infra.crypto.verify_signature(
@@ -693,7 +693,7 @@ def test_npm_app(network, args):
             },
         )
         assert r.status_code == http.HTTPStatus.OK, r.status_code
-        assert r.body.json() == True, r.body
+        assert r.body.json() is True, r.body
 
         try:
             infra.crypto.verify_signature(
@@ -717,7 +717,7 @@ def test_npm_app(network, args):
             },
         )
         assert r.status_code == http.HTTPStatus.OK, r.status_code
-        assert r.body.json() == True, r.body
+        assert r.body.json() is True, r.body
 
         r = c.post(
             "/app/verifySignature",
@@ -729,7 +729,7 @@ def test_npm_app(network, args):
             },
         )
         assert r.status_code == http.HTTPStatus.OK, r.status_code
-        assert r.body.json() == False, r.body
+        assert r.body.json() is False, r.body
 
         curves = [ec.SECP256R1, ec.SECP256K1, ec.SECP384R1]
         for curve in curves:
@@ -747,7 +747,7 @@ def test_npm_app(network, args):
                 },
             )
             assert r.status_code == http.HTTPStatus.OK, r.status_code
-            assert r.body.json() == True, r.body
+            assert r.body.json() is True, r.body
 
         key_priv_pem, key_pub_pem = infra.crypto.generate_eddsa_keypair()
         algorithm = {"name": "EdDSA"}
@@ -763,7 +763,7 @@ def test_npm_app(network, args):
             },
         )
         assert r.status_code == http.HTTPStatus.OK, r.status_code
-        assert r.body.json() == True, r.body
+        assert r.body.json() is True, r.body
 
         r = c.post(
             "/app/digest",

tests/lts_compatibility.py

@@ -524,6 +524,14 @@ def run_ledger_compatibility_since_first(args, local_branch, use_snapshot):
                     network = infra.network.Network(
                         **network_args, existing_network=network
                     )
+                    primary, _ = network.find_primary()
+                    ledger_dir, committed_ledger_dirs = primary.get_ledger()
+                    snapshots_dir = (
+                        network.get_committed_snapshots(primary)
+                        if use_snapshot
+                        else None
+                    )
+
                     network.start_in_recovery(
                         args,
                         ledger_dir,

tests/raft_scenarios_runner.py

@@ -42,10 +42,10 @@ def separate_log_lines(text):
         if line.startswith("<RaftDriver>"):
             mermaid.append(line[len("<RaftDriver>") :])
         elif '"raft_trace"' in line:
-            l = json.loads(line)
-            if "msg" in l:
-                if "configurations" in l["msg"]:
-                    for config in l["msg"]["configurations"]:
+            line_ = json.loads(line)
+            if "msg" in line_:
+                if "configurations" in line_["msg"]:
+                    for config in line_["msg"]["configurations"]:
                         nodes.update(config["nodes"].keys())
             log.append(line)
     return (

tests/reconfiguration.py

@@ -160,13 +160,13 @@ def test_ignore_first_sigterm(network, args):
 
     with new_node.client() as c:
         r = c.get("/node/state")
-        assert r.body.json()["stop_notice"] == False, r
+        assert r.body.json()["stop_notice"] is False, r
 
     new_node.sigterm()
 
     with new_node.client() as c:
         r = c.get("/node/state")
-        assert r.body.json()["stop_notice"] == True, r
+        assert r.body.json()["stop_notice"] is True, r
 
     primary, _ = network.find_primary()
     network.retire_node(primary, new_node)

tests/recovery.py

@@ -390,8 +390,10 @@ def test_persistence_old_snapshot(network, args):
     # ledger directory (note: they used to be marked as ".ignored" by the new node)
     current_ledger_dir, committed_ledger_dirs = old_primary.get_ledger()
     for committed_ledger_dir in committed_ledger_dirs:
-        for l in os.listdir(committed_ledger_dir):
-            shutil.copy(os.path.join(committed_ledger_dir, l), current_ledger_dir)
+        for ledger_file_path in os.listdir(committed_ledger_dir):
+            shutil.copy(
+                os.path.join(committed_ledger_dir, ledger_file_path), current_ledger_dir
+            )
 
     # Capture latest committed TxID on primary so we can check later that the
     # entire ledger has been fully recovered