Python Ledger docs and tutorial (#1435)

cmake/common.cmake

@@ -376,33 +376,47 @@ function(add_e2e_test)
 
     # Make python test client framework importable
     set_property(
-      TEST ${PARSED_ARGS_NAME} APPEND
+      TEST ${PARSED_ARGS_NAME}
+      APPEND
       PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
     )
 
     if(SHUFFLE_SUITE)
       set_property(
-        TEST ${PARSED_ARGS_NAME} APPEND PROPERTY ENVIRONMENT "SHUFFLE_SUITE=1"
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
+        PROPERTY ENVIRONMENT "SHUFFLE_SUITE=1"
       )
     endif()
 
-    set_property(TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS e2e)
     set_property(
-      TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS ${PARSED_ARGS_LABEL}
+      TEST ${PARSED_ARGS_NAME}
+      APPEND
+      PROPERTY LABELS e2e
+    )
+    set_property(
+      TEST ${PARSED_ARGS_NAME}
+      APPEND
+      PROPERTY LABELS ${PARSED_ARGS_LABEL}
     )
 
     if(${PARSED_ARGS_CURL_CLIENT})
       set_property(
-        TEST ${PARSED_ARGS_NAME} APPEND PROPERTY ENVIRONMENT "CURL_CLIENT=ON"
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
+        PROPERTY ENVIRONMENT "CURL_CLIENT=ON"
       )
     endif()
     set_property(
-      TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
+      TEST ${PARSED_ARGS_NAME}
+      APPEND
+      PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
     )
 
     if(DEFINED DEFAULT_ENCLAVE_TYPE)
       set_property(
-        TEST ${PARSED_ARGS_NAME} APPEND
+        TEST ${PARSED_ARGS_NAME}
+        APPEND
         PROPERTY ENVIRONMENT "DEFAULT_ENCLAVE_TYPE=${DEFAULT_ENCLAVE_TYPE}"
       )
     endif()
@@ -451,20 +465,28 @@ function(add_perf_test)
 
   # Make python test client framework importable
   set_property(
-    TEST ${PARSED_ARGS_NAME} APPEND
+    TEST ${PARSED_ARGS_NAME}
+    APPEND
     PROPERTY
       ENVIRONMENT
       "PYTHONPATH=${CCF_DIR}/tests:${CMAKE_CURRENT_BINARY_DIR}:$ENV{PYTHONPATH}"
   )
   if(DEFINED DEFAULT_ENCLAVE_TYPE)
     set_property(
-      TEST ${PARSED_ARGS_NAME} APPEND
+      TEST ${PARSED_ARGS_NAME}
+      APPEND
       PROPERTY ENVIRONMENT "DEFAULT_ENCLAVE_TYPE=${DEFAULT_ENCLAVE_TYPE}"
     )
   endif()
-  set_property(TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS perf)
   set_property(
-    TEST ${PARSED_ARGS_NAME} APPEND PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
+    TEST ${PARSED_ARGS_NAME}
+    APPEND
+    PROPERTY LABELS perf
+  )
+  set_property(
+    TEST ${PARSED_ARGS_NAME}
+    APPEND
+    PROPERTY LABELS ${PARSED_ARGS_CONSENSUS}
   )
 endfunction()
 

cmake/version.cmake

@@ -16,7 +16,8 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git)
   execute_process(
     COMMAND "bash" "-c"
             "${GIT_EXECUTABLE} describe --tags --abbrev=0 | tr -d ccf-"
-    OUTPUT_VARIABLE "CCF_RELEASE_VERSION" OUTPUT_STRIP_TRAILING_WHITESPACE
+    OUTPUT_VARIABLE "CCF_RELEASE_VERSION"
+    OUTPUT_STRIP_TRAILING_WHITESPACE
   )
 endif()
 
@@ -35,7 +36,8 @@ if(NOT CCF_RELEASE_VERSION)
 
   execute_process(
     COMMAND "bash" "-c" "basename ${CMAKE_CURRENT_SOURCE_DIR} | cut -d'-' -f2"
-    OUTPUT_VARIABLE "CCF_VERSION" OUTPUT_STRIP_TRAILING_WHITESPACE
+    OUTPUT_VARIABLE "CCF_VERSION"
+    OUTPUT_STRIP_TRAILING_WHITESPACE
   )
 
   set(CCF_RELEASE_VERSION ${CCF_VERSION})

doc/developers/ledger.rst

@@ -71,7 +71,7 @@ The following diagram describes how deltas committed by the leader are written t
 Reading and Verifing Ledger
 ---------------------------
 
-A Python implementation for parsing the ledger can be found in `ledger.py <https://github.com/microsoft/CCF/blob/master/tests/infra/ledger.py>`_.
+A Python implementation for parsing the ledger can be found in `ledger.py <https://github.com/microsoft/CCF/blob/master/python/ccf/ledger.py>`_.
 
 The ``Ledger`` class is constructed using the path of the ledger. It then exposes an iterator for transaction data structures, where each transaction is composed of the following:
 

doc/operators/index.rst

@@ -13,4 +13,5 @@ This section describes how :term:`Operators` manage the different nodes constitu
     operator_rpc_api
     resource_usage
     ledger
+    ledger_python
     container

doc/operators/ledger_python.rst

@@ -0,0 +1,59 @@
+Parsing the Ledger with Python
+==============================
+
+This page describes the Python API of the :py:class:`ccf.ledger` module which can be used by auditors to parse a CCF ledger.
+
+.. note:: To install the ``ccf`` python package, run:
+
+    .. code-block:: bash
+
+        $ pip install ccf
+
+Tutorial
+--------
+
+This tutorial demonstrates how to parse the ledger produced by a CCF node. It shows a very basic example which loops through all transactions in the ledger and counts how many times all keys in a target key-value store table are updated.
+
+First, the path to the ledger directory should be set:
+
+.. code-block:: python
+
+    ledger_dir = "</path/to/ledger/dir>" # Path to ledger directory
+
+.. note:: By default, the ledger directory is created under the node directory.
+
+Then, import the ledger module:
+
+.. literalinclude:: ../../python/tutorial.py
+    :language: py
+    :start-after: SNIPPET: import_ledger
+    :lines: 1
+
+In this particular example, a target table is set. This is a public table that can be read and audited from the ledger directly. In this example, the target table is the well-known ``ccf.nodes`` table that keeps track of all nodes in the network.
+
+.. literalinclude:: ../../python/tutorial.py
+    :language: py
+    :start-after: SNIPPET: target_table
+    :lines: 1
+
+.. note:: In practice, it is likely that auditors will want to run more complicated logic when parsing the ledger. For example, this might involve verifying signatures transactions or auditing governance operations and looping over multiple tables.
+
+Finally, the ledger can be iterated over. For each transaction in the ledger, the public tables changed within that transaction are observed. If the target table is included, we loop through all keys and values modified in that transaction.
+
+.. literalinclude:: ../../python/tutorial.py
+    :language: py
+    :start-after: SNIPPET_START: iterate_over_ledger
+    :end-before: SNIPPET_END: iterate_over_ledger
+
+API
+---
+
+.. autoclass:: ccf.ledger.Ledger
+    :members:
+
+.. autoclass:: ccf.ledger.Transaction
+    :members:
+
+.. autoclass:: ccf.ledger.PublicDomain
+    :members:
+

python/README.md

@@ -1,3 +1,3 @@
 # CCF Python
 
-Suite of Python tools for the Confidential Consortium Framework (CCF). For more information, visit https://github.com/Microsoft/python/ccf.
+Suite of Python tools for the Confidential Consortium Framework (CCF). For more information, please visit https://github.com/microsoft/CCF/tree/master/python.

python/ccf/ledger.py

@@ -40,7 +40,11 @@ class GcmHeader:
         return GCM_SIZE_TAG + GCM_SIZE_IV
 
 
-class LedgerDomain:
+class PublicDomain:
+    """
+    All public tables within a :py:class:`ccf.ledger.Transaction`.
+    """
+
     def __init__(self, buffer):
         self._buffer = buffer
         self._buffer_size = buffer.getbuffer().nbytes
@@ -109,6 +113,11 @@ class LedgerDomain:
             )
 
     def get_tables(self):
+        """
+        Returns a dictionnary of all public tables (with their content) in a :py:class:`ccf.ledger.Transaction`.
+
+        :return: Dictionnary of public tables with their content.
+        """
         return self._tables
 
 
@@ -123,6 +132,9 @@ def _byte_read_safe(file, num_of_bytes):
 
 
 class Transaction:
+    """
+    A transaction represents one entry in the CCF ledger.
+    """
 
     _file = None
     _total_size = 0
@@ -159,12 +171,22 @@ class Transaction:
         self._public_domain_size = to_uint_64(buffer)
 
     def get_public_domain(self):
+        """
+        Retrieve the public (i.e. non-encrypted) domain for that transaction.
+
+        :return: :py:class:`ccf.ledger.PublicDomain`
+        """
         if self._public_domain == None:
             buffer = io.BytesIO(_byte_read_safe(self._file, self._public_domain_size))
-            self._public_domain = LedgerDomain(buffer)
+            self._public_domain = PublicDomain(buffer)
         return self._public_domain
 
     def get_public_tx(self) -> bytes:
+        """
+        Returns raw transaction bytes.
+
+        :return: Raw transaction bytes.
+        """
         # remember where the pointer is in the file before we go back for the transaction bytes
         header = self._file.tell()
         self._file.seek(self._tx_offset)
@@ -192,32 +214,32 @@ class Transaction:
 
 
 class Ledger:
+    """
+    Class used to parse and iterate over all :py:class:`ccf.ledger.Transaction` stored in a CCF ledger.
 
-    _filenames = []
-    _fileindex = 0
+    :param str name: Ledger directory for a single CCF node.
+    """
 
     def __init__(self, name: str):
 
         self._filenames = []
         self._fileindex = 0
 
-        contents = os.listdir(name)
-        # Sorts the list based off the first number after ledger_ so that the ledger is verified in sequence
-        sort = sorted(
-            contents,
+        ledgers = os.listdir(name)
+        # Sorts the list based off the first number after ledger_ so that
+        # the ledger is verified in sequence
+        sorted_ledgers = sorted(
+            ledgers,
             key=lambda x: int(
                 x.replace(".committed", "").replace("ledger_", "").split("-")[0]
             ),
         )
 
-        for chunk in sort:
-            # Add only the .committed ledgers to be verified
+        for chunk in sorted_ledgers:
             if os.path.isfile(os.path.join(name, chunk)):
-                if chunk.endswith(".committed"):
-                    self._filenames.append(os.path.join(name, chunk))
-                else:
+                if not chunk.endswith(".committed"):
                     LOG.warning(f"The file {chunk} has not been committed")
-                    self._filenames.append(os.path.join(name, chunk))
+                self._filenames.append(os.path.join(name, chunk))
 
         self._current_tx = Transaction(self._filenames[0])
 
@@ -234,6 +256,3 @@ class Ledger:
 
     def __iter__(self):
         return self
-
-    def get_ledger_files(self) -> list:
-        return self._filenames

python/tutorial.py

@@ -22,6 +22,7 @@ with open(sys.argv[1]) as client_info_file:
 
 host = client_info["host"]
 port = client_info["port"]
+ledger_dir = client_info["ledger"]
 common_dir = client_info["common_dir"]
 ca = os.path.join(common_dir, "networkcert.pem")
 cert = os.path.join(common_dir, "user0_cert.pem")
@@ -63,3 +64,25 @@ r = user_client.get("/app/log/public?id=0")
 assert r.status_code == http.HTTPStatus.OK
 assert r.body == {"msg": "Public message"}
 # SNIPPET_END: authenticated_get_requests
+
+# SNIPPET: import_ledger
+import ccf.ledger
+
+# SNIPPET: create_ledger
+ledger = ccf.ledger.Ledger(ledger_dir)
+
+# SNIPPET: target_table
+target_table = "ccf.nodes"
+
+# SNIPPET_START: iterate_over_ledger
+target_table_changes = 0  # Simple counter
+
+for transaction in ledger:
+    # Retrieve all public tables changed in transaction
+    public_tables = transaction.get_public_domain().get_tables()
+
+    # If target_table was changed, count the number of keys changed
+    if target_table in public_tables:
+        for key, value in public_tables[target_table].items():
+            target_table_changes += 1  # A key was changed
+# SNIPPET_END: iterate_over_ledger

tests/start_network.py

@@ -6,7 +6,6 @@ import time
 import sys
 import json
 import os
-
 from loguru import logger as LOG
 
 
@@ -14,6 +13,7 @@ def dump_network_info(path, network, node):
     network_info = {}
     network_info["host"] = node.pubhost
     network_info["port"] = node.rpc_port
+    network_info["ledger"] = node.remote.ledger_path()
     network_info["common_dir"] = network.common_dir
 
     with open(path, "w") as network_info_file: