Split client and ledger tutorial (#2172)

CMakeLists.txt

@@ -782,8 +782,9 @@ if(BUILD_TESTS)
     CONSENSUS cft
     LABEL tutorial
     CURL_CLIENT TRUE
-    ADDITIONAL_ARGS -p liblogging --tutorial
-                    ${CMAKE_SOURCE_DIR}/python/tutorial.py
+    ADDITIONAL_ARGS
+      -p liblogging --client-tutorial ${CMAKE_SOURCE_DIR}/python/tutorial.py
+      --ledger-tutorial ${CMAKE_SOURCE_DIR}/python/ledger_tutorial.py
   )
 
   add_e2e_sandbox_test(

doc/audit/python_library.rst

@@ -22,14 +22,14 @@ First, the path to the ledger directory should be set:
 
 Then, import the ledger module:
 
-.. literalinclude:: ../../python/tutorial.py
+.. literalinclude:: ../../python/ledger_tutorial.py
     :language: py
     :start-after: SNIPPET: import_ledger
     :lines: 1
 
 In this particular example, a target table is set. This is a public table that can be read and audited from the ledger directly. In this example, the target table is the well-known ``public:ccf.gov.nodes`` table that keeps track of all nodes in the network.
 
-.. literalinclude:: ../../python/tutorial.py
+.. literalinclude:: ../../python/ledger_tutorial.py
     :language: py
     :start-after: SNIPPET: target_table
     :lines: 1
@@ -38,7 +38,7 @@ In this particular example, a target table is set. This is a public table that c
 
 Finally, the ledger can be iterated over. For each transaction in the ledger, the public tables changed within that transaction are observed. If the target table is included, we loop through all keys and values modified in that transaction.
 
-.. literalinclude:: ../../python/tutorial.py
+.. literalinclude:: ../../python/ledger_tutorial.py
     :language: py
     :start-after: SNIPPET_START: iterate_over_ledger
     :end-before: SNIPPET_END: iterate_over_ledger

python/ledger_tutorial.py

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache 2.0 License.
+
+import sys
+from loguru import logger as LOG
+
+# Note: It is safer to run the ledger tutorial when the service has stopped
+# as all ledger files will have been written to.
+
+# Change default log format
+LOG.remove()
+LOG.add(
+    sys.stdout,
+    format="<green>[{time:HH:mm:ss.SSS}]</green> {message}",
+)
+
+if len(sys.argv) < 2:
+    print("Error: Ledger directory should be specified as first argument")
+    sys.exit(1)
+
+ledger_dir = sys.argv[1]
+
+# SNIPPET: import_ledger
+import ccf.ledger
+
+# SNIPPET: create_ledger
+ledger = ccf.ledger.Ledger(ledger_dir)
+
+# SNIPPET: target_table
+target_table = "public:ccf.gov.nodes"
+
+# SNIPPET_START: iterate_over_ledger
+target_table_changes = 0  # Simple counter
+
+for chunk in ledger:
+    for transaction in chunk:
+        # Retrieve all public tables changed in transaction
+        public_tables = transaction.get_public_domain().get_tables()
+
+        # If target_table was changed, count the number of keys changed
+        if target_table in public_tables:
+            for key, value in public_tables[target_table].items():
+                target_table_changes += 1  # A key was changed
+# SNIPPET_END: iterate_over_ledger

python/tutorial.py

@@ -18,14 +18,11 @@ LOG.add(
 import ccf.clients
 
 # Load client info file.
-if len(sys.argv) < 3:
-    print(
-        "Error: Ledger directory and common directory should be specified as first and second arguments, respectively"
-    )
+if len(sys.argv) < 2:
+    print("Error: Common directory should be specified as first argument")
     sys.exit(1)
 
-ledger_dir = sys.argv[1]
-common_dir = sys.argv[2]
+common_dir = sys.argv[1]
 
 # Assumes sandbox started with at least one node
 host = "127.0.0.1"
@@ -94,29 +91,6 @@ r = member_client.post("/gov/ack/update_state_digest")
 assert r.status_code == http.HTTPStatus.OK
 # SNIPPET_END: signed_request
 
-# SNIPPET: import_ledger
-import ccf.ledger
-
-# SNIPPET: create_ledger
-ledger = ccf.ledger.Ledger(ledger_dir)
-
-# SNIPPET: target_table
-target_table = "public:ccf.gov.nodes"
-
-# SNIPPET_START: iterate_over_ledger
-target_table_changes = 0  # Simple counter
-
-for chunk in ledger:
-    for transaction in chunk:
-        # Retrieve all public tables changed in transaction
-        public_tables = transaction.get_public_domain().get_tables()
-
-        # If target_table was changed, count the number of keys changed
-        if target_table in public_tables:
-            for key, value in public_tables[target_table].items():
-                target_table_changes += 1  # A key was changed
-# SNIPPET_END: iterate_over_ledger
-
 # SNIPPET: import_proposal_generator
 import ccf.proposal_generator
 

tests/e2e_tutorial.py

@@ -14,20 +14,32 @@ def run(args):
 
         cmd = [
             "python",
-            args.tutorial,
-            primary.get_ledger()[1],
+            args.client_tutorial,
             network.common_dir,
         ]
         rc = infra.proc.ccall(*cmd).returncode
         assert rc == 0, f"Failed to run tutorial script: {rc}"
 
+    cmd = [
+        "python",
+        args.ledger_tutorial,
+        primary.get_ledger()[1],
+    ]
+    rc = infra.proc.ccall(*cmd).returncode
+    assert rc == 0, f"Failed to run tutorial script: {rc}"
+
 
 if __name__ == "__main__":
 
     def add(parser):
         parser.add_argument(
-            "--tutorial",
-            help="Path to tutorial file",
+            "--client-tutorial",
+            help="Path to client tutorial file",
+            type=str,
+        )
+        parser.add_argument(
+            "--ledger-tutorial",
+            help="Path to ledger tutorial file",
             type=str,
         )
 

tests/test_install.sh

@@ -49,7 +49,7 @@ python3.8 -m venv env
 # shellcheck source=/dev/null
 source env/bin/activate
 python -m pip install ../../../python
-python ../../../python/tutorial.py ./workspace/sandbox_0/0.ledger/ ./workspace/sandbox_common/
+python ../../../python/tutorial.py ./workspace/sandbox_common/
 
 # Test Python package CLI
 ../../../tests/test_python_cli.sh > test_python_cli.out
@@ -60,6 +60,9 @@ while [ "$(service_http_status)" == "200" ]; do
     sleep 1
 done
 
+# Now that the service has been stopped, run ledger tutorial
+python ../../../python/ledger_tutorial.py ./workspace/sandbox_0/0.ledger
+
 # Recover network
 cp -r ./workspace/sandbox_0/0.ledger .
 
@@ -69,4 +72,5 @@ timeout --signal=SIGINT --kill-after=${recovered_network_live_time}s --preserve-
     -e release \
     --recover \
     --ledger-dir 0.ledger \
-    --common-dir ./workspace/sandbox_common/
+    --common-dir ./workspace/sandbox_common/
+