[DRAFT] UnitTest based tests

CMakeLists.txt

@@ -307,16 +307,19 @@ set(
 )
 
 set(
-  CCF_NETWORK_TEST_DEFAULT_CONSTITUTION
-  --constitution
+  CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES
   ${CCF_DIR}/samples/constitutions/default/actions.js
-  --constitution
   ${CCF_DIR}/samples/constitutions/default/validate.js
-  --constitution
   ${CCF_DIR}/samples/constitutions/default/resolve.js
-  --constitution
   ${CCF_DIR}/samples/constitutions/default/apply.js
 )
+set(CCF_NETWORK_TEST_DEFAULT_CONSTITUTION "")
+foreach(CONSTITUTION_FILE IN LISTS CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES)
+  list(
+    APPEND CCF_NETWORK_TEST_DEFAULT_CONSTITUTION --constitution
+    ${CONSTITUTION_FILE}
+  )
+endforeach()
 set(
   CCF_NETWORK_TEST_ARGS
   --log-level
@@ -325,6 +328,27 @@ set(
   ${WORKER_THREADS}
 )
 
+# For fast e2e runs, tick node faster than default value (except for
+# instrumented builds which may process ticks slower).
+if(SAN)
+  set(NODE_TICK_MS 10)
+else()
+  set(NODE_TICK_MS 1)
+endif()
+
+list(
+  TRANSFORM CCF_NETWORK_TEST_DEFAULT_CONSTITUTION_FILES
+  REPLACE "(.+)" "\"\\1\""
+  OUTPUT_VARIABLE QUOTED_DEFAULT_CONSTITUTION_FILES
+)
+list(JOIN QUOTED_DEFAULT_CONSTITUTION_FILES ", " DEFAULT_CONSTITUTION_JSON)
+set(DEFAULT_CONSTITUTION_JSON "[${DEFAULT_CONSTITUTION_JSON}]")
+configure_file(
+  ${CCF_DIR}/cmake/test_config.json.in
+  ${CMAKE_BINARY_DIR}/test_config.json
+  @ONLY
+)
+
 # SNIPPET_START: JS generic application
 add_ccf_app(
   js_generic
@@ -1258,6 +1282,27 @@ if(BUILD_TESTS)
   )
   set_tests_properties(schema_test PROPERTIES TIMEOUT 900)
 
+  if(BUILD_END_TO_END_TESTS)
+    add_test(
+      NAME e2e_unittests
+      COMMAND ${PYTHON} -m unittest discover -s ${CMAKE_SOURCE_DIR}/tests -v
+    )
+    set_property(
+      TEST e2e_unittests
+      APPEND
+      PROPERTY ENVIRONMENT "PYTHONPATH=${CCF_DIR}/tests:$ENV{PYTHONPATH}"
+    )
+    set_property(
+      TEST e2e_unittests
+      APPEND
+      PROPERTY ENVIRONMENT "CCF_TEST_CONFIG=${CMAKE_BINARY_DIR}/test_config.json"
+    )
+    set_property(TEST e2e_unittests APPEND PROPERTY LABELS e2e)
+    set_property(TEST e2e_unittests APPEND PROPERTY LABELS e2e_unittest)
+    set_property(TEST e2e_unittests APPEND PROPERTY LABELS bucket_a)
+    add_san_test_properties(e2e_unittests)
+  endif()
+
   add_e2e_test(
     NAME snp_platform_tests
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/amd_snp.py

cmake/common.cmake

@@ -112,14 +112,6 @@ function(add_e2e_test)
   if(BUILD_END_TO_END_TESTS)
     set(PYTHON_WRAPPER ${PYTHON})
 
-    # For fast e2e runs, tick node faster than default value (except for
-    # instrumented builds which may process ticks slower).
-    if(SAN)
-      set(NODE_TICK_MS 10)
-    else()
-      set(NODE_TICK_MS 1)
-    endif()
-
     if(NOT PARSED_ARGS_PERF_LABEL)
       set(PARSED_ARGS_PERF_LABEL ${PARSED_ARGS_NAME})
     endif()

cmake/test_config.json.in

@@ -0,0 +1,9 @@
+{
+  "binary_dir": "@CMAKE_BINARY_DIR@",
+  "log_level": "@TEST_LOGGING_LEVEL@",
+  "worker_threads": @WORKER_THREADS@,
+  "tick_ms": @NODE_TICK_MS@,
+  "default_constitution": @DEFAULT_CONSTITUTION_JSON@,
+  "historical_testdata": "@CMAKE_SOURCE_DIR@/tests/testdata",
+  "jinja_templates_path": "@CMAKE_SOURCE_DIR@/samples/templates"
+}

python/src/ccf/ledger.py

@@ -399,7 +399,8 @@ def clone(self, at_loc: int = 0):
 
     @staticmethod
     def from_file(filename):
-        return SimpleBuffer(filename, open(filename, "rb").read())
+        with open(filename, "rb") as f:
+            return SimpleBuffer(filename, f.read())
 
 
 def _byte_read_safe(file: SimpleBuffer, num_of_bytes):

tests/ci-buckets.txt

@@ -15,6 +15,7 @@
 #   ./scripts/test-buckets-checks.sh -f
 
 bucket_a:
+  e2e_unittests
   lts_compatibility
 
 bucket_b:

tests/infra/e2e_args.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the Apache 2.0 License.
 import argparse
+import json
 import os
 import infra.interfaces
 import infra.path
@@ -46,11 +47,26 @@ def default_platform():
     return "virtual"
 
 
+def _load_test_config():
+    config_path = os.getenv("CCF_TEST_CONFIG")
+    if config_path:
+        with open(config_path, encoding="utf-8") as f:
+            return json.load(f)
+
+    config_path = os.path.join(os.getcwd(), "test_config.json")
+    if os.path.isfile(config_path):
+        with open(config_path, encoding="utf-8") as f:
+            return json.load(f)
+
+    return None
+
+
 def cli_args(
     add=lambda x: None,
     parser=None,
     accept_unknown=False,
     ledger_chunk_bytes_override=None,
+    argv=None,
 ):
     LOG.remove()
     LOG.add(
@@ -400,10 +416,20 @@ def cli_args(
     )
     add(parser)
 
+    test_config = _load_test_config()
+    if test_config is not None:
+        config_defaults = {
+            k: v for k, v in test_config.items() if k != "default_constitution"
+        }
+        parser.set_defaults(**config_defaults)
+
     if accept_unknown:
-        args, unknown_args = parser.parse_known_args()
+        args, unknown_args = parser.parse_known_args(argv)
     else:
-        args = parser.parse_args()
+        args = parser.parse_args(argv)
+
+    if test_config is not None and not args.constitution:
+        args.constitution = test_config.get("default_constitution", [])
 
     args.binary_dir = os.path.abspath(args.binary_dir)
 

tests/infra/network.py

@@ -6,6 +6,7 @@
 
 from contextlib import contextmanager
 from enum import Enum, IntEnum, auto
+import unittest
 from infra.clients import flush_info, CCFConnectionException, CCFIOException
 import infra.crypto
 import infra.member
@@ -2229,3 +2230,80 @@ def network(
     )
     if init_partitioner:
         net.partitioner.cleanup()
+
+
+class NetworkTestCase(unittest.TestCase):
+    label = None
+    package = "samples/apps/logging/logging"
+    test_config_overrides = lambda args: {}
+    network_kwargs = lambda args: {}
+    start_and_open_kwargs = {}
+    failure_stop_kwargs = {
+        "skip_verification": True,
+        "accept_ledger_diff": True,
+        "skip_verify_chunking": True,
+    }
+    success_stop_kwargs = {
+        "skip_verification": False,
+        "accept_ledger_diff": False,
+        "skip_verify_chunking": False,
+    }
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        if cls.label is None:
+            raise TypeError(f"{cls.__name__} must define a label")
+        if not isinstance(cls.label, str):
+            raise TypeError(f"{cls.__name__}.label must be a string")
+
+    @classmethod
+    def resolve_args_overrides(cls, args):
+        return cls.test_config_overrides(args)
+
+    @classmethod
+    def _cleanup(cls):
+        if cls._failing_test_id is None:
+            cls.network.stop_all_nodes(**cls.success_stop_kwargs)
+        else:
+            cls.network.txs = None
+            cls.network.log_stack_traces(timeout=10)
+            cls.network.stop_all_nodes(**cls.failure_stop_kwargs)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.args = infra.e2e_args.cli_args(argv=[])
+        cls.args.label = cls.label
+        cls.args.package = cls.package
+        if not os.path.isabs(cls.args.package):
+            cls.args.package = os.path.join(cls.args.binary_dir, cls.args.package)
+        for name, value in cls.resolve_args_overrides(cls.args).items():
+            setattr(cls.args, name, value)
+
+        cls.network = infra.network.Network(
+            cls.args.nodes,
+            cls.args.binary_dir,
+            cls.args.debug_nodes,
+            **cls.network_kwargs(cls.args),
+        )
+        with infra.network.close_on_error(cls.network, pdb=cls.args.pdb):
+            cls.network.start_and_open(cls.args, **cls.start_and_open_kwargs)
+        cls._failing_test_id = None
+        cls.addClassCleanup(cls._cleanup)
+
+    # Record traces on exceptions
+    def _callTestMethod(self, method):
+        if type(self)._failing_test_id is not None:
+            raise unittest.SkipTest(
+                f"Skipping test due to previous failure: {type(self)._failing_test_id}"
+            )
+        try:
+            return method()
+        except unittest.SkipTest:
+            raise
+        except Exception:
+            type(self)._failing_test_id = self.id()
+            if self.args.pdb:
+                import pdb
+
+                pdb.post_mortem()
+            raise

tests/test_e2e_operations.py

@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache 2.0 License.
+
+import infra.logging_app as app
+import infra
+from e2e_operations import (
+    test_backup_snapshot_fetch,
+    test_backup_snapshot_fetch_max_size,
+    test_error_message_on_failure_to_fetch_snapshot,
+    test_join_idempotency_short_circuits_on_backup,
+    test_join_time_snapshot_fetch_failure,
+)
+
+
+class BackupSnapshotDownload(infra.network.NetworkTestCase):
+    label = "backup_snapshot_download"
+    test_config_overrides = lambda args: {
+        "snapshot_tx_interval": 30,
+        "nodes": infra.e2e_args.max_nodes(args, f=0),
+    }
+    start_and_open_kwargs = {"backup_snapshot_fetch_enabled": True}
+    network_kwargs = lambda _: {"txs": app.LoggingTxs("user0")}
+    success_stop_kwargs = {
+        "skip_verification": True,
+    }
+
+    def test_backup_snapshot_fetch(self):
+        test_backup_snapshot_fetch(self.network, self.args)
+
+    def test_backup_snapshot_fetch_max_size(self):
+        test_backup_snapshot_fetch_max_size(self.network, self.args)
+
+    def test_join_idempotency_short_circuits_on_backup(self):
+        test_join_idempotency_short_circuits_on_backup(self.network, self.args)
+
+    def test_join_time_snapshot_fetch_failure(self):
+        test_join_time_snapshot_fetch_failure(self.network, self.args)
+
+    def test_error_message_on_failure_to_fetch_snapshot(self):
+        test_error_message_on_failure_to_fetch_snapshot(self.network, self.args)