perf1: test cases: fix caching of system settings (#12861)

* test cases: fix caching of system settings

* fix tests

* fix caching for github

* fix caching for github

* simplify cache loading

* fix test cases

* set tags on (re)import

Author: valentijnscholten

dojo/middleware.py

@@ -83,9 +83,11 @@ def __init__(self, get_response):
 
     def __call__(self, request):
         self.load()
-        response = self.get_response(request)
-        self.cleanup()
-        return response
+        try:
+            return self.get_response(request)
+        finally:
+            # ensure cleanup happens even if an exception occurs
+            self.cleanup()
 
     def process_exception(self, request, exception):
         self.cleanup()
@@ -94,7 +96,6 @@ def process_exception(self, request, exception):
     def get_system_settings(cls):
         if hasattr(cls._thread_local, "system_settings"):
             return cls._thread_local.system_settings
-
         return None
 
     @classmethod
@@ -104,6 +105,8 @@ def cleanup(cls, *args, **kwargs):  # noqa: ARG003
 
     @classmethod
     def load(cls):
+        # cleanup any existing settings first to ensure fresh state
+        cls.cleanup()
         from dojo.models import System_Settings
         system_settings = System_Settings.objects.get(no_cache=True)
         cls._thread_local.system_settings = system_settings

run-unittest.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 unset TEST_CASE
+EXTRA_ARGS=()
 
 bash ./docker/docker-compose-check.sh
 if [[ $? -eq 1 ]]; then exit 1; fi
@@ -13,9 +14,11 @@ usage() {
   echo "  --help -h - prints this dialogue."
   echo
   echo "You must specify a test case (arg)!"
+  echo "Any additional arguments will be passed to the test command."
   echo
-  echo "Example command:"
+  echo "Example commands:"
   echo "./run-unittest.sh --test-case unittests.tools.test_stackhawk_parser.TestStackHawkParser"
+  echo "./run-unittest.sh --test-case unittests.tools.test_stackhawk_parser.TestStackHawkParser -v3 --failfast"
 }
 
 while [[ $# -gt 0 ]]; do
@@ -29,19 +32,13 @@ while [[ $# -gt 0 ]]; do
       usage
       exit 0
       ;;
-    -*)
-      echo "Unknown option $1"
-      usage
-      exit 1
-      ;;
     *)
-      POSITIONAL_ARGS+=("$1") # save positional arg
+      EXTRA_ARGS+=("$1") # save extra arg
       shift # past argument
       ;;
   esac
 done
 
-
 if [ -z "$TEST_CASE" ]
 then
   echo "No test case supplied."
@@ -50,7 +47,8 @@ then
 fi
 
 echo "Running docker compose unit tests with test case $TEST_CASE ..."
-# Compose V2 integrates compose functions into the Docker platform, continuing to support
-# most of the  previous docker-compose features and flags. You can run Compose V2 by
-# replacing the hyphen (-) with a space, using docker compose, instead of docker-compose.
-docker compose exec uwsgi bash -c "python manage.py test $TEST_CASE -v2 --keepdb"
+if [ ${#EXTRA_ARGS[@]} -gt 0 ]; then
+  echo "Additional arguments: ${EXTRA_ARGS[*]}"
+fi
+
+docker compose exec uwsgi bash -c "python manage.py test $TEST_CASE -v2 ${EXTRA_ARGS[*]} --keepdb"

unittests/dojo_test_case.py

@@ -16,6 +16,7 @@
 
 from dojo.jira_link import helper as jira_helper
 from dojo.jira_link.views import get_custom_field
+from dojo.middleware import DojoSytemSettingsMiddleware
 from dojo.models import (
     SEVERITIES,
     DojoMeta,
@@ -54,12 +55,15 @@ def decorator(test_func):
         def wrapper(*args, **kwargs):
             # Set the flag to the specified value
             System_Settings.objects.update(**{flag_name: value})
+            # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests
+            DojoSytemSettingsMiddleware.load()
             try:
                 return test_func(*args, **kwargs)
             finally:
                 # Reset the flag to its original state after the test
                 System_Settings.objects.update(**{flag_name: not value})
-
+                # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests
+                DojoSytemSettingsMiddleware.load()
         return wrapper
 
     return decorator
@@ -74,11 +78,15 @@ def wrapper(*args, **kwargs):
             old_value = getattr(System_Settings.objects.get(), field)
             # Set the flag to the specified value
             System_Settings.objects.update(**{field: value})
+            # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests
+            DojoSytemSettingsMiddleware.load()
             try:
                 return test_func(*args, **kwargs)
             finally:
                 # Reset the flag to its original state after the test
                 System_Settings.objects.update(**{field: old_value})
+                # Reinitialize middleware with updated settings as this doesn't happen automatically during django tests
+                DojoSytemSettingsMiddleware.load()
 
         return wrapper
 
@@ -90,28 +98,13 @@ class DojoTestUtilsMixin:
     def get_test_admin(self, *args, **kwargs):
         return User.objects.get(username="admin")
 
-    def system_settings(
-        self,
-        *,
-        enable_jira=False,
-        enable_jira_web_hook=False,
-        disable_jira_webhook_secret=False,
-        jira_webhook_secret=None,
-        enable_github=False,
-        enable_product_tag_inehritance=False,
-        enable_product_grade=True,
-        enable_webhooks_notifications=True,
-    ):
+    def system_settings(self, **kwargs):
         ss = System_Settings.objects.get()
-        ss.enable_jira = enable_jira
-        ss.enable_jira_web_hook = enable_jira_web_hook
-        ss.disable_jira_webhook_secret = disable_jira_webhook_secret
-        ss.jira_webhook_secret = jira_webhook_secret
-        ss.enable_github = enable_github
-        ss.enable_product_tag_inheritance = enable_product_tag_inehritance
-        ss.enable_product_grade = enable_product_grade
-        ss.enable_webhooks_notifications = enable_webhooks_notifications
+        for key, value in kwargs.items():
+            setattr(ss, key, value)
         ss.save()
+        # Refresh the cache with the new settings
+        DojoSytemSettingsMiddleware.load()
 
     def create_product_type(self, name, *args, description="dummy description", **kwargs):
         product_type = Product_Type(name=name, description=description)
@@ -489,6 +482,11 @@ class DojoTestCase(TestCase, DojoTestUtilsMixin):
     def __init__(self, *args, **kwargs):
         TestCase.__init__(self, *args, **kwargs)
 
+    def setUp(self):
+        super().setUp()
+        # Initialize middleware with fresh settings from db
+        DojoSytemSettingsMiddleware.load()
+
     def common_check_finding(self, finding):
         self.assertIn(finding.severity, SEVERITIES)
         finding.clean()

unittests/test_importers_performance.py

@@ -38,9 +38,22 @@ class TestDojoImporterPerformance(DojoTestCase):
 
     def setUp(self):
         super().setUp()
+
         self.system_settings(enable_webhooks_notifications=False)
         self.system_settings(enable_product_grade=False)
         self.system_settings(enable_github=False)
+        # from dojo.models import System_Settings
+
+        # # Configure system settings directly
+        # from dojo.middleware import DojoSytemSettingsMiddleware
+        # from dojo.models import System_Settings
+        # system_settings = System_Settings.objects.get()
+        # system_settings.enable_product_tag_inheritance = True
+        # system_settings.save()
+
+        # Initialize middleware with modified settings
+        # DojoSytemSettingsMiddleware.initialize_for_testing(System_Settings.objects.get())
+
         # Warm up ContentType cache for relevant models. This is needed if we want to be able to run the test in isolation
         # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count.
         # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries.
@@ -106,6 +119,8 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_
                 "sync": True,
                 "scan_type": STACK_HAWK_SCAN_TYPE,
                 "engagement": engagement,
+                "tags": ["performance-test", "tag-in-param", "go-faster"],
+                "apply_tags_to_findings": True,
             }
             importer = DefaultImporter(**import_options)
             test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(scan)
@@ -127,6 +142,8 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_
                 "verified": True,
                 "sync": True,
                 "scan_type": STACK_HAWK_SCAN_TYPE,
+                "tags": ["performance-test-reimport", "reimport-tag-in-param", "reimport-go-faster"],
+                "apply_tags_to_findings": True,
             }
             reimporter = DefaultReImporter(**reimport_options)
             test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan)
@@ -154,11 +171,11 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_
 
     def test_import_reimport_reimport_performance(self):
         self.import_reimport_performance(
-            expected_num_queries1=603,
+            expected_num_queries1=712,
             expected_num_async_tasks1=15,
-            expected_num_queries2=489,
+            expected_num_queries2=656,
             expected_num_async_tasks2=23,
-            expected_num_queries3=347,
+            expected_num_queries3=332,
             expected_num_async_tasks3=20,
         )
 
@@ -172,12 +189,12 @@ def test_import_reimport_reimport_performance_no_async(self, mock):
         so we patch the we_want_async decorator to always return False.
         """
         self.import_reimport_performance(
-            expected_num_queries1=673,
-            expected_num_async_tasks1=25,
-            expected_num_queries2=544,
-            expected_num_async_tasks2=30,
-            expected_num_queries3=387,
-            expected_num_async_tasks3=25,
+            expected_num_queries1=712,
+            expected_num_async_tasks1=15,
+            expected_num_queries2=656,
+            expected_num_async_tasks2=23,
+            expected_num_queries3=332,
+            expected_num_async_tasks3=20,
         )
 
     @patch("dojo.decorators.we_want_async", return_value=False)
@@ -190,11 +207,12 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self
         so we patch the we_want_async decorator to always return False.
         """
         self.system_settings(enable_product_grade=True)
+
         self.import_reimport_performance(
-            expected_num_queries1=673,
+            expected_num_queries1=752,
             expected_num_async_tasks1=25,
-            expected_num_queries2=544,
+            expected_num_queries2=690,
             expected_num_async_tasks2=30,
-            expected_num_queries3=387,
+            expected_num_queries3=357,
             expected_num_async_tasks3=25,
         )

unittests/test_tags.py

@@ -292,7 +292,7 @@ class InheritedTagsTests(DojoAPITestCase):
     def setUp(self, *args, **kwargs):
         super().setUp()
         self.login_as_admin()
-        self.system_settings(enable_product_tag_inehritance=True)
+        self.system_settings(enable_product_tag_inheritance=True)
         self.product = self.create_product("Inherited Tags Test", tags=["inherit", "these", "tags"])
         self.scans_path = get_unit_tests_scans_path("zap")
         self.zap_sample5_filename = self.scans_path / "5_zap_sample_one.xml"