perf2: skip post processing if not needed (#12862)

* test cases: fix caching of system settings

* fix tests

* fix caching for github

* fix caching for github

* simplify cache loading

* fix test cases

* product grade: skip in finding save if disabled

* post process only when needed

* cleanup

* set tags on (re)import

* rebase set tags

* reduce save with options

* update counts, reduce saves with options

Author: valentijnscholten

dojo/importers/default_importer.py

@@ -198,7 +198,8 @@ def process_findings(
 
             # Force parsers to use unsaved_tags (stored in below after saving)
             unsaved_finding.tags = None
-            unsaved_finding.save(dedupe_option=False)
+            # postprocessing will be done on next save.
+            unsaved_finding.save_no_options()
             finding = unsaved_finding
             # Determine how the finding should be grouped
             self.process_finding_groups(

dojo/importers/default_reimporter.py

@@ -480,7 +480,7 @@ def process_matched_mitigated_finding(
                 )
                 note.save()
                 existing_finding.notes.add(note)
-                existing_finding.save(dedupe_option=False)
+                existing_finding.save_no_options()
             # Return True here to force the loop to continue
             return existing_finding, True
         logger.debug(
@@ -498,9 +498,8 @@ def process_matched_mitigated_finding(
         component_version = getattr(unsaved_finding, "component_version", None)
         existing_finding.component_name = existing_finding.component_name or component_name
         existing_finding.component_version = existing_finding.component_version or component_version
-        existing_finding.save(dedupe_option=False)
-        # don't dedupe before endpoints are added
-        existing_finding.save(dedupe_option=False)
+        # don't dedupe before endpoints are added, postprocessing will be done on next save (in calling method)
+        existing_finding.save_no_options()
         note = Notes(entry=f"Re-activated by {self.scan_type} re-upload.", author=self.user)
         note.save()
         endpoint_statuses = existing_finding.status_finding.exclude(
@@ -572,7 +571,7 @@ def process_matched_active_finding(
         ):
             existing_finding.component_name = existing_finding.component_name or component_name
             existing_finding.component_version = existing_finding.component_version or component_version
-            existing_finding.save(dedupe_option=False)
+            existing_finding.save_no_options()
         # Return False here to make sure further processing happens
         return existing_finding, False
 
@@ -595,7 +594,7 @@ def process_finding_that_was_not_matched(
         if self.scan_date_override:
             unsaved_finding.date = self.scan_date.date()
         # Save it. Don't dedupe before endpoints are added.
-        unsaved_finding.save(dedupe_option=False)
+        unsaved_finding.save_no_options()
         finding = unsaved_finding
         # Force parsers to use unsaved_tags (stored in finding_post_processing function below)
         finding.tags = None

dojo/models.py

@@ -2773,7 +2773,8 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru
         self.found_by.add(self.test.test_type)
 
         # only perform post processing (in celery task) if needed. this check avoids submitting 1000s of tasks to celery that will do nothing
-        if dedupe_option or issue_updater_option or product_grading_option or push_to_jira:
+        system_settings = System_Settings.objects.get()
+        if dedupe_option or issue_updater_option or (product_grading_option and system_settings.enable_product_grade) or push_to_jira:
             finding_helper.post_process_finding_save(self, dedupe_option=dedupe_option, rules_option=rules_option, product_grading_option=product_grading_option,
                 issue_updater_option=issue_updater_option, push_to_jira=push_to_jira, user=user, *args, **kwargs)
         else:

unittests/test_importers_performance.py

@@ -42,17 +42,6 @@ def setUp(self):
         self.system_settings(enable_webhooks_notifications=False)
         self.system_settings(enable_product_grade=False)
         self.system_settings(enable_github=False)
-        # from dojo.models import System_Settings
-
-        # # Configure system settings directly
-        # from dojo.middleware import DojoSytemSettingsMiddleware
-        # from dojo.models import System_Settings
-        # system_settings = System_Settings.objects.get()
-        # system_settings.enable_product_tag_inheritance = True
-        # system_settings.save()
-
-        # Initialize middleware with modified settings
-        # DojoSytemSettingsMiddleware.initialize_for_testing(System_Settings.objects.get())
 
         # Warm up ContentType cache for relevant models. This is needed if we want to be able to run the test in isolation
         # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count.
@@ -172,9 +161,9 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_
     def test_import_reimport_reimport_performance(self):
         self.import_reimport_performance(
             expected_num_queries1=712,
-            expected_num_async_tasks1=15,
+            expected_num_async_tasks1=10,
             expected_num_queries2=656,
-            expected_num_async_tasks2=23,
+            expected_num_async_tasks2=22,
             expected_num_queries3=332,
             expected_num_async_tasks3=20,
         )
@@ -190,9 +179,9 @@ def test_import_reimport_reimport_performance_no_async(self, mock):
         """
         self.import_reimport_performance(
             expected_num_queries1=712,
-            expected_num_async_tasks1=15,
+            expected_num_async_tasks1=10,
             expected_num_queries2=656,
-            expected_num_async_tasks2=23,
+            expected_num_async_tasks2=22,
             expected_num_queries3=332,
             expected_num_async_tasks3=20,
         )
@@ -207,12 +196,15 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self
         so we patch the we_want_async decorator to always return False.
         """
         self.system_settings(enable_product_grade=True)
+        # Refresh the cache with the new settings
+        from dojo.middleware import DojoSytemSettingsMiddleware
+        DojoSytemSettingsMiddleware.load()
 
         self.import_reimport_performance(
-            expected_num_queries1=752,
-            expected_num_async_tasks1=25,
-            expected_num_queries2=690,
-            expected_num_async_tasks2=30,
+            expected_num_queries1=732,
+            expected_num_async_tasks1=15,
+            expected_num_queries2=686,
+            expected_num_async_tasks2=28,
             expected_num_queries3=357,
             expected_num_async_tasks3=25,
         )