optimize asyn_dupe_delete grading

Author: valentijnscholten

dojo/models.py

@@ -2254,15 +2254,16 @@ def hash_code_allows_null_cwe(self):
         deduplicationLogger.debug(f"HASHCODE_ALLOWS_NULL_CWE is: {hashCodeAllowsNullCwe}")
         return hashCodeAllowsNullCwe
 
-    def delete(self, *args, **kwargs):
+    def delete(self, *args, product_grading_option=True, **kwargs):
         logger.debug("%d test delete", self.id)
         super().delete(*args, **kwargs)
-        with suppress(Test.DoesNotExist, Engagement.DoesNotExist, Product.DoesNotExist):
-            # Suppressing a potential issue created from async delete removing
-            # related objects in a separate task
-            system_settings = System_Settings.objects.get()
-            if system_settings.enable_product_grade:
-                calculate_grade(self.engagement.product)
+        if product_grading_option:
+            with suppress(Test.DoesNotExist, Engagement.DoesNotExist, Product.DoesNotExist):
+                # Suppressing a potential issue created from async delete removing
+                # related objects in a separate task
+                system_settings = System_Settings.objects.get()
+                if system_settings.enable_product_grade:
+                    calculate_grade(self.engagement.product)
 
     @property
     def statistics(self):
@@ -2863,17 +2864,18 @@ def copy(self, test=None):
 
         return copy
 
-    def delete(self, *args, **kwargs):
+    def delete(self, *args, product_grading_option=True, **kwargs):
         logger.debug("%d finding delete", self.id)
         from dojo.finding import helper
         helper.finding_delete(self)
         super().delete(*args, **kwargs)
-        with suppress(Finding.DoesNotExist, Test.DoesNotExist, Engagement.DoesNotExist, Product.DoesNotExist):
-            # Suppressing a potential issue created from async delete removing
-            # related objects in a separate task
-            system_settings = System_Settings.objects.get()
-            if system_settings.enable_product_grade:
-                calculate_grade(self.test.engagement.product)
+        if product_grading_option:
+            with suppress(Finding.DoesNotExist, Test.DoesNotExist, Engagement.DoesNotExist, Product.DoesNotExist):
+                # Suppressing a potential issue created from async delete removing
+                # related objects in a separate task
+                system_settings = System_Settings.objects.get()
+                if system_settings.enable_product_grade:
+                    calculate_grade(self.test.engagement.product)
 
     # only used by bulk risk acceptance api
     @classmethod

dojo/tasks.py

@@ -129,7 +129,7 @@ def async_dupe_delete(*args, **kwargs):
         logger.info("delete excess duplicates (max_dupes per finding: %s, max deletes per run: %s)", dupe_max, total_duplicate_delete_count_max_per_run)
         deduplicationLogger.info("delete excess duplicates (max_dupes per finding: %s, max deletes per run: %s)", dupe_max, total_duplicate_delete_count_max_per_run)
 
-        # limit to 100 to prevent overlapping jobs
+        # limit to settings.DUPE_DELETE_MAX_PER_RUN to prevent overlapping jobs
         results = Finding.objects \
                 .filter(duplicate=True) \
                 .order_by() \
@@ -146,13 +146,17 @@ def async_dupe_delete(*args, **kwargs):
             queryset=Finding.objects.filter(duplicate=True).order_by("date")))
 
         total_deleted_count = 0
+        affected_products = set()
         for original in originals_with_too_many_duplicates:
             duplicate_list = original.original_finding.all()
             dupe_count = len(duplicate_list) - dupe_max
 
             for finding in duplicate_list:
                 deduplicationLogger.debug(f"deleting finding {finding.id}:{finding.title} ({finding.hash_code}))")
-                finding.delete()
+                # Collect the product for batch grading later
+                affected_products.add(finding.test.engagement.product)
+                # Skip individual product grading during deletion
+                finding.delete(product_grading_option=False)
                 total_deleted_count += 1
                 dupe_count -= 1
                 if dupe_count <= 0:
@@ -165,6 +169,14 @@ def async_dupe_delete(*args, **kwargs):
 
         logger.info("total number of excess duplicates deleted: %s", total_deleted_count)
 
+        # Batch product grading for all affected products
+        if affected_products:
+            system_settings = System_Settings.objects.get()
+            if system_settings.enable_product_grade:
+                logger.info("performing batch product grading for %s products", len(affected_products))
+                for product in affected_products:
+                    calculate_grade(product)
+
 
 @app.task(ignore_result=False)
 def celery_status():