sla_config: use mass update for recalculation (#12133)

* sla_config_updater: add logging

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* sla_config: use mass update on changes

* cleanup

Author: valentijnscholten

dojo/models.py

@@ -1053,7 +1053,7 @@ def save(self, *args, **kwargs):
                     super(Product, product).save()
                 # launch the async task to update all finding sla expiration dates
                 from dojo.sla_config.helpers import update_sla_expiration_dates_sla_config_async
-                update_sla_expiration_dates_sla_config_async(self, tuple(severities), products)
+                update_sla_expiration_dates_sla_config_async(self, products, tuple(severities))
 
     def clean(self):
         sla_days = [self.critical, self.high, self.medium, self.low]
@@ -1214,7 +1214,7 @@ def save(self, *args, **kwargs):
                     sla_config.async_updating = True
                     super(SLA_Configuration, sla_config).save()
                 # launch the async task to update all finding sla expiration dates
-                from dojo.product.helpers import update_sla_expiration_dates_product_async
+                from dojo.sla_config.helpers import update_sla_expiration_dates_product_async
                 update_sla_expiration_dates_product_async(self, sla_config)
 
     def get_absolute_url(self):
@@ -2679,6 +2679,7 @@ def __str__(self):
 
     def save(self, dedupe_option=True, rules_option=True, product_grading_option=True,  # noqa: FBT002
              issue_updater_option=True, push_to_jira=False, user=None, *args, **kwargs):  # noqa: FBT002 - this is bit hard to fix nice have this universally fixed
+        logger.debug("Start saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None")
 
         from dojo.finding import helper as finding_helper
 
@@ -3043,7 +3044,7 @@ def get_sla_start_date(self):
         return self.date
 
     def get_sla_period(self):
-        sla_configuration = SLA_Configuration.objects.filter(id=self.test.engagement.product.sla_configuration_id).first()
+        sla_configuration = self.test.engagement.product.sla_configuration
         sla_period = getattr(sla_configuration, self.severity.lower(), None)
         enforce_period = getattr(sla_configuration, str("enforce_" + self.severity.lower()), None)
         return sla_period, enforce_period
@@ -3140,6 +3141,7 @@ def has_finding_group(self):
         return self.finding_group is not None
 
     def save_no_options(self, *args, **kwargs):
+        logger.debug("save_no_options")
         return self.save(dedupe_option=False, rules_option=False, product_grading_option=False,
              issue_updater_option=False, push_to_jira=False, user=None, *args, **kwargs)
 

dojo/product/helpers.py

@@ -3,31 +3,11 @@
 
 from dojo.celery import app
 from dojo.decorators import dojo_async_task
-from dojo.models import Endpoint, Engagement, Finding, Product, SLA_Configuration, Test
+from dojo.models import Endpoint, Engagement, Finding, Product, Test
 
 logger = logging.getLogger(__name__)
 
 
-@dojo_async_task
-@app.task
-def update_sla_expiration_dates_product_async(product, sla_config, *args, **kwargs):
-    update_sla_expiration_dates_product_sync(product, sla_config)
-
-
-def update_sla_expiration_dates_product_sync(product, sla_config):
-    logger.info(f"Updating finding SLA expiration dates within product {product}")
-    # update each finding that is within the SLA configuration that was saved
-    for f in Finding.objects.filter(test__engagement__product=product):
-        f.save()
-    # reset the async updating flag to false for the sla config assigned to this product
-    if sla_config:
-        sla_config.async_updating = False
-        super(SLA_Configuration, sla_config).save()
-    # set the async updating flag to false for the sla config assigned to this product
-    product.async_updating = False
-    super(Product, product).save()
-
-
 @dojo_async_task
 @app.task
 def propagate_tags_on_product(product_id, *args, **kwargs):

dojo/sla_config/helpers.py

@@ -3,25 +3,50 @@
 from dojo.celery import app
 from dojo.decorators import dojo_async_task
 from dojo.models import Finding, Product, SLA_Configuration
+from dojo.utils import calculate_grade, mass_model_updater
 
 logger = logging.getLogger(__name__)
 
 
 @dojo_async_task
 @app.task
-def update_sla_expiration_dates_sla_config_async(sla_config, severities, products, *args, **kwargs):
-    update_sla_expiration_dates_sla_config_sync(sla_config, severities, products)
+def update_sla_expiration_dates_sla_config_async(sla_config, products, severities, *args, **kwargs):
+    update_sla_expiration_dates_sla_config_sync(sla_config, products, severities)
 
 
-def update_sla_expiration_dates_sla_config_sync(sla_config, severities, products):
+@dojo_async_task
+@app.task
+def update_sla_expiration_dates_product_async(product, sla_config, *args, **kwargs):
+    update_sla_expiration_dates_sla_config_sync(sla_config, [product])
+
+
+def update_sla_expiration_dates_sla_config_sync(sla_config, products, severities=None):
     logger.info(f"Updating finding SLA expiration dates within the {sla_config} SLA configuration")
     # update each finding that is within the SLA configuration that was saved
-    for f in Finding.objects.filter(test__engagement__product__sla_configuration_id=sla_config.id, severity__in=severities):
-        f.save()
+    findings = Finding.objects.filter(test__engagement__product__sla_configuration_id=sla_config.id)
+    if products:
+        findings = findings.filter(test__engagement__product__in=products)
+    if severities:
+        findings = findings.filter(severity__in=severities)
+
+    findings = findings.prefetch_related(
+            "test",
+            "test__engagement",
+            "test__engagement__product",
+            "test__engagement__product__sla_configuration",
+    )
+
+    findings = findings.order_by("id").only("id", "sla_start_date", "date", "severity", "test")
+
+    mass_model_updater(Finding, findings, lambda f: f.set_sla_expiration_date(), fields=["sla_expiration_date"])
+
     # reset the async updating flag to false for all products using this sla config
     for product in products:
         product.async_updating = False
         super(Product, product).save()
+        calculate_grade(product)
+
     # reset the async updating flag to false for this sla config
     sla_config.async_updating = False
     super(SLA_Configuration, sla_config).save()
+    logger.info(f"DONE Updating finding SLA expiration dates within the {sla_config} SLA configuration")

dojo/utils.py

@@ -2241,22 +2241,18 @@ def mass_model_updater(model_type, models, function, fields, page_size=1000, ord
     i = 0
     batch = []
     total_pages = (total_count // page_size) + 2
-    # logger.info('pages to process: %d', total_pages)
+    # logger.debug("pages to process: %d", total_pages)
     logger.debug("%s%s out of %s models processed ...", log_prefix, i, total_count)
     for _p in range(1, total_pages):
-        # logger.info('page: %d', p)
         if order == "asc":
             page = models.filter(id__gt=last_id)[:page_size]
         else:
             page = models.filter(id__lt=last_id)[:page_size]
 
-        # logger.info('page query: %s', page.query)
-        # if p == 23:
-        #     raise ValueError('bla')
+        logger.debug("page query: %s", page.query)
         for model in page:
             i += 1
             last_id = model.id
-            # logger.info('last_id: %s', last_id)
 
             function(model)