perf(endpoint_manager): use prefetched status_finding_non_special to avoid extra DB queries

Add a named Prefetch to build_candidate_scope_queryset that fetches only
non-special endpoint statuses (excluding false_positive, out_of_scope,
risk_accepted) with their endpoint joined in via select_related. This
replaces the two separate "status_finding" and "status_finding__endpoint"
prefetches with a single query and avoids per-finding DB hits in
update_endpoint_status and process_matched_special_status_finding.

Update expected performance test counts to reflect the reduced query counts.

Author: valentijnscholten

dojo/finding/deduplication.py

@@ -8,7 +8,7 @@
 from django.db.models.query_utils import Q
 
 from dojo.celery import app
-from dojo.models import Finding, System_Settings
+from dojo.models import Endpoint_Status, Finding, System_Settings
 
 logger = logging.getLogger(__name__)
 deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication")
@@ -295,12 +295,19 @@ def build_candidate_scope_queryset(test, mode="deduplication", service=None):
     # Base prefetches for both modes
     prefetch_list = ["endpoints", "vulnerability_id_set", "found_by"]
 
-    # Additional prefetches for reimport mode
+    # Additional prefetches for reimport mode: fetch only non-special endpoint statuses with their
+    # endpoint joined in, so endpoint_manager can read status_finding_non_special directly without
+    # any extra DB queries
     if mode == "reimport":
-        prefetch_list.extend([
-            "status_finding",
-            "status_finding__endpoint",
-        ])
+        prefetch_list.append(
+            Prefetch(
+                "status_finding",
+                queryset=Endpoint_Status.objects.exclude(
+                    Q(false_positive=True) | Q(out_of_scope=True) | Q(risk_accepted=True),
+                ).select_related("endpoint"),
+                to_attr="status_finding_non_special",
+            ),
+        )
 
     return (
         queryset

dojo/importers/default_reimporter.py

@@ -766,12 +766,8 @@ def process_matched_mitigated_finding(
         else:
             # TODO: Delete this after the move to Locations
             # Reactivate mitigated endpoints that are not false positives, out of scope, or risk accepted
-            endpoint_statuses = existing_finding.status_finding.exclude(
-                Q(false_positive=True)
-                | Q(out_of_scope=True)
-                | Q(risk_accepted=True),
-            )
-            self.endpoint_manager.chunk_endpoints_and_reactivate(endpoint_statuses)
+            # status_finding_non_special is prefetched by build_candidate_scope_queryset
+            self.endpoint_manager.chunk_endpoints_and_reactivate(existing_finding.status_finding_non_special)
         existing_finding.notes.add(note)
         self.reactivated_items.append(existing_finding)
         # The new finding is active while the existing on is mitigated. The existing finding needs to

dojo/importers/endpoint_manager.py

@@ -1,7 +1,6 @@
 import logging
 
 from django.core.exceptions import MultipleObjectsReturned, ValidationError
-from django.db.models import Q
 from django.urls import reverse
 from django.utils import timezone
 
@@ -158,12 +157,9 @@ def update_endpoint_status(
         """Update the list of endpoints from the new finding with the list that is in the old finding"""
         # New endpoints are already added in serializers.py / views.py (see comment "# for existing findings: make sure endpoints are present or created")
         # So we only need to mitigate endpoints that are no longer present
-        # Evaluate the queryset once into a list to avoid double DB evaluation below
-        existing_finding_endpoint_status_list = list(
-            existing_finding.status_finding.exclude(
-                Q(false_positive=True) | Q(out_of_scope=True) | Q(risk_accepted=True),
-            ).select_related("endpoint"),
-        )
+        # status_finding_non_special is prefetched by build_candidate_scope_queryset with the
+        # special-status exclusion and endpoint select_related already applied at the DB level
+        existing_finding_endpoint_status_list = existing_finding.status_finding_non_special
         new_finding_endpoints_list = new_finding.unsaved_endpoints
         if new_finding.is_mitigated:
             # New finding is mitigated, so mitigate all old endpoints

unittests/test_importers_performance.py

@@ -270,9 +270,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self):
         self._import_reimport_performance(
             expected_num_queries1=295,
             expected_num_async_tasks1=6,
-            expected_num_queries2=232,
+            expected_num_queries2=226,
             expected_num_async_tasks2=17,
-            expected_num_queries3=114,
+            expected_num_queries3=108,
             expected_num_async_tasks3=16,
         )
 
@@ -292,9 +292,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self):
         self._import_reimport_performance(
             expected_num_queries1=302,
             expected_num_async_tasks1=6,
-            expected_num_queries2=239,
+            expected_num_queries2=233,
             expected_num_async_tasks2=17,
-            expected_num_queries3=121,
+            expected_num_queries3=115,
             expected_num_async_tasks3=16,
         )
 
@@ -315,9 +315,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr
         self._import_reimport_performance(
             expected_num_queries1=309,
             expected_num_async_tasks1=8,
-            expected_num_queries2=246,
+            expected_num_queries2=240,
             expected_num_async_tasks2=19,
-            expected_num_queries3=125,
+            expected_num_queries3=119,
             expected_num_async_tasks3=18,
         )