Finding hash/dedupe changes (#10386)

* dedupe-help Move logic to set Finding hash code to its own method

* dedupe-help rework set_hash_code method to accept dedupe_option

* Update versions in application files

* Revert "Update versions in application files"

This reverts commit 7ee4bfa.

* dedupe-help reorder method to make linter happy

* dedupe-help Rework finding hash set/dedupe to attempt to load methods based on settings and fall back to existing implementations as defaults

* dedupe-help add helper method to load custom methods and use it

* dedupe-help bug in load custom helper method

* dedupe-help Linter fix (import ordering)

* dedupe-help Update default_importer to handle .values() call on findings set within close old findings method

* dedupe-help extract get_(re)importer methods from engagement/test (re)import views into a separate method

* dedupe-help extract reimport dedupe alg determination into its own method

* dedupe-help refactor where custom methods for hashing/dedupe are called to minimize changes to existing calls

* dedupe-help linter fixes

---------

Co-authored-by: DefectDojo release bot <dojo-release-bot@users.noreply.github.com>

Author: dogboat

dojo/engagement/views.py

@@ -67,6 +67,7 @@
     TypedNoteForm,
     UploadThreatForm,
 )
+from dojo.importers.base_importer import BaseImporter
 from dojo.importers.default_importer import DefaultImporter
 from dojo.models import (
     Check_List,
@@ -921,6 +922,15 @@ def create_engagement(
         # Return the engagement
         return engagement
 
+    def get_importer(
+        self,
+        context: dict,
+    ) -> BaseImporter:
+        """
+        Gets the importer to use
+        """
+        return DefaultImporter(**context)
+
     def import_findings(
         self,
         context: dict,
@@ -929,7 +939,7 @@ def import_findings(
         Attempt to import with all the supplied information
         """
         try:
-            importer_client = DefaultImporter(**context)
+            importer_client = self.get_importer(context)
             context["test"], _, finding_count, closed_finding_count, _, _, _ = importer_client.process_scan(
                 context.pop("scan", None),
             )

dojo/importers/default_importer.py

@@ -108,7 +108,7 @@ def process_scan(
         new_findings = self.determine_process_method(self.parsed_findings, **kwargs)
         # Close any old findings in the processed list if the the user specified for that
         # to occur in the form that is then passed to the kwargs
-        closed_findings = self.close_old_findings(self.test.finding_set.values(), **kwargs)
+        closed_findings = self.close_old_findings(self.test.finding_set.all(), **kwargs)
         # Update the timestamps of the test object by looking at the findings imported
         self.update_timestamps()
         # Update the test meta
@@ -247,11 +247,12 @@ def close_old_findings(
         logger.debug("REIMPORT_SCAN: Closing findings no longer present in scan report")
         # Close old active findings that are not reported by this scan.
         # Refactoring this to only call test.finding_set.values() once.
+        findings = findings.values()
         mitigated_hash_codes = []
         new_hash_codes = []
         for finding in findings:
             new_hash_codes.append(finding["hash_code"])
-            if getattr(finding, "is_mitigated", None):
+            if finding.get("is_mitigated", None):
                 mitigated_hash_codes.append(finding["hash_code"])
                 for hash_code in new_hash_codes:
                     if hash_code == finding["hash_code"]:

dojo/importers/default_reimporter.py

@@ -147,6 +147,13 @@ def process_scan(
             test_import_history,
         )
 
+    def determine_deduplication_algorithm(self) -> str:
+        """
+        Determines what dedupe algorithm to use for the Test being processed.
+        :return: A string representing the dedupe algorithm to use.
+        """
+        return self.test.deduplication_algorithm
+
     def process_findings(
         self,
         parsed_findings: List[Finding],
@@ -160,7 +167,7 @@ def process_findings(
         at import time
         """
 
-        self.deduplication_algorithm = self.test.deduplication_algorithm
+        self.deduplication_algorithm = self.determine_deduplication_algorithm()
         self.original_items = list(self.test.finding_set.all())
         self.new_items = []
         self.reactivated_items = []

dojo/models.py

@@ -2640,14 +2640,7 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru
             except Exception as ex:
                 logger.error("Can't compute cvssv3 score for finding id %i. Invalid cvssv3 vector found: '%s'. Exception: %s", self.id, self.cvssv3, ex)
 
-        # Finding.save is called once from serializers.py with dedupe_option=False because the finding is not ready yet, for example the endpoints are not built
-        # It is then called a second time with dedupe_option defaulted to true; now we can compute the hash_code and run the deduplication
-        if dedupe_option:
-            if (self.hash_code is not None):
-                deduplicationLogger.debug("Hash_code already computed for finding")
-            else:
-                self.hash_code = self.compute_hash_code()
-                deduplicationLogger.debug("Hash_code computed for finding: %s", self.hash_code)
+        self.set_hash_code(dedupe_option)
 
         if self.pk is None:
             # We enter here during the first call from serializers.py
@@ -3346,6 +3339,20 @@ def inherit_tags(self, potentially_existing_tags):
     def violates_sla(self):
         return (self.sla_expiration_date and self.sla_expiration_date < timezone.now().date())
 
+    def set_hash_code(self, dedupe_option):
+        from dojo.utils import get_custom_method
+        if hash_method := get_custom_method("FINDING_HASH_METHOD"):
+            hash_method(self, dedupe_option)
+        else:
+            # Finding.save is called once from serializers.py with dedupe_option=False because the finding is not ready yet, for example the endpoints are not built
+            # It is then called a second time with dedupe_option defaulted to true; now we can compute the hash_code and run the deduplication
+            if dedupe_option:
+                if self.hash_code is not None:
+                    deduplicationLogger.debug("Hash_code already computed for finding")
+                else:
+                    self.hash_code = self.compute_hash_code()
+                    deduplicationLogger.debug("Hash_code computed for finding: %s", self.hash_code)
+
 
 class FindingAdmin(admin.ModelAdmin):
     # For efficiency with large databases, display many-to-many fields with raw

dojo/test/views.py

@@ -41,6 +41,7 @@
     TestForm,
     TypedNoteForm,
 )
+from dojo.importers.base_importer import BaseImporter
 from dojo.importers.default_reimporter import DefaultReImporter
 from dojo.models import (
     IMPORT_UNTOUCHED_FINDING,
@@ -979,6 +980,15 @@ def process_jira_form(
         context["push_to_jira"] = push_all_jira_issues or (form and form.cleaned_data.get("push_to_jira"))
         return None
 
+    def get_reimporter(
+        self,
+        context: dict,
+    ) -> BaseImporter:
+        """
+        Gets the reimporter to use
+        """
+        return DefaultReImporter(**context)
+
     def reimport_findings(
         self,
         context: dict,
@@ -987,7 +997,7 @@ def reimport_findings(
         Attempt to import with all the supplied information
         """
         try:
-            importer_client = DefaultReImporter(**context)
+            importer_client = self.get_reimporter(context)
             (
                 context["test"],
                 finding_count,

dojo/utils.py

@@ -1,13 +1,15 @@
 import binascii
 import calendar as tcalendar
 import hashlib
+import importlib
 import logging
 import mimetypes
 import os
 import re
 from calendar import monthrange
 from datetime import date, datetime, timedelta
 from math import pi, sqrt
+from typing import Callable, Optional
 
 import bleach
 import crum
@@ -295,6 +297,9 @@ def do_dedupe_finding_task(new_finding, *args, **kwargs):
 
 
 def do_dedupe_finding(new_finding, *args, **kwargs):
+    if dedupe_method := get_custom_method("FINDING_DEDUPE_METHOD"):
+        return dedupe_method(new_finding, *args, **kwargs)
+
     try:
         enabled = System_Settings.objects.get(no_cache=True).enable_deduplication
     except System_Settings.DoesNotExist:
@@ -2594,6 +2599,23 @@ def get_open_findings_burndown(product):
     return past_90_days
 
 
+def get_custom_method(setting_name: str) -> Optional[Callable]:
+    """
+    Attempts to load and return the method specified by fully-qualified name at the given setting.
+
+    :param setting_name: The name of the setting that holds the fqname of the Python method we want to load
+    :return: The callable if it was able to be loaded, else None
+    """
+    if fq_name := getattr(settings, setting_name, None):
+        try:
+            mn, _, fn = fq_name.rpartition(".")
+            m = importlib.import_module(mn)
+            return getattr(m, fn)
+        except ModuleNotFoundError:
+            pass
+    return None
+
+
 def generate_file_response(file_object: FileUpload) -> FileResponse:
     """Serve an uploaded file in a uniformed way.