Release: Merge back 2.53.4 into dev from: master-into-dev/2.53.4-2.54.0-dev

.github/workflows/fetch-oas.yml

@@ -33,11 +33,15 @@ jobs:
              docker images
 
       - name: Start Dojo
-        run: docker compose up --no-deps -d postgres nginx uwsgi
+        run: docker compose up --no-deps -d valkey postgres uwsgi nginx
         env:
           DJANGO_VERSION: ${{ env.release_version }}-alpine
           NGINX_VERSION: ${{ env.release_version }}-alpine
 
+      - name: Wait for Dojo to be ready
+        run: |
+          timeout 120 bash -c 'until curl -f http://localhost:8080/api/v2/oa3/schema/; do sleep 10; done'
+
       - name: Download OpenAPI Specifications
         run: |-
              wget 'http://localhost:8080/api/v2/oa3/schema/?format=${{ matrix.file-type }}' -O oas.${{ matrix.file-type }} --tries=10 --retry-on-http-error=502

.github/workflows/test-helm-chart.yml

@@ -124,7 +124,7 @@ jobs:
           for c in $(echo "$chars" | grep -o .); do
             title="${title//"$c"/_}"
           done
-          yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n  description: '$title'\n"' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n  description: '"$title"'\n"' helm/defectdojo/Chart.yaml
           git add helm/defectdojo/Chart.yaml
           git commit -m "ci: update Chart annotations from PR #${{ github.event.pull_request.number }}" || echo "No changes to commit"
 

.github/workflows/validate_docs_build.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Setup Hugo
         uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
         with:
-          hugo-version: '0.153.0' # renovate: datasource=github-releases depName=gohugoio/hugo
+          hugo-version: '0.153.1' # renovate: datasource=github-releases depName=gohugoio/hugo
           extended: true
 
       - name: Setup Node

Dockerfile.django-alpine

@@ -78,6 +78,7 @@ COPY \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.django-debian

@@ -81,6 +81,7 @@ COPY \
   docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/certs/* \
   /
 COPY wsgi.py manage.py docker/unit-tests.sh ./

Dockerfile.integration-tests-debian

@@ -73,6 +73,7 @@ COPY --from=openapitools /opt/openapi-generator/modules/openapi-generator-cli/ta
 COPY docker/wait-for-it.sh \
   docker/secret-file-loader.sh \
   docker/reach_database.sh \
+  docker/reach_broker.sh \
   docker/entrypoint-integration-tests.sh \
   /
 

docker/entrypoint-celery-beat.sh

@@ -4,6 +4,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 umask 0002
 
@@ -23,6 +24,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 # do the check with Django stack

docker/entrypoint-celery-worker-dev.sh

@@ -7,8 +7,10 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

docker/entrypoint-celery-worker.sh

@@ -7,6 +7,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null || true)
@@ -22,6 +23,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then

docker/entrypoint-uwsgi-dev.sh

@@ -4,8 +4,10 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 cd /app || exit

docker/entrypoint-uwsgi.sh

@@ -3,6 +3,7 @@ set -e  # needed to handle "exit" correctly
 
 . /secret-file-loader.sh
 . /reach_database.sh
+. /reach_broker.sh
 
 # Allow for bind-mount multiple settings.py overrides
 FILES=$(ls /app/docker/extra_settings/* 2>/dev/null || true)
@@ -18,6 +19,7 @@ if [ "$NUM_FILES" -gt 0 ]; then
 fi
 
 wait_for_database_to_be_reachable
+wait_for_broker_to_be_reachable
 echo
 
 umask 0002

docker/reach_broker.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+wait_for_broker_to_be_reachable() {
+    echo -n "Waiting for broker to be reachable "
+    failure_count=0
+    DD_BROKER_READINESS_TIMEOUT=${DD_BROKER_READINESS_TIMEOUT:-10}
+    while true;
+    do 
+        set +e
+        celery --app=dojo status 2>/dev/null >/dev/null
+        BROKER_TEST=$?
+        set -e
+        if [[ "$BROKER_TEST" == "0" ]]; then
+            echo "Broker test was successful. Broker and at least one worker is connected."
+            break
+        fi
+        if [[ "$BROKER_TEST" == "69" ]]; then
+            echo "Broker test was successful. Broker is up. No worker is connected (but we are not testing that here)."
+            break
+        fi
+        echo -n "."
+        failure_count=$((failure_count + 1)) 
+        if [ $DD_BROKER_READINESS_TIMEOUT = $failure_count ]; then
+            echo "Broker test was failed:"
+            # One more time with output
+            celery --app=dojo status
+            exit 1
+        fi
+    done
+}

docs/content/en/customize_dojo/user_management/configure_sso.md

@@ -557,6 +557,7 @@ You can also optionally set the following variables:
     {{< highlight python >}}
     DD_SOCIAL_AUTH_OIDC_ID_KEY=(str, ''),                           #the key associated with the OIDC user IDs
     DD_SOCIAL_AUTH_OIDC_USERNAME_KEY=(str, ''),                     #the key associated with the OIDC usernames
+    DD_SOCIAL_AUTH_CREATE_USER_MAPPING=(str, "username"),           #could also be email or fullname
     DD_SOCIAL_AUTH_OIDC_WHITELISTED_DOMAINS=(list, ['']),           #list of domains allowed for login
     DD_SOCIAL_AUTH_OIDC_JWT_ALGORITHMS=(list, ["RS256","HS256"]),
     DD_SOCIAL_AUTH_OIDC_ID_TOKEN_ISSUER=(str, ''),

docs/content/en/open_source/upgrading/2.53.md

@@ -2,7 +2,7 @@
 title: "Upgrading to DefectDojo Version 2.53.x"
 toc_hide: true
 weight: -20251103
-description: "Helm chart: changes for initializer annotations + Replaced Redis with Valkey + HPA & PDB support"
+description: "Helm chart: changes for initializer annotations + Replaced Redis with Valkey + HPA & PDB support + Batch Deduplication"
 ---
 
 ## Helm Chart Changes
@@ -89,4 +89,26 @@ Both `extraAnnotations` and `initializer.podAnnotations` will now be properly ap
 
 Reimport will update existing findings `fix_available` and `fix_version` fields based on the incoming scan report.
 
+## Batch Deduplication
+
+Before 2.53.0 Defect Dojo has been deduplicating new or updated findings one-by-one. This works well for small imports and has the benefit of an easy to understand codebase and test suite. For larger imports however the performance is bad and resource usage is (very) high. A 1000+ finding import can cause a celery worker to spend minutes on deduplication.
+
+PR [13491](https://github.com/DefectDojo/django-DefectDojo/pull/13491) changes the deduplication process for import and reimport to be done in batches. This biggest benefit is that there now will be 1 database query per batch (1000 findings), instead of 1 query per finding (1000 queries).
+
+A quick test with the `jfrog_xray_unified/very_many_vulns.json` samples scan (10k findings) shwo the obvious huge improvement in deduplication time. Please note that we're not only doing this for performance, but also to reduce the resources (cloud cost) needed to run Defect Dojo.
+
+initial import (no duplicates):
+| branch | import time | dedupe time | total time |
+|--------|:-----------:|:-----------:|:-----------:|
+| dev | ~200s | ~400s | ~600s |
+| dedupe-batching | ~190s | _~12s_ | ~200s |
+
+second import into the same product (all duplicates):
+initial import (no duplicates):
+| branch | import time | dedupe time | total time |
+|--------|:-----------:|:-----------:|:-----------:|
+| dev | ~200s | ~400s | ~600s |
+| dedupe-batching | ~190s | _~180s_ | ~370s |
+
+
 There are no other special instructions for upgrading to 2.53.x. Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.53.0) for the contents of the release.

dojo/api_v2/serializers.py

@@ -2295,6 +2295,7 @@ def process_scan(
         Raises exceptions in the event of an error
         """
         try:
+            logger.debug(f"process_scan called with context: {context}")
             start_time = time.perf_counter()
             importer = self.get_importer(**context)
             context["test"], _, _, _, _, _, _ = importer.process_scan(
@@ -2572,6 +2573,7 @@ def process_scan(
         """
         statistics_before, statistics_delta = None, None
         try:
+            logger.debug(f"process_scan called with context: {context}")
             start_time = time.perf_counter()
             if test := context.get("test"):
                 statistics_before = test.statistics

dojo/api_v2/views.py

@@ -2529,7 +2529,7 @@ def perform_create(self, serializer):
             jira_driver = engagement or (product or None)
             if jira_project := (jira_helper.get_jira_project(jira_driver) if jira_driver else None):
                 push_to_jira = push_to_jira or jira_project.push_all_issues
-        # logger.debug(f"push_to_jira: {push_to_jira}")
+
         serializer.save(push_to_jira=push_to_jira)
 
     def get_queryset(self):

dojo/engagement/views.py

@@ -936,6 +936,30 @@ def import_findings(
     ) -> str | None:
         """Attempt to import with all the supplied information"""
         try:
+            # Log only user-entered form values, excluding internal objects
+            user_values = {
+                "scan_type": context.get("scan_type"),
+                "scan_date": context.get("scan_date"),
+                "minimum_severity": context.get("minimum_severity"),
+                "active": context.get("active"),
+                "verified": context.get("verified"),
+                "test_title": context.get("test_title"),
+                "tags": context.get("tags"),
+                "version": context.get("version"),
+                "branch_tag": context.get("branch_tag"),
+                "build_id": context.get("build_id"),
+                "commit_hash": context.get("commit_hash"),
+                "service": context.get("service"),
+                "close_old_findings": context.get("close_old_findings"),
+                "apply_tags_to_findings": context.get("apply_tags_to_findings"),
+                "apply_tags_to_endpoints": context.get("apply_tags_to_endpoints"),
+                "close_old_findings_product_scope": context.get("close_old_findings_product_scope"),
+                "group_by": context.get("group_by"),
+                "create_finding_groups_for_all_findings": context.get("create_finding_groups_for_all_findings"),
+                "push_to_jira": context.get("push_to_jira"),
+                "push_all_jira_issues": context.get("push_all_jira_issues"),
+            }
+            logger.debug(f"import_findings called with user values: {user_values}")
             importer_client = self.get_importer(context)
             context["test"], _, finding_count, closed_finding_count, _, _, _ = importer_client.process_scan(
                 context.pop("scan", None),

dojo/finding/deduplication.py

@@ -27,6 +27,7 @@ def get_finding_models_for_deduplication(finding_ids):
 
     """
     if not finding_ids:
+        logger.debug("get_finding_models_for_deduplication called with no finding_ids")
         return []
 
     return list(
@@ -659,6 +660,7 @@ def dedupe_batch_of_findings(findings, *args, **kwargs):
         return batch_dedupe_method(findings, *args, **kwargs)
 
     if not findings:
+        logger.debug("dedupe_batch_of_findings called with no findings")
         return None
 
     enabled = System_Settings.objects.get().enable_deduplication

dojo/finding/helper.py

@@ -470,22 +470,30 @@ def post_process_finding_save_internal(finding, dedupe_option=True, rules_option
 @app.task
 def post_process_findings_batch_signature(finding_ids, *args, dedupe_option=True, rules_option=True, product_grading_option=True,
              issue_updater_option=True, push_to_jira=False, user=None, **kwargs):
-    return post_process_findings_batch(finding_ids, dedupe_option, rules_option, product_grading_option,
-                                       issue_updater_option, push_to_jira, user, **kwargs)
+    return post_process_findings_batch(finding_ids, *args, dedupe_option=dedupe_option, rules_option=rules_option, product_grading_option=product_grading_option, issue_updater_option=issue_updater_option, push_to_jira=push_to_jira, user=user, **kwargs)
+    # Pass arguments as keyword arguments to ensure Celery properly serializes them
 
 
 @dojo_async_task
 @app.task
 def post_process_findings_batch(finding_ids, *args, dedupe_option=True, rules_option=True, product_grading_option=True,
              issue_updater_option=True, push_to_jira=False, user=None, **kwargs):
 
+    logger.debug(
+        f"post_process_findings_batch called: finding_ids_count={len(finding_ids) if finding_ids else 0}, "
+        f"args={args}, dedupe_option={dedupe_option}, rules_option={rules_option}, "
+        f"product_grading_option={product_grading_option}, issue_updater_option={issue_updater_option}, "
+        f"push_to_jira={push_to_jira}, user={user.id if user else None}, kwargs={kwargs}",
+    )
     if not finding_ids:
         return
 
     system_settings = System_Settings.objects.get()
 
     # use list() to force a complete query execution and related objects to be loaded once
+    logger.debug(f"getting finding models for batch deduplication with: {len(finding_ids)} findings")
     findings = get_finding_models_for_deduplication(finding_ids)
+    logger.debug(f"found {len(findings)} findings for batch deduplication")
 
     if not findings:
         logger.debug(f"no findings found for batch deduplication with IDs: {finding_ids}")
@@ -517,6 +525,8 @@ def post_process_findings_batch(finding_ids, *args, dedupe_option=True, rules_op
                 jira_helper.push_to_jira(finding)
             else:
                 jira_helper.push_to_jira(finding.finding_group)
+    else:
+        logger.debug("push_to_jira is False, not ushing to JIRA")
 
 
 @receiver(pre_delete, sender=Finding)

dojo/finding/views.py

@@ -14,7 +14,7 @@
 from django.core import serializers
 from django.core.exceptions import PermissionDenied, ValidationError
 from django.db import models
-from django.db.models import F, QuerySet
+from django.db.models import F, QuerySet, Value
 from django.db.models.functions import Coalesce, ExtractDay, Length, TruncDate
 from django.db.models.query import Prefetch
 from django.http import Http404, HttpRequest, HttpResponse, HttpResponseRedirect, JsonResponse, StreamingHttpResponse
@@ -257,6 +257,11 @@ def filter_findings_by_filter_name(self, findings: QuerySet[Finding]):
         return findings
 
     def filter_findings_by_form(self, request: HttpRequest, findings: QuerySet[Finding]):
+        # Apply default ordering if no ordering parameter is provided
+        # This maintains backward compatibility with the previous behavior
+        if not request.GET.get("o"):
+            findings = findings.order_by(self.get_order_by())
+
         # Set up the args for the form
         args = [request.GET, findings]
         # Set the initial form args
@@ -277,11 +282,19 @@ def filter_findings_by_form(self, request: HttpRequest, findings: QuerySet[Findi
     def get_filtered_findings(self):
         findings = get_authorized_findings(Permissions.Finding_View)
         # Annotate computed SLA age in days: sla_expiration_date - (sla_start_date or date)
+        # Handle NULL sla_expiration_date by using Coalesce to provide a large default value
+        # so NULLs sort last when sorting ascending (most urgent first)
         findings = findings.annotate(
-            sla_age_days=ExtractDay(
-                F("sla_expiration_date") - Coalesce(F("sla_start_date"), TruncDate("created")),
+            sla_age_days=Coalesce(
+                ExtractDay(
+                    F("sla_expiration_date") - Coalesce(F("sla_start_date"), TruncDate("created")),
+                ),
+                Value(999999),  # Large value to push NULLs to the end when sorting ascending
+                output_field=models.IntegerField(),
             ),
-        ).order_by(self.get_order_by())
+        )
+        # Don't apply initial order_by here - let OrderingFilter handle it via request.GET['o']
+        # This prevents conflicts between initial ordering and user-requested sorting
         findings = self.filter_findings_by_object(findings)
         return self.filter_findings_by_filter_name(findings)
 

dojo/fixtures/unit_metrics_additional_data.json

@@ -175,7 +175,7 @@
       "description": "TEST finding",
       "mitigated_by": null,
       "reporter": 2,
-      "mitigated": null,
+      "mitigated": "2018-01-02T00:00:00Z",
       "active": false,
       "line": 100,
       "under_review": false,
@@ -416,7 +416,7 @@
       "description": "test finding",
       "mitigated_by": null,
       "reporter": 1,
-      "mitigated": null,
+      "mitigated": "2017-12-28T00:00:00Z",
       "active": true,
       "line": 123,
       "under_review": false,