diff --git a/.github/labeler.yml b/.github/labeler.yml index 5a93262ccd4..f0f52a50f24 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -65,3 +65,8 @@ lint: - changed-files: - any-glob-to-any-file: - ruff.toml + +gha: + - changed-files: + - any-glob-to-any-file: + - .github/workflows diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index c7b55c5c125..7a02aa15919 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -43,6 +43,8 @@ categories: label: 'localization' - title: 'πŸ”§ Improved code quality with linters' label: 'lint' + - title: 'βš™οΈ Improvemets of GitHub Actions' + label: 'gha' - title: '🧰 Maintenance' collapse-after: 3 labels: diff --git a/.github/workflows/test-helm-chart.yml b/.github/workflows/test-helm-chart.yml index c1dac77d12c..a2c8484889e 100644 --- a/.github/workflows/test-helm-chart.yml +++ b/.github/workflows/test-helm-chart.yml @@ -26,14 +26,17 @@ jobs: with: python-version: 3.14 # Renovate helper is not needed here - - name: Configure Helm repos - run: |- - helm dependency list ./helm/defectdojo - helm dependency update ./helm/defectdojo + # Running update is not needed for listing - it just triggers errors as soon as one of the subcharts is not in the latest version (`helm dep. update` change content of the chart) + # As soon as we would like to run more extensive tests, this can be reenabled, but this will need to be placed after "ct list-changed" command + # - name: Configure Helm repos + # run: |- + # helm dependency list ./helm/defectdojo + # helm dependency update ./helm/defectdojo - name: Set up chart-testing uses: helm/chart-testing-action@6ec842c01de15ebb84c8627d2744a0c2f2755c9f # v2.8.0 with: + version: 3.14.0 # renovate: datasource=github-releases depName=helm/chart-testing yamale_version: 6.1.0 # renovate: datasource=pypi depName=yamale versioning=semver yamllint_version: 1.37.1 # renovate: datasource=pypi depName=yamllint versioning=semver @@ -52,6 +55,8 @@ jobs: changed=$(ct list-changed --config ct.yaml --target-branch ${{ env.ct-branch}}) if [[ -n "$changed" ]]; then echo "changed=true" >> $GITHUB_OUTPUT + echo "Content changed:" + echo "$changed" | sed 's/^/ /' fi # run version check only if not dev as in dev we have a `x.y.z-dev` version @@ -114,12 +119,12 @@ jobs: - name: Update values in HELM chart if: startsWith(github.head_ref, 'renovate/') || startsWith(github.head_ref, 'dependabot/') run: | - title=${{ github.event.pull_request.title }} + title='${{ github.event.pull_request.title }}' chars='{}:[],&*#?|-<>=!%@' for c in $(echo "$chars" | grep -o .); do title="${title//"$c"/_}" done - yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n description: $title\n"' helm/defectdojo/Chart.yaml + yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n description: '$title'\n"' helm/defectdojo/Chart.yaml git add helm/defectdojo/Chart.yaml git commit -m "ci: update Chart annotations from PR #${{ github.event.pull_request.number }}" || echo "No changes to commit" diff --git a/components/package.json b/components/package.json index 30ae9191c97..d91b75e32dd 100644 --- a/components/package.json +++ b/components/package.json @@ -1,6 +1,6 @@ { "name": "defectdojo", - "version": "2.53.1", + "version": "2.53.2", "license" : "BSD-3-Clause", "private": true, "dependencies": { diff --git a/docs/assets/images/asset_hierarchy.png b/docs/assets/images/asset_hierarchy.png new file mode 100644 index 00000000000..3bf2ae0fa6d Binary files /dev/null and b/docs/assets/images/asset_hierarchy.png differ diff --git a/docs/assets/images/asset_hierarchy_diagram.png b/docs/assets/images/asset_hierarchy_diagram.png new file mode 100644 index 00000000000..78f6baa97d2 Binary files /dev/null and b/docs/assets/images/asset_hierarchy_diagram.png differ diff --git a/docs/assets/images/asset_hierarchy_node.png b/docs/assets/images/asset_hierarchy_node.png new file mode 100644 index 00000000000..a1b33720379 Binary files /dev/null and b/docs/assets/images/asset_hierarchy_node.png differ diff --git a/docs/assets/images/assets_loadmore.png b/docs/assets/images/assets_loadmore.png new file mode 100644 index 00000000000..7de764b5894 Binary files /dev/null and b/docs/assets/images/assets_loadmore.png differ diff --git a/docs/content/en/about_defectdojo/pro_features.md b/docs/content/en/about_defectdojo/pro_features.md index 8fd83333f07..46eaccb9f0d 100644 --- a/docs/content/en/about_defectdojo/pro_features.md +++ b/docs/content/en/about_defectdojo/pro_features.md @@ -18,6 +18,12 @@ See our [Pro UI Guide](../ui_pro_vs_os) for more information. ![image](images/enabling_deduplication_within_an_engagement_2.png) +### Assets/Organizations + +DefectDojo Pro allows for improved organizational visualization for large lists of repositories or other business structures. See [Assets/Organizations documentation](/en/working_with_findings/organizing_engagements_tests/pro_assets_organizations/) for details. + +![image](images/asset_hierarchy_diagram.png) + ### Finding Priority DefectDojo Pro can pre-triage your Findings by Priority and Risk, allowing your team to identify and fix your most critical issues first. diff --git a/docs/content/en/changelog/changelog.md b/docs/content/en/changelog/changelog.md index e978214f0ac..a747c2c6f8a 100644 --- a/docs/content/en/changelog/changelog.md +++ b/docs/content/en/changelog/changelog.md @@ -8,8 +8,57 @@ Here are the release notes for **DefectDojo Pro (Cloud Version)**. These release For Open Source release notes, please see the [Releases page on GitHub](https://github.com/DefectDojo/django-DefectDojo/releases), or alternatively consult the Open Source [upgrade notes](/en/open_source/upgrading/upgrading_guide/). +## Dec 2025: v2.53 + +### Dec 8, 2025: v2.53.1 + +* **(Assets/Organizations)** Introduced overhaul to Products/Product Types, added the ability to create and diagram relationships between Assets. See [Assets/Organizations documentation](/en/working_with_findings/organizing_engagements_tests/pro_assets_organizations/) for details, and information on opting in to the Beta. +* **(Findings)** Added new KEV fields for ransomware, exploits, and date handling. + +### Dec 1, 2025: v2.53.0 + +* **(Pro UI)** Added Asset Hierarchy. +* **(Priority)** Priority and Risk can now be overridden manually, or through Rules Engine. + +## Nov 2025: v2.52 + +### Nov 24, 2025: v2.52.3 + +* **(Pro UI)** Improved error messaging for failed Imports. +* **(Pro UI)** Added Engagement Tags column to Finding lists + + +### Nov 17, 2025: v2.52.2 + +* No significant feature changes. + +### Nov 10, 2025: v2.52.1 + +* **(Pro UI)** Finding view now shows all associated Endpoints, not just Active Endpoints + + +### Nov 3, 2025: v2.52.0 + +* **(Pro UI)** In-app Contact Support form now requires a valid email address in your user profile. +* **(Pro UI)** You can now Add Files to Findings through the Pro UI directly from Finding Lists. +* **(Pro UI)** Unicode letters are now allowed in Tags. + ## Oct 2025: v2.51 +### Oct 27, 2025: v2.51.3 + +* **(Tools)** Added Nuclei scan support for Smart Upload. +* **(Priority)** Added Prioritization Engine to allow for configurable Priority and Risk calculations for individual Findings under a given Product. +* **(Metrics)** Updated Metrics table to include Products with zero Findings (as a result of filter parameters, or otherwise). +* **(Pro UI)** Added Surveys to Pro UI. + + + +### Oct 20, 2025: v2.51.2 + +* **(Connectors)** Added Anchore Enterprise Connector. + + ### Oct 14, 2025: v2.51.1 * **(Pro UI)** Added Finding Quick Report feature. Quick report allows users to quickly render an HTML report with the currently displayed Findings on a Finding table. diff --git a/docs/content/en/customize_dojo/user_management/configure_sso.md b/docs/content/en/customize_dojo/user_management/configure_sso.md index da6c9ca5f33..1aaed3b95b3 100644 --- a/docs/content/en/customize_dojo/user_management/configure_sso.md +++ b/docs/content/en/customize_dojo/user_management/configure_sso.md @@ -133,7 +133,7 @@ The Azure AD token need to be configured to include Group IDs. Without this step To update the format of the token, add a [Group Claim](https://learn.microsoft.com/en-us/entra/identity/hybrid/connect/how-to-connect-fed-group-claims) that applies to whatever Group type you are using. If unsure of what type that is, select `All Groups`. Do not activate `Emit groups as role claims` within the Azure AD "Token configuration" page. -Application API permissions need to be updated with the `Group.Read.All` permission so that groups can be read on behalf of the user that has successfully signed in. +Application API permissions need to be updated with the `GroupMember.Read.All` or `Group.Read.All` permission so that groups can be read on behalf of the user that has successfully signed in. `GroupMember.Read.All` is recommended as this grants the application fewer permissions. ##### Group Cleaning @@ -169,7 +169,7 @@ The Azure AD token returned by Azure will also need to be configured to include If unsure of what type that is, select `All Groups`. Do not activate `Emit groups as role claims` within the Azure AD "Token configuration" page. -Application API permissions need to be updated with the `Group.Read.All` permission so that groups can be read on behalf of the user that has successfully signed in. +Application API permissions need to be updated with the `GroupMember.Read.All` or `Group.Read.All` permission so that groups can be read on behalf of the user that has successfully signed in. `GroupMember.Read.All` is recommended as this grants the application fewer permissions. To limit the amount of groups imported from Azure AD, a regular expression can be used as the following: diff --git a/docs/content/en/working_with_findings/organizing_engagements_tests/pro_assets_organizations.md b/docs/content/en/working_with_findings/organizing_engagements_tests/pro_assets_organizations.md new file mode 100644 index 00000000000..877097c74d1 --- /dev/null +++ b/docs/content/en/working_with_findings/organizing_engagements_tests/pro_assets_organizations.md @@ -0,0 +1,123 @@ +--- +title: "⚠️ Assets and Organizations (Pro)" +description: "DefectDojo Pro - Product Hierarchy Overhaul" +--- + +DefectDojo Pro is extending the Product/Product Type object classes to provide greater flexibility with the data model. + +Currently, this feature is in Beta. Pro users who are interested in opting in can do so by emailing [support@defectdojo.com](mailto:support@defectdojo.com). + +## Significant Changes + +* **Product Types** have been renamed to "Organizations", and **Products** have been renamed to "Assets". Currently, this name change is opt-in for existing DefectDojo Pro subscriptions. +* **Assets** can now have parent/child relationships with one another to further sub-categorize Organizational components. + +### Organizations + +As with Product Types, **Organizations** should be understood as a top-level category. You can use these to separate your business' core software applications, departments or business functions. + +For example, you could create an Organization for many repository groupings: "Core Application", "Infrastructure", "DevOps", "Analytics", "SDK" could all contain multiple code repos. + +Keep in mind that for reporting purposes, it’s easier to combine multiple Organizations into a single document than it is to subdivide a single Organization into separate documents. Therefore, we recommend setting up Organizations at as granular a level as makes sense for your team's reports. For example, there is no need to represent a large business division as an Organization if you're primarily going to be reporting on individual departments within that division. + +### Assets + +Assets are meant to represent subdivisions of your Organizations. However, unlike Products, Assets can be nested, and have parent-child relationships with one another. + +## Asset Nesting Examples + +### Asset-Level Branch Representation + +Development and feature branches can be represented in a variety of ways; separate Engagements or Tests are existing ways that you can represent the difference between your Production, Dev, and other feature branches. + +You can also represent these using nested Assets. Consider the following Asset tree: + +``` +Core Application [Organization] +└── webapp-frontend + β”œβ”€β”€ webapp-frontend/prod + └── webapp-frontend/dev + β”œβ”€β”€ webapp-frontend/dev/feature-a + └── webapp-frontend/dev/feature-b +``` + +In this environment, each branch (`prod`, `dev`, `feature a`, `feature b`) could have its own Engagements and Tests that are isolated from the other Assets, so that they don't deduplicate against each other. This setup can also ease in navigation, as Asset names can directly correspond to the path on Git. + +### Mono-Repo: Separate Components + +If you use a single repository for all of your code, but have different teams contributing to directories within that repository, you can set up your Asset nesting to represent that structure. + +``` +Core Application [Organization] +β”œβ”€β”€ webapp-frontend [Parent Asset] +β”‚ β”œβ”€β”€ mobile-ios +β”‚ β”œβ”€β”€ mobile-android +β”‚ └── mobile-sdk +β”œβ”€β”€ webapp-backend [Parent Asset] +β”‚ β”œβ”€β”€ database +β”‚ └── api +└── infra [Parent Asset] + β”œβ”€β”€ docker + β”œβ”€β”€ kubernetes + └── nginx +``` + +In this diagram, every element under "Core Application" could be recorded as a separate Asset, with unique business criticality (see: [Priority & Risk](/en/working_with_findings/priority_adjustments/)), RBAC, and corresponding Engagements and Tests. You could continue to test, and store results, on the parent Asset (for example, `webapp-backend`), but you could also run isolated testing on a particular child Asset (for example, `database`). + +### Pen Tests: Isolated RBAC + +If you want to store pen test results within a single asset, but you don't want testers to be able to look at asset data, you could create child assets for each testing group to upload their results. + +``` +Core Application [Organization] +└── webapp-frontend [Parent Asset] + β”œβ”€β”€ Pen Test Group A + └── Pen Test Group B +``` + +Crucially, giving a user RBAC access to a single Child Asset (e.g. `Pen Test Group A`) here does not allow them to see any Findings from other Child Assets (e.g. `Pen Test Group B`), nor does it allow them to see Findings in the Parent Asset (`webapp-frontend`). + +The Parent Asset could contain Engagements representing CI/CD results, internal Testing, historical data, or other Finding data which you do not want 3rd parties to be able to discover. Creating a Child Asset for specific Test results allows your internal team to report on those results in combination with the state of the parent Asset. + +## Visualizing Assets - Hierarchy + +You can visualize the structure of Assets in DefectDojo, and change relationships using the Asset Hierarchy option in the menu. + +![image](images/asset_hierarchy.png) + +Opening Asset Hierarchy will display a table of all of your Assets which can be filtered. Selecting one or more Assets from this table will render a hierarchy diagram. + +![image](images/asset_hierarchy_diagram.png) + +### Diagram navigation + +The icons at the top left of the hierarchy diagram allow you to zoom in and out. Clicking and dragging in this diagram allows you to scroll through the diagram. + +Each Asset is rendered as a single node in this diagram, which can be moved around for display purposes. + +Assets are connected together using labelled paths, which represent the kind of relationship each note has to one another. Currently, `parent` is the only label supported. + +### Exploring Asset nodes + +Each Asset node can be interacted with by clicking on the blue buttons. These buttons appear only when an Asset node is selected (by clicking on the node). + +![image](images/asset_hierarchy_node.png) + +* πŸ‘οΈ (eyeball icon) will take you directly to the corresponding Asset View (formerly known as the Product View). +* ✏️ (pencil icon) will open a modal with the Edit Asset form (formerly known as the Edit Product form) +* βž• (plus icon) will allow you to add a new Child Asset to this Asset. The Asset does not need to be currently visible in the diagram, but must be part of the same Organization. +* βœ₯ (four-arrows icon) allows you to change the Parent Asset of the currently selected Asset. +* πŸ—‘οΈ (trash can icon) allows you to remove an Asset's parent relationship. This icon only appears if an Asset already has a Parent. + +If your diagram displays an Asset with un-selected Parent Assets, you can click the Load More button to populate the diagram with the Parent Asset (as well as that Parent Asset's children). + +![image](images/assets_loadmore.png) + +## Notes + +* Note that deduplication scopes have not changed; Assets only deduplicate Findings within themselves, and do not consider Findings in other Assets, regardless of Parent/Child relationships. +* RBAC scopes have not changed within this system; each Asset is still considered an individual object for the purposes of assigning permissions. No new RBAC inheritance has been created. + * Giving a user access to an entire Organization will still give that user access to all Assets contained within that Organization (as with Product Types). + * Giving a user access to a single Asset does not give that user access to any related Parent or Child Assets, nor access to the Organization. +* There is no limit to the number of Parent/Child relationships that can be created. Theoretically, you could represent a repository's entire directory structure with separate Assets if you wished. +* Cyclical relationships are not allowed: Parent Assets cannot be Children of their Child Assets. \ No newline at end of file diff --git a/dojo/__init__.py b/dojo/__init__.py index 1d5aa3febd8..e4af428e118 100644 --- a/dojo/__init__.py +++ b/dojo/__init__.py @@ -4,6 +4,6 @@ # Django starts so that shared_task will use this app. from .celery import app as celery_app # noqa: F401 -__version__ = "2.53.1" +__version__ = "2.53.2" __url__ = "https://github.com/DefectDojo/django-DefectDojo" __docs__ = "https://documentation.defectdojo.com" diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 7f334236dbf..7297e55fef4 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -147,7 +147,14 @@ def set_duplicate(new_finding, existing_finding): for find in new_finding.original_finding.all(): new_finding.original_finding.remove(find) set_duplicate(find, existing_finding) - existing_finding.found_by.add(new_finding.test.test_type) + # Only add test type to found_by if it is not already present. + # This is efficient because `found_by` is prefetched for candidates via `build_dedupe_scope_queryset()`. + test_type = getattr(getattr(new_finding, "test", None), "test_type", None) + if test_type is not None and test_type not in existing_finding.found_by.all(): + existing_finding.found_by.add(test_type) + + # existing_finding.found_by.add(new_finding.test.test_type) + logger.debug("saving new finding: %d", new_finding.id) super(Finding, new_finding).save() logger.debug("saving existing finding: %d", existing_finding.id) @@ -239,7 +246,7 @@ def build_dedupe_scope_queryset(test): return ( Finding.objects.filter(scope_q) .select_related("test", "test__engagement", "test__test_type") - .prefetch_related("endpoints") + .prefetch_related("endpoints", "found_by") ) diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 277609d3153..19bf9ee6d99 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -236,7 +236,8 @@ def add_to_finding_group(finding_group, finds): finding_group.findings.add(*available_findings) # Now update the JIRA to add the finding to the finding group - if finding_group.has_jira_issue and jira_helper.get_jira_instance(finding_group).finding_jira_sync: + jira_instance = jira_helper.get_jira_instance(finding_group) + if finding_group.has_jira_issue and jira_instance and jira_instance.finding_jira_sync: logger.debug("pushing to jira from finding.finding_bulk_update_all()") jira_helper.push_to_jira(finding_group) @@ -263,7 +264,8 @@ def remove_from_finding_group(finds): # Now update the JIRA to remove the finding from the finding group for group in affected_groups: - if group.has_jira_issue and jira_helper.get_jira_instance(group).finding_jira_sync: + jira_instance = jira_helper.get_jira_instance(group) + if group.has_jira_issue and jira_instance and jira_instance.finding_jira_sync: logger.debug("pushing to jira from finding.finding_bulk_update_all()") jira_helper.push_to_jira(group) @@ -340,7 +342,8 @@ def group_findings_by(finds, finding_group_by_option): # Now update the JIRA to add the finding to the finding group for group in affected_groups: - if group.has_jira_issue and jira_helper.get_jira_instance(group).finding_jira_sync: + jira_instance = jira_helper.get_jira_instance(group) + if group.has_jira_issue and jira_instance and jira_instance.finding_jira_sync: logger.debug("pushing to jira from finding.finding_bulk_update_all()") jira_helper.push_to_jira(group) diff --git a/dojo/finding/views.py b/dojo/finding/views.py index e48554e613d..f49eab8fb5d 100644 --- a/dojo/finding/views.py +++ b/dojo/finding/views.py @@ -983,10 +983,11 @@ def process_jira_form(self, request: HttpRequest, finding: Finding, context: dic jira_helper.finding_link_jira(request, finding, new_jira_issue_key) jira_message = "Linked a JIRA issue successfully." # any existing finding should be updated + jira_instance = jira_helper.get_jira_instance(finding) push_to_jira = ( push_to_jira and not (push_to_jira and finding.finding_group) - and (finding.has_jira_issue or jira_helper.get_jira_instance(finding).finding_jira_sync) + and (finding.has_jira_issue or (jira_instance and jira_instance.finding_jira_sync)) ) # Determine if a message should be added if jira_message: @@ -1251,8 +1252,9 @@ def defect_finding_review(request, fid): # Only push if the finding is not in a group if jira_issue_exists: # Determine if any automatic sync should occur + jira_instance = jira_helper.get_jira_instance(finding) push_to_jira = jira_helper.is_push_all_issues(finding) \ - or jira_helper.get_jira_instance(finding).finding_jira_sync + or (jira_instance and jira_instance.finding_jira_sync) # Add the closing note if push_to_jira and not finding_in_group: if defect_choice == "Close Finding": @@ -1543,8 +1545,9 @@ def request_finding_review(request, fid): # Only push if the finding is not in a group if jira_issue_exists: # Determine if any automatic sync should occur + jira_instance = jira_helper.get_jira_instance(finding) push_to_jira = jira_helper.is_push_all_issues(finding) \ - or jira_helper.get_jira_instance(finding).finding_jira_sync + or (jira_instance and jira_instance.finding_jira_sync) # Add the closing note if push_to_jira and not finding_in_group: jira_helper.add_comment(finding, new_note, force_push=True) @@ -1637,8 +1640,9 @@ def clear_finding_review(request, fid): # Only push if the finding is not in a group if jira_issue_exists: # Determine if any automatic sync should occur + jira_instance = jira_helper.get_jira_instance(finding) push_to_jira = jira_helper.is_push_all_issues(finding) \ - or jira_helper.get_jira_instance(finding).finding_jira_sync + or (jira_instance and jira_instance.finding_jira_sync) # Add the closing note if push_to_jira and not finding_in_group: jira_helper.add_comment(finding, new_note, force_push=True) diff --git a/dojo/finding_group/views.py b/dojo/finding_group/views.py index 7f932cc0f3d..451d4dcd720 100644 --- a/dojo/finding_group/views.py +++ b/dojo/finding_group/views.py @@ -82,7 +82,16 @@ def view_finding_group(request, fgid): if jira_issue: # See if the submitted issue was a issue key or the full URL - jira_instance = jira_helper.get_jira_project(finding_group).jira_instance + jira_project = jira_helper.get_jira_project(finding_group) + if not jira_project or not jira_project.jira_instance: + messages.add_message( + request, + messages.ERROR, + "Cannot process JIRA issue: JIRA instance is not configured or has been deleted.", + extra_tags="alert-danger", + ) + return render(request, "dojo/edit_finding_group.html", {"form": edit_finding_group_form, "finding_group": finding_group}) + jira_instance = jira_project.jira_instance jira_issue = jira_issue.removeprefix(jira_instance.url + "/browse/") if finding_group.has_jira_issue and jira_issue != jira_helper.get_jira_key(finding_group): diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index 212c976dc33..03bb801e32f 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -1,5 +1,6 @@ import base64 import logging +import time from celery import chord, group from django.conf import settings @@ -244,9 +245,14 @@ def parse_findings( """ # Attempt any preprocessing before generating findings scan = self.process_scan_file(scan) + start_time = time.perf_counter() if hasattr(parser, "get_tests"): - return self.parse_findings_dynamic_test_type(scan, parser) - return self.parse_findings_static_test_type(scan, parser) + parsed_findings = self.parse_findings_dynamic_test_type(scan, parser) + else: + parsed_findings = self.parse_findings_static_test_type(scan, parser) + elapsed_time = time.perf_counter() - start_time + logger.info(f"Parsing findings took {elapsed_time:.2f} seconds ({len(parsed_findings) if parsed_findings else 0} findings parsed)") + return parsed_findings def sync_process_findings( self, diff --git a/dojo/jira_link/helper.py b/dojo/jira_link/helper.py index b42d6e10f59..453a33cd23d 100644 --- a/dojo/jira_link/helper.py +++ b/dojo/jira_link/helper.py @@ -369,10 +369,10 @@ def get_jira_project_key(obj): def get_jira_issue_template(obj): jira_project = get_jira_project(obj) - template_dir = jira_project.issue_template_dir + template_dir = jira_project.issue_template_dir if jira_project else None if not template_dir: jira_instance = get_jira_instance(obj) - template_dir = jira_instance.issue_template_dir + template_dir = jira_instance.issue_template_dir if jira_instance else None # fallback to default as before if not template_dir: @@ -902,6 +902,10 @@ def failure_to_add_message(message: str, exception: Exception, _: Any) -> bool: jira_project = get_jira_project(obj) jira_instance = get_jira_instance(obj) + if not jira_instance: + message = f"Object {obj.id} cannot be pushed to JIRA as the JIRA instance has been deleted or is not available." + return failure_to_add_message(message, None, obj) + obj_can_be_pushed_to_jira, error_message, _error_code = can_be_pushed_to_jira(obj) if not obj_can_be_pushed_to_jira: # not sure why this check is not part of can_be_pushed_to_jira, but afraid to change it @@ -1046,10 +1050,17 @@ def failure_to_update_message(message: str, exception: Exception, obj: Any) -> b message = f"Object {obj.id} cannot be pushed to JIRA as there is no JIRA configuration for {to_str_typed(obj)}." return failure_to_update_message(message, None, obj) + if not jira_instance: + message = f"Object {obj.id} cannot be pushed to JIRA as the JIRA instance has been deleted or is not available." + return failure_to_update_message(message, None, obj) + j_issue = obj.jira_issue try: JIRAError.log_to_tempfile = False jira = get_jira_connection(jira_instance) + if not jira: + message = f"Object {obj.id} cannot be pushed to JIRA as the JIRA connection could not be established." + return failure_to_update_message(message, None, obj) issue = jira.issue(j_issue.jira_id) except Exception as e: message = f"The following jira instance could not be connected: {jira_instance} - {e}" @@ -1168,7 +1179,8 @@ def get_jira_issue_from_jira(find): return jira.issue(j_issue.jira_id) except JIRAError as e: - logger.exception("jira_meta for project: %s and url: %s meta: %s", jira_project.project_key, jira_project.jira_instance.url, json.dumps(meta, indent=4)) # this is None safe + jira_url = jira_project.jira_instance.url if (jira_project and jira_project.jira_instance) else "N/A" + logger.exception("jira_meta for project: %s and url: %s meta: %s", jira_project.project_key if jira_project else "N/A", jira_url, json.dumps(meta, indent=4)) log_jira_alert(e.text, find) return None @@ -1199,6 +1211,10 @@ def issue_from_jira_is_active(issue_from_jira): def push_status_to_jira(obj, jira_instance, jira, issue, *, save=False): + if not jira_instance: + logger.warning("Cannot push status to JIRA for %d:%s - jira_instance is None", obj.id, to_str_typed(obj)) + return False + status_list = _safely_get_obj_status_for_jira(obj) issue_closed = False updated = False @@ -1224,6 +1240,7 @@ def push_status_to_jira(obj, jira_instance, jira, issue, *, save=False): if updated and save: obj.jira_issue.jira_change = timezone.now() obj.jira_issue.save() + return updated # gets the metadata for the provided issue type in the provided jira project @@ -1300,6 +1317,8 @@ def get_issuetype_fields( def is_jira_project_valid(jira_project): + if not jira_project or not jira_project.jira_instance: + return False try: jira = get_jira_connection(jira_project) get_issuetype_fields(jira, jira_project.project_key, jira_project.jira_instance.default_issue_type) @@ -1361,7 +1380,11 @@ def close_epic(eng, push_to_jira, **kwargs): jira_project = get_jira_project(engagement) jira_instance = get_jira_instance(engagement) - if jira_project.enable_engagement_epic_mapping: + if not jira_instance: + logger.warning("JIRA close epic failed: jira_instance is None") + return False + + if jira_project and jira_project.enable_engagement_epic_mapping: if push_to_jira: try: jissue = get_jira_issue(eng) @@ -1449,7 +1472,11 @@ def add_epic(engagement, **kwargs): jira_project = get_jira_project(engagement) jira_instance = get_jira_instance(engagement) - if jira_project.enable_engagement_epic_mapping: + if not jira_instance: + logger.warning("JIRA add epic failed: jira_instance is None") + return False + + if jira_project and jira_project.enable_engagement_epic_mapping: epic_name = kwargs.get("epic_name") epic_issue_type_name = getattr(jira_project, "epic_issue_type_name", "Epic") if not epic_name: diff --git a/dojo/management/commands/reimport_unittest_scan.py b/dojo/management/commands/reimport_unittest_scan.py new file mode 100644 index 00000000000..8480962b3bb --- /dev/null +++ b/dojo/management/commands/reimport_unittest_scan.py @@ -0,0 +1,228 @@ +import json +import logging +import time +from importlib import import_module +from importlib.util import find_spec +from inspect import isclass +from pathlib import Path + +from django.core.management.base import BaseCommand, CommandError +from django.urls import reverse +from rest_framework.authtoken.models import Token +from rest_framework.test import APIClient + +from unittests.test_dashboard import User + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + help = ( + "Reimport a specific unittest scan by filename into an existing test. " + "Automatically deduces scan type from path." + ) + + def add_arguments(self, parser): + parser.add_argument( + "test_id", + type=int, + help="ID of the test to reimport into", + ) + parser.add_argument( + "scan_file", + type=str, + help="Path to scan file relative to unittests/scans/ (e.g., 'jfrog_xray_unified/very_many_vulns.json')", + ) + parser.add_argument( + "--minimum-severity", + type=str, + default="Low", + choices=["Critical", "High", "Medium", "Low", "Info"], + help="Minimum severity to import (default: Low)", + ) + parser.add_argument( + "--active", + action="store_true", + default=True, + help="Mark findings as active (default: True)", + ) + parser.add_argument( + "--verified", + action="store_true", + default=False, + help="Mark findings as verified (default: False)", + ) + parser.add_argument( + "--close-old-findings", + action="store_true", + default=False, + help="Close findings not present in the new scan (default: False)", + ) + parser.add_argument( + "--tags", + action="append", + default=[], + help=( + "Tag(s) to apply to the test (repeat --tags to add multiple). " + "Example: --tags perf --tags jfrog" + ), + ) + + def get_test_admin(self): + return User.objects.get(username="admin") + + def reimport_scan(self, payload, expected_http_status_code=201): + testuser = self.get_test_admin() + token = Token.objects.get(user=testuser) + client = APIClient() + client.credentials(HTTP_AUTHORIZATION="Token " + token.key) + + response = client.post(reverse("reimportscan-list"), payload) + if expected_http_status_code != response.status_code: + msg = f"Expected HTTP status code {expected_http_status_code}, got {response.status_code}: {response.content[:1000]}" + raise CommandError(msg) + return json.loads(response.content) + + def deduce_scan_type_from_path(self, scan_file_path): + """ + Deduce the scan type from the file path by finding the corresponding parser. + + Args: + scan_file_path: Path like 'zap/zap_sample.json' or 'jfrog_xray_unified/very_many_vulns.json' + + Returns: + tuple: (scan_type, parser_class) or raises CommandError if not found + + """ + # Extract the directory name (parser module name) + path_parts = Path(scan_file_path).parts + if len(path_parts) < 2: + msg = f"Scan file path must include directory: {scan_file_path}" + raise CommandError(msg) + + module_name = path_parts[0] + + # Try to find and load the parser module + try: + if not find_spec(f"dojo.tools.{module_name}.parser"): + msg = f"No parser module found for '{module_name}'" + raise CommandError(msg) + + module = import_module(f"dojo.tools.{module_name}.parser") + + # Find the parser class + parser_class = None + expected_class_name = module_name.replace("_", "") + "parser" + + for attribute_name in dir(module): + attribute = getattr(module, attribute_name) + if isclass(attribute) and attribute_name.lower() == expected_class_name: + parser_class = attribute + break + + if not parser_class: + msg = f"No parser class found in module '{module_name}'" + raise CommandError(msg) + + # Get the scan type from the parser + parser_instance = parser_class() + scan_types = parser_instance.get_scan_types() + + if not scan_types: + msg = f"Parser '{module_name}' has no scan types" + raise CommandError(msg) + + return scan_types[0], parser_class + + except ImportError as e: + msg = f"Failed to import parser module '{module_name}': {e}" + raise CommandError(msg) + + def reimport_unittest_scan(self, test_id, scan_file, minimum_severity, active, verified, close_old_findings, tags): + """ + Reimport a specific unittest scan file into an existing test. + + Args: + test_id: ID of the test to reimport into + scan_file: Path to scan file relative to unittests/scans/ + minimum_severity: Minimum severity level + active: Whether findings should be active + verified: Whether findings should be verified + close_old_findings: Whether to close findings not in the new scan + tags: List of tags to apply + + """ + # Validate scan file exists + scan_path = Path("unittests/scans") / scan_file + if not scan_path.exists(): + msg = f"Scan file not found: {scan_path}" + raise CommandError(msg) + + # Deduce scan type from path + scan_type, _parser_class = self.deduce_scan_type_from_path(scan_file) + + logger.info(f"Reimporting scan '{scan_file}' using scan type '{scan_type}'") + logger.info(f"Target: Test ID {test_id}") + + # Reimport the scan + with scan_path.open(encoding="utf-8") as testfile: + payload = { + "test": test_id, + "minimum_severity": minimum_severity, + "scan_type": scan_type, + "file": testfile, + "version": "1.0.1", + "active": active, + "verified": verified, + "close_old_findings": close_old_findings, + } + + if tags: + payload["tags"] = tags + + result = self.reimport_scan(payload) + + logger.info(f"Successfully reimported scan. Test ID: {result.get('test')}") + logger.info(f"Reimport summary: {result.get('scan_save_message', 'No summary available')}") + + return result + + def handle(self, *args, **options): + test_id = options["test_id"] + scan_file = options["scan_file"] + minimum_severity = options["minimum_severity"] + active = options["active"] + verified = options["verified"] + close_old_findings = options["close_old_findings"] + tags = options["tags"] + + start_time = time.time() + + try: + self.reimport_unittest_scan( + test_id=test_id, + scan_file=scan_file, + minimum_severity=minimum_severity, + active=active, + verified=verified, + close_old_findings=close_old_findings, + tags=tags, + ) + + end_time = time.time() + duration = end_time - start_time + + self.stdout.write( + self.style.SUCCESS( + f"Successfully reimported '{scan_file}' into test ID {test_id} " + f"(took {duration:.2f} seconds)", + ), + ) + + except Exception as e: + end_time = time.time() + duration = end_time - start_time + logger.exception(f"Failed to reimport scan '{scan_file}' after {duration:.2f} seconds") + msg = f"Reimport failed after {duration:.2f} seconds: {e}" + raise CommandError(msg) diff --git a/dojo/models.py b/dojo/models.py index 282a8c4d667..48c274ae006 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -2766,6 +2766,8 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru logger.debug("Start saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None") from dojo.finding import helper as finding_helper # noqa: PLC0415 circular import + is_new_finding = self.pk is None + # if not isinstance(self.date, (datetime, date)): # raise ValidationError(_("The 'date' field must be a valid date or datetime object.")) @@ -2809,7 +2811,7 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru self.set_hash_code(dedupe_option) - if self.pk is None: + if is_new_finding: # We enter here during the first call from serializers.py from dojo.utils import apply_cwe_to_template # noqa: PLC0415 circular import # No need to use the returned variable since `self` Is updated in memory @@ -2838,7 +2840,9 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru logger.debug("Saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None") super().save(*args, **kwargs) - self.found_by.add(self.test.test_type) + # Only add to found_by for newly-created findings (avoid doing this on every update) + if is_new_finding: + self.found_by.add(self.test.test_type) # only perform post processing (in celery task) if needed. this check avoids submitting 1000s of tasks to celery that will do nothing system_settings = System_Settings.objects.get() @@ -3505,11 +3509,12 @@ def set_hash_code(self, dedupe_option): # Finding.save is called once from serializers.py with dedupe_option=False because the finding is not ready yet, for example the endpoints are not built # It is then called a second time with dedupe_option defaulted to true; now we can compute the hash_code and run the deduplication elif dedupe_option: + finding_id = self.id if self.id is not None else "unsaved" if self.hash_code is not None: - deduplicationLogger.debug("Hash_code already computed for finding %i", self.id) + deduplicationLogger.debug("Hash_code already computed for finding: %s", finding_id) else: self.hash_code = self.compute_hash_code() - deduplicationLogger.debug("Hash_code computed for finding %i: %s", self.id, self.hash_code) + deduplicationLogger.debug("Hash_code computed for finding: %s: %s", finding_id, self.hash_code) class FindingAdmin(admin.ModelAdmin): diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index f13696c586b..003dcb0b3ed 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -1911,10 +1911,10 @@ def saml2_attrib_map_format(din): "BAM-": "https://jira.atlassian.com/browse/", # e.g. https://jira.atlassian.com/browse/BAM-25498 "BSERV-": "https://jira.atlassian.com/browse/", # e.g. https://jira.atlassian.com/browse/BSERV-19020 "C-": "https://hub.armosec.io/docs/", # e.g. https://hub.armosec.io/docs/c-0085 - "CISCO-SA-": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/", # e.g. https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-umbrella-tunnel-gJw5thgE "CAPEC": "https://capec.mitre.org/data/definitions/&&.html", # e.g. https://capec.mitre.org/data/definitions/157.html "CERTFR-": "https://www.cert.ssi.gouv.fr/alerte/", # e.g. https://www.cert.ssi.gouv.fr/alerte/CERTFR-2025-ALE-012" "CGA-": "https://images.chainguard.dev/security/", # e.g. https://images.chainguard.dev/security/CGA-24pq-h5fw-43v3 + "CISCO-SA-": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/", # e.g. https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-umbrella-tunnel-gJw5thgE "CONFSERVER-": "https://jira.atlassian.com/browse/", # e.g. https://jira.atlassian.com/browse/CONFSERVER-93361 "CVE-": "https://nvd.nist.gov/vuln/detail/", # e.g. https://nvd.nist.gov/vuln/detail/cve-2022-22965 "CWE": "https://cwe.mitre.org/data/definitions/&&.html", # e.g. https://cwe.mitre.org/data/definitions/79.html @@ -1931,6 +1931,7 @@ def saml2_attrib_map_format(din): "GLSA": "https://security.gentoo.org/", # e.g. https://security.gentoo.org/glsa/202409-32 "GO-": "https://pkg.go.dev/vuln/", # e.g. https://pkg.go.dev/vuln/GO-2025-3703 "GSD-": "https://cvepremium.circl.lu/vuln/", # e.g. https://cvepremium.circl.lu/vuln/gsd-2021-34715 + "ICSA-": "https://cvepremium.circl.lu/recent/", # e.g. https://cvepremium.circl.lu/vuln/ICSA-25-317-01 "JSDSERVER-": "https://jira.atlassian.com/browse/", # e.g. https://jira.atlassian.com/browse/JSDSERVER-14872 "JVNDB-": "https://jvndb.jvn.jp/en/contents/", # e.g. https://jvndb.jvn.jp/en/contents/2025/JVNDB-2025-004079.html "KB": "https://support.hcl-software.com/csm?id=kb_article&sysparm_article=", # e.g. https://support.hcl-software.com/csm?id=kb_article&sysparm_article=KB0108401 diff --git a/dojo/utils.py b/dojo/utils.py index a00ba7b48f1..931348cfa03 100644 --- a/dojo/utils.py +++ b/dojo/utils.py @@ -6,7 +6,9 @@ import mimetypes import os import pathlib +import random import re +import time from calendar import monthrange from collections.abc import Callable from datetime import date, datetime, timedelta @@ -29,6 +31,7 @@ from django.contrib.auth.signals import user_logged_in, user_logged_out, user_login_failed from django.contrib.contenttypes.models import ContentType from django.core.paginator import Paginator +from django.db import OperationalError from django.db.models import Case, Count, F, IntegerField, Q, Sum, Value, When from django.db.models.query import QuerySet from django.db.models.signals import post_save @@ -2003,22 +2006,51 @@ def __init__(self, *args, **kwargs): @dojo_async_task @app.task def delete_chunk(self, objects, **kwargs): + # Now delete all objects with retry for deadlocks + max_retries = 3 for obj in objects: - try: - obj.delete() - except AssertionError: - logger.debug("ASYNC_DELETE: object has already been deleted elsewhere. Skipping") - # The id must be None - # The object has already been deleted elsewhere - except LogEntry.MultipleObjectsReturned: - # Delete the log entrys first, then delete - LogEntry.objects.filter( - content_type=ContentType.objects.get_for_model(obj.__class__), - object_pk=str(obj.pk), - action=LogEntry.Action.DELETE, - ).delete() - # Now delete the object again - obj.delete() + retry_count = 0 + while retry_count < max_retries: + try: + obj.delete() + break # Success, exit retry loop + except OperationalError as e: + error_msg = str(e) + if "deadlock detected" in error_msg.lower(): + retry_count += 1 + if retry_count < max_retries: + # Exponential backoff with jitter + wait_time = (2 ** retry_count) + random.uniform(0, 1) # noqa: S311 + logger.warning( + f"ASYNC_DELETE: Deadlock detected deleting {self.get_object_name(obj)} {obj.pk}, " + f"retrying ({retry_count}/{max_retries}) after {wait_time:.2f}s", + ) + time.sleep(wait_time) + # Refresh object from DB before retry + obj.refresh_from_db() + else: + logger.error( + f"ASYNC_DELETE: Deadlock persisted after {max_retries} retries for {self.get_object_name(obj)} {obj.pk}: {e}", + ) + raise + else: + # Not a deadlock, re-raise + raise + except AssertionError: + logger.debug("ASYNC_DELETE: object has already been deleted elsewhere. Skipping") + # The id must be None + # The object has already been deleted elsewhere + break + except LogEntry.MultipleObjectsReturned: + # Delete the log entrys first, then delete + LogEntry.objects.filter( + content_type=ContentType.objects.get_for_model(obj.__class__), + object_pk=str(obj.pk), + action=LogEntry.Action.DELETE, + ).delete() + # Now delete the object again (no retry needed for this case) + obj.delete() + break @dojo_async_task @app.task @@ -2037,18 +2069,29 @@ def delete(self, obj, **kwargs): @app.task def crawl(self, obj, model_list, **kwargs): logger.debug("ASYNC_DELETE: Crawling " + self.get_object_name(obj) + ": " + str(obj)) + task_results = [] for model_info in model_list: model = model_info[0] model_query = model_info[1] filter_dict = {model_query: obj} # Only fetch the IDs since we will make a list of IDs in the following function call - objects_to_delete = model.objects.only("id").filter(**filter_dict) + objects_to_delete = model.objects.only("id").filter(**filter_dict).distinct().order_by("id") logger.debug("ASYNC_DELETE: Deleting " + str(len(objects_to_delete)) + " " + self.get_object_name(model) + "s in chunks") chunks = self.chunk_list(model, objects_to_delete) for chunk in chunks: logger.debug(f"deleting {len(chunk)} {self.get_object_name(model)}") - self.delete_chunk(chunk) - self.delete_chunk([obj]) + result = self.delete_chunk(chunk) + # Collect async task results to wait for them all at once + if hasattr(result, "get"): + task_results.append(result) + # Wait for all chunk deletions to complete (they run in parallel) + for task_result in task_results: + task_result.get(timeout=300) # 5 minute timeout per chunk + # Now delete the main object after all chunks are done + result = self.delete_chunk([obj]) + # Wait for final deletion to complete + if hasattr(result, "get"): + result.get(timeout=300) # 5 minute timeout logger.debug("ASYNC_DELETE: Successfully deleted " + self.get_object_name(obj) + ": " + str(obj)) def chunk_list(self, model, full_list): diff --git a/helm/defectdojo/Chart.lock b/helm/defectdojo/Chart.lock index ea0498f4989..1d693b06484 100644 --- a/helm/defectdojo/Chart.lock +++ b/helm/defectdojo/Chart.lock @@ -5,5 +5,5 @@ dependencies: - name: valkey repository: oci://registry-1.docker.io/cloudpirates version: 0.10.2 -digest: sha256:65773fc2a992a5688995a98ed396ca54de12b88fd7b124459a39961801ae62f3 -generated: "2025-11-25T20:48:39.324383+01:00" +digest: sha256:ffaad5072467386baa7b7142dde88375c007fd72f6bd7e97008107d1fc37751d +generated: "2025-12-15T10:35:03.656421-06:00" diff --git a/helm/defectdojo/Chart.yaml b/helm/defectdojo/Chart.yaml index 6050a4a021b..38568c07cfe 100644 --- a/helm/defectdojo/Chart.yaml +++ b/helm/defectdojo/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: "2.53.1" +appVersion: "2.53.2" description: A Helm chart for Kubernetes to install DefectDojo name: defectdojo -version: 1.9.1 +version: 1.9.2 icon: https://defectdojo.com/hubfs/DefectDojo_favicon.png maintainers: - name: madchap @@ -10,11 +10,11 @@ maintainers: url: https://github.com/DefectDojo/django-DefectDojo dependencies: - name: postgresql - version: ~16.7.0 + version: 16.7.27 repository: "oci://us-docker.pkg.dev/os-public-container-registry/defectdojo" condition: postgresql.enabled - name: valkey - version: ~0.10.0 + version: 0.10.2 repository: "oci://registry-1.docker.io/cloudpirates" condition: valkey.enabled # For correct syntax, check https://artifacthub.io/docs/topics/annotations/helm/ @@ -35,7 +35,7 @@ dependencies: annotations: artifacthub.io/prerelease: "false" artifacthub.io/changes: | - - kind: deprecated - description: djnago.mediaPersistentVolume.fsGroup was removed because it was replaced with django.podSecurityContext.fsGroup + - kind: fixed + description: Use non-tilde notation of versions in subcharts - kind: changed - description: Bump DefectDojo to 2.53.1 + description: Bump DefectDojo to 2.53.2 diff --git a/helm/defectdojo/README.md b/helm/defectdojo/README.md index 8016350fb8a..385a80e18ad 100644 --- a/helm/defectdojo/README.md +++ b/helm/defectdojo/README.md @@ -511,7 +511,7 @@ The HELM schema will be generated for you. # General information about chart values -![Version: 1.9.1](https://img.shields.io/badge/Version-1.9.1-informational?style=flat-square) ![AppVersion: 2.53.1](https://img.shields.io/badge/AppVersion-2.53.1-informational?style=flat-square) +![Version: 1.9.2](https://img.shields.io/badge/Version-1.9.2-informational?style=flat-square) ![AppVersion: 2.53.2](https://img.shields.io/badge/AppVersion-2.53.2-informational?style=flat-square) A Helm chart for Kubernetes to install DefectDojo @@ -525,8 +525,8 @@ A Helm chart for Kubernetes to install DefectDojo | Repository | Name | Version | |------------|------|---------| -| oci://registry-1.docker.io/cloudpirates | valkey | ~0.10.0 | -| oci://us-docker.pkg.dev/os-public-container-registry/defectdojo | postgresql | ~16.7.0 | +| oci://registry-1.docker.io/cloudpirates | valkey | 0.10.2 | +| oci://us-docker.pkg.dev/os-public-container-registry/defectdojo | postgresql | 16.7.27 | ## Values diff --git a/readme-docs/RELEASING.md b/readme-docs/RELEASING.md index 55e1e7d3eb9..c8fc6f89a56 100644 --- a/readme-docs/RELEASING.md +++ b/readme-docs/RELEASING.md @@ -4,8 +4,10 @@ We have two types of releases: -1. **Feature releases** - These are created from the `dev` branch, via a new release branch, i.e. `release/x.y.z`. -2. **Bugfix releases** - These are created from the `bugfix` branch, via a new release branch, i.e. `release/x.y.z`. +1. **Feature releases** - These are monthly releases created from the `dev` branch, via a new release branch, i.e. `release/x.y.z`. +2. **Bugfix releases** - These are weekly created from the `bugfix` branch, via a new release branch, i.e. `release/x.y.z`. + +Dependency updates and security patches go into the monthly releases. Urgent crirtical security patches will go into the bugfix releases. The release process will then: diff --git a/unittests/scans/rubocop/many_vulns.json b/unittests/scans/rubocop/many_vulns.json index 408008c7b06..8cc3435a7dd 100644 --- a/unittests/scans/rubocop/many_vulns.json +++ b/unittests/scans/rubocop/many_vulns.json @@ -1 +1,133 @@ -{"metadata":{"rubocop_version":"1.24.1","ruby_engine":"ruby","ruby_version":"3.0.3","ruby_patchlevel":"157","ruby_platform":"x86_64-linux"},"files":[{"path":"fake_app_unsecure/app/main.rb","offenses":[{"severity":"convention","message":"`File.read` is safer than `IO.read`.","cop_name":"Security/IoMethods","corrected":false,"correctable":true,"location":{"start_line":12,"start_column":1,"last_line":12,"last_column":13,"length":13,"line":12,"column":1}},{"severity":"convention","message":"`File.read` is safer than `IO.read`.","cop_name":"Security/IoMethods","corrected":false,"correctable":true,"location":{"start_line":13,"start_column":1,"last_line":13,"last_column":15,"length":15,"line":13,"column":1}},{"severity":"convention","message":"Prefer `JSON.parse` over `JSON.load`.","cop_name":"Security/JSONLoad","corrected":false,"correctable":true,"location":{"start_line":19,"start_column":6,"last_line":19,"last_column":9,"length":4,"line":19,"column":6}},{"severity":"convention","message":"Prefer using `YAML.safe_load` over `YAML.load`.","cop_name":"Security/YAMLLoad","corrected":false,"correctable":true,"location":{"start_line":22,"start_column":6,"last_line":22,"last_column":9,"length":4,"line":22,"column":6}},{"severity":"convention","message":"The use of `Kernel#open` is a serious security risk.","cop_name":"Security/Open","corrected":false,"correctable":false,"location":{"start_line":25,"start_column":1,"last_line":25,"last_column":4,"length":4,"line":25,"column":1}},{"severity":"convention","message":"The use of `Kernel#open` is a serious security risk.","cop_name":"Security/Open","corrected":false,"correctable":false,"location":{"start_line":26,"start_column":1,"last_line":26,"last_column":4,"length":4,"line":26,"column":1}},{"severity":"convention","message":"The use of `URI.open` is a serious security risk.","cop_name":"Security/Open","corrected":false,"correctable":false,"location":{"start_line":27,"start_column":5,"last_line":27,"last_column":8,"length":4,"line":27,"column":5}}]}],"summary":{"offense_count":7,"target_file_count":1,"inspected_file_count":1}} \ No newline at end of file +{ + "metadata": { + "rubocop_version": "1.24.1", + "ruby_engine": "ruby", + "ruby_version": "3.0.3", + "ruby_patchlevel": "157", + "ruby_platform": "x86_64-linux" + }, + "files": [ + { + "path": "fake_app_unsecure/app/main.rb", + "offenses": [ + { + "severity": "convention", + "message": "`File.read` is safer than `IO.read`.", + "cop_name": "Security/IoMethods", + "corrected": false, + "correctable": true, + "location": { + "start_line": 12, + "start_column": 1, + "last_line": 12, + "last_column": 13, + "length": 13, + "line": 12, + "column": 1 + } + }, + { + "severity": "convention", + "message": "`File.read` is safer than `IO.read`.", + "cop_name": "Security/IoMethods", + "corrected": false, + "correctable": true, + "location": { + "start_line": 13, + "start_column": 1, + "last_line": 13, + "last_column": 15, + "length": 15, + "line": 13, + "column": 1 + } + }, + { + "severity": "convention", + "message": "Prefer `JSON.parse` over `JSON.load`.", + "cop_name": "Security/JSONLoad", + "corrected": false, + "correctable": true, + "location": { + "start_line": 19, + "start_column": 6, + "last_line": 19, + "last_column": 9, + "length": 4, + "line": 19, + "column": 6 + } + }, + { + "severity": "convention", + "message": "Prefer using `YAML.safe_load` over `YAML.load`.", + "cop_name": "Security/YAMLLoad", + "corrected": false, + "correctable": true, + "location": { + "start_line": 22, + "start_column": 6, + "last_line": 22, + "last_column": 9, + "length": 4, + "line": 22, + "column": 6 + } + }, + { + "severity": "convention", + "message": "The use of `Kernel#open` is a serious security risk.", + "cop_name": "Security/Open", + "corrected": false, + "correctable": false, + "location": { + "start_line": 25, + "start_column": 1, + "last_line": 25, + "last_column": 4, + "length": 4, + "line": 25, + "column": 1 + } + }, + { + "severity": "convention", + "message": "The use of `Kernel#open` is a serious security risk.", + "cop_name": "Security/Open", + "corrected": false, + "correctable": false, + "location": { + "start_line": 26, + "start_column": 1, + "last_line": 26, + "last_column": 4, + "length": 4, + "line": 26, + "column": 1 + } + }, + { + "severity": "convention", + "message": "The use of `URI.open` is a serious security risk.", + "cop_name": "Security/Open", + "corrected": false, + "correctable": false, + "location": { + "start_line": 27, + "start_column": 5, + "last_line": 27, + "last_column": 8, + "length": 4, + "line": 27, + "column": 5 + } + } + ] + } + ], + "summary": { + "offense_count": 7, + "target_file_count": 1, + "inspected_file_count": 1 + } +} \ No newline at end of file diff --git a/unittests/test_import_reimport.py b/unittests/test_import_reimport.py index 2f71c720e02..f51a462045a 100644 --- a/unittests/test_import_reimport.py +++ b/unittests/test_import_reimport.py @@ -11,7 +11,7 @@ from django.urls import reverse from django.utils import timezone -from dojo.models import Finding, Test, Test_Type, User +from dojo.models import Engagement, Finding, Product, Product_Type, Test, Test_Type, User from .dojo_test_case import DojoAPITestCase, get_unit_tests_scans_path from .test_utils import assertTestImportModelsCreated @@ -2033,6 +2033,118 @@ def test_reimport_set_scan_date_parser_sets_date(self): date = findings["results"][0]["date"] self.assertEqual(date, "2006-12-26") + @override_settings( + IMPORT_REIMPORT_DEDUPE_BATCH_SIZE=50, + IMPORT_REIMPORT_MATCH_BATCH_SIZE=50, + ) + def test_reimport_with_internal_duplicates_same_batch(self): + """ + Test that duplicates within the same scan report are detected during reimport + even when they're processed in the same batch. + + This test mimics the integration test test_import_path_tests in tests/dedupe_test.py, + which tests the same scenario but uses Selenium UI testing. + + BACKGROUND: + When reimporting a scan report, the system uses batch processing to optimize + database queries. Findings are processed in batches (default: 1000), and for + each batch, candidate matches are fetched from the database once. However, this + creates a bug: if multiple duplicate findings exist within the same scan report + and they're processed in the same batch, they cannot match each other because + the candidate dictionary is built before any findings in the batch are saved. + + THE BUG: + - Finding #1 in batch: No matches found β†’ Creates new finding + - Finding #2 in batch: No matches found (Finding #1 not in DB yet) β†’ Creates duplicate + - Finding #3 in batch: No matches found (Findings #1 and #2 not in DB yet) β†’ Creates duplicate + Result: 3 findings created instead of 1 unique finding + + THIS TEST: + This test reproduces the bug by: + 1. Creating an empty test (no existing findings to match against) + 2. Reimporting dedupe_path_1.json which contains 3 duplicate findings: + - All have same file_path: "/opt/appsecpipeline/source/dojo/cred/views.py" + - All have same line_number: 524 + - All have same test_id: "B110" + - All should have the same hash_code and match each other + 3. Using small batch size (50) to ensure all 3 duplicates are in the same batch + 4. Verifying that only 1 finding is created (not 3) + + EXPECTED BEHAVIOR: + - Total findings: 1 (the 3 duplicates should match to 1 unique finding) + - Active findings: 1 + - Duplicate findings: 0 (duplicates within report should match, not be marked as duplicates) + + CURRENT BUGGY BEHAVIOR: + - Total findings: 3 (all duplicates are created because they can't match each other) + - Active findings: 1 (first one) + - Duplicate findings: 2 (second and third are marked as duplicates of first) + + This test should fail with the current buggy implementation and pass after + implementing intra-batch duplicate tracking. + """ + # Use the dedupe_path_1.json file from integration tests (has 3 duplicate findings) + # Note: This file is in tests/dedupe_scans, not unittests/scans + dedupe_path_file = Path(__file__).parent.parent / "tests" / "dedupe_scans" / "dedupe_path_1.json" + + # Create engagement and test + product_type, _ = Product_Type.objects.get_or_create(name="PT Bandit Internal Dupes") + product, _ = Product.objects.get_or_create(name="P Bandit Internal Dupes", prod_type=product_type) + engagement = Engagement.objects.create( + name="E Bandit Internal Dupes", + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + engagement.deduplication_on_engagement = True + engagement.save() + + # Create an empty test first + # Get or create the test type + test_type, _ = Test_Type.objects.get_or_create(name="Bandit Scan") + test = Test.objects.create( + engagement=engagement, + test_type=test_type, + title="Path Test 1", + target_start=timezone.now(), + target_end=timezone.now(), + ) + test_id = test.id + + # Now reimport the file with duplicates to the empty test + # This tests reimport batch matching where duplicates within the same report need to match each other + reimport_result = self.reimport_scan_with_params( + test_id, + dedupe_path_file, + scan_type="Bandit Scan", + ) + + test_id = reimport_result["test"] + + # Verify findings count + # dedupe_path_1.json has 3 findings that are duplicates (same file_path, line_number, test_id) + # They should all match each other, resulting in 1 unique finding + # However, if intra-batch matching fails, we'll get 3 findings created instead of 1 + total_findings = Finding.objects.filter(test_id=test_id).count() + active_findings = Finding.objects.filter(test_id=test_id, active=True, duplicate=False).count() + duplicate_findings = Finding.objects.filter(test_id=test_id, duplicate=True).count() + + # Expected: 1 active finding (the 3 duplicates should match each other) + # If intra-batch matching fails, we'll get 3 findings instead + with self.subTest(metric="total_findings"): + self.assertEqual(total_findings, 1, + f"Expected 1 total finding (3 duplicates should match to 1), got {total_findings}. " + f"If this is 3, intra-batch duplicates weren't detected.") + + with self.subTest(metric="active_findings"): + self.assertEqual(active_findings, 1, + f"Expected 1 active finding, got {active_findings}. " + f"If this fails, intra-batch duplicates weren't detected.") + + with self.subTest(metric="duplicate_findings"): + self.assertEqual(duplicate_findings, 0, + f"Expected 0 duplicate findings (duplicates within report should match), got {duplicate_findings}") + class ImportReimportTestUI(DojoAPITestCase, ImportReimportMixin): fixtures = ["dojo_testdata.json"] diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 9da777ccecc..83058b29f10 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -217,11 +217,11 @@ def test_import_reimport_reimport_performance_no_async(self): testuser.usercontactinfo.block_execution = True testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=345, + expected_num_queries1=346, expected_num_async_tasks1=6, - expected_num_queries2=293, + expected_num_queries2=294, expected_num_async_tasks2=17, - expected_num_queries3=180, + expected_num_queries3=181, expected_num_async_tasks3=16, ) @@ -239,11 +239,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=311, + expected_num_queries1=312, expected_num_async_tasks1=6, - expected_num_queries2=286, + expected_num_queries2=287, expected_num_async_tasks2=17, - expected_num_queries3=175, + expected_num_queries3=176, expected_num_async_tasks3=16, ) @@ -265,11 +265,11 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=347, + expected_num_queries1=348, expected_num_async_tasks1=8, - expected_num_queries2=295, + expected_num_queries2=296, expected_num_async_tasks2=19, - expected_num_queries3=182, + expected_num_queries3=183, expected_num_async_tasks3=18, ) @@ -288,11 +288,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=313, + expected_num_queries1=314, expected_num_async_tasks1=8, - expected_num_queries2=288, + expected_num_queries2=289, expected_num_async_tasks2=19, - expected_num_queries3=177, + expected_num_queries3=178, expected_num_async_tasks3=18, ) @@ -449,9 +449,9 @@ def test_deduplication_performance_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=316, + expected_num_queries1=317, expected_num_async_tasks1=7, - expected_num_queries2=287, + expected_num_queries2=282, expected_num_async_tasks2=7, ) @@ -469,8 +469,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=280, + expected_num_queries1=281, expected_num_async_tasks1=7, - expected_num_queries2=250, + expected_num_queries2=245, expected_num_async_tasks2=7, )