Merge branch 'dev' into merge_mobsf

Author: manuel-sommer

.github/pull_request_template.md

@@ -26,7 +26,7 @@ This checklist is for your information.
 - [ ] Bugfixes should be submitted against the `bugfix` branch.
 - [ ] Give a meaningful name to your PR, as it may end up being used in the release notes.
 - [ ] Your code is flake8 compliant.
-- [ ] Your code is python 3.12 compliant.
+- [ ] Your code is python 3.13 compliant.
 - [ ] If this is a new feature and not a bug fix, you've included the proper documentation in the docs at https://github.com/DefectDojo/django-DefectDojo/tree/dev/docs as part of this PR.
 - [ ] Model changes must include the necessary migrations in the dojo/db_migrations folder.
 - [ ] Add applicable tests to the unit tests.

.github/workflows/close-stale.yml

@@ -15,13 +15,24 @@ jobs:
   close-stale:
     runs-on: ubuntu-latest
     steps:
+      - name: Close issues and PRs that are pending closure
+        uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0
+        with:
+          # Disable automatic stale marking - only close manually labeled items
+          days-before-stale: -1
+          days-before-close: 7
+          stale-issue-label: 'pending-closure'
+          stale-pr-label: 'pending-closure'
+          close-issue-message: 'This issue has been automatically closed because it was manually labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'
+          close-pr-message: 'This PR has been automatically closed because it was manually labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'
+
       - name: Close stale issues and PRs
-        uses: actions/stale@3a9db7e6a41a89f618792c92c0e97cc736e1b13f # v10.0.0
+        uses: actions/stale@5f858e3efba33a5ca4407a664cc011ad407f2008 # v10.1.0
         with:
           # Disable automatic stale marking - only close manually labeled items
           days-before-stale: -1
           days-before-close: 7
           stale-issue-label: 'stale'
           stale-pr-label: 'stale'
-          close-issue-message: 'This issue has been automatically closed because it was manually labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'
-          close-pr-message: 'This PR has been automatically closed because it was manually labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'
+          close-issue-message: 'This issue has been automatically closed because it was labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'
+          close-pr-message: 'This PR has been automatically closed because it was labeled as stale. If you believe this was closed in error, please reopen it and remove the stale label.'

.github/workflows/gh-pages.yml

@@ -21,10 +21,10 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
-          node-version: '22.19.0'
+          node-version: '22.20.0'
 
       - name: Cache dependencies
-        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

.github/workflows/integration-tests.yml

@@ -2,12 +2,18 @@ name: Integration tests
 
 on:
   workflow_call:
+    inputs:
+        auditlog_type:
+          type: string
+          default: "django-auditlog"
 
 jobs:
   integration_tests:
     # run tests with docker compose
     name: User Interface Tests
     runs-on: ubuntu-latest
+    env:
+      AUDITLOG_TYPE: ${{ inputs.auditlog_type }}
     strategy:
       matrix:
         test-case: [

.github/workflows/k8s-tests.yml

@@ -5,15 +5,6 @@ on:
 
 env:
   DD_HOSTNAME: defectdojo.default.minikube.local
-  HELM_REDIS_BROKER_SETTINGS: " \
-    --set redis.enabled=true \
-    --set celery.broker=redis \
-    --set createRedisSecret=true \
-    "
-  HELM_PG_DATABASE_SETTINGS: " \
-    --set postgresql.enabled=true \
-    --set createPostgresqlSecret=true \
-   "
 jobs:
   setting_minikube_cluster:
     name: Kubernetes Deployment
@@ -25,9 +16,7 @@ jobs:
           # databases, broker and k8s are independent, so we don't need to test each combination
           # lastest k8s version (https://kubernetes.io/releases/) and oldest supported version from aws
           # are tested (https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html#available-versions)
-          - databases: pgsql
-            brokers: redis
-            k8s: 'v1.33.4'
+          - k8s: 'v1.34.0'
             os: debian
     steps:
       - name: Checkout
@@ -36,7 +25,7 @@ jobs:
       - name: Setup Minikube
         uses: manusa/actions-setup-minikube@b589f2d61bf96695c546929c72b38563e856059d # v2.14.0
         with:
-          minikube version: 'v1.33.1'
+          minikube version: 'v1.37.0'
           kubernetes version: ${{ matrix.k8s }}
           driver: docker
           start args: '--addons=ingress --cni calico'
@@ -68,12 +57,6 @@ jobs:
              helm dependency list ./helm/defectdojo
              helm dependency update ./helm/defectdojo
 
-      - name: Set confings into Outputs
-        id: set
-        run: |-
-              echo "pgsql=${{ env.HELM_PG_DATABASE_SETTINGS }}" >> $GITHUB_ENV
-              echo "redis=${{ env.HELM_REDIS_BROKER_SETTINGS }}" >> $GITHUB_ENV
-
       - name: Deploying Django application with ${{ matrix.databases }} ${{ matrix.brokers }}
         timeout-minutes: 15
         run: |-
@@ -86,8 +69,10 @@ jobs:
                --set django.ingress.enabled=true \
                --set imagePullPolicy=Never \
                --set initializer.keepSeconds="-1" \
-               ${{ env[matrix.databases] }} \
-               ${{ env[matrix.brokers] }} \
+               --set redis.enabled=true \
+               --set createRedisSecret=true \
+               --set postgresql.enabled=true \
+               --set createPostgresqlSecret=true \
                --set createSecret=true
 
       - name: Check deployment status
@@ -121,12 +106,15 @@ jobs:
              to_complete "condition=ready" pod "defectdojo.org/component=django"
              echo "Pods up and ready to rumbole"
              kubectl get pods
+
+      - name: Test login page
+        timeout-minutes: 10
+        run: |-
              RETRY=0
              while :
                do
                DJANGO_IP=$(kubectl get svc defectdojo-django -o jsonpath='{.spec.clusterIP}')
                OUT=$(kubectl run  curl --quiet=true --image=curlimages/curl:8.15.0 \
-                  --overrides='{ "apiVersion": "v1" }' \
                   --restart=Never -i --rm -- \
                     --silent \
                     --max-time 20 \
@@ -144,7 +132,7 @@ jobs:
                     echo "ERROR: cannot display login screen; got HTTP code $CR"
                     exit 1
                   else
-                    ((RETRY++))
+                    RETRY=$((RETRY+1))
                     echo "Attempt $RETRY to get login page"
                     sleep 5
                   fi
@@ -153,26 +141,48 @@ jobs:
                 break
                fi
              done
+
+      - name: Test API auth call
+        timeout-minutes: 10
+        run: |-
              ADMIN_PASS=$(kubectl get secret/defectdojo -o jsonpath='{.data.DD_ADMIN_PASSWORD}' | base64 -d)
              echo "Simple API check"
              DJANGO_IP=$(kubectl get svc defectdojo-django -o jsonpath='{.spec.clusterIP}')
-             CR=$(kubectl run  curl --quiet=true --image=curlimages/curl:8.15.0 \
-               --overrides='{ "apiVersion": "v1" }' \
-               --restart=Never -i --rm -- \
-                 --silent \
-                 --max-time 20 \
-                 --header "Host: $DD_HOSTNAME" \
-                 --data-raw "username=admin&password=$ADMIN_PASS" \
-                 --output /dev/null \
-                 --write-out "%{http_code}\n" \
-                 "http://${DJANGO_IP}/api/v2/api-token-auth/")
-             echo $CR
-             if [[ $CR -ne 200 ]]; then
-              echo "ERROR: login is not possible; got HTTP code $CR"
-              exit 1
-             else
-              echo "Result received"
-             fi
+             RETRY=0
+             while :
+               do
+                  OUT=$(kubectl run  curl --quiet=true --image=curlimages/curl:8.15.0 \
+                    --restart=Never -i --rm -- \
+                      --dump-header - \
+                      --no-progress-meter \
+                      --max-time 20 \
+                      --header "Host: $DD_HOSTNAME" \
+                      --data-raw "username=admin&password=$ADMIN_PASS" \
+                      "http://${DJANGO_IP}/api/v2/api-token-auth/")
+                  CR=$(echo $OUT | egrep "^HTTP" | cut -d' ' -f2)
+                  echo "Return code $CR"
+                  if [[ $CR -ne 200 ]]; then
+                    echo "Retry: $RETRY"
+                    if [[ $RETRY -gt 2 ]]; then
+                      kubectl get pods
+                      echo $(kubectl logs --tail=30 -l defectdojo.org/component=django -c uwsgi)
+                      echo "ERROR: cannot perform API login; got HTTP code $CR; Full response:"
+                      echo $OUT
+                      exit 1
+                    else
+                      RETRY=$((RETRY+1))
+                      echo "Attempt $RETRY to perform API login"
+                      sleep 5
+                    fi
+                  else
+                    echo "Result received"
+                    break
+                  fi
+              done
+
+      - name: Check of logs
+        timeout-minutes: 10
+        run: |-
              echo "Final Check of components"
              errors=$(kubectl get pods | grep Error | awk '{print $1}')
              if [[ ! -z $errors ]]; then
@@ -185,3 +195,11 @@ jobs:
              else
                echo "DD K8S successfully deployed"
              fi
+
+      - name: Failed Logs
+        if: failure()
+        run: |-
+             echo "ERROR: Here are logs from deployment/defectdojo-django containers:"
+             kubectl logs deployment/defectdojo-django --all-pods=true --all-containers=true --tail=100
+             echo "And all pod status one more time"
+             kubectl get pods

.github/workflows/release-1-create-pr.yml

@@ -80,13 +80,23 @@ jobs:
               sed -ri "0,/version/s/version: \S+/$NEW_CHART_VERSION/" helm/defectdojo/Chart.yaml
           fi
 
+      - name: Update values in HELM chart
+        run: |
+          yq -i '.annotations."artifacthub.io/prerelease" = "false"' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/changes" += "- kind: changed\n  description: Bump DefectDojo to ${{ inputs.release_number }}\n"' helm/defectdojo/Chart.yaml
+
       - name: Check version numbers
         run: |
           grep -H version dojo/__init__.py
           grep -H version components/package.json
           grep -H appVersion helm/defectdojo/Chart.yaml
           grep -H version helm/defectdojo/Chart.yaml
 
+      - name: Run helm-docs
+        uses: losisin/helm-docs-github-action@a57fae5676e4c55a228ea654a1bcaec8dd3cf5b5 # v1.6.2
+        with:
+          chart-search-root: "helm/defectdojo"
+
       - name: Push version changes
         uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1
         with:

.github/workflows/release-3-master-into-dev.yml

@@ -74,6 +74,17 @@ jobs:
           git add docs/content/en/open_source/upgrading/$minorv.md
         if: endsWith(inputs.release_number_new, '.0') && endsWith(inputs.release_number_dev, '.0-dev')
 
+      - name: Update values in HELM chart
+        run: |
+          yq -i '.annotations = {}' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/prerelease" = "true"' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/changes" = ""' helm/defectdojo/Chart.yaml
+
+      - name: Run helm-docs
+        uses: losisin/helm-docs-github-action@a57fae5676e4c55a228ea654a1bcaec8dd3cf5b5 # v1.6.2
+        with:
+          chart-search-root: "helm/defectdojo"
+
       - name: Push version changes
         uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1
         with:
@@ -139,6 +150,17 @@ jobs:
           grep appVersion helm/defectdojo/Chart.yaml
           grep version components/package.json
 
+      - name: Update values in HELM chart
+        run: |
+          yq -i '.annotations = {}' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/prerelease" = "true"' helm/defectdojo/Chart.yaml
+          yq -i '.annotations."artifacthub.io/changes" = ""' helm/defectdojo/Chart.yaml
+
+      - name: Run helm-docs
+        uses: losisin/helm-docs-github-action@a57fae5676e4c55a228ea654a1bcaec8dd3cf5b5 # v1.6.2
+        with:
+          chart-search-root: "helm/defectdojo"
+
       - name: Push version changes
         uses: stefanzweifel/git-auto-commit-action@778341af668090896ca464160c2def5d1d1a3eb0 # v6.0.1
         with:

.github/workflows/release-x-manual-docker-containers.yml

@@ -52,7 +52,7 @@ jobs:
         run: echo "DOCKER_ORG=$(echo ${GITHUB_REPOSITORY%%/*} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
 
       - name: Login to DockerHub
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/release-x-manual-helm-chart.yml

@@ -87,7 +87,7 @@ jobs:
           echo "chart_version=$(ls build | cut -d '-' -f 2,3 | sed 's|\.tgz||')" >> $GITHUB_ENV
 
       - name: Create release ${{ inputs.release_number }}
-        uses: softprops/action-gh-release@6cbd405e2c4e67a21c47fa9e383d020e4e28b836 # v2.3.3
+        uses: softprops/action-gh-release@aec2ec56f94eb8180ceec724245f64ef008b89f5 # v2.4.0
         with:
           name: '${{ inputs.release_number }} 🌈'
           tag_name: ${{ inputs.release_number }}

.github/workflows/release-x-manual-merge-container-digests.yml

@@ -48,7 +48,7 @@ jobs:
         merge-multiple: true
 
     - name: Login to DockerHub
-      uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
+      uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}