From 32fa4a6b1de1d8c344cf12521d70ad336618bade Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 11:34:58 +0200
Subject: [PATCH 01/19] separate sonar from unit tests

---
 .github/workflows/CI-unittests.yml | 67 +++++++++++++++++++++++-------
 1 file changed, 53 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index b7f86e0be5..8abc96dcca 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -1,6 +1,6 @@
 ---
 name: CI-unittests
-# Unit tests (includes CTAO-SonarQube)
+# Unit tests and CTAO-SonarQube
 
 on:
   workflow_dispatch:
@@ -72,7 +72,7 @@ jobs:
 
           - os: ubuntu-latest
             python-version: "3.13"
-            extra-args: ["sonarqube", "random-order"]
+            extra-args: ["random-order"]
 
           - os: ubuntu-latest
             python-version: "3.14"
@@ -115,27 +115,66 @@ jobs:
           pytest --durations=10 --color=yes -n 4 --dist loadscope \
             --cov=simtools --cov-report=xml --retries 2 --retry-delay 5
 
+      - name: Upload coverage report
+        if: matrix.python-version == '3.13'
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-python-3.13
+          path: coverage.xml
+          if-no-files-found: error
+          retention-days: 3
+
+      - name: Random order
+        if: github.event_name == 'schedule' && contains(matrix.extra-args, 'random-order')
+        shell: bash -l {0}
+        env:
+          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
+          SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}
+        run: |
+          pytest --color=yes -n 4 --dist loadscope --count 5 --random-order \
+           --retries 2 --retry-delay 5
+
+  sonarqube:
+    needs: unit_tests
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Download coverage report
+        uses: actions/download-artifact@v8
+        with:
+          name: coverage-python-3.13
+          path: .
+
+      - name: Check Sonar scanner mirror configuration
+        env:
+          SONAR_SCANNER_BINARIES_URL: ${{ vars.SONAR_SCANNER_BINARIES_URL }}
+        run: |
+          if [[ -z "${SONAR_SCANNER_BINARIES_URL}" ]]; then
+            echo "Set the SONAR_SCANNER_BINARIES_URL repository or organization variable to the mirrored sonar-scanner-cli Distribution URL."
+            exit 1
+          fi
+
       # CTAO-DPPS-SonarQube
       - uses: SonarSource/sonarqube-scan-action@v8.1.0
-        if: contains(matrix.extra-args, 'sonarqube')
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
+          scannerVersion: 8.1.0.6389
+          scannerBinariesUrl: ${{ vars.SONAR_SCANNER_BINARIES_URL }}
           args: >
             -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14
             -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de
             -Dsonar.qualitygate.wait=true
             -Dsonar.python.coverage.reportPaths=coverage.xml
-            -Dsonar.python.version=${{ matrix.python-version }}
+            -Dsonar.python.version=3.13
             -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py
             -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**
-
-      - name: Random order
-        if: github.event_name == 'schedule' && contains(matrix.extra-args, 'random-order')
-        shell: bash -l {0}
-        env:
-          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
-          SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}
-        run: |
-          pytest --color=yes -n 4 --dist loadscope --count 5 --random-order \
-           --retries 2 --retry-delay 5

From aea4aab3b71a0c3d3a05a236f04e494303d63560 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 11:44:54 +0200
Subject: [PATCH 02/19] remove mirror

---
 .github/workflows/CI-unittests.yml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 8abc96dcca..3be2139fdf 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -154,22 +154,12 @@ jobs:
           name: coverage-python-3.13
           path: .
 
-      - name: Check Sonar scanner mirror configuration
-        env:
-          SONAR_SCANNER_BINARIES_URL: ${{ vars.SONAR_SCANNER_BINARIES_URL }}
-        run: |
-          if [[ -z "${SONAR_SCANNER_BINARIES_URL}" ]]; then
-            echo "Set the SONAR_SCANNER_BINARIES_URL repository or organization variable to the mirrored sonar-scanner-cli Distribution URL."
-            exit 1
-          fi
-
       # CTAO-DPPS-SonarQube
       - uses: SonarSource/sonarqube-scan-action@v8.1.0
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
           scannerVersion: 8.1.0.6389
-          scannerBinariesUrl: ${{ vars.SONAR_SCANNER_BINARIES_URL }}
           args: >
             -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14
             -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de

From 624ab47bd018151a0d10e5633a44fbcaf6d65189 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 11:54:27 +0200
Subject: [PATCH 03/19] Fallback for simulation models

---
 .github/workflows/CI-integrationtests.yml | 47 +++++++++++++++++++++--
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index f9a4df6d02..b6684437df 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -103,6 +103,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Determine simulation model branch
+        id: simulation_model_branch
         env:
           HEAD_REF: ${{ github.head_ref }}
           REF_NAME: ${{ github.ref_name }}
@@ -126,17 +127,55 @@ jobs:
             BRANCH="$SIMTOOLS_DB_SIMULATION_MODEL_VERSION"
           fi
           echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV"
+          echo "branch=$BRANCH" >> "$GITHUB_OUTPUT"
+
+      - name: Restore cached simulation models repository
+        if: github.event_name != 'schedule'
+        id: restore_simulation_models
+        uses: actions/cache/restore@v4
+        with:
+          path: simulation-models
+          key: simulation-models-${{ steps.simulation_model_branch.outputs.branch }}-${{ github.run_id }}
+          restore-keys: |
+            simulation-models-${{ steps.simulation_model_branch.outputs.branch }}-
 
       - name: Clone simulation models repository
         if: github.event_name != 'schedule'
         run: |
+          CLONE_DIR="simulation-models-clone"
+          CACHE_DIR="simulation-models"
+          rm -rf "$CLONE_DIR"
+
           for attempt in 1 2 3; do
-            git clone --depth 1 --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" "$SIM_MODELS_REPO" "simulation-models" && break
-            echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
-            rm -rf simulation-models
-            sleep 120
+            if timeout 180 git clone --depth 1 --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" "$SIM_MODELS_REPO" "$CLONE_DIR"; then
+              rm -rf "$CACHE_DIR"
+              mv "$CLONE_DIR" "$CACHE_DIR"
+              echo "Successfully cloned simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
+              exit 0
+            fi
+
+            rm -rf "$CLONE_DIR"
+            if [[ "$attempt" -lt 3 ]]; then
+              echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
+              sleep 120
+            fi
           done
 
+          if [[ -d "$CACHE_DIR" ]]; then
+            echo "GitLab clone failed; using cached simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
+            exit 0
+          fi
+
+          echo "Failed to clone simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH and no cache is available."
+          exit 1
+
+      - name: Save cached simulation models repository
+        if: github.event_name != 'schedule' && steps.restore_simulation_models.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: simulation-models
+          key: simulation-models-${{ steps.simulation_model_branch.outputs.branch }}-${{ github.run_id }}
+
       - name: Upload simulation models repository
         if: github.event_name != 'schedule'
         uses: actions/upload-artifact@v7

From 91dc94b638022929921ee3d1a803d55e339d849a Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 11:58:38 +0200
Subject: [PATCH 04/19] cached version of interaction tables

---
 .github/workflows/CI-integrationtests.yml | 49 ++++++++++++++++++++---
 1 file changed, 43 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index b6684437df..20f222069b 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -68,18 +68,55 @@ jobs:
       CORSIKA_TABLES: "v0.1.0"
 
     steps:
+      - name: Restore cached CORSIKA interaction tables
+        id: restore_interaction_tables
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.IT_NAME }}
+          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
+          restore-keys: |
+            corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-
+
       - name: Clone CORSIKA interaction tables (exclude QGSJet tables)
         run: |
           REPO_URL=https://gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/${IT_NAME}
           export GIT_LFS_SKIP_SMUDGE=1  # disable automatic downloading of all LFS files
+
+          CLONE_DIR="${IT_NAME}-clone"
+          CACHE_DIR="$IT_NAME"
+          rm -rf "$CLONE_DIR"
+
           for attempt in 1 2 3; do
-            git clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$IT_NAME" && break
-            echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
-            rm -rf "$IT_NAME"
-            sleep 120
+            if timeout 180 git clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$CLONE_DIR"; then
+              git lfs install
+              (cd "$CLONE_DIR" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III")
+              rm -rf "$CACHE_DIR"
+              mv "$CLONE_DIR" "$CACHE_DIR"
+              echo "Successfully cloned CORSIKA interaction tables $CORSIKA_TABLES."
+              exit 0
+            fi
+
+            rm -rf "$CLONE_DIR"
+            if [[ "$attempt" -lt 3 ]]; then
+              echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
+              sleep 120
+            fi
           done
-          git lfs install
-          (cd "$IT_NAME" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III")
+
+          if [[ -d "$CACHE_DIR" ]]; then
+            echo "GitLab clone failed; using cached CORSIKA interaction tables $CORSIKA_TABLES."
+            exit 0
+          fi
+
+          echo "Failed to clone CORSIKA interaction tables $CORSIKA_TABLES and no cache is available."
+          exit 1
+
+      - name: Save cached CORSIKA interaction tables
+        if: steps.restore_interaction_tables.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.IT_NAME }}
+          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
 
       - name: Upload CORSIKA interaction tables
         uses: actions/upload-artifact@v7

From 926abc2b271d0eeb19242bcbce886f2499e760b6 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:01:22 +0200
Subject: [PATCH 05/19] Update cache action

---
 .github/workflows/CI-integrationtests.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index 20f222069b..a285e06b0a 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -70,7 +70,7 @@ jobs:
     steps:
       - name: Restore cached CORSIKA interaction tables
         id: restore_interaction_tables
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@v5
         with:
           path: ${{ env.IT_NAME }}
           key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
@@ -113,7 +113,7 @@ jobs:
 
       - name: Save cached CORSIKA interaction tables
         if: steps.restore_interaction_tables.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@v5
         with:
           path: ${{ env.IT_NAME }}
           key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
@@ -169,7 +169,7 @@ jobs:
       - name: Restore cached simulation models repository
         if: github.event_name != 'schedule'
         id: restore_simulation_models
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@v5
         with:
           path: simulation-models
           key: simulation-models-${{ steps.simulation_model_branch.outputs.branch }}-${{ github.run_id }}
@@ -208,7 +208,7 @@ jobs:
 
       - name: Save cached simulation models repository
         if: github.event_name != 'schedule' && steps.restore_simulation_models.outputs.cache-hit != 'true'
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@v5
         with:
           path: simulation-models
           key: simulation-models-${{ steps.simulation_model_branch.outputs.branch }}-${{ github.run_id }}

From 2ba2805578f455c85b3f8c909709713356ccd7ee Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:14:15 +0200
Subject: [PATCH 06/19] use cached sonar

---
 .github/workflows/CI-unittests.yml | 39 ++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 3be2139fdf..fce809eebc 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -137,6 +137,10 @@ jobs:
   sonarqube:
     needs: unit_tests
     runs-on: ubuntu-latest
+    env:
+      SONAR_SCANNER_VERSION: 8.1.0.6389
+      SONAR_SCANNER_PLATFORM: linux-x64
+      SONAR_SCANNER_CACHE_DIR: .sonar/sonar-scanner
 
     defaults:
       run:
@@ -155,16 +159,31 @@ jobs:
           path: .
 
       # CTAO-DPPS-SonarQube
-      - uses: SonarSource/sonarqube-scan-action@v8.1.0
+      - name: Restore Sonar scanner cache
+        id: sonar_scanner_cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.SONAR_SCANNER_CACHE_DIR }}
+          key: sonar-scanner-${{ runner.os }}-${{ runner.arch }}-${{ env.SONAR_SCANNER_VERSION }}
+
+      - name: Install Sonar scanner
+        if: steps.sonar_scanner_cache.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p "$SONAR_SCANNER_CACHE_DIR"
+          curl --fail --location --retry 5 --retry-delay 20 --retry-all-errors \
+            "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}.zip" \
+            --output sonar-scanner.zip
+          unzip -q sonar-scanner.zip -d "$SONAR_SCANNER_CACHE_DIR"
+
+      - name: Run Sonar scanner
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-        with:
-          scannerVersion: 8.1.0.6389
-          args: >
-            -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14
-            -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de
-            -Dsonar.qualitygate.wait=true
-            -Dsonar.python.coverage.reportPaths=coverage.xml
-            -Dsonar.python.version=3.13
-            -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py
+        run: |
+          "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
+            -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
+            -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de \
+            -Dsonar.qualitygate.wait=true \
+            -Dsonar.python.coverage.reportPaths=coverage.xml \
+            -Dsonar.python.version=3.13 \
+            -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py \
             -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**

From 920a7ca7a3c493e38a348df424e65983c1b5c555 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:22:46 +0200
Subject: [PATCH 07/19] changelog

---
 .github/workflows/CI-unittests.yml | 2 +-
 docs/changes/2239.maintenance.md   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 docs/changes/2239.maintenance.md

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index fce809eebc..09930f3b88 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -161,7 +161,7 @@ jobs:
       # CTAO-DPPS-SonarQube
       - name: Restore Sonar scanner cache
         id: sonar_scanner_cache
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ${{ env.SONAR_SCANNER_CACHE_DIR }}
           key: sonar-scanner-${{ runner.os }}-${{ runner.arch }}-${{ env.SONAR_SCANNER_VERSION }}
diff --git a/docs/changes/2239.maintenance.md b/docs/changes/2239.maintenance.md
new file mode 100644
index 0000000000..92dde254bc
--- /dev/null
+++ b/docs/changes/2239.maintenance.md
@@ -0,0 +1 @@
+Increase robustness of CI unit and integration tests. Add caching steps for CORSIKA interaction tables, simulation models, and sonar binary as fall back.

From 048e2615bb622b63e7d73b72aebf00957e01ef5c Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:28:12 +0200
Subject: [PATCH 08/19] retry sonar

---
 .github/workflows/CI-unittests.yml | 34 +++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 09930f3b88..1f10e63e9f 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -179,11 +179,29 @@ jobs:
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
-            -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
-            -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de \
-            -Dsonar.qualitygate.wait=true \
-            -Dsonar.python.coverage.reportPaths=coverage.xml \
-            -Dsonar.python.version=3.13 \
-            -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py \
-            -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**
+          for attempt in 1 2 3; do
+            if "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
+              -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
+              -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de \
+              -Dsonar.qualitygate.wait=true \
+              -Dsonar.scanner.connectTimeout=60 \
+              -Dsonar.scanner.socketTimeout=120 \
+              -Dsonar.scanner.responseTimeout=120 \
+              -Dsonar.plugins.download.timeout=600 \
+              -Dsonar.python.coverage.reportPaths=coverage.xml \
+              -Dsonar.python.version=3.13 \
+              -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py \
+              -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**
+            then
+              exit 0
+            fi
+
+            if [[ "$attempt" -lt 3 ]]; then
+              delay=$((attempt * 60))
+              echo "Sonar scanner attempt $attempt failed. Retrying in $delay seconds..."
+              sleep "$delay"
+            fi
+          done
+
+          echo "Sonar scanner failed after 3 attempts."
+          exit 1

From 5ce8cd49ffd5a47b830b70317acd9228e319352a Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:33:49 +0200
Subject: [PATCH 09/19] fix success

---
 .github/workflows/CI-integrationtests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index a285e06b0a..a91ad2f22b 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -182,13 +182,15 @@ jobs:
           CLONE_DIR="simulation-models-clone"
           CACHE_DIR="simulation-models"
           rm -rf "$CLONE_DIR"
+          CLONE_SUCCESS=false
 
           for attempt in 1 2 3; do
             if timeout 180 git clone --depth 1 --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" "$SIM_MODELS_REPO" "$CLONE_DIR"; then
               rm -rf "$CACHE_DIR"
               mv "$CLONE_DIR" "$CACHE_DIR"
               echo "Successfully cloned simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
-              exit 0
+              CLONE_SUCCESS=true
+              break
             fi
 
             rm -rf "$CLONE_DIR"
@@ -198,13 +200,11 @@ jobs:
             fi
           done
 
-          if [[ -d "$CACHE_DIR" ]]; then
+          if [ "$CLONE_SUCCESS" = false ] && [ ! -d "$CACHE_DIR" ]; then
             echo "GitLab clone failed; using cached simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
-            exit 0
+            exit 1
           fi
 
-          echo "Failed to clone simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH and no cache is available."
-          exit 1
 
       - name: Save cached simulation models repository
         if: github.event_name != 'schedule' && steps.restore_simulation_models.outputs.cache-hit != 'true'

From f09201eaa38c9bad6dea89af7ff19e6ea035796e Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:43:01 +0200
Subject: [PATCH 10/19] error reporting

---
 .github/workflows/CI-unittests.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 1f10e63e9f..7bd5503af6 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -180,7 +180,8 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
           for attempt in 1 2 3; do
-            if "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
+            set +e
+            "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
               -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
               -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de \
               -Dsonar.qualitygate.wait=true \
@@ -192,7 +193,12 @@ jobs:
               -Dsonar.python.version=3.13 \
               -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py \
               -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**
-            then
+            status=$?
+            set -e
+
+            echo "Sonar scanner attempt $attempt exited with status $status."
+
+            if [[ "$status" -eq 0 ]]; then
               exit 0
             fi
 

From 8f81acdd63fbd90453be742260b3838b82db19f8 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 12:53:45 +0200
Subject: [PATCH 11/19] retry on connection issues only

---
 .github/workflows/CI-unittests.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 7bd5503af6..404d3a16d0 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -180,6 +180,7 @@ jobs:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
           for attempt in 1 2 3; do
+            scanner_log="sonar-scanner-attempt-${attempt}.log"
             set +e
             "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
               -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
@@ -192,8 +193,9 @@ jobs:
               -Dsonar.python.coverage.reportPaths=coverage.xml \
               -Dsonar.python.version=3.13 \
               -Dsonar.exclusions=**/docs/**,src/simtools/applications/**,**/__init__.py \
-              -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/**
-            status=$?
+              -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/** \
+              2>&1 | tee "$scanner_log"
+            status=${PIPESTATUS[0]}
             set -e
 
             echo "Sonar scanner attempt $attempt exited with status $status."
@@ -202,6 +204,16 @@ jobs:
               exit 0
             fi
 
+            if grep -Eqi "QUALITY GATE STATUS: FAILED|quality gate failed" "$scanner_log"; then
+              echo "Sonar quality gate failed; not retrying."
+              exit "$status"
+            fi
+
+            if ! grep -Eqi "HTTP connect timed out|connect timed out|Connection refused|Connection reset|Failed to query server version|Failed to upload report|Failed to request|timeout|timed out|temporarily unavailable|502|503|504" "$scanner_log"; then
+              echo "Sonar scanner failed with a non-transient error; not retrying."
+              exit "$status"
+            fi
+
             if [[ "$attempt" -lt 3 ]]; then
               delay=$((attempt * 60))
               echo "Sonar scanner attempt $attempt failed. Retrying in $delay seconds..."

From dc42ea54708bd945908a0b4d0d37eb9352fb7ee2 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:10:01 +0200
Subject: [PATCH 12/19] improve efficiency

---
 .github/workflows/CI-integrationtests.yml | 58 ++++++++++++++++-------
 .github/workflows/CI-unittests.yml        |  2 +
 2 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index a91ad2f22b..15178fc58d 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -3,6 +3,13 @@ name: CI-integrationtests
 
 permissions: {}
 
+env:
+  CLONE_ATTEMPTS: 2
+  GIT_HTTP_CONNECT_TIMEOUT: 20
+  GIT_HTTP_LOW_SPEED_LIMIT: 1000
+  GIT_HTTP_LOW_SPEED_TIME: 30
+  RETRY_SLEEP_SECONDS: 60
+
 on:
   workflow_dispatch:
     inputs:
@@ -85,31 +92,39 @@ jobs:
           CLONE_DIR="${IT_NAME}-clone"
           CACHE_DIR="$IT_NAME"
           rm -rf "$CLONE_DIR"
+          CLONE_SUCCESS=false
 
-          for attempt in 1 2 3; do
-            if timeout 180 git clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$CLONE_DIR"; then
+          for attempt in $(seq 1 "$CLONE_ATTEMPTS"); do
+            if git \
+              -c "http.connectTimeout=$GIT_HTTP_CONNECT_TIMEOUT" \
+              -c "http.lowSpeedLimit=$GIT_HTTP_LOW_SPEED_LIMIT" \
+              -c "http.lowSpeedTime=$GIT_HTTP_LOW_SPEED_TIME" \
+              clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$CLONE_DIR"
+            then
               git lfs install
               (cd "$CLONE_DIR" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III")
               rm -rf "$CACHE_DIR"
               mv "$CLONE_DIR" "$CACHE_DIR"
               echo "Successfully cloned CORSIKA interaction tables $CORSIKA_TABLES."
-              exit 0
+              CLONE_SUCCESS=true
+              break
             fi
 
             rm -rf "$CLONE_DIR"
-            if [[ "$attempt" -lt 3 ]]; then
-              echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
-              sleep 120
+            if [[ "$attempt" -lt "$CLONE_ATTEMPTS" ]]; then
+              echo "Clone attempt $attempt failed. Retrying in $RETRY_SLEEP_SECONDS seconds..."
+              sleep "$RETRY_SLEEP_SECONDS"
             fi
           done
 
-          if [[ -d "$CACHE_DIR" ]]; then
-            echo "GitLab clone failed; using cached CORSIKA interaction tables $CORSIKA_TABLES."
-            exit 0
+          if [ "$CLONE_SUCCESS" = false ] && [ ! -d "$CACHE_DIR" ]; then
+            echo "Failed to clone CORSIKA interaction tables $CORSIKA_TABLES and no cache is available."
+            exit 1
           fi
 
-          echo "Failed to clone CORSIKA interaction tables $CORSIKA_TABLES and no cache is available."
-          exit 1
+          if [ "$CLONE_SUCCESS" = false ]; then
+            echo "GitLab clone failed; using cached CORSIKA interaction tables $CORSIKA_TABLES."
+          fi
 
       - name: Save cached CORSIKA interaction tables
         if: steps.restore_interaction_tables.outputs.cache-hit != 'true'
@@ -184,8 +199,13 @@ jobs:
           rm -rf "$CLONE_DIR"
           CLONE_SUCCESS=false
 
-          for attempt in 1 2 3; do
-            if timeout 180 git clone --depth 1 --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" "$SIM_MODELS_REPO" "$CLONE_DIR"; then
+          for attempt in $(seq 1 "$CLONE_ATTEMPTS"); do
+            if git \
+              -c "http.connectTimeout=$GIT_HTTP_CONNECT_TIMEOUT" \
+              -c "http.lowSpeedLimit=$GIT_HTTP_LOW_SPEED_LIMIT" \
+              -c "http.lowSpeedTime=$GIT_HTTP_LOW_SPEED_TIME" \
+              clone --depth 1 --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH" "$SIM_MODELS_REPO" "$CLONE_DIR"
+            then
               rm -rf "$CACHE_DIR"
               mv "$CLONE_DIR" "$CACHE_DIR"
               echo "Successfully cloned simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
@@ -194,17 +214,21 @@ jobs:
             fi
 
             rm -rf "$CLONE_DIR"
-            if [[ "$attempt" -lt 3 ]]; then
-              echo "Clone attempt $attempt failed. Retrying in 120 seconds..."
-              sleep 120
+            if [[ "$attempt" -lt "$CLONE_ATTEMPTS" ]]; then
+              echo "Clone attempt $attempt failed. Retrying in $RETRY_SLEEP_SECONDS seconds..."
+              sleep "$RETRY_SLEEP_SECONDS"
             fi
           done
 
           if [ "$CLONE_SUCCESS" = false ] && [ ! -d "$CACHE_DIR" ]; then
-            echo "GitLab clone failed; using cached simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
+            echo "Failed to clone simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH and no cache is available."
             exit 1
           fi
 
+          if [ "$CLONE_SUCCESS" = false ]; then
+            echo "GitLab clone failed; using cached simulation models branch $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH."
+          fi
+
 
       - name: Save cached simulation models repository
         if: github.event_name != 'schedule' && steps.restore_simulation_models.outputs.cache-hit != 'true'
diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 404d3a16d0..c5432e458b 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -176,6 +176,8 @@ jobs:
           unzip -q sonar-scanner.zip -d "$SONAR_SCANNER_CACHE_DIR"
 
       - name: Run Sonar scanner
+        # Soft fail for draft PRs (unstable network connections)
+        continue-on-error: ${{ github.event.pull_request.draft == true }}
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |

From a930b08c1cf0b11d2fde4b8bcdd6aaa5b707a1b7 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:23:03 +0200
Subject: [PATCH 13/19] failure logs

---
 .github/workflows/CI-unittests.yml | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index c5432e458b..bee34886f2 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -176,6 +176,7 @@ jobs:
           unzip -q sonar-scanner.zip -d "$SONAR_SCANNER_CACHE_DIR"
 
       - name: Run Sonar scanner
+        id: sonar_scan
         # Soft fail for draft PRs (unstable network connections)
         continue-on-error: ${{ github.event.pull_request.draft == true }}
         env:
@@ -225,3 +226,23 @@ jobs:
 
           echo "Sonar scanner failed after 3 attempts."
           exit 1
+
+      - name: Report soft-failed Sonar scan
+        if: always() && github.event.pull_request.draft == true && steps.sonar_scan.outcome == 'failure'
+        run: |
+          echo "::warning::Sonar scan failed but was allowed to continue because this is a draft PR."
+          {
+            echo "## Sonar scan soft failure"
+            echo
+            echo "The Sonar scan failed, but the workflow continued because this is a draft PR."
+            echo "Open the \`Run Sonar scanner\` step logs for details."
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload soft-failed Sonar logs
+        if: always() && github.event.pull_request.draft == true && steps.sonar_scan.outcome == 'failure'
+        uses: actions/upload-artifact@v7
+        with:
+          name: sonar-scanner-logs
+          path: sonar-scanner-attempt-*.log
+          if-no-files-found: ignore
+          retention-days: 7

From ce32dbb648b3bc7636ede9fb5889c14870aca0a8 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:44:44 +0200
Subject: [PATCH 14/19] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .github/workflows/CI-integrationtests.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index 15178fc58d..98f5034d40 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -102,7 +102,11 @@ jobs:
               clone --depth 1 --branch "$CORSIKA_TABLES" "$REPO_URL" "$CLONE_DIR"
             then
               git lfs install
-              (cd "$CLONE_DIR" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III")
+              if ! (cd "$CLONE_DIR" && git lfs pull --exclude="interaction-tables/qgsdat-II-04,interaction-tables/qgsdat-III"); then
+                echo "git lfs pull failed; will retry and/or fall back to the cache."
+                rm -rf "$CLONE_DIR"
+                continue
+              fi
               rm -rf "$CACHE_DIR"
               mv "$CLONE_DIR" "$CACHE_DIR"
               echo "Successfully cloned CORSIKA interaction tables $CORSIKA_TABLES."

From 64420f63a6d493bfb6affae62ddba4d8d7113a1f Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:45:41 +0200
Subject: [PATCH 15/19] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .github/workflows/CI-integrationtests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index 98f5034d40..3ff33097ca 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -80,7 +80,7 @@ jobs:
         uses: actions/cache/restore@v5
         with:
           path: ${{ env.IT_NAME }}
-          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
+          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}
           restore-keys: |
             corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-
 

From 861b3d1aafdc2cd7a8d735f15e494a09df55bc8c Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:45:49 +0200
Subject: [PATCH 16/19] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 docs/changes/2239.maintenance.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/changes/2239.maintenance.md b/docs/changes/2239.maintenance.md
index 92dde254bc..d76a289c85 100644
--- a/docs/changes/2239.maintenance.md
+++ b/docs/changes/2239.maintenance.md
@@ -1 +1 @@
-Increase robustness of CI unit and integration tests. Add caching steps for CORSIKA interaction tables, simulation models, and sonar binary as fall back.
+Increase robustness of CI unit and integration tests. Add caching steps for CORSIKA interaction tables, simulation models, and sonar binary as a fallback.

From 33ecde688c5066c084627bbdb110d53ac446313b Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:45:59 +0200
Subject: [PATCH 17/19] Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
---
 .github/workflows/CI-integrationtests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
index 3ff33097ca..8bf985fc60 100644
--- a/.github/workflows/CI-integrationtests.yml
+++ b/.github/workflows/CI-integrationtests.yml
@@ -135,7 +135,7 @@ jobs:
         uses: actions/cache/save@v5
         with:
           path: ${{ env.IT_NAME }}
-          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}-${{ github.run_id }}
+          key: corsika-interaction-tables-${{ env.CORSIKA_TABLES }}
 
       - name: Upload CORSIKA interaction tables
         uses: actions/upload-artifact@v7

From d357b4ea57d1681332fd14139740f5ddeaaaf90c Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 13:58:42 +0200
Subject: [PATCH 18/19] fs

---
 .github/workflows/CI-unittests.yml | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index bee34886f2..26f62ccf32 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -182,6 +182,7 @@ jobs:
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
+          final_status=1
           for attempt in 1 2 3; do
             scanner_log="sonar-scanner-attempt-${attempt}.log"
             set +e
@@ -204,17 +205,21 @@ jobs:
             echo "Sonar scanner attempt $attempt exited with status $status."
 
             if [[ "$status" -eq 0 ]]; then
-              exit 0
+              final_status=0
+              echo "Sonar scanner succeeded."
+              break
             fi
 
             if grep -Eqi "QUALITY GATE STATUS: FAILED|quality gate failed" "$scanner_log"; then
               echo "Sonar quality gate failed; not retrying."
-              exit "$status"
+              final_status="$status"
+              break
             fi
 
             if ! grep -Eqi "HTTP connect timed out|connect timed out|Connection refused|Connection reset|Failed to query server version|Failed to upload report|Failed to request|timeout|timed out|temporarily unavailable|502|503|504" "$scanner_log"; then
               echo "Sonar scanner failed with a non-transient error; not retrying."
-              exit "$status"
+              final_status="$status"
+              break
             fi
 
             if [[ "$attempt" -lt 3 ]]; then
@@ -224,8 +229,10 @@ jobs:
             fi
           done
 
-          echo "Sonar scanner failed after 3 attempts."
-          exit 1
+          if [[ "$final_status" -ne 0 ]]; then
+            echo "Sonar scanner failed with status $final_status."
+          fi
+          exit "$final_status"
 
       - name: Report soft-failed Sonar scan
         if: always() && github.event.pull_request.draft == true && steps.sonar_scan.outcome == 'failure'

From 8010dae7903e43c23698e5679360a8544e4aef99 Mon Sep 17 00:00:00 2001
From: Gernot Maier <gernot.maier@desy.de>
Date: Wed, 3 Jun 2026 15:15:47 +0200
Subject: [PATCH 19/19] sonar success

---
 .github/workflows/CI-unittests.yml | 19 ++++++-------------
 1 file changed, 6 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
index 26f62ccf32..434539e108 100644
--- a/.github/workflows/CI-unittests.yml
+++ b/.github/workflows/CI-unittests.yml
@@ -182,10 +182,9 @@ jobs:
         env:
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          final_status=1
+          set +e
           for attempt in 1 2 3; do
             scanner_log="sonar-scanner-attempt-${attempt}.log"
-            set +e
             "${SONAR_SCANNER_CACHE_DIR}/sonar-scanner-${SONAR_SCANNER_VERSION}-${SONAR_SCANNER_PLATFORM}/bin/sonar-scanner" \
               -Dsonar.projectKey=gammasim_simtools_0d23837b-8b2d-4e54-9a98-2f1bde681f14 \
               -Dsonar.host.url=https://sonar-ctao.zeuthen.desy.de \
@@ -200,26 +199,22 @@ jobs:
               -Dsonar.coverage.exclusions=**/tests/**,src/simtools/applications/** \
               2>&1 | tee "$scanner_log"
             status=${PIPESTATUS[0]}
-            set -e
 
             echo "Sonar scanner attempt $attempt exited with status $status."
 
             if [[ "$status" -eq 0 ]]; then
-              final_status=0
               echo "Sonar scanner succeeded."
-              break
+              exit 0
             fi
 
             if grep -Eqi "QUALITY GATE STATUS: FAILED|quality gate failed" "$scanner_log"; then
               echo "Sonar quality gate failed; not retrying."
-              final_status="$status"
-              break
+              exit "$status"
             fi
 
             if ! grep -Eqi "HTTP connect timed out|connect timed out|Connection refused|Connection reset|Failed to query server version|Failed to upload report|Failed to request|timeout|timed out|temporarily unavailable|502|503|504" "$scanner_log"; then
               echo "Sonar scanner failed with a non-transient error; not retrying."
-              final_status="$status"
-              break
+              exit "$status"
             fi
 
             if [[ "$attempt" -lt 3 ]]; then
@@ -229,10 +224,8 @@ jobs:
             fi
           done
 
-          if [[ "$final_status" -ne 0 ]]; then
-            echo "Sonar scanner failed with status $final_status."
-          fi
-          exit "$final_status"
+          echo "Sonar scanner failed after 3 attempts."
+          exit 1
 
       - name: Report soft-failed Sonar scan
         if: always() && github.event.pull_request.draft == true && steps.sonar_scan.outcome == 'failure'