ci(backend): run mutation testing monthly in shards

.github/workflows/ops.yml

@@ -8,6 +8,8 @@ on:
   # between the CI network split and the first run after it.
   schedule:
     - cron: "0 5 * * 1"
+    # Monthly: mutation testing takes hours of runner time across its shards.
+    - cron: "0 2 1 * *"
   workflow_dispatch:
     inputs:
       operation:
@@ -17,6 +19,7 @@ on:
         options:
           - perf-baseline
           - bundle-budget
+          - mutation
       git_ref:
         description: Git ref to test
         required: false
@@ -33,7 +36,7 @@ defaults:
 jobs:
   app-bundle-budget:
     name: App Bundle Budget
-    if: github.event_name == 'schedule' || inputs.operation == 'bundle-budget'
+    if: github.event.schedule == '0 5 * * 1' || inputs.operation == 'bundle-budget'
     runs-on: ubuntu-26.04
     timeout-minutes: 20
     steps:
@@ -64,7 +67,7 @@ jobs:
 
   backend-perf-baseline:
     name: Backend Perf Baseline
-    if: github.event_name == 'schedule' || inputs.operation == 'perf-baseline'
+    if: github.event.schedule == '0 5 * * 1' || inputs.operation == 'perf-baseline'
     runs-on: ubuntu-26.04
     timeout-minutes: 45
     steps:
@@ -104,3 +107,74 @@ jobs:
             repo/backend/reports/performance/latest-k6-summary.json
             repo/backend/reports/performance/*.md
           retention-days: 14
+
+  # Tracks the survivor count rather than gating on it: it never reaches zero. Triage
+  # the functions listed above baseline, then rebaseline (backend/tests/README.md).
+  # One run over the whole app outlasts a job's time limit, so it is sharded by
+  # mutant name; the globs must cover every package under app/ exactly once.
+  backend-mutation:
+    name: Backend Mutation Testing (${{ matrix.shard }})
+    if: github.event.schedule == '0 2 1 * *' || inputs.operation == 'mutation'
+    runs-on: ubuntu-26.04
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: auth-flows
+            mutants: app.api.auth.services.[e-o]*
+          - shard: auth-core
+            mutants: app.api.auth.services.[!e-o]* app.api.auth.[!s]* app.api.auth.schemas*
+          - shard: plugins
+            mutants: app.api.plugins.*
+          - shard: common
+            mutants: app.api.common.* app.api.stats.* app.api.application.* app.api.reference_data.*
+          - shard: data
+            mutants: app.api.data_collection.* app.api.file_storage.*
+          - shard: core
+            mutants: app.core.*
+    steps:
+      - name: Checkout trusted workflow ref
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up runtime
+        uses: ./.github/actions/setup-runtime
+        with:
+          setup-python: "true"
+          uv-cache-dependency-glob: backend/uv.lock
+
+      - name: Checkout requested repository ref
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.git_ref || github.ref_name }}
+          path: repo
+          persist-credentials: false
+
+      - name: Install dependencies
+        working-directory: repo
+        run: just backend/install
+
+      - name: Run mutation testing
+        working-directory: repo/backend
+        env:
+          MUTANTS: ${{ matrix.mutants }}
+        run: |
+          # Split MUTANTS into words without expanding the globs in it.
+          set -f
+          # shellcheck disable=SC2086
+          MUTMUT_JOBS="$(nproc)" just mutation $MUTANTS
+          {
+            echo "### Mutation testing: ${{ matrix.shard }}"
+            echo "$(wc -l < reports/mutation/survivors.txt) survivors"
+            echo "$(wc -l < reports/mutation/new-survivors.txt) functions above baseline"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Archive survivor lists
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ops-backend-mutation-${{ matrix.shard }}
+          path: repo/backend/reports/mutation/
+          retention-days: 30

backend/.gitignore

@@ -61,3 +61,7 @@ reports/performance/*.md
 
 # Derived upload fixtures; rebuilt by `just perf-fixtures` from the committed sample.
 perf/fixtures/generated/
+
+# Mutation testing working copy (`just mutation`)
+mutants/
+reports/mutation/

backend/justfile

@@ -76,6 +76,42 @@ test-ci:
     mkdir -p reports/coverage
     {{ pytest }} tests/unit tests/integration -n auto --dist=loadgroup --durations=20 --junitxml=reports/junit.xml --cov --cov-report=xml --cov-report=term
 
+# Mutation testing against one throwaway Postgres; the full app takes hours. Narrow
+# it with mutant globs: `just mutation 'app.core.images*'`. Survivors never fail the
+# run; it lists functions with more survivors than tests/mutation-baseline.txt.
+[doc('Mutation testing over app/, reporting survivors beyond the baseline')]
+mutation *mutants:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    # Mutant globs pass through to mutmut, not the shell.
+    set -f
+    container=$(docker run -d --rm -p 127.0.0.1::5432 -e POSTGRES_PASSWORD=postgres postgres:18-alpine)
+    trap 'docker stop "$container" >/dev/null' EXIT
+    export TEST_POSTGRES_HOST=127.0.0.1
+    TEST_POSTGRES_PORT=$(docker port "$container" 5432/tcp | head -1 | cut -d: -f2)
+    export TEST_POSTGRES_PORT
+    # TCP only answers once initdb's socket-only server has handed over.
+    for _ in $(seq 60); do
+        docker exec "$container" pg_isready -h 127.0.0.1 -U postgres -q && break
+        sleep 1
+    done
+    docker exec "$container" pg_isready -h 127.0.0.1 -U postgres -q
+    # Low priority on half the cores by default, so a dev machine stays usable.
+    nice -n 10 uv run mutmut run --max-children "${MUTMUT_JOBS:-$(( ($(nproc) + 1) / 2 ))}" {{ mutants }}
+    mkdir -p reports/mutation
+    uv run mutmut results | sed -n 's/^ *\(.*\): survived$/\1/p' | sort > reports/mutation/survivors.txt
+    # Mutant numbers shift whenever a function changes, so compare survivor counts per function.
+    sed 's/__mutmut_[0-9]*$//' reports/mutation/survivors.txt | sort | uniq -c \
+        | awk 'FILENAME == ARGV[1] { base[$2] = $1; next } $1 > base[$2] { print $2, base[$2] + 0, "->", $1 }' \
+            tests/mutation-baseline.txt - > reports/mutation/new-survivors.txt
+    echo "$(wc -l < reports/mutation/survivors.txt) survivors; functions above baseline:"
+    cat reports/mutation/new-survivors.txt
+
+# Rebaseline after triaging survivors. Only from a full run (or the Ops artifact):
+# it replaces the whole file, so a narrowed run would drop every other function.
+mutation-baseline:
+    sed 's/__mutmut_[0-9]*$//' reports/mutation/survivors.txt | sort | uniq -c | awk '{ print $1, $2 }' > tests/mutation-baseline.txt
+
 # ============================================================================
 # Database & Migrations
 # ============================================================================

backend/pyproject.toml

@@ -117,6 +117,9 @@
     "dirty-equals>=0.8.0", # Flexible assertions for dynamic data
     "pytest-mock>=3.14.0", # Better mocking with cleaner pytest integration
   ]
+mutation = [
+    "mutmut>=3.8.0",
+]
 
 ### Tool configuration
 [tool.alembic]
@@ -289,3 +292,13 @@
 
   [tool.uv.exclude-newer-package]
     relab-rpi-cam-models = "3000-01-01T00:00:00Z" # self-managed package, exempt from the supply-chain delay
+
+[tool.mutmut]
+  # Scheduled weekly in .github/workflows/ops.yml; run locally with `just mutation`.
+  also_copy                          = [".env.test", "Dockerfile", "alembic", "data/seed", "scripts", "tests"]
+  # Script tests read repo-root files outside the mutants copy, and scripts are not mutated.
+  pytest_add_cli_args_test_selection = ["tests/unit", "tests/integration", "--ignore=tests/unit/scripts"]
+  # Log calls only change diagnostics; their mutants would all survive.
+  do_not_mutate_patterns             = ['^\s*(logger|logging)\.\w+\(']
+  do_not_mutate                      = ["app/__init__.py", "app/__version__.py", "app/main.py"]
+  source_paths                       = ["app"]

backend/tests/README.md

@@ -50,3 +50,24 @@ camera setup -> record -> persist. Keep them sparse; they are slower than the ot
 - Break what a new test covers and watch it fail before trusting it.
 - To find gaps, replace a guard with `if False:` and run `tests/unit tests/integration`. One that
   breaks nothing is untested, or covered only by an assertion another path also satisfies.
+
+## Mutation testing
+
+`just mutation` runs mutmut over `app/` against one throwaway Postgres; the Ops workflow runs it
+monthly in shards by package and uploads each shard's survivor list. It never fails on survivors. It lists the functions with more
+survivors than `tests/mutation-baseline.txt`: add the missing assertion, or rebaseline with
+`just mutation-baseline` when the survivor is deliberate (a guard for a state that cannot occur).
+A full run is heavy, so leave it to the Ops job; locally it runs at low priority on half the cores
+(set `MUTMUT_JOBS` to change that). Pass mutant globs to narrow a local run:
+`just mutation 'app.core.images*'`. Results are cached in
+`mutants/`: after adding an assertion, rerun with that function's glob or delete `mutants/`, or the
+old verdict stands. Rebaseline only from a full run, since it replaces the file. From an Ops run:
+
+```sh
+gh run download <run-id> -p 'ops-backend-mutation-*' -D reports/mutation/ops
+cat reports/mutation/ops/*/survivors.txt | sort > reports/mutation/survivors.txt
+just mutation-baseline
+```
+
+Log calls are not mutated, but only on the call's first line: the argument lines of a multi-line
+call still are, and their survivors belong in the baseline.

backend/tests/conftest.py

@@ -86,11 +86,7 @@ def _absorb_shared_container_coords(config: pytest.Config) -> None:
     """Reuse the controller's shared Postgres container on an xdist worker."""
     global _external_container
     workerinput = _worker_input(config)
-    os.environ["DATABASE_HOST"] = workerinput["relab_db_host"]
-    os.environ["DATABASE_PORT"] = workerinput["relab_db_port"]
-    os.environ["POSTGRES_USER"] = "postgres"
-    os.environ["POSTGRES_PASSWORD"] = "postgres"  # Test-password only
-    os.environ["POSTGRES_DB"] = "postgres"
+    _publish_postgres_coords(workerinput["relab_db_host"], workerinput["relab_db_port"])
     _external_container = True
 
 
@@ -125,6 +121,11 @@ def _ensure_testcontainers_postgres() -> None:
     if _postgres_container is not None or _external_container:
         return
 
+    if external_host := os.getenv("TEST_POSTGRES_HOST"):
+        # An already-running Postgres: `just mutation` shares one across its forked runs.
+        _publish_postgres_coords(external_host, os.getenv("TEST_POSTGRES_PORT", "5432"))
+        return
+
     logger.info("Starting Testcontainers Postgres...")
     _postgres_container = PostgresContainer(
         "postgres:18-alpine",
@@ -136,15 +137,17 @@ def _ensure_testcontainers_postgres() -> None:
 
     host = _postgres_container.get_container_host_ip()
     port = _postgres_container.get_exposed_port(5432)
+    _publish_postgres_coords(str(host), str(port))
+    logger.info("Testcontainers Postgres started: %s:%s", host, port)
+
 
-    os.environ["DATABASE_HOST"] = str(host)
-    os.environ["DATABASE_PORT"] = str(port)
+def _publish_postgres_coords(host: str, port: str) -> None:
+    os.environ["DATABASE_HOST"] = host
+    os.environ["DATABASE_PORT"] = port
     os.environ["POSTGRES_USER"] = "postgres"
     os.environ["POSTGRES_PASSWORD"] = "postgres"  # Test-password only
     os.environ["POSTGRES_DB"] = "postgres"
 
-    logger.info("Testcontainers Postgres started: %s:%s", host, port)
-
 
 def _validate_test_database_name(database_name: str) -> str:
     """Return a validated test database name."""
@@ -177,6 +180,9 @@ def _get_worker_test_db_name() -> str:
     db_name = base_name
     if worker_id and worker_id != _MASTER_WORKER:
         db_name = f"{base_name}_{worker_id}"
+    elif os.getenv("MUTANT_UNDER_TEST"):
+        # mutmut runs mutants in parallel forked processes against one shared server.
+        db_name = f"{base_name}_{os.getpid()}"
 
     return _validate_test_database_name(db_name)