ci: run checks on pull requests into dev, including the integration suite

.config/nextest.toml

@@ -0,0 +1,13 @@
+# Integration tests (crates/test-support) each boot the whole server
+# against real Postgres, Redis and ClickHouse; see the `integration` job
+# in .github/workflows/ci.yml.
+
+[profile.ci]
+# Report every failure, not just the first.
+fail-fast = false
+# A runner has 4 vCPUs. The harness gives each running test its own
+# Redis logical DB (1 + slot), so this must stay at 15 or below.
+test-threads = 4
+# A hung test fails after 3 minutes instead of holding the runner until
+# the job timeout.
+slow-timeout = { period = "60s", terminate-after = 3 }

.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
   pull_request:
-    branches: [main]
+    branches: [main, dev]
 
 env:
   CARGO_TERM_COLOR: always
@@ -77,6 +77,93 @@ jobs:
       - name: Format check
         run: cargo fmt --all -- --check
 
+  # The ~260 tests in crates/test-support/tests/ are `#[ignore]`d so the
+  # unit job (and a plain `cargo test`) skips them: each one boots the
+  # whole server against its own Postgres database, Redis logical DB and,
+  # when it asks for one, ClickHouse database. A separate job so a red
+  # check says which suite broke.
+  integration:
+    name: Integration Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    services:
+      postgres:
+        image: postgres:18-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: think_watch_test
+        ports:
+          - 5432:5432
+        # Every test creates its own database. With Docker's default
+        # 64MB /dev/shm, Postgres fails a few hundred databases in with
+        # "could not resize shared memory segment".
+        options: >-
+          --shm-size=1g
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      redis:
+        image: redis:8-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      clickhouse:
+        image: clickhouse/clickhouse-server:26.3-alpine
+        env:
+          CLICKHOUSE_USER: default
+          CLICKHOUSE_PASSWORD: chtest
+          # The analytics schema starts with `USE think_watch`.
+          CLICKHOUSE_DB: think_watch
+          CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+        ports:
+          - 8123:8123
+        options: >-
+          --ulimit nofile=262144:262144
+          --health-cmd "wget -qO- http://127.0.0.1:8123/ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+
+    env:
+      TEST_DATABASE_BASE_URL: postgres://postgres:postgres@localhost:5432
+      # Base logical DB. nextest runs tests in parallel and the harness
+      # moves each running test to DB 1 + its slot, so base + jobs must
+      # stay within Redis's 16 DBs.
+      TEST_REDIS_URL: redis://localhost:6379/1
+      TEST_CLICKHOUSE_URL: http://localhost:8123
+      TEST_CLICKHOUSE_USER: default
+      TEST_CLICKHOUSE_PASSWORD: chtest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      # Same disk pressure as the unit job: the server plus 50-odd test
+      # binaries do not fit in the runner's default free space.
+      - name: Reclaim runner disk
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+                      /opt/hostedtoolcache/CodeQL /usr/local/.ghcup
+          df -h /
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest
+
+      - name: Build tests
+        run: cargo nextest run -p think-watch-test-support --run-ignored only --no-run
+
+      - name: Integration tests
+        run: cargo nextest run -p think-watch-test-support --run-ignored only --profile ci
+
   frontend:
     name: Frontend Build
     runs-on: ubuntu-latest
@@ -124,7 +211,7 @@ jobs:
   docker-server-build:
     name: Server Build (${{ matrix.platform }})
     runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
-    needs: [rust, frontend]
+    needs: [rust, integration, frontend]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
       contents: read
@@ -239,7 +326,7 @@ jobs:
   docker-web:
     name: Web Build & Push
     runs-on: ubuntu-latest
-    needs: [rust, frontend]
+    needs: [rust, integration, frontend]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     # A healthy run takes minutes. With no limit, a hung build holds the
     # runner for GitHub's six-hour maximum — which is what happened when

Cargo.toml

@@ -41,6 +41,17 @@ strip = "symbols"
 [profile.dev]
 debug = "line-tables-only"
 
+# Every login in the integration suite hashes a password with Argon2 and
+# grinds a proof-of-work over SHA-256. Unoptimised, those two dominate:
+# the login-heavy tests ran for one to two minutes each in CI. Optimising
+# just the hash crates keeps the rest of the build fast to compile.
+[profile.dev.package.argon2]
+opt-level = 3
+[profile.dev.package.blake2]
+opt-level = 3
+[profile.dev.package.sha2]
+opt-level = 3
+
 [workspace.dependencies]
 
 # ── thinkwatch-core (MIT) ────────────────────────────────────────────

crates/test-support/src/lib.rs

@@ -9,9 +9,11 @@
 //! - `TEST_DATABASE_BASE_URL` (default `postgres://thinkwatch:thinkwatch@localhost:5432`)
 //! - `TEST_REDIS_URL` (default `redis://localhost:6379`)
 //!
-//! Tests run against the same Redis; isolation is achieved by giving
-//! every fixture a fresh UUID-suffixed email / user id, which ensures
-//! the rate-limit, lockout, and signing keys never collide.
+//! Tests run against the same Redis instance. Each `TestApp` FLUSHDBs
+//! its logical DB on spawn, so concurrent tests need separate DBs:
+//! under nextest each running test gets DB `base + slot` (see
+//! `redis_url_for_slot`); plain `cargo test` must run with
+//! `--test-threads=1`.
 
 pub mod ch;
 pub mod client;
@@ -149,6 +151,7 @@ impl TestApp {
         let redis_url = std::env::var("TEST_REDIS_URL").unwrap_or_else(|_| {
             "redis://:225b3facaf55212ff86ad6595e6d6471@localhost:6379/1".into()
         });
+        let redis_url = redis_url_for_slot(&redis_url)?;
 
         // Per-test database with migrations applied.
         let db_owner = IsolatedDatabase::create(&base_url)
@@ -158,8 +161,11 @@ impl TestApp {
 
         // Redis: shared instance on a dedicated logical DB. We
         // FLUSHDB at spawn time to clear any stragglers from prior
-        // tests. Tests must therefore run serially
-        // (`--test-threads=1`) — the Makefile target enforces it.
+        // tests, so two tests must never share a logical DB at the
+        // same time: either run serially (`--test-threads=1`, the
+        // Makefile target) or under nextest, where
+        // `redis_url_for_slot` gives each concurrently running test
+        // its own DB.
         let redis = build_redis(&redis_url).await?;
         // fred 10 doesn't expose FLUSHDB directly (only FLUSHALL),
         // and we don't want to nuke the dev DB. Send the raw
@@ -396,6 +402,33 @@ impl Drop for TestApp {
     }
 }
 
+/// Under nextest, move the Redis URL to logical DB `base + slot`.
+///
+/// Every `TestApp` FLUSHDBs its Redis DB on spawn, so tests that run
+/// at the same time must not share one. nextest runs each test in its
+/// own process and hands it `NEXTEST_TEST_GLOBAL_SLOT`, a number in
+/// `0..jobs` that no other running test holds; offsetting the DB by it
+/// keeps parallel tests apart. Outside nextest the URL is unchanged.
+fn redis_url_for_slot(redis_url: &str) -> anyhow::Result<String> {
+    let Ok(slot) = std::env::var("NEXTEST_TEST_GLOBAL_SLOT") else {
+        return Ok(redis_url.to_string());
+    };
+    let slot: u32 = slot.parse().context("parse NEXTEST_TEST_GLOBAL_SLOT")?;
+    let mut url = url::Url::parse(redis_url).context("parse TEST_REDIS_URL")?;
+    let base: u32 = match url.path().trim_start_matches('/') {
+        "" => 0,
+        db => db.parse().context("parse the Redis DB in TEST_REDIS_URL")?,
+    };
+    let db = base + slot;
+    // Redis ships with 16 logical DBs (0..=15).
+    anyhow::ensure!(
+        db <= 15,
+        "Redis DB {db} (base {base} + nextest slot {slot}) is past DB 15; lower the test threads"
+    );
+    url.set_path(&format!("/{db}"));
+    Ok(url.to_string())
+}
+
 async fn build_redis(redis_url: &str) -> anyhow::Result<RedisClient> {
     let cfg = RedisConfig::from_url(redis_url).context("parse REDIS_URL")?;
     let client = Builder::from_config(cfg).build()?;