diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 848bb6f3..8619c07b 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -7,6 +7,14 @@ # scope". Suppress only that exact message — a genuine scope typo (e.g. # `contents` → `conten`) still produces a different message and fails. Drop # this once actionlint ships the scope. +# +# nektos/act has the same gap and no equivalent escape hatch: `act` (0.2.89) +# refuses ci.yml outright with `Unknown Property code-quality` from its own +# schema validator, before running anything. That is the tool, not the +# workflow — GitHub accepts the scope. To dry-run or run ci.yml under act, +# copy the workflows to a scratch dir and strip the one `code-quality: write` +# line there. (act's default image also ships no Go, so `setup-env`'s +# `go version` step exits 127 under act regardless.) paths: "**/*.yml": ignore: diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index df58eef4..123d011c 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -44,6 +44,15 @@ # lockfile + astro.config.mjs. Speeds up warm `astro check` / # `astro build` — unchanged content skips the parse + transform # pipeline. See #132. +# 7. chtypes artifact cache (~/.cache/chtypes/artifacts/abi6) keyed on +# chtypes.lock — the pinned ClickHouse-version .so/.dylib the SDK +# dlopens. Measured for today's one pinned line (26.6, linux-amd64): +# 316 MiB of libchtypes.so on disk, ~65 MiB as a stored archive, and +# 85 MiB over the wire on a MISS (the upstream .tar.gz). Fetched via +# scripts/fetch-chtypes.sh (--frozen: refuses anything the lock +# doesn't name), which the CLI makes idempotent even on a cache hit +# (a lightweight manifest check, not a re-download). Go test jobs +# that dial chtypes need this; see .github/workflows/README.md. name: Setup CI environment description: Caches (restore + automatic post-job save) and toolchains for WaveHouse CI @@ -66,6 +75,9 @@ inputs: astro: description: "Cache the Astro content collections (docs check/build)" default: "false" + chtypes: + description: "Fetch + cache the chtypes artifact(s) pinned in chtypes.lock (Go test jobs that dial chtypes only)" + default: "false" # Exact-key cache-hit flags, for consumers that want to skip work on a # warm cache (e.g. docs-build's pnpm-store prune). Saves are NOT gated on @@ -178,6 +190,36 @@ runs: restore-keys: | golangci-${{ runner.os }}- + # chtypes artifact cache — keyed on chtypes.lock (not go.mod/go.sum): + # the pinned file+sha256 per platform/line is the only thing that + # changes its contents. runner.arch is in the key on principle (all CI + # runners are ubuntu-latest amd64 today; a future arm64 runner must not + # restore amd64 .so files into its search path). + # + # The SDK's default fetch dir is revision-scoped: + # ~/.cache/chtypes/artifacts/abi/-. The path and the key + # prefix both name the ABI revision (6 at SDK v0.4.0), so a cache saved + # by an older SDK is never restored into the search path. When the SDK's + # ABI revision changes, bump `abi6` in both places and re-lock (see + # docs/development.md). + - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + if: ${{ inputs.chtypes == 'true' }} + id: chtypes-cache + with: + path: ~/.cache/chtypes/artifacts/abi6 + key: chtypes-abi6-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('chtypes.lock') }} + restore-keys: | + chtypes-abi6-${{ runner.os }}-${{ runner.arch }}- + + # Runs on both cache hit and miss: scripts/fetch-chtypes.sh --frozen is + # cheap on a hit (a manifest check against chtypes.lock, not a + # re-download — see the script) and it is what turns a restored-but- + # unverified cache entry back into a hash-checked one every run. + - name: Fetch pinned chtypes artifact + if: ${{ inputs.chtypes == 'true' }} + shell: bash + run: scripts/fetch-chtypes.sh + # No actions/setup-go: it spends ~9s/job downloading + extracting a # toolchain the runner can already provide. The image's preinstalled # `go` + GOTOOLCHAIN=auto (the Go ≥1.21 default) resolve go.mod's @@ -189,6 +231,16 @@ runs: # this step makes that cost visible and the post-job save captures it. # Trade-off: setup-go's inline problem matchers (PR-file annotations # on compile errors) are gone; the log output is unchanged. + # + # On a chtypes job the cold cost is paid TWICE, and the second fetch is + # invisible because it happens inside the step above: `go run @ver` + # resolves its toolchain from the pinned CLI module, not from this + # repo's go.mod, so it can land on a different patch than the one this + # step materialises (measured on an ambient go1.24.13: go1.27.1 for the + # CLI, go1.27.0 for `go 1.27` here). Both are ~80 MB and both live in + # ~/go/pkg/mod/golang.org/toolchain, so gomod-v1 absorbs both after the + # first save. A `toolchain` line in go.mod naming the same patch the CLI + # resolves would collapse it to one — not worth pinning for ~7s. - name: Verify Go resolves go.mod's toolchain if: ${{ inputs.go == 'true' }} shell: bash diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 575a8389..ee04e39f 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -40,7 +40,7 @@ Solid arrows are `needs` edges. Dotted arrows are **artifact polls** (invariant Break one of these knowingly or not at all. -1. **The aggregator job named `CI` is the only required status check.** The `main branch protection` ruleset requires `CI` and nothing else. The aggregator fails on any `failure`/`cancelled` need and treats `skipped` as passing — so path-filtered jobs (docs-only PRs skip the Go suites) and event-filtered jobs (title on pushes, deploys on PRs) never orphan the required check, and adding/renaming jobs never requires a ruleset edit. Consequence: every job that must gate merges **must be in the aggregator's `needs` list**. Two jobs are deliberately non-gating and excluded: `timing` (advisory wall-clock table) and `docs-preview` (the convenience Cloudflare preview deploy — `docs-build` already validates the build and *is* a need, so only the build gates; the preview deploy reports its own "Docs preview" check but, slow or failed, never delays or reds `CI`). +1. **The aggregator job named `CI` is the only required status check.** The `main branch protection` ruleset requires `CI` and nothing else. The aggregator fails on any `failure`/`cancelled` need and treats `skipped` as passing — so path-filtered jobs (docs-only PRs skip the Go suites) and event-filtered jobs (title on pushes, deploys on PRs) never orphan the required check, and adding/renaming jobs never requires a ruleset edit. Consequence: every job that must gate merges **must be in the aggregator's `needs` list**. Two jobs are deliberately non-gating and excluded: `timing` (advisory wall-clock table) and `docs-preview` (the convenience Cloudflare preview deploy — `docs-build` already validates the build and *is* a need, so only the build gates; the preview deploy reports its own "Docs preview" check but, slow or failed, never delays or reds `CI`). Outside `ci.yml` entirely, `.github/workflows/goreleaser-validate.yml` ("Release build validation") is advisory the same way — it runs on its own `pull_request` path filter and `workflow_dispatch`, never appears in the aggregator's `needs`, and making it gating would mean adding it there. 2. **A dedicated `coverage` job applies the consolidated gate, polling — not `needs`-ing — the suites.** Each suite (`unit`, `integration`, `e2e`) runs with `COV_DEFER=1` and uploads a `coverage-` fragment; the `coverage` job runs `make cov` (merge + every threshold gate) over all three — exactly like local `make ci`'s final step. Keeping it a separate job (not folded into e2e's tail) decouples the gate result from the e2e suite's pass/fail. Crucially it is `needs: changes` **only, not the suites**: a `needs` edge is a *scheduling* barrier — GitHub won't pick up a runner, check out, restore caches, or `pnpm install` until the needed jobs finish — so needing the suites would serialize this job's ~50s of setup onto the critical path after the last suite, for nothing (the setup doesn't depend on their results). Instead it starts at run creation, runs its setup in parallel with the suites, and blocks only at the merge by polling for the three fragments with [`scripts/ci/wait-artifact.sh`](../../scripts/ci/wait-artifact.sh) (fails fast if a producer concluded without producing). Tail on the critical path: ~10s, not ~50s. **The aggregator and `docs-deploy` must keep `coverage` *and* every suite in their `needs`** — the suites directly (a suite failure must red the gate even though `coverage` no longer needs them), and `coverage` (else a coverage-gate failure wouldn't block merge or a prod deploy). @@ -50,7 +50,7 @@ Break one of these knowingly or not at all. 5. **Trust domains.** Jobs that can reach deploy secrets (`docs-preview`, `docs-deploy`) check out **trusted `main`** and execute only files resolved from it — wrangler, the worker source, `wrangler.jsonc`, and any `scripts/ci/*.sh` they call ([#305](https://github.com/Wave-RF/WaveHouse/issues/305)). The only PR-derived input they touch is the static `docs-dist` artifact, consumed as data. Inline `run:` blocks in those jobs are acceptable (the workflow file itself is the reviewed surface); PR-tree *files* are not. Everything else (suites, lint, docs-build) runs the PR tree with no secrets beyond a read-mostly `GITHUB_TOKEN`. Fork PRs: secrets are absent and `docs-preview` skips itself. -6. **`ci.yml`'s caches are owned end-to-end by `setup-env`** ([.github/actions/setup-env](../actions/setup-env/action.yml)): each cache is a nested `actions/cache` step that restores inline and saves automatically at job end on an exact-key miss. No save steps in `ci.yml`. Trade-offs accepted: failed jobs don't save (restore-keys cushion the next run), and concurrent same-key misses produce benign "already exists" warnings. **Two cache steps live outside it**, both in `publish-dev.yml`, because that workflow doesn't use `setup-env` at all (it runs GoReleaser, not the test suites): a bare `actions/cache` owning the release build cache, and an `actions/cache/restore` that *reads* `gomod-v1` and owns nothing. Those two are the only `actions/cache*` steps outside the composite — keep it that way. A workflow that needs the shared module tree reads it restore-only; writing it belongs to the `ci.yml` jobs that run a full `go mod download`. +6. **`ci.yml`'s caches are owned end-to-end by `setup-env`** ([.github/actions/setup-env](../actions/setup-env/action.yml)): each cache is a nested `actions/cache` step that restores inline and saves automatically at job end on an exact-key miss. No save steps in `ci.yml`. Trade-offs accepted: failed jobs don't save (restore-keys cushion the next run), and concurrent same-key misses produce benign "already exists" warnings. **Two cache step definitions live outside it**, both in `publish-dev.yml`, because that workflow doesn't use `setup-env` at all (it runs GoReleaser, not the test suites): a bare `actions/cache` owning the release build cache, and an `actions/cache/restore` that *reads* `gomod-v1` and owns nothing. Both are instantiated once per matrix leg (`amd64`, `arm64` — each its own native `--single-target` build), so four cache-step runs total; still the only `actions/cache*` steps outside the composite — keep it that way. A workflow that needs the shared module tree reads it restore-only; writing it belongs to the `ci.yml` jobs that run a full `go mod download`. ## Coverage publishing @@ -80,15 +80,16 @@ Queue settings live in the `main branch protection` ruleset's `merge_queue` rule | pnpm store | `pnpm--` | any node job on miss | Store path resolved from pnpm at runtime. docs-build prunes before its save on a key rotation. | | Playwright Chromium | `playwright--` | docs-build | rehype-mermaid renders via headless Chrome at docs build. | | Astro content collections | `astro--` | lint / docs-build | Warm `astro check`/`build` skip unchanged content. | -| Go build objects (release) | `gobuild-v3--go-release-` | publish-dev (hand-rolled, not `setup-env`) | `~/.cache/go-build` from GoReleaser's 8-target cross-compile (~0.5 GB). Same family and key inputs as the CI flavors, `-release` suffix because cross-compiled objects share nothing with the native-only ones. Worth ≈2.5–7 min on every push to main (mean delta ≈4.8 min). | -| Go modules (release read) | `gomod-v1--` | nobody — **restore-only** | `publish-dev` reads `ci.yml`'s shared entry from `main`'s scope via `actions/cache/restore`, so its cross-compile isn't slowed by a cold module tree. No post-step save, so 0 GB of budget and no risk of a partial write to the shared key. | +| chtypes artifact | `chtypes-abi6---` | unit / integration / e2e via `setup-env` (shared) | `~/.cache/chtypes/artifacts/abi6` — the pinned ClickHouse-version `.so` the SDK dlopens; the SDK's default cache is per ABI revision (`abi/-`), and the revision is in both the path and the key prefix so an older SDK's cache is never restored. Measured for today's one pinned line (26.6, linux-amd64): **316 MiB on disk, ~0.06 GB stored**, and 85 MiB over the wire on a miss. Fetched by [`scripts/fetch-chtypes.sh`](../../scripts/fetch-chtypes.sh) (`--frozen`, refuses anything `chtypes.lock` doesn't name), which the CLI makes idempotent even on a cache hit — a manifest check, not a re-download. | +| Go build objects (release) | `gobuild-v3---go-release-` | publish-dev (hand-rolled, not `setup-env`) | `~/.cache/go-build` from each `publish-dev` leg's own native `goreleaser build --single-target` (~0.5 GB per arch). `runner.arch` is load-bearing in the key: both legs (`ubuntu-latest`, `ubuntu-24.04-arm`) report `runner.os == 'Linux'`, so without it they'd overwrite the same entry every run — a guaranteed miss on one of them, forever. Same family and key inputs as the CI flavors; `-release` suffix because these objects share nothing with the native test-build flavors. Budget is now **2 arches × 2 generations** of a native single-target cache, not the old 1 × 2 generations of a single 3-target cross-compile cache — still narrower than the pre-chtypes 8-target matrix (Windows/FreeBSD/darwin-amd64 dropped — chtypes publishes no artifact for any of them). | +| Go modules (release read) | `gomod-v1--` | nobody — **restore-only** | `publish-dev` reads `ci.yml`'s shared entry from `main`'s scope via `actions/cache/restore`, so its build isn't slowed by a cold module tree. No post-step save, so 0 GB of budget and no risk of a partial write to the shared key. | | CodeQL DB + deps | `codeql-dependencies-*`, `codeql-overlay-base-database-*` | GHAS default setup | **Not ours** — minted by GitHub's default CodeQL setup, not by any workflow in this repo, and not configurable here. ~0.4 GB. Listed so the budget arithmetic below is honest. | Deliberately **not** cached: `actions/setup-go`'s bundled cache (`cache: false` in `publish-dev.yml`, `release.yml` and `goreleaser-validate.yml`) — for different reasons per job. -It stores `~/go/pkg/mod` **and** `~/.cache/go-build` under one entry (~1 GB stored), keyed on the root `go.mod` — setup-go hashed `go.sum` through v6.2.0 and `go.mod` from v6.3.0, see [actions/setup-go#705](https://github.com/actions/setup-go/pull/705) — so roughly half of it re-stores the module tree `gomod-v1` already keeps once. `publish-dev.yml` opts out of that entry and caches the half that pays for itself on its own key (`gobuild-v3--go-release-`, ~0.5 GB): its GoReleaser step takes 36–246 s warm versus 401–446 s cold, so dropping the build objects outright would cost roughly 2.5–7 minutes on every push to main (mean delta ≈4.8 min across those runs). Those timings were measured with setup-go's bundled entry, which also held `~/go/pkg/mod` — so `publish-dev` additionally *restores* (never saves) `gomod-v1` from `main`'s scope, keeping the module tree warm too. Without that restore the job would re-download ~112 MB per push and land above the warm range this table quotes. +It stores `~/go/pkg/mod` **and** `~/.cache/go-build` under one entry (~1 GB stored), keyed on the root `go.mod` — setup-go hashed `go.sum` through v6.2.0 and `go.mod` from v6.3.0, see [actions/setup-go#705](https://github.com/actions/setup-go/pull/705) — so roughly half of it re-stores the module tree `gomod-v1` already keeps once. `publish-dev.yml` opts out of that entry and caches the half that pays for itself on its own key (`gobuild-v3---go-release-`, ~0.5 GB per arch): its GoReleaser step takes 36–246 s warm versus 401–446 s cold, so dropping the build objects outright would cost roughly 2.5–7 minutes on every push to main (mean delta ≈4.8 min across those runs). Those timings were measured with setup-go's bundled entry, which also held `~/go/pkg/mod` — so `publish-dev` additionally *restores* (never saves) `gomod-v1` from `main`'s scope, keeping the module tree warm too. Without that restore the job would re-download ~112 MB per push and land above the warm range this table quotes. -`release.yml` keeps the plain opt-out — no re-cache. After this change nothing mints a `setup-go-*` key at all, so turning its bundled cache back on would be a cold miss *and* a fresh ~1 GB save rather than a hit. What is warm is `publish-dev`'s `gobuild-v3--go-release-` entry, which a tag run could restore from the default branch's scope — but a tagged release is rare and not latency-sensitive, so it isn't worth a hand-rolled restore step. +`release.yml` keeps the plain opt-out — no re-cache. After this change nothing mints a `setup-go-*` key at all, so turning its bundled cache back on would be a cold miss *and* a fresh ~1 GB save rather than a hit. What is warm is `publish-dev`'s `gobuild-v3---go-release-` entries (one per arch), which a tag run could restore from the default branch's scope — but a tagged release is rare and not latency-sensitive, so it isn't worth a hand-rolled restore step. Re-enabling the bundled cache there would be strictly negative, not merely unhelpful: cache writes are scoped to the ref that made them, so a save from `refs/tags/v1.0.0` can never be read by `refs/tags/v1.0.1`, by `main`, or by a PR — only by a re-run of that same tag. It would be a ~1 GB entry per release that nothing but a retry can ever read. If release wall-clock ever does matter, the lever is `actions/cache/restore` on `publish-dev`'s key: restore-only, so it reads `main`'s warm entry and never writes a tag-scoped one. @@ -110,7 +111,7 @@ Include every family the rotation orphans, not just the renamed one — e.g. tur **Sizing policy — the repo cache budget is 10 GB, hard.** Past it GitHub LRU-evicts, so warm entries disappear mid-run and builds silently get slower. Budget for **two live generations**: a `go.mod`/`go.sum` or lockfile bump mints a whole new set while the previous one is still warm, so the steady state is ~2× a single generation. That is why `~/go/pkg/mod` is cached **once** (`gomod-v1`) rather than folded into each suffixed build cache — doing the latter stored the module tree five times over, five entries of ~0.9-1.2 GB each, ~5.2 GB per generation, and #438's 24-module bump pushed the repo to 10.53 GB ([#443](https://github.com/Wave-RF/WaveHouse/issues/443)). -Steady state after the split is roughly 5 GB of the 10 — two generations of `gomod-v1` + the five `gobuild-v3` flavors + the release build cache, plus the node-side caches and CodeQL. Before adding a cache or widening an existing `path:`, check the current footprint and confirm two generations still fit: +Steady state after the split is roughly 5 GB of the 10 — two generations of `gomod-v1` + the five `gobuild-v3` flavors + the release build cache (now two entries, one per arch, since the key carries `runner.arch`) — plus the node-side caches and CodeQL — plus ~0.13 GB for two generations of the `chtypes-abi6` artifact cache (~0.06 GB stored per generation, one pinned line today — the `.so` is 316 MiB on disk but compresses ~5x). Before adding a cache or widening an existing `path:`, check the current footprint and confirm two generations still fit: ```bash gh api repos/Wave-RF/WaveHouse/actions/cache/usage \ @@ -121,6 +122,12 @@ gh api repos/Wave-RF/WaveHouse/actions/caches --paginate \ Never add a per-job copy of content that is a pure function of a lockfile — key it once, unsuffixed, and let every job share it. +## chtypes artifacts + +`unit`, `integration` and `e2e` link the chtypes SDK (cgo dlopen of a per-ClickHouse-version `.so`/`.dylib`) and need the artifact for the line the test suite dials — today ClickHouse 26.6, matching `tests/integration/setup_test.go`'s pinned container. `WAVEHOUSE_TEST_REQUIRE_CHTYPES=1` (job-level `env:` on all three) makes `typelayer.TestEngine` `t.Fatal()` if the artifact is missing instead of `t.Skip()`ing — CI must never quietly skip chtypes-backed tests. + +`chtypes.lock` (repo root) pins the exact file + sha256 per platform/line; `scripts/fetch-chtypes.sh` wraps the SDK's own CLI with `--frozen --lock chtypes.lock`, so a fetch here can only install what the lock names, never the rolling `artifacts` release. `setup-env`'s `chtypes: "true"` input (see [Cache inventory](#cache-inventory)) restores `~/.cache/chtypes/artifacts/abi6` and always re-runs the fetch script afterward — cheap on a hit (a manifest check, not a re-download) and what turns a restored-but-unverified cache entry back into a hash-checked one every run. A lock is specific to the SDK's ABI revision: a build from another revision is never selected, so after an SDK bump that changes the revision (6 at v0.4.0) `--frozen` fails with `CHTYPES_ARTIFACT_PINNED` or `CHTYPES_ARTIFACT_UNPUBLISHED` until the lock is regenerated the same way, and the `abi6` path and key prefix in `setup-env` move with it. Widening the pinned line set is a two-step: `scripts/fetch-chtypes.sh ` locally to update `chtypes.lock`, then add the line to `LOCK_LINES` in that script. Two hash mismatches are possible and they behave differently — don't read one as the other. **Upstream republished a pinned line under a new sha256**: `--frozen` refuses the artifact the lock does not name, and both `Dockerfile.goreleaser`'s fetch and CI's cache-miss fetch fail, until `chtypes.lock` is regenerated per platform (`go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch 26.6 --lock chtypes.lock --platform `, once each for `darwin-arm64`, `linux-amd64`, `linux-arm64`, without `--frozen`) — and `goreleaser-validate.yml`'s image job, which exercises this same fetch on every PR touching `chtypes.lock` or the release workflows, is what surfaces a republish at PR time rather than mid-release. **The cache holds a library the current lock no longer names** (a re-lock landed, so the exact key missed and `restore-keys` handed back the previous generation): this does *not* fail — measured, the CLI reports `is present but hashes … (want …) — replacing` and re-downloads, then the post-job save mints the new generation. So a re-lock costs one cold fetch per Go job on the first run and nothing after. + ## Timing (steady state, full pipeline) The non-gating **Timing summary** job writes a per-job wall-clock table to every run's Summary page. Reference shape: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4761927..29297f73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,11 @@ jobs: if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest timeout-minutes: 20 + # WAVEHOUSE_TEST_REQUIRE_CHTYPES=1: typelayer.TestEngine t.Fatal()s instead of + # t.Skip()ing when the pinned artifact isn't installed — CI must never + # silently skip the chtypes-backed tests it has the artifact for. + env: + WAVEHOUSE_TEST_REQUIRE_CHTYPES: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -206,6 +211,7 @@ jobs: uses: ./.github/actions/setup-env with: go-cache-suffix: "-unit" + chtypes: "true" - name: Run Go unit tests + SDK vitest tests run: make test-unit test-ts COV_DEFER=1 - name: Upload coverage fragment @@ -223,6 +229,9 @@ jobs: if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest timeout-minutes: 20 + # See the unit job's comment on WAVEHOUSE_TEST_REQUIRE_CHTYPES. + env: + WAVEHOUSE_TEST_REQUIRE_CHTYPES: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -237,7 +246,8 @@ jobs: uses: ./.github/actions/setup-env with: go-cache-suffix: "-integration" - node: "false" # pure Go + testcontainers ClickHouse + node: "false" # Go + testcontainers ClickHouse only + chtypes: "true" - name: Run Go integration tests run: make test-integration COV_DEFER=1 - name: Upload coverage fragment @@ -259,7 +269,12 @@ jobs: needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest - timeout-minutes: 25 + # 35, not 25: a cold first push pays the cover-instrumented cgo build plus + # the chtypes artifact fetch on top of the suite (W7 estimated 14-19 min). + timeout-minutes: 35 + # See the unit job's comment on WAVEHOUSE_TEST_REQUIRE_CHTYPES. + env: + WAVEHOUSE_TEST_REQUIRE_CHTYPES: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -279,6 +294,7 @@ jobs: uses: ./.github/actions/setup-env with: go-cache-suffix: "-e2e-cov" + chtypes: "true" # `-j` builds the prereqs (build-ts ∥ build-cover) concurrently, # then runs the orchestrator: ClickHouse testcontainer + the cover # binary + the SDK vitest suite. diff --git a/.github/workflows/goreleaser-validate.yml b/.github/workflows/goreleaser-validate.yml index b09cd84e..ee4112c9 100644 --- a/.github/workflows/goreleaser-validate.yml +++ b/.github/workflows/goreleaser-validate.yml @@ -1,18 +1,47 @@ -name: GoReleaser snapshot validation +name: Release build validation -# Catches goreleaser config + Go-compile breakage at PR time, before -# it could block a real tag-time release. Snapshot mode implies -# --skip=publish, so nothing is pushed. +# PR-time proof that a real release would build. Nothing is published: +# `--snapshot` implies `--skip=publish`, and the image is built to +# `type=cacheonly`, so no registry is touched and no GitHub Release is made. # -# Path-filtered to goreleaser config / dockerfile changes only; -# go.mod / go.sum bumps are validated post-merge by publish-dev.yml -# running the full multi-target pipeline. `goreleaser build -# --single-target` exercises config parseability + Go compile for -# the host platform (release does not accept --single-target). +# It covers the three things a tag-time failure would otherwise block on: +# +# 1. `goreleaser check` — .goreleaser.yaml's schema. +# 2. a build per target — all THREE, each on its own native runner, which +# is the only way to compile them now that cgo +# rules out cross-compiling (darwin in particular: +# see the header of .goreleaser.yaml). The old +# `--single-target` job ran on linux/amd64 only and +# was blind to the other two by construction. +# 3. the image — the real multi-arch `docker buildx build` over +# deployments/Dockerfile.goreleaser with the +# snapshot binaries in the exact context layout +# release.yml assembles, including the chtypes +# artifact fetch from chtypes.lock. A stale lock, +# an unfetchable artifact or a Dockerfile mistake +# fails here rather than mid-release. +# +# Path filter: go.mod / go.sum are in it because a dependency bump is the +# realistic way a platform breaks (the darwin cgo C file that killed +# cross-compiling arrived with a prometheus/client_golang bump), and +# publish-dev.yml no longer builds darwin at all, so nothing else on main +# would catch it. chtypes.lock / fetch-chtypes.sh / _colors.sh are in it +# because the image build consumes all three — the assemble step copies +# _colors.sh into the context and the Dockerfile's fetch stage sources it, so +# an edit there can break the image with nothing else on a PR to catch it. +# +# NOT a required check: the `main branch protection` ruleset requires only +# `CI`, the aggregator in ci.yml. Making this gating means adding it to that +# aggregator's `needs`, which is a ci.yml change. on: pull_request: paths: - .goreleaser.yaml + - go.mod + - go.sum + - chtypes.lock + - scripts/fetch-chtypes.sh + - scripts/_colors.sh - deployments/Dockerfile.goreleaser - .github/workflows/release.yml - .github/workflows/publish-dev.yml @@ -27,34 +56,143 @@ concurrency: cancel-in-progress: true jobs: - validate: - name: Validate snapshot build + config: + name: Validate .goreleaser.yaml runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Runs on PR-authored code; nothing here needs authenticated git. + persist-credentials: false + + - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + with: + # Same v2 line as release.yml so this validates against the same + # goreleaser version a real release uses. + version: "~> v2" + args: check + + build: + name: Build ${{ matrix.goos }}/${{ matrix.goarch }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + # Keep going: knowing that two of three targets are broken is more useful + # on a PR than the first failure alone. (release.yml is fail-fast for the + # opposite reason — there, a partial result is worthless.) + fail-fast: false + matrix: + # Must mirror release.yml's matrix and .goreleaser.yaml's declared + # goos/goarch. + include: + - goos: linux + goarch: amd64 + runner: ubuntu-latest + - goos: linux + goarch: arm64 + runner: ubuntu-24.04-arm + - goos: darwin + goarch: arm64 + runner: macos-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # goreleaser uses `git describe` to derive the snapshot # version; needs full tag history. fetch-depth: 0 - # Runs on PR-authored code; nothing here needs authenticated git. persist-credentials: false - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" - # Single-target build is fast enough that the bundled - # cache's post-step save costs more than a cold - # `go mod download`. + # A single-target build is fast enough that the bundled cache's + # post-step save costs more than a cold `go mod download` — and a + # cache saved from a PR's ref can only ever be read by that same PR. cache: false - - name: Run GoReleaser snapshot build + - name: Assert the runner is ${{ matrix.goos }}/${{ matrix.goarch }} + shell: bash + run: | + set -euo pipefail + host_os=$(go env GOOS) + host_arch=$(go env GOARCH) + echo "runner=${{ matrix.runner }} host=${host_os}/${host_arch}" + if [ "$host_os" != "${{ matrix.goos }}" ] || [ "$host_arch" != "${{ matrix.goarch }}" ]; then + echo "::error::${{ matrix.runner }} is ${host_os}/${host_arch} but the matrix says ${{ matrix.goos }}/${{ matrix.goarch }} — this leg is not validating what it claims to" + exit 1 + fi + + - name: Snapshot build uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: - # Same v2 line as release.yml so the snapshot validates - # against the same goreleaser version a real release uses. version: "~> v2" - # `build` (not `release`) so --single-target is accepted; - # release-only steps (archive / docker / checksums) are - # validated post-merge by publish-dev.yml. - args: build --snapshot --clean --single-target + # `--snapshot` because a PR head carries no release tag. + args: build --snapshot --clean --single-target --output dist/wavehouse + + - name: Describe the binary + shell: bash + run: | + set -euo pipefail + file dist/wavehouse + ls -l dist/wavehouse + + # Only the Linux binaries feed the image job; uploading the darwin one + # too would cost a transfer nothing reads. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: matrix.goos == 'linux' + with: + name: snapshot-linux-${{ matrix.goarch }} + path: dist/wavehouse + if-no-files-found: error + retention-days: 1 + + image: + name: Build the multi-arch image + # A plain `needs` on a fail-fast: false matrix means ANY failing leg skips + # this job — including the darwin one, which contributes nothing here. That + # is the intended trade: if a target cannot even compile, the image result + # is noise, and the build job's own red is the finding. + needs: build + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: snapshot-linux-* + path: staging + + - name: Assemble the image build context + shell: bash + run: | + set -euo pipefail + mkdir -p ctx/scripts + cp chtypes.lock ctx/ + cp scripts/fetch-chtypes.sh scripts/_colors.sh ctx/scripts/ + for goarch in amd64 arm64; do + mkdir -p "ctx/linux/${goarch}" + install -m 0755 "staging/snapshot-linux-${goarch}/wavehouse" \ + "ctx/linux/${goarch}/wavehouse" + done + find ctx -type f -printf '%M %s %p\n' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + # `type=cacheonly`: run the full build for both platforms — including the + # chtypes artifact fetch and sha256 verification against chtypes.lock — + # and throw the result away. No registry credentials, so this works on + # fork PRs too. + - name: Build (no push) + shell: bash + run: | + set -euo pipefail + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --file deployments/Dockerfile.goreleaser \ + --output type=cacheonly \ + ctx diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index c026cfc2..d8c46e10 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -4,15 +4,20 @@ name: Publish dev image # - :dev rolling pointer to the latest main commit # - :dev- immutable point-in-time reference # -# Reuses the goreleaser pipeline from release.yml. A synthetic local -# tag (v0.0.0-dev.) anchors goreleaser's .Version on -# HEAD; the tag is never pushed to origin. WAVEHOUSE_DEV=1 switches -# .goreleaser.yaml to dev image tags and suppresses the GitHub -# Release. +# Same shape as release.yml — native build jobs, then one job that assembles +# and pushes — because cgo makes cross-compiling impossible here (see the +# header of .goreleaser.yaml for the measurements). The difference is scope: +# a dev build publishes ONLY the image, so it builds only the two Linux +# targets that go into it. darwin/arm64 is not built here; its PR-time proof +# is goreleaser-validate.yml, which builds all three on a config or +# go.mod/go.sum change. # -# Real tagged releases (`v*`) flow through release.yml against the -# same .goreleaser.yaml without WAVEHOUSE_DEV set, producing `:vX.Y.Z` -# plus a moving channel pointer — `:latest` for a stable tag, but +# A synthetic local tag (v0.0.0-dev.) anchors goreleaser's .Version +# on HEAD; the tag is never pushed to origin. +# +# Real tagged releases (`v*`) flow through release.yml against the same +# .goreleaser.yaml, producing archives, a GitHub Release and `:vX.Y.Z` plus a +# moving channel pointer — `:latest` for a stable tag, but # `:alpha`/`:beta`/`:rc`/`:next` for a prerelease, which therefore never # touches `:latest`. Cleanup of old dev- tags is handled by # cleanup-ghcr.yml. @@ -23,11 +28,9 @@ on: branches: [main] workflow_dispatch: +# Least privilege by default; only `publish` is elevated. permissions: contents: read - packages: write - id-token: write # OIDC for the build-provenance attestation (Sigstore) - attestations: write # write the build-provenance attestation concurrency: # The :dev tag should track HEAD, so cancel any in-flight build @@ -38,18 +41,28 @@ concurrency: cancel-in-progress: true jobs: - publish: - name: Build + push :dev image - runs-on: ubuntu-latest - timeout-minutes: 30 + build: + name: Build linux/${{ matrix.goarch }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + fail-fast: true + matrix: + # Only what the image needs. Must stay a subset of + # .goreleaser.yaml's declared goos/goarch. + include: + - goarch: amd64 + runner: ubuntu-latest + - goarch: arm64 + runner: ubuntu-24.04-arm steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # goreleaser reads full git history for changelog + commit - # info. Matches release.yml. + # goreleaser reads full git history for commit info. Matches + # release.yml. fetch-depth: 0 - # Same reasoning as release.yml: a third-party action and a - # cross-compile run here, and nothing needs authenticated git. + # Same reasoning as release.yml: a third-party action and a full + # `go mod download` run here, and nothing needs authenticated git. persist-credentials: false - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 @@ -64,32 +77,30 @@ jobs: # `gomod-v1` — the duplication #443 is about. cache: false - # The other half is the reason this job is fast, so cache it on its - # own. GoReleaser cross-compiles 8 targets here (4 goos × 2 goarch), - # and the last 20 runs split cleanly on whether these objects were - # warm: GoReleaser finished in 36-246s with them, 401-446s without. - # (Measured on setup-go's bundled entry, which carried this same - # ~/.cache/go-build tree — this key is new here.) Dropping them - # outright would cost roughly 2.5-7 min per push to main (the envelope - # between those two clusters; mean delta ~4.8 min). + # The build objects are the reason this job is fast, so cache them on + # their own key. # - # Release-scoped suffix: these objects are cross-compiled for 8 - # GOOS/GOARCH pairs and share nothing with ci.yml's native-only - # flavors, so `-release` keeps both sides from restoring bytes the - # other can't use. Same `gobuild-v3` family and key inputs as - # setup-env's (see .github/workflows/README.md); ~0.5 GB rather than - # the ~1 GB the bundled cache held. + # `runner.arch` is new in the key and load-bearing: both legs of this + # matrix report `runner.os == 'Linux'`, so the pre-chtypes key + # (`gobuild-v3--go-release-…`) would have the amd64 and arm64 legs + # overwriting each other's entry every run — a guaranteed miss on one of + # them, forever. Still the `gobuild-v3` family and the same key inputs as + # setup-env's (see .github/workflows/README.md); `-release` keeps these + # from restoring into ci.yml's native test-build flavors, which carry + # different tags and a coverage instrumentation variant. - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/go-build - key: gobuild-v3-${{ runner.os }}-go-release-${{ hashFiles('**/go.mod', '**/go.sum') }} + key: gobuild-v3-${{ runner.os }}-${{ runner.arch }}-go-release-${{ hashFiles('**/go.mod', '**/go.sum') }} restore-keys: | - gobuild-v3-${{ runner.os }}-go-release- + gobuild-v3-${{ runner.os }}-${{ runner.arch }}-go-release- # And read — never write — ci.yml's shared module tree, so dropping # setup-go's bundled cache doesn't leave this job re-downloading ~112 MB # of modules on every push. This workflow runs on main, the same scope - # ci.yml saves `gomod-v1` into, so the entry is there to hit. + # ci.yml saves `gomod-v1` into, so the entry is there to hit. No arch in + # this key: the module cache is downloaded source, identical on both + # legs, so the arm64 leg reads the entry amd64 jobs wrote. # # restore, not cache: a full actions/cache would add a post-step save, # and this job has no business writing the entry every ci.yml Go job @@ -103,11 +114,91 @@ jobs: restore-keys: | gomod-v1-${{ runner.os }}- - # dockers_v2 builds the linux/amd64+arm64 manifest via `docker buildx`; - # the GitHub-hosted runner's default docker driver can't build - # multi-platform images, so create a docker-container builder. No QEMU - # needed — Dockerfile.goreleaser stages are $BUILDPLATFORM-pinned / - # COPY-only by design. + # Before the build: if `ubuntu-latest` ever moves to arm64 this would + # otherwise publish two identical arm64 binaries under different names. + - name: Assert the runner is linux/${{ matrix.goarch }} + shell: bash + run: | + set -euo pipefail + host_os=$(go env GOOS) + host_arch=$(go env GOARCH) + echo "runner=${{ matrix.runner }} host=${host_os}/${host_arch}" + if [ "$host_os" != linux ] || [ "$host_arch" != "${{ matrix.goarch }}" ]; then + echo "::error::${{ matrix.runner }} is ${host_os}/${host_arch} but the matrix says linux/${{ matrix.goarch }} — cgo cannot cross-compile these targets, so this run would ship the wrong binary" + exit 1 + fi + + - name: Create synthetic dev tag + # `goreleaser build` needs a tag on HEAD to derive .Version. Created + # locally with --force (retries on the same commit don't collide) and + # never pushed to origin. The pushed image tags are computed in the + # publish job below. + run: git tag --force "v0.0.0-dev.${GITHUB_SHA::7}" + + - name: Build + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + with: + # Same v2 line as release.yml so dev exercises the same + # goreleaser version a real release will. + version: "~> v2" + args: build --clean --single-target --output dist/wavehouse + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wavehouse-linux-${{ matrix.goarch }} + path: dist/wavehouse + if-no-files-found: error + retention-days: 1 + + publish: + name: Push :dev image + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + # A job-level block REPLACES the workflow-level one outright, so + # `contents: read` has to be restated here or `actions/checkout` has no + # token to read the repo with. + contents: read + packages: write + id-token: write # OIDC for the build-provenance attestation (Sigstore) + attestations: write # write the build-provenance attestation + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: wavehouse-linux-* + path: staging + + - name: Assemble the image build context + shell: bash + run: | + set -euo pipefail + mkdir -p ctx/scripts + # Dockerfile.goreleaser's chtypes-fetch stage needs these three at + # their normal repo-relative paths. + cp chtypes.lock ctx/ + cp scripts/fetch-chtypes.sh scripts/_colors.sh ctx/scripts/ + + for goarch in amd64 arm64; do + bin="staging/wavehouse-linux-${goarch}/wavehouse" + if [ ! -f "$bin" ]; then + echo "::error::missing build artifact for linux/${goarch} (${bin})" + exit 1 + fi + # actions/upload-artifact's zip carries no unix mode bits, so the + # executable bit does not survive the round trip. + mkdir -p "ctx/linux/${goarch}" + install -m 0755 "$bin" "ctx/linux/${goarch}/wavehouse" + done + find ctx -type f -printf '%M %s %p\n' + + # The default docker driver can't build multi-platform images, so create + # a docker-container builder. No QEMU needed — Dockerfile.goreleaser's + # stages are $BUILDPLATFORM-pinned or COPY-only by design. - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 @@ -118,30 +209,36 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Create synthetic dev tag - # `goreleaser release` requires a tag on HEAD to derive - # .Version. Created locally with --force (retries on the - # same commit don't collide) and never pushed to origin. - # The pushed image tags come from .goreleaser.yaml's - # WAVEHOUSE_DEV branch (`dev-` + `dev`). - run: git tag --force "v0.0.0-dev.${GITHUB_SHA::7}" - - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + # `:dev-` matches cleanup-ghcr.yml's `^dev-[0-9a-f]+$` regex. + # `:dev` is written as a literal rather than derived from a channel + # variable: a push to main that forgot to set something must never be + # able to publish `:latest`. + - name: Build and push the multi-arch dev image + shell: bash env: - # Switches .goreleaser.yaml into dev mode: dev/dev- - # image tags, dev OCI labels, GitHub Release suppressed. - WAVEHOUSE_DEV: "1" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - # Same v2 line as release.yml so dev exercises the same - # goreleaser version a real release will. - version: "~> v2" - args: release --clean + IMAGE: ghcr.io/wave-rf/wavehouse + run: | + set -euo pipefail + version="0.0.0-dev.${GITHUB_SHA::7}" + created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --file deployments/Dockerfile.goreleaser \ + --tag "${IMAGE}:dev-${GITHUB_SHA}" \ + --tag "${IMAGE}:dev" \ + --label "org.opencontainers.image.title=WaveHouse (dev)" \ + --label "org.opencontainers.image.description=Schema-aware real-time API gateway for ClickHouse — rolling dev build from main" \ + --label "org.opencontainers.image.url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.version=${version}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "org.opencontainers.image.created=${created}" \ + --push \ + ctx # Attest the rolling dev image (multi-arch manifest-list digest) and store # the attestation alongside it in GHCR. Free for public repos via Sigstore. - # Dev binaries aren't distributed (the GitHub Release is suppressed), so + # Dev binaries aren't distributed (no GitHub Release, no archives), so # only the image is attested. Mirrors release.yml. - name: Resolve pushed dev image digest id: image diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd6fa9c4..0672b07b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,30 @@ name: Release +# Shape: three NATIVE build jobs, one publish job. +# +# cgo made cross-compiling impossible for darwin (measured — see the header of +# .goreleaser.yaml: a dependency's own darwin C file needs Mach headers no +# Linux runner can supply), and the GoReleaser features that solve it — +# split/merge, `builder: prebuilt` — are Pro-only, with no `--skip=build` in +# OSS. So GoReleaser compiles, once per runner, and this workflow does the +# assembling: archives, checksums, GitHub Release, and the multi-arch GHCR +# image built straight from `deployments/Dockerfile.goreleaser` (the same +# `//wavehouse` context layout GoReleaser's dockers_v2 used to hand +# it, so that file is unchanged). +# +# Every runner label here is free for public repos: ubuntu-latest, +# ubuntu-24.04-arm, macos-latest. + on: push: tags: - "v*" +# Least privilege by default. Only `publish` is elevated — the three build +# jobs compile a tag's code on three runners and must not be able to write a +# release, push a package, or mint an attestation. permissions: - contents: write # create the GitHub Release (goreleaser) - packages: write # push the image to GHCR - id-token: write # OIDC for build-provenance attestations (Sigstore) - attestations: write # write the build-provenance attestations + contents: read concurrency: # Per-tag, never cancelling: two runs of the SAME tag (a re-run after a @@ -19,54 +34,195 @@ concurrency: cancel-in-progress: false jobs: - release: - name: Release - runs-on: ubuntu-latest - timeout-minutes: 30 + build: + name: Build ${{ matrix.goos }}/${{ matrix.goarch }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 25 + strategy: + # A release is all-or-nothing: there is no such thing as publishing two + # of the three platforms, so stop the others as soon as one fails. + fail-fast: true + matrix: + # Must mirror .goreleaser.yaml's `builds[0].goos/goarch/ignore`. + # `--single-target` compiles the RUNNER's platform and ignores that + # list, so this matrix is what actually decides what gets built — the + # guard step below is what keeps the two from drifting apart. + include: + - goos: linux + goarch: amd64 + runner: ubuntu-latest + - goos: linux + goarch: arm64 + runner: ubuntu-24.04-arm + - goos: darwin + goarch: arm64 + runner: macos-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + # goreleaser derives .Version from the tag and .ShortCommit from the + # commit; `git describe` needs the tag history. fetch-depth: 0 - # The most privileged job here: contents+packages+attestations write, - # running a third-party action, a downloaded goreleaser binary, a full - # `go mod download` + cross-compile, and a Docker build — any one of - # which could read a persisted token out of .git/config and push. No - # authenticated git is needed: GoReleaser talks to the API via - # GITHUB_TOKEN, `git describe` is local, and the fetch is already done. + # Nothing here needs authenticated git: `git describe` is local and + # this job pushes nothing. A downloaded goreleaser binary and a full + # `go mod download` run before anything else, and a persisted token + # in .git/config would be readable by both. persist-credentials: false - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: "go.mod" - # Opting out keeps a ~1 GB archive — the module tree plus this - # job's cross-compile objects — out of the 10 GB repo budget. - # (That entry is keyed on the root go.mod, not go.sum: setup-go - # hashed go.sum through v6.2.0 and go.mod from v6.3.0 — - # actions/setup-go#705.) - # - # Turning it back on here would be strictly negative. Nothing - # mints that key any more — publish-dev.yml and - # goreleaser-validate.yml opt out too, and ci.yml runs no - # setup-go — so the restore is a guaranteed miss. Worse, cache - # writes are scoped to the ref that made them: a save from - # refs/tags/v1.0.0 can never be read by refs/tags/v1.0.1, by main, - # or by a PR — only by a re-run of that same tag. It would be a - # ~1 GB entry per release that nothing but a retry can ever read. - # - # What IS warm is publish-dev's gobuild-v3--go-release- entry, - # restorable from the default branch's scope. A tagged release is - # rare and not latency-sensitive, so it isn't worth a hand-rolled - # restore step here — but that's the lever if it ever is. - # - # Same cache: false as publish-dev.yml and goreleaser-validate.yml; - # different follow-up (publish-dev re-caches the useful half). #443. + # No cache. Cache writes are scoped to the ref that made them, so a + # save from refs/tags/v1.0.0 can never be read by refs/tags/v1.0.1, + # by main, or by a PR — only by a re-run of that same tag. It would + # be a ~1 GB entry per release that nothing but a retry can read. + # A tagged release is rare and not latency-sensitive. (publish-dev + # caches instead, on main's scope, where it is read back. #443.) cache: false - # dockers_v2 builds the linux/amd64+arm64 manifest via `docker buildx`; - # the GitHub-hosted runner's default docker driver can't build + # Before the build, not after: if a runner label ever changes + # architecture under us (ubuntu-latest moving to arm64 would do it), + # `--single-target` would silently produce the wrong binary and this + # workflow would publish it under the matrix's name. + - name: Assert the runner is ${{ matrix.goos }}/${{ matrix.goarch }} + shell: bash + run: | + set -euo pipefail + host_os=$(go env GOOS) + host_arch=$(go env GOARCH) + echo "runner=${{ matrix.runner }} host=${host_os}/${host_arch}" + if [ "$host_os" != "${{ matrix.goos }}" ] || [ "$host_arch" != "${{ matrix.goarch }}" ]; then + echo "::error::${{ matrix.runner }} is ${host_os}/${host_arch} but the matrix says ${{ matrix.goos }}/${{ matrix.goarch }} — cgo cannot cross-compile these targets, so this run would ship the wrong binary" + exit 1 + fi + + - name: Build + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 + with: + # Pin to the v2 line so a v3 release breaks loudly instead of + # silently changing release behavior. + version: "~> v2" + # `build`, not `release`: this job only compiles. `--output` copies + # the binary to a fixed path so nothing downstream has to know + # GoReleaser's per-target dist directory name (`wavehouse_darwin_ + # arm64_v8.0` and friends — the GOARM64 suffix is not obvious). + args: build --clean --single-target --output dist/wavehouse + + - name: Describe the binary + shell: bash + run: | + set -euo pipefail + file dist/wavehouse + ls -l dist/wavehouse + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: wavehouse-${{ matrix.goos }}-${{ matrix.goarch }} + path: dist/wavehouse + if-no-files-found: error + # An intra-run hand-off to `publish`, not a deliverable — the + # deliverables are the release assets and the image. + retention-days: 1 + + publish: + name: Publish + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # create the GitHub Release + packages: write # push the image to GHCR + id-token: write # OIDC for build-provenance attestations (Sigstore) + attestations: write # write the build-provenance attestations + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # `git describe` picks the previous v* tag for the release notes. + fetch-depth: 0 + # The most privileged job here: contents+packages+attestations write, + # running a Docker build that fetches from the network — which could + # read a persisted token out of .git/config and push. No + # authenticated git is needed: `gh` reaches the API via GH_TOKEN, + # `git describe` is local, and the fetch is already done. + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: wavehouse-* + path: staging + + # Resolve the moving GHCR tag BEFORE anything is published, so a + # malformed tag fails here rather than after a multi-arch image push. + # The script rejects anything that isn't semver-shaped, so a typo can't + # fall through to `latest`. It is also the single source of truth for + # "is this a prerelease" below — `latest` means stable, by definition — + # rather than a second semver parser that could disagree with it. + - name: Resolve release channel + shell: bash + run: | + set -euo pipefail + channel="$(scripts/ci/release-channel.sh "$GITHUB_REF_NAME")" + echo "WAVEHOUSE_CHANNEL=${channel}" >> "$GITHUB_ENV" + echo "::notice::${GITHUB_REF_NAME} publishes ghcr.io/wave-rf/wavehouse:${GITHUB_REF_NAME} + :${channel}" + + - name: Assemble archives, checksums, and the image build context + shell: bash + run: | + set -euo pipefail + mkdir -p dist ctx/scripts + + # Dockerfile.goreleaser's chtypes-fetch stage needs these three at + # their normal repo-relative paths — the same files GoReleaser's + # dockers_v2.extra_files used to copy into its temp context. + cp chtypes.lock ctx/ + cp scripts/fetch-chtypes.sh scripts/_colors.sh ctx/scripts/ + + for target in linux/amd64 linux/arm64 darwin/arm64; do + goos="${target%/*}" + goarch="${target#*/}" + bin="staging/wavehouse-${goos}-${goarch}/wavehouse" + if [ ! -f "$bin" ]; then + echo "::error::missing build artifact for ${target} (${bin})" + exit 1 + fi + # actions/upload-artifact zips its input and that zip carries no + # unix mode bits, so the executable bit does NOT survive the round + # trip. Restore it before anything tars or COPYs the file. + chmod +x "$bin" + + stage="$(mktemp -d)" + cp "$bin" "$stage/wavehouse" + # NOTICE is not optional: the repo is Apache-2.0 and §4(d) makes + # every redistributor of these archives inherit an attribution + # obligation they cannot satisfy from a tarball carrying only + # LICENSE. CHANGELOG.md is deliberately absent — 300+ KB of + # development history, one click away on GitHub, and the release + # page already renders the notes. + cp LICENSE NOTICE README.md "$stage/" + tar -czf "dist/wavehouse_${goos}_${goarch}.tar.gz" \ + -C "$stage" wavehouse LICENSE NOTICE README.md + rm -rf "$stage" + + # Dockerfile.goreleaser COPYs `$TARGETPLATFORM/wavehouse`. + if [ "$goos" = linux ]; then + mkdir -p "ctx/linux/${goarch}" + cp "$bin" "ctx/linux/${goarch}/wavehouse" + fi + done + + # Base names only, two-space separator — the format GoReleaser wrote, + # and what `actions/attest-build-provenance` and `sha256sum -c` both + # expect. + (cd dist && sha256sum wavehouse_*.tar.gz > checksums.txt) + echo "--- dist/" + ls -l dist + echo "--- checksums.txt" + cat dist/checksums.txt + + # The GitHub-hosted runner's default docker driver can't build # multi-platform images, so create a docker-container builder. No QEMU - # needed — Dockerfile.goreleaser stages are $BUILDPLATFORM-pinned / - # COPY-only by design. Mirrors publish-dev.yml. + # needed — Dockerfile.goreleaser's stages are $BUILDPLATFORM-pinned or + # COPY-only by design, so nothing foreign-arch ever executes. - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 @@ -77,36 +233,90 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - # Resolve the moving GHCR tag BEFORE the build, so a malformed tag - # fails here rather than after an 8-target cross-compile and a - # multi-arch image push. The script rejects anything that isn't - # semver-shaped, so a typo can't fall through to `latest`. - - name: Resolve release channel + # Before the GitHub Release, deliberately: a failed image build then + # leaves nothing public to reconcile, and the job is re-runnable. The + # reverse order would publish a release announcing an image that does + # not exist. + # + # Every build gets an immutable reference plus one moving pointer: + # :{tag} plus the channel release-channel.sh derived — :latest for a + # stable tag, :alpha/:beta/:rc/:next for a prerelease. That indirection + # is why `v1.3.0-rc.1` can't take :latest away from a shipped `v1.2.0`. + # + # The labels mirror what GoReleaser's dockers_v2.labels injected; the + # static floor for all of them lives in the Dockerfile itself. + - name: Build and push the multi-arch image shell: bash + env: + IMAGE: ghcr.io/wave-rf/wavehouse run: | set -euo pipefail - channel="$(scripts/ci/release-channel.sh "$GITHUB_REF_NAME")" - echo "WAVEHOUSE_CHANNEL=${channel}" >> "$GITHUB_ENV" - echo "::notice::${GITHUB_REF_NAME} publishes ghcr.io/wave-rf/wavehouse:${GITHUB_REF_NAME} + :${channel}" + version="${GITHUB_REF_NAME#v}" + created="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --file deployments/Dockerfile.goreleaser \ + --tag "${IMAGE}:${GITHUB_REF_NAME}" \ + --tag "${IMAGE}:${WAVEHOUSE_CHANNEL}" \ + --label "org.opencontainers.image.title=WaveHouse" \ + --label "org.opencontainers.image.description=Schema-aware real-time API gateway for ClickHouse" \ + --label "org.opencontainers.image.url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.version=${version}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --label "org.opencontainers.image.created=${created}" \ + --push \ + ctx - - name: Run GoReleaser - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - with: - # Pin to the v2 line so a v3 release breaks loudly instead - # of silently changing release behavior. - version: "~> v2" - args: release --clean + # GoReleaser's `release.mode` defaulted to `keep-existing`, which is what + # made "publish from the Releases UI, let the tag fire this workflow" + # work without clobbering a hand-written body. Reproduce that: if the + # release already exists (UI-created, or this is a re-run), upload the + # assets and leave the notes alone. + # + # `--notes-start-tag` is not optional. GitHub's own generator picks the + # previous release itself, and this repo publishes `clients/ts/v*` + # releases too — so without it a server release's notes can be diffed + # against an SDK release. `git describe --match 'v*'` is the same rule + # .goreleaser.yaml's `git.ignore_tags` applied. + - name: Create or update the GitHub Release + shell: bash env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Second dockers_v2 tag in .goreleaser.yaml. Stable → latest; - # prerelease → alpha/beta/rc/next, so an rc can't displace the - # :latest a shipped stable release owns. - WAVEHOUSE_CHANNEL: ${{ env.WAVEHOUSE_CHANNEL }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + tag="$GITHUB_REF_NAME" + + if gh release view "$tag" >/dev/null 2>&1; then + echo "::notice::release ${tag} already exists — uploading assets, leaving its notes untouched" + gh release upload "$tag" dist/wavehouse_*.tar.gz dist/checksums.txt --clobber + exit 0 + fi + + args=(--verify-tag --title "$tag" --generate-notes) + + # `latest` is release-channel.sh's answer for a stable tag and only + # for a stable tag, so this is the same judgement GoReleaser's + # `prerelease: auto` made — and GitHub's "Latest release" badge keys + # off exactly this flag. + if [ "$WAVEHOUSE_CHANNEL" != latest ]; then + args+=(--prerelease) + fi + + if previous="$(git describe --tags --abbrev=0 --match 'v*' "${tag}^" 2>/dev/null)"; then + echo "::notice::release notes for ${tag} generated since ${previous}" + args+=(--notes-start-tag "$previous") + else + echo "::notice::no earlier v* tag is reachable from ${tag} — notes will cover the full history" + fi + + gh release create "$tag" "${args[@]}" \ + dist/wavehouse_*.tar.gz dist/checksums.txt # Build-provenance attestations — free for public repos via Sigstore's # public-good infra. Binaries: one attestation over every artifact listed - # in goreleaser's checksums.txt. Image: attest the multi-arch - # manifest-list digest and store the attestation alongside it in GHCR. + # in checksums.txt. Image: attest the multi-arch manifest-list digest and + # store the attestation alongside it in GHCR. - name: Attest binary provenance uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: @@ -132,10 +342,10 @@ jobs: # Prove the attestations we just wrote actually verify against the # artifacts people will download — the same `gh attestation verify` - # command, flags included, that the install docs tell users to run. Attesting and verifying - # are different code paths (subject digests, the checksums-file - # expansion, the registry round-trip), so a release that publishes - # unverifiable provenance should go red rather than look green. + # command, flags included, that the install docs tell users to run. + # Attesting and verifying are different code paths (subject digests, the + # checksums-file expansion, the registry round-trip), so a release that + # publishes unverifiable provenance should go red rather than look green. # Runs last: the release is already public by this point, so this is a # loud alarm, not a gate. - name: Verify published provenance @@ -152,7 +362,7 @@ jobs: # attestation. Verify them all rather than a representative one — # a per-GOOS/GOARCH gap is exactly what this would miss. shopt -s nullglob - archives=(dist/*.tar.gz dist/*.zip) + archives=(dist/*.tar.gz) if [ ${#archives[@]} -eq 0 ]; then echo "::error::no release archives in dist/ to verify" exit 1 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d809ce7d..c18851f2 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,21 +1,61 @@ version: 2 +# ───────────────────────────────────────────────────────────────────────────── +# GoReleaser is now ONLY the compiler. It is invoked once per native runner as +# `goreleaser build --single-target`, and everything downstream of the binary — +# archives, checksums, the GitHub Release, the multi-arch GHCR image — is done +# by .github/workflows/release.yml. This file therefore contains `builds:` and +# nothing else. +# +# Why, measured rather than assumed (2026-09-16, this tree at the cgo switch): +# +# cgo means each target needs a C toolchain that can target that OS/ABI, and +# for darwin that means real Apple SDK headers. In golang:1.27-bookworm +# (linux/amd64 — the shape of `ubuntu-latest`): +# +# linux/amd64, ambient gcc 12.2 OK 44,931,216 B max GLIBC_2.34 +# linux/arm64, apt gcc-aarch64-linux-gnu OK 41,400,592 B max GLIBC_2.34 +# linux/arm64, zig cc aarch64-linux-gnu.2.17 OK 51,094,216 B max GLIBC_2.17, +# but NEEDED libresolv/libdl/ +# libpthread and `-s` ignored +# darwin/arm64, zig cc -target aarch64-macos FAILS, with AND without +# `-tags netgo,osusergo`: +# +# # github.com/prometheus/client_golang/prometheus +# process_collector_mem_cgo_darwin.c:18:10: fatal error: +# 'mach/mach_vm.h' file not found +# +# That is a *compile* failure inside a dependency's own C file, not the +# `-lresolv` *link* failure recorded earlier — netgo/osusergo cannot help, +# because the build never reaches the linker. client_golang's darwin process +# collector switches on `cgo` with no opt-out tag, and zig ships no Mach +# headers. Supplying them needs an Apple SDK, which cannot be fetched on a +# GitHub-hosted Linux runner. So: no darwin cross-compile, at all. +# +# The two GoReleaser features that solve this natively — split/merge +# (`goreleaser continue --merge`) and `builder: prebuilt` — are Pro-only, and +# `goreleaser release --skip=` accepts no `build` value in OSS (checked +# against v2.18.1, whose accepted set is announce, archive, aur, aur-source, +# before, chocolatey, docker, flatpak, homebrew, iru, ko, makeself, mcp, +# nfpm, nix, notarize, publish, sbom, scoop, sign, snapcraft, srpm, validate, +# winget). There is therefore no OSS way to have GoReleaser assemble a +# release from binaries built elsewhere — hence the split above. +# ───────────────────────────────────────────────────────────────────────────── + # GoReleaser defaults ProjectName to the repo directory name — "WaveHouse" — # which is the only place the product's display casing leaked into an artifact # identifier: archives built as `WaveHouse_linux_amd64.tar.gz` while the binary # inside, the GHCR image, and the npm package are all lowercase `wavehouse`. # Asset URLs are permanent once a release is published, so this is pinned -# rather than inherited. +# rather than inherited. release.yml names its archives from this same value. project_name: wavehouse -# GoReleaser's tag detection — which tag it is releasing, and which one the -# release notes are diffed from — walks git history without caring which tag -# family it lands on. This repo has several: `v*` for the server and -# `clients//v*` for each client SDK. Without this, cutting `v0.1.0` after -# an SDK release describes the server release as "everything since -# clients/ts/v0.1.0" — verified: `previous=clients/ts/v0.1.0 current=v0.1.0`. -# Ignoring the client families makes the built-in detection correct, which is -# why no workflow needs to pass GORELEASER_PREVIOUS_TAG. +# GoReleaser's tag detection — which tag it is building from — walks git +# history without caring which tag family it lands on. This repo has several: +# `v*` for the server and `clients//v*` for each client SDK. Without +# this, a snapshot build cut after an SDK release stamps itself from +# `clients/ts/v0.1.0`. release.yml applies the same rule to the release notes +# by deriving `--notes-start-tag` from `git describe --match 'v*'`. git: ignore_tags: - "clients/*" @@ -24,114 +64,36 @@ builds: - id: wavehouse main: ./cmd/wavehouse binary: wavehouse + # cgo is unconditional: the chtypes binding dlopens the per-version + # artifact, so CGO_ENABLED=1 on every target (dlfcn only — no C library + # linked, no header). The platform set follows from that artifact, not + # from Go: Windows and FreeBSD are dropped + # (chtypes publishes no artifact for either) along with darwin/amd64 (no + # artifact either — only darwin-arm64, linux-amd64, linux-arm64 exist at + # https://artifacts.wavehouse.dev/artifacts/index.json). goos x goarch is + # a cross product, hence the explicit `ignore` to drop darwin/amd64 + # rather than a matrix that only ever had one cell per OS. env: - - CGO_ENABLED=0 - goos: [linux, darwin, windows, freebsd] + - CGO_ENABLED=1 + # `--single-target` builds the runner's own GOOS/GOARCH *regardless of what + # is set here*, so on the release path these three keys are a declaration + # rather than a driver: they are the supported set, and the `build` + # matrices in release.yml / publish-dev.yml / goreleaser-validate.yml must + # mirror them. Keep them accurate — `goreleaser check` validates this + # file's schema, nothing validates that a workflow matrix agrees with it, + # and a plain `goreleaser build` with no flag would still try (and fail) to + # cross-compile all three. + # + # There are deliberately NO `overrides:` with CC/CXX any more: every target + # is compiled by a native toolchain on its own runner, so the ambient `cc` + # is always the right one. + goos: [linux, darwin] goarch: [amd64, arm64] + ignore: + - goos: darwin + goarch: amd64 ldflags: - -s -w - -X main.Version={{.Version}} - -X main.BuildTime={{.Date}} - -X main.GitCommit={{.ShortCommit}} - -checksum: - name_template: "checksums.txt" - -archives: - - name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" - # Windows gets .zip; everything else keeps GoReleaser's tar.gz default. - # Windows has shipped bsdtar since 10 1803, so a .tar.gz is *openable* — - # but not from Explorer, which still only double-clicks into .zip. The - # convention costs nothing and this is the first release whose asset - # names people will script against. - format_overrides: - - goos: windows - formats: [zip] - # Narrowed from GoReleaser's default, which also globs CHANGELOG* — 324 KB - # of development history in every one of the eight archives, for a file - # that is a click away on GitHub and whose contents are the release notes - # the download page already shows. - # NOTICE is not optional: the repo is Apache-2.0 and §4(d) makes every - # redistributor of these archives inherit an attribution obligation they - # cannot satisfy from a tarball carrying only LICENSE. GoReleaser's default - # glob never included it either, so this hunk is the moment to fix it — - # archive contents are effectively permanent once published. - files: - - LICENSE* - - NOTICE* - - README* - -# Suppress the GitHub Release on dev pushes. publish-dev.yml sets -# WAVEHOUSE_DEV=1 + a synthetic tag so it can reuse this same pipeline; -# we don't want a release entry on the repo for every push to main. -# Real tag releases (release.yml) leave WAVEHOUSE_DEV unset → defaults -# to "0" → the field evaluates to "false" → release runs normally. -# `envOrDefault` is required because goreleaser templates fail strict -# on missing env vars; bare `.Env.WAVEHOUSE_DEV` would error on the -# release path. -release: - disable: '{{ eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}' - # `auto` marks the GitHub Release as a pre-release whenever the tag - # carries a prerelease identifier (v0.1.0-alpha.1 → yes; v0.1.0 → no). - # GoReleaser's default is a flat `false`, which would have published the - # first alpha as a full stable release — and GitHub's "Latest release" - # badge keys off exactly this field. - prerelease: auto - -dockers_v2: - - dockerfile: deployments/Dockerfile.goreleaser - ids: - - wavehouse - images: - - "ghcr.io/wave-rf/wavehouse" - # Every build gets an immutable reference plus one moving pointer. - # - # immutable release: :{{ .Tag }} (e.g. :v1.2.3) - # dev: :dev- (matches cleanup-ghcr.yml's - # `^dev-[0-9a-f]+$` regex) - # moving release: the channel release.yml derived from the tag via - # scripts/ci/release-channel.sh — :latest for a - # stable tag, :alpha/:beta/:rc/:next for a - # prerelease - # dev: :dev (rolling, follows main) - # - # The channel indirection is why `v1.3.0-rc.1` can't take :latest away - # from a shipped `v1.2.0`. The dev branch stays an explicit literal - # rather than leaning on WAVEHOUSE_CHANNEL's default: a push to main that - # forgot to set the env var must never be able to publish :latest. - tags: - - '{{ if eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}dev-{{ .FullCommit }}{{ else }}{{ .Tag }}{{ end }}' - - '{{ if eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}dev{{ else }}{{ envOrDefault "WAVEHOUSE_CHANNEL" "latest" }}{{ end }}' - platforms: - - linux/amd64 - - linux/arm64 - labels: - "org.opencontainers.image.title": '{{ if eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}WaveHouse (dev){{ else }}WaveHouse{{ end }}' - "org.opencontainers.image.description": '{{ if eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}Schema-aware real-time API gateway for ClickHouse — rolling dev build from main{{ else }}Schema-aware real-time API gateway for ClickHouse{{ end }}' - "org.opencontainers.image.url": "https://github.com/Wave-RF/WaveHouse" - "org.opencontainers.image.source": "{{ .GitURL }}" - "org.opencontainers.image.version": "{{ .Version }}" - "org.opencontainers.image.revision": "{{ .FullCommit }}" - "org.opencontainers.image.created": "{{ .Date }}" - -changelog: - # The changelog pipe is NOT skipped by `release.disable` — only by - # `--snapshot` or this key (verified: a run with `release.disable: true` - # still logs "generating changelog"). Left on, `github-native` would make - # publish-dev.yml POST /releases/generate-notes on every push to main — an - # endpoint needing `contents: write`, which that workflow deliberately does - # not grant — to build a body that is then discarded, for a synthetic tag - # that exists only on the runner. goreleaser-validate.yml cannot catch it - # either, since `build --snapshot` skips this pipe entirely. - disable: '{{ eq (envOrDefault "WAVEHOUSE_DEV" "0") "1" }}' - # Delegate the release body to GitHub's own "generate release notes" — the - # grouped, linked, per-PR list you get from the Releases UI button, with - # authors and a New Contributors section. `use: git` built the body from raw - # commit subjects instead: nearly the same content (main is squash-merged, so - # each subject IS a PR title) but no links, no authors, and no grouping. - # - # Categorization and exclusions move to .github/release.yml with this. The - # `sort` and `filters` keys that used to live here are gone rather than left - # in place: github-native renders the body on GitHub's side, so GoReleaser - # never sees the commits and both would be silently dead config. - use: github-native diff --git a/AGENTS.md b/AGENTS.md index 893d3020..8de53f29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,24 +26,25 @@ One binary: - **`cmd/wavehouse/`** — Standalone mode (all-in-one with embedded NATS, optional Pebble dedup) -Sixteen internal packages under `internal/` (plus `internal/testutil/` for shared test helpers): +Seventeen internal packages under `internal/` (plus `internal/testutil/` for shared test helpers): -- **`api/`** — Chi HTTP router, JWT/JWKS middleware (from `auth/`), ingest/query/structured-query/SSE/schema/DLQ/pipes handlers +- **`api/`** — Chi HTTP router, JWT/JWKS middleware (from `auth/`), ingest/query/structured-query/SSE/schema/DLQ/pipes handlers. On the ingest path, `content_type.go` is the Content-Type → format table (and the RFC 9110 resolution rules behind the `415`) and `ingest_framing.go` is the only code that reads body bytes itself — the array re-frame, the record count, and the positional dedupe-id read. `clickhouse_http.go` runs structured queries and pipes over ClickHouse's HTTP interface with `default_format=JSONEachRow` - **`auth/`** — JWT auth middleware: HMAC **or** JWKS verification with `alg` pinned to the active verifier, role extraction from a configurable claim path; always runs, never rejects (bad token → empty role + stashed reason) - **`cache/`** — `Cache` interface → `LocalCache` (Ristretto) + `SharedCache` (TBD) + `TieredCache` (singleflight) - **`chconn/`** — `Manager`, the one ClickHouse `driver.Conn` every consumer holds; `Reconfigure` swaps the connection behind it after a settings reload changes the wiring (never dials; the old connection closes after a `query_timeout` grace) - **`chsql/`** — dependency-free ClickHouse SQL helpers shared by `query`/`policy` (avoids an import cycle): `QuoteIdent` (backtick-quote every identifier) + `BindUnsafe` (reject names with a literal `?`) - **`config/`** — YAML + env var config loading (cleanenv); strict on both sides (undeclared YAML key, unbound `WH_*` variable) and probes `data_dir` writability — boot is the validator, there is no dry run - **`dedupe/`** — `Deduplicator` interface → `Embedded` (Pebble), wrapped by `Managed` whose open/closed state follows the hot-reloadable `dedupe.enabled` in the settings directory's `config.json` -- **`discovery/`** — `SchemaRegistry` that introspects ClickHouse `system.columns` (name/type/nullability plus `default_expression` and 1-based `position`) and `system.tables` (each table's `create_table_query`, kept in-process and never serialized — an external-engine table renders its wiring there unconditionally — endpoint, bucket/host, database, username, S3 access key id; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so the exposure is the topology, not the secret), records the server version, + `Validate()` for ingest payloads + `CanonicalizeTimestamps()` rewriting top-level `DateTime`/`DateTime64` column values to the canonical RFC 3339 UTC wire form pre-publish (Key Design Decision #19) -- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else — `row` is one positional `JSONCompactEachRow` line and `columns` names its slots, the table's **insertable** columns (a `MATERIALIZED`/`ALIAS` column cannot be named in an `INSERT`); the worker batches per (table, column list); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`), so non-admin callers never reach the proxy. A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) +- **`discovery/`** — `SchemaRegistry` that introspects ClickHouse `system.columns` (name/type/nullability plus `default_expression` and 1-based `position`) and `system.tables` (each table's `create_table_query`, kept in-process and never serialized — an external-engine table renders its wiring there unconditionally — endpoint, bucket/host, database, username, S3 access key id; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so the exposure is the topology, not the secret), records the server version and default timezone (`ServerTimezone()`), and fires an `OnRefresh(serverVersion, serverTZ, tables)` hook after every successful refresh — `typelayer.Engine.Bind` is its only consumer (Key Design Decision #20) +- **`ingest/`** — Ingest worker pipeline (`worker.go`: JetStream input → per-table batch INSERT with DLQ output). The pipeline is **insert-only**. The wire format `EventMessage` (`types.go`) carries `{table_name, scope, received_timestamp, format, columns, row}` and nothing else; `row` is the exact `JSONCompactEachRow` bytes ClickHouse's own writer produced for that stored record (via `typelayer.Table.Ingest`), `columns` names its positions (`typelayer.Table.WireColumns`, or the narrower list a column-restricted role produces), and the worker batches per (table, column list); the worker accepts whatever table name the envelope carries (table existence was already checked by the HTTP ingest handler, which `404`s an unknown table before publish; the worker doesn't re-validate), then bulk-INSERTs with `typelayer.InsertSettings()` plus `async_insert=0`. In the embedded-NATS deployment (the default), the server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (the same `RequireAdmin` gate as the rest of `/v1/ops/*`), so non-admin callers never reach the proxy. A request with no token (or an invalid one) resolves to the `default_role`, which in a production config is not the admin role (setting them equal is a loudly-warned dev-only setting), so it can't reach this endpoint. Plus `Sweeper` (Active Sweeper for NATS message lifecycle) + `EventMessage`/`BufferConsumerName` types (`types.go`) - **`mq/`** — `Publisher`/`Subscriber` interfaces → `EmbeddedNATS` + `RemoteNATS` - **`observability/`** — OpenTelemetry pipeline: `InitProvider` wires trace/metric/log providers via OTLP gRPC (each signal independently gated). A top-level `Prometheus` config block drives an optional `/metrics` scrape endpoint that runs independently of OTLP push — standalone (Alloy/Mimir scrape, no collector), alongside OTLP, or off. `NewLogger` produces a slog handler that fans out to stdout AND OTLP (stdout always 100%, OTLP sample-rate-aware). `TraceHandler` injects trace_id/span_id from active spans. `tracer.go` provides W3C trace context propagation over NATS headers. - **`pipes/`** — Named query pipes: `NamedQuery` type + `BindParams` + `Source` (read per request; `settings.Store` in production, `Static(q...)` in tests) - **`policy/`** — Hasura-style access control, **role-first**: `TablePolicy` is `map[string]RolePermissions`, and a role's grant splits by operation into `SelectPermissions` (columns, row `filter`, aggregations, the `max_*` limits) and `InsertPermissions` (columns, `check`) — so a field only one side honors does not exist on the other. `Evaluate()` resolves ONE operation and leaves the other side **nil** (`Select *ResolvedSelect` / `Insert *ResolvedInsert`), which every accessor fails closed on — nil is "not resolved", distinct from an empty side, which is "unrestricted" (what the admin return builds). Claim templating (`{{ jwt.claim.path }}`) resolves during that call. Policies come from `Source`, a `func() *Policy` read per call (`settings.Store.Policy` in production, `Static(p)` in tests) - **`query/`** — Structured query AST types + SQL builder with schema validation, structural policy predicate/limit emission, timestamp bucketing - **`settings/`** — the settings directory: `Validate` (strict JSON, per-file rules, cross-file role references), `Store` (the adopted snapshot + serialized `Reload`, typed accessors read per call, `AfterAdopt` hooks), the fsnotify `Watch`, and the `go:embed`ded seed `wavehouse bootstrap` writes -- **`stream/`** — SSE fan-out: rows travel POSITIONALLY, so each connection is told its projected column list in an `event: schema` frame before its first row and again on drift — **not** guaranteed after a gap-fill across a column change, which can leave a connection reading live rows against a stale list until it reconnects ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)) — (tracked per connection; replay tracks its own). The event `Hub` (registers subscribers by `(topic, role)`; `Broadcast` projects + serializes each event once per role, the #294 delivery hot path — a role carrying a row-level `filter` keeps the shared projection but delivers per subscriber, each subscriber's claims evaluated against the row, #319), `Subscriber` (per-connection outbound `Frame` queue, `Send`/`Frames`; claims fixed at construction, immutable), the `Bucket` fan-out set (`subscriberSet`, one per `(topic, role)`), the `Heartbeater` keepalive wheel, and `Metrics` (the `wavehouse_sse_*` stream instruments) +- **`stream/`** — SSE fan-out: rows travel POSITIONALLY, so each connection is told its projected column list in an `event: schema` frame before its first row and again on drift — **not** guaranteed after a gap-fill across a column change, which can leave a connection reading live rows against a stale list until it reconnects ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)) — (tracked per connection; replay tracks its own). The event `Hub` (registers subscribers by `(topic, role)`; `Broadcast` projects + serializes each event once per role, the #294 delivery hot path — a role carrying a row-level `filter` keeps the shared projection but delivers per subscriber, each subscriber's claims evaluated against the row via `typelayer`, #319), `Subscriber` (per-connection outbound `Frame` queue, `Send`/`Frames`; claims fixed at construction, immutable), the `Bucket` fan-out set (`subscriberSet`, one per `(topic, role)`), the `Heartbeater` keepalive wheel, and `Metrics` (the `wavehouse_sse_*` stream instruments) +- **`typelayer/`** — the only package that imports `github.com/wave-rf/chtypes/go/chtypes` (the sole exception: `cmd/wavehouse/main.go` references `typelayer` itself). Wraps one `chtypes.Registry`, opened once at boot from a registry directory (`clickhouse.chtypes_registry` / `WH_CHTYPES_REGISTRY`); `Engine.Bind` (fired from `discovery`'s `OnRefresh`) resolves the artifact matching the server's minor line — no nearest-version fallback — and recompiles a `Table` handle per changed schema. `Engine.RoleTable(table, RoleShape)` compiles and caches the role's own schema — its insertable columns, plus a `DEFAULT ''` per `_eq` check column — which is how column policy and auto-inject are answered with no Go-side record inspection. `Table.Ingest(format, body)` runs one request body through ClickHouse's own reader (`JSONEachRow`/`CSV`/`TSV`/`CSVWithNames`/`TSVWithNames`), returning a verdict per input record (accepted / rejected with ClickHouse's code and message / declined) plus the accepted rows as `JSONCompactEachRow` bytes; the role's insert checks run in that same parse as a compiled row filter (parse outcome first, then the check verdict), and `Table.ParseRow` / `Row.Visible` judge a subscriber's row filter over one parsed event — one compiled-filter mechanism, values bound as `{p:String}` (Key Design Decision #20) ## Key Design Decisions @@ -51,7 +52,7 @@ The invariant index — what must stay true. Full narrative and rationale live i 1. **Interface-first** — core behaviors are Go interfaces (`Cache`, `Deduplicator`, `Publisher`, `Subscriber`); standalone vs. future-clustered swap implementations. 2. **Bring Your Own Schema** — users create ClickHouse tables; WaveHouse discovers them via `system.columns` and never auto-migrates. -3. **Schema-driven ingest** — `POST /v1/ingest?table={table}` takes flat JSON, validated against the discovered schema (unknown fields rejected, types/nullability enforced). No envelope. The **declared `Content-Type` chooses the format and the bytes never do** (arity within the JSON family is still the body's): no declaration, one whose **media type** is unsupported or unparseable, a comma-bearing value that, as a whole, does not parse as one media type, or repeated lines that **disagree**, is a `415` decided *before* the body is read. A malformed *parameter* on a comma-free line never costs the request (`; charset=a; charset=b` still reads as its media type), and repeated lines are accepted only when they all resolve to the same **supported** format — two agreeing `text/csv` lines are still a `415`. A body declared NDJSON stays NDJSON whatever its bytes, so a bad line is a per-record error rather than a silent re-framing; the reverse (NDJSON sent as `application/json`) is deliberately **not** caught — record one, `200`, the rest ignored ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)). Fail-closed — preserve it when touching `internal/api`. +3. **Schema-driven ingest: the body goes to ClickHouse's parser as-is** — `POST /v1/ingest?table={table}` never decodes a record. The **declared `Content-Type` chooses the format and the bytes never do** (`internal/api/content_type.go` is the whole table: the `application/json` and four NDJSON spellings → `JSONEachRow`, `text/csv` → `CSV`, `text/tab-separated-values` → `TSV`, `; header=present` → `CSVWithNames` / `TSVWithNames`, `; header=absent` → the same with header detection off, a bare type → ClickHouse's default auto-detection); anything else — no declaration, an unsupported or unparseable media type, a comma-bearing value that does not parse as one media type, or repeated lines that **disagree** — is a `415` decided *before* the body is read, while a malformed *parameter* on a comma-free line never costs the request. The body decides exactly one thing, in `internal/api/ingest_framing.go`: the first non-whitespace byte picks array-vs-single inside `application/json`, which sets the response shape. A top-level array is re-framed in place (outer brackets and depth-1 commas blanked) so a compact array keeps per-record salvage; that rewrite must never run on a bare object or NDJSON, which it destroys. NDJSON sent as `application/json` is still not caught — record one, `200`, the rest ignored ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)). CSV/TSV are positional over the table's wire columns. Fail-closed — preserve it when touching `internal/api`. 4. **Async ingestion** — ingest returns 200 after optional dedup + MQ publish; ClickHouse writes happen later via `StartIngestWorker`. NATS full → 503 + Retry-After. 5. **Per-table batching** — the worker groups events by table, then splits each batch by column list (`groupByColumns`), emitting one `INSERT INTO … (cols) FORMAT JSONCompactEachRow` per distinct list so a schema change mid-stream can't corrupt a statement. Each table's batch is independent. 6. **Dead Letter Queue** — failed batch inserts publish to `WAVEHOUSE_DLQ` (`dlq.`), gated per table by `dlq.enabled` in the settings directory's `config.json` (hot-reloadable; off = leave the row unacked for redelivery). No silent data loss on the insert path. The one drop is an envelope the worker cannot READ (malformed JSON, an unknown **or absent** `format` — a pre-v2 envelope carries none — or columns and row that don't pair): it is poison by construction, so with the DLQ off it is acked-and-dropped rather than redelivered forever — logged at `ERROR` and counted by `wavehouse_ingest_poison_total` under `disposition="dropped"`. With the DLQ on it is parked like any other failure, and counted under `disposition="parked"`. @@ -60,18 +61,19 @@ The invariant index — what must stay true. Full narrative and rationale live i 9. **Singleflight** — `TieredCache` coalesces concurrent misses (`x/sync/singleflight`) to prevent cache stampede. 10. **Active Sweeper** — purges NATS messages that are both ACKed (written to CH) and older than the gap window; SSE gap-fill uses `DeliverByStartTime`, no in-process ring buffer. 11. **Hasura-style access control: fail-closed (security)** — `policy.IsAdmin` (role == `admin_role`, **exact case-sensitive**, default `"admin"`) is the single admin check, shared by `Evaluate`/`ResolveRole`/`Validate`/the `/v1/ops` gate/`RoleAllowed`. Empty/absent role matches nothing (no `"*"` wildcard); `Validate` rejects empty role keys; a `nil` policy (deleted) denies **everyone incl. admin** via a role — a total lockout for token-based callers, so recovery is writing `policies.json` and reloading, never an implicit admin grant (**exception:** the operator key's `auth.IsOperator` bit passes the `/v1/ops` gate even under a `nil` policy — a deliberate break-glass that can `POST /v1/ops/settings/reload` over HTTP, see #7). `default_role` is the one sanctioned roleless exception (`ResolveRole` maps empty → it pre-eval); `default_role == admin_role` is permitted but dev-only and loudly warned (`policy.DefaultRoleGrantsAdmin`). Preserve when touching `internal/policy` (policy twin of #13; see #159). Detail: architecture.md § `policy/`. -12. **Structured queries: column authz fail-closed (security)** — `POST /v1/query?table={table}`: typed AST validated against schema, permission-enforced, timestamp-bucketed for cache, `DefaultMaxRows` (10,000) cap. Every column reference — projection, aggregation args, `filters`, `group_by`, `order_by`, `time_range` — is authorized inside `query.Build` (the single chokepoint that enumerates them all), so no clause can skip the role's `allow_columns`/`deny_columns` check (#223). A `select_all` read by a *column-restricted* role expands to its allowed columns via `policy.AllowedProjection`, never a bare `SELECT *`; *unrestricted*/admin roles keep `SELECT *` (`policy.RestrictsColumns` decides). Omitting `columns` selects nothing (`ErrEmptyProjection` → `200 []`); `["*"]` is the literal column `*` (schema-gated, not a wildcard); a table-granted role with no readable columns fails closed (`ErrNoReadableColumns` → `403`). Structured and live-stream (`stream.projectIndices`) reads share the one per-column decision `policy.IsColumnAllowed`, so column visibility can't drift. Row visibility has the same one-source guarantee (#319): `Evaluate` resolves a role's row-`filter` once (`resolvePredicates`), and both surfaces consume that single resolution — the query path renders it to SQL (`predicatesToSQL`), the stream evaluates it in memory per subscriber (`ResolvedPermissions.RowVisible`, whose type-aware comparison fails closed on anything it can't prove about the ingested payload — `policy.ColumnSpec`, with `DateTime`/`DateTime64` operands compared as instants through the ingest grammar (`discovery.Column.TimeParser`) and claim constants rendered canonically and digit-exact by the one shared rule `policy.CanonicalScalar` (#457 — which also refuses a float64 at/past 2^53 rather than match a neighboring ID, and whose ok=false — an absent claim, a structured value, no canonical form — makes the predicate match no rows on BOTH surfaces: `1 = 0` in SQL, every row withheld in memory); numeric comparison runs in the column's STORAGE domain (`policy.NumericSpec`, classified by `discovery.NumericStorageOf` — Float width rounding, Decimal scale truncation, integer exactness, both operands narrowed as ClickHouse narrows stored value and bound constant, out-of-range operands refused rather than modeled; the `tests/integration` differential oracle holds in-range verdicts equal to a live ClickHouse's and the never-admit-where-SQL-hides direction for the refused out-of-range ones); an event whose insert later fails into the DLQ is the one residual payload-vs-stored asymmetry, documented in the access-control enforcement caution) — so row visibility can't drift either. Preserve when touching `internal/query` or the structured-query handler. Detail: architecture.md § `query/`. +12. **Structured queries: column authz fail-closed (security)** — `POST /v1/query?table={table}`: typed AST validated against schema, permission-enforced, timestamp-bucketed for cache, `DefaultMaxRows` (10,000) cap. Every column reference — projection, aggregation args, `filters`, `group_by`, `order_by`, `time_range` — is authorized inside `query.Build` (the single chokepoint that enumerates them all), so no clause can skip the role's `allow_columns`/`deny_columns` check (#223). A `select_all` read by a *column-restricted* role expands to its allowed columns via `policy.AllowedProjection`, never a bare `SELECT *`; *unrestricted*/admin roles keep `SELECT *` (`policy.RestrictsColumns` decides). Omitting `columns` selects nothing (`ErrEmptyProjection` → `200 []`); `["*"]` is the literal column `*` (schema-gated, not a wildcard); a table-granted role with no readable columns fails closed (`ErrNoReadableColumns` → `403`). Structured and live-stream (`stream.projectIndices`) reads share the one per-column decision `policy.IsColumnAllowed`, so column visibility can't drift. Row visibility has the same one-source guarantee (#319): `Evaluate` resolves a role's row-`filter` once (`resolvePredicates`, exposed via `Predicates()`), and both surfaces consume that resolution — the query path renders it to SQL, the stream compiles it through `internal/typelayer` and evaluates it per subscriber. Only a definite true admits; error, decline, drift and an unavailable engine all withhold, counted in `wavehouse_sse_rows_withheld_total{table,role,reason}`. The one residual payload-vs-stored asymmetry is an event whose insert later fails into the DLQ, documented in the access-control enforcement caution. Preserve when touching `internal/query` or the structured-query handler. Detail: architecture.md § `query/`. 13. **Named query pipes: fail-closed (security)** — pre-defined SQL templates (Tinybird-style) with param binding + caching; `GET/POST /v1/pipes/{name}` sit outside `RequireAdmin`, so per-pipe `allowed_roles` is the *only* execute-path gate, via `policy.RoleAllowed`: exact allowlist membership (no `"*"`), admin always passes, empty/absent role and empty-string entries authorize nobody, and no `allowed_roles` → admin-only. Preserve and exercise via `testutil.RunRoleMatrix` / `StandardRoleMatrix` (see #159). Detail: architecture.md § `pipes/`. 14. **TypeScript SDK** — `@wavehouse/sdk`: typed query builder, real-time SSE over `fetch`, live queries (incrementable/decomposable/poll aggregation), codegen CLI. Exactly one runtime dependency — `eventsource-parser` (SSE framing, itself dependency-free); adding a second needs the same scrutiny the first got. The canonical client (see §SDK Sync). 15. **Observability invariants** — stdout always 100% (sampling is OTLP-push-only); WARN+ERROR always export at 100% (a non-configurable floor — don't expose it); gRPC OTel exporters dial lazily so an unreachable collector never blocks startup; the OTel Prometheus exporter uses a **private** `prometheus.Registry`. The OTLP endpoint/TLS/custom-CA/mTLS/headers are delegated to the OpenTelemetry SDK's standard `OTEL_EXPORTER_OTLP_*` env vars — `InitProvider` passes **no** endpoint/header options. Known gap, intentionally not patched in WaveHouse app code: the pinned gRPC logs exporter (`otlploggrpc` v0.19/v0.20) ignores the env TLS-cert vars, so a custom/private CA and mutual TLS apply to traces/metrics but **not** the logs signal (public-CA/system-roots TLS and plaintext still work for logs) — upstream bug open-telemetry/opentelemetry-go#6661. A malformed `OTEL_EXPORTER_OTLP_HEADERS` is logged and skipped by the SDK (fail-soft), not fatal. Preserve when touching the logger/sampler/provider. Detail: architecture.md § `observability/`. 16. **Bearer-token-only CORS posture (security)** — Bearer JWT on every request, no cookies/sessions; `corsMiddleware` deliberately **never** emits `Access-Control-Allow-Credentials` (not needed, and `*` + credentials is a spec violation browsers reject). `cors.allowed_origins` (settings directory) controls who can *read* responses, not cookie scope; CSRF protection is structural. Don't reintroduce cookie auth or `Allow-Credentials` without a design discussion — answers GitHub #29/#30. Code: `internal/api/router.go`. 17. **Non-fatal boot** — schema-discovery failure on boot is non-fatal: `cmd/wavehouse` records an `api.BootState`, binds `:8080`, serves 503 on `/livez`/`/readyz` with the diagnostic, and retries via `SchemaRegistry.RetryRefresh` (backoff 2s → 60s). Bounds supervisor restart loops. 18. **Health endpoints** — liveness `/livez`, readiness `/readyz` (k8s convention); `/healthz` is a permanent alias of `/livez`; `/health` + `/ready` are deprecated (removal v0.2.0, CHANGELOG #144). `/v1/health` is the SDK's content-free public ping (no ClickHouse check), a `/v1` route so it survives reverse-proxy probe-path filtering. Point k8s at `/livez`/`/readyz`, SDK/online-checks at `/v1/health`, never the deprecated aliases. -19. **Canonical timestamp wire form (fail-open at ingest)** — the HTTP ingest handler rewrites every top-level `DateTime`/`DateTime64` column value it can parse to RFC 3339 UTC (`discovery.CanonicalizeTimestamps`; per-column precision + zone precomputed at schema refresh) after validation + policy checks and **before** the NATS publish, so the one payload every consumer shares — SSE subscribers, the ClickHouse insert, the DLQ — carries the same spelling `/v1/query` renders: live and query reads can't drift on the instant (#372). Zone-less inputs are read in the column's declared zone, else the discovered server default — ClickHouse's own rule, so the spelling changes but never the instant. Deliberately **fail-open**: an unparseable value or unresolvable zone (no tzdata embedded — never a failed refresh, never a silent UTC reinterpretation, which would move instants) publishes verbatim; ingest must not reject a record over its timestamp spelling — fail-closed enforcement belongs to the stream row-filter (#381). Don't re-spell timestamps downstream. Preserve when touching `internal/discovery`, the ingest handler, or the SSE fan-out. Detail: architecture.md § `discovery/` + §Ingest Path; the exact spelling spec (truncation, zero-trimming, `Z`-only) lives in api.md §Timestamp canonicalization — keep it in sync with `canonicalTimestamp`. +19. **Timestamps agree on the wire by construction, not by rewriting** — the NATS/SSE `row` for `DateTime`/`DateTime64` columns is the exact bytes ClickHouse's own writer produced for the stored record (`typelayer.Table.Ingest`, via chtypes), in the column's declared zone else the server's default (`"2026-06-21 04:00:00.123"`, never RFC 3339's `Z` suffix) — there is no separate WaveHouse rewrite step to keep in sync with what `/v1/query` renders, so live and query reads can't drift on spelling *or* instant (#372). Preserve when touching `internal/typelayer`, the ingest handler, or the SSE fan-out. Detail: architecture.md § `typelayer/` + §Ingest Path; the wire shape lives in api.md §Timestamp rendering. +20. **ClickHouse's own parser validates ingest and evaluates row-level security, in-process (security)** — `internal/typelayer` is the only importer of `github.com/wave-rf/chtypes/go/chtypes`, a per-ClickHouse-minor-version shared library loaded via `dlopen` and matched to the connected server's line with **no nearest-version fallback**; a table with no matching artifact, or one caught mid a server-timezone change, is `Unavailable` (ingest → `503`, stream → every row withheld) rather than silently approximated. Ingest validation, type coercion, and `DEFAULT` substitution run ClickHouse's real parser over the whole request body in one call, so a rejection carries ClickHouse's own error code and message instead of a WaveHouse-authored sentence — an unknown column, a computed-only column and **a column the role may not write** are all **117**, because column policy is answered by compiling the role its own schema (`Engine.RoleTable`) rather than by walking a decoded record; a record the engine cannot answer for is **declined** (`422`), distinct from and never conflated with a data rejection (`400`). Predicates — a role's row `filter` and its insert `check` alike — compile through chtypes with every bound value a `{p:String}` parameter, never interpolated, and are evaluated the way the server's `WHERE` clause would evaluate them, for every column type. Only a definite true admits; error, decline, schema drift, or an unavailable engine withhold (fail closed), each counted separately in `wavehouse_sse_rows_withheld_total{table,role,reason}`. Consequence: the binary requires cgo (dlopen only, no static link to the artifact) and glibc, so supported platforms are Linux amd64/arm64 and macOS arm64 — see [Deployment → chtypes artifacts](docs/src/content/docs/deployment.md#chtypes-artifacts). Preserve when touching `internal/typelayer`, ingest, or the stream row-filter; change the artifact-matching or fail-closed behavior only with a security review. Detail: architecture.md § `typelayer/`. ## Code Conventions -- **Go 1.26**, strict formatting (`gofumpt`, enforced by CI) +- **Go 1.27**, strict formatting (`gofumpt`, enforced by CI); cgo enabled (`internal/typelayer`'s chtypes dlopen shim needs a C toolchain + glibc) - **Structured logging** with `log/slog` (JSON handler) - **Chi v5** for HTTP routing - **Error handling**: Return errors, don't panic. Wrap with `fmt.Errorf("context: %w", err)`. @@ -428,7 +430,7 @@ internal/chconn/ → ClickHouse connection manager (driver.Conn swapped o internal/chsql/ → Shared ClickHouse SQL helpers (identifier quoting + bind-safety) internal/config/ → Configuration structs + loader internal/dedupe/ → Optional deduplication (interface + embedded/distributed) -internal/discovery/ → ClickHouse schema introspection + ingest validation +internal/discovery/ → ClickHouse schema introspection (system.columns/system.tables, server version + timezone) internal/ingest/ → Batch buffer with DLQ + Active Sweeper (NATS message lifecycle) internal/mq/ → MQ abstraction (interface + embedded/remote NATS) internal/observability/ → OpenTelemetry pipeline (traces/metrics/logs providers, Prometheus exporter, slog fan-out, NATS trace propagation) @@ -437,6 +439,7 @@ internal/policy/ → Access control policies (types, evaluation, Source) internal/query/ → Structured query AST + SQL builder internal/settings/ → Settings directory (validate, adopted snapshot + reload, watcher, embedded seed) internal/stream/ → SSE fan-out (event Hub: project once per role, Subscriber outbound queue, Bucket fan-out, keepalive Heartbeater wheel) +internal/typelayer/ → In-process ClickHouse parser (chtypes): ingest validation/coercion + row-level-security compilation internal/testutil/ → Shared test helpers (NopLogger, etc.) tests/ → Integration & E2E tests tests/integration/ → Go integration tests (//go:build integration; ClickHouse testcontainer) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b53b190..a644e74e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Added +- **Ingest accepts CSV and TSV bodies** (`internal/api/content_type.go`, `internal/api/ingest.go`, `internal/typelayer/typelayer.go`, `docs/src/content/docs/api.md`): `Content-Type: text/csv` and `text/tab-separated-values` are read by ClickHouse's own `CSV`/`TSV` readers. `text/csv; header=present` and `text/tab-separated-values; header=present` read `CSVWithNames` / `TSVWithNames` (the first line names the columns, in any order; an omitted column takes its `DEFAULT`; an unknown or duplicate name is code 117 for the whole request). `header` is RFC 4180 §3's optional parameter, mapped three ways onto ClickHouse: `header=present` is `CSVWithNames`, `header=absent` is strictly positional (`input_format_csv_detect_header=0` / `input_format_tsv_detect_header=0`), and a bare type is ClickHouse's default reading with header auto-detection on, so a first line that spells the column names is consumed as a header. Positionally, the fields are the table's wire columns — declaration order minus every `MATERIALIZED`, `ALIAS` and `EPHEMERAL` column — and every one of them must be present, in that order. An empty CSV field (and `\N` in TSV) takes the column's `DEFAULT`; too few fields is code 27, too many is 117, and under `header=absent` a header line is one record that fails to parse with code 27. Always the batch response shape. + - **Schema discovery captures each table's DDL, its columns' ordinals and default expressions, and the server version** (`internal/discovery/discovery.go`, `internal/testutil/testutil.go`): `Column` gains `DefaultExpression` and `Position` (both from a widened `system.columns` select), `TableSchema` gains `DDL` from `system.tables.create_table_query`, and `SchemaRegistry` gains `ServerVersion()` from a `SELECT version()` probe next to the existing `SELECT timezone()`. Groundwork for the native type layer, captured on the same refresh as the columns so a stale version cannot outlive the schemas it describes. That is a publication guarantee, not a same-server one: `chconn.Manager` resolves the connection per call, so a reload changing `clickhouse.addr` mid-refresh can still pair a version from one server with schemas from another — narrow, and self-correcting on the next refresh. `DDL` is `json:"-"` and does **not** appear in `/v1/ops/schema`: that endpoint marshals `TableSchema` straight to the client, and an external-engine table (S3, MySQL, PostgreSQL, Kafka) renders its wiring there unconditionally — endpoint, bucket or host, database, username, S3 access key id. ClickHouse masks the password itself as `[HIDDEN]` from ~23.9 (verified on 26.7.3), so the exposure is the topology rather than the secret — except on an older server, or one with `display_secrets_in_show_and_select` enabled. `position` and `default_expression` are additive fields in the response. A table listed in `system.tables` with no `system.columns` rows is skipped rather than published column-less, and both new queries fail the refresh on error exactly as `timezone()` and `system.columns` do — callers keep the prior cache and retry. - **Settings-directory hot reload — boot loading, three reload triggers, and the config-key migration** (`internal/settings/` (new: `store.go`, `watch.go`, + tests), `internal/api/settings.go` (new, + tests), `internal/api/{router,ingest,structured_query}.go`, `internal/discovery/discovery.go`, `internal/config/config.go`, `cmd/wavehouse/main.go`, `config.yaml`, `deployments/compose/standalone.yaml`, `docs/src/content/docs/settings-directory.mdx` (new — the hot-reloadable half of configuration gets its own page; `configuration.mdx` is boot config only); closes the loop [#500](https://github.com/Wave-RF/WaveHouse/pull/500) opened, tracked by [#48](https://github.com/Wave-RF/WaveHouse/issues/48)): the server now *consumes* the settings directory instead of only validating it. `settings.Store` owns the adopted snapshot: `settings.dir` / `WH_SETTINGS_DIR` is now **required**, boot validates and adopts the directory (missing or invalid refuses to start); a running instance then re-validates and re-adopts on any of three triggers — a **directory watch** (fsnotify on the directory, not the files, so atomic-writer replaces and Kubernetes ConfigMap symlink swaps aren't lost; bursts debounce into one reload), **`SIGHUP`**, and **`POST /v1/ops/settings/reload`** (admin-gated; returns `{"adopted", "findings"}`, `200` adopted / `422` rejected) — all funneling through one serialized reload path. A reload that fails validation keeps the previous good snapshot (an operator mid-edit degrades to a log line, never a broken server); warnings don't block adoption, matching `wavehouse validate`. The tenant tunables **migrate out of boot config** into the directory's `config.json`: `dedupe.id_field` / `dedupe.require_id` (now with the per-table overrides under `dedupe.tables` that [#222](https://github.com/Wave-RF/WaveHouse/issues/222) asked for, resolved per record through the table → global cascade in one atomic snapshot read, so a reload lands at a record boundary and never mixes documents within one record), `query.default_max_rows` and `query.timestamp_bucket_seconds` (read per query), `schema.refresh_interval` (re-read after each tick, so a change applies from the next cycle), `stream.keepalive_interval` / `stream.keepalive_buckets` (a reload calls the new `Heartbeater.Reconfigure`, which rebuilds the keepalive wheel in place with every live subscriber carried over and re-times the running ticker) and `stream.gap_window_minutes` (the sweeper re-reads it every sweep), `mq.max_bytes_gb` (an after-adopt hook updates the `WAVEHOUSE` and `WAVEHOUSE_DLQ` stream limits in place via `EmbeddedNATS.Resize` — shrinking below the buffered size backpressures until the worker drains, nothing is dropped), `dlq.enabled` with per-table overrides under `dlq.tables` (resolved by the ingest worker at the moment a poison row is isolated: on → park it on `WAVEHOUSE_DLQ` and ack; off → leave it unacked for redelivery, never dropped; the DLQ stream and `GET /v1/ops/dlq/stats` now always exist, so the switch is purely behavioral), the **ClickHouse wiring** (`clickhouse.addr` / `http_port` / `http_scheme` / `database` / `username` / `query_timeout`: the new `chconn.Manager` is the one `driver.Conn` every consumer holds and swaps the connection behind it on reload — unconditionally, since the adopted settings are the authority and reachability already surfaces through schema discovery and `/readyz`; the replaced one closes after a `query_timeout` grace; the ingest worker, raw-SQL proxy, and schema registry read the HTTP target, timeout, and database per call), the **auth verifier wiring** (`auth.jwks_url` / `auth.role_claim`: the new `auth.Authenticator` swaps a whole verifier — key source plus its pinned algorithm allowlist — atomically per reload, unconditionally, so an unreachable JWKS fails closed until it can be fetched; `auth.Middleware` is gone — `Authenticator` is the one constructor), and the CORS allowlist (`cors.allowed_origins`, resolved per request). The corresponding YAML/env keys are **removed**: `server.cors_allowed_origins`, `query.default_max_rows`, `schema.refresh_interval`, `dedupe.enabled`, `dedupe.id_field`, `dedupe.require_id`, `stream.keepalive_interval`, `stream.keepalive_buckets`, `mq.gap_window_minutes`, `cache.timestamp_bucket_seconds`, `mq.max_bytes_gb`, `dlq.enabled`, `clickhouse.addr`, `clickhouse.http_port`, `clickhouse.http_scheme`, `clickhouse.database`, `clickhouse.username`, `clickhouse.query_timeout`, `auth.jwks_url`, `auth.role_claim` (and `WH_SERVER_CORS_ALLOWED_ORIGINS`, `WH_QUERY_DEFAULT_MAX_ROWS`, `WH_SCHEMA_REFRESH_INTERVAL`, `WH_DEDUPE_ENABLED`, `WH_DEDUPE_ID_FIELD`, `WH_DEDUPE_REQUIRE_ID`, `WH_STREAM_KEEPALIVE_INTERVAL`, `WH_STREAM_KEEPALIVE_BUCKETS`, `WH_MQ_GAP_WINDOW_MINUTES`, `WH_CACHE_TIMESTAMP_BUCKET_SECONDS`, `WH_MQ_MAX_BYTES_GB`, `WH_DLQ_ENABLED`, `WH_CH_ADDR`, `WH_CH_HTTP_PORT`, `WH_CH_HTTP_SCHEME`, `WH_CH_DATABASE`, `WH_CH_USERNAME`, `WH_CH_QUERY_TIMEOUT`, `WH_AUTH_JWKS_URL`, `WH_AUTH_ROLE_CLAIM`); the secrets — `clickhouse.password`, `auth.jwt_secret`, `auth.operator_key` — stay boot config on purpose (never in a tracked JSON file; combined with the adopted wiring on every reconnect, rotating one is a restart), and boot config is now **strict**: `config.Load` re-reads the YAML against the struct's tags and refuses to start naming every undeclared key, so a `dlq:` or `clickhouse: addr:` left behind can't be read, ignored, and believed; the binary carries **no compiled defaults** — every `config.json` key is required (validation names each missing one), so the adopted snapshot is what the files say, and once adopted it outlives its files (a deleted file or vanished directory is just a rejected reload). Defaults live in one checked-in seed directory (`internal/settings/seed/`, `go:embed`ded): the new **`wavehouse bootstrap [dir]`** writes it (refusing a non-empty directory, the `initdb` contract; the directory resolves exactly as it does for `validate` — the argument, else `WH_SETTINGS_DIR`, usage error with neither — so the two commands are interchangeable on one path and a bare `bootstrap` inside the container images seeds `/app/settings`), the dev `config.yaml` points at a gitignored `./settings` that `make dev` seeds from it, and the e2e fixture ships a copy. The container images ship **no** settings directory: `WH_SETTINGS_DIR` is preset to `/app/settings`, the operator mounts a directory there (`standalone.yaml` bind-mounts the checked-in `deployments/compose/settings/`), and a missing mount refuses to boot rather than running on defaults nobody chose. `dedupe.enabled` moves too: the new `dedupe.Managed` wraps the Pebble store and a `Store.AfterAdopt` hook opens or closes it after every adoption, so flipping the switch is a reload, not a restart (seen ids persist across an off/on cycle; a failed open on reload is logged and ingest fails closed with `500` until the next reload, since the files asked for dedupe — at boot it still refuses to start; a record caught in the instant of the flip is published un-deduped and counted by `wavehouse_ingest_dedupe_disabled_total` rather than failed, and the hook is registered before the boot apply so a reload can never leave the settings and the store out of step). The watcher reloads once as soon as its watch exists, closing the gap between the boot read and the watch — an edit landing in between (a ConfigMap update during a rolling restart) is adopted, not silently missed. `dedupe.enabled` / `WH_DEDUPE_ENABLED` are removed from boot config alongside the other keys. What stays in boot config is only what cannot change under a running process — resource sizing (`data_dir`, `cache.l1_max_cost`), the listeners, the observability exporters — and the secrets. The compose stack now bind-mounts a checked-in `deployments/compose/settings/` (the seed with `clickhouse.addr` pointed at the `clickhouse` service) instead of a volume seeded with `bootstrap`, so the quickstart is `up -d` again; the e2e orchestrator copies the fixture settings per run and patches the testcontainer's ClickHouse ports into `config.json`, since that wiring no longer has an env override. Every after-adopt hook (dedupe, keepalive wheel) is registered before the reload triggers start, so the watcher's first reload can never be missed by a hook. Consumers take functions, not values (`IngestHandler.DedupeSettings`, the structured-query handler's `defaultMaxRows` / `bucketSecs func() int`, the ingest worker's `dlqEnabled func(table) bool`, the sweeper's `gapWindow func() time.Duration`, `corsMiddleware`'s origins getter, `SchemaRegistry`'s database and refresh-interval sources, the query handlers' timeout sources), so `internal/api` stays testable without materializing settings directories. The settings directory is also the **runtime authority for access control and named pipes** (`internal/settings/store.go`, `internal/policy/source.go` (new), `internal/pipes/pipes.go`, `internal/api/{policy,pipes,router}.go`, `internal/stream/hub.go`, `internal/auth/auth.go`, `cmd/wavehouse/main.go`, `Makefile`, `deployments/compose/settings/{policies,roles}.json`, `clients/ts/src/settings.ts` (new); closes [#229](https://github.com/Wave-RF/WaveHouse/issues/229), [#33](https://github.com/Wave-RF/WaveHouse/issues/33), [#461](https://github.com/Wave-RF/WaveHouse/issues/461), [#514](https://github.com/Wave-RF/WaveHouse/issues/514), [#460](https://github.com/Wave-RF/WaveHouse/issues/460), [#363](https://github.com/Wave-RF/WaveHouse/issues/363); advances [#48](https://github.com/Wave-RF/WaveHouse/issues/48) and [#214](https://github.com/Wave-RF/WaveHouse/issues/214)): `roles.json`, `policies.json`, and `pipes.json` are adopted with `config.json` as one snapshot and re-adopted on the same three triggers, and **files are the only write path** — standalone, the operator edits them on the host; on WaveHouse Cloud the control plane writes them — so there is no stored copy that can skip validation: every adoption runs the current rules (strict decode rejecting unknown and duplicate keys, the full policy validation including the claim-template grammar, pipe name/SQL/parameter-type rules, and the cross-file check that every role a grant or `allowed_roles` names is declared in `roles.json`), and a rejected edit keeps the previous good policy and pipes in effect. `policies.json` is one policy document (`{}` = no policy, adopted fail-closed with a warning); `pipes.json` carries full definitions (`allowed_roles`, `parameters`, `description`), so a file-defined pipe is no longer admin-only by construction. Consumers read the adopted snapshot per request through `policy.Source` (a `func() *policy.Policy`; `settings.Store.Policy` in production, `policy.Static(p)` in tests) and `pipes.Source` (`settings.Store`; `pipes.Static(q...)` in tests), so a reload applies to the very next request, including the SSE hub's per-event policy read. `GET /v1/ops/policy`, `POST /v1/ops/policy/validate`, `GET /v1/ops/pipes`, `GET /v1/ops/pipes/{name}`, and pipe execution are unchanged; the operator key still passes the `/v1/ops/*` gate under no policy, now as the break-glass that inspects the policy and triggers `POST /v1/ops/settings/reload` after `policies.json` is fixed. The SDK gains `wh.settings.reload()` (`POST /v1/ops/settings/reload`, returning `{ adopted, findings }`). The compose stack's trial `public` policy moves into the bind-mounted `deployments/compose/settings/policies.json` + `roles.json`, and `make dev` copies the same two files into its seeded `./settings` so a fresh dev server works tokenless. **Removed** — the write endpoints `PUT /v1/ops/policy`, `PUT /v1/ops/pipes/{name}`, and `DELETE /v1/ops/pipes/{name}`; the NATS KV buckets `WAVEHOUSE_POLICY` and `WAVEHOUSE_PIPES` and their KV Watch sync (`internal/policy/store.go`, the pipes KV store); the boot-config keys `policy.file_path` / `WH_POLICY_FILE_PATH` and `pipes.dir` / `WH_PIPES_DIR` (a leftover `policy:` or `pipes:` YAML block now refuses boot by name, like the other moved keys) and the `.sql`-directory pipes bootstrap; `deployments/compose/dev-policy.yaml`; the SDK methods `wh.policy.set`, `wh.pipes.set`, and `wh.pipes.delete`; and the test helpers `policy.NewMemoryStore`, `pipes.NewMemoryStore`, and `testutil/natsjs.go`. @@ -20,20 +22,30 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Changed +- **The chtypes SDK is `go/v0.4.0` (ABI revision 6)** (`go.mod`, `chtypes.lock`, `scripts/fetch-chtypes.sh`, `.github/actions/setup-env/action.yml`): the lock pins the revision-6 26.6.8.7 build (`b1790767905`) on darwin-arm64, linux-amd64 and linux-arm64, and must be regenerated whenever the SDK's ABI revision changes. The default artifact cache moved to `~/.cache/chtypes/artifacts/abi6/-`, so the first fetch after upgrading downloads again (explicit `--dest` / `CHTYPES_REGISTRY` directories, as the Docker images use, are unaffected); CI's cache key and path carry the revision. Insert `check` clauses now cost one parse instead of two: they are judged inside the same `RowsExportWith` call that validates the body, through a compiled row filter, and the five ingest formats (JSON family, CSV, TSV, CSV and TSV with `header=present`) all take that path. + +- **The release pipeline builds each binary on its own native runner; GoReleaser is now only the compiler** (`.goreleaser.yaml`, `.github/workflows/release.yml`, `.github/workflows/publish-dev.yml`, `.github/workflows/goreleaser-validate.yml`, `deployments/Dockerfile.goreleaser`, `docs/src/content/docs/development.md`): cgo cannot cross-compile darwin from Linux. Measured rather than inferred — `zig cc -target aarch64-macos` fails at *compile* time on `prometheus/client_golang`'s `process_collector_mem_cgo_darwin.c`, which `#include`s ``; `-tags netgo,osusergo` does not help, because the build never reaches the linker that the earlier `-lresolv` finding was about, and no Apple SDK can be fetched onto a GitHub-hosted Linux runner. GoReleaser's answers to this (split/merge, `builder: prebuilt`) are Pro-only and OSS `goreleaser release` accepts no `--skip=build`, so it cannot assemble a release from binaries built elsewhere. `release.yml` therefore runs `goreleaser build --single-target` on `ubuntu-latest`, `ubuntu-24.04-arm` and `macos-latest` — all free for public repos — and one `ubuntu-latest` job assembles the `.tar.gz` archives, `checksums.txt`, the multi-arch GHCR image (`docker buildx build` over the unchanged `Dockerfile.goreleaser`, given the same `//wavehouse` context layout `dockers_v2` used to produce), the GitHub Release and both provenance attestations. `.goreleaser.yaml` shrinks to `builds:` and keeps being the one declaration of the ldflags, binary name and supported platform set; its per-target `CC`/`CXX` overrides are gone. Behaviour is preserved deliberately, not incidentally: archive names and contents, `checksums.txt` format, the immutable-tag-plus-channel-pointer scheme via `scripts/ci/release-channel.sh`, `prerelease: auto` (now "the channel is not `latest`"), `mode: keep-existing` (now a `gh release view` guard, which also makes the job re-runnable) and `changelog.use: github-native` with `git.ignore_tags` (now `gh release create --generate-notes --notes-start-tag "$(git describe --match 'v*')"` — without that flag GitHub would happily diff a server release against a `clients/ts/v*` one). `publish-dev.yml` follows the same shape with only the two Linux targets, since a dev build publishes only the image. `goreleaser-validate.yml` becomes a real proof instead of a host-platform-only smoke test: `goreleaser check`, all three targets in `--snapshot`, and a genuine multi-arch `docker buildx build` to `--output type=cacheonly`, which also exercises the `chtypes.lock` fetch — the one PR-time signal that would have caught an upstream artifact republish before a tag did. Note the released **Linux binaries are now dynamically linked and require `GLIBC_2.34`** (measured on `ubuntu-24.04`, both architectures: Debian 12 / Ubuntu 22.04 / RHEL 9 and newer); the pre-cgo builds were static. Container images are unaffected — `distroless/cc-debian12` is glibc 2.36. + +- **The type layer is ClickHouse's own: ingest validation, row-level security and insert checks all run through chtypes** (BREAKING; new `internal/typelayer` package wrapping `github.com/wave-rf/chtypes/go` v0.4.0, a cgo dlopen of a per-ClickHouse-version shared library; `internal/discovery`, `internal/api/{ingest,content_type,ingest_framing}.go`, `internal/ingest/worker.go`, `internal/stream/hub.go`, `internal/policy`): the hand-written type-coercion, validation and row-filter code is replaced by calls into the same parser/analyzer ClickHouse's own server runs, loaded per ClickHouse minor line rather than compiled in. **The request body is no longer decoded in Go at all** — it goes to that parser as-is, in one call per request, and what comes back is a verdict per record plus the accepted rows as the exact `JSONCompactEachRow` bytes ClickHouse's writer produced. Consequences, all BREAKING: per-record errors carry an integer `code` (`{"code": , "error": ""}`) with ClickHouse's own wording — `27`/`26` unparseable, `117` unknown field, `6` out of range — so `400 {"error":"invalid json"}` is gone from this endpoint; a record the engine cannot answer for is `422 "validation engine declined: …"`, never a `400`; and **timestamp values on the wire — ingest responses, SSE rows, `/v1/query` results — carry ClickHouse's own rendering** (`"2026-06-21 04:00:00.123"`, in the column's zone) instead of the RFC 3339 `Z`-suffixed form WaveHouse used to canonicalize to, by construction rather than by a rewriting step (closes [#372](https://github.com/Wave-RF/WaveHouse/issues/372) a different way than originally planned). Row `filter` grants and insert `check` clauses are one mechanism now: both compile to a chtypes filter with every bound value a `{p:String}` parameter, and only a definite true admits — a compile failure, an evaluation error or a decline fails closed. + +- **A column the role may not insert is now ClickHouse's code 117, not a WaveHouse 403** (BREAKING; `internal/api/ingest.go`, `internal/typelayer/typelayer.go`, `clients/ts/src/types.ts`, `tests/e2e/sdk/ingest.test.ts`): column policy on the write path is answered by compiling the role its **own** copy of the table schema, without the columns it may not write, instead of walking a decoded record's keys. A record naming one is therefore refused by ClickHouse's parser exactly as an unknown column is — `400 {"code":117,"error":"Unknown field found while parsing JSONEachRow format: x"}` where 0.1.0 answered `403 {"error":"column \"x\" not allowed for insert"}`. The message no longer confirms whether the column exists, which is arguably the better answer. The read paths are unchanged: a denied column is still `403 column "x" not allowed` on `/v1/query` and still stripped from SSE events. Two further consequences of the same mechanism: an `_eq` insert check auto-injects by way of a `DEFAULT ''` on that compiled schema, so a supplied value still wins and an absent one is filled — but an `_in` check, which has no single value to stamp, now tests **the table's own default** against the claim-derived set rather than rejecting an absent column outright; and an explicit `null` on a checked column behaves exactly like omitting it. + +- **WaveHouse now requires cgo, and supported platforms narrow to darwin/arm64, linux/amd64, linux/arm64** (BREAKING; `go.mod`, `scripts/build.sh`, `.goreleaser.yaml`, `deployments/Dockerfile`, `deployments/Dockerfile.goreleaser`, `Makefile`, `internal/config/config.go`, `config.yaml`, `cmd/wavehouse/main.go`, `chtypes.lock` (new), `scripts/fetch-chtypes.sh` (new), `.github/actions/setup-env/action.yml`, `.github/workflows/ci.yml`): the native type layer above needs cgo for `dlfcn` (no C library linked, no header). cgo is now unconditional: `CGO_ENABLED=0` is gone from every build path, and the `make audit-cgo` target that policed the old no-cgo build has been **removed** along with it (`make binary-analysis` is now `size` + `deadcode`). Because chtypes publishes artifacts only for darwin-arm64, linux-amd64 and linux-arm64, **Windows, FreeBSD and darwin/amd64 builds are discontinued** — `.goreleaser.yaml`'s matrix drops from 8 targets to 3, and the release archives/checksums/GHCR image narrow to match. The runtime image moves from an Alpine/musl builder + `distroless/static` to `golang:1.27-bookworm` (glibc, ships gcc) + `distroless/cc-debian12` (glibc + libstdc++, which the SDK's shared library needs) and bakes the pinned chtypes artifact into the image at `/opt/chtypes/artifacts` via a new `chtypes.lock` (exact file + sha256 per platform/line) and `scripts/fetch-chtypes.sh --frozen` wrapper, so the container has no first-request download. `go.mod` moves to `go 1.27`. New boot config: `clickhouse.chtypes_registry` / `WH_CHTYPES_REGISTRY` lets an operator point at an explicit registry directory instead of the SDK's own search path (the shipped image instead sets the SDK's own `CHTYPES_REGISTRY` env var directly). CI's `unit`/`integration`/`e2e` jobs fetch and cache the pinned artifact (`setup-env`'s new `chtypes` input) and set `WAVEHOUSE_TEST_REQUIRE_CHTYPES=1` so a missing artifact fails the job instead of silently skipping the chtypes-backed tests. `GOLANGCI_LINT_VERSION` bumped `v2.11.4` → `v2.13.2`: the `go 1.27` bump panics `v2.11.4`'s type checker on every package; `v2.13.0` is the oldest release whose changelog claims go1.27 support, but it panics in this tree for a different reason (`nilness`/`honnef.co/go/tools@v0.8.0-rc.1` crashing while analyzing a third-party dependency), fixed once that dependency moves past its release candidate in `v2.13.1`. The cross-toolchain approach this bullet originally described was replaced before landing — see the release-pipeline entry above. + - **Boot refuses an unbound `WH_*` environment variable and an unusable `data_dir`** (BREAKING; `internal/config/check.go` (new, + tests), `internal/config/{config,persistence}.go`, `cmd/wavehouse/main.go`, `docs/src/integrations/diagram-png.mjs`): the environment half of the strict YAML loader. `config.Load` now errors, naming every offender, on a `WH_*` variable that no `Config` field binds — the two variables read outside the struct, `WH_CONFIG` and `WH_LOG_LEVEL`, are exempt — `WH_DEDUPE_ENABLED=true` left in a compose file from before the settings-directory move, or a misspelling, was set, ignored, and believed. **An existing deployment that still exports a variable this release moved to the settings directory stops booting until it is unset**; the upgrade runbook in `deployment.md` gains that audit. Only the `WH_` prefix is checked, since the environment always carries unrelated names; the one outside source that shares it — Kubernetes service-link variables for a Service named `wh` or `wh-*` — is named in the error with the `enableServiceLinks: false` remediation, and the docs build's opt-out knob is renamed from `WH_SKIP_DIAGRAM_PNG` to `DOCS_SKIP_DIAGRAM_PNG` so an exported one no longer refuses a local boot. Right after `Load`, before ClickHouse or the settings directory are touched, `config.CheckDataDir` probes `data_dir` and refuses boot on any of: an empty or blank value (reachable through `WH_DATA_DIR=`), refused outright since the ancestor walk would otherwise fall back to the working directory and NATS and Pebble state would land under it; a path that exists and is not a directory; a dangling symlink at `data_dir` or any component above it (the walk to the nearest existing ancestor uses `Lstat`, so a failed mount is not skipped over as "does not exist" and passed in an unrelated directory); and a directory the process cannot write to — or, when it does not exist, an unwritable nearest ancestor — probed by creating and removing one temp file. So an unusable `data_dir` refuses boot before schema discovery rather than after it; a permission denial — on the probe, or on reaching the path at all through a parent without search permission — carries the UID-65532 remediation (a bind mount owned by root is the typical cause), and that hint string is now shared with `LogStorageInitError`. `EnvConfig` and `EnvLogLevel` join `EnvSettingsDir` as the exported names for the process-level variables. Boot is the validator for the non-hot-reloadable half — there is no dry-run subcommand, by decision on #530: boot config only takes effect through a restart, so the restart is where it is checked, and the docs say so. Closes #530. - **The docs site now consumes the *published* `@wavehouse/sdk`, not the workspace one** (`docs/package.json`, `pnpm-workspace.yaml`, `Makefile`, `scripts/classify-paths.sh`). The landing page's live demo streams against a separately-deployed backend on its own release cadence, but took its SDK from the tree — so this release's wire change would have reached the deployed site the moment it merged, while the backend still spoke the old envelope: the panel keeps reporting "live" while every frame is dropped for want of a schema announcement, with no `error` callback ([#568](https://github.com/Wave-RF/WaveHouse/issues/568)). `docs` now pins `^0.1.1` from the registry, which takes `0.1.x` patches and stops short of `0.2.0`, so moving the site onto the new wire is a deliberate bump lined up with tagging the SDK release rather than a side effect of merging. `tests/e2e/sdk` deliberately keeps `workspace:*`. Depending on our own package from the registry also made `minimumReleaseAge` apply to it for the first time, and the exclude list named only `@wave-rf/*` (the plugin scope), so a freshly tagged SDK would have been uninstallable by the docs site for seven days — `@wavehouse/*` is now exempt too. The docs build no longer needs `build-ts`. - **Policy format v2: `policies.json` is role-first, and the two operations are separate permission types** (BREAKING; `internal/policy/policy.go`, `internal/policy/rowfilter.go`, `internal/settings/validate.go`, `internal/query/builder.go`, `internal/api/{ingest,structured_query}.go`, `internal/stream/hub.go`, `clients/ts/src/{types,index}.ts`, `deployments/compose/settings/policies.json`, `docs/src/content/docs/{access-control.mdx,settings-directory.mdx,architecture.md}`, `AGENTS.md`, `tests/e2e/sdk/`): a table entry was keyed `tables.
.select.`; it is now keyed `tables.
..select`. A role appears once per table and its grant carries two optional blocks, so a role that could both read and write no longer has to be written out twice, and "this role has no insert grant" is a missing block rather than an absence you have to notice in a second map. The blocks are now distinct types rather than one struct whose halves were inert per operation: `select` takes `allow_columns`, `deny_columns`, `filter`, `allowed_aggregations`, `denied_aggregations` and the four `max_*` limits; `insert` takes `allow_columns`, `deny_columns`, `check`. Field names and semantics are unchanged — only the nesting moves — but a field on the wrong side that the old layout *accepted* — the four `max_*` limits and the two aggregation rules — is now a validation error instead of being silently ignored. `filter` under an `insert` grant and `check` under a `select` one are rejected too, but that is not new here — [#541](https://github.com/Wave-RF/WaveHouse/pull/541), also unreleased, added the runtime check; the split types now refuse them one layer earlier, as unknown keys at the strict decode. Upgrading from **0.1.0**, though, none of the eight were enforced: a 0.1.0 policy could carry an insert-side `filter` that resolved into a `WHERE` the insert path never read, as well as an ignored limit. Converting to the role-first layout drops both. Internally `ResolvedPermissions` splits the same way (`.Select` / `.Insert`), and `IsColumnAllowed` takes the side to consult, which is what stops the read allowlist from ever answering a write question or vice versa. **There is no automatic conversion** — the settings files are the source of truth and WaveHouse has no write path back to them — so `policies.json` must be converted by hand; run `wavehouse validate` before restarting. A document still in the old layout is reported as one clear finding naming the table and operation and pointing at [the migration note](https://wavehouse.dev/access-control#migrating-from-the-operation-first-layout), instead of the confusing strict-decode "unknown field" error it would otherwise produce (or, for an empty operation block, silently decoding as a role named `select` with no grants — which fails the undeclared-role check when `roles.json` does not declare a role named `select` — the usual case, since `checkRoleRefs` errors and moves on before reaching the "grant sets neither select nor insert" warning. If such a role *is* declared, you get that warning instead and the document adopts). One shape is refused differently: a grant keyed by a role named after the *other* operation (`tables.t.select.insert`) reads as a different grant under each layout — different role, different operation, or both — so it gets its own error asking you to rename the role rather than the migration pointer. A role named after its *own* operation (`tables.t.select.select`) means the same thing either way and is accepted. -- **Ingest now requires a declared `Content-Type`, and it is authoritative** (BREAKING; `internal/api/record_reader.go`, `internal/api/ingest.go`, `clients/ts/src/table.ts`, `docs/src/content/docs/{api.md,architecture.md,sdk/queries.md}`): `POST /v1/ingest` used to sniff the body and treat the header as a hint — the first non-whitespace byte chose between a single object and an array, and an `application/x-ndjson` body that happened to start with `[` was silently re-read as a JSON array. A request that declares **no** `Content-Type`, or one whose media type is not in the accepted list, is now rejected with `415` before the body is parsed, naming every accepted type (`application/json`, `application/x-ndjson`, `application/ndjson`, `application/jsonl`, `application/jsonlines`) and quoting what was declared, bounded to four distinct header lines each capped at 128 bytes. The header is parsed per RFC 9110 §8.3 via `mime.ParseMediaType` rather than by hand, so the grammar's rules apply — parameters never affect the format (with the one exception below), and a comma inside a quoted value is data. Because `Content-Type` is a **singleton** field (§5.3 forbids repeating it), anything that is not exactly one readable media type is refused; the one accommodation is that repeated header lines are all resolved and accepted when they agree, since honoring just the first would let an NDJSON body be read as one JSON object and drop every record past it. A comma-joined value gets no such accommodation — §8.3 warns that taking a member of the pseudo-list is itself an interoperability and security hazard. **Four additional shapes 0.1.0 accepted now `415`** (beyond repeated header lines that disagree, which it also accepted): a present-but-empty header (a bare `Content-Type:` line, or one that is only whitespace); a value with a trailing or leading comma (`application/json,`); a comma-joined value that does not parse as a single media type (`application/json, application/json` — but a comma *inside a quoted parameter value* is legal data, so `application/json; a=", application/x-ndjson; b="` is one media type and is accepted); and a malformed parameter on a line that *also* carries a comma, which is refused rather than guessed at because the comma may be a second declaration joined on — so `application/json; profile="a,b"; charset` is a `415` while `application/json; profile="a,b"` and `application/json; charset` are each accepted ([#563](https://github.com/Wave-RF/WaveHouse/issues/563)). A repeated parameter name is *not* among them: the media type is re-parsed alone, so `; charset=a; charset=b` reads as `application/json` like every other malformed parameter. The declaration now decides the format outright: a body declared as NDJSON is read as NDJSON whatever its first byte, so a line that isn't a JSON object fails as a **per-record** error through the existing batch-result path instead of re-framing the whole request. The body still picks arity *within* the JSON family — `[` is an array, anything else a single object — because those are the same format at different lengths. Clients that relied on the sniffer must now send a header; the TS SDK already sent one on both paths (`application/json` for a single object, `application/x-ndjson` for arrays and `insertNDJSON`) and now states it at each call site rather than leaning on the request default, and every `curl` example in the docs already carried one. The format is modeled as an `IngestFormat` where the sniffing used to live, with the slot for CSV kept where the old comment marked it. +- **Ingest now requires a declared `Content-Type`, and it is authoritative** (BREAKING; `internal/api/content_type.go`, `internal/api/ingest.go`, `clients/ts/src/table.ts`, `docs/src/content/docs/{api.md,architecture.md,sdk/queries.md}`): `POST /v1/ingest` used to sniff the body and treat the header as a hint — the first non-whitespace byte chose between a single object and an array, and an `application/x-ndjson` body that happened to start with `[` was silently re-read as a JSON array. A request that declares **no** `Content-Type`, or one whose media type is not in the accepted list, is now rejected with `415` before the body is parsed, naming every accepted type (`application/json`, `application/x-ndjson`, `application/ndjson`, `application/jsonl`, `application/jsonlines`, `text/csv`, `text/tab-separated-values`) and quoting what was declared, bounded to four distinct header lines each capped at 128 bytes. The header is parsed per RFC 9110 §8.3 via `mime.ParseMediaType` rather than by hand, so the grammar's rules apply — parameters never affect the format (with the one exception below), and a comma inside a quoted value is data. Because `Content-Type` is a **singleton** field (§5.3 forbids repeating it), anything that is not exactly one readable media type is refused; the one accommodation is that repeated header lines are all resolved and accepted when they agree, since honoring just the first would let an NDJSON body be read as one JSON object and drop every record past it. A comma-joined value gets no such accommodation — §8.3 warns that taking a member of the pseudo-list is itself an interoperability and security hazard. **Four additional shapes 0.1.0 accepted now `415`** (beyond repeated header lines that disagree, which it also accepted): a present-but-empty header (a bare `Content-Type:` line, or one that is only whitespace); a value with a trailing or leading comma (`application/json,`); a comma-joined value that does not parse as a single media type (`application/json, application/json` — but a comma *inside a quoted parameter value* is legal data, so `application/json; a=", application/x-ndjson; b="` is one media type and is accepted); and a malformed parameter on a line that *also* carries a comma, which is refused rather than guessed at because the comma may be a second declaration joined on — so `application/json; profile="a,b"; charset` is a `415` while `application/json; profile="a,b"` and `application/json; charset` are each accepted ([#563](https://github.com/Wave-RF/WaveHouse/issues/563)). A repeated parameter name is *not* among them: the media type is re-parsed alone, so `; charset=a; charset=b` reads as `application/json` like every other malformed parameter. The declaration now decides the format outright: a body declared as NDJSON is read as NDJSON whatever its first byte, so a line that isn't a JSON object fails as a **per-record** error through the existing batch-result path instead of re-framing the whole request. The body still picks arity *within* the JSON family — `[` is an array, anything else a single object — because those are the same format at different lengths. Clients that relied on the sniffer must now send a header; the TS SDK already sent one on both paths (`application/json` for a single object, `application/x-ndjson` for arrays and `insertNDJSON`) and now states it at each call site rather than leaning on the request default, and every `curl` example in the docs already carried one. One correction since: a JSON **array** declared `application/x-ndjson` now ingests every element, because the declaration only picks the format and ClickHouse's reader takes the brackets. - **A policy `check` on a column the table cannot accept is now refused instead of silently unenforced** (BREAKING; `internal/api/ingest.go`, `internal/discovery/discovery.go`, `docs/src/content/docs/{api.md,access-control.mdx}`): a `check` clause naming a column the table does not have, one it computes (`MATERIALIZED`/`ALIAS`), or an `EPHEMERAL` one can never be enforced — the published row carries one slot per insertable column, so an auto-injected value for anything outside that set is dropped on the way out, and an ephemeral column is never stored even though the row does carry it. The record inserted **without** the value the policy required and answered `200 {"ok":true}`. Demonstrated on this branch: a `check` of `tenant _eq {{ jwt.tenant }}` against a `MATERIALIZED tenant` column published `columns:["page"], row:["/a"]` — the tenant constraint absent from the row entirely. It is now refused, naming every offending column and the reason, on **every** insert by that role until the policy or the table is corrected: a single-object request answers `403`, while a batch answers `200` with the same message against each record in `results` — the batch is still read to the end and reports per record, as it does for any other rejection. Policy validation cannot catch this — it never sees the ClickHouse schema — so **audit your `check` blocks against their tables before upgrading**; `wavehouse validate` will not tell you. -- **The ingest envelope carries only insertable columns** (BREAKING; `internal/discovery/discovery.go`, `internal/api/ingest.go`, `internal/stream/hub.go`, `internal/testutil/testutil.go`, `tests/integration/ingest_test.go`, `clients/ts/src/types.ts`): naming columns explicitly in the `INSERT` — the change above — makes a computed column fatal, so the envelope, the compact encoder and the SSE connect-time announcement now use the table's **insertable** subset. Verified against ClickHouse 26.6.3: a `MATERIALIZED` column in an `INSERT` column list is `Cannot insert column …, because it is MATERIALIZED column` (code 44, and `insert_allow_materialized_columns` defaults to `0`); an `ALIAS` column is `No such column …` (code 16). Schema discovery reads every row of `system.columns` with no `default_kind` filter, so without this both would land in the envelope and then in the statement, and **a table carrying either could ingest under the previous column-less `FORMAT JSONEachRow` and could not ingest at all** — every row to the DLQ, or redelivered forever where the DLQ is off. `Column` gains `DefaultKind`; `TableSchema` gains `IsInsertable` / `InsertableColumns` / `InsertableColumnNames`, memoized per table since the ingest path would otherwise rebuild them once per record. `EPHEMERAL` stays insertable — it is insert-only by construction, never stored, confirmed on the same server rather than assumed. `GET /v1/ops/schema` still reports the whole table, now including `default_kind`, `default_expression` and `position`: a computed column stays queryable, it just cannot be written. No fixture in the suite declared a computed column, which is why every gate was green while this was broken; `tests/integration` now creates one and drives HTTP ingest → NATS → the worker's `INSERT` end to end. **BREAKING:** a record that *supplies* a value for a `MATERIALIZED`/`ALIAS` column is now rejected (`400 … cannot be inserted`) where it was previously accepted and silently dropped by the positional encoder. +- **The ingest envelope carries only insertable columns** (BREAKING; `internal/discovery/discovery.go`, `internal/api/ingest.go`, `internal/stream/hub.go`, `internal/testutil/testutil.go`, `tests/integration/ingest_test.go`, `clients/ts/src/types.ts`): naming columns explicitly in the `INSERT` — the change above — makes a computed column fatal, so the envelope, the compact encoder and the SSE connect-time announcement now use the table's **insertable** subset. Verified against ClickHouse 26.6.3: a `MATERIALIZED` column in an `INSERT` column list is `Cannot insert column …, because it is MATERIALIZED column` (code 44, and `insert_allow_materialized_columns` defaults to `0`); an `ALIAS` column is `No such column …` (code 16). Schema discovery reads every row of `system.columns` with no `default_kind` filter, so without this both would land in the envelope and then in the statement, and **a table carrying either could ingest under the previous column-less `FORMAT JSONEachRow` and could not ingest at all** — every row to the DLQ, or redelivered forever where the DLQ is off. `Column` gains `DefaultKind`; `TableSchema` gains `IsInsertable` / `InsertableColumns` / `InsertableColumnNames`, memoized per table since the ingest path would otherwise rebuild them once per record. `EPHEMERAL` stays insertable — it is insert-only by construction, never stored, confirmed on the same server rather than assumed. `GET /v1/ops/schema` still reports the whole table, now including `default_kind`, `default_expression` and `position`: a computed column stays queryable, it just cannot be written. No fixture in the suite declared a computed column, which is why every gate was green while this was broken; `tests/integration` now creates one and drives HTTP ingest → NATS → the worker's `INSERT` end to end. **BREAKING:** a record that *supplies* a value for a `MATERIALIZED`, `ALIAS` or `EPHEMERAL` column is now rejected where it was previously accepted and silently dropped by the positional encoder — as ClickHouse's `400 {"code":117,"error":"Unknown field found while parsing JSONEachRow format: x"}`, since none of the three is part of the role's compiled schema. -- **Ingest reads the request body up front, and the per-record decisions sit behind interfaces** (`internal/api/{ingest,ingest_seams,bufpool,record_reader}.go`, `internal/stream/hub.go`, `internal/ingest/compact.go`): responses are unchanged except at the body cap and one new read-failure body (`400 {"error":"invalid request body"}`, when the body cannot be read at all — a malformed transfer encoding or a truncated upload, which previously surfaced through the decoder as `invalid json`), and at the cap the `413` is now decided before any record is processed: an over-cap batch no longer ingests the prefix it had already decoded, and an over-cap single-object body whose first object was followed by an oversized tail — which used to answer `200` after ingesting that one object — now answers `413`. Both are improvements, since a client retrying a `413` can no longer double-insert a prefix, but they are behavior changes and the memory profile changes too (see below); this is the seam work the native type layer lands against. The handler now reads the whole (already `MaxBytesReader`-capped) body into a pooled `*bytes.Buffer` and runs the record readers over those bytes rather than the live connection — so the `413` surfaces at that read instead of mid-iteration (same status, same message), and the `415` is decided from the header before a single byte is read. Three decision points became interfaces with default implementations that delegate to today's code unchanged: `RecordValidator` (schema validation + timestamp canonicalization — the two calls stay where they are, with the check-clause block between them, since merging them would move checks onto canonicalized values), `InsertChecker` (the `_eq` and `_in` comparisons), and `stream.RowEvaluator` (row visibility, reached by both the live fan-out and replay through the one shared admission step). **The memory profile is not unchanged, and that is the deliberate part.** Streaming meant peak resident bytes on the order of one record: the NDJSON path scanned line by line and the array path let `json.Decoder` compact after each element. Peak is now O(body) per in-flight request — and `bytes.Buffer` grows by doubling, so the peak allocation can exceed the body cap before `MaxBytesReader` errors. `maxPooledBufferBytes` (1 MiB) caps what a request hands *back* to the pool, not its peak, and nothing in `internal/api` bounds total in-flight bytes, so the ceiling is concurrency × the 16 MiB data-plane cap — which has no operator knob (`maxRequestBytes` is test-only), so the outer limit is the reverse proxy's, which the reverse-proxy guide already advises setting. Kept because it is the shape the native type layer lands against, which needs the body addressable rather than consumed; a bound on total in-flight ingest bytes is tracked in [#544](https://github.com/Wave-RF/WaveHouse/issues/544). Operators fronting large batches at high concurrency should size for it or cap body size at the proxy. All three are nil-safe: an un-wired handler or `Hub` uses the default rather than panicking past the check. Also new: `ingest.EncodeCompactRow`, which renders a record as one `JSONCompactEachRow` line — inert in this commit, and the encoder every published row goes through by the end of the release. +- **Ingest reads the request body up front, and the record readers are gone** (BREAKING at the body cap; `internal/api/{ingest,bufpool,content_type,ingest_framing}.go`, `internal/stream/{hub,roweval}.go`): the handler reads the whole (already `MaxBytesReader`-capped) body into a pooled `*bytes.Buffer` and hands those bytes to ClickHouse's parser in one call, so the `415` is decided from the header before a byte is read and the `413` before any record is processed — an over-cap batch no longer ingests the prefix it had already decoded, and an over-cap single-object body whose first object was followed by an oversized tail, which used to answer `200`, now answers `413`. A body that cannot be read at all is `400 {"error":"invalid request body"}`, where the decoder used to surface it as `invalid json`. The per-format record readers, the 500-record chunking and the 10 MiB per-NDJSON-line bound are all deleted with them; the only bytes WaveHouse still inspects itself are in `ingest_framing.go` — the first non-whitespace byte, the in-place re-frame of a top-level array, and the positional read of the dedupe id out of the exported row. **The memory profile changes and that is deliberate.** Streaming meant peak resident bytes on the order of one record; peak is now O(body) per in-flight request, and `bytes.Buffer` grows by doubling, so the peak allocation can exceed the body cap before `MaxBytesReader` errors. `maxPooledBufferBytes` (1 MiB) caps what a request hands *back* to the pool, not its peak, and nothing in `internal/api` bounds total in-flight bytes, so the ceiling is concurrency × the 16 MiB data-plane cap — the outer limit is the reverse proxy's, which the reverse-proxy guide already advises setting. A bound on total in-flight ingest bytes is tracked in [#544](https://github.com/Wave-RF/WaveHouse/issues/544). -- **NATS envelope v2: the row travels positionally, with the column names sent alongside** (BREAKING; `internal/ingest/{types,compact,worker}.go`, `internal/api/ingest.go`, `docs/src/content/docs/{api.md,architecture.md,ingest-pipeline.md}`): `EventMessage`'s `data` object is replaced by `format` (`"JSONCompactEachRow"`), `columns` (the table's declaration order) and `row` (one compact line — a positional JSON array). The `INSERT` the worker emits carries the column names once for a whole group instead of every row repeating every key (each NATS envelope still carries its own `columns`, since a message must stand alone), and a reader can tell a schema change mid-stream from a reordering. **In-flight NATS messages published by an older version are not readable by the new worker** — an envelope whose `format` is absent or unknown, or whose `columns` and `row` can't be paired (a length mismatch, an undecodable row), carries no way to say which value belongs to which column. Such an envelope is parked on the DLQ (see the Fixed entry below), never inserted. **Drain the ingest queue before deploying.** The worker groups a batch by column list, so a schema change mid-stream splits the INSERT rather than corrupting it, and writes `INSERT INTO {table} (cols) FORMAT JSONCompactEachRow` — the table still binds as a server-side `Identifier` parameter, while the column list, which has no such parameter, is quoted client-side by the same `chsql.QuoteIdent` the query builder uses. A positional row has one value per column and no way to say "absent", so a field the record omitted now rides as an explicit `null` in its slot; `input_format_null_as_default=1` (already the server default — set explicitly for one configured otherwise) turns that back into the column's default for a **non-nullable** column, matching what omitting the key did under `JSONEachRow`. **Transitional divergence, on `Nullable` columns only, and it is not what that setting controls:** ClickHouse stores an explicit `null` as `NULL` on a nullable column whatever the setting says — only an *absent* key ever took the default — so a `Nullable(T) DEFAULT …` column now stores `NULL` where it previously took its default. Verified against ClickHouse 26.6.3 (omitted key → default; explicit null → `NULL` at either setting). *Against a server explicitly running `input_format_null_as_default=0`*, the reverse also changes: an explicit `null` for a non-nullable column with a default now takes the default rather than failing the row into the DLQ, because WaveHouse pins the setting instead of inheriting it. On a default-configured server that was already the behavior. Every other column type behaves as before. The DLQ flow is unchanged — its payload is the new envelope. Row cells are copied as their original bytes rather than re-encoded at each hop, so a 64-bit id past 2^53 keeps every digit end to end. +- **NATS envelope v2: the row travels positionally, with the column names sent alongside** (BREAKING; `internal/ingest/{types,compact,worker}.go`, `internal/api/ingest.go`, `docs/src/content/docs/{api.md,architecture.md,ingest-pipeline.md}`): `EventMessage`'s `data` object is replaced by `format` (`"JSONCompactEachRow"`), `columns` (the table's declaration order) and `row` (one compact line — a positional JSON array). The `INSERT` the worker emits carries the column names once for a whole group instead of every row repeating every key (each NATS envelope still carries its own `columns`, since a message must stand alone), and a reader can tell a schema change mid-stream from a reordering. **In-flight NATS messages published by an older version are not readable by the new worker** — an envelope whose `format` is absent or unknown, or whose `columns` and `row` can't be paired (a length mismatch, an undecodable row), carries no way to say which value belongs to which column. Such an envelope is parked on the DLQ (see the Fixed entry below), never inserted. **Drain the ingest queue before deploying.** The worker groups a batch by column list, so a schema change mid-stream splits the INSERT rather than corrupting it, and writes `INSERT INTO {table} (cols) FORMAT JSONCompactEachRow` — the table still binds as a server-side `Identifier` parameter, while the column list, which has no such parameter, is quoted client-side by the same `chsql.QuoteIdent` the query builder uses. A positional row has one value per column and no way to say "absent", but nothing has to stand in for one: the row is what ClickHouse's own writer produced, so an omitted field's `DEFAULT` — including a volatile one like `now()` — was already evaluated before the line existed, on a `Nullable(T) DEFAULT …` column as on any other. Verified against ClickHouse 26.6.3. The DLQ flow is unchanged — its payload is the new envelope. Row cells are copied as their original bytes rather than re-encoded at each hop, so a 64-bit id past 2^53 keeps every digit end to end. - **SSE: the column list is announced as an `event: schema` frame, and rows arrive positionally** (BREAKING for raw consumers; `internal/stream/{hub,subscriber,metrics}.go`, `internal/api/stream.go`, `clients/ts/src/stream/sse.ts`, `docs/src/content/docs/{api.md,sdk/streaming.md}`): a data frame's `data` object is replaced by `row`, the compact array reduced to the caller's projected positions, and each connection is sent `event: schema` — `{"table_name", "columns"}` — before its first row and again whenever the projected list changes. The announcement is **per connection**: it goes out at subscribe time from the schema registry, so a client on a quiet table knows the shape before any row arrives, and a late joiner or a reconnect is told again. It deliberately carries **no** `id:` line — an empty one would clear the client's `Last-Event-ID` and cost the connection its resumption point, and a schema frame has no event position of its own to offer. Replay follows the identical contract with its own drift state, because it writes straight to the socket while live events queue behind it — sharing the connection's state would let a live announcement claim the slot and leave a replayed row ahead of it with nothing to zip against. **Known limitation, deferred to the schema-versioning work:** those two states are not reconciled when they disagree, so if a table's column set changes while a client is connected *and* that client gap-fill-replays across the change, live rows arriving after the replay may carry no fresh schema frame until the next drift or a reconnect ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)). The SDK's arity check catches most of it — a row whose **length** disagrees with the announced list is dropped rather than guessed at — but not a **same-length** change (a `RENAME COLUMN`, or a drop paired with an add), where values zip under the wrong names until the next announcement or a reconnect. So the residual case costs correctness, not only availability. The TypeScript SDK consumes the schema event and zips each row back into an object, so `.stream()`, `.liveQuery()` and `StreamEvent.data` are **unchanged** — two things are visible: a column the producer omitted now arrives as an explicit `null` rather than an absent key, and a row object has a **null prototype** (`Object.create(null)`), so a ClickHouse column legitimately named `__proto__` becomes an own property instead of vanishing into the inherited setter — at the cost of `row.hasOwnProperty(…)`, `` `${row}` `` and `row.constructor` no longer working on it. Use `Object.hasOwn(row, …)`. The client-side `.select(…)` projection now builds its row the same way: `projectColumns` used a plain object literal, so a `__proto__` column survived an unprojected stream and vanished from a projected one — the guarantee held for the transport but not for the path most callers use. **The SSE reader and writer changed together, so a version-skewed pair is a silently dead stream — upgrade both.** `@wavehouse/sdk` publishes independently of the server, so a pinned frontend against a self-scheduled backend is the normal shape, not an edge case. A **new SDK against an older server** never receives an `event: schema` frame, so `_columns` stays unset and every data frame is dropped — no `error` callback fires, the stream simply delivers nothing. The warnings are bounded (three per cause per connection, then a suppression line), so **a quiet console is not evidence the stream is healthy**. An **older SDK against a new server** reads the removed `data` key and yields `data: undefined` for every event. Neither surfaces as a catchable error. A **raw** SSE consumer (a hand-rolled `EventSource`) must keep the announced list and zip against it; the SDK drops — with a warning, never an error — a row it has no list for or whose length disagrees with one, rather than delivering values under guessed names. @@ -41,10 +53,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **The landing page's live demo now reads from the stats deployment's new WaveHouse Cloud backend** (`docs/src/components/LiveDemo.astro`, `docs/scripts/screenshot.mjs`): the GitHub-activity dogfood deployment behind the hero panel (Wave-RF/WaveHouse-Stats) moved off its self-managed AWS infrastructure onto WaveHouse Cloud, so `BASE_URL` — the origin `@wavehouse/sdk` queries in the visitor's browser — points at `https://iefrrvavd5akvphk7pq3.wavehouse.app` instead of `https://stats.wavehouse.dev`, ahead of the AWS stack being torn down. The `PUBLIC_WAVEHOUSE_STATS_URL` build-time override is unchanged, so a fork or staging docs build still redirects the panel without a code edit. **`DEMO_HOST` deliberately stays `stats.wavehouse.dev`** — the demo *site* is still served there and is still what the panel's chrome label and "Full demo" link should show; the migration splits the site from the API origin behind it, and the two constants now carry comments saying so. Verified against the new deployment before the switch: all five pipes the panel reads (`gh_summary`, `gh_activity_recent`, `gh_events_per_minute`, and the pre-#19 `gh_stars_total` / `gh_forks_total` fallbacks) return `200` with the same row shapes, the structured-query backfill fallback (`POST /v1/query?table=gh_events`) matches its old-backend response byte for byte, `GET /v1/stream?table=gh_events` opens an SSE stream, and CORS is unchanged (`Access-Control-Allow-Origin: *`, `X-Cache` exposed) so the cross-origin browser reads keep working from the docs site. The new backend is already the live ingest target — it reported more recent events than the old one at cutover (5,175 vs 5,038 over 7d) — which is the other half of why the panel had to follow it. `screenshot.mjs`'s `networkidle` note is retargeted to "the stats demo backend" rather than naming a host it no longer connects to. +- **Structured queries and pipes are rendered by ClickHouse, not by WaveHouse** (BREAKING; `internal/api/clickhouse_exec.go` (deleted), `internal/api/clickhouse_http.go` (new), `internal/api/{structured_query,pipes,cache_key,ch_settings}.go`, `internal/query/builder.go`, `internal/chsql/chsql.go`): `POST /v1/query` and `GET/POST /v1/pipes/{name}` used to run through `clickhouse-go`'s native driver and re-render every row in Go; they now go over ClickHouse's HTTP interface with `default_format=JSONEachRow`, bind each value as a named `{pN:String}` parameter, and the cache stores ClickHouse's own bytes. The whole Exec-vs-Query mutation classifier goes with the driver. **`Decimal*` values are now a JSON number (`12.5`) where they were a string (`"12.5"`)** — which is what `clients/ts` codegen and the SDK reference already claimed, so the generated types and the docs were wrong before and are right now. Response object keys come back in **SELECT order** rather than alphabetical. Three further behaviour changes: a `null` filter value is now `400 {"error":"filter value must not be null"}` instead of a silently empty result (`col = NULL` is never true); a ClickHouse refusal is still a `500` but the body carries ClickHouse's own wording (`Code: 158. DB::Exception: … (TOO_MANY_ROWS)`); and the RFC3339 filter-value rewrite is deleted, so a `"…Z"` value goes to the server verbatim — correct on every ClickHouse this repo pins (`cast_string_to_date_time_mode` defaults to `best_effort` from 26.5), a per-query `500` below that line. An `in` list binds as one `Array(String)` parameter, which ClickHouse caps at roughly 64 KiB of literal text (~6,000 short elements); past that the query fails with ClickHouse's own `500` rather than a clean `400`. Cache keys change value, so a deploy serves one cold cache; `X-Cache` semantics, the namespace deps and the singleflight are untouched. `/v1/ops/query` is unaffected. + - **WH001 (no hard-wrapped prose) now applies to every tracked Markdown file, with no carve-out** (`.github/.markdownlint.json` (deleted), `.claude/.markdownlint.json` (deleted), `.claude/skills/integration-astro-view-transitions/` (deleted), `.markdownlint-cli2.jsonc`, `.github/workflows/README.md`, `.claude/skills/pm-triage/references/routine.md`, `AGENTS.md`, `scripts/docs-prose.sh`, `.github/prompts/docs-review.md`, `docs/src/content/docs/claude-code.md`, `docs/src/content/docs/development.md`, `.claude/agents/docs-reviewer.md`): two path-scoped configs had switched WH001 off under `.github/` and `.claude/` ever since [#489](https://github.com/Wave-RF/WaveHouse/pull/489) introduced the rule — baked in from the start rather than added in response to a discovered problem — which left the repo documenting the rule three ways and disagreeing with itself: `CONTRIBUTING.md` promises contributors `make lint` enforces it *everywhere*, while `AGENTS.md` and the `.markdownlint-cli2.jsonc` header wrote up the carve-out. Not theoretical: on [#520](https://github.com/Wave-RF/WaveHouse/pull/520) a reviewer correctly flagged a hard-wrapped bullet in `.github/workflows/README.md`, an agent pointed at `"WH001": false` for that path and pushed back, and the reviewer recorded a *learning* never to flag WH001 there — the wrong invariant, learned off the wrong side of the contradiction ([#521](https://github.com/Wave-RF/WaveHouse/issues/521)). Both configs are deleted — each held nothing but the override, so the root `.markdownlint.json` governs again — and the 51 hard-wrapped paragraphs they were hiding are joined: 41 in `.github/workflows/README.md` and 10 in `.claude/skills/pm-triage/references/routine.md`, mechanical joins with no wording changed and every fenced block, table row, and heading byte-identical either side of the reflow. Deleted with them: the wizard-installed PostHog skill at `.claude/skills/integration-astro-view-transitions/` — 9 files, ~1,456 lines, including an 809-line `EXAMPLE.md` copied wholesale from `PostHog/context-mill`. Its integration job finished in [#277](https://github.com/Wave-RF/WaveHouse/pull/277), nothing in the repo calls it, and the docs-site setup it once described is documented where it belongs — in `docs/src/components/PostHog.astro` and this file. Keeping unowned third-party prose in the tree means content that drifts silently on every upstream bump and that nobody here reviews; it was also the single file that would have needed a special-case lint exclusion, so removing it is what lets WH001 apply with **no exception at all** rather than one documented one. Its two inventory rows in `claude-code.md` go with it, as does the now-dead `docs/posthog-setup-report.md` entry in the `scripts/docs-prose.sh` denylist (the wizard's other artifact, deleted back in [#502](https://github.com/Wave-RF/WaveHouse/pull/502)) and the copies of that denylist in `AGENTS.md` and `.github/prompts/docs-review.md`, which the script's header requires be kept in lockstep. Review of the change then turned up four more things the exclusion had been hiding, all fixed here: **WH001 has a blind spot** — `no-hard-wrapped-prose.mjs` classifies any line indented four or more spaces as an indented code block, so a *nested* list item is never joined, which left three hard-wrapped bullets in `.github/workflows/README.md` §"Adding a job" that the autofix could not see (unwrapped by hand; they were the last hard-wrapped prose paragraphs in the repo) and made `AGENTS.md`'s and `development.md`'s "a list item is joined as a unit" wrong for nested items (both now state the four-space caveat); the `scripts/docs-prose.sh` header told readers to keep its denylist in lockstep with **two** sibling copies when there are **three** — the missed one being `.claude/agents/docs-reviewer.md`, the gating subagent's own system prompt, which had in fact been silently out of sync for the whole life of the `posthog-setup-report.md` exclusion; the `.markdownlint-cli2.jsonc` header's "applies to every tracked Markdown file" was exact for WH001 but not WH002, which returns early on anything that isn't `.mdx`; and the job-graph diagram omitted `docs-deploy`'s `needs` edges from `unit`, `integration`, and `e2e`, contradicting invariant 2 three lines below it. The denylist also drops its `PERF-CLAIMS-REVIEW.md` entry — unlike the wizard artifact this one names a file that was **never tracked** at all, so it guarded a hypothetical; the list's other general cases are patterns (`*.draft.md`, `*.old.md`) that already cover a one-off review document, and a literal filename restated in four places is the outlier. `scripts/docs-prose.sh all` still resolves the same 27-file prose set. ### Removed +- **`policy.LiteralValue` and `policy.CanonicalNumericLiteral`** (`internal/policy/{canonical,policy}.go`): the marker type and the numeric re-reading of a policy-authored insert-check literal. Insert checks are chtypes filters now, so a literal binds as written and ClickHouse reads it under the column's type — there is no second, numeric reading at compare time. A `_eq: "1.0"` against a `UInt64` used to admit a stored `1`; it is now ClickHouse's code 53 `TYPE_MISMATCH` per row (`422`), and the fix is to write a literal the column can read. `CanonicalScalar` stays: it is still the one rendering layer for a JWT claim. The released-version entry further down this file describing `LiteralValue` as shipped behaviour is left as history. + - **The policy's entire HTTP surface — `GET /v1/ops/policy`, `POST /v1/ops/policy/validate`, and the SDK's `wh.policy` namespace** (`internal/api/policy.go` + `policy_test.go` (deleted), `internal/api/{router,router_test}.go`, `cmd/wavehouse/main.go`, `clients/ts/src/policy.ts` (deleted), `clients/ts/src/{client,types,index}.ts`, `tests/e2e/sdk/{admin,query,ingest,streaming}.test.ts` + `settings.ts`, `docs/src/content/docs/{api.md,access-control.mdx,settings-directory.mdx,architecture.md,configuration.mdx,development.md,reverse-proxy.mdx,sdk/admin.md,sdk/reference.md}`, `AGENTS.md`; closes [#514](https://github.com/Wave-RF/WaveHouse/issues/514)): both endpoints were born alongside `PUT /v1/ops/policy` and outlived it when [#508](https://github.com/Wave-RF/WaveHouse/pull/508) deleted the policy write API; with files as the only write path, the policy is read, edited, and validated where it lives, so the whole read/dry-run surface goes too. The dry run had also kept its original lenient decoder while adoption became strict, certifying `{"valid": true}` for documents a reload would refuse — a misspelled operator key (`"eq"` for `"_eq"`) silently dropped into a filter that disables row security, the exact fail-open [#460](https://github.com/Wave-RF/WaveHouse/issues/460) demonstrated; deleting it removes the last non-strict policy decode site, closing #514 (the other five sites it cites were deleted or made strict by #508). Its replacement is `wavehouse validate`, which enforces strictly more (the cross-file role references against `roles.json` were invisible to a single-document dry run). The break-glass story narrows accordingly: the operator key can still trigger `POST /v1/ops/settings/reload`, whose findings report exactly why a rejected directory was refused — what it can no longer do is read back the adopted snapshot over HTTP; a bad edit still never breaks a running server (the previous good snapshot stays adopted). The e2e suite's read-modify-write helper pattern moves from `wh.policy.get()` to reading the harness-owned `policies.json` directly (`readPolicyFile()` — the file *is* the adopted policy there, since `setPolicy` fails unless the reload reports adoption). The policy document types (`Policy`, `TablePolicy`, `RolePermissions`) stay exported from the SDK — they describe `policies.json` and the e2e harness consumes them; `GET /v1/ops/pipes[/{name}]` is untouched. - **The README coverage badge and its whole publishing pipeline** (`.github/workflows/ci.yml`, `scripts/ci/publish-badge.sh` (deleted), `scripts/cov/main.go`, `.github/workflows/README.md`, `.testcoverage.yml`, `AGENTS.md`): [#502](https://github.com/Wave-RF/WaveHouse/pull/502) rewrote the README badge row and dropped the Go Coverage badge, but nothing removed what fed it — so for the six days until this landed the non-gating `badge` job kept running on every main push, holding `ci.yml`'s only `contents: write`, publishing `coverage-go.json` to the orphan `badges` branch for a badge no page rendered. Retired rather than restored ([#509](https://github.com/Wave-RF/WaveHouse/issues/509)): the `badge` job, its two producer steps in `coverage` (`cov badge` + the `go-coverage-badge` artifact), `scripts/ci/publish-badge.sh`, and the `cov badge` subcommand (with `badgeData`/`badgeColor`). The orphan `badges` branch is deleted separately once this lands — while the job still exists on main, the next code push would recreate it. **The gate is untouched** — `make cov`, `.testcoverage.yml`'s `threshold.total` and per-suite minima, and the GitHub Code Quality PR comments (the other half of [#133](https://github.com/Wave-RF/WaveHouse/issues/133)) all still run; only the published badge surface is gone. The security consequence is the reason to prefer retiring over restoring: **`ci.yml` now declares no `contents: write` in any job**, so the workflow that executes PR-authored code can no longer write to the repository under any path. Also fixed in passing: `timing`'s `needs` still listed `badge` (a dangling `needs` is a workflow-level error once the job is gone), and two permission comments were wrong: the "sole holder of `contents:write`" claims were repo-wide statements only ever true within `ci.yml` (`release.yml` and `publish-npm.yml` hold it too), and the workflow header claimed `docs-preview` was the only job with a write scope, undercounting `coverage`'s `code-quality: write` — which it holds while executing the PR tree. @@ -53,6 +69,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **A row-filter claim containing a backslash, tab or newline no longer withholds rows the query path returns** (`internal/chsql/chsql.go`, `internal/chsql/chsql_test.go`, `internal/query/builder.go`, `internal/typelayer/filter.go`, `internal/typelayer/filter_test.go`, `tests/integration/rowfilter_stream_test.go`): ClickHouse reads a scalar `{p:String}` parameter with its escaped-text reader, so an unescaped `a\b` arrived holding a backspace and a raw tab or newline was a hard parse error. The stream compared such a claim false and the query path could misread it. Both surfaces now encode `\`, tab, newline and carriage return through one `chsql.EscapeStringParam`, measured byte-for-byte against ClickHouse 26.6.3.62 and against the chtypes artifact. The old behaviour was fail-closed — rows withheld, never leaked — so this is availability, not confidentiality. + - **`classify-paths.sh` no longer reads a `grep` failure as "no match"** (`scripts/classify-paths.sh`, `scripts/classify-paths.test.sh`): both decisions were `if printf … | grep -qE …; then A; else B; fi`. `grep` exits `0` on match, `1` on no match and **`2` on error** (can't fork/exec, read error, bad pattern), and the `else` branch collapsed `1` and `2` into the same answer — `set -euo pipefail` does not help, since `set -e` is suppressed for a command used as an `if` condition. Observed twice in local `make ci` runs whose static checks run at `-j 14`: a different single case failed each time (`mixed-docs-go` answering `docs=false`, then `dep-bump-go` answering `code=false`) while every other case passed, which is the signature of a transient `grep` failure rather than a pattern bug. The test caught it only because it asserts expected values; **the production path has no such check** — CI's `changes` job gates the docs pipeline on this answer, so a `docs=false` produced by an errored `grep` silently skips the docs build and still reports success. The two greps now go through a `matches` helper that aborts with a diagnostic on any exit above 1, and the test suite stubs `grep` onto `PATH` to prove the abort fires (that case fails against the previous script). A second instance of the same class, found reviewing the first fix: the helper piped its input into `grep -q`, which exits at the first match — so once the file list outgrew the pipe buffer (a few thousand paths) the upstream `printf` died of SIGPIPE, `pipefail` reported 141, and the new error arm aborted on an ordinary large change set. Reproduced at 5,000 paths. It now reads from a here-string instead, and the suite pins that case. `scripts/ci/classify-changes.sh` also stopped reading the classifier through process substitution, which discarded its exit status: a classifier that aborted left `code`/`docs` empty, every `needs.changes.outputs.code == 'true'` job skipped, and the `CI` aggregator reported green having run nothing. It now captures the status, and fails closed — running everything — on a failed *or* partial classification, matching the rule already used for an empty file list. Also here, unrelated and one line: `biome.json` declared `$schema` 2.4.15 while the lockfile pins the 2.5.8 CLI, so `biome check --error-on-warnings` failed on the config itself for any change touching TypeScript. Bumped to match; it changes no lint rule. - **The role-first split makes two of #541's rule rejections structural, and adds the resolver-side half of a third** (`internal/policy/policy.go`): [#541](https://github.com/Wave-RF/WaveHouse/pull/541) rejects `filter` under an `insert` grant and `check` under a `select` one at validation time. With `select` and `insert` as separate types those fields do not exist on the wrong side at all, so the strict decode refuses them as unknown keys and the runtime checks are gone — the same document is still refused, one layer earlier. #541's operator-less `filter`/`check` rejection is unchanged and keeps its message; what is added here is the matching deny in `evaluateSelect`/`evaluateInsert` — for the operator-less shape and, separately, for a check using an operator the resolver does not honor (`_neq`/`_gt`/`_lt`, or the ambiguous `_eq`+`_in`), which main's `Evaluate` resolved to no clause at all and authorized the insert with the rule silently gone; `Evaluate` does not re-validate the policy it is handed and `policy.Static` is a Validate-free `policy.Source`, so without the operator-less deny a policy reaching the resolvers unvalidated still resolved an operator-less `filter` entry to no predicate and answered `RowVisible` true for every row. `evaluateInsert` also gained the bind-unsafe check-column deny — the insert-side mirror of the `filter`-side guard main already had. @@ -63,6 +81,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Security +- **A policy claim compared against an integer column goes through a strict cast, so a claim that does not fit the column matches nothing instead of wrapping** (`internal/chsql/chsql.go`, `internal/policy/policy.go`, `internal/query/builder.go`, `internal/typelayer/{filter,typelayer}.go`, `docs/src/content/docs/{access-control.mdx,api.md,architecture.md}`, + tests): a claim bound as a plain `{p:String}` wrapped modulo 2^64 on every integer column (and at their own width on `[U]Int128`/`[U]Int256`), identically on `/v1/query`, the stream and the insert check, so a claim of `18446744073709551621` read and wrote tenant `5`. Row filters and insert checks now compare an integer column (`Nullable`/`LowCardinality` included) against `if(toString(accurateCastOrNull({p:String}, 'T')) = {p:String}, accurateCastOrNull({p:String}, 'T'), NULL)`: an in-range canonical claim answers exactly as before and keeps the primary key in use, while an out-of-range or non-canonical one (`007`, `+5`, `1.0`) matches no row on any operator and an insert check refuses it with `403` (a non-canonical claim used to be a per-row code 53 — no rows on a read, `422` on an insert). Other column types and a caller's own filters keep the plain form. + - **Policy validation now rejects the fail-open rule shapes strict decoding can't see** (`internal/policy/policy.go`, `docs/src/content/docs/access-control.mdx`; closes [#460](https://github.com/Wave-RF/WaveHouse/issues/460)): four new `validateRolePerms` rejections close the fail-open shapes strict decoding can't see because the document is syntactically innocent. A `filter` entry with no operator (`"tenant_id": {}`) resolved to zero predicates — no `WHERE` clause, row security silently off, the same shape a misspelled `"eq"` for `"_eq"` used to decode to before strict decoding closed that route; it is now rejected, as is its check-path twin (an operator-less `check` entry, skipped by `Evaluate`'s resolve switch — accepted but constraining nothing) and `filter:` under an `insert:` grant (resolved and then ignored by the ingest path — the same accept-but-ignore family as [#224](https://github.com/Wave-RF/WaveHouse/issues/224), and the pointed asymmetry #460 called out against the loud `check` `_neq`/`_gt`/`_lt` rejection) along with its mirror, `check:` under a `select:` grant — the likelier authoring slip and the fail-open direction: the author believes reads are row-scoped while `Evaluate` resolves the entry and nothing on the select or stream paths reads it. Because [#508](https://github.com/Wave-RF/WaveHouse/pull/508) funneled every adoption through the one `policy.Validate` path, the four checks land on boot, the directory watch, `SIGHUP`, `POST /v1/ops/settings/reload`, and `wavehouse validate` at once. #460's migration caveat (a stored policy hard-failing at boot) has evaporated with the settings directory being new and unreleased; no shipped seed, compose, or fixture policy carries any of the rejected shapes. ## [0.1.0] - 2026-08-19 diff --git a/Makefile b/Makefile index 8dcbe35d..bbc92ea7 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,19 @@ GSA := GOEXPERIMENT=jsonv2 go tool gsa # Externally-installed tools — version is encoded in the path so bumping the # version invalidates the file rule and triggers a reinstall. -GOLANGCI_LINT_VERSION := v2.11.4 +# +# v2.11.4 panics in its type-checker on every package once go.mod says +# `go 1.27` (measured, this migration). v2.13.0 is the OLDEST release whose +# changelog claims go1.27 support ("go1.27 support (#6642)"), but it panics +# too — a DIFFERENT bug: `nilness`/`honnef.co/go/tools@v0.8.0-rc.1` crashes +# analyzing a third-party dependency's source (measured: `internal error: +# unhandled builtin recover`, package "sentry", i.e. getsentry/sentry-go). +# v2.13.1 bumps that dependency past its release candidate to the real +# 0.8.0 and the panic is gone; v2.13.2 (bumping it again, to 0.8.1) is the +# newest confirmed-clean release at time of writing — pinned here rather +# than v2.13.1 since nothing points at 2.13.1 specifically being the fix, +# only that 2.13.0 is broken and 2.13.2 is verified clean in this tree. +GOLANGCI_LINT_VERSION := v2.13.2 GOLANGCI_LINT := $(LOCAL_BIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) # air is the hot-reload runner used by `make dev`. We install it to .bin/ @@ -171,7 +183,7 @@ AIR_VERSION := v1.65.1 AIR := $(LOCAL_BIN)/air-$(AIR_VERSION) # misspell: curated common-typo corrector + US/UK locale enforcer. Installed -# standalone to .bin/ (pure Go, `go install` — same pattern as air) so it can +# standalone to .bin/ via `go install`, same pattern as air, so it can # lint Markdown/MDX prose. DISTINCT from the misspell analyzer bundled inside # golangci-lint, which only inspects Go source; same maintained fork # (github.com/golangci/misspell), two entry points. Drives `make lint-prose`. @@ -903,20 +915,6 @@ release-sdk-go: ## Tag a Go SDK release — go get (VERSION=X.Y.Z) # `verify` or `ci` — `deadcode` has false positives on reflection / HTTP # routers, and the size/dep tools are too slow for a pre-push gate. -# audit-cgo: WaveHouse builds with CGO_ENABLED=0. Listed packages have pure-Go -# fallbacks today, but a new dep could quietly break that constraint — this -# audit surfaces every transitively-reachable package with C files so the drift -# is visible before a release-time cross-compile breaks. -.PHONY: audit-cgo -audit-cgo: ## Audit dependency tree for CGO files (informational) - @echo "$(CYAN)==> Scanning dependency tree for packages with C files...$(RESET)" - @printf " WaveHouse builds with %sCGO_ENABLED=0%s — listed packages have pure-Go fallbacks\n" "$(YELLOW)" "$(RESET)" - @echo " and their C code is never compiled. This audit catches new CGO deps." - @echo - @CGO_ENABLED=1 go list -deps -f '{{if .CgoFiles}} ⚠ {{.ImportPath}} ({{len .CgoFiles}} C files){{end}}' ./cmd/... - @echo - @echo "$(GREEN)==> CGO audit complete$(RESET)" - # deadcode: whole-program reachability analysis, complementary to # golangci-lint's `unused` (which is locally scoped). False positives are # common for HTTP routers, reflection-based dispatch, and init() registration @@ -943,9 +941,9 @@ dep-cut: ## Top cuttable dependencies by transitive weight (LIMIT=N to override) @LIMIT='$(LIMIT)' scripts/dep-cut.sh # binary-analysis: one command for "what's in my binary, and what's wrong with -# it." Runs in dep-order: build → size → audit-cgo → deadcode. +# it." Runs in dep-order: build → size → deadcode. .PHONY: binary-analysis -binary-analysis: size audit-cgo deadcode ## Combined: size + audit-cgo + deadcode +binary-analysis: size deadcode ## Combined: size + deadcode @echo @echo "$(GREEN)==> Binary analysis complete$(RESET)" @printf " Cuttable dependencies: %smake dep-cut%s\n" "$(CYAN)" "$(RESET)" @@ -1037,7 +1035,7 @@ $(GOLANGCI_LINT): @mv $(LOCAL_BIN)/golangci-lint $@ @echo "$(GREEN)==> Installed: $@$(RESET)" -# air installs cleanly via `go install` (pure Go, no shell-piping). GOBIN +# air installs cleanly via `go install` (no shell-piping). GOBIN # pins the install location to our .bin/ rather than the user's $GOPATH/bin. $(AIR): @echo "$(YELLOW)==> Installing air $(AIR_VERSION) for $(OS)_$(ARCH)...$(RESET)" @@ -1046,7 +1044,7 @@ $(AIR): @mv $(LOCAL_BIN)/air $@ @echo "$(GREEN)==> Installed: $@$(RESET)" -# misspell installs cleanly via `go install` (pure Go), GOBIN-pinned to .bin/ +# misspell installs cleanly via `go install`, GOBIN-pinned to .bin/ # like air. cmd/misspell is the CLI entry point of the golangci fork — the same # codebase golangci-lint vendors as a library for its Go-only misspell linter. $(MISSPELL): diff --git a/README.md b/README.md index fce01a0a..574bfa05 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@

The open-source real-time API gateway for ClickHouse: schema-aware ingest, async batching, real-time SSE streaming, and tiered query caching. - All in a single binary. + All in one binary, plus the per-ClickHouse-version artifact it loads at start.

@@ -67,7 +67,7 @@ Full walkthrough at **[wavehouse.dev/getting-started](https://wavehouse.dev/gett ## Why WaveHouse? -ClickHouse is a phenomenal OLAP database, but pointing a frontend right at it leaves a lot to be desired: one-row inserts trigger `Too many parts`, there's no backpressure or edge validation, no real-time push, and no row/column security. You end up building custom APIs, a Kafka queue, a batch consumer, a cache tier, and an auth service. **WaveHouse is that whole stack as one binary** — the only external dependency is ClickHouse. +ClickHouse is a phenomenal OLAP database, but pointing a frontend right at it leaves a lot to be desired: one-row inserts trigger `Too many parts`, there's no backpressure or edge validation, no real-time push, and no row/column security. You end up building custom APIs, a Kafka queue, a batch consumer, a cache tier, and an auth service. **WaveHouse is that whole stack as one binary** (plus a per-ClickHouse-version artifact it loads at start, for ClickHouse-native ingest validation and row-level security) — the only external network dependency is ClickHouse. If you're building user-facing analytics, WaveHouse is like **Supabase for ClickHouse**. Or an **open-source Tinybird** that pushes data to the frontend in real time over SSE, not just pull-based REST. @@ -81,7 +81,7 @@ If you're building user-facing analytics, WaveHouse is like **Supabase for Click | | Direct ClickHouse | Kafka + CH (DIY) | Tinybird | **WaveHouse** | | ----------------------------- | :---------------: | :--------------: | :-----------: | :------------: | -| Self-hosted, single binary | — | — | ✗ (SaaS) | ✓ | +| Self-hosted, one binary | — | — | ✗ (SaaS) | ✓ | | Safe high-rate inserts | ✗ | ✓ (via Kafka) | ✓ | ✓ | | Schema validation at the edge | ✗ | custom | ✓ | ✓ | | Real-time push (SSE) | ✗ | custom service | ✗ | ✓ native | @@ -127,6 +127,14 @@ Swap in `:vX.Y.Z` and `release.yml` for a release image. Pin the signer either w go install github.com/Wave-RF/WaveHouse/cmd/wavehouse@latest ``` +`go install` compiles from source with cgo enabled (requires a C toolchain and glibc — Linux amd64/arm64 or macOS arm64) but does not fetch the [chtypes artifact](https://wavehouse.dev/deployment#chtypes-artifacts) WaveHouse loads at start. Fetch it once before the first run: + +```bash +go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch +``` + +This downloads 160–290 MB into the default local cache (`~/.cache/chtypes/artifacts/abi6/-`, one directory per SDK ABI revision); point `WH_CHTYPES_REGISTRY` elsewhere if you keep it somewhere else. + ```bash wavehouse bootstrap ./settings # starter settings directory, every key at its default WH_SETTINGS_DIR=./settings wavehouse @@ -144,7 +152,7 @@ Track what's shipped, in progress, and planned on the [**project board**](https: ## Local Development -You'll need **Go 1.26+, GNU Make 4+, Docker (Compose v2), Node.js 22 LTS, and pnpm 11.21+**. See [development docs](https://wavehouse.dev/development) for the authoritative source of truth with the full list, version requirements, and gotchas. +You'll need **Go 1.27+, GNU Make 4+, Docker (Compose v2), Node.js 22 LTS, and pnpm 11.21+**. See [development docs](https://wavehouse.dev/development) for the authoritative source of truth with the full list, version requirements, and gotchas. ```bash make tools # one-time bootstrap diff --git a/chtypes.lock b/chtypes.lock new file mode 100644 index 00000000..f65d419f --- /dev/null +++ b/chtypes.lock @@ -0,0 +1,17 @@ +{ + "schema": 1, + "artifacts": { + "darwin-arm64/26.6": { + "file": "chtypes-26.6.8.7-stable-darwin-arm64-b1790767905.tar.gz", + "sha256": "11df5c31e638990c595670d6b92e9e8382a7216fcba433ae0f6fbe5a325ab8f6" + }, + "linux-amd64/26.6": { + "file": "chtypes-26.6.8.7-stable-linux-amd64-b1790767905.tar.gz", + "sha256": "2fcbe729ae110507af22d35a19c290c97c1fae9f036aa2183841331b8ec9770e" + }, + "linux-arm64/26.6": { + "file": "chtypes-26.6.8.7-stable-linux-arm64-b1790767905.tar.gz", + "sha256": "c85effdb5512aac0e012304249b09c86b516f53b401613f251222267453351d3" + } + } +} diff --git a/clients/ts/src/types.ts b/clients/ts/src/types.ts index 4736c12a..c55ef456 100644 --- a/clients/ts/src/types.ts +++ b/clients/ts/src/types.ts @@ -4,7 +4,19 @@ // --- Database type helper --- -/** User-provided database schema mapping table names to row types. */ +/** + * User-provided database schema mapping table names to row types. + * + * A queried value is rendered by ClickHouse itself, so the JSON spelling is + * the server's, not the client's: `Decimal*` and every integer/float arrive + * as numbers, `FixedString`/`UUID`/`Enum*`/`IPv4`/`IPv6` and the date-time + * family as strings, `Array`/`Map` as their JSON equivalents, and `Nullable` + * as the value or `null` — which is what `wavehouse codegen` generates. Two + * consequences worth knowing: `DateTime`/`DateTime64` use ClickHouse's own + * `YYYY-MM-DD HH:MM:SS[.fff]` spelling (the same bytes the stream carries), + * not ISO-8601, and a 64-bit integer past 2^53 arrives as an unquoted JSON + * number, so it loses precision in JavaScript. + */ export type Database = Record>; // --- Result types --- @@ -292,9 +304,9 @@ export type Schemas = Record; // --- Insert result --- /** - * A per-record outcome from a batch (array / NDJSON) insert. Mirrors the - * single-object response shape plus the record's position. Exactly one of - * `ok` / `duplicate` / `error` is set. + * A per-record outcome from a batch (array / NDJSON / CSV / TSV) insert. + * Mirrors the single-object response shape plus the record's position. Exactly + * one of `ok` / `duplicate` / `error` is set. */ export interface InsertRecordResult { /** 1-based index of the record within the submitted batch. */ @@ -305,6 +317,20 @@ export interface InsertRecordResult { duplicate?: boolean; /** Set (with `ok`/`duplicate` absent) when the record was rejected. */ error?: string; + /** + * ClickHouse's own error code, present only when the server's parser is what + * refused the record — 117 unknown field, 27 unparseable value, 6 out of + * range. Absent for a gateway rejection (a failed policy check, a missing + * dedupe id), so `code !== undefined` means "ClickHouse answered". + * + * 117 now also covers **a column the caller's role may not write**. Column + * policy is enforced by compiling the role's own schema without the denied + * columns, so naming one is an unknown field to the parser rather than a + * separate gateway refusal: it is a `400` with this code, where it used to be + * a `403 column "x" not allowed for insert`. The message is ClickHouse's own + * and does not reveal whether the column exists. + */ + code?: number; } export interface InsertResult { diff --git a/cmd/wavehouse/main.go b/cmd/wavehouse/main.go index 20afec50..f74b2781 100644 --- a/cmd/wavehouse/main.go +++ b/cmd/wavehouse/main.go @@ -28,6 +28,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/settings" "github.com/Wave-RF/WaveHouse/internal/stream" + "github.com/Wave-RF/WaveHouse/internal/typelayer" ) // Pre-populated build info variables, set via ldflags by the Makefile and by @@ -321,9 +322,24 @@ func run() int { // only after the first successful Refresh (sync or retry) so it never // races RetryRefresh on Refresh calls or on bootState writes. bootState := api.NewBootState(nil) + + // chtypes engine — opened once at boot, like data_dir. Failure here is + // fatal (explicit registry dir missing/unreadable, or no artifact + // anywhere when unset): unlike schema discovery, there is no useful + // degraded mode without a validation engine. + types, err := typelayer.NewEngine(typelayer.Config{RegistryDir: cfg.ClickHouse.ChtypesRegistry}, logger) + if err != nil { + logger.Error("chtypes engine init", "error", err) + return 1 + } + // Both sources are read per refresh, so a settings reload retunes the // cadence and a ClickHouse reconfigure moves the database without a restart. registry := discovery.NewSchemaRegistry(chConn, chConn.Database, settingsStore.SchemaRefreshInterval, logger) + // Bind (re)compiles per-table handles after every successful Refresh — + // registered before the first Refresh call below so boot itself binds, + // not just later reloads. + registry.OnRefresh(types.Bind) if err := registry.Refresh(ctx); err != nil { logger.Warn("schema discovery failed on boot, retrying in background", "error", err) bootState.Set(fmt.Errorf("schema discovery: %w", err)) @@ -458,6 +474,7 @@ func run() int { // per (topic, role) and pushes it to that role's subscribers. sseMetrics := stream.NewMetrics() streamHub := stream.NewHub(policySource, registry, sseMetrics) + streamHub.RowEvaluator = stream.NewRowEvaluator(types, logger) // Start batch consumer → ClickHouse. ingestCleanup, err := ingest.StartIngestWorker( @@ -494,6 +511,7 @@ func run() int { ingestHandler.PolicySource = policySource ingestHandler.Dedup = dedup ingestHandler.DedupeSettings = settingsStore.DedupeFor + ingestHandler.Types = types dlqHandler := api.NewDLQHandler(js, logger) @@ -569,6 +587,8 @@ func run() int { } }() + // Pipes and StructuredQuery take the HTTP target rather than the native + // driver: they ask ClickHouse for JSON (internal/api/clickhouse_http.go). deps := api.Dependencies{ Ingest: ingestHandler, Query: queryHandler, @@ -577,8 +597,8 @@ func run() int { Version: api.NewVersionHandler(Version, GitCommit, BuildTime), Schema: api.NewSchemaHandler(registry), DLQ: dlqHandler, - Pipes: api.NewPipesHandler(settingsStore, policySource, chConn, cache, chConn.QueryTimeout, logger), - StructuredQuery: api.NewStructuredQueryHandler(chConn, cache, registry, policySource, settingsStore.TimestampBucketSeconds, chConn.QueryTimeout, settingsStore.DefaultMaxRows, logger), + Pipes: api.NewPipesHandler(settingsStore, policySource, chConn.Target, cache, chConn.QueryTimeout, logger), + StructuredQuery: api.NewStructuredQueryHandler(chConn.Target, cache, registry, policySource, settingsStore.TimestampBucketSeconds, chConn.QueryTimeout, settingsStore.DefaultMaxRows, logger), AuthMW: authMW, PolicySource: policySource, diff --git a/config.yaml b/config.yaml index f1300837..8b2f2757 100644 --- a/config.yaml +++ b/config.yaml @@ -32,10 +32,16 @@ prometheus: path: /metrics port: 0 # 0 = mount on server.port; non-zero = sidecar listener -# Only the password is boot config; addr, http_port, http_scheme, database, -# username, query_timeout are the settings directory's clickhouse block. +# Only the password and chtypes_registry are boot config; addr, http_port, +# http_scheme, database, username, query_timeout are the settings +# directory's clickhouse block. clickhouse: password: "" + # Explicit chtypes artifact registry directory. Empty (default) defers to + # the SDK's own search path ($CHTYPES_REGISTRY, the per-user cache + # ~/.cache/chtypes/artifacts/abi6/-, system dirs) — set this + # only to point at a non-default location. + chtypes_registry: "" # In-process L1 cache size. The query time-bucket # (query.timestamp_bucket_seconds) is a settings key. diff --git a/deployments/Dockerfile b/deployments/Dockerfile index bb74ecd7..3cfff440 100644 --- a/deployments/Dockerfile +++ b/deployments/Dockerfile @@ -1,15 +1,20 @@ # syntax=docker/dockerfile:1 # Stage 1: Build -FROM golang:1.26-alpine AS builder +# +# bookworm (glibc), not alpine (musl): the chtypes SDK's dlopen path needs +# cgo (dlfcn), and the image the SDK builds its own artifacts against — plus +# the artifact's own glibc floor (2.17, 2.29 for the 24.8/25.3 lines) — is +# glibc, not musl. The non-alpine golang image bundles gcc out of the box, +# unlike alpine (which needs an explicit apk add build-base for cgo). +FROM golang:1.27-bookworm AS builder WORKDIR /src # Pass build tags as an argument (defaults to empty) ARG BUILD_TAGS="" -# Download dependencies first into the module cache. The `golang:*-alpine` -# image sets GOPATH=/go, so `go mod download` writes to /go/pkg/mod — that's -# the cache mount target. Splitting this into its own layer means dependency -# changes (go.mod / go.sum) don't bust the source-code COPY layer's cache. +# Download dependencies first into the module cache. Splitting this into its +# own layer means dependency changes (go.mod / go.sum) don't bust the +# source-code COPY layer's cache. COPY go.mod go.sum ./ RUN --mount=type=cache,target=/go/pkg/mod go mod download @@ -20,9 +25,25 @@ COPY . . # tree (read by `go build` for imports) and /root/.cache/go-build for the # compiled-package cache (Go's content-addressed build cache). Without both, # every `docker build` is a cold compile. +# +# CGO_ENABLED=1 is unconditional (see the bookworm comment above). +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=1 go build -tags="${BUILD_TAGS}" -ldflags="-s -w" -o /bin/wavehouse ./cmd/wavehouse + +# Bake the pinned chtypes artifact(s) into the image so the container has no +# runtime network dependency and no first-request download stall. chtypes.lock +# is this repo's pin (schema-1, exact file + sha256 per platform/line); +# scripts/fetch-chtypes.sh wraps the SDK's own CLI with --frozen, which +# refuses anything the lock doesn't name. `go run @` +# resolves and builds the SDK's CLI straight from its module proxy — it +# needs no entry in THIS repo's go.mod/go.sum, so this works even before +# `go mod tidy` has picked up the chtypes/go dependency. +COPY chtypes.lock ./ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -tags="${BUILD_TAGS}" -ldflags="-s -w" -o /bin/wavehouse ./cmd/wavehouse + mkdir -p /opt/chtypes/artifacts && \ + scripts/fetch-chtypes.sh --dest /opt/chtypes/artifacts # Create the parent state and settings directories owned by the nonroot # user (UID 65532 in distroless). The binary creates `nats/` and `pebble/` @@ -42,13 +63,24 @@ RUN --mount=type=cache,target=/go/pkg/mod \ RUN mkdir -p /app/data /app/settings && chown -R 65532:65532 /app # Stage 2: Final minimal image -FROM gcr.io/distroless/static-debian12 +# +# distroless/cc, not distroless/static: chtypes' dlopen path needs glibc's +# dynamic loader plus libstdc++ (the artifact is a C++-built .so) — the +# "cc" variant bundles exactly that (glibc, libgcc1, libstdc++6) and nothing +# else, still with no shell or package manager. +FROM gcr.io/distroless/cc-debian12 WORKDIR /app USER nonroot:nonroot ENV WH_SETTINGS_DIR=/app/settings +# The SDK's own registry search path (R3 §7) reads this directly — no +# WaveHouse config change is needed to find what the image already baked +# in. WH_CHTYPES_REGISTRY (internal/config) is for an operator who wants to +# point at a different, bind-mounted registry directory instead. +ENV CHTYPES_REGISTRY=/opt/chtypes/artifacts COPY --from=builder --chown=nonroot:nonroot /app /app +COPY --from=builder --chown=nonroot:nonroot /opt/chtypes /opt/chtypes COPY --from=builder --chown=nonroot:nonroot /bin/wavehouse /app/wavehouse # OCI image metadata. Static labels live here; per-build dynamic ones diff --git a/deployments/Dockerfile.goreleaser b/deployments/Dockerfile.goreleaser index 8d22201c..71113150 100644 --- a/deployments/Dockerfile.goreleaser +++ b/deployments/Dockerfile.goreleaser @@ -1,8 +1,19 @@ +# The prebuilt-binary image. Despite the name, GoReleaser no longer drives +# this file: cgo made cross-compiling impossible for darwin, so the release +# pipeline builds each binary on its own native runner and +# .github/workflows/release.yml (and publish-dev.yml, and the no-push +# validation in goreleaser-validate.yml) invokes `docker buildx build` on it +# directly. The build CONTRACT is unchanged and those workflows reproduce it +# exactly — a context holding `//wavehouse` per target plus +# chtypes.lock and scripts/{fetch-chtypes,_colors}.sh — which is why this file +# needed no edit beyond these comments. The name is kept so published image +# history, `docker history` output and every doc reference stay valid. +# # Tiny stage just to mkdir + chown the state directories. Distroless has no # shell, so the runtime image can't run RUN commands itself; we materialise -# the layout here and COPY it into the final image. Keeps the goreleaser -# variant in lockstep with deployments/Dockerfile (source-build) so the two -# images expose the same /app filesystem layout to operators. +# the layout here and COPY it into the final image. Keeps this variant in +# lockstep with deployments/Dockerfile (source-build) so the two images expose +# the same /app filesystem layout to operators. # # Only the parents are pre-created — the binary mkdirs `nats/` and `pebble/` # subdirs itself when NATS/Pebble open their stores. Named-volume copy-up @@ -21,28 +32,62 @@ FROM --platform=$BUILDPLATFORM alpine:3 AS layout # RUN in deployments/Dockerfile; keep the two in lockstep. RUN mkdir -p /app/data /app/settings && chown -R 65532:65532 /app -FROM gcr.io/distroless/static-debian12 +# Fetches the pinned chtypes artifact for THIS image's target platform (not +# necessarily the builder's — buildx may emulate or cross-fetch). Built +# natively on $BUILDPLATFORM like `layout` above: no compiled Go code +# crosses an arch boundary here, just an HTTPS download + sha256 check via +# the SDK CLI's own --platform flag, so QEMU emulation would buy nothing. +# Needs chtypes.lock + scripts/fetch-chtypes.sh + scripts/_colors.sh at their +# normal repo-relative paths. The build context is assembled by the workflow +# rather than being the repo root, so those three are copied into it +# explicitly — see the "Assemble the image build context" step in +# .github/workflows/release.yml, publish-dev.yml and goreleaser-validate.yml. +# +# This stage is also the pipeline's chtypes.lock check: `--frozen` refuses any +# artifact the lock does not name, so an upstream republish of a pinned line +# fails the build here rather than silently baking a different library. +FROM --platform=$BUILDPLATFORM golang:1.27-bookworm AS chtypes-fetch +ARG TARGETPLATFORM +WORKDIR /src +COPY chtypes.lock ./ +COPY scripts/fetch-chtypes.sh scripts/_colors.sh ./scripts/ +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + plat="$(echo "$TARGETPLATFORM" | tr / -)"; \ + mkdir -p /opt/chtypes/artifacts && \ + scripts/fetch-chtypes.sh --platform "$plat" --dest /opt/chtypes/artifacts + +FROM gcr.io/distroless/cc-debian12 -# GoReleaser automatically injects this based on the platform it's building for +# buildx injects this per target platform of a `--platform a,b` build ARG TARGETPLATFORM WORKDIR /app USER nonroot:nonroot ENV WH_SETTINGS_DIR=/app/settings +# The SDK's own registry search path (R3 §7) reads this directly — no +# WaveHouse config change is needed to find what the image already baked +# in. WH_CHTYPES_REGISTRY (internal/config) is for an operator who wants to +# point at a different, bind-mounted registry directory instead. +ENV CHTYPES_REGISTRY=/opt/chtypes/artifacts # Pre-created state directories owned by the nonroot user (UID 65532) so # the binary can mkdir under /app/data without a volume mount, and so # bind-mounting empty volumes at /app/data inherits correct ownership. COPY --from=layout --chown=nonroot:nonroot /app /app +COPY --from=chtypes-fetch --chown=nonroot:nonroot /opt/chtypes /opt/chtypes -# Copy the binaries from the platform-specific subdirectories GoReleaser creates +# Copy the binary from its platform-specific subdirectory of the context — +# `linux/amd64/wavehouse`, `linux/arm64/wavehouse`. The workflow stages each +# native runner's binary there with mode 0755 (actions/upload-artifact's zip +# carries no unix mode bits, so the executable bit has to be restored). COPY --chown=nonroot:nonroot $TARGETPLATFORM/wavehouse /app/wavehouse -# OCI image metadata. Static labels here; goreleaser injects per-build -# ones (revision, version, created, image.title with the version) via -# `dockers_v2.labels` in `.goreleaser.yaml`, which take precedence on -# release builds. The static set below is the floor — guaranteed even -# if a label is missing from the goreleaser config. +# OCI image metadata. Static labels here; the publishing workflows pass the +# per-build ones (revision, version, created, and a dev-suffixed title and +# description on the :dev channel) as `docker buildx build --label`, which +# take precedence. The static set below is the floor — guaranteed even when a +# label is missing from the command line, e.g. on a local `docker build`. LABEL org.opencontainers.image.title="WaveHouse" LABEL org.opencontainers.image.description="Schema-aware real-time API gateway for ClickHouse" LABEL org.opencontainers.image.url="https://github.com/Wave-RF/WaveHouse" diff --git a/deployments/compose/standalone.yaml b/deployments/compose/standalone.yaml index cdba543e..63a3e991 100644 --- a/deployments/compose/standalone.yaml +++ b/deployments/compose/standalone.yaml @@ -21,6 +21,12 @@ services: # are convention, not config. The Dockerfile pre-creates /app/data # owned by the nonroot user; bind-mount any persistent volume here. WH_DATA_DIR: /app/data + # deployments/Dockerfile already bakes the pinned chtypes artifact + # (chtypes.lock) into the image at /opt/chtypes/artifacts and sets + # CHTYPES_REGISTRY there — no override needed for the quickstart. + # Point it elsewhere only if you bind-mount a different/updated + # registry directory instead of rebuilding the image: + # CHTYPES_REGISTRY: /opt/chtypes/artifacts # ClickHouse wiring (addr, ports, database, user) is in ./settings/config.json # — the settings directory below — where it hot-reloads. Only the # password is env (a secret); the bundled ClickHouse has none. diff --git a/docs/src/content/docs/access-control.mdx b/docs/src/content/docs/access-control.mdx index 1962a11f..d06e79fb 100644 --- a/docs/src/content/docs/access-control.mdx +++ b/docs/src/content/docs/access-control.mdx @@ -190,11 +190,11 @@ The rules, in order: 2. **An empty (or `["*"]`) `allow_columns` means "all columns"** — every column not in `deny_columns` is permitted. Use this with `deny_columns` for a blocklist posture: see everything *except* a few sensitive columns. 3. **A non-empty `allow_columns` is an allowlist** — only the named columns (and never the denied ones) are permitted. -On a structured query (`POST /v1/query?table={table}`) the allowlist is a **hard cap on every column the query references — in any clause**: the projection, an aggregation argument, `filters`, `group_by`, `order_by`, and `time_range`. Naming a disallowed column anywhere is rejected with `403 column "x" not allowed`. A full-row read is requested explicitly with `"select_all": true`, which expands to exactly the columns the role may read — never a raw `SELECT *` that could include a denied column; if the role is allowed *no* columns, the read is rejected (`403`) rather than returning empty rows. **Omitting `columns` (or sending `[]` / `""`) returns nothing** — a request for no data — so a hidden column can't leak by being left out, grouped on, or filtered on to infer its values. (Note: in a query, `["*"]` is the *literal column named `*`*, not a wildcard — use `select_all` for all columns. In `allow_columns`, `["*"]` is still the all-columns wildcard.) On insert (`POST /v1/ingest?table={table}`) the body is `403 column "x" not allowed for insert`. On live streams, denied columns are silently **stripped** from each event rather than rejecting the connection. The structured-query and live-stream paths defer to the **same** per-column decision (`IsColumnAllowed`), so the two read surfaces enforce identical column visibility and can't drift apart. +On a structured query (`POST /v1/query?table={table}`) the allowlist is a **hard cap on every column the query references — in any clause**: the projection, an aggregation argument, `filters`, `group_by`, `order_by`, and `time_range`. Naming a disallowed column anywhere is rejected with `403 column "x" not allowed`. A full-row read is requested explicitly with `"select_all": true`, which expands to exactly the columns the role may read — never a raw `SELECT *` that could include a denied column; if the role is allowed *no* columns, the read is rejected (`403`) rather than returning empty rows. **Omitting `columns` (or sending `[]` / `""`) returns nothing** — a request for no data — so a hidden column can't leak by being left out, grouped on, or filtered on to infer its values. (Note: in a query, `["*"]` is the *literal column named `*`*, not a wildcard — use `select_all` for all columns. In `allow_columns`, `["*"]` is still the all-columns wildcard.) On insert (`POST /v1/ingest?table={table}`) the rule is enforced by compiling the role its *own* copy of the table schema, without the columns it may not write — so a record naming one is refused by ClickHouse's parser with code **117**, `Unknown field found while parsing JSONEachRow format: x`, the same answer a column the table does not have gets. The message no longer confirms whether the column exists. On live streams, denied columns are silently **stripped** from each event rather than rejecting the connection. The structured-query and live-stream paths defer to the **same** per-column decision (`IsColumnAllowed`), so the two read surfaces enforce identical column visibility and can't drift apart. ## Row-level security -`filter` restricts *which rows* a role can read. Like the per-column decision above, one resolution drives **both** read surfaces: a structured query gets the predicates injected as a `WHERE` clause in the generated SQL, and the live stream evaluates the *same* resolved predicates in memory against each subscriber's claims before delivering an event — so row visibility can't drift between the two paths (the stream's in-memory comparison has a fail-closed boundary; see [the enforcement caution below](#where-each-rule-is-enforced)). Each entry maps a column to a comparison whose value is usually a **JWT claim template**: +`filter` restricts *which rows* a role can read. One resolution drives **both** read surfaces: the structured query injects the predicates as a `WHERE` clause, and the live stream compiles the *same* resolved predicates through chtypes and evaluates them per subscriber before delivering an event — so row visibility cannot drift between the two paths (fail-closed boundary; see [the enforcement caution below](#where-each-rule-is-enforced)). Each entry maps a column to a comparison whose value is usually a **JWT claim template**: ```json { @@ -235,13 +235,25 @@ Any `filter` or `check` operator value may interpolate token claims with `{{ jwt - `{{ jwt.sub }}` → the token's `sub` claim. - `{{ jwt.app_metadata.tenant_id }}` → a nested claim. -Values are always bound as SQL **parameters**, never concatenated into the query, so templating is injection-safe. If a claim path in a `filter` template can't be resolved (a validly-signed token that simply doesn't carry the claim), that filter **fails closed**: the predicate becomes constant-false, so on the structured-query path (`POST /v1/query`) the role sees **no rows**, and the live stream withholds every event for that subscriber — one resolution drives both read surfaces (see [where each rule is enforced](#where-each-rule-is-enforced)). This holds for every operator — `_eq`, `_neq`, `_gt`, `_lt`, and `_in` alike. The alternative, binding the empty string the template would render to, would leave a live predicate against `''`: `_eq` would match every empty-valued row, and `_neq`/`_gt` on a string column would match essentially *all* rows, erasing the restriction. A literal value with no template in it — including an explicit `""` — binds exactly as written, even when it spells a number ([insert checks](#insert-checks) additionally accept such a literal's numeric reading at compare time, since a policy literal carries no JSON type — but what *binds* is always the spelling you wrote; whether it then matches follows the column's type: a `String` or byte-equality column compares that exact text, while a numeric column reads the constant as a number on both surfaces — on a `Float` or `Decimal` column `_eq: "1.0"` admits a stored or streamed `1`, but an integer column accepts only the plain digit form: `1.0` and `1e3` alike are refused, matching the cast error the query path raises for them (see [the enforcement caution](#where-each-rule-is-enforced))). +Values are always bound as SQL **parameters**, never concatenated into the query, so templating is injection-safe. If a claim path in a `filter` template can't be resolved (a validly-signed token that simply doesn't carry the claim), that filter **fails closed**: the predicate becomes constant-false, so on the structured-query path (`POST /v1/query`) the role sees **no rows**, and the live stream withholds every event for that subscriber — one resolution drives both read surfaces (see [where each rule is enforced](#where-each-rule-is-enforced)). This holds for every operator — `_eq`, `_neq`, `_gt`, `_lt`, and `_in` alike. The alternative, binding the empty string the template would render to, would leave a live predicate against `''`: `_eq` would match every empty-valued row, and `_neq`/`_gt` on a string column would match essentially *all* rows, erasing the restriction. A literal value with no template in it — including an explicit `""` — binds exactly as written, and ClickHouse reads it under the column's type: a `String` column compares that exact text, while a numeric column reads the constant as a number. A spelling the column cannot read is not a second chance — `_eq: "1.0"` against a `UInt64` matches no rows on a read filter and is refused with `403` on an insert (an integer column takes only the canonical spelling; see the caution below), and on another numeric column such as `Decimal` it is ClickHouse's own code 53 `TYPE_MISMATCH` per row, which withholds on a read filter and is a `422` on an insert. Write a literal the column can read. -"Can't be resolved" means the claim path is **absent** from the token (or `null`) — or resolves to a JSON **object or array** rather than a scalar, which usually means a dropped path segment (`{{ jwt.app_metadata }}` where `{{ jwt.app_metadata.tenant_id }}` was meant); the one structured shape with defined semantics is the bare-claim `_in` array above. Scalar claims — strings, booleans, and numbers — resolve normally. A numeric claim binds in **canonical decimal form**, not the token's spelling: an integer id keeps every digit — up to a 100-digit bound, far past any real id — while `1.0` or `1e3` binds as `1` and `1000`, so spelling differences between issuers never change the bound value. What must fit the bound is the value's **exact decimal form**, exponent applied — roughly 100 digits (the exact-form gate allows 102 characters, whatever they are) — so `1e400` can't be resolved, but neither can `1e150` or `1e-150`, whose short spellings expand to 151- and 152-character exact forms even though a float64 could hold them; prefer issuing integer ids as integers. Type the scoped column to match: an integer column — or a `Decimal` whose scale covers the claim's fractional digits — keeps the comparison exact, while a `Float32`/`Float64` column rounds the stored value and quietly gives that exactness back (`col = '9007199254740993'` matches a stored `9007199254740992` there). A claim that is *present but empty* is a value the token vouches for: it resolves to `''` and binds normally, so `_neq` against an empty-string claim still emits `col != ''`. Make sure your identity provider actually issues the claims your policy templates reference — and omits unset claims rather than issuing them as empty strings. +"Can't be resolved" means the claim path is **absent** from the token (or `null`) — or resolves to a JSON **object or array** rather than a scalar, which usually means a dropped path segment (`{{ jwt.app_metadata }}` where `{{ jwt.app_metadata.tenant_id }}` was meant); the one structured shape with defined semantics is the bare-claim `_in` array above. Scalar claims — strings, booleans, and numbers — resolve normally. A numeric claim binds in **canonical decimal form**, not the token's spelling: `1.0` and `1e3` bind as `1` and `1000`, so spelling differences between issuers never change the bound value, and an integer id keeps every digit up to a ~100-digit bound on the value's *exact* decimal form (so `1e150` is refused even though a float64 holds it — prefer issuing integer ids as integers). Type the scoped column to match: an integer column, or a `Decimal` whose scale covers the claim's fractional digits, keeps the comparison exact for ids that fit the column's type, while a `Float32`/`Float64` column rounds the stored value and gives that exactness back (`col = '9007199254740993'` matches a stored `9007199254740992` there). A claim that is *present but empty* is a value the token vouches for: it resolves to `''` and binds normally. Make sure your identity provider issues the claims your policy templates reference — and omits unset claims rather than issuing them as empty strings. + +:::caution[Choosing columns to compare claims against] +Every claim is bound as a string parameter and read under its column's type — on `/v1/query` and on the stream alike, and for an auto-injected `_eq` value too — so the column's type decides how the claim is read. + +Compare claims against `String` or `UUID` columns (tenant ids, org ids, user ids): those compare the claim exactly as written. + +On an integer column (`UInt8` through `UInt256`, `Int8` through `Int256`, `Nullable` included) the claim must still fit the column's type, and is compared through a strict cast: a claim that is not the canonical spelling of a value the column can hold — out of range such as `18446744073709551616` (2^64), or spelled `007` or `+5` — matches no rows on any operator, and an insert `check` refuses the record with `403`. It never wraps onto another value. + +On a timestamp column (`Date`, `DateTime`, `DateTime64`) the claim is parsed the way ClickHouse parses a string compared with that type. A claim without a UTC offset is read in the server's time zone, and one carrying an offset such as `2026-01-01T00:00:00Z` is accepted only on ClickHouse 26.5 and later — earlier lines refuse the comparison. + +Prefer identity columns over time columns, and express time windows in the query instead. +::: Claim paths may contain only letters, digits, `_`, and `.` (the segment separator). Any `{{ … }}` fragment that is not a well-formed `{{ jwt. }}` template — a path outside that grammar (a hyphen: `{{ jwt.tenant-id }}`; a namespaced claim: `{{ jwt.https://app.example.com/tenant_id }}`), a missing or misspelled `jwt.` prefix, or an unterminated `{{` — is **not** recognized as a template, and left unchecked the resolver would bind the literal `{{ … }}` text as a value. (Policy values have no other placeholder syntax; a pipe's `{{param}}` placeholders are a different mechanism and are not accepted here.) That is *not* fail-closed: on a read filter `_neq`/`_lt` would then match essentially every row (a leak), and on a write `check` the literal text would be stamped into every inserted row (silent corruption). So such a policy is **rejected when it is validated**: a `policies.json` carrying one refuses boot, is refused by a reload (the previous good policy stays in effect), and fails `wavehouse validate`. Every adoption runs the current rules, so an upgrade that tightens validation re-checks the file on the next boot or reload. Flatten hyphenated or namespaced claims into a supported path at your identity provider — on Auth0, an [Action can set a flat custom claim](https://auth0.com/docs/secure/tokens/json-web-tokens/create-custom-claims) outside the registered OIDC names (its legacy Rules required URL namespacing, which established tenants often still carry), and namespacing is a common OIDC convention elsewhere, so a namespaced claim is the shape you are most likely to meet first. -Row filters apply on the structured-query path and, per subscriber, on the live SSE stream — the stream evaluates the same resolved predicates in memory against each subscriber's token claims (see the [enforcement caution](#where-each-rule-is-enforced) for what its in-memory comparison can and cannot decide). Named pipes authorize by `allowed_roles` membership alone — scope a pipe's exposure in its SQL text, since neither the table policy's row `filter` nor its column allow/deny list is applied on the pipe path (see [Named Pipes](/pipes#authorizing-a-pipe)). +Row filters apply on the structured-query path and, per subscriber, on the live SSE stream — the stream compiles the same resolved predicates through the in-process ClickHouse parser and evaluates them against each subscriber's token claims (see the [enforcement caution](#where-each-rule-is-enforced) for the fail-closed reasons a compile or evaluation can withhold a row). Named pipes authorize by `allowed_roles` membership alone — scope a pipe's exposure in its SQL text, since neither the table policy's row `filter` nor its column allow/deny list is applied on the pipe path (see [Named Pipes](/pipes#authorizing-a-pipe)). ## Insert checks @@ -265,13 +277,14 @@ Row filters apply on the structured-query path and, per subscriber, on the live } ``` -For each checked column, on `POST /v1/ingest?table={table}`: +Checks are compiled into **one chtypes filter** — `col = {p:String}` for `_eq`, `col IN (…)` for `_in`, AND-joined over every checked column, with an integer column's claim compared through the strict cast described under [JWT claim templating](#jwt-claim-templating) — and evaluated against the row ClickHouse's own parser produced, by the same engine that evaluates a row `filter`. So a check sees the **stored** value, after coercion and after `DEFAULT`s: what it admits is what the table will hold. -- **If the request body includes the column**, its value must satisfy the check — equal the claim-derived value (`_eq`), or be one of the claim-derived set (`_in`) — or the insert is rejected with `403 check failed for column "x"`. Equality is decided on **canonical scalar form**, the same rule the claim side follows: a numeric body value matches by value, not spelling (`1.0` satisfies a check against claim `1`); a body value with no canonical form — an object, array, or `null` — satisfies no check, so a `check` on a `Map` or `Array` column rejects every insert that names it; and the 100-digit bound applies to the body's literal too, so an over-long numeric value is this same `403`, not a schema error. A **template-free** `_eq` check value carries no JSON type, so it matches by either reading — a static `_eq: "1.0"` accepts an inserted `1.0` (number) and an inserted `"1.0"` (string) alike, while auto-injecting exactly as written. A claim-derived value keeps strict canonical equality: a string-typed claim matches only its exact text, so a claim of `"1e3"` never accepts an inserted `1000`. On an integer or `Decimal` column a `writer` therefore cannot forge a row for another user or tenant — equal canonical forms store equal values. On a `Float32`/`Float64` column the rounding [noted above](#jwt-claim-templating) applies to the write side too: the stored value can land on a neighboring id. And on a `String` column the guarantee is narrower in a different way: the check compares canonical forms but ClickHouse stores the body's raw spelling, so a body value that is an alternate numeric spelling of the claim (`1e3` for a claim of `1000`) passes the check yet stores differently than the auto-injected value would — and a literal's numeric reading shares this property (`_eq: "1.0"` accepting an inserted `1` stores the text `1`, not `1.0`). -- **If the request body omits the column**, an `_eq` check **auto-injects** the claim-derived value before publishing, so clients can send just the business fields (`page`, `score`) and let the policy stamp `user_id` and `tenant_id` from the token. An `_in` check has no single value to stamp, so an omitted column is **rejected** — the writer must name a value within their allowed set. -- **The check's column must be one a record can actually carry.** A `check` naming a column the table does not have, one ClickHouse computes (`MATERIALIZED`/`ALIAS`), or an `EPHEMERAL` one is refused with a per-record `403` naming the column — on *every* insert by that role, until the policy or the table is corrected. None of the three can be enforced: the published row has one slot per insertable column, so an injected value for a computed or unknown column is dropped on the way out, and an ephemeral column is accepted by the `INSERT` but never stored and never selectable. Each would have answered `200` while enforcing nothing, which is worse than refusing. `wavehouse validate` cannot catch this — it never sees the ClickHouse schema — so **audit your `check` blocks against their tables before upgrading**. (The `EPHEMERAL` case is refused even when the body *supplies* the value: it passes schema validation, since an ephemeral column is legal in an `INSERT`.) +- **If the request body includes the column**, its value must satisfy the check or the record is rejected with `403 check failed for column "x"` (`… for columns "x", "y"` when more than one is checked — the filter is AND-joined, so it names the set tested rather than inventing an attribution). Comparison is ClickHouse's, under the column's own type: on an integer, `Decimal`, `UUID` or `String` column a writer cannot forge a row for another tenant, because equal values store equal. A `Float32`/`Float64` column rounds the stored value and gives that exactness back — the row can land on a neighboring id. A check the engine cannot evaluate at all is a `422`, never a silent pass. +- **If the request body omits the column** — or sends an explicit `null`, which `input_format_null_as_default` resolves the same way — an `_eq` check **auto-injects** the claim-derived value, so clients can send just the business fields and let the policy stamp `user_id` and `tenant_id` from the token. It is implemented as a `DEFAULT` on the role's compiled schema, so a value the caller *does* send still wins. An `_in` check has no single value to stamp, so the **table's own default** is what gets tested: the record is admitted if that default is in the claim-derived set and rejected if it is not. +- **The check's column must be one a record can actually carry.** A `check` naming a column the table does not have, one ClickHouse computes (`MATERIALIZED`/`ALIAS`), or an `EPHEMERAL` one is refused with a per-record `403` naming the column — on *every* insert by that role, until the policy or the table is corrected. None of the three can be enforced: the published row has one slot per wire column, so an injected value for a computed or unknown column is dropped on the way out, and an ephemeral column is never stored. Each would have answered `200` while enforcing nothing. `wavehouse validate` cannot catch this — it never sees the ClickHouse schema — so **audit your `check` blocks against their tables before upgrading**. +- **A claim literal the column cannot read fails closed, not loosely.** A `_eq` value that is not a legal literal for the column (`"1.0"` on a `UInt64`) cannot be compiled as that column's `DEFAULT`, so the injection is dropped and logged; the filter then judges the record as sent, which an absent column loses. On an integer column every record then fails the check — a `403`; on another column a supplied value is ClickHouse's code 53 `TYPE_MISMATCH` per row — a `422`. -**When the claim can't be resolved** (a validly-signed token that doesn't carry it), an `_eq` check is — unlike a row filter — **not** fail-closed: the template still renders, the unresolvable placeholder replaced by the empty string and any surrounding literal text kept (`"acct-{{ jwt.org_id }}"` → `acct-`, a bare `"{{ jwt.sub }}"` → `''`), and that rendered value becomes the required value — so an omitted column is auto-injected with it and any other supplied value is rejected. On an integer or `Decimal` column that stamped `''` does not stay empty: ClickHouse coerces it to `0`, so every such row lands on tenant/user `0` — and since the read side of the same claim fails closed, those rows are also invisible to the writer that produced them. (A `Float` column may instead reject the coercion — ClickHouse-version-dependent — failing the insert into the DLQ.) An unresolvable `_in` check instead fails closed: the claim-derived set is empty, so every insert by that role into that table is rejected (there is no single value to auto-inject). The `_in` element rule from [row-level security](#row-level-security) applies here too — one non-scalar element (`null`, object, nested array) in the claim array collapses the allowed set the same way. Whether the `_eq` write path should match the read path and reject the insert rather than stamp `''` is tracked in [#463](https://github.com/Wave-RF/WaveHouse/issues/463). +**When the claim can't be resolved** (a validly-signed token that doesn't carry it), an `_eq` check is — unlike a row filter — **not** fail-closed on a `String` column: the template still renders, the unresolvable placeholder replaced by the empty string and any surrounding literal text kept (`"acct-{{ jwt.org_id }}"` → `acct-`, a bare `"{{ jwt.sub }}"` → `''`), and that rendered value becomes the required value — so an omitted column is auto-injected with it and any other supplied value is rejected. On a **numeric** column it now fails closed: `''` is not a literal a `UInt64` can read, so the role's schema will not compile with it as a default and no record satisfies the check — every insert by that role into that table is refused (a `403` on an integer column, a `422` on a `Decimal` or `Float` one), rather than the rows landing on tenant/user `0` as they used to ([#463](https://github.com/Wave-RF/WaveHouse/issues/463)). An unresolvable `_in` check fails closed everywhere: the claim-derived set is empty, so nothing satisfies it. The `_in` element rule from [row-level security](#row-level-security) applies here too — one non-scalar element (`null`, object, nested array) in the claim array collapses the allowed set the same way. This pairs naturally with a matching `filter` on the `select` side: `check` stamps the tenant on write, `filter` scopes reads to that tenant. @@ -346,23 +359,13 @@ The same policy drives every data path, but not every field is meaningful on eve | Named pipe | `GET/POST /v1/pipes/{name}` | per-pipe `allowed_roles` (not the policy engine; see [Named Pipes](/pipes)). Resource limits come from ClickHouse's [server-wide settings](/configuration#server-side-resource-limits), not per-role policy caps | :::caution[Live streams enforce column and row policy, but not resource limits] -SSE subscribers are checked for table-level `select` permission, have denied columns stripped from each event, and — like the query path — receive only the rows their role's row-level `filter` predicates admit, evaluated per subscriber against their JWT claims. Predicates are evaluated against the **full ingested event**, so a filter may key on a column the role can't `select` (denied columns are still stripped from what's delivered). The stream evaluates predicates in memory — reproducing ClickHouse's coercion for the types it can classify and refusing the rest — so how much it can enforce depends on what the schema says about the filtered column, and every case it cannot decide **fails closed**. Provided the filter constant is one the query path's SQL also accepts (see the per-type guidance below — the *every other type* bucket constrains your constant to the event's own text rendering, and timestamp constants should be zone-less or Unix-seconds strings — the two spellings every ClickHouse release converts in a `WHERE` comparison), ambiguity only ever *withholds* a row the query path would return, never delivers one it would hide. The one residual payload-vs-stored case — an event whose insert later fails outright into the DLQ — is called out below. - -- **Numeric columns** (`Int*`/`UInt*`/`Float*`/`Decimal*`, unwrapping `Nullable`/`LowCardinality` in any nesting): all five operators compare numerically in the column's **storage domain**, matching ClickHouse. Both operands render to the claim side's exact [canonical decimal form](#jwt-claim-templating) — so a 64-bit ID never falsely matches a neighbor, string-encoded or bare — and are then narrowed the way ClickHouse narrows the stored value and the bound constant: `Float32`/`Float64` round to the column's width, `Decimal` truncates at its scale, and integer columns are exact at any width. An operand outside the JSON number grammar (`NaN`, any `Inf`/`Infinity` spelling, hex), past roughly the [100-digit bound](#jwt-claim-templating), or beyond the float domain's range withholds the row — as does, on an integer column, a fractional operand or a constant spelled any way but the plain form ClickHouse's integer cast accepts (`1e3` errors the query there, so the stream withholds to match). Operands outside the column's **numeric range** (a negative bound on an unsigned column, a value past the integer width, more integer digits than a Decimal's precision budget) are refused on both sides too: ClickHouse's own reading of such a constant varies by pair — an error, a mathematical promotion, or a width-boundary wrap onto a *different* value than written — so the stream withholds rather than model any one behavior, and an out-of-range payload was never storable anyway. Write bounds within the column's range. -- **`String` columns** (again under any `Nullable`/`LowCardinality` wrapping): byte comparison *is* ClickHouse's String comparison — equality and ordering are both exact. -- **`DateTime`/`DateTime64` columns** (again under any wrapping): both operands are parsed as **instants** — through the same grammar [ingest canonicalization](/api#timestamp-canonicalization) reads — and compared chronologically, so all five operators are exact and the constant's spelling doesn't need to match the event's: ingest rewrites payload values to RFC 3339 UTC before publishing, and a zone-less constant (`2026-06-21 04:00:00`, read in the column's declared zone, else the discovered server default — ClickHouse's own rule) still matches the rewritten payload denoting that instant. **Write timestamp constants zone-less like that, or as 9–10-digit Unix-seconds strings**: those two spellings are converted to the column type in a `WHERE` comparison on every ClickHouse release. The RFC 3339 `Z` form is instead rejected there with a type error on older releases (verified on 25.x; 26.6 accepts it — the exact release that changed isn't pinned here, so prefer the two always-safe spellings) — a property of the release's constant conversion, not of `date_time_input_format`, which governs the [ingest parser WaveHouse pins](/api#timestamp-canonicalization) rather than comparison constants, so setting `best_effort` server-side does not rescue it. The stream accepts every grammar spelling regardless. An operand the grammar can't read — on either side — withholds the row, as does an instant outside the column type's range (which insert-time *saturation* would have moved anyway). A column whose **declared** zone can't be loaded at runtime has no timestamp parser at all — it falls into the byte-equality bucket below (its values aren't canonicalized at ingest either), so `_neq`/`_gt`/`_lt` withhold every row. A column relying on the **server default** zone when that couldn't be resolved keeps instant comparison for zone-explicit operands (RFC 3339, Unix seconds) but refuses zone-less ones rather than guess the zone. -- **Every other type** (`Enum`, `UUID`, `Date`/`Date32`, `Bool`, `IPv4`/`IPv6`, `FixedString`, …): only byte-equality is trusted. `_eq`/`_in` admit exactly the event's own text rendering — write the filter value the way your events carry it (`true`, not `1`; a lowercase UUID if that's what clients send; `Date` values keep the producer's spelling — unlike `DateTime`, they are not canonicalized at ingest). The same constant is bound into the query path's SQL, where ClickHouse compares it against the column's declared type rather than the event's text rendering — so pick a value that works on both surfaces, and verify the query path returns what you expect before relying on the filter. `_neq`, `_gt` and `_lt` withhold **every** row: a text difference can be pure representation (an uppercase UUID, an alternate date format, an Enum name vs. its number), so inequality and order are unprovable without ClickHouse — on these columns, use the query path for ordering/exclusion filters. -- **No usable schema** — the table is unknown to schema discovery, or discovery is still failing at boot (the server serves while retrying in the background): every column is treated as the "other" bucket above (timestamp columns lose their parser too). Equality scoping keeps working; ordering and `_neq` withhold until a schema is available. - -A few more edges worth knowing when you write a policy — the stream evaluates the **ingested event payload**, not the stored row, and each of these follows from that: +SSE subscribers are checked for table-level `select` permission, have denied columns stripped from each event, and receive only the rows their role's row `filter` admits, evaluated per subscriber against their JWT claims. Row-level security on the stream is evaluated by the **same engine as the server's own `WHERE` clause**: each event is parsed once (`internal/typelayer.Table.ParseRow`) and each subscriber's resolved predicates are compiled — claim values bound as `{p:String}` parameters, never interpolated — and evaluated against it (`Row.Visible`). Because this is ClickHouse's own parsing and comparison, every column type compares exactly as it would in a real `WHERE` clause, and there is no per-type comparison table to reconcile with the server. Predicates are evaluated against the **full ingested event**, so a filter may key on a column the role cannot `select`. -- **A filtered column the payload doesn't carry withholds *every* event** for that subscriber, even though the same filter matches normally on the query path. That bites a `MATERIALIZED`/`ALIAS` column, which is never part of an ingest payload — and note the `check` half of the pairing is no longer available for those columns, since a `check` on one is now refused outright (see Insert checks above). A `DEFAULT` column your clients omit is withheld too, but by the next rule rather than this one: a positional row carries one slot per insertable column, so an omitted column arrives as an explicit `null` rather than being absent. The recommended [`check` + `filter` pairing](#insert-checks) is unaffected: an `_eq` insert `check` auto-injects its claim value into any payload that omits the column *before* the event is published, so the streamed event carries it and the matching row filter evaluates normally. (That holds for timestamp columns too: the injected claim value is canonicalized with the rest of the payload before publish, and the stream compares timestamps as instants, so the claim's spelling and the canonical wire spelling meet.) -- **A non-scalar event value** (array/object/null) under a filtered column withholds the row. -- **Insert-time numeric narrowing is simulated, not skipped.** The insert narrows a payload carrying more precision than the column's declared type — a `Decimal` **truncates** at its scale (`1.005`, `1.006` and `1.009` all store as `1.00` in a `Decimal(10, 2)`), a `Float32` **rounds** to its nearest representable value (`16777217` stores as `16777216`) — and ClickHouse applies the same narrowing to a bound filter constant at compare time. The stream narrows **both operands** identically before comparing, so its verdict matches the query path's on narrowing columns under every operator: a `_gt: "1.004"` filter on a `Decimal(10, 2)` column withholds a `1.005` payload exactly as the query path hides the stored `1.00`. (An earlier revision of this feature compared the raw payload and could deliver such an event; that fail-open is closed, and an integration test holds every in-range numeric stream verdict equal to a live ClickHouse's — for the out-of-range operands the range gate refuses, it asserts the half that matters: the stream never admits a row ClickHouse hides.) What remains payload-vs-stored: an event whose insert later **fails outright** (an out-of-range value, a batch error, the DLQ) was already streamed to whichever subscribers the filter admitted, and its row never becomes queryable. +Only a **definite true** admits a row. Everything else withholds, and `wavehouse_sse_rows_withheld_total{table,role,reason}` counts each cause separately so a quiet stream's reason is visible rather than guessed: `filter` (a definite non-match), `error` (the predicate errored on this row — no supertype between constant and column, or a constant a non-integer column's type cannot read, ClickHouse's code 53; on an integer column such a claim is a `filter`), `decline` (the engine would not answer), `unavailable` (no chtypes artifact for the connected server's line, or a server timezone change since boot — see [Deployment → chtypes artifacts](/deployment#chtypes-artifacts)), and `drift` (the event's column list and the live table disagree after a mid-stream `ALTER`). A filter naming a column the table no longer has, or a `MATERIALIZED`/`ALIAS`/`EPHEMERAL` one, fails to **compile**, which withholds every row for that role until the filter or the table changes — logged once, not per event. -One more boundary is temporal: a subscriber's claims (and role) are captured when the SSE connection is established and are never re-read, while the policy itself is re-read on every live event. A gap-fill replay is the one exception: it runs under the single policy snapshot taken when the connection opened, so a policy change landing mid-replay applies from the first live event after it. Tightening a policy therefore applies from the next live event, but a token that expires — or claims revoked at the identity provider — keeps its open stream until the client disconnects, so treat connection lifetime as the revocation window for stream row-scoping. +Two edges follow from the stream evaluating the **ingested event** rather than re-reading the stored row. An **omitted `DEFAULT` column is not a problem case**: chtypes evaluated the `DEFAULT` before publish, so the event carries the real value, and the [`check` + `filter` pairing](#insert-checks) works — an `_eq` insert check stamps its claim into any payload that omits the column *before* publish, so the streamed event carries it and the matching row filter evaluates normally. What remains is the other direction: an event whose insert later **fails outright** at the ClickHouse level (a connectivity fault, a batch error unrelated to the record's own shape — chtypes already caught the shape problems at ingest) was already streamed to whichever subscribers the filter admitted, and its row never becomes queryable. -Each row withheld **from a subscriber** by row-level security is counted in `wavehouse_sse_rows_withheld_total` (labeled by table and role; a row withheld from three subscribers counts three times) — check it before concluding a quiet stream simply has no matching rows. +One more boundary is temporal: a subscriber's claims and role are captured when the SSE connection is established and never re-read, while the policy itself is re-read on every live event. A gap-fill replay is the one exception: it runs under the single policy snapshot taken when the connection opened, so a policy change landing mid-replay applies from the first live event after it. Tightening a policy therefore applies from the next live event, but a token that expires — or claims revoked at the identity provider — keeps its open stream until the client disconnects, so treat connection lifetime as the revocation window. The resource limits (`max_rows`, `max_execution_time`, `max_rows_to_read`, `max_memory_usage`) remain a property of the SQL query path and are **not** applied to the live event stream; if those caps are part of a role's isolation story, don't rely on them over the stream. ::: diff --git a/docs/src/content/docs/api.md b/docs/src/content/docs/api.md index b1f7bc66..0b7dfa43 100644 --- a/docs/src/content/docs/api.md +++ b/docs/src/content/docs/api.md @@ -64,19 +64,12 @@ X-Content-Type-Options: nosniff The body is always a JSON object that includes an `error` field describing the failure: ```json -{"error": "invalid json"} +{"error": "unknown table: clicks"} ``` Some endpoints attach extra fields alongside `error` on their **failure** responses — e.g. a failing `/readyz` returns `{"status":"not ready","error":"…"}`. The guarantee is scoped to failures: whenever a response signals an error (any 4xx/5xx), an `error` field is present and parseable. Success responses carry each endpoint's own shape and need **not** include `error` — a healthy `/readyz` returns just `{"status":"ready"}`. -This contract holds for: - -- Handler-emitted errors — validation (4xx), permission denials (403), not-found (404), backend errors (5xx). -- Router-level **404 Not Found** when the URL does not match any registered route. -- Router-level **405 Method Not Allowed** when the URL matches a route but the method is not registered. -- Server-level **500 Internal Server Error** when a handler panics — recovered, logged with stack, and reported to the client as JSON **when the handler has not yet committed any response headers or body bytes**. - -Historically some error paths defaulted to `text/plain` because they were emitted via `http.Error` or chi's default handlers; those paths now route through a shared `writeJSONError` helper so strict clients can branch on `Content-Type` consistently. +The contract holds for handler-emitted errors (validation, permission denials, not-found, backend failures), for router-level `404`s and `405`s, and for the `500` a recovered handler panic produces — the last one only while no response headers or body bytes have been committed. Everything routes through one `writeJSONError` helper, so strict clients can branch on `Content-Type` consistently. The per-endpoint error tables below list the bodies you can expect for each status code; the `Content-Type` and `X-Content-Type-Options` headers above apply uniformly and are not repeated. @@ -109,7 +102,7 @@ Returns `200 OK` once the gateway has discovered ClickHouse table schemas at lea Status code: `503 Service Unavailable` -The boot-degraded response lets an operator `curl /livez` to learn why the gateway isn't ready to serve traffic yet, instead of grepping a restart-loop log. The binary is bound on `:8080` and serves diagnostics, but is not yet accepting ingest/query traffic. Schema discovery retries with exponential backoff (2s → 60s); once a Refresh succeeds, `/livez` flips to `200` and stays there for the rest of the process lifetime — transient ClickHouse blips after that point are reflected in `/readyz`, not `/livez`. +The boot-degraded response lets an operator `curl /livez` to learn why the gateway isn't serving yet instead of grepping a restart-loop log: the binary is bound on `:8080` and serves diagnostics, but is not yet accepting ingest/query traffic. Schema discovery retries with exponential backoff (2s → 60s). --- @@ -135,7 +128,7 @@ Status code: `503 Service Unavailable` ### Liveness vs readiness — behavior matrix -`/livez` (liveness) and `/readyz` (readiness) answer different questions, so they diverge once the process has booted. `/livez` is **sticky**: after the first successful schema discovery it stays `200` for the rest of the process lifetime, even if ClickHouse later becomes unreachable — liveness asks "is the process alive and past boot," not "is its backend up right now." `/readyz` stays **conditional**: it pings ClickHouse on every call and drops back to `503` whenever ClickHouse is unreachable. +`/livez` is **sticky**: after the first successful schema discovery it stays `200` for the rest of the process lifetime, even if ClickHouse later becomes unreachable — liveness asks "is the process alive and past boot," not "is its backend up right now." `/readyz` stays **conditional**: it pings ClickHouse on every call and drops back to `503` whenever ClickHouse is unreachable. | State | `/livez` | `/readyz` | |----------------------------|:--------:|:---------:| @@ -167,7 +160,7 @@ Returns the build metadata embedded in the running binary — `version`, `git_co "version": "1.2.3", "git_commit": "a1b2c3d", "build_time": "2026-06-02T12:00:00Z", - "go_version": "go1.26.3" + "go_version": "go1.27.0" } ``` @@ -186,166 +179,143 @@ Where those values come from depends on how the binary was built: ### `POST /v1/ingest?table={table}` — Ingest Data -Accepts a single flat JSON object, a JSON array of objects, or a newline-delimited JSON (NDJSON) batch, validates each record against the ClickHouse schema for `{table}`, and publishes it to the message queue. Returns immediately — ClickHouse insertion happens asynchronously via the batch consumer. +Validates a body of records against the ClickHouse schema for `{table}` and publishes each accepted one to the message queue. Returns immediately — ClickHouse insertion happens asynchronously via the batch consumer. A single-object body answers `{"ok":true}` (or `{"duplicate":true}` when dedup is on); every other body answers the [batch summary](#batch-ingest). -**`Content-Type` is required and authoritative.** The format is what the caller declares, not what the bytes look like: a body declared as NDJSON is read as NDJSON whatever its first byte, so a line that isn't a JSON object fails as a per-record error rather than silently re-framing the whole request. A request with **no** `Content-Type`, or one whose media type is not in the accepted list, is rejected with `415` and a message listing the accepted types — nothing is guessed. The one thing the body still decides is *arity within the JSON family*: the first non-whitespace byte picks a top-level array (`[`) or a single object. The reverse mis-declaration is **not** caught: NDJSON sent as `application/json` is read as the single object it starts with and the remaining lines are ignored — a `200` for one record. Declare `application/x-ndjson` for anything line-framed ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)). +**The body goes to ClickHouse's own parser as-is.** WaveHouse never decodes a record: validation, type coercion, `DEFAULT` substitution and timestamp parsing are ClickHouse's own, running in-process via [chtypes](/deployment#chtypes-artifacts) (`internal/typelayer`) — the exact code path a real `INSERT` runs. A rejection therefore carries ClickHouse's own integer `code` and message rather than a WaveHouse-authored sentence, and there is no separate coercion table to keep in sync with the server. -:::note[What counts as a valid declaration] +**`Content-Type` is required and authoritative**: it declares the format and the bytes never override it. -The header is parsed with Go's `mime.ParseMediaType`, which implements RFC 9110 §8.3 `media-type`, and only the media type decides the format. Parameters are ignored, so no malformed parameter costs you the request — `application/json; charset`, `application/json;;`, a value left mid-quote, even a name repeated with different values all read as `application/json`. One exception: a malformed parameter on a line that **also contains a comma** is refused, because the comma may be a second declaration joined on and the error cannot distinguish that from a comma inside data ([#563](https://github.com/Wave-RF/WaveHouse/issues/563)). So `application/json; profile="a,b"` is one media type and is accepted, while `application/json; profile="a,b"; charset` is a `415` — each half alone is fine. +| `Content-Type` | Body | +| --- | --- | +| `application/json` | one flat object, **or** a top-level array of them | +| `application/x-ndjson`, `application/ndjson`, `application/jsonl`, `application/jsonlines` | one object per line; always a batch | +| `text/csv`, `text/tab-separated-values` | positional, with ClickHouse's own header auto-detection — see [Positional formats](#positional-formats-csv--tsv) | +| `text/csv; header=absent`, `text/tab-separated-values; header=absent` | strictly positional, no header detection | +| `text/csv; header=present`, `text/tab-separated-values; header=present` | a header line naming the columns, in any order — see [Header formats](#header-formats-headerpresent) | +| anything else, or none | `415`, listing the accepted types | -`Content-Type` is also a **singleton** field, and §5.3 forbids repeating it. So anything that isn't exactly one readable media type is a `415`: no header, an unsupported type, one whose **media type** doesn't parse, or more than one declaration. The single accommodation is for intermediaries that duplicate the header — **repeated header lines** are all resolved and accepted when they agree on the format. A **comma-joined** value is not — §8.3 warns that picking a member of the resulting pseudo-list is itself an interoperability and security hazard. Precisely: a value carrying a comma is refused whenever the value as a whole does not parse as one media type — whatever made it unparseable. A comma *inside a quoted parameter value* is legal data, so `application/json; a=", application/x-ndjson; b="` is one media type and is accepted, even though an intermediary may have built it by illegally joining two lines — the server cannot tell. - -The 415 body quotes what you declared, bounded: at most **four distinct** header lines, each capped at 128 bytes and marked `…(truncated)` when cut, followed by `"…and N more"`. N counts every header line not quoted — *including duplicates of one that is* — so five copies of the same header show it once, then `"…and 4 more"`. When declarations conflict, the one that actually disagreed is always quoted, even when four agreeing spellings would otherwise fill the list. -::: +The two JSON families are one format to ClickHouse; the declaration decides only how the body frames its records. The single thing the body still chooses is *arity within `application/json`*: the first non-whitespace byte picks an array (`[`) or a single object. Under a single-object body only the first object is read — concatenated objects after it are ignored, a `200` for one record; declare NDJSON for anything line-framed ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)). The reverse now works: a JSON array declared `application/x-ndjson` ingests every element. -| Body | `Content-Type` | Response | -| ---- | -------------- | -------- | -| one flat JSON object | `application/json` | `{"ok":true}` (or `{"duplicate":true}`) | -| a JSON array of objects (any length, even 1) | `application/json` | per-record summary — see [Batch Ingest](#batch-ingest) | -| one JSON object per line (NDJSON) | `application/x-ndjson` (also `application/ndjson`, `application/jsonl`, `application/jsonlines`) | per-record summary — see [Batch Ingest](#batch-ingest) | +:::note[What counts as a valid declaration] +The header is parsed with Go's `mime.ParseMediaType` (RFC 9110 §8.3) and the **media type** decides the format, so no malformed *parameter* costs the request — `application/json; charset`, `application/json;;`, a value left mid-quote, a name repeated with different values all read as `application/json`. The one parameter that also decides a format is `header`, on `text/csv` and `text/tab-separated-values` only: `present` selects the header format, `absent` the strictly positional one, no `header` at all ClickHouse's default reading, and any other value is a `415`. A line whose parameters did not parse and that mentions `header` is a `415` as well, because guessing at it could ingest a declared header line as data or drop a data row as a header. Two more things are refused. A malformed parameter on a line that **also contains a comma** is a `415`, because the comma may be a second declaration joined on and the error cannot tell that from a comma inside data ([#563](https://github.com/Wave-RF/WaveHouse/issues/563)) — so `application/json; profile="a,b"` is fine and `application/json; profile="a,b"; charset` is not. And `Content-Type` is a **singleton** field (§5.3 forbids repeating it), so repeated header *lines* are accepted only when they agree, while a comma-joined value is refused outright: §8.3 warns that picking a member of the resulting pseudo-list is itself an interoperability and security hazard. -The inbound request body is capped at 16 MiB; a body over the cap is rejected with `413` (matching [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse)). The cap applies to **every** body shape, NDJSON included — the whole body is read before it is parsed, so a line-framed batch is bounded by the same 16 MiB cap as a JSON array (NDJSON carries one additional, tighter bound: a single line over 10 MiB fails the request). The `413` is decided before any record is processed, so nothing is published — including for a single-object body whose trailing bytes push it over the cap, which is now rejected rather than accepted on its first object. Split an upload larger than the cap across several requests, and set your own outer limit at the [reverse proxy](/reverse-proxy#request-body-size-limits). +The 415 body quotes what you declared, bounded: at most **four distinct** header lines, each capped at 128 bytes and marked `…(truncated)` when cut, then `"…and N more"` counting every line not quoted, duplicates included. When declarations conflict, the one that actually disagreed is always quoted. +::: -The `{table}` URL query must match a table that exists in ClickHouse. WaveHouse discovers table schemas on startup and refreshes them periodically. +The inbound request body is capped at 16 MiB and the `413` is decided before any record is processed, so nothing is published. The cap applies to every body shape — the whole body is read before it is parsed, so a line-framed batch is bounded exactly as a JSON array is. Split a larger upload across several requests, and set your own outer limit at the [reverse proxy](/reverse-proxy#request-body-size-limits). The `{table}` query parameter must name a table WaveHouse has discovered in ClickHouse; schemas refresh periodically. :::note[Insert-only] -The ingest pipeline accepts only inserts. All other mutations — `DELETE`, `UPDATE`, `TRUNCATE`, `DROP`, `ALTER`, `REPLACE`, etc. — must be issued through [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse), which is restricted to the admin role (`admin_role`, the same gate as the rest of `/v1/ops/*`). - -The policy engine authorizes mutations by inspecting the columns being written. That works for inserts but not for predicate-driven mutations like `DELETE … WHERE` — there's no way to prove the predicate matches only rows the caller is allowed to touch. Routing those statements through the admin-gated raw-SQL surface keeps the policy contract honest. +The ingest pipeline accepts only inserts. Every other mutation — `DELETE`, `UPDATE`, `TRUNCATE`, `DROP`, `ALTER`, `REPLACE` — goes through [`POST /v1/ops/query`](#post-v1opsquery--query-clickhouse) under the admin role (`admin_role`, the same gate as the rest of `/v1/ops/*`). The policy engine authorizes a write by the columns it names, which works for an insert but not for a predicate-driven `DELETE … WHERE`: nothing can prove the predicate matches only rows the caller may touch. ::: -**Request:** +**What ClickHouse decides, and what WaveHouse decides.** Everything about a *value* is ClickHouse's: -```json -{ - "url": "https://example.com/dashboard", - "user_name": "Alice", - "verified": true, - "score": 42.5 -} -``` +- A field the role may not write is indistinguishable from one the table does not have: both are code **117**, `Unknown field found while parsing JSONEachRow format: x`. So are `MATERIALIZED`, `ALIAS` and `EPHEMERAL` columns — none of the three is ever part of a published row. +- An omitted column, or an explicit `null` on one (WaveHouse pins `input_format_null_as_default`), takes its `DEFAULT` expression — evaluated by ClickHouse, including a volatile one like `now()` — or the type's implicit zero where none is declared, exactly as an `INSERT` naming fewer columns does. +- A coercion ClickHouse would make it makes here (a numeric string into an `Int*`, `"true"` into a `Bool`, an out-of-range integer wrapping); anything it would refuse fails synchronously in the ingest response with its real code, rather than surfacing later in the DLQ. `Nullable()` and `LowCardinality()` wrappers are transparent. -The body is a **flat JSON object** whose keys must match column names in the target ClickHouse table. Values must be type-compatible (see schema validation below). +WaveHouse decides only policy: whether the role may insert at all, and whether the record satisfies the role's [`check` clauses](/access-control#insert-checks) — evaluated by the same compiled-filter engine as row-level security, in the same parse that validates the record and against the row ClickHouse produced, so a check sees stored values rather than the payload's spelling. A record ClickHouse refuses reports that refusal, never a check result. A record chtypes cannot evaluate at all — as opposed to accepting or rejecting it — is **declined** (`422`), which is not a data verdict. -**Schema Validation:** - -- Unknown fields (not in the ClickHouse schema) are rejected. -- Type mismatches are rejected (e.g., sending a boolean for a `Float64` column). -- Missing required columns (non-nullable without a default) are rejected. -- Null values for non-nullable columns without a default are rejected. -- A value for a `MATERIALIZED` or `ALIAS` column is rejected — ClickHouse computes those, and the published row has no slot for one. -- **An omitted `Nullable(T) DEFAULT …` column now stores `NULL`, not the default.** The published row is positional, with one slot per insertable column and no way to express "absent", so an omitted key rides as an explicit `null`; `input_format_null_as_default` rescues that only for a **non-nullable** column. On a nullable column only an absent key ever took the default, and a positional row cannot express absence. Verified on ClickHouse 26.6.3. (One case changes only on a server explicitly running `input_format_null_as_default=0`: an explicit `null` for a non-nullable column with a default now takes the default there rather than failing the row into the DLQ, because WaveHouse pins the setting instead of inheriting it. On a default-configured server this was already the behavior.) -- Type compatibility: `String` accepts JSON strings, numbers, and booleans (ClickHouse coerces the non-strings); `FixedString`/`UUID` accept the same at validation, but ClickHouse rejects a non-string value there, so it surfaces in the DLQ; `DateTime`/`Date`/`Enum` accept JSON strings or numbers; `IPv*` accepts JSON strings (a number passes validation but ClickHouse rejects it → DLQ); `Int*`/`Float*`/`Decimal` accept JSON numbers or strings — a string lets JavaScript callers avoid 64-bit precision loss, and its contents are ClickHouse's to judge (a non-numeric string is accepted here and surfaces in the DLQ, not as a `400`); `Bool` accepts JSON booleans and the numbers `0`/`1` (any other number, and *any* string — including `"true"` — passes validation but is rejected by ClickHouse → DLQ); `Array` accepts JSON arrays; `Map` accepts JSON objects; `Tuple` accepts JSON arrays or objects at validation, but ClickHouse takes an array only for an *unnamed* tuple and an object only for a *named* one — the other shape surfaces in the DLQ; any other ClickHouse type (`JSON`, `Variant`, `Dynamic`, geo, …) accepts any JSON value — WaveHouse defers to ClickHouse, so a bad value surfaces in the DLQ rather than as a `400`. -- `Nullable()` and `LowCardinality()` wrappers are handled transparently. -- Top-level `DateTime`/`DateTime64` values are rewritten to a canonical wire form on ingest — see [Timestamp canonicalization](#timestamp-canonicalization). - -**Response (accepted):** - -```json -{"ok": true} -``` - -**Response (duplicate):** *(only when dedup is enabled)* - -```json -{"duplicate": true} -``` - -**Error responses:** +**Error responses.** Rows marked **per-record** are reported in `results` on a batch body (the request itself stays `200`) and become the response status on a single-object body; every other row fails the whole request. | Status | Body | Cause | | ------ | ---- | ----- | -| 400 | `{"error":"invalid request body"}` | The body could not be read at all — a malformed transfer encoding, or a truncated upload (a body cut off *in transit*). A body that arrived complete but ends mid-value is `invalid json` | -| 400 | `{"error":"invalid json"}` | Malformed request body | -| 400 | `{"error":"unknown column ... for table ..."}` (also: `missing required column ...`, `type mismatch for column ...`, `null value for non-nullable column ...`) | Schema validation failure (unknown fields, type mismatches, missing required columns, null in a non-nullable column with no default). The body is the validator's message verbatim — there is no `validation failed:` prefix. | -| 400 | `{"error":"column \"x\" of table \"t\" is materialized and cannot be inserted"}` (also `… is alias …`) | The record supplies a value for a column ClickHouse computes. Omit it — the server fills it in. Refused rather than dropped: the published row has one slot per insertable column, so the value would otherwise vanish behind a `200` | -| 400 | `{"error":"missing dedupe id field \"event_id\""}` | Only when dedupe is enabled with `dedupe.require_id: true` and the row lacks the configured `id_field`. With `require_id: false` (the default) the row is instead published un-deduped. Either way — reject or publish — the row is logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total`. In a batch this is a per-record failure, not a whole-request error. | +| 400 | `{"code":,"error":""}` | **Per-record.** ClickHouse's parser refused the record; `code` and message are its own. `117` is an unknown field — which now includes a column the role may not write, and any `MATERIALIZED`/`ALIAS`/`EPHEMERAL` column; `27`/`26` are unparseable input; `6` out of range | +| 400 | `{"code":117,"error":"Unknown field found in format header: 'x' at position 1 …"}` | A `header=present` body whose header names a column the table — or the role's writable set — does not have, or names one twice. ClickHouse refuses the body before reading any record, so the whole request fails and nothing is published | +| 400 | `{"error":"invalid request body"}` | The body could not be read at all — a malformed transfer encoding, or an upload cut off *in transit* | +| 400 | `{"error":"empty body"}` (declared variants: `empty ndjson body`, `empty csv body`, `empty tsv body`, `empty csvwithnames body`, `empty tsvwithnames body`) | The body holds no bytes. A `header=present` body holding only its header line is a valid record-less batch (`200`, `total: 0`) | +| 400 | `{"error":"invalid json: unterminated json array"}` | A body declared `application/json` opening with `[` whose brackets do not balance — truncated, or structurally broken. It cannot be salvaged per record, so the whole request fails | +| 400 | `{"error":"missing dedupe id field \"event_id\""}` | **Per-record.** Only with `dedupe.require_id: true`, when the row carries no value for the configured `id_field`. With `require_id: false` (the default) the row is published un-deduped instead. Either way it is logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total` | | 401 | `{"error":"invalid token"}` / `{"error":"token expired"}` | A present-but-invalid/expired token was supplied and denied (the gate surfaces the token reason rather than silently falling back to `default_role`) | -| 403 | `{"error":"forbidden"}` (empty-role variant: `forbidden: request has no role and no public default_role is configured`) | The resolved role lacks `insert` on the table | -| 403 | `{"error":"column \"x\" not allowed for insert"}` | The record names a column the role's `allow_columns`/`deny_columns` forbids ([Access control → Column permissions](/access-control#column-permissions)) | -| 403 | `{"error":"check failed for column \"x\""}` | The record's value for a checked column doesn't satisfy the policy `check` (`_eq`/`_in`), or an `_in`-checked column is omitted ([Access control → Insert checks](/access-control#insert-checks)). On the batch path both this and the column error above are per-record failures reported in `results`, not whole-request rejections | -| 403 | `{"error":"policy check references column \"x\", which table \"t\" does not have"}` (also `… which is materialized and cannot be inserted`, the same for `alias`, and `… which is ephemeral and is never stored`) | A **policy misconfiguration**, not a bad request: the role's `check` names a column the table lacks, one ClickHouse computes, or an `EPHEMERAL` one. None can be enforced — the published row carries one slot per insertable column, and an ephemeral column is never stored — so the check would have passed silently while enforcing nothing. Like the rejections above this is decided per record, so the **status depends on the body shape**: a single-object request answers `403`, while a batch answers `200` and carries the same message against each record in `results`. It fires on **every** insert by that role until the policy or the table is corrected, and names every offending column rather than one of them. `wavehouse validate` cannot catch it: it never sees the ClickHouse schema | -| 404 | `{"error":"unknown table: ..."}` | Table not found in ClickHouse schema | +| 403 | `{"error":"forbidden"}` (empty-role variant: `forbidden: request has no role and no public default_role is configured`) | The resolved role lacks `insert` on the table — checked once, before any record | +| 403 | `{"error":"check failed for column \"x\""}` (several: `check failed for columns "x", "y"`) | **Per-record.** The row does not satisfy the role's insert [`check`](/access-control#insert-checks). The filter is AND-joined over every checked column, so with more than one it names the set that was tested rather than guessing an attribution | +| 403 | `{"error":"policy check references column \"x\", which table \"t\" does not have"}` (also `… which is materialized and cannot be inserted`, the same for `alias`, and `… which is ephemeral and is never stored`) | **Per-record.** A **policy misconfiguration**, not a bad request: the role's `check` names a column the table lacks, one ClickHouse computes, or an `EPHEMERAL` one. None can be enforced, so the check would have passed silently while enforcing nothing. It fires on every insert by that role until the policy or the table is corrected, and names every offending column. `wavehouse validate` cannot catch it — it never sees the ClickHouse schema | +| 404 | `{"error":"unknown table: ..."}` | Table not found in the discovered schema | | 413 | `{"error":"request body exceeded 16777216 bytes"}` | Request body over the 16 MiB cap | -| 415 | `{"error":"no Content-Type: ingest requires one of application/json, application/x-ndjson, …"}` (declared variant: `Content-Type "text/plain": ingest requires one of …` — see the note above on how declarations are echoed; conflicting variant: `conflicting Content-Type declarations "application/json", "application/x-ndjson": ingest reads one format per request, and requires one of …`) | The request declared no `Content-Type`, one whose media type is unsupported or does not parse, a comma-bearing value that does not parse as a single media type, or repeated header lines that disagree — different formats, or one supported and one not. Checked before the body is parsed | -| 500 | `{"error":"dedupe failed"}` | Deduplication backend error | -| 500 | `{"error":"publish failed"}` | Message queue error | -| 503 | `{"error":"service unavailable"}` | NATS JetStream stream full (backpressure). Response includes `Retry-After: 30` header. | - -**curl example:** +| 415 | `{"error":"no Content-Type: ingest requires one of application/json, application/x-ndjson, application/ndjson, application/jsonl, application/jsonlines, text/csv, text/csv; header=present, text/csv; header=absent, text/tab-separated-values, text/tab-separated-values; header=present, text/tab-separated-values; header=absent"}` (declared variant: `Content-Type "text/plain": ingest requires one of …`; conflicting variant: `conflicting Content-Type declarations "application/json", "application/x-ndjson": ingest reads one format per request, and requires one of …`) | No `Content-Type`, an unsupported or unparseable one, a `header` value other than `present`/`absent`, a comma-bearing value that does not parse as a single media type, or repeated lines that disagree. Checked before the body is read | +| 422 | `{"error":"validation engine declined: "}` | **Per-record.** chtypes could not evaluate the record at all — the artifact declined the shape, rather than the data being wrong. A `check` clause that could not be evaluated lands here too (`validation engine declined: the insert check for column "x" could not be evaluated`) | +| 500 | `{"error":"dedupe failed"}` / `{"error":"publish failed"}` | Deduplication backend or message-queue error | +| 503 | `{"error":"service unavailable"}` | NATS JetStream stream full (backpressure). Carries `Retry-After: 30` | +| 503 | `{"error":""}` | No chtypes artifact matches the connected ClickHouse server's minor version, or the server's reported timezone changed since WaveHouse started — checked once for the whole table, so it aborts before any record. Carries `Retry-After: 30`; logged at most once per table per minute. See [Deployment → chtypes artifacts](/deployment#chtypes-artifacts) | ```bash curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ -H "Content-Type: application/json" \ -d '{"page": "/home", "button": "signup", "score": 42.5}' +# → {"ok":true} ``` -#### Timestamp canonicalization - -**Send the canonical form — RFC 3339 UTC with the fraction already truncated to the column's precision and trailing zeros trimmed (spelled out below) — and the value is republished byte-for-byte.** A `DateTime`/`DateTime64` column value sent in any other accepted form — including RFC 3339 UTC with extra or trailing-zero fraction digits — is rewritten to that **one canonical wire form — RFC 3339 UTC** (`2026-06-21T04:00:00Z`, fraction truncated to the column's precision) before publishing, so the stored instant never changes but every consumer (the ClickHouse insert, [SSE subscribers](#get-v1stream--server-sent-events-stream), the DLQ) sees the same spelling `/v1/query` renders. The accepted input forms: - -- RFC 3339, any offset (`.`-fractions only — ClickHouse has no `,` separator). -- `YYYY-MM-DD[ T]HH:MM:SS[.fff]` or `YYYY-MM-DD`, zone-less — interpreted in the column's time zone, else the ClickHouse server's, exactly as ClickHouse itself would. -- A Unix-seconds string of exactly 9–10 digits (a `.fff` fraction is honored only for `DateTime64` columns, as ClickHouse does). -- A **non-negative integer** JSON number, unquoted — read the way ClickHouse reads bare numbers: Unix **seconds** for a `DateTime` column, but the column's raw **tick count** for `DateTime64` (a `DateTime64(3)` stores milliseconds, so `1750478400500` is the millisecond epoch `2025-06-21T04:00:00.5Z` — and `1750478400` is January 1970, not June 2025). +#### Timestamp rendering -**Fail-open**: a value in none of those forms is published verbatim — ClickHouse's more liberal parser decides insertability, and a value it too rejects surfaces via the DLQ, as before. `Date`/`Date32` columns pass through untouched. +WaveHouse rewrites timestamps in neither direction. **Inbound**, any spelling ClickHouse's own parser accepts under `date_time_input_format=best_effort` (the setting WaveHouse pins, both at chtypes' ingest compile and on the worker's `INSERT`) is accepted — RFC 3339 with any offset, a zone-less `YYYY-MM-DD[ T]HH:MM:SS[.fff]` read in the column's declared zone else the server's default, a Unix-seconds string, a bare integer at the column's tick scale, among the other forms its lenient parser reads. It is ClickHouse's grammar, not a reimplementation of it, so whatever a real `INSERT` into this table would accept, ingest accepts, with the same coercions and the same refusals. -:::note[Pass-through edge cases] +**Outbound**, `DateTime`/`DateTime64` values in the NATS/SSE wire row and in `/v1/query` / `/v1/pipes/{name}` results are the exact bytes ClickHouse's writer produces, in the column's declared zone else the server's default: `"2026-06-21 04:00:00.123"`, space-separated, no `Z` suffix, never RFC 3339. Every consumer renders from the same stored value the same way, so SSE and `/v1/query` agree on spelling for a given row **by construction**, with no WaveHouse rewriting step to keep in sync ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). The raw-SQL proxy `/v1/ops/query` is the exception: it sets `date_time_output_format=iso`, which keeps trailing fraction zeros and an ISO-8601 `Z`, and is not expected to match the other two byte-for-byte. -- Digit-strings of lengths other than 9–10 are ClickHouse's own forms — calendar shapes like `YYYYMMDD`, or its 13/16/19-digit ms/µs/ns epochs — and pass through untouched. -- A bare number with a fraction or exponent (`1750478400.5`) is passed through un-rewritten, and ClickHouse then fails the row for any timestamp column: it parses bare numbers as integers only — its lenient timestamp parsing, which accepts `"1750478400.5"` for a `DateTime64` column (on a plain `DateTime` the leftover fraction still fails the row), applies solely to quoted strings. -- An instant outside the column type's range also passes through — ClickHouse *saturates* out-of-range values spelling-dependently (and a `DateTime64(9)` column rejects the insert outright past the Int64-nanosecond ceiling, 2262-04-11 — a bound WaveHouse conservatively applies to every `DateTime64` of precision ≥ 7 when deciding what it may rewrite), so no rewrite there is safe. -- A time zone that doesn't resolve at runtime also causes pass-through: the binary embeds no tzdata, so named zones resolve from the runtime's zone database (the bundled distroless images ship one; a stripped-down custom runtime, or a server zone newer than the image's snapshot, may not resolve). An unresolvable *column* zone skips canonicalization for that column entirely; an unresolvable *server* default skips only zone-less values of columns without a declared zone — warned at schema refresh either way, and never guessed as UTC, which could move the stored instant. Remedy: install `tzdata` in a custom image, or point Go at a zone database via the `ZONEINFO` environment variable. -- Timestamps nested inside a composite column (`Array(DateTime)`, `Map(K, DateTime64)`, `Tuple(…, DateTime)`) pass through untouched; only top-level `DateTime`/`DateTime64` columns (including `Nullable`/`LowCardinality` wrappers) are canonicalized. -- The accepted grammar is differentially tested against a live ClickHouse: raw and canonicalized spellings must insert identically, or both fail. +**Row-level security compares instants, not spellings.** A stream row filter on a `DateTime`/`DateTime64` column is compiled and evaluated by the same engine that validates ingest ([internal/typelayer](/access-control#where-each-rule-is-enforced)), so a filter constant in any spelling ClickHouse would accept in a `WHERE` clause matches the stored instant however the payload spelled it, and a predicate the engine cannot compile withholds every row for that role. -::: +#### Positional formats (CSV / TSV) -:::caution[Upgrading WaveHouse against a pre-26.5 ClickHouse] -WaveHouse pins `date_time_input_format=best_effort` on its inserts — the ClickHouse server default since 26.5. On an older server whose default was `basic`, a plain `DateTime` column read an all-digit timestamp string of five or more digits as Unix seconds (shorter runs it rejected outright, where `best_effort` reads `"2026"` as a year); under `best_effort`, `"20260711"` stores 2026-07-11, not 1970-08-23, and some lengths (e.g. 12 digits) are rejected outright. (`DateTime64` columns diverge the same way on calendar-shaped runs — `"20260711"` is 1970-08-23 under `basic`, 2026-07-11 under `best_effort` — and additionally whenever an epoch run's unit doesn't match the column scale, e.g. a 16-digit microsecond epoch into a `DateTime64(3)`; an epoch run whose unit matches the column scale (a 13-digit millisecond epoch into a `DateTime64(3)`) reads identically too — only 9–10-digit Unix-seconds runs, with an optional fraction, agree at *every* scale.) The canonical form itself is what the pin rescues: under `basic` an RFC 3339 value's `Z` suffix is rejected outright (the row fails and lands in the DLQ), and the pin is what makes it insertable regardless of server version. Zone-less date-times and 9–10-digit Unix-seconds strings parse identically under both settings. -::: +`text/csv` and `text/tab-separated-values` are **positional**; `header` is [RFC 4180 §3](https://www.rfc-editor.org/rfc/rfc4180#section-3)'s optional parameter, and WaveHouse maps it onto ClickHouse's own behavior three ways: -**The canonical form, precisely.** This is the one strict timestamp spelling in WaveHouse — the same one `/v1/query` and `/v1/pipes/{name}` render for top-level timestamp columns and the SSE stream carries (the raw-SQL proxy `/v1/ops/query` instead renders server-side via `date_time_output_format=iso`, which keeps trailing fraction zeros): +| Content-Type | Reading | +| --- | --- | +| `text/csv; header=present` | `CSVWithNames`: a header line is required and columns are addressed by name — see [Header formats](#header-formats-headerpresent) | +| `text/csv; header=absent` | `CSV`, strictly positional: header detection is off (`input_format_csv_detect_header=0`), so every line is a record | +| `text/csv` (no `header` parameter) | ClickHouse's default `CSV`: header auto-detection stays on | -- `YYYY-MM-DDTHH:MM:SSZ`, or `YYYY-MM-DDTHH:MM:SS.FZ` when there is a sub-second part: uppercase `T` separator, uppercase `Z` suffix, always UTC — never a numeric offset — and seconds always present. -- The fraction is **truncated** (never rounded) to the column's precision: a `DateTime` column (whole seconds) never carries a fraction; a `DateTime64(3)` column carries at most three digits. -- Trailing fractional zeros are trimmed and an all-zero fraction is dropped (Go's `time.RFC3339Nano` rendering): `.120` becomes `.12Z`, `.000` becomes plain `Z` — byte-for-byte what `/v1/query` returns for the same stored value. -- A column's declared time zone changes only how zone-less *inputs* are interpreted, never the output: every canonical value ends in `Z`. +`text/tab-separated-values` maps the same way, with `input_format_tsv_detect_header`. The auto-detection is ClickHouse's own heuristic, not WaveHouse's: send `header=absent` when a data row could spell the column names or you need the first line always read as a record. With no parameter, the positional fields are the table's **wire columns** — declaration order minus every `MATERIALIZED`, `ALIAS` and `EPHEMERAL` column — and a producer must send **every one of them, in that order**. `GET /v1/ops/schema?table={table}` returns the columns in `position` order; drop the three computed kinds and that is the field order. -Examples for a `DateTime64(3, 'America/New_York')` column: `"2026-06-21 00:00:00.1239"` (zone-less, read in New York) → `"2026-06-21T04:00:00.123Z"`; `"1750478400.5"` (Unix-seconds string) → `"2025-06-21T04:00:00.5Z"`; the integer number `1750478400500` (ticks at the column's millisecond scale) → `"2025-06-21T04:00:00.5Z"`. +| Body | Outcome | +| --- | --- | +| every field, in order | accepted | +| an empty field (CSV) or `\N` (TSV) | that column takes its `DEFAULT` | +| too few fields | rejected, code **27** — ClickHouse's own message, e.g. `Cannot parse input: expected ',' before: …` | +| too many fields | rejected, code **117** — `Expected end of line` | +| a header line, no `header` parameter | ClickHouse detects it and **consumes** it as a header: `total` and every `index` count data rows only | +| a header line, `header=absent` | **not a header** — read as a data row, so it fails to parse (code 27) wherever a column cannot read its own name; the data rows after it still parse | -**The stream row-filter doesn't require this spelling.** Row-level enforcement compares timestamp operands as **instants** under the same input grammar, so a filter constant in any accepted spelling — zone-less, RFC 3339, Unix seconds — matches the canonical payload denoting the same instant, and an operand the grammar can't read withholds the row. Instant comparison also needs the column's timestamp parser from schema discovery — with no usable schema, or a declared zone that can't be loaded at runtime, the column falls back to byte-equality, where only an exactly matching spelling admits. See [the enforcement caution](/access-control#where-each-rule-is-enforced) for per-type comparison rules and the spelling that also works in query-path SQL. +The messages are ClickHouse's own and differ between ClickHouse lines; branch on the `code`. An empty **TSV** field is the empty string, not a default: `\N` is TSV's spelling for "take the default", and a `DateTime64` cannot read `""`. -#### Batch Ingest +A positional producer cannot self-describe, so a column-order change silently re-assigns values — pin it to the schema and re-check it after any `ALTER`, or send a header with `header=present`. -A **JSON array** of objects (`[{…}, {…}]`) or an **NDJSON** body (`Content-Type: application/x-ndjson`, one JSON object per line) ingests a batch in a single request. Each record is validated, authorized, deduplicated, and published independently, so **one malformed or rejected record never blocks the rest of the batch**. (The SDK's `insert([...])` array helper uses the NDJSON form automatically; both forms return the same response.) +```bash +curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ + -H "Content-Type: text/csv" \ + --data-binary $'"/home","signup",42.5,\n"/about","nav",3,\n' +# → {"total":2,"succeeded":2,"failed":0,"duplicates":0,"results":[{"index":1,"ok":true},{"index":2,"ok":true}]} +``` -- **JSON array** — the most convenient form from most HTTP clients. A structural JSON syntax error fails the whole request (`400`), but a wrong-typed element (a non-object) is reported per-record like any other rejection. An explicit empty array (`[]`) is a valid, record-less batch (`200`, `total: 0`). -- **NDJSON** — one record per line. Its advantages are tolerance of a malformed record and cheap client-side generation, not a larger ceiling: the body cap applies to it exactly as to a JSON array. Blank lines are skipped, and a single malformed *line* is reported and skipped (the newline reframes the next record) — where a structural syntax error anywhere in a JSON array fails the whole request. Both forms report a wrong-typed record per-record. +#### Header formats (`header=present`) -**Request (JSON array):** +`text/csv; header=present` and `text/tab-separated-values; header=present` open with a header line naming the columns, and the fields are addressed by it rather than by position. IANA's `text/tab-separated-values` registration defines no parameters, so `header` on TSV is WaveHouse's mirror of the CSV one. -```http -POST /v1/ingest?table=clicks -Content-Type: application/json +| Body | Outcome | +| --- | --- | +| a header naming the columns, in any order | the header is **not a record**: `total` and every `index` count data lines only | +| a column the header omits | takes its `DEFAULT`, exactly as an omitted JSON field does — including a check clause's injected value | +| a header name in a different case | matched case-insensitively, as ClickHouse does from 26.5 | +| a header naming a column the table, or the role's writable set, lacks — or a name given twice | the whole request is a `400` with code **117**; nothing is published | +| a bad data row | rejected per record with its code; the rows around it still ingest | +| only the header line | a valid record-less batch: `200`, `total: 0` | -[{"page": "/home", "button": "signup", "score": 42.5}, {"page": "/about", "button": "nav", "score": 3}, {"page": "/pricing", "button": "cta", "score": 7, "referrer": "/home"}] +```bash +curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ + -H "Content-Type: text/csv; header=present" \ + --data-binary $'button,page\nsignup,/home\nnav,/about\n' +# → {"total":2,"succeeded":2,"failed":0,"duplicates":0,"results":[{"index":1,"ok":true},{"index":2,"ok":true}]} ``` -**Request (NDJSON):** +#### Batch Ingest -```http -POST /v1/ingest?table=clicks -Content-Type: application/x-ndjson +A JSON array, an NDJSON body, a CSV body or a TSV body ingests a batch in one request. Each record is validated, authorized, deduplicated and published independently, so **one malformed or rejected record never blocks the rest of the batch** — including inside a single-line (compact) JSON array, which WaveHouse re-frames in place before handing it over. An explicit empty array (`[]`) is a valid record-less batch (`200`, `total: 0`); blank lines in an NDJSON body are skipped. (The SDK's `insert([...])` array helper uses the NDJSON form automatically; every form returns the same response.) -{"page": "/home", "button": "signup", "score": 42.5} -{"page": "/about", "button": "nav", "score": 3} -{"page": "/pricing", "button": "cta", "score": 7, "referrer": "/home"} -``` +The response counts records read, published, rejected and deduplicated, then lists per-record outcomes: each entry mirrors the single-object response (`ok` / `duplicate` / `error`) plus its 1-based `index`, and carries ClickHouse's integer `code` when the rejection was its parser's. `results` is truncated to the first 10,000 entries; the four counts stay authoritative. -**Response (`200`):** a per-record summary. Each `results` entry mirrors the single-object response (`ok` / `duplicate` / `error`) plus its 1-based `index`. +```bash +curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ + -H "Content-Type: application/json" \ + -d '[{"page":"/home","button":"signup","score":42.5},{"page":"/about","button":"nav","score":3},{"page":"/pricing","button":"cta","score":7,"referrer":"/home"}]' +``` ```json { @@ -356,53 +326,17 @@ Content-Type: application/x-ndjson "results": [ { "index": 1, "ok": true }, { "index": 2, "ok": true }, - { "index": 3, "error": "unknown column \"referrer\" for table \"clicks\"" } + { "index": 3, "error": "Unknown field found while parsing JSONEachRow format: referrer", "code": 117 } ] } ``` -| Field | Meaning | -| ----- | ------- | -| `total` | records read from the body | -| `succeeded` | records validated and published | -| `failed` | records rejected — see `results` | -| `duplicates` | records skipped by dedup (when enabled) | -| `results` | per-record outcomes, each `{ index, ok\|duplicate\|error }` with `index` the 1-based record position. Truncated to the first 10,000 entries for very large batches (the counts stay authoritative). | - -A `200` is returned whenever the body was read and the records were processed — **even if every record failed**, so branch on `failed`/`results`, not the status code. Per-record problems (a malformed NDJSON line, a non-object array element, schema validation, column/check permission failures) are reported in `results` and the batch continues. Whole-request conditions abort with a non-`200` instead: - -| Status | Body | Cause | -| ------ | ---- | ----- | -| 400 | `{"error":"empty body"}` / `{"error":"empty ndjson body"}` | The body has no records | -| 400 | `{"error":"invalid request body"}` | The body could not be read at all — a malformed transfer encoding, or a truncated upload (a body cut off *in transit*). A body that arrived complete but ends mid-value is `invalid json` | -| 400 | `{"error":"invalid json: ..."}` | A structural JSON syntax error, a single NDJSON line over 10 MiB, or a JSON array that ends before its closing `]` — a body that transferred completely but was generated truncated. The whole request fails rather than reporting a partial success | -| 401 | `{"error":"invalid token"}` / `{"error":"token expired"}` | A present-but-invalid/expired token was supplied and denied (same auth gate as the single-object path; surfaces the token reason) | -| 403 | `{"error":"forbidden"}` (empty-role variant: `forbidden: request has no role and no public default_role is configured`) | The resolved role lacks `insert` on the table (checked once, before any record) | -| 413 | `{"error":"request body exceeded 16777216 bytes"}` | Request body over the 16 MiB cap | -| 415 | `{"error":"no Content-Type: ingest requires one of application/json, application/x-ndjson, …"}` (declared variant: `Content-Type "text/plain": ingest requires one of …` — see the note above on how declarations are echoed; conflicting variant: `conflicting Content-Type declarations "application/json", "application/x-ndjson": ingest reads one format per request, and requires one of …`) | The request declared no `Content-Type`, one whose media type is unsupported or does not parse, a comma-bearing value that does not parse as a single media type, or repeated header lines that disagree — different formats, or one supported and one not. Checked before the body is parsed | -| 500 | `{"error":"publish failed"}` / `{"error":"dedupe failed"}` | Message-queue or dedup-backend failure mid-batch | -| 503 | `{"error":"service unavailable"}` | NATS JetStream full (backpressure) mid-batch; includes `Retry-After: 30` | +A `200` is returned whenever the body was read and the records were processed — **even if every record failed**, so branch on `failed`/`results`, not the status code. :::caution[At-least-once on retry] -A batch aborted partway (a `503`/`500`, a JSON-array syntax error, or an NDJSON line over the 10 MiB line bound, after some leading records were already published) re-publishes those leading records when the whole batch is retried. A whole-body read failure is **not** one of these: a `413`, or the `400 invalid request body` of an upload cut off in transit, is decided before any record is processed, so nothing is published — safe to retry, once split for a `413`. Enable deduplication if duplicate suppression matters — this is the same at-least-once property the single-object path already has (the SDK retries both on `503`). +A batch aborted partway — a `503` or `500` after some leading records were already published — re-publishes those leading records when the whole batch is retried. Failures decided before any record is processed are not in this class: a `413`, a `415`, the `400 invalid request body` of an upload cut off in transit, and the unterminated-array `400` all publish nothing and are safe to retry as-is (once split, for a `413`). Enable deduplication if duplicate suppression matters — the single-object path has the same at-least-once property, and the SDK retries both on `503`. ::: -**curl example (JSON array):** - -```bash -curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ - -H "Content-Type: application/json" \ - -d '[{"page":"/home","button":"signup","score":42.5},{"page":"/about","button":"nav","score":3}]' -``` - -**curl example (NDJSON):** - -```bash -curl -X POST "http://localhost:8080/v1/ingest?table=clicks" \ - -H "Content-Type: application/x-ndjson" \ - --data-binary $'{"page":"/home","button":"signup","score":42.5}\n{"page":"/about","button":"nav","score":3}\n' -``` - --- ### `POST /v1/ops/query` — Query ClickHouse @@ -514,19 +448,38 @@ Every column the query references — in `columns`, an aggregation argument, `fi | `columns` | string \| string[] | No | Columns to SELECT — an array, or a single string for one column. A literal `"*"` is the column *named* `*`, **not** a wildcard. Omit (or send `[]` / `""`) to select nothing; use `select_all` for a full-row read. Mutually exclusive with `select_all`. | | `select_all` | bool | No | Select every column the role may read (the all-columns wildcard, expanded server-side to the allow/deny set). Mutually exclusive with a non-empty `columns`, and with `aggregations`. | | `aggregations` | object[] | No | Aggregation functions (`fn`, `column`, `alias`). | -| `filters` | object[] | No | WHERE conditions (`column`, `op`, `value`). Ops: eq, neq, gt, gte, lt, lte, in, like. | +| `filters` | object[] | No | WHERE conditions (`column`, `op`, `value`). Ops: eq, neq, gt, gte, lt, lte, in, like. A `null` value is a `400`. | | `group_by` | string[] | No | GROUP BY columns. | | `order_by` | object[] | No | ORDER BY clauses (`column`, `dir`). | | `limit` | int | No | Max rows. Omitted or above the configured `query.default_max_rows` (default 10,000) → silently capped at that value; a policy `max_rows` can lower it further (see [Access Control](/access-control#resource-limits)). | | `time_range` | object | No | Time window (`column`, `since`, `until`). `since`/`until` accept RFC3339 or Go-duration relative values ("1h", "30m", "7d", "2w" — day and week suffixes expand to hours). Relative values mean that long *ago*. The window applies only when `column` and `since` are set — an `until` without `since` is ignored. | +:::note[Filter values bind as strings] +Every bound value — a caller's filter, a policy row filter, an insert `check` — binds as a `{p:String}` parameter and is compared under the column's own type. One rule across all three surfaces, and the answer is the server's: send ClickHouse's own spelling for a value and it reads it. A policy claim on an integer column is compared through a strict cast on top, so a claim that is not the canonical spelling of a value the column can hold matches nothing instead of wrapping (see [Access Control](/access-control#jwt-claim-templating)); a caller's own filter keeps the plain form, since it can only narrow what the policy admits. There is no RFC3339 sniff any more, so a `"2026-01-01T00:00:00Z"` filter on a `DateTime` column is handed over verbatim — fine on every ClickHouse this repo pins (≥ 26.5 reads it natively), a per-query `500` below that line. + +On **this endpoint** an `in` list binds as one `Array(String)` parameter, and ClickHouse caps a query-string parameter at about 64 KiB of literal text — roughly 6,000 short elements. Past that the query fails with ClickHouse's own `500` rather than a clean `400`; the 1 MiB request-body cap alone would have allowed more. Stream row filters bind their `in` elements one parameter each and carry no such ceiling. +::: + :::note[Identifier names] -Table, column, and alias names may contain any characters ClickHouse accepts — dots, spaces, unicode, reserved keywords — because every identifier is backtick-quoted automatically. The one exception is a name containing a literal `?`, which is rejected with `400` (a clickhouse-go positional-binder limitation tracked in [#279](https://github.com/Wave-RF/WaveHouse/issues/279)). +Table, column, and alias names may contain any characters ClickHouse accepts — dots, spaces, unicode, reserved keywords — because every identifier is backtick-quoted automatically. The one exception is a name containing a literal `?`, which is rejected with `400`: the builder assembles positional placeholders before rewriting them to ClickHouse's named parameters, and a `?` inside an identifier would desync that rewrite ([#279](https://github.com/Wave-RF/WaveHouse/issues/279)). ::: **Response:** -JSON array of result rows. Top-level `DateTime`/`DateTime64` values are returned in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`) — `Nullable` timestamp columns included (a SQL `NULL` renders as JSON `null`), while timestamps nested inside `Array`/`Map`/`Tuple` columns are rendered in the column's declared zone, else the ClickHouse server's, as the driver returns them — byte-identical to the [SSE stream](#get-v1stream--server-sent-events-stream) for values [canonicalized at ingest](#timestamp-canonicalization) (a fail-open pass-through that ClickHouse accepted still comes back canonical here, though it streamed in the producer's spelling). The response carries an `X-Cache: HIT` or `X-Cache: MISS` header — this endpoint shares the in-process L1 (Ristretto) + singleflight machinery (unlike `/v1/ops/query`, which always hits ClickHouse). +JSON array of result rows, **rendered by ClickHouse**: the query runs over its HTTP interface with `FORMAT JSONEachRow`, and WaveHouse frames the lines into an array without re-encoding a value. So every type is spelled the way the connected server spells it, per version, with no WaveHouse conversion table in between. + +| ClickHouse type | JSON | +| --- | --- | +| `DateTime`, `DateTime64` | `"2026-06-21 04:00:00.123"` — space-separated, no `Z`, in the column's declared zone else the server's; byte-identical to the [SSE stream](#get-v1stream--server-sent-events-stream) for the same row (see [Timestamp rendering](#timestamp-rendering)) | +| `Decimal*` | a JSON **number** (`12.5`), not a string | +| `Int64`/`UInt64` past 2^53 | an unquoted number — still lossy in a JavaScript `number`; read it as text if you need every digit | +| `FixedString(n)` | a string padded to `n` bytes with `\u0000` | +| `Enum*` | the name, not the ordinal | +| `Nullable(T)` | `null` for a SQL `NULL` | +| `Array`, `Map`, `Tuple` | ClickHouse's own JSON for the container | +| `NaN` / `Inf` | `null` (ClickHouse's default rendering) | + +Keys come back in **SELECT order**, not alphabetical. The response carries `X-Cache: HIT` or `X-Cache: MISS` — this endpoint shares the in-process L1 (Ristretto) + singleflight machinery (unlike `/v1/ops/query`, which always hits ClickHouse) and the cache stores ClickHouse's own bytes. The inbound request body is capped at 1 MiB; a body over the cap is rejected with `413`. A query AST is bounded by nature (far under 1 MiB even with a large `in`-list), and the cap blocks a single-request memory-exhaustion vector on this public endpoint. Set a tighter or higher outer limit at your [reverse proxy](/reverse-proxy#request-body-size-limits) — but it can only narrow the effective limit, not raise it past this cap. @@ -534,7 +487,9 @@ The inbound request body is capped at 1 MiB; a body over the cap is rejected wit | Status | Body | Cause | | ------ | ---- | ----- | -| 400 | `{"error":"..."}` | Schema validation error (unknown column, bad aggregation, or an unparseable `time_range` `since`/`until` — neither a relative duration nor an RFC3339 timestamp) | +| 400 | `{"error":"unknown column: x"}` | Schema validation error — an unknown column, a bad aggregation, or an unparseable `time_range` `since`/`until` (neither a relative duration nor an RFC3339 timestamp) | +| 400 | `{"error":"filter value must not be null"}` | A filter carries `"value": null`. It is refused rather than answered: `col = NULL` is never true, and an empty parameter would silently ask a different question | +| 500 | `{"error":"clickhouse query: Code: 158. DB::Exception: … (TOO_MANY_ROWS) …"}` | ClickHouse refused the query — a resource limit, a type mismatch, anything else its engine raises. The body carries ClickHouse's own wording | | 403 | `{"error":"forbidden"}` | Role lacks select permission on table | | 403 | `{"error":"column \"x\" not allowed"}` | Column denied by policy | | 403 | `{"error":"aggregation \"x\" not allowed"}` | Aggregation fn denied by policy | @@ -597,26 +552,26 @@ Opens a persistent SSE connection for real-time event streaming. Supports histor **Response:** SSE stream (`text/event-stream`). Data events include an `id:` field set to the event's `received_timestamp`. The stream opens with a `: connected` comment and emits a minimal `:` keepalive comment periodically (every 30 seconds by default), which keeps a quiet connection from being closed by a proxy; both are standard SSE comments that `EventSource` ignores (raw consumers should skip `:`-prefixed lines). -**Row values arrive positionally, and the column names are announced separately.** Before the first row, and again whenever the column list changes, the stream sends an `event: schema` frame naming the columns of the rows that follow — in order, already reduced to what the caller's role may read. That re-announcement is **not** guaranteed after a gap-fill across a column change; see the arity note below. Every data frame's `row` array then has exactly one value per announced column, in that order. `schema` is a **named** SSE event, so a browser `EventSource` must `addEventListener('schema', …)` — it never reaches `onmessage`. A schema frame carries **no** `id:` line, so it never moves the client's `Last-Event-ID`. In the example below the table has its own `received_timestamp` **column**, which collides by name with the frame's top-level `received_timestamp` **field** — they are different values: the field is when WaveHouse received the event, the row slot is that column as published (`null` where the record omitted it, which ClickHouse replaces with the column's default on insert). +**Row values arrive positionally, and the column names are announced separately.** Before the first row, and again whenever the column list changes, the stream sends an `event: schema` frame naming the columns of the rows that follow — in order, already reduced to what the caller's role may read. That re-announcement is **not** guaranteed after a gap-fill across a column change; see the arity note below. Every data frame's `row` array then has exactly one value per announced column, in that order. `schema` is a **named** SSE event, so a browser `EventSource` must `addEventListener('schema', …)` — it never reaches `onmessage`. A schema frame carries **no** `id:` line, so it never moves the client's `Last-Event-ID`. In the example below the table has its own `received_timestamp` **column**, which collides by name with the frame's top-level `received_timestamp` **field** — they are different values: the field is when WaveHouse received the event (WaveHouse's own RFC 3339 timestamp), the row slot is that column as ClickHouse rendered it (a record that omitted it carries the evaluated `DEFAULT`, not `null` — see [Timestamp rendering](#timestamp-rendering)). ```text event: schema data: {"table_name":"clicks","columns":["page","button","score","received_timestamp"]} id: 2026-03-24T12:00:00.123Z -data: {"table_name":"clicks","received_timestamp":"2026-03-24T12:00:00.123Z","row":["/home","signup",42.5,"2026-03-24T11:59:58.512Z"]} +data: {"table_name":"clicks","received_timestamp":"2026-03-24T12:00:00.123Z","row":["/home","signup",42.5,"2026-03-24 11:59:58.512"]} id: 2026-03-24T12:00:01.456Z -data: {"table_name":"clicks","received_timestamp":"2026-03-24T12:00:01.456Z","row":["/pricing","cta",7,null]} +data: {"table_name":"clicks","received_timestamp":"2026-03-24T12:00:01.456Z","row":["/pricing","cta",7,"2026-03-24 12:00:01.456"]} ``` A raw consumer must keep the most recent announced column list and zip each `row` against it; a value the record did not carry arrives as `null` in its slot rather than being omitted, so positions never shift. **Check arity before zipping:** drop a `row` whose length disagrees with the last announced list rather than zipping it, because the announcement is not guaranteed in one case — a connection that gap-fills across a column change may receive live rows with no fresh announcement until the columns next change or it reconnects ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)). An arity check covers an added or removed column; a *same-length* change (a `RENAME COLUMN`, or a drop paired with an add) it cannot see, and reconnecting is what resynchronizes. Separately, a replay spanning a server upgrade across the v2 ingest envelope silently omits the pre-upgrade events — see [Upgrading across the v2 ingest envelope](/deployment#upgrading-across-the-v2-ingest-envelope). The TypeScript SDK does this for you and still yields row objects — `.stream()` and `.liveQuery()` are unchanged. The announcement is **per connection**, so a client that joins mid-stream is told the columns before it is sent a row, and a reconnect is told again. Each SSE connection is bound to a single `?table=`; to consume multiple tables, open one connection per table. -Values of top-level `DateTime`/`DateTime64` columns inside `row` arrive in the canonical RFC 3339 UTC form (ingest rewrites them before publishing — see [timestamp canonicalization](#timestamp-canonicalization)), so a live event and a `/v1/query` read of the same row agree on the instant in zone-explicit form — a zone-less spelling no longer parses as local time in a browser ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). The two renderings are byte-identical regardless of the declared time zone or a `Nullable` wrapper — a column declared with a non-UTC zone also streams as `Z`, and `/v1/query` normalizes it (nullable or not) to UTC before rendering. Canonicalization is fail-open at ingest, so a value outside the accepted input forms streams in whatever spelling the producer sent — and for exactly those events the byte-identity above does not hold: a spelling ClickHouse accepts anyway is stored and still queries back canonical, while one it too rejects lands in the DLQ and never becomes queryable at all. +Values of top-level `DateTime`/`DateTime64` columns inside `row` are ClickHouse's own rendering of the stored value — the exact bytes chtypes' `RowsExport` produced for that record (see [Timestamp rendering](#timestamp-rendering)), not a WaveHouse rewrite — so a live event and a `/v1/query` read of the same row agree on spelling **by construction**, with no separate canonicalization step to keep in sync ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). A column declared with a non-UTC zone streams in that zone, not normalized to UTC; parse the timestamp with a zone-aware parser rather than assuming `Z`. -**Note:** When access control policies are active, streamed events are filtered per the caller's role: tables without `select` permission are skipped, denied columns are removed from each event, and the role's [row-level `filter`](/access-control#row-level-security) is evaluated per subscriber against the caller's JWT claims — supplied by the connection's token (the `Authorization` header, or the `?token=` fallback above), with replayed gap-fill events filtered the same way. For a filter constant the query path's SQL also accepts ([the enforcement caution](/access-control#where-each-rule-is-enforced) gives per-type guidance), a connection is never delivered a row the query path would hide for that role — every comparison the stream can't prove fails closed and withholds the row instead. Numeric comparisons run in the column's storage domain — both operands narrowed the way ClickHouse narrows the stored value and the bound constant — so columns that narrow on insert (`Float32`/`Float64` width, a `Decimal`'s scale) agree with the query path too; the residual payload-vs-stored case is an event whose insert later fails into the DLQ, which the caution documents. The connection's claims are captured once, when the stream is established — a policy change applies from the next live event (an in-flight gap-fill finishes under the policy snapshot taken when the stream opened), but an expired token or changed claims take effect only when the client reconnects. +**Note:** When access control policies are active, streamed events are filtered per the caller's role: tables without `select` permission are skipped, denied columns are removed from each event, and the role's [row-level `filter`](/access-control#row-level-security) is compiled and evaluated per subscriber against the caller's JWT claims — supplied by the connection's token (the `Authorization` header, or the `?token=` fallback above), with replayed gap-fill events filtered the same way. This runs through the same in-process ClickHouse parser (chtypes) that validates ingest, so every column type compares exactly as it would in the query path's `WHERE` clause — a connection is never delivered a row the query path would hide for that role, and a predicate that can't compile or evaluate withholds the row instead of guessing (see [the enforcement caution](/access-control#where-each-rule-is-enforced) for the fail-closed reasons). The residual payload-vs-stored case is an event whose insert later fails outright at ClickHouse — a connectivity fault or batch error, not a data-shape problem chtypes would already have caught — which the caution documents. The connection's claims are captured once, when the stream is established — a policy change applies from the next live event (an in-flight gap-fill finishes under the policy snapshot taken when the stream opened), but an expired token or changed claims take effect only when the client reconnects. **CORS:** `/v1/stream` honors the `cors.allowed_origins` allowlist (settings directory) like every endpoint. Note that a **header-authenticated stream preflights before it connects** — `Authorization` is not CORS-safelisted — where a bare `EventSource` never preflighted at all: its request is not a `fetch()`, so Fetch's unsafe-request flag is never set and `Last-Event-ID` rides on the plain `GET`. Both headers are allow-listed, so an allowed origin connects *and* resumes cross-origin. @@ -805,18 +760,20 @@ The message format used on NATS JetStream between ingest and the batch consumer: "received_timestamp": "2026-03-24T12:00:00.123456789Z", "format": "JSONCompactEachRow", "columns": ["page", "button", "score", "received_timestamp"], - "row": ["/home", "signup", 42.5, null] + "row": ["/home", "signup", 42.5, "2026-03-24 12:00:00.123"] } ``` +The request that produced this envelope omitted `received_timestamp` (`DEFAULT now64(3, 'UTC')` on the table); chtypes evaluated the default before publish, so `row` carries the resulting timestamp — ClickHouse's own rendering, not `null` and not a WaveHouse rewrite. + | Field | Type | Description | | ----- | ---- | ----------- | | `table_name` | string | Target ClickHouse table (from URL). | | `scope` | string | Reserved; currently always empty. | | `received_timestamp` | string | RFC 3339 nano timestamp when WaveHouse received the event. | | `format` | string | Row format. Always `JSONCompactEachRow` today; stated on the wire so a reader can tell an envelope it understands from one it doesn't. | -| `columns` | string[] | The table's **insertable** column names, in declaration order — what each position in `row` means. A `MATERIALIZED` or `ALIAS` column is computed by ClickHouse and cannot be named in an `INSERT`, so it never appears here. | -| `row` | array | One `JSONCompactEachRow` line: one value per entry in `columns`, in that order. A column the request body omitted is `null` here; for a **non-nullable** column the insert turns that back into the column's default (`input_format_null_as_default`), but a `Nullable(T) DEFAULT …` column stores `NULL` — only an *absent* key ever took the default, and a positional row has one slot per column and no way to express absence. Parseable `DateTime`/`DateTime64` values are rewritten to canonical RFC 3339 UTC (see [timestamp canonicalization](#timestamp-canonicalization)); other values as originally sent. | +| `columns` | string[] | The table's **wire** column names, in declaration order — what each position in `row` means (`internal/typelayer.Table.WireColumns`: the insertable subset minus any `MATERIALIZED`, `ALIAS`, or `EPHEMERAL` column — none of the three can be named in an `INSERT`, or is ever part of a published row). | +| `row` | array | One `JSONCompactEachRow` line: one value per entry in `columns`, in that order — the exact bytes ClickHouse's own writer produced for this stored row (`internal/typelayer.Table.Ingest`, via chtypes). A column the request body omitted carries its evaluated `DEFAULT` (or the type's implicit zero value where none is declared), not `null` — the same as a native `INSERT` naming fewer columns than the table has. `DateTime`/`DateTime64` values are ClickHouse's own rendering (see [Timestamp rendering](#timestamp-rendering)), and an out-of-range integer is wrapped the way a real `INSERT` wraps it. | `columns` and `row` are only meaningful together: a reader that cannot pair them — a length mismatch, an undecodable row, a `columns` list naming one column twice — has no way to map a value to a column. Both readers also refuse an envelope whose `format` they do not recognize, which is what a pre-v2 message looks like. Either way the SSE fan-out withholds such an envelope rather than guess, and the batch consumer parks it on the DLQ with `X-DLQ-*` headers — acking and dropping it only where the DLQ is switched off for that table, since it can never insert on retry. Both outcomes increment `wavehouse_ingest_poison_total`, separated by its `disposition` label (`parked` / `dropped`). @@ -836,7 +793,7 @@ Three values, where the envelope above has four: this is the frame a role restri ## Dead Letter Queue (DLQ) -When a batch insert to ClickHouse fails (e.g., type errors, connection issues), the worker re-inserts the batch row by row: rows that succeed are acked, and only the rows that fail again are published to the DLQ NATS stream (`WAVEHOUSE_DLQ`) under subjects `dlq.{table}`. This prevents infinite retry loops — those messages are ACKed from the main stream and moved to the DLQ for inspection. A second class lands here too: an envelope the worker cannot *read* at all — malformed JSON, an unknown **or absent** `format` (a pre-v2 message has no `format` field at all, which is how it presents here), or `columns` and `row` that do not pair — is parked without ever reaching a table batch, which is what an operator sees after upgrading across the wire change without draining first. **Two different body shapes land here, and a consumer must not assume one decoder.** A row that failed its INSERT is parked as the `EventMessage` envelope above. An envelope the worker could not *read* is parked as **its original bytes, verbatim** — `parkOnDLQ` republishes what arrived — so it is whatever the producer sent: a pre-v2 `data` object, malformed JSON, or a v2 envelope whose `columns` and `row` do not pair. Being undecodable as an `EventMessage` is precisely why it was parked, so decode defensively and fall back on the `X-DLQ-Error` header, which names the reason. For the first shape the body is the published `EventMessage` envelope (`{"table_name":…,"scope":"","received_timestamp":…,"format":…,"columns":[…],"row":[…]}` — the failed row is the `row` array, read against `columns`, its `DateTime`/`DateTime64` values as published: canonicalized where WaveHouse could parse them, otherwise the producer's original spelling — see [timestamp canonicalization](#timestamp-canonicalization)); the failure reason, table, and time travel in the `X-DLQ-Table` / `X-DLQ-Error` / `X-DLQ-Timestamp` message headers. +When a batch insert to ClickHouse fails (e.g., type errors, connection issues), the worker re-inserts the batch row by row: rows that succeed are acked, and only the rows that fail again are published to the DLQ NATS stream (`WAVEHOUSE_DLQ`) under subjects `dlq.{table}`. This prevents infinite retry loops — those messages are ACKed from the main stream and moved to the DLQ for inspection. A second class lands here too: an envelope the worker cannot *read* at all — malformed JSON, an unknown **or absent** `format` (a pre-v2 message has no `format` field at all, which is how it presents here), or `columns` and `row` that do not pair — is parked without ever reaching a table batch, which is what an operator sees after upgrading across the wire change without draining first. **Two different body shapes land here, and a consumer must not assume one decoder.** A row that failed its INSERT is parked as the `EventMessage` envelope above. An envelope the worker could not *read* is parked as **its original bytes, verbatim** — `parkOnDLQ` republishes what arrived — so it is whatever the producer sent: a pre-v2 `data` object, malformed JSON, or a v2 envelope whose `columns` and `row` do not pair. Being undecodable as an `EventMessage` is precisely why it was parked, so decode defensively and fall back on the `X-DLQ-Error` header, which names the reason. For the first shape the body is the published `EventMessage` envelope (`{"table_name":…,"scope":"","received_timestamp":…,"format":…,"columns":[…],"row":[…]}` — the failed row is the `row` array, read against `columns`, its `DateTime`/`DateTime64` values exactly as published: ClickHouse's own rendering of the stored value, since chtypes already validated and coerced the record before it was ever published — see [Timestamp rendering](#timestamp-rendering)); the failure reason, table, and time travel in the `X-DLQ-Table` / `X-DLQ-Error` / `X-DLQ-Timestamp` message headers. Because chtypes catches the type/shape problems synchronously at ingest, a row that reaches this DLQ path failed for a reason chtypes couldn't have caught up front — a ClickHouse-side outage or a genuine insert-time fault — not a data mismatch. Use `GET /v1/ops/dlq/stats` to monitor DLQ depth. diff --git a/docs/src/content/docs/architecture.md b/docs/src/content/docs/architecture.md index 72c2e766..ef35075b 100644 --- a/docs/src/content/docs/architecture.md +++ b/docs/src/content/docs/architecture.md @@ -45,7 +45,7 @@ flowchart TD ## Binaries -WaveHouse ships a single binary, `wavehouse`: an all-in-one process running the API, batch worker, embedded NATS JetStream, and optional embedded Pebble dedup. The only external dependency is ClickHouse. +WaveHouse ships one binary, `wavehouse` — an all-in-one process running the API, batch worker, embedded NATS JetStream, and optional embedded Pebble dedup — plus a second artifact it loads at start: a per-ClickHouse-version shared library (`internal/typelayer`, via [chtypes](/deployment#chtypes-artifacts)) that runs ClickHouse's own parser in-process for ingest validation, type coercion, and row-level security. The binary requires cgo (dlopen only — no static link to the artifact) and glibc, so supported platforms are Linux amd64/arm64 and macOS arm64. The only external network dependency is ClickHouse. ## Internal Packages @@ -55,18 +55,19 @@ internal/ ├── auth/ JWT/JWKS authentication middleware (HMAC or JWKS, role extraction) ├── cache/ In-process Ristretto cache with singleflight coalescing ├── chconn/ The one ClickHouse driver.Conn every consumer holds; reload swaps the connection behind it -├── chsql/ Shared ClickHouse SQL helpers (identifier quoting, bind-safety) +├── chsql/ Shared ClickHouse SQL helpers (identifier quoting, bind-safety, strict integer cast) ├── config/ YAML + env var configuration loading ├── dedupe/ Optional deduplication (Pebble) -├── discovery/ ClickHouse schema introspection and validation +├── discovery/ ClickHouse schema introspection (system.columns/system.tables, server version + timezone) ├── ingest/ Batch buffering, DLQ, and Active Sweeper ├── mq/ Message queue abstraction (embedded NATS) ├── observability/ OpenTelemetry pipeline (traces/metrics/logs + Prometheus exposition) ├── pipes/ Named query pipes (NamedQuery type, parameter binding, Source) -├── policy/ Hasura-style access control (policy types, evaluation, Source) +├── policy/ Hasura-style access control (policy types, claim resolution, Source) ├── query/ Structured query AST, SQL builder, and timestamp bucketing ├── settings/ Settings directory: validate the JSON files, hold the adopted snapshot, reload on watch / SIGHUP / API -└── stream/ SSE fan-out: event Hub (project once per role), Subscriber queue, Bucket fan-out, keepalive Heartbeater wheel +├── stream/ SSE fan-out: event Hub (project once per role), Subscriber queue, Bucket fan-out, keepalive Heartbeater wheel +└── typelayer/ In-process ClickHouse parser (chtypes): ingest validation/coercion + predicate compilation ``` ### `api/` — HTTP Layer @@ -77,8 +78,8 @@ The API layer uses [Chi](https://github.com/go-chi/chi) for routing with Request - **auth middleware** — the JWT/JWKS authentication middleware is its own package, [`auth/`](#auth--authentication); the router runs it on every `/v1/*` route. - **pipes.go** — Named query pipe handlers: admin listing (`GET /v1/ops/pipes[/{name}]`, read per request from its `pipes.Source`) and execution with parameter binding. `pipes.json` is the only write path. - **structured_query.go** — Handler for `POST /v1/query?table={table}`: validates query AST, enforces permissions, builds and executes SQL. -- **ingest.go** — Accepts `POST /v1/ingest?table={table}` in three body shapes: one flat JSON object, a JSON array of them, or NDJSON. The **required** `Content-Type` chooses the format *family* — `application/json` versus the four NDJSON spellings — and within the JSON family the body's first non-whitespace byte picks array versus single object; the bytes never choose the family. Anything that is not exactly one readable media type is a `415`, decided before the body is read: the header is parsed per RFC 9110 §8.3, and because `Content-Type` is a singleton field, repeated header lines must all resolve to the same format and a value carrying a comma is refused unless the value as a whole parses as one media type — a comma inside a *quoted* parameter value is data, so `application/json; a=", application/x-ndjson; b="` is accepted. It then reads the whole (`MaxBytesReader`-capped) body into a pooled buffer and runs the per-format record readers over those bytes, so the `413` lands before any record is processed and peak memory per request is O(body) rather than O(record). Then it validates each record against the discovered schema, optional dedup, publishes to NATS subject `ingest.{table}`. When dedup is on, a row missing the configured `id_field` can't be deduped: it is logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total` (labeled by `table`), then published un-deduped — or rejected when `dedupe.require_id` is set ([#219](https://github.com/Wave-RF/WaveHouse/issues/219)). -- **query.go** — Proxies raw SQL for `POST /v1/ops/query` straight to ClickHouse's HTTP interface. **Not cached** — sets `Cache-Control: no-store` so every request hits ClickHouse; DateTime is rendered ISO-8601 via `date_time_output_format=iso` (the Go-side type conversion lives in the structured-query / pipes path, not here). +- **ingest.go** — Accepts `POST /v1/ingest?table={table}` and hands the body to ClickHouse's own parser in one call. The **required** `Content-Type` chooses the format (`content_type.go`: the `application/json` and NDJSON spellings → `JSONEachRow`, `text/csv` → `CSV`, `text/tab-separated-values` → `TSV`, and each of those two with `; header=present` → `CSVWithNames` / `TSVWithNames`; `; header=absent` → the same formats with header detection off (`typelayer.IngestOptions.StrictPositional`), a bare type leaves ClickHouse's auto-detection on, any other `header` value is a `415`); the bytes never choose it. Anything that is not exactly one readable media type is a `415`, decided before the body is read: the header is parsed per RFC 9110 §8.3, and because `Content-Type` is a singleton field, repeated header lines must all resolve to the same format and a value carrying a comma is refused unless the value as a whole parses as one media type. It then reads the whole (`MaxBytesReader`-capped) body into a pooled buffer, so the `413` lands before any record is processed. `ingest_framing.go` is the only code that reads those bytes itself: the first non-whitespace byte answers the one remaining question inside the JSON family (array → batch response, otherwise single object), a top-level array is re-framed in place — outer brackets and depth-1 commas blanked to newlines — so one bad record cannot cost the batch, and the dedupe id is read positionally out of the exported row. One `typelayer.Table.Ingest` call per body parses, validates and checks in the same pass (one `RowsExportWith` with the role's insert `check` clauses compiled into a row filter): it returns a verdict per record and the accepted rows as `JSONCompactEachRow` bytes, with no second parse for the checks. Accepted rows are deduped and published to NATS subject `ingest.{table}`. When dedup is on, a row missing the configured `id_field` can't be deduped: it is logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total` (labeled by `table`), then published un-deduped — or rejected when `dedupe.require_id` is set ([#219](https://github.com/Wave-RF/WaveHouse/issues/219)). +- **query.go** — Proxies raw SQL for `POST /v1/ops/query` straight to ClickHouse's HTTP interface. **Not cached** — sets `Cache-Control: no-store` so every request hits ClickHouse; DateTime is rendered ISO-8601 via `date_time_output_format=iso` — a deliberately different audience from the structured-query path, which leaves ClickHouse's default spelling alone so it matches the SSE wire. - **stream.go** — Real-time streaming via SSE. Callers select a table with the `?table=` query parameter. Each connection registers one `Subscriber` (the `stream/` package) with both the event `Hub` (under its `(topic, role)`) and the shared keepalive wheel, then drains both from a single byte-pump — so idle streams keep emitting `:` keepalive comments (surviving reverse-proxy idle timeouts) while live events arrive already projected and serialized. Per-event projection/serialization happens **once per role** in the `Hub`, not once per subscriber ([#294](https://github.com/Wave-RF/WaveHouse/issues/294)); the handler also snapshots the connection's JWT claims onto the `Subscriber`, which the `Hub` evaluates per subscriber when the role carries a row-level `filter` ([#319](https://github.com/Wave-RF/WaveHouse/issues/319)). Gap-fill replay from NATS JetStream (`DeliverByStartTime`) stays per-connection (low-volume, one-time on connect). - **schema.go** — Schema discovery API: list all schemas, get one table, trigger refresh. - **dlq.go** — DLQ stats endpoint and `EnsureDLQStream` helper for creating the `WAVEHOUSE_DLQ` NATS stream. @@ -88,11 +89,11 @@ The API layer uses [Chi](https://github.com/go-chi/chi) for routing with Request The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https://github.com/Wave-RF/WaveHouse/issues/294)) lives next to the keepalive primitives it shares. One abstraction per file. -- **hub.go** — `Hub`, the event fan-out. Subscribers register under `(topic, role)`; `Broadcast` decodes each event once, applies each subscribed role's column policy once, builds one SSE frame per role, and fans it to every member of that role's `Bucket` — prepending a per-connection `event: schema` frame wherever that connection's announced column list has drifted, and withholding the row if the announcement cannot be queued — collapsing the prior per-subscriber `unmarshal → evaluate → filter → marshal` into one pass per distinct `(role, table)` output shape (the [#294](https://github.com/Wave-RF/WaveHouse/issues/294) lever; the measured ceiling was ~2 270 deliveries/s from re-projecting per subscriber). That schema-before-row guarantee is the LIVE path's: `ReplayProjector` tracks drift in its own state and the two are not reconciled ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)). The column projection is claims-independent, so it is shared across a role's whole bucket; the role's row-level `filter` predicate is not — it is resolved against each subscriber's JWT claims, so for a role that carries a filter `Broadcast` keeps the shared column projection but delivers it only to the subscribers whose claims admit each row (`ResolvedPermissions.RowVisible`, evaluated against the full event via the type-aware comparison seeded from the schema registry — `policy.ColumnSpec`: numeric columns compare numerically, `String` bytewise, `DateTime`/`DateTime64` as instants through the same parser ingest canonicalization uses (`discovery.Column.TimeParser` — one grammar, so filter constants and canonicalized payloads can't disagree on the instant), everything else admits byte-equality only and fails ordering/`!=` closed, so a missing schema can never downgrade the comparison to a leak). Each row withheld this way increments `wavehouse_sse_rows_withheld_total`. This is the [#319](https://github.com/Wave-RF/WaveHouse/issues/319) fix that closes the query/stream row-level-security drift; roles without a filter keep the pure once-per-role fast path. `ReplayProjector` shares the same projection and per-connection row check for the handler's gap-fill, holding one policy snapshot per gap-fill and caching the per-table column-kind lookup across the replay loop. +- **hub.go** — `Hub`, the event fan-out. Subscribers register under `(topic, role)`; `Broadcast` decodes each event once, applies each subscribed role's column policy once, builds one SSE frame per role, and fans it to every member of that role's `Bucket` — prepending a per-connection `event: schema` frame wherever that connection's announced column list has drifted, and withholding the row if the announcement cannot be queued — collapsing the prior per-subscriber `unmarshal → evaluate → filter → marshal` into one pass per distinct `(role, table)` output shape (the [#294](https://github.com/Wave-RF/WaveHouse/issues/294) lever; the measured ceiling was ~2 270 deliveries/s from re-projecting per subscriber). That schema-before-row guarantee is the LIVE path's: `ReplayProjector` tracks drift in its own state and the two are not reconciled ([#543](https://github.com/Wave-RF/WaveHouse/issues/543)). The column projection is claims-independent, so it is shared across a role's whole bucket; the role's row-level `filter` predicate is not — it is resolved against each subscriber's JWT claims, so for a role that carries a filter `Broadcast` keeps the shared column projection but delivers it only to the subscribers whose claims admit each row. Visibility itself is decided by `internal/typelayer` (`Table.ParseRow` once per event, `Row.Visible` per subscriber) — the same ClickHouse parsing and comparison semantics the server's own `WHERE` clause applies, for every column type, rather than a hand-written per-type comparator; a predicate error, a policy column the table no longer has, a schema drift between the event and the live table, or an unavailable engine all withhold the row rather than guessing. Each row withheld this way increments `wavehouse_sse_rows_withheld_total{table,role,reason}`. This is the [#319](https://github.com/Wave-RF/WaveHouse/issues/319) fix that closes the query/stream row-level-security drift; roles without a filter keep the pure once-per-role fast path. `ReplayProjector` shares the same projection and per-connection row check for the handler's gap-fill, holding one policy snapshot per gap-fill and caching the per-table column-kind lookup across the replay loop. - **subscriber.go** — `Subscriber`, the per-connection handle. It carries the connection's JWT claims, fixed at construction (`NewSubscriber(claims, metrics)`, no setter) — the claims the `Hub` resolves a role's row-level `filter` against, and immutability is what makes the fan-out's unsynchronized claims read race-free structurally. It owns a single ready-to-write outbound queue of `Frame`s (each tagged with its `kind`, so the handler labels the write where it happens): producers — the keepalive wheel and the event `Hub` — fan frames in with `Send` (non-blocking; a full queue drops, and `Send` itself counts the drop by frame kind, so no producer can forget to), and the handler drains `Frames()` to the client verbatim. The queue is sized for buffering live events (cap 64, up from the keepalive-only cap 1; #152 will make it a knob), and an `Evicted()` channel is the seam the slow-consumer follow-up closes to disconnect a wedged consumer. - **bucket.go** — `Bucket`, the reusable fan-out primitive: a concurrency-safe set of subscribers. `Push` fans one `Frame` to every member fire-and-forget — the keepalive wheel's ring is its only caller now that both `Hub` paths iterate `Snapshot`, since the schema announcement is per connection even where the projection is shared per role; `Snapshot` exposes the members so the event `Hub` can evaluate row visibility per subscriber before sending (drop counting lives in `Send` itself). The `Hub` holds one `Bucket` per `(topic, role)` so a projected frame is built once and sent to every member instead of re-projected per subscriber. - **heartbeat.go** — The keepalive wheel (`Heartbeater`). A single process-wide ticker fans a minimal `:` comment across the ring of `Bucket`s, waking ~1/N of live streams per tick so the writes don't synchronize. The effective per-connection keepalive period is `stream.keepalive_interval` in the settings directory (the wheel ticks every `keepalive_interval ÷ keepalive_buckets`, so one rotation spans the interval; a reload calls `Reconfigure`, which rebuilds the ring in place with every live subscriber carried over); the owning handler goroutine does the actual write, so the shared ticker never touches a `ResponseWriter` directly. -- **metrics.go** — `Metrics`, the SSE instrument set: `wavehouse_sse_active_streams` (open streams), `wavehouse_sse_stream_duration_seconds` (lifetime), `wavehouse_sse_frames_sent_total` / `wavehouse_sse_bytes_sent_total` (labeled by `kind`: `keepalive`, `event`, `replay`, `schema`), `wavehouse_sse_dropped_frames_total` (frames dropped to a full subscriber queue — the slow-consumer signal that was silent before #294), and `wavehouse_sse_rows_withheld_total` (rows withheld from a subscriber by the row-level-security filter, labeled by table and role — the signal that separates "no matching rows" from "a fail-closed filter is withholding everything"). Nil-safe, so the handler holds one unconditionally and tests skip wiring it; one shared instance records the handler's write sites, each `Subscriber`'s queue-full drops (counted inside `Send`, by frame kind), and the `Hub`'s row-withheld counts. Separate from `observability.RegisterSystemMetrics`, which covers only the NATS/Pebble system gauges. Streams are observed through these metrics rather than per-event traces (the router excludes `/v1/stream` from the HTTP tracer). +- **metrics.go** — `Metrics`, the SSE instrument set: `wavehouse_sse_active_streams` (open streams), `wavehouse_sse_stream_duration_seconds` (lifetime), `wavehouse_sse_frames_sent_total` / `wavehouse_sse_bytes_sent_total` (labeled by `kind`: `keepalive`, `event`, `replay`, `schema`), `wavehouse_sse_dropped_frames_total` (frames dropped to a full subscriber queue — the slow-consumer signal that was silent before #294), and `wavehouse_sse_rows_withheld_total` (rows withheld from a subscriber by row-level security, labeled by `table`, `role`, and `reason` — `filter` (a definite non-match) vs. `error`/`decline`/`unavailable`/`drift` — the signal that separates "no matching rows" from "a fail-closed filter is withholding everything"). Nil-safe, so the handler holds one unconditionally and tests skip wiring it; one shared instance records the handler's write sites, each `Subscriber`'s queue-full drops (counted inside `Send`, by frame kind), and the `Hub`'s row-withheld counts. Separate from `observability.RegisterSystemMetrics`, which covers only the NATS/Pebble system gauges. Streams are observed through these metrics rather than per-event traces (the router excludes `/v1/stream` from the HTTP tracer). ### `auth/` — Authentication @@ -120,16 +121,25 @@ The SSE fan-out, factored out of `api/` so the delivery hot path ([#294](https:/ ### `discovery/` — Schema Discovery & Validation -- **discovery.go** — `SchemaRegistry` queries `system.columns` to discover ClickHouse table schemas, keeping each column's `default_kind` so `IsInsertable` / `InsertableColumns` / `InsertableColumnNames` (memoized per table at refresh) can decide the insertable subset the ingest envelope and the SSE announcement are both built from. Each refresh also records the server version (`SELECT version()`), joins `system.tables` for each table's `create_table_query` (kept in-process as `TableSchema.DDL` and marked `json:"-"` — an external-engine table renders its wiring in that statement — endpoint, bucket/host, database, username, access key id — so it must never reach `/v1/ops/schema`; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so what is withheld here is the topology), reads each column's `default_expression` and 1-based `position` alongside its type, discovers the server's default time zone (`SELECT timezone()`) and bakes every `DateTime`/`DateTime64` column's canonicalization spec (precision + resolved zone) into the cached schema, so the per-record ingest path parses no type strings and loads no zones ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). Supports periodic auto-refresh, on-demand refresh, and `RetryRefresh` (boot-time exponential backoff loop used by `cmd/wavehouse` so a transiently unreachable ClickHouse doesn't crash-loop the binary). Thread-safe via `sync.RWMutex`. -- **timestamp.go** — `CanonicalizeTimestamps(schema, data)` rewrites every parseable value in a top-level `DateTime`/`DateTime64` column to the canonical RFC 3339 UTC wire form before the event is published ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)): zone-less values are interpreted in the column's declared zone, else the discovered server default — ClickHouse's own rule, so the spelling changes but never the instant. Fail-open: an unparseable value or unresolvable zone passes through verbatim for ClickHouse's own parser to judge; ingest never rejects a record over its timestamp spelling. `Column.TimeParser()` exposes the same grammar as a value→instant parser (nil for a column with no resolved timestamp spec — a non-timestamp column, or one whose declared zone couldn't be loaded), which the stream row-filter uses so filter constants and canonicalized payloads can't disagree on the instant ([#381](https://github.com/Wave-RF/WaveHouse/issues/381)). -- **validation.go** — `Validate(schema, data)` checks incoming JSON against the discovered schema: unknown fields, type compatibility, missing required columns, null handling. Also exports the type classifiers `IsNumericType` / `IsStringType` and the storage-model classifier `NumericStorageOf` (all unwrapping `Nullable`/`LowCardinality`; the latter yields a numeric column's float width, `Decimal` scale, or integer exactness), which — together with `Column.TimeParser` from timestamp.go — seed the stream row-filter's `policy.ColumnSpec` comparison. -- **discovery_test.go** — Unit tests for validation logic. +- **discovery.go** — `SchemaRegistry` queries `system.columns` to discover ClickHouse table schemas, keeping each column's `default_kind`, `default_expression`, and 1-based `position` so `IsInsertable` / `InsertableColumns` / `InsertableColumnNames` (memoized per table at refresh) can decide the insertable subset the ingest envelope and the SSE announcement are both built from. Each refresh also records the server version (`SELECT version()`) and default time zone (`SELECT timezone()`, exposed via `ServerTimezone()`), and joins `system.tables` for each table's `create_table_query` (kept in-process as `TableSchema.DDL` and marked `json:"-"` — an external-engine table renders its wiring in that statement — endpoint, bucket/host, database, username, access key id — so it must never reach `/v1/ops/schema`; ClickHouse masks the password as `[HIDDEN]` from ~23.9, so what is withheld here is the topology). An `OnRefresh(func(serverVersion, serverTZ string, tables []*TableSchema))` hook fires synchronously right after the atomic swap on every successful refresh — `internal/typelayer.Engine.Bind` is its only registered consumer, and it is what resolves the chtypes artifact matching the connected server's line and recompiles per-table handles ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). Supports periodic auto-refresh, on-demand refresh, and `RetryRefresh` (boot-time exponential backoff loop used by `cmd/wavehouse` so a transiently unreachable ClickHouse doesn't crash-loop the binary). Thread-safe via `sync.RWMutex`. +- **discovery_test.go** — Unit tests for schema discovery. + +### `typelayer/` — In-Process ClickHouse Parser + +The only package that imports `github.com/wave-rf/chtypes/go/chtypes` (the sole exception: `cmd/wavehouse/main.go` references `typelayer` itself, never chtypes directly). It wraps one `chtypes.Registry` for the process, opened once at boot from a registry directory (`clickhouse.chtypes_registry` / `WH_CHTYPES_REGISTRY`; empty means the chtypes search path) — see [Deployment → chtypes artifacts](/deployment#chtypes-artifacts) for what ships where and how large it is. + +- **`Engine.Bind`** runs synchronously from `discovery.SchemaRegistry`'s `OnRefresh` hook: it resolves the artifact matching the connected server's **minor** version — never a nearest-version fallback, so an unmatched line leaves the engine `Unavailable` for every table — sets the process-global ClickHouse timezone from the discovered server timezone before the first library call (a later refresh reporting a *different* server timezone is a hard error: every table becomes `Unavailable` until the process restarts), and recompiles a handle per table whose column signature changed since the last bind. +- **`Engine.RoleTable(table, RoleShape)`** compiles the role's *own* schema — the columns it may insert, plus a `DEFAULT ''` on each `_eq` check column — and caches it per `(generation, shape)`. That is how column policy and auto-inject are answered without WaveHouse looking at a record: a denied column is simply not in the schema, so naming it is ClickHouse's code 117, and an absent check column takes the claim as its default while a supplied value still wins. +- **`Table.Ingest(format, body)`** runs the whole request body through ClickHouse's own reader in one call (`JSONEachRow`, `CSV`, `TSV`, `CSVWithNames` or `TSVWithNames`), with the parsing settings the worker's `INSERT` pins (`date_time_input_format=best_effort`, `input_format_null_as_default=1`) and `input_format_skip_unknown_fields=0`. It returns one verdict per input record — **accepted**, **rejected** with ClickHouse's real code and message, or **declined** (chtypes could not answer at all, a distinct condition never conflated with a rejection) — plus the accepted rows as `JSONCompactEachRow` bytes, exactly what ClickHouse's own writer produced: `DEFAULT`s evaluated, out-of-range integers wrapped, computed columns absent. +- **Predicates** compile through chtypes with every bound value as a `{pN:String}` parameter, never interpolated — on an integer column wrapped in the same strict round-trip cast (`chsql.StrictInt`) the query builder emits, so a claim that does not fit the column matches nothing instead of wrapping: `Table.Ingest` judges an ingest `check` in the same parse that validates the body, `Table.ParseRow` / `Row.Visible` judge a subscriber's row filter over one parsed event, cached per `(generation, expression, params)`. Only a definite true admits; a predicate error, a policy column the table no longer has, schema drift, or an unavailable engine all withhold (fail closed), counted in `wavehouse_sse_rows_withheld_total{table,role,reason}`. +- A table with no matching artifact for the server's line, or one caught mid a server-timezone change, returns an `Unavailable` error naming the cause: ingest maps it to `503`, the stream withholds every row for that table. + +See [API → Ingest](/api#post-v1ingesttabletable--ingest-data) for the ingest error-response shape and [Access Control → Where each rule is enforced](/access-control#where-each-rule-is-enforced) for how predicates are compiled and evaluated. ### `ingest/` — Ingest Pipeline, DLQ & Sweeping -- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a JetStream consumer reads from the `WAVEHOUSE` stream via a durable `buffer-consumer` pull subscription, batches events per table, and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, scope, received_timestamp, format, columns, row}` — the row positionally as one `JSONCompactEachRow` line, with `columns` naming its positions (the table's insertable columns — a computed one cannot be named in an `INSERT`); the worker batches per (table, column list) and writes `INSERT INTO … (cols) FORMAT JSONCompactEachRow`. It accepts any table name (the table name in the NATS subject is `query.SafeEncodeNATS(rawUnsafeTableName)`), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/ops/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy. On a bulk-insert failure the batch is re-inserted row by row; rows that succeed are acked, and only the rows that fail again are routed to the DLQ (`sendToDLQ`), which republishes the as-published `EventMessage` envelope to `dlq.{table}` NATS subjects with the failure context in `X-DLQ-*` headers when DLQ is enabled — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. -- **types.go** — `EventMessage` struct (TableName, Scope — reserved, always empty today, ReceivedTimestamp, Format, Columns, Row; `Format` is `FormatJSONCompactEachRow` and `Row` is one positional line whose slots `Columns` names) and `BufferConsumerName` constant, shared across API handlers and the ingest pipeline. -- **compact.go** — `EncodeCompactRow`, the positional row encoder every published row goes through, rendering one record over the table's **insertable** columns in declaration order. Serialization only: it validates nothing and judges no value. +- **worker.go** — `StartIngestWorker` launches an ingest pipeline: a JetStream consumer reads from the `WAVEHOUSE` stream via a durable `buffer-consumer` pull subscription, batches events per table, and performs bulk INSERTs to ClickHouse. The pipeline is **insert-only**. The wire format `EventMessage` carries `{table_name, scope, received_timestamp, format, columns, row}` — the row positionally as one `JSONCompactEachRow` line, with `columns` naming its positions (the table's **wire** columns — `internal/typelayer.Table.WireColumns`, the insertable subset minus any `MATERIALIZED`/`ALIAS`/`EPHEMERAL` column); the worker batches per (table, column list) and writes `INSERT INTO … (cols) FORMAT JSONCompactEachRow` with `internal/typelayer.InsertSettings()` (`date_time_input_format=best_effort`, `input_format_null_as_default=1`) plus `async_insert=0`, the same parsing settings chtypes compiled the row with. It accepts any table name (the table name in the NATS subject is `query.SafeEncodeNATS(rawUnsafeTableName)`), then bulk-INSERTs. The embedded NATS server runs with `DontListen: true` (`internal/mq/embedded.go`), so the only Publishers reachable on the `ingest.>` subjects are in-process Go code — today, only the HTTP `/v1/ingest?table={table}` handler. Non-insert mutations (`DELETE`/`UPDATE`/`TRUNCATE`/…) must go through `POST /v1/ops/query` under the admin role (`policy.admin_role`) — see the Query Path section below; the `/v1/ops/*` `RequireAdmin` middleware enforces the check at the API layer, so a no/invalid-token request (resolved to `default_role`, not admin in a production config) never reaches the proxy. On a bulk-insert failure the batch is re-inserted row by row; rows that succeed are acked, and only the rows that fail again are routed to the DLQ (`sendToDLQ`), which republishes the as-published `EventMessage` envelope to `dlq.{table}` NATS subjects with the failure context in `X-DLQ-*` headers when DLQ is enabled — see [Ingest Pipeline](/ingest-pipeline) for the worker internals. +- **types.go** — `EventMessage` struct (TableName, Scope — reserved, always empty today, ReceivedTimestamp, Format, Columns, Row; `Format` is `FormatJSONCompactEachRow` and `Row` is one positional line whose slots `Columns` names) and `BufferConsumerName` constant, shared across API handlers and the ingest pipeline. `Row` is now the exact bytes `internal/typelayer.Table.Ingest` returned for an accepted record — ClickHouse's own `JSONCompactEachRow` writer output, `DEFAULT`s already filled in — not a value WaveHouse encodes itself. - **sweeper.go** — `Sweeper` implements the Active Sweeper pattern. It runs every minute and purges NATS JetStream messages that are **both** ACKed by the buffer consumer (written to ClickHouse) **and** older than the configurable gap window. ### `mq/` — Message Queue @@ -148,12 +158,12 @@ The package's design invariants — stdout always 100%, WARN+ERROR always export ### `policy/` — Access Control -- **policy.go** — Hasura-style policy types, now **role-first**: `TablePolicy` is `map[string]RolePermissions`, and a role's grant carries a separate `SelectPermissions` and `InsertPermissions` — so a field only one side ever honored (`filter`, aggregations and the `max_*` limits on select; `check` on insert) does not exist on the other, and a document that puts one there fails the strict decode as an unknown key. `Evaluate()` resolves permissions against JWT claims (including `{{ jwt.claim.path }}` template resolution) for ONE operation, leaving the side it did not resolve **nil**. That is what the pointers buy: an *empty* side means "no restrictions" — what the admin return constructs on both sides — while a *nil* side means "you asked the wrong operation", and as value types the two were the same zero value. Every accessor fails closed on a nil side. The handful of bare field reads outside this package each sit past an accessor that denies an unresolved side first, so a nil `Select` never reaches one; if that ordering ever changed they would panic rather than silently widen. A nil guard that skips such a read must never be added, since an absent `WhereClause` is an unfiltered query. The per-column decision `IsColumnAllowed(col, insert)` takes the side it is being asked about, alongside its batch/projection forms `AllowedProjection()` and `RestrictsColumns()`, `IsAggregationAllowed()`, `CheckClauses()` — the write-side accessor for the one consumer that iterates a side's map instead of asking about a column, whose `ok=false` a caller must treat as *refuse the write*, never as *no checks to run* — `resolvePredicates()` — the one resolution both read surfaces render from, so the SQL `WHERE` and the in-memory row check cannot drift — and `Validate()`, split into `validateSelectPerms`/`validateInsertPerms` and run from `settings.Validate` on every adoption, which is where the rules in [Access Control](/access-control) are actually enforced. -- **rowfilter.go** — the in-memory row-visibility twin of the SQL `WHERE`: `HasRowFilter`, `RowVisible` (evaluates the resolved predicates against a decoded event, per subscriber), and `ColumnSpec` — the per-column comparison contract (`ColumnKind` `Numeric`/`Text`/`Time`/`Opaque`, plus each kind's parameters: the caller-supplied instant parser for `Time`, the `NumericSpec` storage model for `Numeric`) whose zero value is the fail-closed floor: numeric columns compare in the column's **storage domain** (operands rendered by canonical.go, compared by numeric.go — next two bullets), `String` bytewise, `DateTime`/`DateTime64` chronologically (both operands through the ingest grammar; either side unreadable ⇒ withheld), and everything else (including any column with no usable schema) admits byte-equality only, failing `!=`/`>`/`<` closed. Both `HasRowFilter` and `RowVisible` fail closed on a denied or unresolved grant: `HasRowFilter` is the gate in front of `RowVisible`, so it must answer *true* there or the whole-bucket fast path skips the check entirely. -- **canonical.go** — the one rendering layer for comparison operands: every value a `filter` or `check` compares — a JWT claim (`CanonicalScalar`), a policy-authored literal (`CanonicalNumericLiteral`), an ingested payload value (`numericCanonical`) — converges on one exact canonical decimal form (positional, digit-bounded, never a float64 round-trip), so what a read filter binds and what the stream compares can't drift; `scalarString` is the deliberate exception, the raw byte rendering that `Text`/`Opaque` equality compares. -- **numeric.go** — compares canonical forms the way the column that stores them would: `compareCanonicalDecimals` orders by exact digit-string arithmetic, and `NumericSpec` first narrows both operands the way ClickHouse narrows the stored value and the bound constant — `Float32`/`Float64` width rounding, `Decimal` scale truncation, integers exact at any width, with an operand outside the column's width or a `Decimal`'s precision budget refused rather than modeled; the `tests/integration` differential oracle holds in-range verdicts equal to a live ClickHouse's and asserts the never-admit-where-SQL-hides direction for the refused out-of-range operands. +- **policy.go** — Hasura-style policy types, now **role-first**: `TablePolicy` is `map[string]RolePermissions`, and a role's grant carries a separate `SelectPermissions` and `InsertPermissions` — so a field only one side ever honored (`filter`, aggregations and the `max_*` limits on select; `check` on insert) does not exist on the other, and a document that puts one there fails the strict decode as an unknown key. `Evaluate()` resolves permissions against JWT claims (including `{{ jwt.claim.path }}` template resolution) for ONE operation, leaving the side it did not resolve **nil**. That is what the pointers buy: an *empty* side means "no restrictions" — what the admin return constructs on both sides — while a *nil* side means "you asked the wrong operation", and as value types the two were the same zero value. Every accessor fails closed on a nil side. The handful of bare field reads outside this package each sit past an accessor that denies an unresolved side first, so a nil `Select` never reaches one; if that ordering ever changed they would panic rather than silently widen. A nil guard that skips such a read must never be added, since a skipped `WhereSQL` is an unfiltered query. The per-column decision `IsColumnAllowed(col, insert)` takes the side it is being asked about, alongside its batch/projection forms `AllowedProjection()` and `RestrictsColumns()`, `IsAggregationAllowed()`, `CheckClauses()` — the write-side accessor for the one consumer that iterates a side's map instead of asking about a column, whose `ok=false` a caller must treat as *refuse the write*, never as *no checks to run* — `resolvePredicates()` and its exported `Predicates()` accessor — the one resolution every read surface renders from, so the SQL `WHERE` (rendered by `ResolvedSelect.WhereSQL`, which takes the column types so an integer column's claims bind through `chsql.StrictInt`), and the `internal/typelayer` row-level-security engine that replaced the in-memory evaluator below, cannot drift apart — and `Validate()`, split into `validateSelectPerms`/`validateInsertPerms` and run from `settings.Validate` on every adoption, which is where the rules in [Access Control](/access-control) are actually enforced. +- **canonical.go** — the one rendering layer for policy comparison operands: every JWT claim value (`CanonicalScalar`) is rendered into one exact canonical decimal form (positional, digit-bounded, never a float64 round-trip) before it reaches a `filter`/`check` predicate, so every comparison surface binds the same value the same way, and a null/object/array claim fails closed. A policy-authored literal is *not* re-rendered — it binds exactly as written, and a spelling the column cannot read is ClickHouse's own type error at evaluation time on both surfaces. - **source.go** — `Source`, a `func() *Policy` every consumer (the auth middleware, ingest, structured query, pipes, the stream hub, the `/v1/ops` gate) reads per call, so a settings reload applies to the very next request. In production it is `settings.Store.Policy`; `Static(p)` fixes one for tests. A `nil` result is a deliberate lockout. +Predicate *evaluation* lives elsewhere: `internal/typelayer` compiles a role's resolved predicates through chtypes and evaluates them — `Table.ParseRow` / `Row.Visible` for a streamed event, `Table.Ingest` (a row filter inside the validating parse) for an ingest `check` — with ClickHouse's own comparison semantics for every column type. `policy` only resolves the values both the SQL path and that engine bind. + ### `pipes/` — Named Query Pipes - **pipes.go** — `NamedQuery` type with SQL template and parameter definitions, and `Source` (`Pipe(name)` / `Pipes()`), read per request — `settings.Store` in production (`pipes.json`), `Static(q...)` in tests. `BindParams()` resolves `{{param}}` / `{{param:default}}` placeholders by inlining escaped literal values into the SQL (strings single-quote-escaped; arrays rendered as escaped `(…)` `IN`-lists). A non-scalar value with no SQL form (a JSON object, or an empty array) is rejected rather than emitted raw. @@ -161,7 +171,7 @@ The package's design invariants — stdout always 100%, WARN+ERROR always export ### `query/` — Structured Query Engine - **ast.go** — `StructuredQuery` AST types: columns, aggregations, filters, group by, order by, limit, time range. -- **builder.go** — `Build()` converts AST to parameterized SQL. It is the single chokepoint that validates every referenced identifier against the schema **and** authorizes every column reference — projection, aggregation args, filters, group_by, order_by, time_range — against the role's column allowlist (the [#223](https://github.com/Wave-RF/WaveHouse/issues/223) hard cap). A full-row read is requested with `select_all`, which expands to the role's allowed columns rather than emitting a raw `SELECT *`; an omitted projection selects nothing, and `*` in `columns` is a literal column name. Every identifier is backtick-quoted via `internal/chsql` (`QuoteIdent`) so any ClickHouse-legal name is accepted — a name containing `?` is refused fail-closed ([#279](https://github.com/Wave-RF/WaveHouse/issues/279)). The role's row-level-security predicate and `max_rows` cap are emitted by `Build()` itself, as part of the WHERE and LIMIT assembly — policy SQL is never spliced into rendered text ([#322](https://github.com/Wave-RF/WaveHouse/issues/322)). Timestamp bucketing for cache optimization. +- **builder.go** — `Build()` converts AST to parameterized SQL. It is the single chokepoint that validates every referenced identifier against the schema **and** authorizes every column reference — projection, aggregation args, filters, group_by, order_by, time_range — against the role's column allowlist (the [#223](https://github.com/Wave-RF/WaveHouse/issues/223) hard cap). A full-row read is requested with `select_all`, which expands to the role's allowed columns rather than emitting a raw `SELECT *`; an omitted projection selects nothing, and `*` in `columns` is a literal column name. Every identifier is backtick-quoted via `internal/chsql` (`QuoteIdent`) so any ClickHouse-legal name is accepted — a name containing `?` is refused fail-closed, because `Build` emits positional placeholders that a later pass rewrites to ClickHouse's named parameters ([#279](https://github.com/Wave-RF/WaveHouse/issues/279)). The role's row-level-security predicate and `max_rows` cap are emitted by `Build()` itself, as part of the WHERE and LIMIT assembly — policy SQL is never spliced into rendered text ([#322](https://github.com/Wave-RF/WaveHouse/issues/322)). Timestamp bucketing for cache optimization. ### `settings/` — Settings Directory @@ -179,7 +189,7 @@ The hot-reloadable half of configuration: a directory of four JSON files (`confi ### `chsql/` — ClickHouse SQL Helpers -- **chsql.go** — Dependency-free ClickHouse SQL helpers shared by `query/` and `policy/`, kept in their own package to break an import cycle. `QuoteIdent` is the single place every identifier — column, table, alias — becomes SQL text: always backtick-quoted and escaped, so any ClickHouse-legal name (dots, spaces, unicode, keywords) is safe. `BindUnsafe` reports whether a name contains a literal `?`, which would desync clickhouse-go's positional binder; such names are rejected fail-closed rather than silently mis-bound. +- **chsql.go** — Dependency-free ClickHouse SQL helpers shared by `query/` and `policy/`, kept in their own package to break an import cycle. `QuoteIdent` is the single place every identifier — column, table, alias — becomes SQL text: always backtick-quoted and escaped, so any ClickHouse-legal name (dots, spaces, unicode, keywords) is safe. `BindUnsafe` reports whether a name contains a literal `?`, which would desync the positional-to-named parameter rewrite; such names are rejected fail-closed rather than silently mis-bound. `IntegerType` and `StrictInt` pick and render the strict round-trip cast a policy claim is compared through on an integer column, identically for the query builder and the type layer, so a claim that does not fit the column's type is NULL and matches nothing instead of wrapping. ## Data Flows @@ -194,16 +204,25 @@ Client POST /v1/ingest?table={table} unsupported, or if declarations disagree; before the body is read) → Read the whole body into a pooled buffer, bounded by the 16 MiB cap (413 before any record is processed, so nothing is published) - → Validate JSON body against schema (type checks, required columns) - → Policy column rules + check clauses (disallowed columns rejected; - claim-derived values enforced or injected) - → Canonicalize top-level DateTime/DateTime64 column values to RFC 3339 UTC - (rewrites the payload so every consumer shares one spelling; fail-open — - an unparseable value passes through verbatim for ClickHouse's parser to judge) - → Optional deduplication check (configurable ID field; a row missing that - field is published un-deduped + logged/counted, or rejected under require_id) - → Publish to NATS JetStream (ingest.{table}) - → 200 OK returned immediately + → Compile the ROLE's schema: the columns it may insert, plus a DEFAULT per + _eq check column carrying the claim (cached per generation+shape) + → Validate the whole body through chtypes (internal/typelayer), one call per + request: ClickHouse's own parser type-checks, coerces, and fills DEFAULTs + (including now()) per record. A rejected record carries ClickHouse's real + error code and message — an unknown field, a MATERIALIZED/ALIAS/EPHEMERAL + column, or a column this role may not write is 117; a record chtypes + cannot answer for is a distinct "declined" outcome (422), never a data + rejection. No artifact matching the server's line, or a server timezone + change since boot, fails the whole table closed (503) + → Evaluate the role's check clauses over the accepted rows with one compiled + chtypes filter — false is 403 for that record, unevaluable is 422 + → Optional deduplication check, on records chtypes accepted (configurable + ID field; a row missing that field is published un-deduped + logged/ + counted, or rejected under require_id) — deliberately after validation, + so a chtypes-rejected record is never marked seen + → Publish the ClickHouse-rendered row (DEFAULTs filled, MATERIALIZED/ + ALIAS/EPHEMERAL columns absent) to NATS JetStream (ingest.{table}) + → 200 OK returned immediately, per-record outcomes in the response body → (If NATS stream is full: 503 + Retry-After header) Ingest worker pipeline (StartIngestWorker): @@ -213,8 +232,10 @@ Ingest worker pipeline (StartIngestWorker): DLQ, or acked-and-dropped where the DLQ is off for the table; either way counted by wavehouse_ingest_poison_total under its disposition) → Batch events per table, bulk INSERT to ClickHouse - (INSERTs pin date_time_input_format=best_effort — the server default since - ClickHouse 26.5; see /ingest-pipeline for the basic-vs-best_effort divergence) + (INSERTs pin the same parsing settings chtypes compiled the row with — + date_time_input_format=best_effort, input_format_null_as_default=1 — + plus async_insert=0, so per-row error attribution isn't lost to an + async flush wait; see /ingest-pipeline for detail) → On success: DoubleAck messages → On failure: re-insert row by row; each row that fails again → DLQ output (dlq.{table}), then Ack to prevent infinite retry @@ -273,7 +294,7 @@ Client POST /v1/ops/query (browser, CDN, corp proxy) caches the result. ``` -The proxy-pattern wins are: zero classification logic on the WaveHouse side (no isMutation heuristic to maintain), and any ClickHouse statement type — including verbs added in future versions and inline FORMAT overrides — works without WaveHouse code changes. Multi-statement input (`SELECT 1; TRUNCATE t`) is supported when the upstream ClickHouse has multi-query enabled, which is the default on recent versions; older or restrictively-configured servers will return a clear error from ClickHouse itself for the second statement. The proxy buffers the response in memory with a 64 MiB cap (502 with `clickhouse response exceeded N bytes` on overflow, to keep a runaway `SELECT *` from pinning RAM on the API server), and passes ClickHouse's `Content-Type` through when an inline `FORMAT` directive overrides the default JSON envelope. The structured query endpoint and pipes still go through `clickhouse-go`'s native driver (Query/Exec) for performance and to keep the cached row-array shape consistent. +The proxy-pattern wins are: zero classification logic on the WaveHouse side (no isMutation heuristic to maintain), and any ClickHouse statement type — including verbs added in future versions and inline FORMAT overrides — works without WaveHouse code changes. Multi-statement input (`SELECT 1; TRUNCATE t`) is supported when the upstream ClickHouse has multi-query enabled, which is the default on recent versions; older or restrictively-configured servers will return a clear error from ClickHouse itself for the second statement. The proxy buffers the response in memory with a 64 MiB cap (502 with `clickhouse response exceeded N bytes` on overflow, to keep a runaway `SELECT *` from pinning RAM on the API server), and passes ClickHouse's `Content-Type` through when an inline `FORMAT` directive overrides the default JSON envelope. The structured query endpoint and pipes reach ClickHouse the same way, over its HTTP interface, but ask for `default_format=JSONEachRow` and bind their values as named `{pN:String}` parameters. ClickHouse renders the JSON; WaveHouse frames the lines into an array and caches those bytes, so no Go-side row conversion sits between the server and the response. ### Streaming Path @@ -298,7 +319,8 @@ Client GET /v1/stream → fan the finished frame to every Subscriber of that (topic, role); a role carrying a row-level filter delivers per subscriber instead: the shared frame goes only to subscribers whose JWT claims admit - the row (RowVisible) + the row (internal/typelayer's Row.Visible, compiled and evaluated by + the same engine as the server's WHERE clause) → a connection whose column list drifts (a schema change mid-stream) is re-announced before the next row, per connection → Live and replay track that drift in SEPARATE state and do not reconcile @@ -310,7 +332,9 @@ Client GET /v1/stream under the wrong names until it reconnects → Handler drains keepalives + event frames from one byte-pump → client → Policy filtering (historical + live): denied tables skipped, denied - columns stripped, row filter evaluated per subscriber against claims. + columns stripped, row filter compiled and evaluated per subscriber + against claims by internal/typelayer (fail closed on a compile/parse + error, a missing policy column, schema drift, or an unavailable engine). Column projection runs once per role (Hub.Broadcast) — per-subscriber work only where a row filter makes visibility per-connection; replay shares the same column policy + row check but projects per-connection @@ -320,7 +344,7 @@ Client GET /v1/stream | Component | Technology | Purpose | | --------- | ---------- | ------- | -| Language | Go 1.26 | Core runtime | +| Language | Go 1.27 | Core runtime | | HTTP Router | Chi v5 | Request routing and middleware | | Authentication | golang-jwt v5 + keyfunc v3 | JWT (HMAC + JWKS) parsing and validation | | Analytics DB | ClickHouse | Primary data store + schema source of truth | @@ -328,5 +352,6 @@ Client GET /v1/stream | L1 Cache | Ristretto v2 | In-process memory cache | | Embedded KV | Pebble | Optional deduplication | | Config | cleanenv | YAML + env var config loading | -| Release | GoReleaser | Cross-platform binary builds | -| Containers | Docker (distroless) | Minimal production images | +| Type engine | [chtypes](https://github.com/wave-rf/chtypes) (cgo/dlopen) | In-process ClickHouse parser: ingest validation/coercion + row-level-security compilation | +| Release | GoReleaser | Binary builds for Linux amd64/arm64 and macOS arm64 | +| Containers | Docker (`distroless/cc`, glibc) | Minimal production images | diff --git a/docs/src/content/docs/configuration.mdx b/docs/src/content/docs/configuration.mdx index 791f15ba..f1148c7a 100644 --- a/docs/src/content/docs/configuration.mdx +++ b/docs/src/content/docs/configuration.mdx @@ -53,6 +53,7 @@ Only the secret is boot config. The wiring — native address, HTTP port and sch | YAML Key | Env Var | Default | Description | | --- | --- | ------- | ----------- | | `clickhouse.password` | `WH_CH_PASSWORD` | *(empty)* | Authentication password, combined with the settings directory's `clickhouse.username` on every (re)connect. A secret, so it never lives in a tracked JSON file; rotating it is a restart. | +| `clickhouse.chtypes_registry` | `WH_CHTYPES_REGISTRY` | *(empty)* | Directory holding the chtypes artifacts (one `/` per ClickHouse line). Empty defers to the chtypes search path — `$CHTYPES_REGISTRY`, `~/.cache/chtypes/artifacts/abi6/-`, then the system directories. Either way a library is opened lazily, on first use of its line; an explicit directory is searched first, then the rest of the path. Boot-tier: changing it is a restart. See [chtypes artifacts](/deployment#chtypes-artifacts). | ### Server-side resource limits diff --git a/docs/src/content/docs/deployment.md b/docs/src/content/docs/deployment.md index 1a478570..72247467 100644 --- a/docs/src/content/docs/deployment.md +++ b/docs/src/content/docs/deployment.md @@ -7,11 +7,11 @@ sidebar: order: 10 --- -How to run WaveHouse in production — single binary, Docker images, releases, health checks, and the required ClickHouse schema. +How to run WaveHouse in production — one binary, Docker images, releases, health checks, and the required ClickHouse schema. -## Single binary +## One binary, plus a per-version artifact -WaveHouse runs as one process with embedded NATS and optional Pebble dedup. The only external dependency is ClickHouse. +WaveHouse runs as one process with embedded NATS and optional Pebble dedup. At start it also loads a second artifact — a per-ClickHouse-version shared library (`internal/typelayer`, via [chtypes](#chtypes-artifacts)) that runs ClickHouse's own parser in-process for ingest validation and row-level security. The only external network dependency is ClickHouse. ### Quick Start with Docker Compose @@ -78,7 +78,7 @@ docker build -f deployments/Dockerfile -t wavehouse:latest . This builds the runtime image `wavehouse:latest`. (The published `ghcr.io` images are built by GoReleaser from `deployments/Dockerfile.goreleaser`, not this command — see Registry below.) -All images use multi-stage builds (Go Alpine builder → distroless runtime) for minimal attack surface. +All images use multi-stage builds (`golang:1.27-bookworm` glibc builder → `gcr.io/distroless/cc-debian12` runtime — cgo needs glibc, so the previous Alpine/musl builder and `distroless/static` runtime no longer work) for minimal attack surface. The build also fetches the [chtypes artifact(s)](#chtypes-artifacts) pinned in `chtypes.lock` into the image. ### Registry @@ -104,18 +104,48 @@ gh attestation verify oci://ghcr.io/wave-rf/wavehouse:vX.Y.Z \ --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml ``` +## chtypes artifacts + +**What it is.** WaveHouse validates ingest data, coerces types, substitutes `DEFAULT`s, and evaluates row-level security by running ClickHouse's own parser in-process, through [chtypes](https://github.com/wave-rf/chtypes) (`internal/typelayer`, `github.com/wave-rf/chtypes/go` v0.4.0). The parser itself ships as a shared library (`libchtypes.so` / `.dylib`) built per **ClickHouse minor line** (e.g. `26.6`) and per platform — WaveHouse loads the artifact matching the connected server's minor version at boot; there is no nearest-version fallback. If the connected server's line has no matching artifact, or the server's reported timezone changes after boot, ingest and streaming for the affected tables fail closed with `503` / a withheld row — see [API → Ingest error responses](/api#error-responses) and [Access Control → Where each rule is enforced](/access-control#where-each-rule-is-enforced). + +**Where it lives.** WaveHouse looks for the artifact in a registry directory, in order: an explicit `clickhouse.chtypes_registry` (`WH_CHTYPES_REGISTRY`) if set, then chtypes' own default search path — `$CHTYPES_REGISTRY`, the per-user cache `~/.cache/chtypes/artifacts/abi6/-` (one directory per SDK ABI revision, so an older SDK's downloads are never picked up), then the system directories `/usr/local/share/chtypes/artifacts/` and `/opt/chtypes/artifacts/`. WaveHouse does not autofetch on a miss in production — an unmatched line is a boot-time or refresh-time failure, not a background download. + +**Size.** Each artifact is roughly 160–290 MB on disk; a running process holding several loaded versions (e.g. across a rolling ClickHouse upgrade) costs roughly 120 MB of resident memory per loaded version (the chtypes multi-version guide's figure; a library is opened on first use of its line, not at registry construction). + +**Docker images** ship the artifact(s) baked in: the image build fetches whatever `chtypes.lock` names (see below), so a container never needs network access to ClickHouse's artifact store at runtime. `WH_CHTYPES_REGISTRY` (default `/opt/chtypes/artifacts`) points at the directory inside the image. + +**Release archives and `go install` / building from source** do not carry or fetch an artifact — only the Docker images bake one in. See the [README's `go install` caveat](https://github.com/Wave-RF/WaveHouse#c-go-install-binary-no-docker). Fetch one yourself before first run: + +```bash +scripts/fetch-chtypes.sh # wraps: go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch --frozen --lock chtypes.lock 26.6 +``` + +or, for a line not in the repo's lock file: + +```bash +go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch +``` + +### Pinning with `chtypes.lock` + +`chtypes.lock`, checked in at the repo root, records the exact artifact file and SHA-256 per platform/line the project builds and tests against. CI restores from it with `--frozen` (refusing anything the lock doesn't name) rather than fetching the rolling artifact release, so a pipeline never silently starts testing a new build. Refresh it deliberately — `go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch --lock chtypes.lock --platform `, once per platform (`darwin-arm64`, `linux-amd64`, `linux-arm64`), without `--frozen` — and commit the result; don't regenerate it implicitly. + +A lock is specific to the SDK's ABI revision (6 at v0.4.0): the fetcher never selects a build from another revision, so after an SDK bump that changes the revision, `--frozen` fails (`CHTYPES_ARTIFACT_PINNED` or `CHTYPES_ARTIFACT_UNPUBLISHED`) until the lock is regenerated the same way, and the CI cache key and path (`abi6`) move with it. + ## Releases Releases are built with [GoReleaser](https://goreleaser.com/). The configuration is in `.goreleaser.yaml`. The release archives attached to each GitHub Release carry a signed [Sigstore](https://www.sigstore.dev/) build-provenance attestation — verify a downloaded archive with `gh attestation verify --repo Wave-RF/WaveHouse --signer-workflow Wave-RF/WaveHouse/.github/workflows/release.yml`. (This covers the prebuilt archives, not `go install`, which compiles from source.) ### Supported Platforms +The binary requires cgo (dlopen only — no static link to the chtypes artifact) and glibc, which sets the supported platform matrix: + | OS | Architecture | | -- | ----------- | | Linux | amd64, arm64 | -| macOS | amd64, arm64 | -| Windows | amd64, arm64 | -| FreeBSD | amd64, arm64 | +| macOS | arm64 only | + +Windows, FreeBSD, and darwin/amd64 are no longer built — there is no chtypes artifact for them, and the binary cannot run without one. If you need one of these, [open an issue](https://github.com/Wave-RF/WaveHouse/issues) describing your use case. ### Creating a Release @@ -318,7 +348,7 @@ WaveHouse serves plain HTTP on `:8080` and does **not** terminate TLS, manage ce WaveHouse uses a **Bring Your Own Schema** model. You create your tables in ClickHouse with whatever columns and engines you need. WaveHouse discovers the schemas automatically via `system.columns` and validates ingest data against them — see [Schema Validation](/api#post-v1ingesttabletable--ingest-data) for the rules a record must satisfy. -Three schema-design consequences are worth knowing before you write the DDL. A `MATERIALIZED` or `ALIAS` column is computed by ClickHouse and cannot be inserted: omit it from your records, and a record that names one is rejected. An `EPHEMERAL` column is the awkward one — it *is* insertable, but it is never stored and no query can read it back, so it is only useful as an input to another column's `DEFAULT` expression, and a policy `check` naming one is refused outright. And a `Nullable(T) DEFAULT …` column never takes its default through ingest: an omitted key stores `NULL`, not the default — see [the journey of one event](/ingest-pipeline#the-journey-of-one-event) for why. A **non-nullable** column with a default is unaffected. +Two schema-design consequences are worth knowing before you write the DDL. A `MATERIALIZED`, `ALIAS`, or `EPHEMERAL` column is never part of a published row: WaveHouse's ingest validation runs ClickHouse's own parser in-process (via [chtypes](#chtypes-artifacts)), and a record that names one is rejected with ClickHouse's own code (117) rather than published. An omitted column — on any table — takes its `DEFAULT` expression, or the type's implicit zero value where none is declared, evaluated by that same parser before the row is published; there is no longer a positional-encoding quirk that stores `NULL` on a `Nullable(T) DEFAULT …` column instead — see [the journey of one event](/ingest-pipeline#the-journey-of-one-event) for detail. Example table: @@ -344,7 +374,7 @@ On the worker side the outcome depends on the DLQ. **With the DLQ enabled for th Three audits belong **before** the drain, because none of them announces itself afterwards: -- **`Nullable(T) DEFAULT …` columns now store `NULL` where they took their default.** A positional row has one slot per insertable column and no way to say *absent*, so a key the record omits rides as an explicit `null`. `input_format_null_as_default=1` turns that back into the default for a **non-nullable** column, but ClickHouse stores `NULL` on a nullable one whatever the setting says — only an absent key ever took the default. Following this runbook exactly still changes what lands in those columns, silently. See [the ingest note](/ingest-pipeline#the-journey-of-one-event). +- **The wire envelope's `row` is positional** (`columns` names each slot), so a message from before this migration and one from after it look the same shape-wise; what changed underneath is how an omitted field is resolved into that slot — see [the ingest note](/ingest-pipeline#the-journey-of-one-event) for the current behavior. - **Policy `check` blocks are now validated against the table.** A `check` naming a column the table lacks, one it computes (`MATERIALIZED`/`ALIAS`), or an `EPHEMERAL` one is a per-record `403` on *every* insert by that role. `wavehouse validate` cannot catch it — it never sees the ClickHouse schema — so audit them against their tables first. See [Access control → Insert checks](/access-control#insert-checks). - **Every `WH_*` variable the binary does not bind refuses boot.** The old binary ignored a variable it did not read; the new one names every unbound one and exits before it opens the queue, so a pod spec or compose file that still carries one comes back from the upgrade as a container that will not start. Diff the environment against the [Configuration Reference](/configuration) first: a `WH_*` variable that is not in its tables is unbound, and whatever it used to configure now lives in the [settings directory](/settings-directory) or is gone. A Kubernetes Service in the pod's namespace named `wh` or `wh-*` counts too: it injects link variables under the `WH_` prefix (`WH_SERVICE_HOST` and `WH_PORT` for `wh`, `WH_FOO_SERVICE_HOST` and `WH_FOO_PORT` for `wh-foo`), so set `enableServiceLinks: false` on the pod spec. diff --git a/docs/src/content/docs/development.md b/docs/src/content/docs/development.md index e550024b..44040038 100644 --- a/docs/src/content/docs/development.md +++ b/docs/src/content/docs/development.md @@ -13,7 +13,7 @@ You need these on your `PATH` before any `make` recipe will work end-to-end: | Tool | Required version | Why | Install | | ---- | ---------------- | --- | ------- | -| **Go** | 1.26+ (matches `go.mod`) | Compiles `cmd/wavehouse`; also runs the pinned `tool` deps (`gotestsum`, `gofumpt`, `goimports`, `govulncheck`, `deadcode`, `gsa`, `goda`) via `go tool` | [go.dev/dl](https://go.dev/dl/) | +| **Go** | 1.27+ (matches `go.mod`) | Compiles `cmd/wavehouse` with cgo enabled (needed by chtypes' dlopen shim — a C toolchain and glibc must be present); also runs the pinned `tool` deps (`gotestsum`, `gofumpt`, `goimports`, `govulncheck`, `deadcode`, `gsa`, `goda`) via `go tool` | [go.dev/dl](https://go.dev/dl/) | | **GNU Make** | **4.0+** | The Makefile uses `--output-sync=target` (Make 4 only) and bash-pinned recipes. macOS ships with BSD Make 3.81, which **will not work** | macOS: `brew install make` then use `gmake` or put `$(brew --prefix make)/libexec/gnubin` on your PATH. Linux: usually already installed | | **bash** | 4+ recommended | Recipes are pinned to `bash`; the helper scripts under `scripts/` use `set -euo pipefail` and bash arrays | macOS default is bash 3.2 (works for current recipes, but `brew install bash` is safer); Linux distros ship 4+ | | **Docker** *(or Podman)* | Engine 20.10+ with the Compose **v2** plugin (`docker compose`, no hyphen) | Compose stacks under `deployments/compose/`; the E2E and integration suites boot ClickHouse via testcontainers (no compose file) | [Docker Desktop](https://docs.docker.com/get-docker/), [colima](https://github.com/abiosoft/colima), or [Podman](https://podman.io) with `podman-compose` / the `podman compose` plugin. The testcontainers Go library also honors `DOCKER_HOST` for rootless Podman setups | @@ -21,6 +21,16 @@ You need these on your `PATH` before any `make` recipe will work end-to-end: | **pnpm** | 11.21+ (pinned via `packageManager` in the root `package.json`) | Package manager for the TypeScript SDK, E2E test harness, and docs site (managed as a single pnpm workspace from the repo root); `make build-ts`, `make test-ts`, `make test-e2e`, `make build-docs`, `make dev-docs`, `make preview-docs` all shell out to `pnpm` | `corepack enable && corepack prepare pnpm@11.21.0 --activate` (recommended), or `npm i -g pnpm` | | **git** + **curl** | any recent | `git` for source + version metadata in builds; `curl` is used by the Makefile to fetch the pinned `golangci-lint` binary into `.bin/` | usually preinstalled | +### The chtypes artifact — fetch it once per machine + +`internal/typelayer` loads a per-ClickHouse-version shared library at start to run ingest validation and row-level security through ClickHouse's own parser (see [Deployment → chtypes artifacts](/deployment#chtypes-artifacts)). It is not source code and `make tools` does not fetch it for you — pull it once with: + +```bash +scripts/fetch-chtypes.sh # wraps: go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch --frozen --lock chtypes.lock 26.6 +``` + +It lands in the default local cache (`~/.cache/chtypes/artifacts/abi6/-`, one directory per SDK ABI revision) and is 160–290 MB — expect the first run to take a minute or two. Without it, `make dev` / `make test` / `make test-e2e` fail closed (a `503` on ingest, every stream row withheld) until a matching artifact exists for the ClickHouse line the tests or your local server run against. + ### Auto-installed by `make tools` Run `make tools` once after cloning to populate everything that doesn't have to be on your PATH: @@ -34,7 +44,7 @@ Run `make tools` once after cloning to populate everything that doesn't have to ### Verify your setup ```bash -go version # go1.26+ +go version # go1.27+ make --version # GNU Make 4.x docker compose version node --version # v22.x (matches .nvmrc and CI) @@ -541,10 +551,9 @@ Run `make help` to see all targets. Key ones: | `make release-sdk-go VERSION=X.Y.Z` | Tag a Go SDK release — `go get` (pending [#434](https://github.com/Wave-RF/WaveHouse/pull/434)) | | **Analysis** (informational, not in CI) | | | `make size` | Binary size analysis → `tmp/analysis/` (text + SVG + interactive HTML) | -| `make audit-cgo` | Audit dependency tree for C files (builds use `CGO_ENABLED=0`) | | `make deadcode` | Find unreachable functions | | `make dep-cut` | Top cuttable deps by transitive weight (`LIMIT=N` to override) | -| `make binary-analysis` | Combined: `size` + `audit-cgo` + `deadcode` | +| `make binary-analysis` | Combined: `size` + `deadcode` | | **Cleanup** (tiered — compose explicitly for partial resets) | | | `make clean` | Build outputs only (`bin/`, `dist/`, `clients/ts/dist/`, `docs/dist/`, `docs/.dev-dist/`) | | `make clean-test` | Test outputs only (`tmp/` — coverage data, logs, NATS state) | @@ -613,7 +622,7 @@ The **tag is the version**, everywhere: | Component | Where the version comes from | | --- | --- | -| Server | GoReleaser's `-ldflags` at build time, from the tag | +| Server | GoReleaser's `-ldflags` at build time, from the tag (`goreleaser build --single-target`, once per platform) | | Go SDK | The tag itself — a Go module has no version file | | TypeScript SDK | `publish-npm.yml` stamps `clients/ts/package.json` from the tag before publishing | @@ -633,7 +642,7 @@ Tag globs are anchored at the start of the ref name, so `v*` never matches a `cl ### What a release publishes -- **Server —** a **GitHub Release** with the cross-compiled archives (linux/darwin/windows/freebsd × amd64/arm64; `.zip` on Windows, `.tar.gz` elsewhere) and `checksums.txt`. A tag carrying a prerelease suffix (`v0.1.0-alpha.1`) is marked as a GitHub pre-release, so it never takes the "Latest release" badge from a shipped stable version. +- **Server —** a **GitHub Release** with archives for the three supported platforms (linux/amd64, linux/arm64, darwin/arm64 — cgo's dlopen requirement and the lack of a chtypes artifact elsewhere dropped Windows, FreeBSD, and darwin/amd64; see [Deployment → Supported Platforms](/deployment#supported-platforms)), each `.tar.gz`, and `checksums.txt`. A tag carrying a prerelease suffix (`v0.1.0-alpha.1`) is marked as a GitHub pre-release, so it never takes the "Latest release" badge from a shipped stable version. - **Both —** **release notes generated by GitHub** from the PRs merged since the previous tag *in the same family* — one line per PR, since `main` is squash-merged, grouped into the categories defined in [`.github/release.yml`](https://github.com/Wave-RF/WaveHouse/blob/main/.github/release.yml). Grouping is by **PR label**: `github_actions` / `documentation` are applied automatically by `actions/labeler`, but `breaking-change`, `security`, `bug`, and `enhancement` are applied by hand — an unlabelled PR lands in "Other changes". Dependabot is split out by **author** rather than by label, because the labels `actions/labeler` applies by path — `github_actions`, `documentation` — mark our own PRs too; our CI work gets its own "CI & build" section — ordered above Documentation, since a CI PR here nearly always updates docs too — and Dependencies is pure Dependabot residue. **Any category keyed on a label a Dependabot PR can carry needs that author exclude** — labeler's path labels *and* the ecosystem labels Dependabot applies itself (`dependencies`, `javascript`, `go`, `github_actions`; `javascript` is in neither `labeler.yml` nor our categories) — or that category intercepts bumps before they reach the `📦 Dependencies` catch-all. `CHANGELOG.md` is *not* the source of the release body; it is the longer-form record of why each change was made. - **Server —** a **GHCR image** at `ghcr.io/wave-rf/wavehouse`, with two tags: the immutable `:vX.Y.Z`, and one moving *channel* pointer. A stable release moves `:latest`; a prerelease moves `:alpha` / `:beta` / `:rc` / `:next` instead, matching the npm dist-tag it would get. The channel comes from the **first** prerelease identifier, matched **exactly**: `v0.2.0-rc.1` → `:rc`, while `-alpha1`, `-preview.1`, or any other form → `:next`. `scripts/ci/release-channel.sh` is the single rule every publisher uses, so `ghcr.io/wave-rf/wavehouse:rc` and `@wavehouse/sdk@rc` can't drift apart. **A prerelease-only project therefore has no `:latest` tag** — that is deliberate; `:latest` starts existing when the first stable release ships. - **TypeScript SDK —** an **npm publish** of `@wavehouse/sdk` under `latest` (stable) or `alpha`/`beta`/`rc`/`next` (prerelease), plus its own GitHub Release. @@ -657,14 +666,32 @@ gh attestation verify oci://ghcr.io/wave-rf/wavehouse:v0.1.0 \ `--signer-workflow` is not optional garnish: `--repo` alone accepts an attestation produced by *any* workflow in the repo. Same point, and the `:dev` equivalent, in [Deployment → Registry](/deployment#registry) and `SECURITY.md`. :::note[Releasing from the GitHub UI instead] -Publishing a release from **Releases → Draft a new release** creates the tag, which fires the same workflow — so it works, and GoReleaser's default `mode: keep-existing` (the key is not set in `.goreleaser.yaml`) means it will not overwrite notes you wrote. Two things the `make` targets do for you and the UI does not: none of the preflight checks run, and you must click **Generate release notes** yourself, because a body you publish empty stays empty. +Publishing a release from **Releases → Draft a new release** creates the tag, which fires the same workflow — so it works, and it will not overwrite notes you wrote: `release.yml` checks `gh release view` first and, when the release already exists, only uploads the assets. (That check also makes the job re-runnable, which is why it is not conditional on how the release was created.) Two things the `make` targets do for you and the UI does not: none of the preflight checks run, and you must click **Generate release notes** yourself, because a body you publish empty stays empty. ::: +### How the server release is built + +Since the switch to cgo the three binaries are built on **three native runners**, not cross-compiled from one: + +| Target | Runner | +| --- | --- | +| `linux/amd64` | `ubuntu-latest` | +| `linux/arm64` | `ubuntu-24.04-arm` | +| `darwin/arm64` | `macos-latest` | + +All three are free for public repositories. Each runs `goreleaser build --single-target --output dist/wavehouse` — so `.goreleaser.yaml` is still the one place the build's `-ldflags`, binary name and supported platform set are declared — and uploads the binary as a run artifact. A final `ubuntu-latest` job assembles everything: the three `.tar.gz` archives, `checksums.txt`, the multi-arch GHCR image via `docker buildx build` over [`deployments/Dockerfile.goreleaser`](https://github.com/Wave-RF/WaveHouse/blob/main/deployments/Dockerfile.goreleaser), the GitHub Release, and the provenance attestations. + +**Why not one runner and cross-compilers?** cgo needs a C toolchain per target, and for darwin that means real Apple SDK headers. `zig cc -target aarch64-macos` cross-compiles most Go programs happily, but not this one: `prometheus/client_golang`'s darwin process collector is a C file that `#include`s ``, which zig does not ship and which cannot legally be fetched onto a GitHub-hosted Linux runner. Measured, with and without `-tags netgo,osusergo`; the two GoReleaser features that would solve it — split/merge and `builder: prebuilt` — are Pro-only, and OSS `goreleaser release` has no `--skip=build`, so there is no way to have GoReleaser assemble a release from binaries built elsewhere. The two Linux targets *can* be cross-compiled (`gcc-aarch64-linux-gnu` works), but building them natively alongside darwin costs nothing extra and keeps one rule instead of two. + +A consequence worth knowing when you file a bug: the released Linux binaries are **dynamically linked against glibc**, minimum `GLIBC_2.34` (measured on `ubuntu-24.04`, both architectures) — Debian 12, Ubuntu 22.04 and RHEL 9 or newer. The pre-cgo builds were static and ran anywhere. The container images are unaffected; their `distroless/cc-debian12` base is glibc 2.36. + +`goreleaser-validate.yml` is the PR-time proof of all of this. On a change to `.goreleaser.yaml`, `go.mod`/`go.sum`, `chtypes.lock`, `scripts/fetch-chtypes.sh`, `deployments/Dockerfile.goreleaser` or the release workflows it runs `goreleaser check`, the same three-runner matrix in `--snapshot` mode, and a real multi-arch image build (to `--output type=cacheonly`, so nothing is pushed). It is advisory, not a required check. + ## The `dev` channel Between releases, every push to `main` republishes both artifacts so `@dev` always means "current `main`": -- **`ghcr.io/wave-rf/wavehouse:dev`** — a rolling pointer, plus an immutable `:dev-` (pruned after 30 days by `cleanup-ghcr.yml`, newest 5 always kept). Built by the same GoReleaser pipeline with `WAVEHOUSE_DEV=1`, which suppresses the GitHub Release. Note a Docker tag is only a pointer: `docker run …:dev` reuses a stale local image unless you `docker pull` first or pass `--pull=always`. +- **`ghcr.io/wave-rf/wavehouse:dev`** — a rolling pointer, plus an immutable `:dev-` (pruned after 30 days by `cleanup-ghcr.yml`, newest 5 always kept). Built by `publish-dev.yml`, the same shape as a real release minus everything that isn't the image: two native Linux build jobs and one push job, no archives and no GitHub Release. Note a Docker tag is only a pointer: `docker run …:dev` reuses a stale local image unless you `docker pull` first or pass `--pull=always`. - **`@wavehouse/sdk@dev`** — `0.0.1-dev..h`, published only when the published package actually changes. The trailing hash covers every file `npm pack` would ship — the built `dist/`, `package.json` minus its `version`, and the bundled `README`/`LICENSE` — so a push whose package would be byte-identical to the current `dev` publish is skipped, while a change to `exports`, `files`, `bin`, or `engines` republishes even though `dist/` is untouched. The `` is load-bearing, not decoration. `npm install …@dev` records a *range* in your `package.json`, not the dist-tag, so what you get on the next install is the highest version matching that range. Under the old `0.0.0-dev.h` scheme, semver's lexical ordering of alphanumeric prerelease identifiers meant the newest publish routinely wasn't the highest one, and a range could resolve *backwards* — an `@dev` install landed on a two-month-old build ([#475](https://github.com/Wave-RF/WaveHouse/issues/475)). A numeric identifier compares numerically, so the channel now orders by publish time. The `0.0.1` base keeps the channel below every real release (so a dev build can never satisfy `^0.1.0`) and above the legacy `0.0.0-dev.*` publishes, which npm's 72-hour unpublish window makes permanent. diff --git a/docs/src/content/docs/getting-started.md b/docs/src/content/docs/getting-started.md index f24c3ee6..94d42412 100644 --- a/docs/src/content/docs/getting-started.md +++ b/docs/src/content/docs/getting-started.md @@ -5,13 +5,13 @@ sidebar: order: 2 --- -Run WaveHouse locally in under five minutes. WaveHouse ships as a single binary with ClickHouse as the only external dependency; this walkthrough covers ingest, query, and real-time streaming. +Run WaveHouse locally in under five minutes. WaveHouse ships as one binary plus the per-ClickHouse-version [chtypes artifact](/deployment#chtypes-artifacts) it loads at start, with ClickHouse as the only external network dependency; this walkthrough covers ingest, query, and real-time streaming. ## Prerequisites - **Docker** — for running ClickHouse (and optionally WaveHouse itself). - **curl** and **jq** (optional) — for poking the API. -- **Go 1.26+** — only required if you want to build from source; skip it for the Docker path below. +- **Go 1.27+** — only required if you want to build from source; skip it for the Docker path below. Building from source also requires cgo (a C toolchain) and glibc — see [Deployment → Supported Platforms](/deployment#supported-platforms). ## 1. Start WaveHouse @@ -62,7 +62,7 @@ curl -s -X POST "http://localhost:8080/v1/ingest?table=clicks" \ # → {"ok":true} ``` -WaveHouse validates the body against the ClickHouse schema before acknowledging. Unknown fields, type mismatches, and missing required columns are rejected with a `400`. +WaveHouse validates the body against the ClickHouse schema before acknowledging — using ClickHouse's own parser, running in-process (`internal/typelayer`, via [chtypes](/deployment#chtypes-artifacts)), so a rejection carries ClickHouse's own error code and message, the same as a native `INSERT` would produce. Unknown fields and type mismatches are rejected with a `400`. ## 4. Query diff --git a/docs/src/content/docs/index.mdx b/docs/src/content/docs/index.mdx index 4c36a3ea..4d095e29 100644 --- a/docs/src/content/docs/index.mdx +++ b/docs/src/content/docs/index.mdx @@ -1,6 +1,6 @@ --- title: Real-time API gateway for ClickHouse -description: The open-source real-time API gateway for ClickHouse — schema-aware ingest, async batching, real-time streaming, and tiered query caching in a single binary. +description: The open-source real-time API gateway for ClickHouse — schema-aware ingest, async batching, real-time streaming, and tiered query caching in one binary. template: splash # Homepage-only structured data: lets Google/GitHub attach the canonical # SoftwareSourceCode + Organization entities to the project's root URL. @@ -10,10 +10,10 @@ head: attrs: type: application/ld+json content: | - {"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://wave-rf.com/#org","name":"Wave RF","url":"https://wave-rf.com","email":"hello@wave-rf.com","sameAs":["https://github.com/Wave-RF"]},{"@type":"SoftwareSourceCode","@id":"https://wavehouse.dev/#project","name":"WaveHouse","description":"The open-source real-time API gateway for ClickHouse — schema-aware ingest, async batching, real-time streaming, and tiered query caching in a single binary.","codeRepository":"https://github.com/Wave-RF/WaveHouse","programmingLanguage":["Go","TypeScript"],"license":"https://opensource.org/license/apache-2.0/","image":"https://wavehouse.dev/og.png","url":"https://wavehouse.dev","author":{"@id":"https://wave-rf.com/#org"}}]} + {"@context":"https://schema.org","@graph":[{"@type":"Organization","@id":"https://wave-rf.com/#org","name":"Wave RF","url":"https://wave-rf.com","email":"hello@wave-rf.com","sameAs":["https://github.com/Wave-RF"]},{"@type":"SoftwareSourceCode","@id":"https://wavehouse.dev/#project","name":"WaveHouse","description":"The open-source real-time API gateway for ClickHouse — schema-aware ingest, async batching, real-time streaming, and tiered query caching in one binary.","codeRepository":"https://github.com/Wave-RF/WaveHouse","programmingLanguage":["Go","TypeScript"],"license":"https://opensource.org/license/apache-2.0/","image":"https://wavehouse.dev/og.png","url":"https://wavehouse.dev","author":{"@id":"https://wave-rf.com/#org"}}]} hero: title: WaveHouse - tagline: The open-source real-time API gateway for ClickHouse. Schema-aware ingest, async batching, real-time streaming, and tiered query caching — in a single binary. + tagline: The open-source real-time API gateway for ClickHouse. Schema-aware ingest, async batching, real-time streaming, and tiered query caching — in one binary. actions: - text: Read the docs link: /getting-started @@ -220,7 +220,7 @@ Self-hosting WaveHouse is deliberately boring — one binary, one dependency. Bu

WaveHouse is alpha — and built entirely in the open.

-

Apache-2.0-licensed, single binary, no vendor lock-in — self-host it forever, or let us run it on WaveHouse Cloud. We ship with honest expectations — see the support cadence and security policy. Kick the tires and tell us where it breaks.

+

Apache-2.0-licensed, one binary, no vendor lock-in — self-host it forever, or let us run it on WaveHouse Cloud. We ship with honest expectations — see the support cadence and security policy. Kick the tires and tell us where it breaks.

Get started in five minutes Star on GitHub diff --git a/docs/src/content/docs/ingest-pipeline.md b/docs/src/content/docs/ingest-pipeline.md index a1defa39..3b31c1b7 100644 --- a/docs/src/content/docs/ingest-pipeline.md +++ b/docs/src/content/docs/ingest-pipeline.md @@ -14,11 +14,10 @@ It is deliberately detailed: this is a hot, concurrency-heavy path, and the goro | File | Contents | | --- | --- | | `worker.go` | `StartIngestWorker`, the `dispatchLoop`, `parseMsg` (+ `rejectPoison` for an envelope it cannot read), the per-table `tableBatcher`/`tableLoop`, `flushTable` (splits a batch per column list via `groupByColumns`) and `flushGroup` (bulk insert with a row-by-row poison-isolation fallback), `insertToClickHouse`, `handleSuccess` (cache invalidation + acks), `sendToDLQ`/`parkOnDLQ` | -| `compact.go` | `EncodeCompactRow` — renders one record as a `JSONCompactEachRow` line over the table's **insertable** columns, in declaration order. Serialization only: it validates nothing and judges no value | | `sweeper.go` | The **Active Sweeper** — purges stream messages that are both written to ClickHouse and past the SSE gap window | | `types.go` | `EventMessage` wire format and the `BufferConsumerName` constant | -The pipeline is **insert-only**. (Upgrading across the v2 envelope? [Drain the queue first](/deployment#upgrading-across-the-v2-ingest-envelope).) The wire format carries `{table_name, scope, received_timestamp, format, columns, row}`: `row` is one `JSONCompactEachRow` line — a positional JSON array — and `columns` names its positions — the table's insertable columns, in declaration order (a `MATERIALIZED` or `ALIAS` column cannot be named in an `INSERT`, so it is not part of the row's contract). (`scope` is reserved and always `""` today.) Each NATS message is its own envelope, so the names ride along per record; where they are carried once is the `INSERT` the worker emits per group. The worker parses the envelope, groups a batch by column list, and bulk-`INSERT`s each group as `INSERT INTO … (cols) FORMAT JSONCompactEachRow` — schema validation already happened at the HTTP ingest handler, before publish. Non-insert mutations go through `POST /v1/ops/query` (admin-only). +The pipeline is **insert-only**; non-insert mutations go through `POST /v1/ops/query` (admin-only). (Upgrading across the v2 envelope? [Drain the queue first](/deployment#upgrading-across-the-v2-ingest-envelope).) Each NATS message is one envelope — `{table_name, scope, received_timestamp, format, columns, row}`, documented field by field in [API → Internal Wire Format](/api#internal-wire-format-nats). What matters here: `row` is not something WaveHouse encodes, it is the exact `JSONCompactEachRow` bytes ClickHouse's own writer produced at ingest time (`internal/typelayer.Table.Ingest`), and `columns` names its positions. The worker parses the envelope, groups a batch by column list, and bulk-`INSERT`s each group as `INSERT INTO … (cols) FORMAT JSONCompactEachRow` with the settings chtypes compiled the row with (`internal/typelayer.InsertSettings()`) plus `async_insert=0`. A role that may not write every column produces a shorter `columns` list, which is simply another batch group. ## High-level shape @@ -55,12 +54,12 @@ flowchart LR Note the stream is **dual-use**: it is both the durable buffer feeding the worker and the replay buffer that SSE clients gap-fill from. That is why a custom sweeper exists instead of plain work-queue auto-deletion (see [Scaling out](#scaling-to-multiple-instances)). -:::note[Omitted columns on `Nullable` columns with a default] -Inserts also pin `input_format_null_as_default=1`. A positional row has one value per insertable column and no way to say "absent", so a field the record omitted rides as an explicit `null` in its slot. That setting turns the `null` back into the column's default for a **non-nullable** column, matching what omitting the key did under `JSONEachRow` — but on a `Nullable(T) DEFAULT …` column ClickHouse stores `NULL` whatever the setting says, because only an *absent* key ever took the default. So such a column now stores `NULL` where it previously took its default. Verified on ClickHouse 26.6.3. +:::note[Omitted columns take their real DEFAULT, not `null`] +The batch that reaches `insertToClickHouse` is not assembled from the request body — it is the bytes `Table.Ingest` returned for each accepted record, produced by ClickHouse's own writer. An omitted field's `DEFAULT` (or the type's implicit zero) was evaluated before that line existed, so a `Nullable(T) DEFAULT …` column takes its default exactly as an `INSERT` naming fewer columns would. Verified on ClickHouse 26.6.3. ::: -:::note[ClickHouse timestamp parsing] -Inserts pin `date_time_input_format=best_effort` — the server default since ClickHouse 26.5, but on older servers the `basic` default rejects the canonical RFC 3339 form's `Z` suffix ([#372](https://github.com/Wave-RF/WaveHouse/issues/372)). The ordinary spellings (zone-less date-times, 9–10-digit Unix-seconds strings) parse identically under both settings. (This is moot for anything still buffered from an older build: a message published before the v2 envelope cannot be read at all — see [Upgrading across the v2 ingest envelope](/deployment#upgrading-across-the-v2-ingest-envelope).) Bare digit-strings of other lengths are the exception: `best_effort` reads them as ClickHouse's calendar/epoch shapes, where `basic` read a plain `DateTime` column's digit string of five or more digits as Unix seconds (shorter runs it rejected outright, where `best_effort` reads `"2026"` as a year): under `best_effort` `"20260711"` stores 2026-07-11, where `basic` stored 1970-08-23. `DateTime64` columns diverge the same way on calendar-shaped runs, and additionally whenever an epoch run's unit doesn't match the column scale (under `basic`, runs longer than 10 digits are ticks at the column's own scale; `best_effort` unit-detects 13/16/19-digit runs as ms/µs/ns). A producer relying on the old `basic` reading changes meaning as soon as this WaveHouse version is deployed — the pin, not a ClickHouse upgrade, is what flips the parse. +:::note[Insert settings pinned] +Inserts pin the same parsing settings chtypes compiled the row with — `date_time_input_format=best_effort` and `input_format_null_as_default=1` — plus `async_insert=0`, which the worker adds itself (not a parsing setting, so it's never passed to chtypes): unpinned, ClickHouse 26.2+'s server-default async insert costs a flush-wait floor per statement and loses per-row error attribution (`While executing WaitForAsyncInsert` instead of `(at row N)`). `deduplicate_insert`'s server default (block-hash dedup, `enable` from 26.2 for plain `MergeTree`) is deliberately left unpinned — fine for retried-identical-batch at-least-once delivery, but worth knowing about if the worker's retry classifier ever needs to distinguish it from two legitimately identical batches. ::: ## The journey of one event diff --git a/docs/src/content/docs/reverse-proxy.mdx b/docs/src/content/docs/reverse-proxy.mdx index 8705956b..3b97e76c 100644 --- a/docs/src/content/docs/reverse-proxy.mdx +++ b/docs/src/content/docs/reverse-proxy.mdx @@ -104,7 +104,7 @@ Set your own **outer** limit at the proxy, sized to your real needs: The effective limit is the smaller of the proxy's and WaveHouse's. For ingest, WaveHouse's 16 MiB is the ceiling — raising the proxy above it won't help, because the server rejects first. The cap applies to **every** body shape, NDJSON included: WaveHouse reads the whole body before parsing it, so a line-framed batch is bounded by the same 16 MiB cap as a JSON array. Split an upload larger than the cap across several requests. See [Batch Ingest](/api#batch-ingest). :::note[Size the container for concurrency, not just for one request] -Ingest reads the whole body into memory before parsing it, so peak memory per in-flight request tracks the **body**, not one record — and it is a multiple of the body. `bytes.Buffer` grows by doubling to the next power of two and then once more to probe for EOF, so any body above ~16 MiB − 512 B lands in a **32 MiB** allocation (a 9 MiB body already costs 16 MiB), and that final doubling *copies*: the old 16 MiB array and the new 32 MiB one are both live while it runs, a transient of roughly **48 MiB — 3× the body — for a single request**. Hitting the cap does not avoid it, since a rejected over-cap body allocates the same 32 MiB before the `413`. On top of that sits the decoded Go value of the record in flight; ingest still decodes record by record, so for a batch of small rows that term is minor. A single-object body **is** one record, and the decode runs *before* schema validation: the body is materialized as a `map[string]any` carrying whatever keys it arrived with, and only then is an unknown column rejected. So a flat object of many tiny keys is fully in memory before anything bounds it — a 16 MiB body of ~1.4M one-byte values decodes to well over 100 MiB, which *is* the order-of-magnitude amplification quoted above. (A single oversized array element or NDJSON line costs the same, bounded by 16 MiB and 10 MiB respectively.) The 3× figure covers the buffering term alone. Nothing bounds the *total* across concurrent uploads either, so size for concurrent uploads × **at least 10×** the largest body you accept — an array-valued key amplifies harder still, around 18× measured, and nesting goes higher — or lower the proxy's body cap for `/v1/ingest` below 16 MiB. Size the container memory limit and the proxy's body cap together, and cap concurrency at the proxy if you accept large batches. A server-side bound on total in-flight bytes is tracked in [#544](https://github.com/Wave-RF/WaveHouse/issues/544). +Ingest reads the whole body into memory before parsing it, so peak memory per in-flight request tracks the **body**, not one record — and it is a multiple of the body. `bytes.Buffer` grows by doubling to the next power of two and then once more to probe for EOF, so any body above ~16 MiB − 512 B lands in a **32 MiB** allocation (a 9 MiB body already costs 16 MiB), and that final doubling *copies*: the old 16 MiB array and the new 32 MiB one are both live while it runs, a transient of roughly **48 MiB — 3× the body — for a single request**. Hitting the cap does not avoid it, since a rejected over-cap body allocates the same 32 MiB before the `413`. WaveHouse no longer decodes a record into Go values, so the old per-record amplification is gone; what sits on top of the buffering term now is ClickHouse's own parse of those bytes and the rows it renders back. Nothing bounds the *total* across concurrent uploads, so size the container memory limit and the proxy's body cap together, and cap concurrency at the proxy if you accept large batches. A server-side bound on total in-flight bytes is tracked in [#544](https://github.com/Wave-RF/WaveHouse/issues/544). ::: ## Server-Sent Events (SSE) @@ -200,7 +200,7 @@ These limit the *whole* request regardless of traffic, so no keepalive extends t WaveHouse does **not** derive a client IP from forwarded headers — it does no per-IP logic (rate limiting and IP allow/deny are the proxy's job) and does not trust `X-Forwarded-For` / `X-Real-IP` / `True-Client-IP` to rewrite the connection's source address. So a forged forwarded header has no effect on WaveHouse, and `r.RemoteAddr` (what OpenTelemetry records as the peer) is the honest immediate peer — your proxy, when one is in front. Still, don't expose `:8080` to untrusted clients: bind WaveHouse to a private interface or firewall the port so the proxy is the only path in. Capturing the real client IP in WaveHouse's own traces and logs — trusted-proxy-aware, so it can't be spoofed — is tracked in [#333](https://github.com/Wave-RF/WaveHouse/issues/333). ::: -- **`Content-Type`** — forward it **verbatim**; ingest reads the format from it and refuses anything it cannot read ([details](/api#post-v1ingesttabletable--ingest-data)). Appending rather than replacing is safe *only* if the proxy sends a second header **line** — those are resolved together and accepted when they agree. A proxy that **merges** duplicates into one comma-joined value (Envoy's `append: true`, and some WAF rewrite rules) produces `application/json, application/json`, which is a `415` on every ingest **even though both halves agree**: a comma-joined value is refused unless the value as a whole still parses as one media type. Replacing it is worse than merging, because it fails silently: an NDJSON batch declared `application/json` is read as the single object it starts with and the remaining lines are dropped behind a `200` ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)) — no error to alert on. If ingest starts returning `415` fleet-wide after a proxy change, look here first. +- **`Content-Type`** — forward it **verbatim**; ingest reads the format from it and refuses anything it cannot read ([details](/api#post-v1ingesttabletable--ingest-data)). Appending rather than replacing is safe *only* if the proxy sends a second header **line** — those are resolved together and accepted when they agree. A proxy that **merges** duplicates into one comma-joined value (Envoy's `append: true`, and some WAF rewrite rules) produces `application/json, application/json`, which is a `415` on every ingest **even though both halves agree**. Replacing it is worse than merging, because it fails silently: an NDJSON batch declared `application/json` is read as the single object it starts with and the remaining lines are dropped behind a `200` ([#561](https://github.com/Wave-RF/WaveHouse/issues/561)) — no error to alert on. And a proxy that rewrites `text/csv` or `text/tab-separated-values` (including their `; header=present` variants, which select the header-line formats) will turn a valid positional batch into a `415` or, if it drops the parameter, into a different reading of the header line (a rejected record under `header=absent`, a silently consumed header on a bare type). If ingest starts returning `415` fleet-wide after a proxy change, look here first. - **CORS** — WaveHouse applies its own CORS from the settings directory's `cors.allowed_origins`. Let one layer own CORS: either pass it through the proxy untouched (recommended), or strip it from WaveHouse and do it at the proxy — not both, or browsers see duplicate `Access-Control-Allow-Origin` headers and reject the response. diff --git a/docs/src/content/docs/sdk/queries.md b/docs/src/content/docs/sdk/queries.md index aded70c1..9a8afc53 100644 --- a/docs/src/content/docs/sdk/queries.md +++ b/docs/src/content/docs/sdk/queries.md @@ -40,9 +40,9 @@ const { data } = await clicks.insert([ // data: { ok, total, succeeded, failed, duplicates, results? } ``` -For an array insert, `data.ok` is `true` only when every record succeeded (`failed === 0`). Inspect `data.failed` and `data.results` (each `{ index, ok|duplicate|error }`, 1-based `index`) for partial failures — the call's top-level `error` is reserved for whole-request failures (network, `404` unknown table, `403` forbidden, `503` backpressure). An empty array is a no-op and sends no request. The array path sends one request regardless of size, so it is bound by the same 16 MiB [request-body cap](/reverse-proxy#request-body-size-limits); bounded-concurrency chunking of very large arrays is tracked in [#196](https://github.com/Wave-RF/WaveHouse/issues/196). +For an array insert, `data.ok` is `true` only when every record succeeded (`failed === 0`). Inspect `data.failed` and `data.results` (each `{ index, ok|duplicate|error, code? }`, 1-based `index`, `code` being ClickHouse's own error code on a parser rejection) for partial failures — the call's top-level `error` is reserved for whole-request failures (network, `404` unknown table, `403` forbidden, `503` backpressure). An empty array is a no-op and sends no request. The array path sends one request regardless of size, so it is bound by the same 16 MiB [request-body cap](/reverse-proxy#request-body-size-limits); bounded-concurrency chunking of very large arrays is tracked in [#196](https://github.com/Wave-RF/WaveHouse/issues/196). -> `POST /v1/ingest` also accepts a raw JSON array or a single object directly, so non-SDK clients can send whichever shape is convenient — but the `Content-Type` is **required** and decides the format (`application/json` or `application/x-ndjson`); a request without one is rejected with `415`. The SDK always sets it. See the [API reference](/api#post-v1ingesttabletable--ingest-data). +> `POST /v1/ingest` also accepts a raw JSON array or a single object directly, and positional `text/csv` / `text/tab-separated-values` bodies (add `; header=present` to send a header line of column names, or `; header=absent` to turn ClickHouse's header auto-detection off), so non-SDK clients can send whichever shape is convenient — but the `Content-Type` is **required** and decides the format; a request without one is rejected with `415`. The SDK always sets it. See the [API reference](/api#post-v1ingesttabletable--ingest-data). ### `.insertNDJSON(source, opts?)` diff --git a/docs/src/content/docs/sdk/reference.md b/docs/src/content/docs/sdk/reference.md index d5f3f3a2..3bd7fde3 100644 --- a/docs/src/content/docs/sdk/reference.md +++ b/docs/src/content/docs/sdk/reference.md @@ -139,7 +139,7 @@ Codegen reads `/v1/ops/schema`, which is **admin-only**. Against a non-dev serve | `--out`, `-o` | Output .d.ts file path | `./wavehouse.d.ts` | | `--auth`, `-a` | Bearer token (if auth required) | — | -The generated row type is the **read** shape — with one exception running the other way: an `EPHEMERAL` column declares a default, so codegen emits it too, yet no query can ever return it. There the type says readable where only the write is real. A `MATERIALIZED` or `ALIAS` column declares a default, so it is emitted as optional — but supplying one on `insert` is a `400` (`column "x" of table "t" is materialized and cannot be inserted`), and the type will not catch it. Omit computed columns; the server fills them in. +The generated row type is the **read** shape, and computed columns are where it and the server disagree. An `EPHEMERAL` column declares a default, so codegen emits it, yet no query can ever return it — the type says readable where only the write is real. `MATERIALIZED` and `ALIAS` columns declare defaults too, so they are emitted as optional, but supplying any of the three on `insert` is a `400` carrying ClickHouse's own code 117 (`Unknown field found while parsing JSONEachRow format: x`), and the type will not catch it. Omit computed columns; the server fills them in. **Example output:** @@ -164,7 +164,7 @@ export interface ClicksRow { | ClickHouse Type | TypeScript Type | |----------------|-----------------| | `String`, `FixedString`, `UUID`, `DateTime*`, `Date*`, `Enum*`, `IPv4/6` | `string` | -| `UInt*`, `Int*`, `Float*`, `Decimal*` | `number` | +| `UInt*`, `Int*`, `Float*`, `Decimal*` | `number` — `Decimal*` comes back as a JSON number, not a string | | `Bool` | `boolean` | | `Nullable(T)` | `T \| null` | | `Array(T)` | `T[]` | diff --git a/docs/src/content/docs/sdk/streaming.md b/docs/src/content/docs/sdk/streaming.md index eadfcb6d..100c929e 100644 --- a/docs/src/content/docs/sdk/streaming.md +++ b/docs/src/content/docs/sdk/streaming.md @@ -97,9 +97,9 @@ interface StreamEvent { } ``` -`data` is a row **object**, as it always has been — but the wire underneath is positional. The server sends the column list in its own `event: schema` frame — before the first row, and again whenever the list drifts on the **live** path (with one exception after a gap-fill, below) — and each row as a JSON array; the SDK keeps the announced list and zips every row against it, so this shape is unchanged and nothing in your code moves. It matters in three places. A column the producer omitted arrives as an explicit `null` rather than an absent key. The row object has a **null prototype**: a ClickHouse column may legitimately be named `__proto__`, and on an ordinary object that assignment hits the inherited setter and the value disappears — so the SDK builds each row with `Object.create(null)`. Property access, spreading, `JSON.stringify` and destructuring all behave normally; what does not is anything inherited from `Object.prototype`, so use `Object.hasOwn(row, "x")` rather than `row.hasOwnProperty("x")`, and don't rely on `` `${row}` `` or `row.constructor`. (`liveQuery`'s REST backfill half still yields ordinary objects.) And a **raw** SSE consumer (a hand-rolled `EventSource`) must do the zipping itself — see [the wire format](/api#get-v1stream--server-sent-events-stream). +`data` is a row **object**, as it always has been — but the wire underneath is positional. The server sends the column list in its own `event: schema` frame — before the first row, and again whenever the list drifts on the **live** path (with one exception after a gap-fill, below) — and each row as a JSON array; the SDK keeps the announced list and zips every row against it, so this shape is unchanged and nothing in your code moves. It matters in two places. The row object has a **null prototype**: a ClickHouse column may legitimately be named `__proto__`, and on an ordinary object that assignment hits the inherited setter and the value disappears — so the SDK builds each row with `Object.create(null)`. Property access, spreading, `JSON.stringify` and destructuring all behave normally; what does not is anything inherited from `Object.prototype`, so use `Object.hasOwn(row, "x")` rather than `row.hasOwnProperty("x")`, and don't rely on `` `${row}` `` or `row.constructor`. (`liveQuery`'s REST backfill half still yields ordinary objects.) And a **raw** SSE consumer (a hand-rolled `EventSource`) must do the zipping itself — see [the wire format](/api#get-v1stream--server-sent-events-stream). A column the producer omitted is no longer `null` on the wire: WaveHouse's ingest validation runs ClickHouse's own parser in-process, which evaluates the column's `DEFAULT` (or its implicit zero value) before the row is published — the same as a native `INSERT` naming fewer columns than the table has. -Row values of top-level `DateTime`/`DateTime64` columns inside `data` (not timestamps nested in `Array`/`Map`/`Tuple` columns) arrive in canonical RFC 3339 UTC (`2026-06-21T04:00:00.123Z`), matching what `/v1/query` returns for the same row — `new Date(value)` parses correctly with no zone fix-up. Values WaveHouse couldn't canonicalize (ingest is fail-open) stream in the producer's original spelling, and the `/v1/query` match doesn't hold for them: a spelling ClickHouse accepted anyway still queries back in canonical UTC (one it rejected never lands in the table at all), and a zone-less date-time is what `new Date()` reads as *local* time — though a date-only `YYYY-MM-DD` string is read as UTC, an ECMAScript quirk (see [Timestamp canonicalization](/api#timestamp-canonicalization)). +Row values of top-level `DateTime`/`DateTime64` columns inside `data` (not timestamps nested in `Array`/`Map`/`Tuple` columns) arrive as ClickHouse itself renders them — `"2026-06-21 04:00:00.123"`, space-separated, no `Z` suffix, in the column's declared zone else the server's default — matching what `/v1/query` returns for the same row byte-for-byte, by construction (see [Timestamp rendering](/api#timestamp-rendering)). `new Date(value)` does **not** parse this form correctly out of the box: ECMAScript's `Date` constructor reads a bare `YYYY-MM-DD HH:MM:SS` string as *local* time, not UTC, so replace the space with `T` and append the zone before parsing, or parse it with a zone-aware library. ### Transport Behavior diff --git a/docs/src/content/docs/settings-directory.mdx b/docs/src/content/docs/settings-directory.mdx index 10e2046e..d0dd252f 100644 --- a/docs/src/content/docs/settings-directory.mdx +++ b/docs/src/content/docs/settings-directory.mdx @@ -164,8 +164,8 @@ What stays in boot config is only what cannot change under a running process — Every dedupe knob lives here — there are no boot-config keys for it. The switch and its fields are resolved per record from one snapshot (table override → global value): - `dedupe.enabled` (seed default `false`) — turns deduplication on. Hot-reloadable: a reload that flips it opens the embedded Pebble store at `/pebble` (or closes it), so no restart is needed; seen ids persist across an off/on cycle. If the store fails to open on a reload, the failure is logged and ingest fails closed (`500 dedupe failed`) until the next reload or restart — the files asked for dedupe, so publishing un-deduped is not a fallback. At boot a failed open refuses to start, like every other store. A record that lands in the instant of the flip itself is published un-deduped: if the settings already say on but the store is not yet open, it's counted by `wavehouse_ingest_dedupe_disabled_total`; in the reverse case (settings already say off, store still open) the handler skips dedupe like any other disabled record and nothing is counted. That counter should only ever tick during a reload, so a steadily climbing rate means the store and the settings have come apart. -- `dedupe.id_field` (seed default `event_id`) — JSON field name in the ingest body used as the dedup key. -- `dedupe.require_id` (seed default `false`) — controls what happens to a row missing `id_field` (which can't be deduped, so idempotency wouldn't apply to it). Such a row is always logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total`, in both modes. `false`: it is then published un-deduped. `true` rejects it instead (`400` for a single insert; a per-record failure in a batch) — a server-side tripwire for producers that must guarantee the id. +- `dedupe.id_field` (seed default `event_id`) — the **column** whose stored value is the dedup key. It is read positionally out of the row ClickHouse rendered, not out of the request body, so the key is the value that will be stored rather than the caller's spelling (`256` into a `UInt8` keys on `0`) — identical for the documented case, a string id. "Missing" therefore means the row carries no value for it, which is what an omitted `String` column produces; a numeric id column cannot tell an omitted `0` from a supplied one. +- `dedupe.require_id` (seed default `false`) — controls what happens to a row with no value for `id_field` (which can't be deduped, so idempotency wouldn't apply to it). Such a row is always logged at `WARN` and counted by `wavehouse_ingest_dedupe_missing_id_total`, in both modes. `false`: it is then published un-deduped. `true` rejects it instead (`400` for a single insert; a per-record failure in a batch) — a server-side tripwire for producers that must guarantee the id. - `dedupe.tables.
.{id_field, require_id}` — per-table overrides; each entry overrides only the fields it names and inherits the rest. ## ClickHouse diff --git a/docs/src/content/docs/why-wavehouse.md b/docs/src/content/docs/why-wavehouse.md index a5b63e21..c9d5d319 100644 --- a/docs/src/content/docs/why-wavehouse.md +++ b/docs/src/content/docs/why-wavehouse.md @@ -171,7 +171,7 @@ Where it differs from WaveHouse: | Dimension | Tinybird | WaveHouse | | --------- | -------- | --------- | -| Hosting | SaaS only (managed tiers: Developer $49/mo → Enterprise custom) | Self-host, single binary | +| Hosting | SaaS only (managed tiers: Developer $49/mo → Enterprise custom) | Self-host, one binary + local artifact | | Pricing model | Pay for allocated vCPU/QPS/storage; egress fees for cross-region | Your infra; no per-query or per-GB fee | | Data residency | Their infrastructure | Your infrastructure | | Source of truth for schema | Tinybird datasource definitions | Your ClickHouse tables (`system.columns`) | diff --git a/go.mod b/go.mod index 3208b94f..ded1e3d8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Wave-RF/WaveHouse -go 1.26.6 +go 1.27 tool ( github.com/Zxilly/go-size-analyzer/cmd/gsa @@ -23,7 +23,6 @@ require ( github.com/fsnotify/fsnotify v1.10.1 github.com/go-chi/chi/v5 v5.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 - github.com/google/uuid v1.6.0 github.com/ilyakaznacheev/cleanenv v1.5.0 github.com/nats-io/nats-server/v2 v2.14.6 github.com/nats-io/nats.go v1.53.1 @@ -32,6 +31,7 @@ require ( github.com/samber/slog-sampling v1.7.0 github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.44.0 + github.com/wave-rf/chtypes/go v0.4.0 go.opentelemetry.io/contrib/bridges/otelslog v0.20.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 go.opentelemetry.io/contrib/instrumentation/runtime v0.71.0 @@ -127,6 +127,7 @@ require ( github.com/google/go-tpm v0.9.8 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/subcommands v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/jedib0t/go-pretty/v6 v6.7.10 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index 71dc2727..289d45b9 100644 --- a/go.sum +++ b/go.sum @@ -363,6 +363,8 @@ github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqo github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/vladopajic/go-test-coverage/v2 v2.18.7 h1:Kfpv8jWoC0muCAgk4bR3SvhNfqSQvO9iKZkrhdAZ4dM= github.com/vladopajic/go-test-coverage/v2 v2.18.7/go.mod h1:sTDv3QDUo3Vjo2azG9hyHfMlXH5ugnF1kqDr/4O3w4g= +github.com/wave-rf/chtypes/go v0.4.0 h1:bRNYUzNB+9MoPHmfbE0ebhHipbHPe04jW/7GClI7U4s= +github.com/wave-rf/chtypes/go v0.4.0/go.mod h1:gQE6FgwdtsXvpWlDr79q8XSwDmhiY1mIRa3gj3bOtgo= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= diff --git a/internal/api/bufpool.go b/internal/api/bufpool.go index 7ff917da..6a51495a 100644 --- a/internal/api/bufpool.go +++ b/internal/api/bufpool.go @@ -25,10 +25,10 @@ func getBodyBuffer() *bytes.Buffer { } // putBodyBuffer returns buf to the pool unless it outgrew -// maxPooledBufferBytes. The caller must be done reading records out of it: the -// record readers decode into freshly allocated values (encoding/json copies -// every string, json.Number included), so "done" means the last Next has -// returned — nothing a handed-back record holds points into these bytes. +// maxPooledBufferBytes. The caller must be done with the request: the body +// goes to the type layer as-is and the exported rows point into the type +// layer's own payload, which is copied into the envelope before the table +// handle is released — nothing published points into these bytes. func putBodyBuffer(buf *bytes.Buffer) { if buf == nil || buf.Cap() > maxPooledBufferBytes { return diff --git a/internal/api/cache_key.go b/internal/api/cache_key.go index ecc40769..c0ed7311 100644 --- a/internal/api/cache_key.go +++ b/internal/api/cache_key.go @@ -4,8 +4,6 @@ import ( "crypto/sha256" "encoding/binary" "encoding/hex" - "encoding/json" - "fmt" ) // queryCacheKey produces a deterministic L1/L2 cache key for a (sql, params) @@ -14,44 +12,34 @@ import ( // (`GET/POST /v1/pipes/{name}`) handlers do — they share this helper so a // key change in one place propagates to every cached read path. // +// params are the ClickHouse query-parameter values, positionally, exactly as +// they go on the wire: already rendered as text and already encoded for +// ClickHouse's parameter reader. That encoding is part of the key's input on +// purpose — two reads whose parameters reach ClickHouse as the same bytes are +// the same query, and two that do not are not. (The values used to be typed +// Go scalars, which is why this framing used to carry a `{type, value}` JSON +// payload; every value is a String parameter now, so the type is constant.) +// // Every section is framed with a 1-byte type marker (0x01 for sql, 0x00 for // param) plus an 8-byte big-endian length, then the payload. Without // length-prefixing the sql itself, a SQL string crafted to end with the // exact bytes of a param frame (`\x00` + 8 length bytes + payload) would // hash identically to a shorter SQL plus a real param — distinct -// `(sql, params)` tuples, same digest. Per-param framing also kept its -// own length prefix so embedded `\x00` inside a string param can't be -// confused for a frame boundary, and the JSON `{type, value}` payload -// shape additionally separates `"42"` (string) from `42` (int) so the -// cache can't serve a string-typed row to an int-typed lookup. -func queryCacheKey(sql string, params []any) string { +// `(sql, params)` tuples, same digest. Per-param framing keeps its own +// length prefix so an embedded `\x00` inside a value can't be confused for +// a frame boundary. +func queryCacheKey(sql string, params []string) string { h := sha256.New() - var sqlLen [8]byte - binary.BigEndian.PutUint64(sqlLen[:], uint64(len(sql))) + var n [8]byte + binary.BigEndian.PutUint64(n[:], uint64(len(sql))) _, _ = h.Write([]byte{1}) // sql frame marker - _, _ = h.Write(sqlLen[:]) + _, _ = h.Write(n[:]) _, _ = h.Write([]byte(sql)) for _, p := range params { - payload, err := json.Marshal(struct { - Type string `json:"type"` - Value any `json:"value"` - }{ - Type: fmt.Sprintf("%T", p), - Value: p, - }) - if err != nil { - // Marshal can only fail on unserialisable types (channels, - // funcs, cyclic structures) that shouldn't reach this path — - // pipes/structured-query params are scalars from JSON. Fall - // back to a type+%v rendering so we still produce a key and - // don't take down the request path. - payload = fmt.Appendf(nil, "%T:%v", p, p) - } - var n [8]byte - binary.BigEndian.PutUint64(n[:], uint64(len(payload))) + binary.BigEndian.PutUint64(n[:], uint64(len(p))) _, _ = h.Write([]byte{0}) // param frame marker _, _ = h.Write(n[:]) - _, _ = h.Write(payload) + _, _ = h.Write([]byte(p)) } return "query:" + hex.EncodeToString(h.Sum(nil)) } diff --git a/internal/api/cache_key_test.go b/internal/api/cache_key_test.go index 11c36e5a..f8e346a1 100644 --- a/internal/api/cache_key_test.go +++ b/internal/api/cache_key_test.go @@ -16,9 +16,9 @@ func TestQueryCacheKey(t *testing.T) { tests := []struct { name string sqlA string - paramsA []any + paramsA []string sqlB string - paramsB []any + paramsB []string expectEqual bool }{ { @@ -42,23 +42,26 @@ func TestQueryCacheKey(t *testing.T) { sqlA: "SELECT 1", paramsA: nil, sqlB: "SELECT 1", - paramsB: []any{"a"}, + paramsB: []string{"a"}, expectEqual: false, }, { name: "embedded NUL byte does not collide with split params", sqlA: "SELECT 1", - paramsA: []any{"foo\x00bar"}, + paramsA: []string{"foo\x00bar"}, sqlB: "SELECT 1", - paramsB: []any{"foo", "bar"}, + paramsB: []string{"foo", "bar"}, expectEqual: false, }, { - name: "string and int with same textual value are distinct", + // Every parameter is a String on the wire now, so "42" and 42 + // ARE the same read. What must still stay apart is two values + // whose CONCATENATION matches — the framing's job. + name: "adjacent params are not confusable with one joined param", sqlA: "SELECT 1", - paramsA: []any{"42"}, + paramsA: []string{"4", "2"}, sqlB: "SELECT 1", - paramsB: []any{42}, + paramsB: []string{"42"}, expectEqual: false, }, { @@ -66,25 +69,21 @@ func TestQueryCacheKey(t *testing.T) { sqlA: "SELECT 1", paramsA: nil, sqlB: "SELECT 1", - paramsB: []any{}, + paramsB: []string{}, expectEqual: true, }, { - // Constructed as an actual collision pair under the old - // "raw sql + framed params" format. The param frame for - // `"y"` is 0x00 + 8-byte BE length (0x1D = 29) + - // `{"type":"string","value":"y"}` (29 bytes), so the byte - // stream `("X", ["y"])` produces under the old framing is - // `"X" + 0x00 + 0x00…0x1D + {"type":"string","value":"y"}`. - // Setting sqlA to exactly those bytes and paramsA to nil - // reproduces that stream — under the old framing the two - // inputs hashed identically. The new 0x01-marker + 8-byte - // length prefix on sql forces them apart. + // Constructed as an actual collision pair under the pre-#315 + // "raw sql + framed params" format: sqlA is byte-for-byte the + // stream `("X", ["y"])` used to produce (the param frame is + // 0x00 + an 8-byte BE length + the payload), so the two inputs + // hashed identically. The 0x01 marker + 8-byte length prefix on + // the sql section forces them apart. name: "sql crafted to mimic a param-frame stream does not collide with shorter sql + real param", - sqlA: "X\x00\x00\x00\x00\x00\x00\x00\x00\x1d{\"type\":\"string\",\"value\":\"y\"}", + sqlA: "X\x00\x00\x00\x00\x00\x00\x00\x00\x01y", paramsA: nil, sqlB: "X", - paramsB: []any{"y"}, + paramsB: []string{"y"}, expectEqual: false, }, } diff --git a/internal/api/ch_settings.go b/internal/api/ch_settings.go index 9f04a8b8..9a8b020b 100644 --- a/internal/api/ch_settings.go +++ b/internal/api/ch_settings.go @@ -1,9 +1,8 @@ package api import ( + "strconv" "time" - - "github.com/ClickHouse/clickhouse-go/v2" ) // chQueryLimits is the per-request resource budget a single read runs under, @@ -14,12 +13,9 @@ import ( // by ClickHouse's own config. type chQueryLimits struct { // ExecutionTime is the wall-clock budget, emitted as max_execution_time in - // fractional seconds. clickhouse-go already derives max_execution_time from - // the context deadline, but only for deadlines > 1s — so a sub-second cap - // would otherwise reach the server with no time bound, and a context cancel - // can't interrupt an already-running server-side phase. Emitting it - // explicitly closes that hole; for >1s budgets the driver overwrites it with - // deadline+5s, a fine backstop. + // fractional seconds. Cancelling the HTTP request cannot interrupt a + // server-side phase that has already started, so the bound has to reach + // ClickHouse as a setting rather than only as a client deadline. ExecutionTime time.Duration // MaxResultRows caps rows RETURNED (max_result_rows + result_overflow_mode= // throw) — defense-in-depth behind the SQL LIMIT the structured builder @@ -33,29 +29,32 @@ type chQueryLimits struct { MaxMemoryBytes int64 } -// chReadSettings builds the per-query ClickHouse Settings that enforce a read's -// resource budget SERVER-SIDE, so it can't outrun the budget during a +// chReadSettings builds the per-query ClickHouse settings that enforce a +// read's resource budget SERVER-SIDE, so it can't outrun the budget during a // server-side scan / merge / aggregation phase (#316). Without these, the only -// budget reaching ClickHouse is whatever clickhouse-go derives from the context -// deadline — which never bounds memory or rows scanned. Returns nil when no cap -// applies, so the caller can skip wrapping the context. -func chReadSettings(l chQueryLimits) clickhouse.Settings { - settings := clickhouse.Settings{} +// budget reaching ClickHouse is the HTTP request's own deadline — which never +// bounds memory or rows scanned. Returns nil when no cap applies. +// +// The values are text because they ride on the ClickHouse HTTP interface's +// query string, but they are the numeric spellings ClickHouse expects: +// max_execution_time in fractional seconds, the rest as plain integers. +func chReadSettings(l chQueryLimits) map[string]string { + settings := map[string]string{} if l.ExecutionTime > 0 { // Fractional seconds — ClickHouse accepts them, preserving a sub-second // cap that a whole-second representation would round away. - settings["max_execution_time"] = l.ExecutionTime.Seconds() + settings["max_execution_time"] = strconv.FormatFloat(l.ExecutionTime.Seconds(), 'f', -1, 64) } if l.MaxResultRows > 0 { - settings["max_result_rows"] = l.MaxResultRows + settings["max_result_rows"] = strconv.Itoa(l.MaxResultRows) settings["result_overflow_mode"] = "throw" } if l.MaxRowsToRead > 0 { - settings["max_rows_to_read"] = l.MaxRowsToRead + settings["max_rows_to_read"] = strconv.FormatInt(l.MaxRowsToRead, 10) settings["read_overflow_mode"] = "throw" } if l.MaxMemoryBytes > 0 { - settings["max_memory_usage"] = l.MaxMemoryBytes + settings["max_memory_usage"] = strconv.FormatInt(l.MaxMemoryBytes, 10) } if len(settings) == 0 { return nil diff --git a/internal/api/ch_settings_test.go b/internal/api/ch_settings_test.go index 930b5eec..38070848 100644 --- a/internal/api/ch_settings_test.go +++ b/internal/api/ch_settings_test.go @@ -17,7 +17,7 @@ func TestChReadSettings(t *testing.T) { limits chQueryLimits // want is the exact settings map expected; nil means chReadSettings // must return nil (no caps → no context wrapping). - want map[string]any + want map[string]string }{ { name: "no caps set", @@ -27,35 +27,36 @@ func TestChReadSettings(t *testing.T) { { name: "sub-second execution time is a fractional max_execution_time", limits: chQueryLimits{ExecutionTime: 500 * time.Millisecond}, - // The driver only auto-derives max_execution_time for deadlines > 1s, - // so a 500ms cap MUST be emitted explicitly or it reaches CH unbounded. - want: map[string]any{"max_execution_time": 0.5}, + // A request deadline is not a server-side bound, so a sub-second cap + // MUST be emitted explicitly — and as fractional seconds, which a + // whole-second spelling would round away to "no cap at all". + want: map[string]string{"max_execution_time": "0.5"}, }, { name: "multi-second execution time", limits: chQueryLimits{ExecutionTime: 3 * time.Second}, - want: map[string]any{"max_execution_time": 3.0}, + want: map[string]string{"max_execution_time": "3"}, }, { name: "max_result_rows caps result rows with throw mode", limits: chQueryLimits{MaxResultRows: 1000}, - want: map[string]any{ - "max_result_rows": 1000, + want: map[string]string{ + "max_result_rows": "1000", "result_overflow_mode": "throw", }, }, { name: "max_rows_to_read caps rows scanned with throw mode", limits: chQueryLimits{MaxRowsToRead: 1_000_000}, - want: map[string]any{ - "max_rows_to_read": int64(1_000_000), + want: map[string]string{ + "max_rows_to_read": "1000000", "read_overflow_mode": "throw", }, }, { name: "max_memory_usage caps peak query memory", limits: chQueryLimits{MaxMemoryBytes: 4 << 30}, // 4 GiB > int32 - want: map[string]any{"max_memory_usage": int64(4 << 30)}, + want: map[string]string{"max_memory_usage": "4294967296"}, }, { name: "all caps together", @@ -65,20 +66,20 @@ func TestChReadSettings(t *testing.T) { MaxRowsToRead: 2_000_000, MaxMemoryBytes: 8 << 30, }, - want: map[string]any{ - "max_execution_time": 2.0, - "max_result_rows": 500, + want: map[string]string{ + "max_execution_time": "2", + "max_result_rows": "500", "result_overflow_mode": "throw", - "max_rows_to_read": int64(2_000_000), + "max_rows_to_read": "2000000", "read_overflow_mode": "throw", - "max_memory_usage": int64(8 << 30), + "max_memory_usage": "8589934592", }, }, { name: "zero caps are omitted even when others are set", limits: chQueryLimits{MaxRowsToRead: 42}, - want: map[string]any{ - "max_rows_to_read": int64(42), + want: map[string]string{ + "max_rows_to_read": "42", "read_overflow_mode": "throw", }, }, @@ -99,12 +100,12 @@ func TestChReadSettings(t *testing.T) { t.Fatalf("expected settings %#v, got nil", tt.want) } if len(got) != len(tt.want) { - t.Fatalf("settings key count mismatch: got %#v, want %#v", map[string]any(got), tt.want) + t.Fatalf("settings key count mismatch: got %#v, want %#v", got, tt.want) } for k, wantV := range tt.want { gotV, ok := got[k] if !ok { - t.Errorf("missing setting %q (got %#v)", k, map[string]any(got)) + t.Errorf("missing setting %q (got %#v)", k, got) continue } if gotV != wantV { diff --git a/internal/api/clickhouse_exec.go b/internal/api/clickhouse_exec.go deleted file mode 100644 index 83258993..00000000 --- a/internal/api/clickhouse_exec.go +++ /dev/null @@ -1,350 +0,0 @@ -package api - -import ( - "context" - "fmt" - "reflect" - "strings" - "time" - - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" - "github.com/google/uuid" -) - -// executeCHQuery runs sql against the native-protocol driver conn, -// classifying by leading SQL verb to pick the Exec-vs-Query path — -// clickhouse-go's driver.Query() errors on statements that return no -// result set, so the dispatch is correctness, not optimisation. Returns -// a row-array suitable for JSON marshalling; mutations marshal to `[]`, -// preserving the "always-an-array" response shape callers depend on. -// -// Used by the structured-query and pipes handlers — those are the cached -// read paths that need explicit Query/Exec dispatch and per-row scanning. -// The raw-SQL endpoint (/v1/ops/query) proxies straight to ClickHouse -// over HTTP and never calls this; see internal/api/query.go. -func executeCHQuery(ctx context.Context, conn driver.Conn, sql string, params []any) ([]map[string]any, error) { - if isMutation(sql) { - if err := conn.Exec(ctx, sql, params...); err != nil { - return nil, fmt.Errorf("clickhouse exec: %w", err) - } - return []map[string]any{}, nil - } - - rows, err := conn.Query(ctx, sql, params...) - if err != nil { - return nil, fmt.Errorf("clickhouse query: %w", err) - } - defer func() { _ = rows.Close() }() - - columns := rows.ColumnTypes() - // Initialize as empty (not nil) so a zero-row result marshals to `[]`, - // not `null`. SDK consumers do `data!.length` on the response; a `null` - // crashes the client on every empty fetch. - results := []map[string]any{} - - for rows.Next() { - valPtrs := make([]any, len(columns)) - for i, col := range columns { - valPtrs[i] = reflect.New(col.ScanType()).Interface() - } - if err := rows.Scan(valPtrs...); err != nil { - return nil, fmt.Errorf("scan clickhouse row: %w", err) - } - row := make(map[string]any) - for i, col := range columns { - row[col.Name()] = reflect.ValueOf(valPtrs[i]).Elem().Interface() - } - results = append(results, transformRow(row)) - } - // rows.Next() returns false both when iteration completes successfully - // AND when the driver hits an error mid-stream (network drop, decode - // failure on a row past the first). Without this check, a partial - // result set silently masquerades as a complete one. - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate clickhouse rows: %w", err) - } - return results, nil -} - -// mutationVerbs is a set of SQL leading keywords that don't return a result -// set — anything that mutates schema or data. Routed through Exec rather -// than Query (see executeCHQuery). Sourced from the ClickHouse statement -// reference: DML, DDL, role/privilege management, and runtime control -// (SYSTEM/KILL/SET). Read-only verbs (SELECT/WITH/SHOW/DESCRIBE/EXPLAIN/ -// EXISTS) intentionally fall through to the default Query path. -var mutationVerbs = map[string]struct{}{ - "INSERT": {}, - "UPDATE": {}, - "DELETE": {}, - "TRUNCATE": {}, - "DROP": {}, - "ALTER": {}, - "CREATE": {}, - "RENAME": {}, - "EXCHANGE": {}, - "OPTIMIZE": {}, - "REPLACE": {}, - "GRANT": {}, - "REVOKE": {}, - "ATTACH": {}, - "DETACH": {}, - "KILL": {}, - "SET": {}, - "USE": {}, - "SYSTEM": {}, -} - -// isMutation reports whether sql's leading statement is a non-SELECT — i.e. -// one that returns no result set and must go through Exec, not Query. -// Leading whitespace and SQL line/block comments are skipped, then the first -// alphabetic token is matched case-insensitively against mutationVerbs. A -// leading WITH clause (CTE) routes through a paren-aware scan because -// ClickHouse accepts `WITH cte AS (...) INSERT INTO t SELECT * FROM cte` as -// equivalent to `INSERT INTO t WITH cte AS (...) SELECT * FROM cte` (see -// https://clickhouse.com/docs/sql-reference/statements/insert-into). Without -// the skip, the WITH form would classify as a read, route through Query, -// silently succeed, and return `[]` — the same silent-success class that -// motivated the original cache-bypass guard. -func isMutation(sql string) bool { - s := stripLeadingSQLComments(sql) - end := 0 - for end < len(s) { - c := s[end] - if (c < 'A' || c > 'Z') && (c < 'a' || c > 'z') { - break - } - end++ - } - if end == 0 { - return false - } - first := strings.ToUpper(s[:end]) - if first != "WITH" { - _, ok := mutationVerbs[first] - return ok - } - return containsMutationVerbAtTopLevel(s[end:]) -} - -// nonMutationVerbs is the read/metadata-statement counterpart to -// mutationVerbs. Together they cover every ClickHouse statement-introducing -// keyword that can legally follow a CTE list. The CTE-aware scanner in -// containsMutationVerbAtTopLevel needs the union to identify *which* token -// is the statement keyword — without it, ordinary identifiers in the CTE -// list (table names, database names like the ClickHouse-built-in `system`) -// can collide with mutation-verb names and false-positive the classifier. -var nonMutationVerbs = map[string]struct{}{ - "SELECT": {}, - "SHOW": {}, - "DESCRIBE": {}, - "DESC": {}, - "EXPLAIN": {}, - "EXISTS": {}, - "CHECK": {}, -} - -// containsMutationVerbAtTopLevel scans s for the statement-introducing -// keyword at paren-depth 0, stepping over SQL string literals (`'…'` with -// `”` escape), quoted identifiers (`"…"` and “ `…` “), parenthesized CTE -// subqueries, and SQL comments. The CTE list contains ordinary identifiers -// (CTE names, table/database names) that must not be matched as mutation -// verbs — `system` would otherwise pattern-match `SYSTEM` and route a -// `WITH … SELECT * FROM system.tables` read through `Exec` (silent empty- -// array result instead of the actual rows). Two-part fix: -// -// 1. Skip identifiers whose next non-whitespace, non-comment token is -// `AS` (case-insensitive) or `(` — those are CTE definition names -// (with optional column list before AS). This catches the harder -// class where the CTE alias is itself a mutation-verb name -// (`WITH set AS (…) SELECT …`, `WITH alter AS (…) …`, etc.). -// 2. Among the remaining identifiers, stop on the FIRST that's a -// known statement keyword (mutation OR read-class), and decide -// based on mutationVerbs membership. -// -// Tokens that aren't CTE names and aren't statement keywords (RECURSIVE, -// MATERIALIZED, scalar CTE aliases, etc.) are skipped silently. Returns -// false if no statement keyword is found — the SQL is syntactically -// incomplete or unrecognised; safer to treat as non-mutation than to -// silently route an unknown verb through Exec (an Exec'd SELECT returns -// `[]` with no error; a Query'd unrecognised statement surfaces a clear -// error). -func containsMutationVerbAtTopLevel(s string) bool { - depth := 0 - i := 0 - for i < len(s) { - c := s[i] - switch { - case c == '(': - depth++ - i++ - case c == ')': - if depth > 0 { - depth-- - } - i++ - case c == '\'': - i++ - for i < len(s) { - if s[i] == '\'' { - if i+1 < len(s) && s[i+1] == '\'' { - i += 2 - continue - } - i++ - break - } - i++ - } - case c == '"' || c == '`': - q := c - i++ - for i < len(s) && s[i] != q { - i++ - } - if i < len(s) { - i++ - } - case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'): - start := i - for i < len(s) { - c2 := s[i] - if (c2 < 'A' || c2 > 'Z') && (c2 < 'a' || c2 > 'z') && (c2 < '0' || c2 > '9') && c2 != '_' { - break - } - i++ - } - if depth == 0 { - kw := strings.ToUpper(s[start:i]) - // Check non-mutation statement keywords (SELECT, SHOW, - // DESCRIBE, …) FIRST — these can legitimately be followed - // by `(` (e.g. `SELECT (1) FROM …`, `SELECT (a, b) FROM …` - // for tuple syntax), so we must not let the CTE-name - // lookahead below misclassify them as CTE aliases. - if _, ok := nonMutationVerbs[kw]; ok { - return false - } - // CTE name suppression: an identifier that ISN'T a - // non-mutation statement keyword and is followed by `AS` - // or `(` is a CTE definition name (with optional column - // list before AS). Skip without checking mutationVerbs - // — protects against CTE aliases that share a spelling - // with a mutation verb (`WITH set AS (...)`, - // `WITH alter AS (...)`, etc.). - if isCTENameLookahead(s, i) { - continue - } - if _, ok := mutationVerbs[kw]; ok { - return true - } - } - case c == '-' && i+1 < len(s) && s[i+1] == '-', c == '#': - for i < len(s) && s[i] != '\n' { - i++ - } - case c == '/' && i+1 < len(s) && s[i+1] == '*': - i += 2 - for i+1 < len(s) { - if s[i] == '*' && s[i+1] == '/' { - i += 2 - break - } - i++ - } - default: - i++ - } - } - return false -} - -// isCTENameLookahead returns true if the next non-whitespace, non-comment -// token at or after pos is `AS` (case-insensitive, word-boundary terminated) -// or `(` — signaling that whatever identifier just ended at pos is a CTE -// definition name (with optional column list before AS). Walks past space / -// tab / newline / `--` line comments / `#` line comments / `/* … */` block -// comments. Returns false on EOF or any other token. -func isCTENameLookahead(s string, pos int) bool { - i := pos - for i < len(s) { - c := s[i] - switch { - case c == ' ' || c == '\t' || c == '\r' || c == '\n': - i++ - case c == '-' && i+1 < len(s) && s[i+1] == '-', c == '#': - for i < len(s) && s[i] != '\n' { - i++ - } - case c == '/' && i+1 < len(s) && s[i+1] == '*': - i += 2 - for i+1 < len(s) { - if s[i] == '*' && s[i+1] == '/' { - i += 2 - break - } - i++ - } - case c == '(': - return true - case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'): - end := i - for end < len(s) { - c2 := s[end] - if (c2 < 'A' || c2 > 'Z') && (c2 < 'a' || c2 > 'z') && (c2 < '0' || c2 > '9') && c2 != '_' { - break - } - end++ - } - return strings.EqualFold(s[i:end], "AS") - default: - return false - } - } - return false -} - -// stripLeadingSQLComments trims whitespace plus line comments (`-- …` and -// MySQL-compat `# …`, both accepted by ClickHouse) and `/* block */` -// comments from the front of sql, returning the remainder with no leading -// whitespace. Unclosed block comments swallow the rest of the string — -// matches what ClickHouse itself would do at parse time. -func stripLeadingSQLComments(sql string) string { - s := strings.TrimLeft(sql, " \t\r\n") - for { - switch { - case strings.HasPrefix(s, "--"), strings.HasPrefix(s, "#"): - if i := strings.IndexByte(s, '\n'); i >= 0 { - s = strings.TrimLeft(s[i+1:], " \t\r\n") - } else { - return "" - } - case strings.HasPrefix(s, "/*"): - if i := strings.Index(s[2:], "*/"); i >= 0 { - s = strings.TrimLeft(s[2+i+2:], " \t\r\n") - } else { - return "" - } - default: - return s - } - } -} - -// transformRow converts ClickHouse-specific types to JSON-friendly values. -func transformRow(row map[string]any) map[string]any { - for k, v := range row { - switch val := v.(type) { - case uuid.UUID: - row[k] = val.String() - case [16]byte: - row[k] = uuid.UUID(val).String() - case time.Time: - row[k] = val.UTC().Format(time.RFC3339Nano) - case *time.Time: - // Nullable(DateTime…) scans as a pointer; NULL stays nil (JSON null). - if val != nil { - row[k] = val.UTC().Format(time.RFC3339Nano) - } - } - } - return row -} diff --git a/internal/api/clickhouse_exec_test.go b/internal/api/clickhouse_exec_test.go deleted file mode 100644 index 947413b7..00000000 --- a/internal/api/clickhouse_exec_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package api - -import ( - "context" - "reflect" - "testing" - "time" - - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// stubConn records how many times Exec or Query was called and returns -// canned results. The embedded nil driver.Conn keeps every method we don't -// override undefined-method-call-panic'd, which is what we want — the test -// fails loudly if executeCHQuery starts touching new surface area. -type stubConn struct { - driver.Conn - execCount int - queryCount int - execErr error - // queryRows, when non-nil, is returned by Query in place of the default - // empty rows. Tests that exercise row-scan / transformRow paths use this. - queryRows driver.Rows -} - -func (c *stubConn) Exec(_ context.Context, _ string, _ ...any) error { - c.execCount++ - return c.execErr -} - -func (c *stubConn) Query(_ context.Context, _ string, _ ...any) (driver.Rows, error) { - c.queryCount++ - if c.queryRows != nil { - return c.queryRows, nil - } - return &chainEmptyRows{}, nil -} - -func TestIsMutation(t *testing.T) { - t.Parallel() - tests := []struct { - name string - sql string - want bool - }{ - {"select", "SELECT 1", false}, - {"select lower", "select 1", false}, - {"with cte", "WITH x AS (SELECT 1) SELECT * FROM x", false}, - {"show", "SHOW TABLES", false}, - {"describe", "DESCRIBE clicks", false}, - {"explain", "EXPLAIN SELECT 1", false}, - {"exists", "EXISTS TABLE clicks", false}, - - {"insert", "INSERT INTO t VALUES (1)", true}, - {"update", "UPDATE t SET a=1 WHERE b=2", true}, - {"delete", "DELETE FROM t WHERE id=1", true}, - {"truncate", "TRUNCATE TABLE t", true}, - {"truncate lower", "truncate table t", true}, - {"drop", "DROP TABLE t", true}, - {"alter", "ALTER TABLE t ADD COLUMN c String", true}, - {"create", "CREATE TABLE t (a Int)", true}, - {"rename", "RENAME TABLE a TO b", true}, - {"exchange", "EXCHANGE TABLES t1 AND t2", true}, - {"optimize", "OPTIMIZE TABLE t", true}, - {"replace", "REPLACE INTO t VALUES (1)", true}, - {"grant", "GRANT SELECT ON t TO u", true}, - {"revoke", "REVOKE SELECT ON t FROM u", true}, - {"system", "SYSTEM RELOAD CONFIG", true}, - {"attach", "ATTACH TABLE t FROM '/path'", true}, - {"detach", "DETACH TABLE t", true}, - {"kill", "KILL QUERY WHERE query_id = 'abc'", true}, - {"set", "SET max_threads = 4", true}, - {"use", "USE mydb", true}, - - {"leading whitespace", " \n\tTRUNCATE TABLE t", true}, - {"line comment then mutation", "-- drop guard\nDROP TABLE t", true}, - {"hash line comment then mutation", "# audit\nDROP TABLE t", true}, - {"block comment then mutation", "/* admin */ ALTER TABLE t ADD COLUMN c Int", true}, - {"mixed comments then select", "-- foo\n# bar\n/* baz */ SELECT 1", false}, - {"with insert", "WITH cte AS (SELECT 1) INSERT INTO t SELECT * FROM cte", true}, - {"with insert lower", "with cte as (select 1) insert into t select * from cte", true}, - {"with delete", "WITH cte AS (SELECT id FROM x) DELETE FROM t WHERE id IN (SELECT id FROM cte)", true}, - {"with update", "WITH cte AS (SELECT 1) ALTER TABLE t UPDATE a=1 WHERE id IN (SELECT id FROM cte)", true}, - {"with truncate", "WITH cte AS (SELECT 1) TRUNCATE TABLE t", true}, - {"with multi-cte insert", "WITH a AS (SELECT 1), b AS (SELECT 2) INSERT INTO t SELECT * FROM a JOIN b", true}, - {"with nested parens insert", "WITH cte AS (SELECT id FROM t WHERE id IN (1,2,3)) INSERT INTO t2 SELECT * FROM cte", true}, - {"with paren-in-string insert", "WITH cte AS (SELECT ')' AS x) INSERT INTO t2 SELECT * FROM cte", true}, - {"with materialized insert", "WITH cte AS MATERIALIZED (SELECT 1) INSERT INTO t SELECT * FROM cte", true}, - {"with recursive select", "WITH RECURSIVE x AS (SELECT 1 UNION ALL SELECT * FROM x) SELECT * FROM x", false}, - {"with nested select", "WITH x AS (SELECT 1) SELECT * FROM (SELECT * FROM x)", false}, - {"with scalar insert", "WITH '/path' AS p INSERT INTO files VALUES (p)", true}, - {"with line comment containing DELETE then select", "WITH cte AS (SELECT 1) -- old DELETE approach\nSELECT * FROM cte", false}, - {"with hash comment containing TRUNCATE then select", "WITH cte AS (SELECT 1) # was TRUNCATE\nSELECT * FROM cte", false}, - {"with block comment containing INSERT then select", "WITH cte AS (SELECT 1) /* INSERT reminder */ SELECT * FROM cte", false}, - {"with comment then real mutation", "WITH cte AS (SELECT 1) -- explanatory\nINSERT INTO t SELECT * FROM cte", true}, - {"with unclosed block comment", "WITH cte AS (SELECT 1) /* unterminated comment DELETE", false}, - {"with select from system tables (collision regression)", "WITH x AS (SELECT 1) SELECT * FROM system.tables", false}, - {"with select from system columns lower (collision regression)", "with x as (select 1) select name from system.columns", false}, - {"with select aliased as set (false positive regression)", "WITH cte AS (SELECT 1) SELECT * FROM cte AS set", false}, - {"with select from system tables then real insert", "WITH x AS (SELECT * FROM system.tables) INSERT INTO snapshot SELECT * FROM x", true}, - {"with CTE alias named set (read)", "WITH set AS (SELECT 1) SELECT * FROM set", false}, - {"with CTE alias named alter (read)", "WITH alter AS (SELECT 1) SELECT id FROM alter", false}, - {"with CTE alias named drop lowercase (read)", "with drop as (select 1) select * from drop", false}, - {"with CTE alias named update then real update", "WITH update AS (SELECT id FROM x) ALTER TABLE other UPDATE c=1 WHERE id IN (SELECT id FROM update)", true}, - {"with CTE name with column list (read)", "WITH cte (a, b) AS (SELECT 1, 2) SELECT * FROM cte", false}, - {"with multi-CTE both with verb-name aliases (read)", "WITH set AS (SELECT 1), kill AS (SELECT 2) SELECT * FROM set JOIN kill", false}, - {"with parenthesized SELECT then system table (CTE-lookahead ordering regression)", "WITH x AS (SELECT 1) SELECT (1) FROM system.tables", false}, - {"with tuple-shape SELECT then system table", "WITH x AS (SELECT 1) SELECT (a, b) FROM system.parts", false}, - - {"empty", "", false}, - {"comment only", "-- just a comment", false}, - {"unclosed block comment", "/* never closed", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, isMutation(tt.sql)) - }) - } -} - -func TestExecuteCHQuery_MutationRoutesToExec(t *testing.T) { - t.Parallel() - // Mutations route through driver.Exec because clickhouse-go's - // driver.Query() errors on statements that return no result set. - // executeCHQuery marshals the no-rows case to `[]` so the response shape - // stays "always an array" regardless of whether the SQL was a read or - // a mutation. Used by structured_query and pipes handlers; the raw-SQL - // handler bypasses this entirely (HTTP proxy). - for _, sql := range []string{ - "TRUNCATE TABLE clicks", - "DROP TABLE clicks", - "DELETE FROM clicks WHERE id = 1", - "ALTER TABLE clicks ADD COLUMN c String", - "INSERT INTO clicks VALUES (1)", - " -- audit log\n UPDATE clicks SET v = 1 WHERE id = 2", - } { - t.Run(sql, func(t *testing.T) { - t.Parallel() - conn := &stubConn{} - rows, err := executeCHQuery(context.Background(), conn, sql, nil) - require.NoError(t, err) - assert.Equal(t, 1, conn.execCount, "Exec must be used for mutations") - assert.Zero(t, conn.queryCount, "Query must not be used for mutations") - assert.Equal(t, []map[string]any{}, rows, "mutation result must marshal to [] not null") - }) - } -} - -func TestExecuteCHQuery_SelectRoutesToQuery(t *testing.T) { - t.Parallel() - conn := &stubConn{} - rows, err := executeCHQuery(context.Background(), conn, "SELECT 1", nil) - require.NoError(t, err) - assert.Zero(t, conn.execCount, "Exec must not be used for SELECT") - assert.Equal(t, 1, conn.queryCount, "Query must be used for SELECT") - assert.Equal(t, []map[string]any{}, rows, "zero-row SELECT must marshal to [] not null") -} - -// TestExecuteCHQuery_TransformsClickHouseTypes pins transformRow's contract -// at the unit level: UUIDs become canonical strings, time.Time — including the -// *time.Time a Nullable(DateTime…) column scans as — becomes RFC3339Nano UTC -// (NULL stays JSON null), and other scalars pass through unchanged. The -// integration suite exercises the same path against a real ClickHouse, but -// this unit test catches regressions in the type-conversion branches without -// standing up testcontainers. -func TestExecuteCHQuery_TransformsClickHouseTypes(t *testing.T) { - t.Parallel() - id := uuid.MustParse("11111111-2222-3333-4444-555555555555") - ts := time.Date(2026, 5, 19, 12, 30, 45, 123456789, time.FixedZone("EST", -5*3600)) - conn := &stubConn{queryRows: &chainOneRow{ - columns: []chainColumnType{ - {name: "id", scanType: reflect.TypeFor[uuid.UUID]()}, - {name: "received_at", scanType: reflect.TypeFor[time.Time]()}, - {name: "updated_at", scanType: reflect.TypeFor[*time.Time]()}, - {name: "deleted_at", scanType: reflect.TypeFor[*time.Time]()}, - {name: "n", scanType: reflect.TypeFor[int64]()}, - }, - values: []any{id, ts, &ts, (*time.Time)(nil), int64(42)}, - }} - - rows, err := executeCHQuery(context.Background(), conn, "SELECT id, received_at, updated_at, deleted_at, n FROM t", nil) - require.NoError(t, err) - require.Len(t, rows, 1) - assert.Equal(t, id.String(), rows[0]["id"], "UUID must be stringified") - assert.Equal(t, ts.UTC().Format(time.RFC3339Nano), rows[0]["received_at"], "time must be RFC3339Nano in UTC") - assert.Equal(t, ts.UTC().Format(time.RFC3339Nano), rows[0]["updated_at"], "nullable time must be RFC3339Nano in UTC") - assert.Nil(t, rows[0]["deleted_at"], "NULL nullable time must stay nil") - assert.Equal(t, int64(42), rows[0]["n"], "scalar must pass through unchanged") -} - -// chainOneRow implements driver.Rows for a single canned row. Scan -// reflect-writes values[i] into the i-th destination pointer that -// executeCHQuery allocates from ColumnTypes()[i].ScanType(). -type chainOneRow struct { - driver.Rows - columns []chainColumnType - values []any - yielded bool -} - -func (r *chainOneRow) Next() bool { - if r.yielded { - return false - } - r.yielded = true - return true -} - -func (r *chainOneRow) Scan(dest ...any) error { - for i, d := range dest { - reflect.ValueOf(d).Elem().Set(reflect.ValueOf(r.values[i])) - } - return nil -} - -func (*chainOneRow) Close() error { return nil } -func (*chainOneRow) Err() error { return nil } - -func (r *chainOneRow) ColumnTypes() []driver.ColumnType { - out := make([]driver.ColumnType, len(r.columns)) - for i := range r.columns { - out[i] = &r.columns[i] - } - return out -} - -type chainColumnType struct { - driver.ColumnType - name string - scanType reflect.Type -} - -func (c *chainColumnType) Name() string { return c.name } -func (c *chainColumnType) ScanType() reflect.Type { return c.scanType } diff --git a/internal/api/clickhouse_http.go b/internal/api/clickhouse_http.go new file mode 100644 index 00000000..ed190108 --- /dev/null +++ b/internal/api/clickhouse_http.go @@ -0,0 +1,193 @@ +package api + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/Wave-RF/WaveHouse/internal/chconn" +) + +// chReader runs the cached read paths — the structured query and named pipes +// — against ClickHouse's HTTP interface and hands back ClickHouse's own JSON +// rendering of the rows. +// +// Asking ClickHouse for JSON rather than scanning rows through the native +// driver is what keeps the response correct across server versions: the +// spelling of a Decimal, a DateTime64's scale, an Enum's name and an IPv6's +// compression are the server's to decide, and they change between versions. +// It is also why the SELECT-vs-mutation classifier this file replaced is +// gone: the classifier existed because clickhouse-go's Query() errors on a +// statement with no result set, and a structured query or a pipe is a read by +// construction. The raw-SQL escape hatch (/v1/ops/query) has proxied to the +// same interface for the same reasons since it was written; see query.go. +type chReader struct { + client *http.Client + // target resolves the ClickHouse HTTP wiring per request (chconn.Manager + // in production) so a settings reload applies to the next read. + target func() chconn.Target + // maxResponseBytes optionally overrides maxCHResponseBytes. Test-only + // seam for the cap-overflow path; not a production knob. + maxResponseBytes int64 +} + +// newCHReader builds the shared reader. The client has no Timeout — every +// read carries a context deadline derived from the inbound request, which +// bounds the whole exchange including the body read. +func newCHReader(target func() chconn.Target) *chReader { + return &chReader{ + client: &http.Client{ + // The target is operator config, not user input, and ClickHouse + // does not redirect in normal operation: surface a 3xx as itself + // rather than chase it. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + target: target, + } +} + +// chOutputFormat is the format and the rendering knobs that make up the +// response contract. JSONEachRow rather than JSON because the framing is one +// row per line and the array wrapper costs a single pass; the rows themselves +// are byte-identical between the two. +// +// date_time_output_format is deliberately absent: ClickHouse's default is the +// `YYYY-MM-DD HH:MM:SS[.fff]` spelling the SSE wire already carries, and the +// two surfaces must agree byte for byte (#372). /v1/ops/query sets `iso` +// because an admin reading raw SQL output is a different audience. +// +// 64-bit integers are pinned UNQUOTED because that is what the endpoint has +// always returned; it is lossy in a JavaScript consumer past 2^53, but that +// was already true and quoting them now would break every consumer that does +// arithmetic on one. The setting is sent explicitly rather than inherited so +// a server-side profile cannot silently change the contract. +// +// Denormal floats are deliberately NOT quoted: ClickHouse's default renders a +// NaN or an Inf as JSON `null`, which is valid JSON, where quoting them puts +// the string "nan" in a numeric field. (Go's marshaller used to fail the whole +// response with "unsupported value: NaN", so anything is an improvement.) +var chOutputFormat = map[string]string{ + "default_format": "JSONEachRow", + "output_format_json_quote_64bit_integers": "0", +} + +// query executes sql and returns the result rows as a JSON array — the +// response body both cached read paths serve. +// +// params supply `param_p0` … `param_pN-1` positionally, for the `{pN:String}` +// / `{pN:Array(String)}` placeholders query.BuildResult.NamedParams emitted; +// they are already encoded for ClickHouse's parameter reader. settings are the +// role's resource caps from chReadSettings. +func (c *chReader) query(ctx context.Context, sql string, params []string, settings map[string]string) ([]byte, error) { + if c == nil || c.client == nil || c.target == nil { + return nil, fmt.Errorf("clickhouse reader not configured") + } + target := c.target() + u, err := url.Parse(target.URL) + if err != nil { + return nil, fmt.Errorf("invalid clickhouse endpoint: %w", err) + } + q := u.Query() + for k, v := range chOutputFormat { + q.Set(k, v) + } + if target.Database != "" { + q.Set("database", target.Database) + } + for k, v := range settings { + q.Set(k, v) + } + for i, p := range params { + q.Set("param_p"+strconv.Itoa(i), p) + } + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), strings.NewReader(sql)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + if target.Username != "" { + req.Header.Set("X-ClickHouse-User", target.Username) + } + if target.Password != "" { + req.Header.Set("X-ClickHouse-Key", target.Password) + } + + // #nosec G704 -- the destination is operator config, not caller input: the + // scheme, host and path come from chconn.Target and only RawQuery is + // replaced, with url.Values.Encode() percent-encoding every key and value. + // A bound value therefore cannot introduce a host, a path or a fragment. + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("clickhouse request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respCap := int64(maxCHResponseBytes) + if c.maxResponseBytes > 0 { + respCap = c.maxResponseBytes + } + // Read one byte past the cap so "exactly cap or more" is detectable + // without a second read. + body, err := io.ReadAll(io.LimitReader(resp.Body, respCap+1)) + if err != nil { + return nil, fmt.Errorf("read clickhouse response: %w", err) + } + if int64(len(body)) > respCap { + return nil, fmt.Errorf("clickhouse response exceeded %d bytes; narrow the query", respCap) + } + if resp.StatusCode != http.StatusOK { + msg := strings.TrimSpace(string(body)) + if msg == "" { + msg = fmt.Sprintf("clickhouse returned status %d", resp.StatusCode) + } + return nil, fmt.Errorf("clickhouse query: %s", msg) + } + return jsonEachRowArray(body) +} + +// jsonEachRowArray frames ClickHouse's newline-delimited JSONEachRow output as +// the JSON array this endpoint has always returned. A JSONEachRow row is one +// complete JSON object on one line — a newline inside a string is escaped — +// so splitting on '\n' cannot cut a row in half, and the rows themselves are +// copied through untouched. +// +// A zero-row result is an empty body and becomes `[]`, not `null`: SDK +// consumers do `data.length` on every response. +// +// A line that does not open an object is ClickHouse's exception text appended +// after the stream had already started (it cannot revise the 200 it sent), so +// it becomes the error it would have been had it arrived in time. +func jsonEachRowArray(body []byte) ([]byte, error) { + out := make([]byte, 0, len(body)+2) + out = append(out, '[') + first := true + for len(body) > 0 { + line := body + if i := bytes.IndexByte(body, '\n'); i >= 0 { + line, body = body[:i], body[i+1:] + } else { + body = nil + } + if len(line) == 0 { + continue + } + if line[0] != '{' { + return nil, fmt.Errorf("clickhouse query: %s", strings.TrimSpace(string(line))) + } + if !first { + out = append(out, ',') + } + out = append(out, line...) + first = false + } + return append(out, ']'), nil +} diff --git a/internal/api/clickhouse_http_test.go b/internal/api/clickhouse_http_test.go new file mode 100644 index 00000000..ff468b4c --- /dev/null +++ b/internal/api/clickhouse_http_test.go @@ -0,0 +1,185 @@ +package api + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/chconn" +) + +// chStub is a ClickHouse HTTP interface that answers with whatever the test +// wants and records what it was asked. +type chStub struct { + server *httptest.Server + status int + body string + + gotSQL string + gotQuery url.Values + gotHeader http.Header +} + +func newCHStub(t testing.TB) *chStub { + t.Helper() + s := &chStub{status: http.StatusOK} + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + s.gotSQL = string(body) + s.gotQuery = r.URL.Query() + s.gotHeader = r.Header.Clone() + w.WriteHeader(s.status) + _, _ = io.WriteString(w, s.body) + })) + t.Cleanup(s.server.Close) + return s +} + +func (s *chStub) reader(target chconn.Target) *chReader { + if target.URL == "" { + target.URL = s.server.URL + } + return newCHReader(func() chconn.Target { return target }) +} + +// TestCHReader_ResponseShape pins the body the cached read paths serve. +// ClickHouse's JSONEachRow output is newline-delimited, the endpoint's +// contract is a JSON array, and a zero-row result must be `[]` and never +// `null` — every SDK consumer does `data.length` on it. +func TestCHReader_ResponseShape(t *testing.T) { + t.Parallel() + tests := []struct { + name string + body string + want string + }{ + {"no rows is an empty body", "", "[]"}, + {"one row", "{\"a\":1}\n", `[{"a":1}]`}, + {"several rows", "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n", `[{"a":1},{"a":2},{"a":3}]`}, + {"no trailing newline", "{\"a\":1}", `[{"a":1}]`}, + { + // A newline inside a value is escaped by JSONEachRow, so splitting + // on '\n' cannot cut a row in half. + name: "an escaped newline inside a value is not a row boundary", + body: "{\"a\":\"x\\ny\"}\n", + want: `[{"a":"x\ny"}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + stub := newCHStub(t) + stub.body = tt.body + got, err := stub.reader(chconn.Target{}).query(context.Background(), "SELECT 1", nil, nil) + require.NoError(t, err) + assert.Equal(t, tt.want, string(got)) + }) + } +} + +// TestCHReader_Request pins what reaches ClickHouse: the SQL as the POST body, +// the output-format contract and the database on the query string, the bound +// values as param_pN in placeholder order, the role's caps as settings, and +// the credentials as ClickHouse's own headers. +func TestCHReader_Request(t *testing.T) { + t.Parallel() + stub := newCHStub(t) + r := stub.reader(chconn.Target{Username: "u", Password: "p", Database: "warehouse"}) + + _, err := r.query(context.Background(), + "SELECT * FROM `t` WHERE `a` = {p0:String} AND `b` IN {p1:Array(String)}", + []string{"/home", "['x','y']"}, + map[string]string{"max_rows_to_read": "1", "read_overflow_mode": "throw"}, + ) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM `t` WHERE `a` = {p0:String} AND `b` IN {p1:Array(String)}", stub.gotSQL) + assert.Equal(t, "JSONEachRow", stub.gotQuery.Get("default_format")) + // Unquoted 64-bit integers are the contract this endpoint has always + // returned; pinned explicitly so a server-side profile cannot change it. + assert.Equal(t, "0", stub.gotQuery.Get("output_format_json_quote_64bit_integers")) + // DateTime rendering is deliberately NOT overridden: ClickHouse's default + // is the same spelling the SSE wire carries (#372). + assert.Empty(t, stub.gotQuery.Get("date_time_output_format")) + assert.Equal(t, "warehouse", stub.gotQuery.Get("database")) + assert.Equal(t, "/home", stub.gotQuery.Get("param_p0")) + assert.Equal(t, "['x','y']", stub.gotQuery.Get("param_p1")) + assert.Equal(t, "1", stub.gotQuery.Get("max_rows_to_read")) + assert.Equal(t, "throw", stub.gotQuery.Get("read_overflow_mode")) + assert.Equal(t, "u", stub.gotHeader.Get("X-ClickHouse-User")) + assert.Equal(t, "p", stub.gotHeader.Get("X-ClickHouse-Key")) +} + +// TestCHReader_Errors covers every way a read fails. ClickHouse's own message +// is carried through verbatim in each case — it is the only diagnostic an +// operator gets. +func TestCHReader_Errors(t *testing.T) { + t.Parallel() + + t.Run("non-200 surfaces ClickHouse's message", func(t *testing.T) { + t.Parallel() + stub := newCHStub(t) + stub.status = http.StatusInternalServerError + stub.body = "Code: 158. DB::Exception: Limit for rows exceeded. (TOO_MANY_ROWS)\n" + _, err := stub.reader(chconn.Target{}).query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "Code: 158") + assert.Contains(t, err.Error(), "TOO_MANY_ROWS") + }) + + t.Run("an empty non-200 body still names the status", func(t *testing.T) { + t.Parallel() + stub := newCHStub(t) + stub.status = http.StatusBadGateway + _, err := stub.reader(chconn.Target{}).query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "502") + }) + + t.Run("an exception appended mid-stream is an error, not a row", func(t *testing.T) { + t.Parallel() + // ClickHouse had already sent 200 and some rows when the query failed, + // so it appends the exception to the body. Without this check the text + // would be spliced into the JSON array as if it were data. + stub := newCHStub(t) + stub.body = "{\"a\":1}\nCode: 241. DB::Exception: Memory limit exceeded. (MEMORY_LIMIT_EXCEEDED)\n" + _, err := stub.reader(chconn.Target{}).query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "Code: 241") + }) + + t.Run("an oversized response is refused, not buffered", func(t *testing.T) { + t.Parallel() + stub := newCHStub(t) + stub.body = strings.Repeat("{\"a\":1}\n", 100) + r := stub.reader(chconn.Target{}) + r.maxResponseBytes = 32 + _, err := r.query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "exceeded 32 bytes") + }) + + t.Run("an unconfigured reader reports itself", func(t *testing.T) { + t.Parallel() + // A zero-value handler (routing-only tests) must surface a diagnostic + // rather than panic on a nil target. + _, err := newCHReader(nil).query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "not configured") + }) + + t.Run("an unreachable endpoint is an error", func(t *testing.T) { + t.Parallel() + r := newCHReader(func() chconn.Target { return chconn.Target{URL: "http://127.0.0.1:1"} }) + _, err := r.query(context.Background(), "SELECT 1", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "clickhouse request failed") + }) +} diff --git a/internal/api/record_reader.go b/internal/api/content_type.go similarity index 59% rename from internal/api/record_reader.go rename to internal/api/content_type.go index cef51468..b0db3731 100644 --- a/internal/api/record_reader.go +++ b/internal/api/content_type.go @@ -1,54 +1,19 @@ package api import ( - "bufio" - "bytes" - "encoding/json" "errors" "fmt" - "io" "mime" "slices" "strings" -) - -const ( - // maxNDJSONLineBytes caps a single NDJSON record so one pathological line - // can't force an unbounded read buffer. 10 MiB is far above any realistic - // flat ingest record; a line larger than this aborts the whole request. - maxNDJSONLineBytes = 10 << 20 // 10 MiB - // maxSniffBytes bounds how far the arity peek looks for the first - // non-whitespace byte. Far beyond any reasonable amount of leading - // whitespace; a body that is only whitespace within this window is treated - // as empty. - maxSniffBytes = 512 + "github.com/Wave-RF/WaveHouse/internal/typelayer" ) -// recordReader yields ingest records one at a time from a request body. Each -// concrete reader covers one wire format (single JSON object, JSON array, -// NDJSON, and — later — CSV), so the handler stays format-agnostic and new -// formats / transports (streaming uploads) slot in behind this one interface. -// -// Next returns io.EOF at the clean end of the body. A *recordSyntaxError is a -// recoverable per-record decode failure — the framing let the reader resync, so -// the handler records it and continues. Any other non-EOF error is fatal to the -// request. -type recordReader interface { - Next() (map[string]any, error) -} - -// recordSyntaxError marks a per-record decode failure the reader recovered from -// (the framing let it skip to the next record). The batch handler turns it into -// a recordResult error; it carries no HTTP status because the decode layer sits -// below the validation/permission layer that owns status codes. -type recordSyntaxError struct{ msg string } - -func (e *recordSyntaxError) Error() string { return e.msg } - -// errEmptyBody is returned by newRecordReader when the body has no content (no -// non-whitespace byte within the sniff window). The handler maps it to a 400. -var errEmptyBody = errors.New("empty body") +// maxSniffBytes bounds how far the arity peek looks for the first +// non-whitespace byte. Far beyond any reasonable amount of leading whitespace; +// a body that is only whitespace within this window is treated as empty. +const maxSniffBytes = 512 // errUnsupportedContentType is returned when the request declares no // Content-Type, one whose media type is not in the accepted list, several that @@ -69,25 +34,47 @@ var errConflictingContentType = errors.New("conflicting content type") // IngestFormat is the wire format of an ingest request body. It comes from the // request's declared Content-Type and nothing else: the body never overrides -// what the client said it sent, so a caller can always tell how their bytes -// will be read without knowing what the first one happens to be. +// what the client said it sent, so a caller can always tell how their bytes will +// be read without knowing what the first one happens to be. +// +// It is NOT the same thing as the chtypes format the parser is handed — see +// wire(). The two JSON families are one format to ClickHouse and two formats +// here, because they differ in how a body frames its records, which is what +// decides the response shape and how many records the request contains. type IngestFormat int const ( // FormatJSON is the application/json family: one flat object, or a // top-level array of them. Which of the two is the body's own business — // the first non-whitespace byte picks it — because both are the same - // format, differing only in arity. + // format to the parser, differing only in arity. FormatJSON IngestFormat = iota - // FormatNDJSON is newline-delimited JSON: one flat object per line. A line - // that is not a JSON object is a per-record error, never a reason to - // re-read the body as something else. + // FormatNDJSON is newline-delimited JSON: one flat object per line. Byte + // for byte the same format ClickHouse calls JSONEachRow; the distinction + // from FormatJSON survives only so a declared-NDJSON body is always a + // batch and never has its arity sniffed. FormatNDJSON - // FormatCSV plugs in here once ingest reads CSV: add the media types to - // acceptedContentTypes, which advertises them in the 415 automatically, and - // the reader to newRecordReader. Do NOT add them in ingestFormatOne — it - // resolves by scanning that one table, and a second list is the drift this - // arrangement exists to prevent. + // FormatCSV is bare `text/csv`: positional in the table's declaration + // order (see wireColumns), under ClickHouse's own header auto-detection — a + // first line that spells the column names is consumed as a header, not a + // record. + FormatCSV + // FormatTSV is FormatCSV's tab-separated twin. + FormatTSV + // FormatCSVWithNames is CSV whose first line names the columns, in any + // order — `text/csv; header=present`, RFC 4180 §3's parameter. The header + // is not a record; a column it omits takes its DEFAULT, and a name the + // table (or the role's projection of it) lacks refuses the request with + // ClickHouse's code 117. + FormatCSVWithNames + // FormatTSVWithNames is FormatCSVWithNames' tab-separated twin. + FormatTSVWithNames + // FormatCSVPositional is `text/csv; header=absent`: strictly positional, + // detection off, so a header line is one record that fails to parse with + // ClickHouse's code 27. + FormatCSVPositional + // FormatTSVPositional is FormatCSVPositional's tab-separated twin. + FormatTSVPositional ) // String renders a format for error messages and logs. @@ -97,11 +84,54 @@ func (f IngestFormat) String() string { return "json" case FormatNDJSON: return "ndjson" + case FormatCSV: + return "csv" + case FormatTSV: + return "tsv" + case FormatCSVWithNames: + return "csvwithnames" + case FormatTSVWithNames: + return "tsvwithnames" + case FormatCSVPositional: + return "csv" + case FormatTSVPositional: + return "tsv" default: return "unknown" } } +// wire is the format chtypes parses the body as. Both JSON families collapse +// onto JSONEachRow: an NDJSON body, a bare object, concatenated objects and a +// newline-framed array are all the same input to ClickHouse's own reader. +func (f IngestFormat) wire() typelayer.Format { + switch f { + case FormatCSV, FormatCSVPositional: + return typelayer.FormatCSV + case FormatTSV, FormatTSVPositional: + return typelayer.FormatTSV + case FormatCSVWithNames: + return typelayer.FormatCSVWithNames + case FormatTSVWithNames: + return typelayer.FormatTSVWithNames + case FormatJSON, FormatNDJSON: + return typelayer.FormatJSONEachRow + default: + return typelayer.FormatJSONEachRow + } +} + +// options are the parse options the format adds to wire(): header detection is +// off exactly for the header=absent pair. +func (f IngestFormat) options() typelayer.IngestOptions { + return typelayer.IngestOptions{StrictPositional: f == FormatCSVPositional || f == FormatTSVPositional} +} + +// alwaysBatch reports whether this format's response is the per-record batch +// shape whatever the body holds. Only the JSON family has an arity question, +// and the first non-whitespace byte answers it (see firstNonSpace). +func (f IngestFormat) alwaysBatch() bool { return f != FormatJSON } + // acceptedContentTypes maps every media type ingest reads to the format it // selects, in the order the 415 body advertises them. The first entry of each // family is the canonical spelling — the TS SDK sends those two. It is the @@ -113,15 +143,31 @@ func (f IngestFormat) String() string { // architecture.md all failed to name — and no test can close that direction by // enumeration, because the complement is unbounded. One table closes it by // construction. +// +// header is the `header` parameter an entry requires: "" matches a declaration +// without one; "present" and "absent" match only that value. It is the one +// parameter that decides a format, and only for the two media types that list +// non-empty entries. RFC 4180 §3 defines it for text/csv, and it maps onto +// ClickHouse's own behaviour three ways: present is the WithNames format, +// absent is strictly positional (detection off), and none leaves ClickHouse's +// default header auto-detection on. text/tab-separated-values takes the same +// mapping (IANA defines no parameters for it). var acceptedContentTypes = []struct { mediaType string + header string format IngestFormat }{ - {"application/json", FormatJSON}, - {"application/x-ndjson", FormatNDJSON}, - {"application/ndjson", FormatNDJSON}, - {"application/jsonl", FormatNDJSON}, - {"application/jsonlines", FormatNDJSON}, + {"application/json", "", FormatJSON}, + {"application/x-ndjson", "", FormatNDJSON}, + {"application/ndjson", "", FormatNDJSON}, + {"application/jsonl", "", FormatNDJSON}, + {"application/jsonlines", "", FormatNDJSON}, + {"text/csv", "", FormatCSV}, + {"text/csv", "present", FormatCSVWithNames}, + {"text/csv", "absent", FormatCSVPositional}, + {"text/tab-separated-values", "", FormatTSV}, + {"text/tab-separated-values", "present", FormatTSVWithNames}, + {"text/tab-separated-values", "absent", FormatTSVPositional}, } // supportedContentTypes is what the 415 body lists and the docs quote, derived @@ -130,134 +176,13 @@ var supportedContentTypes = func() []string { out := make([]string, len(acceptedContentTypes)) for i, a := range acceptedContentTypes { out[i] = a.mediaType + if a.header != "" { + out[i] += "; header=" + a.header + } } return out }() -// errUnterminatedArray marks a JSON array body that ended before its closing -// ']' (a truncated / cut-off upload). It is deliberately NOT io.EOF so the -// batch loop fails the whole request (400) instead of treating the records that -// did arrive as a complete, successful batch. -var errUnterminatedArray = errors.New("unterminated json array") - -// objectReader decodes exactly one flat JSON object — the single-object ingest -// path. A second Next returns io.EOF. Trailing bytes after the first object are -// ignored (matching the historical single-object behavior), so the response -// shape never depends on what follows the object. -type objectReader struct { - dec *json.Decoder - done bool -} - -func (o *objectReader) Next() (map[string]any, error) { - if o.done { - return nil, io.EOF - } - o.done = true - var m map[string]any - if err := o.dec.Decode(&m); err != nil { - return nil, err // fatal: handler maps this to 400 invalid json - } - return m, nil -} - -// arrayReader streams the elements of a top-level JSON array. A wrong-typed -// element (a scalar/array where an object was expected) yields a -// *json.UnmarshalTypeError, which leaves the decoder in sync — recoverable, so -// it becomes a per-record error and iteration continues. A *json.SyntaxError -// desyncs the decoder and is returned as fatal. -type arrayReader struct { - dec *json.Decoder - started bool - done bool -} - -func (a *arrayReader) Next() (map[string]any, error) { - if a.done { - return nil, io.EOF - } - if !a.started { - if _, err := a.dec.Token(); err != nil { // consume the opening '[' - a.done = true - return nil, err - } - a.started = true - } - if !a.dec.More() { - a.done = true - // More() reports false not only at a clean ']', but also on a read error - // and on a truncated array (EOF before ']'), swallowing both — which - // would let dropped records masquerade as a complete partial-200 insert. - // Read the closing token to tell the cases apart: only a ']' ends the - // batch; a missing or non-']' close means the upload was cut off (→ 400, - // via errUnterminatedArray, which is NOT io.EOF) and fails the whole - // request. - // - // The body-cap case no longer reaches here — the handler buffers the - // whole body first, so a cap overflow is a 413 before any reader exists, - // and this operates on an in-memory slice that cannot fail a read. - tok, err := a.dec.Token() - if err != nil { - if errors.Is(err, io.EOF) { - return nil, errUnterminatedArray - } - return nil, err - } - if d, ok := tok.(json.Delim); !ok || d != ']' { - return nil, errUnterminatedArray - } - return nil, io.EOF - } - var m map[string]any - if err := a.dec.Decode(&m); err != nil { - if _, ok := errors.AsType[*json.UnmarshalTypeError](err); ok { - // Decoder stayed in sync past the bad element — recoverable. - return nil, &recordSyntaxError{"record must be a JSON object"} - } - // Syntax/read error: the decoder is desynced, the rest of the array is - // unrecoverable. Don't try to read the closing ']'. - a.done = true - if errors.Is(err, io.EOF) { - // More() said an element followed (e.g. after a trailing comma) but - // the stream ended — a truncated array, not a clean close. Map to a - // fatal error rather than the io.EOF the batch loop treats as "done". - return nil, errUnterminatedArray - } - return nil, err - } - return m, nil -} - -// lineReader yields one record per non-blank line of an NDJSON body. It recovers -// from both type and syntax errors per line (the newline reframes the next -// record), so a single malformed line never blocks the rest of the batch. -type lineReader struct { - sc *bufio.Scanner -} - -func (l *lineReader) Next() (map[string]any, error) { - for l.sc.Scan() { - line := bytes.TrimSpace(l.sc.Bytes()) - if len(line) == 0 { - continue // skip blank lines between records - } - var m map[string]any - dec := json.NewDecoder(bytes.NewReader(line)) - dec.UseNumber() - if err := dec.Decode(&m); err != nil { - return nil, &recordSyntaxError{"invalid json"} - } - return m, nil - } - if err := l.sc.Err(); err != nil { - // A line exceeding maxNDJSONLineBytes (bufio.ErrTooLong) — the scanner - // can't resume, so fail the request. Not the body cap: that trips in the - // handler before this reader is built. - return nil, err - } - return nil, io.EOF -} - // resolveContentType resolves the Content-Type header set to the format ingest // reads the body as. Content-Type is a singleton field (RFC 9110 §8.3) and §5.3 // forbids repeating it, so a duplicate is malformed however it is spelled. §8.3 @@ -283,8 +208,10 @@ func resolveContentType(values []string) (IngestFormat, int, error) { return f, -1, err } -// ingestFormatOne resolves ONE header line, parsed per RFC 9110 §8.3. Only the -// media type decides the format; no malformed parameter costs the request. +// ingestFormatOne resolves ONE header line, parsed per RFC 9110 §8.3. The media +// type decides the format, plus the `header` parameter for the two CSV-family +// types (see acceptedContentTypes); no other malformed parameter costs the +// request. // // That rule needs two steps, because Go splits parse failures in a way the rule // does not. ErrInvalidMediaParameter leaves the media type parsed and returned, @@ -299,8 +226,13 @@ func resolveContentType(values []string) (IngestFormat, int, error) { // member there reads an NDJSON body as one object, dropping every record past // it behind a 200. The error cannot distinguish that from a comma inside data, // so such a line is refused rather than guessed at (#563). +// +// The same caution applies to `header`: on a line whose parameters did not +// parse, a header parameter cannot be read, and guessing "none" would ingest +// a declared header line as data. Such a line is refused when it mentions one; +// a header value other than present/absent is refused too. func ingestFormatOne(v string) (IngestFormat, error) { - mediaType, _, err := mime.ParseMediaType(v) + mediaType, params, err := mime.ParseMediaType(v) if err != nil { if strings.ContainsRune(v, ',') { return FormatJSON, errUnsupportedContentType @@ -313,14 +245,38 @@ func ingestFormatOne(v string) (IngestFormat, error) { mediaType = base } } + header := "" + if headerDecides(mediaType) { + if err != nil && strings.Contains(strings.ToLower(v), "header") { + return FormatJSON, errUnsupportedContentType + } + switch strings.ToLower(params["header"]) { + case "": + case "present", "absent": + header = strings.ToLower(params["header"]) + default: + return FormatJSON, errUnsupportedContentType + } + } for _, a := range acceptedContentTypes { - if a.mediaType == mediaType { + if a.mediaType == mediaType && a.header == header { return a.format, nil } } return FormatJSON, errUnsupportedContentType } +// headerDecides reports whether the `header` parameter selects the format for +// this media type — true only for a type with header-qualified entries. +func headerDecides(mediaType string) bool { + for _, a := range acceptedContentTypes { + if a.mediaType == mediaType && a.header != "" { + return true + } + } + return false +} + // mediaTypePrefix is everything before the first ";" — the media type without // its parameters. Only ingestFormatOne's re-parse uses it, and only on a line // with no comma, so it cannot resurrect a joined declaration. @@ -329,40 +285,6 @@ func mediaTypePrefix(v string) string { return base } -// newRecordReader picks a reader for an already-resolved format over an -// already-buffered body, looking at the bytes only to choose arity within the -// JSON family ('[' → array, else → single object). The declared format is -// authoritative: an NDJSON body is read as NDJSON whatever its first byte, so a -// line that isn't a JSON object fails as a per-record error rather than silently -// re-framing the whole request. batch is false only for the single-object path; -// true for array/NDJSON. -// -// It takes the format rather than a Content-Type on purpose: resolving here as -// well as in the handler would put one rule in two places, which is how the -// joined and repeated paths came to disagree before. The only error this can -// return is errEmptyBody. The caller resolves the format with resolveContentType -// (which owns the 415) and is expected to have bounded the body via -// http.MaxBytesReader before buffering it. -func newRecordReader(format IngestFormat, body []byte) (rr recordReader, batch bool, err error) { - first, ok := firstNonSpace(body) - if !ok { - return nil, false, errEmptyBody - } - - if format == FormatNDJSON { - sc := bufio.NewScanner(bytes.NewReader(body)) - sc.Buffer(make([]byte, 0, 64*1024), maxNDJSONLineBytes) - return &lineReader{sc: sc}, true, nil - } - - dec := json.NewDecoder(bytes.NewReader(body)) - dec.UseNumber() - if first == '[' { - return &arrayReader{dec: dec}, true, nil - } - return &objectReader{dec: dec}, false, nil -} - // firstNonSpace returns the body's first non-whitespace byte. ok is false when // there is none within the sniff window — the same bound the streaming reader // used, kept so a body of leading whitespace longer than the window still reads @@ -382,10 +304,10 @@ func firstNonSpace(body []byte) (byte, bool) { // emptyBodyMessage tailors the empty-body 400 message to the declared format so // an NDJSON caller still gets the familiar "empty ndjson body". func emptyBodyMessage(format IngestFormat) string { - if format == FormatNDJSON { - return "empty ndjson body" + if format == FormatJSON { + return "empty body" } - return "empty body" + return "empty " + format.String() + " body" } // Bounds on what a caller-supplied Content-Type may cost us when echoed back. diff --git a/internal/api/errors.go b/internal/api/errors.go index 876b397b..ddbeb22b 100644 --- a/internal/api/errors.go +++ b/internal/api/errors.go @@ -18,10 +18,23 @@ import ( // match the success-path handlers and RFC 8259 (which does not define a // charset for application/json — JSON is required to be UTF-8 already). func writeJSONError(w http.ResponseWriter, status int, message string) { + writeJSONErrorCode(w, status, message, 0) +} + +// writeJSONErrorCode is writeJSONError plus ClickHouse's own error code, which +// the ingest path carries when the server's parser is the one that refused the +// record (117 unknown field, 27 unparseable value, 6 out of range). A zero code +// is omitted rather than sent as 0, so a body carrying "code" always means +// ClickHouse answered — a gateway rejection never invents one. +func writeJSONErrorCode(w http.ResponseWriter, status int, message string, code int) { w.Header().Set("Content-Type", "application/json") w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(map[string]string{"error": message}) + body := map[string]any{"error": message} + if code != 0 { + body["code"] = code + } + _ = json.NewEncoder(w).Encode(body) } // writeAuthzDenied writes the response for an authorization denial and emits a diff --git a/internal/api/errors_test.go b/internal/api/errors_test.go index 03875c79..0289c814 100644 --- a/internal/api/errors_test.go +++ b/internal/api/errors_test.go @@ -161,7 +161,7 @@ func TestPipesHandler_Execute_DenialLogsAllowedRoles(t *testing.T) { func TestIngest_DenialLogsPolicyGate(t *testing.T) { t.Parallel() logger, buf := warnBufLogger() - h := NewIngestHandler(testRegistry(t), &testutil.MockPublisher{}, logger) + h := newTestIngestHandler(t, testRegistry(t), &testutil.MockPublisher{}, logger) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": {"viewer": {Select: &policy.SelectPermissions{}}}, // no insert for viewer @@ -192,7 +192,7 @@ func TestAuthzDenied_LogsChiRoutePattern(t *testing.T) { logger, buf := warnBufLogger() reg := testutil.NewTestSchemaRegistry(t, nil) router := NewRouter(Dependencies{ - Ingest: NewIngestHandler(reg, &testutil.MockPublisher{}, logger), + Ingest: newTestIngestHandler(t, reg, &testutil.MockPublisher{}, logger), Query: &QueryHandler{}, SSE: NewStreamHandler(stream.NewHub(nil, nil, nil), nil), Health: &HealthHandler{}, diff --git a/internal/api/ingest.go b/internal/api/ingest.go index 2fbd6c4d..4a3963bd 100644 --- a/internal/api/ingest.go +++ b/internal/api/ingest.go @@ -5,11 +5,14 @@ import ( "encoding/json" "errors" "fmt" - "io" "log/slog" + "maps" "net/http" + "reflect" + "slices" "sort" "strings" + "sync" "time" "github.com/Wave-RF/WaveHouse/internal/auth" @@ -19,6 +22,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/mq" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/query" + "github.com/Wave-RF/WaveHouse/internal/typelayer" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -38,7 +42,11 @@ const maxReportedResults = 10000 // IngestHandler handles POST /v1/ingest?table={table} type IngestHandler struct { Registry *discovery.SchemaRegistry - Dedup dedupe.Deduplicator // nil when no dedupe store is wired (tests) + // Types answers "would this record insert?" with ClickHouse's own parser. + // Nil is never "skip validation": the handler refuses the request, because + // nothing else on this path looks at a value. + Types *typelayer.Engine + Dedup dedupe.Deduplicator // nil when no dedupe store is wired (tests) // DedupeSettings resolves the effective dedupe id_field/require_id for a // table (settings.Store.DedupeFor in production). Called once per record so // a settings reload lands at a record boundary — one record never mixes two @@ -48,17 +56,18 @@ type IngestHandler struct { PolicySource policy.Source logger *slog.Logger - // Validator and Checker are the per-record seams a native type layer will - // take over (see ingest_seams.go). Both are optional: nil means the default - // implementation, which is today's behavior unchanged. - Validator RecordValidator - Checker InsertChecker - // maxRequestBytes optionally overrides the default inbound request body cap // (maxRequestBodyBytes). When 0, the default applies. Exists so same-package // tests can pin the cap-overflow path without allocating 16 MiB per run; not // a production tuning knob, hence unexported. Mirrors QueryHandler. maxRequestBytes int64 + + // noticeMu guards noticeLast, the last time each rate-limited notice was + // logged. A missing artifact or a policy whose injected literal will not + // compile is a standing condition: one line per request would bury the rest + // of the log under it. + noticeMu sync.Mutex + noticeLast map[string]time.Time } func NewIngestHandler(registry *discovery.SchemaRegistry, pub mq.Publisher, logger *slog.Logger) *IngestHandler { @@ -79,14 +88,14 @@ var dedupeDisabledCounter, _ = otel.Meter("wavehouse-ingest").Int64Counter( metric.WithDescription("Ingested records published without dedupe because the store was switched off while settings said enabled (reload window)"), ) -// batchResult is the response body for any multi-record ingest (a JSON array, -// an NDJSON batch, and — later — CSV). The status is 200 whenever the body was -// readable and the records were processed; per-record rejections (malformed -// JSON, schema or permission failures) are reported in Results without failing -// the whole request, so one bad record never obscures the rest of the batch -// (issue #195). Whole-request conditions abort with a non-200 status instead — -// see requestAbort for the list, which lives there and only there, because -// stating it in three places is how two of them went stale. +// batchResult is the response body for any multi-record ingest: a JSON array, +// an NDJSON batch, a CSV or a TSV body. The status is 200 whenever the body was +// readable and the records were processed; per-record rejections (unparseable +// values, unknown columns, failed check clauses) are reported in Results without +// failing the whole request, so one bad record never obscures the rest of the +// batch (issue #195). Whole-request conditions abort with a non-200 status +// instead — see requestAbort for the list, which lives there and only there, +// because stating it in three places is how two of them went stale. type batchResult struct { Total int `json:"total"` // records read from the body Succeeded int `json:"succeeded"` // records validated + published @@ -104,15 +113,22 @@ type recordResult struct { Ok bool `json:"ok,omitempty"` Duplicate bool `json:"duplicate,omitempty"` Error string `json:"error,omitempty"` + // Code is ClickHouse's own error code when the record was refused by the + // server's parser (117 unknown field — which now includes a column the role + // may not write, 27 unparseable value, 6 out of range). Absent for a gateway + // rejection — a failed check clause is our verdict, not ClickHouse's, and + // must not be dressed as one. + Code int `json:"code,omitempty"` } -// recordReject is a per-record rejection: this record is bad (failed schema -// validation or a column/check permission rule), but the rest of the batch can -// still proceed. The single-object path maps Status to the HTTP code; the batch -// path records Message against the index and keeps going. +// recordReject is a per-record rejection: this record is bad (ClickHouse's +// parser refused it, or a policy check clause did), but the rest of the batch +// can still proceed. The single-object path maps Status to the HTTP code; the +// batch path records Message against the index and keeps going. type recordReject struct { Status int Message string + Code int // ClickHouse's code; 0 for a gateway rejection (see recordResult.Code) } // requestAbort is a whole-request failure: this record and every one that @@ -121,19 +137,47 @@ type recordReject struct { // // Most causes are TRANSIENT system conditions, where abandoning the tail is what // makes the batch safe to retry: publish backpressure (503), a publish/marshal -// failure (500), a dedup backend error (500). +// failure (500), a dedup backend error (500), a type layer that cannot answer +// (503). // -// One is not. An insert grant that resolved for the other operation is a 403 and -// a caller/config bug — retrying cannot help. It aborts rather than rejecting -// per record because the grant is resolved ONCE per request, so it is true for -// every record or none; as a per-record reject a 10k batch would report 10k -// independent permission failures for a single mis-wired grant. +// Two are not, and retrying either unchanged cannot help: +// - An insert grant that resolved for the other operation is a 403 and a +// caller/config bug. It aborts rather than rejecting per record because the +// grant is resolved ONCE per request, so it is true for every record or +// none; as a per-record reject a 10k batch would report 10k independent +// permission failures for a single mis-wired grant. +// - A header-format body whose header ClickHouse refuses — a name the table +// or the role lacks, or a name given twice — is a 400 with ClickHouse's +// code (117). The header is not a record, and no record was read past it. type requestAbort struct { Status int Message string + Code int // ClickHouse's code, when its parser refused the body as a whole RetryAfter string // non-empty → emit a Retry-After header (503 backpressure) } +// ingestRun is one request's state after the body has been framed and ruled on: the +// handle that produced the bytes and chtypes' verdict per record, its check +// answer included. Both response shapes read their records out of it, so the +// single-object and batch paths cannot disagree about what a record's outcome +// is — only about how it is rendered. +type ingestRun struct { + table, scope string + tbl *typelayer.Table + batch typelayer.Batch + // records is how many records the request contains: the body's own framing + // before chtypes has read it (an empty array is zero, anything else is at + // least one), then len(batch.Rows) once it has answered. + records int + // checkColumns names the check clauses a record's check answer came from, + // for the rejection message. The filter is AND-joined over all of them, so + // a false verdict does not say which one failed — with one clause it does. + checkColumns []string + // checkGuard is the rejection every otherwise-acceptable record gets when + // the role's check clauses name columns no record can carry. + checkGuard *recordReject +} + func (h *IngestHandler) Handle(w http.ResponseWriter, r *http.Request) { now := time.Now().UTC() table := r.URL.Query().Get("table") @@ -151,15 +195,12 @@ func (h *IngestHandler) Handle(w http.ResponseWriter, r *http.Request) { r = r.WithContext(ctx) - // TODO: what should the order of these be to maximize speed + limit risk of data leakage or DoS/resource exhaustion? - if table == "" { h.logger.ErrorContext(ctx, "missing table parameter in request") writeJSONError(w, http.StatusBadRequest, "missing table") return } - // TODO: prevent table-enumeration... schema := h.Registry.Get(table) if schema == nil { h.logger.WarnContext(ctx, "unknown table requested", "table", table) @@ -231,18 +272,18 @@ func (h *IngestHandler) Handle(w http.ResponseWriter, r *http.Request) { return } - // Bound the inbound body (parity with /v1/ops/query; also caps the - // array/stream decode vectors). See query.go for maxRequestBodyBytes. + // Bound the inbound body (parity with /v1/ops/query). See query.go for + // maxRequestBodyBytes. reqCap := int64(maxRequestBodyBytes) if h.maxRequestBytes > 0 { reqCap = h.maxRequestBytes } r.Body = http.MaxBytesReader(w, r.Body, reqCap) - // Read the whole (already-capped) body up front and run the readers over - // those bytes rather than the live connection. The buffer comes from a pool - // and goes back on the way out — every record the readers hand back is - // freshly allocated, so nothing downstream points into it. + // Read the whole (already-capped) body up front and hand those bytes to + // ClickHouse's own parser. The buffer comes from a pool and goes back on the + // way out; the exported rows point INTO the type layer's payload, not into + // this buffer, and are copied into the envelope before it is released. body := getBodyBuffer() defer putBodyBuffer(body) if _, err := body.ReadFrom(r.Body); err != nil { @@ -254,159 +295,398 @@ func (h *IngestHandler) Handle(w http.ResponseWriter, r *http.Request) { return } - // The reader is built from the RESOLVED format, not from a second look at - // the header. Re-resolving here would duplicate the rule in two places, - // which is how the joined and repeated paths came to disagree before. The - // only error left is an empty body. - rr, batch, err := newRecordReader(format, body.Bytes()) - if err != nil { + // Framing: the declared format says how the body frames its records, and the + // first non-whitespace byte answers the one question left inside the JSON + // family — array or single object. That byte is the ONLY thing the body gets + // to decide, and it decides the response shape (AUDIT §A.6), never the + // format. + first, ok := firstNonSpace(body.Bytes()) + if !ok { h.logger.ErrorContext(ctx, "empty ingest body", "table", table, "format", format.String()) writeJSONError(w, http.StatusBadRequest, emptyBodyMessage(format)) return } - - if batch { - h.handleBatch(ctx, w, rr, reqCap, table, scope, schema, perms, role, now, h.policyCheckGuard(ctx, table, role, schema, perms)) - return - } - h.handleSingle(ctx, w, rr, reqCap, table, scope, schema, perms, role, now, h.policyCheckGuard(ctx, table, role, schema, perms)) -} - -// handleSingle ingests a lone flat JSON object and preserves the GA response -// contract: 200 {"ok":true} (or {"duplicate":true} when dedup skips it), or the -// matching non-200 on validation / permission / whole-request failure. -func (h *IngestHandler) handleSingle( - ctx context.Context, - w http.ResponseWriter, - rr recordReader, - reqCap int64, - table, scope string, - schema *discovery.TableSchema, - perms *policy.ResolvedPermissions, - role string, - now time.Time, - checkGuard *recordReject, -) { - data, err := rr.Next() - if err != nil { - // Unreachable while the body is buffered — a bytes.Reader cannot produce - // a *http.MaxBytesError, and the cap is enforced at body.ReadFrom. Kept - // because it is the correct mapping if a reader ever streams again. - if writeMaxBytesError(w, err, reqCap) { + batchShape := format.alwaysBatch() || first == '[' + var records int + switch { + case format == FormatJSON && first == '[': + n, framed := reframeArray(body.Bytes()) + if !framed { + // Brackets that do not balance: a truncated upload or a structural + // syntax error. Neither can be salvaged per record, and reporting the + // records that did arrive as a complete batch is the failure this + // refusal exists to prevent. + h.logger.WarnContext(ctx, "ingest read error", "error", "unterminated json array", "table", table) + writeJSONError(w, http.StatusBadRequest, "invalid json: unterminated json array") return } - h.logger.ErrorContext(ctx, "invalid json payload", "error", err, "table", table) - writeJSONError(w, http.StatusBadRequest, "invalid json") + records = n + default: + // A single-object body is one record (concatenated objects after it are + // ignored, as they always have been — declare NDJSON to batch them, #561); + // a line-framed body has at least the record its first byte starts. The + // real count is chtypes' own, taken from the batch once it has answered. + records = 1 + } + + // The role's projection of the table answers column policy and the auto + // -injected check values inside ClickHouse's own parser, so no Go code walks + // the record's keys. It is taken once and held for the whole request, so + // every record is judged by one generation and one shape: a refresh that + // recompiles the table waits for this request rather than changing the + // answer halfway through a batch. Resolved after the framing checks so a + // 415/413/empty-body/unterminated request is still refused for its own + // reason when the type layer happens to be down. + guard := h.policyCheckGuard(ctx, table, role, schema, perms) + shape, preds, checkColumns, abort := h.insertShape(ctx, table, role, schema, perms, guard) + if abort != nil { + writeAbort(w, abort) return } - - dup, reject, abort := h.processRecord(ctx, table, scope, schema, perms, role, data, now, checkGuard) + tbl, abort := h.roleTable(ctx, table, shape) if abort != nil { writeAbort(w, abort) return } - if reject != nil { - writeJSONError(w, reject.Status, reject.Message) - return + defer tbl.Release() + + st := &ingestRun{table: table, scope: scope, tbl: tbl, records: records, checkColumns: checkColumns, checkGuard: guard} + if records > 0 { + if abort := h.judge(ctx, st, format, body.Bytes(), preds); abort != nil { + writeAbort(w, abort) + return + } } - if dup { - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]bool{"duplicate": true}) + + if batchShape { + h.writeBatch(ctx, w, st, now) return } - - h.logger.InfoContext(ctx, "event successfully ingested", "table", table) - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(map[string]bool{"ok": true}) + h.writeSingle(ctx, w, st, now) } -// handleBatch ingests a multi-record body (JSON array or NDJSON), running each -// record through the same validate → authorize → dedup → publish pipeline as a -// single insert. A record that fails validation or a PER-RECORD permission rule -// (a denied column, a failed check clause) — or that the reader couldn't decode -// — is recorded against its index and the batch continues; a whole-request -// condition aborts it (see requestAbort). Returns 200 with a per-record summary -// once the body is consumed. -func (h *IngestHandler) handleBatch( - ctx context.Context, - w http.ResponseWriter, - rr recordReader, - reqCap int64, - table, scope string, - schema *discovery.TableSchema, - perms *policy.ResolvedPermissions, - role string, - now time.Time, - checkGuard *recordReject, -) { - result := batchResult{Results: []recordResult{}} - - for { - data, err := rr.Next() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - if rse, ok := errors.AsType[*recordSyntaxError](err); ok { - result.Total++ - result.Failed++ - appendResult(&result, recordResult{Index: result.Total, Error: rse.Error()}) - continue - } - // Unreachable while the body is buffered — a bytes.Reader cannot produce - // a *http.MaxBytesError, and the cap is enforced at body.ReadFrom. Kept - // because it is the correct mapping if a reader ever streams again. - if writeMaxBytesError(w, err, reqCap) { - return - } - // A fatal stream error (JSON array syntax error, or an oversized - // NDJSON line) — the reader can't resume, so fail the request rather - // than report a misleading partial summary. Not a body read error: - // the readers run over an in-memory slice now. - h.logger.WarnContext(ctx, "ingest read error", "error", err, "table", table) - writeJSONError(w, http.StatusBadRequest, "invalid json: "+err.Error()) - return +// judge asks ClickHouse's own parser for a verdict per record — ONE call for the +// whole body, no chunking (AUDIT §A.9) — with the role's insert check clauses, +// if any, compiled to ONE filter and answered by that same parse. +// +// A Go-level failure is an unavailable handle (the type layer's outage) or ours, +// and neither is the caller's record to fix. A body ClickHouse refused as a +// whole is the caller's to fix, and is a 400 with its code. +func (h *IngestHandler) judge(ctx context.Context, st *ingestRun, format IngestFormat, body []byte, preds []policy.Predicate) *requestAbort { + batch, err := st.tbl.IngestWith(format.wire(), format.options(), body, preds...) + if err != nil { + var un *typelayer.Unavailable + if errors.As(err, &un) { + h.logUnavailable(ctx, st.table, un.Cause) + return unavailableAbort(un.Cause) } + h.logger.ErrorContext(ctx, "record validation failed", "error", err, "table", st.table) + return &requestAbort{Status: http.StatusInternalServerError, Message: "validation failed"} + } + if r := batch.Refused; r != nil { + h.logger.WarnContext(ctx, "ingest body refused by the parser", "error", r.Message, "code", r.Code, "table", st.table) + return &requestAbort{Status: http.StatusBadRequest, Message: r.Message, Code: r.Code} + } + st.batch = batch + // chtypes' per-record answer is the record count: a JSON array sent as + // NDJSON is however many elements its reader took, a blank line is nothing. + // When it gave no per-record detail the padded batch is still index-shaped, + // so the same rule keeps every later index in range. + st.records = len(batch.Rows) + return nil +} + +// resolveRecord decides the i-th record's outcome and, when it survives every +// rule, publishes it. Exactly one of the returns is set, or none (published). +// +// The order is the one the type layer imposes and is a documented change: a +// record that both fails to parse and violates a check clause now reports the +// PARSE error — chtypes answers the check only for a record it accepted. +// Nothing is published either way, so no enforcement is lost (AUDIT §A.2). +func (h *IngestHandler) resolveRecord(ctx context.Context, st *ingestRun, i int, now time.Time) (duplicate bool, reject *recordReject, abort *requestAbort) { + verdict := verdictAt(st.batch, i) + if !verdict.Accepted { + h.logVerdict(ctx, st.table, verdict) + return false, verdictReject(verdict), nil + } + if st.checkGuard != nil { + return false, st.checkGuard, nil + } + if verdict.CheckReason != "" { + h.logger.WarnContext(ctx, "check clause failed", "columns", st.checkColumns, + "reason", verdict.CheckReason, "cause", verdict.Message, "table", st.table) + return false, checkReject(verdict.CheckReason, st.checkColumns), nil + } + return h.publishAccepted(ctx, st, verdict.Line, now) +} - result.Total++ - idx := result.Total - dup, reject, abort := h.processRecord(ctx, table, scope, schema, perms, role, data, now, checkGuard) +// writeSingle answers a lone flat JSON object and preserves the GA response +// contract: 200 {"ok":true} (or {"duplicate":true} when dedup skips it), or the +// matching non-200 on refusal / permission / whole-request failure. +func (h *IngestHandler) writeSingle(ctx context.Context, w http.ResponseWriter, st *ingestRun, now time.Time) { + dup, reject, abort := h.resolveRecord(ctx, st, 0, now) + switch { + case abort != nil: + writeAbort(w, abort) + case reject != nil: + writeJSONErrorCode(w, reject.Status, reject.Message, reject.Code) + case dup: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"duplicate": true}) + default: + h.logger.InfoContext(ctx, "event successfully ingested", "table", st.table) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"ok": true}) + } +} + +// writeBatch answers a multi-record body (JSON array, NDJSON, CSV or TSV). A +// record ClickHouse's parser refused, or one a check clause denied, is recorded +// against its index and the batch continues; a whole-request condition aborts it +// (see requestAbort). Returns 200 with a per-record summary. +func (h *IngestHandler) writeBatch(ctx context.Context, w http.ResponseWriter, st *ingestRun, now time.Time) { + result := batchResult{Total: st.records, Results: []recordResult{}} + for i := range st.records { + dup, reject, abort := h.resolveRecord(ctx, st, i, now) if abort != nil { // Whole-request failure: surface the status rather than recording a // request-scoped condition as per-record loss (see requestAbort). writeAbort(w, abort) return } - if reject != nil { + entry := recordResult{Index: i + 1} + switch { + case reject != nil: + entry.Error, entry.Code = reject.Message, reject.Code result.Failed++ - appendResult(&result, recordResult{Index: idx, Error: reject.Message}) - continue - } - if dup { + case dup: + entry.Duplicate = true result.Duplicates++ - appendResult(&result, recordResult{Index: idx, Duplicate: true}) - continue + default: + entry.Ok = true + result.Succeeded++ + } + // Up to maxReportedResults entries are echoed; the counts above stay + // authoritative even when the slice is truncated. + if len(result.Results) < maxReportedResults { + result.Results = append(result.Results, entry) } - result.Succeeded++ - appendResult(&result, recordResult{Index: idx, Ok: true}) } - h.logger.InfoContext(ctx, "batch ingested", "table", table, + h.logger.InfoContext(ctx, "batch ingested", "table", st.table, "total", result.Total, "succeeded", result.Succeeded, "failed", result.Failed, "duplicates", result.Duplicates) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(result) } -// appendResult records a per-record outcome up to maxReportedResults. The -// batchResult counts are incremented by the caller and stay authoritative even -// when the Results slice is truncated. -func appendResult(result *batchResult, entry recordResult) { - if len(result.Results) < maxReportedResults { - result.Results = append(result.Results, entry) +// insertShape turns the role's resolved insert grant into the projection of the +// table ClickHouse itself will enforce, plus the predicates the check clauses +// become. +// +// Columns is the allow/deny decision, answered by omitting the denied columns +// from the compiled schema: a record naming one is then refused per row with +// ClickHouse's own code 117 rather than by a Go walk over the record's keys +// (AUDIT §A.1, decision D1). nil means the role may write every column, which +// compiles to no second handle at all. +// +// Defaults is the `_eq` auto-inject: the required value becomes the column's +// DEFAULT, so a record that omits it is filled and a record that supplies one +// still wins (measured, AUDIT §A.3). An `_in` check has no single value to +// inject, so the column keeps the TABLE's own default and the filter tests that +// — a documented behaviour change (D3). +func (h *IngestHandler) insertShape( + ctx context.Context, + table, role string, + schema *discovery.TableSchema, + perms *policy.ResolvedPermissions, + guard *recordReject, +) (typelayer.RoleShape, []policy.Predicate, []string, *requestAbort) { + if perms == nil { + return typelayer.RoleShape{}, nil, nil, nil + } + + // Through the accessor, not a bare read: a bare read presents an empty map + // on an unresolved side and every check then passes vacuously. ok=false + // ABORTS the request; it must never be read as "no checks to run". + checks, resolved := perms.CheckClauses() + if !resolved { + // ABORT, not a per-record reject: perms is resolved once per request, so + // this is true for every record or none. As a reject, a 10k-record batch + // would emit 10k ERROR lines and report 10k independent permission + // failures for one mis-wired grant. + h.logger.ErrorContext(ctx, "insert checks consulted on a grant resolved for another operation", + "table", table, "role", role) + return typelayer.RoleShape{}, nil, nil, &requestAbort{ + Status: http.StatusForbidden, + Message: "insert permissions were not resolved for this request", + } + } + + shape := typelayer.RoleShape{Columns: allowedInsertColumns(schema, perms)} + if guard != nil { + // The guard's own columns cannot be injected into or filtered on — that + // is what it refuses — so the shape stays bare and every record that + // parses gets the guard's rejection instead. + return shape, nil, nil, nil + } + + cols := slices.Sorted(maps.Keys(checks)) + preds := make([]policy.Predicate, 0, len(cols)) + for _, col := range cols { + switch v := checks[col].(type) { + case []any: + // An _in set. A nil/empty set is an unresolvable claim and matches + // nothing — Ingest fails every record's check without compiling + // anything (#224). + vals := make([]string, 0, len(v)) + for _, e := range v { + s, ok := scalarString(e) + if !ok { + vals = nil + break + } + vals = append(vals, s) + } + preds = append(preds, policy.Predicate{Column: col, Op: "in", Values: vals}) + default: + s, ok := scalarString(checks[col]) + if !ok { + // A required value with no string form cannot be expressed as a + // filter constant, and admitting the record would drop the check + // entirely. An empty Values list matches nothing. + preds = append(preds, policy.Predicate{Column: col, Op: "="}) + continue + } + if shape.Defaults == nil { + shape.Defaults = make(map[string]string, len(cols)) + } + shape.Defaults[col] = s + preds = append(preds, policy.Predicate{Column: col, Op: "=", Values: []string{s}}) + } + } + return shape, preds, cols, nil +} + +// allowedInsertColumns is the role's writable column set, or nil when it may +// write every column the table has. nil is the identity shape, which reuses the +// table's own compiled handle instead of a second one. +// +// Every column is asked through IsColumnAllowed so the allow/deny precedence +// stays in the one place that owns it; the computed kinds are included for the +// same reason, and typelayer keeps them whatever this list says (they are the +// server's to compute, and a MATERIALIZED expression over a dropped column would +// not compile at all). +func allowedInsertColumns(schema *discovery.TableSchema, perms *policy.ResolvedPermissions) []string { + allowed := make([]string, 0, len(schema.Columns)) + for _, c := range schema.Columns { + if perms.IsColumnAllowed(c.Name, true) { + allowed = append(allowed, c.Name) + } + } + if len(allowed) == len(schema.Columns) { + return nil + } + return allowed +} + +// scalarString renders a check clause's required value as the string the filter +// binds. Every filter parameter binds as {pN:String} whatever the column's +// declared type (AUDIT §C.1), so this is the only conversion the check path +// needs. +// +// The reflect.Kind test rather than a type switch is deliberate: policy marks a +// placeholder-free check value with its own string-kinded named type, a +// distinction that existed only for the Go-side numeric re-reading ClickHouse +// now answers and that policy drops with it. Naming the type here would keep it +// alive; asking for its kind works across the change. +func scalarString(v any) (string, bool) { + if s, ok := v.(string); ok { + return s, true + } + rv := reflect.ValueOf(v) + if rv.Kind() == reflect.String { + return rv.String(), true + } + return "", false +} + +// roleTable resolves the compiled handle for this role's projection, or the 503 +// every record of this request gets instead. The type layer being down is an +// outage, never a verdict about the data: a caller must be able to retry the +// same body unchanged. +// +// An injected literal the column cannot read (`count UInt64 DEFAULT 'abc'`) is a +// compile refusal, ClickHouse code 6 — measured. That must not become a 503 for +// a policy that is simply unsatisfiable, so the shape is retried without its +// defaults: the check filter then judges the record as sent, which fails closed +// (an absent column takes the table default and the filter refuses it). The +// type layer logs the refusal once per generation and shape; this adds one +// rate-limited line saying what was done about it. +func (h *IngestHandler) roleTable(ctx context.Context, table string, shape typelayer.RoleShape) (*typelayer.Table, *requestAbort) { + if h.Types == nil { + // Nothing else on this path inspects a value, so an unwired type layer + // cannot mean "accept anything" — the same fail-closed direction the + // stream's row evaluator takes. + h.logUnavailable(ctx, table, "no type layer is wired") + return nil, unavailableAbort("ingest validation is unavailable") + } + tbl, err := h.Types.RoleTable(table, shape) + if err == nil { + return tbl, nil + } + if len(shape.Defaults) > 0 { + bare := typelayer.RoleShape{Columns: shape.Columns} + if t, bareErr := h.Types.RoleTable(table, bare); bareErr == nil { + if h.noticeDue("inject:" + table) { + h.logger.WarnContext(ctx, "insert check value cannot be injected as a column default; records omitting it will fail the check", + "table", table, "columns", slices.Sorted(maps.Keys(shape.Defaults)), "cause", err.Error()) + } + return t, nil + } + } + var un *typelayer.Unavailable + if errors.As(err, &un) { + h.logUnavailable(ctx, table, un.Cause) + return nil, unavailableAbort(un.Cause) + } + h.logger.ErrorContext(ctx, "could not resolve the compiled schema", "error", err, "table", table) + return nil, unavailableAbort(err.Error()) +} + +// unavailableAbort is the 503 for a type layer that cannot answer. Retry-After +// matches the publish-backpressure 503: both are "come back, nothing is wrong +// with your request". +func unavailableAbort(cause string) *requestAbort { + return &requestAbort{Status: http.StatusServiceUnavailable, Message: cause, RetryAfter: "30"} +} + +// logUnavailable emits at most one line per table per minute. A missing +// artifact or a timezone mismatch persists until an operator acts, so the +// per-request line says nothing the first one didn't. +func (h *IngestHandler) logUnavailable(ctx context.Context, table, cause string) { + if h.noticeDue("unavailable:" + table) { + h.logger.ErrorContext(ctx, "ingest type layer unavailable", "table", table, "cause", cause) } } +// noticeDue rate-limits a standing-condition notice to one line per key per +// minute, so a condition that persists until an operator acts does not bury the +// rest of the log under one line per request. +func (h *IngestHandler) noticeDue(key string) bool { + now := time.Now() + h.noticeMu.Lock() + defer h.noticeMu.Unlock() + if last, seen := h.noticeLast[key]; seen && now.Sub(last) < time.Minute { + return false + } + if h.noticeLast == nil { + h.noticeLast = make(map[string]time.Time) + } + h.noticeLast[key] = now + return true +} + // writeAbort emits a whole-request failure response: the status and message, // plus a Retry-After header when one is set (503 backpressure). Shared by the // single-object and batch paths. @@ -414,7 +694,7 @@ func writeAbort(w http.ResponseWriter, abort *requestAbort) { if abort.RetryAfter != "" { w.Header().Set("Retry-After", abort.RetryAfter) } - writeJSONError(w, abort.Status, abort.Message) + writeJSONErrorCode(w, abort.Status, abort.Message, abort.Code) } // writeMaxBytesError writes a 413 if err is the inbound body-cap overflow and @@ -432,27 +712,34 @@ func writeMaxBytesError(w http.ResponseWriter, err error, limit int64) bool { // returns the rejection every record should get when they do not. // // A check the row cannot carry can never be enforced: the row holds one slot -// per INSERTABLE column, by position, so an auto-injected value for anything -// outside that set is dropped on the way out and the record inserts WITHOUT the -// value the policy requires — answering 200. Policy validation cannot catch -// this; it never sees the ClickHouse schema. Three ways in, all refused. +// per WIRE column, by position, so an injected value for anything outside that +// set is dropped on the way out and the record inserts WITHOUT the value the +// policy requires — answering 200. Policy validation cannot catch this; it never +// sees the ClickHouse schema. Three ways in, all refused. +// +// It is not redundant now that the compiled schema answers column policy: a +// Defaults entry for a column the shape cannot carry is a COMPILE refusal, so +// without this guard a mis-wired policy would be a 503 naming a ClickHouse +// internal rather than a 403 naming the column the operator has to fix. // // Evaluated here rather than per record because the condition is a property of -// (table, role, policy) and is identical for every record in the request — the -// same reasoning as the !resolved abort in processRecord. Doing it per record -// would emit one ERROR line per record for a single mis-wired policy, which on -// a 16 MiB body of small records is ~1.2M lines. The reject is still returned -// per record, so a batch reports each record's own cause: one that SUPPLIES the -// column fails schema validation first, with a different message. +// (table, role, policy) and is identical for every record in the request. Doing +// it per record would emit one ERROR line per record for a single mis-wired +// policy, which on a 16 MiB body of small records is ~1.2M lines. The reject is +// still returned per record, so a batch reports each record's own cause: one +// that SUPPLIES the column is refused by ClickHouse first, with its own message. func (h *IngestHandler) policyCheckGuard( ctx context.Context, table, role string, schema *discovery.TableSchema, perms *policy.ResolvedPermissions, ) *recordReject { + if perms == nil { + return nil + } checks, resolved := perms.CheckClauses() if !resolved { - return nil // the !resolved abort in processRecord owns this case + return nil // the !resolved abort in insertShape owns this case } // Sorted, and every offender — not the first one a map range happens to @@ -500,149 +787,107 @@ func (h *IngestHandler) policyCheckGuard( } } -// processRecord runs the per-record pipeline shared by the single-object and -// batch ingest paths: schema validation → column/check permission enforcement -// (with claim-derived auto-injection) → optional dedup → publish. The -// table-level insert grant is checked once by the caller before any record is -// processed, so perms here drives only the per-column and per-row checks (it is -// nil when no policy store is configured). data may be mutated to auto-inject -// check-clause values. -// -// Exactly one of the outcomes is meaningful per call: -// - duplicate true: the record was skipped by dedup (reject/abort nil). -// - reject non-nil: the record is bad; the rest of a batch may still proceed. -// - abort non-nil: a whole-request failure; the caller stops and returns it. -func (h *IngestHandler) processRecord( - ctx context.Context, - table, scope string, - schema *discovery.TableSchema, - perms *policy.ResolvedPermissions, - role string, - data map[string]any, - now time.Time, - checkGuard *recordReject, -) (duplicate bool, reject *recordReject, abort *requestAbort) { - if err := h.validator().Validate(schema, data); err != nil { - h.logger.WarnContext(ctx, "schema validation failed", "error", err, "table", table) - return false, &recordReject{Status: http.StatusBadRequest, Message: err.Error()}, nil - } - - // DEEP AUTH: column-level allow/deny + check clauses. - if perms != nil { - for col := range data { - if !perms.IsColumnAllowed(col, true) { - h.logger.WarnContext(ctx, "column insertion forbidden", "column", col, "role", role) - return false, &recordReject{ - Status: http.StatusForbidden, - Message: fmt.Sprintf("column %q not allowed for insert", col), - }, nil - } - } - // Through the accessor, not a bare read. The check loop iterates a side's - // map rather than asking about a column, so IsColumnAllowed cannot cover it, - // and a bare read presents an empty map on an unresolved side — every check - // then passes vacuously. ok=false ABORTS the request; it must never be read - // as "no checks to run". - // - // Reachable, unlike the query path's bare reads: discovery.Validate only - // requires a column that is neither nullable nor defaulted, so a table whose - // columns are all nullable or all defaulted accepts `{}`, and the column loop - // above then runs zero times. - checks, resolved := perms.CheckClauses() - if !resolved { - // ABORT, not a per-record reject: perms is resolved once per request, - // so this is true for every record or none. As a reject, a 10k-record - // batch would emit 10k ERROR lines and report 10k independent - // permission failures for one mis-wired grant. - h.logger.ErrorContext(ctx, "insert checks consulted on a grant resolved for another operation", - "table", table, "role", role) - return false, nil, &requestAbort{ - Status: http.StatusForbidden, - Message: "insert permissions were not resolved for this request", - } - } - for col, requiredVal := range checks { - // The guard for this is evaluated ONCE per request in - // policyCheckGuard (see Handle) and only consulted here: the condition - // is a property of (table, role, policy), identical for every record, - // so evaluating it per record would emit one ERROR line per record for - // a single mis-wired policy — the same amplification the !resolved - // abort above exists to avoid. The REJECT is still per record, because - // a record that supplies the column fails schema validation first with - // a different message, and a batch should report each its own cause. - if checkGuard != nil { - return false, checkGuard, nil - } - // A []any value is an _in check: the inserted value must be present and - // one of the allowed set. Unlike the scalar _eq case there is no single - // value to auto-inject, so an absent column fails closed. - if set, isSet := requiredVal.([]any); isSet { - actual, ok := data[col] - if !ok || !h.checker().InSet(actual, set) { - h.logger.WarnContext(ctx, "check clause failed", "column", col, "allowed", set, "actual", actual, "present", ok) - return false, &recordReject{ - Status: http.StatusForbidden, - Message: fmt.Sprintf("check failed for column %q", col), - }, nil - } - continue - } - if actual, ok := data[col]; ok { - // Both sides canonicalize through policy.CanonicalScalar, so a - // numeric insert value matches a numeric claim by value, not by - // spelling (payload 1.0 vs claim 1), and a value with no canonical - // form (object/array/null) matches nothing. A policy.LiteralValue — - // a placeholder-free check value, which carries no JSON type — - // additionally matches by its numeric reading, so `_eq: "1.0"` - // accepts an inserted 1.0 as well as an inserted "1.0". The type is - // what scopes that second reading to author-written literals: a - // claim-derived value arrives as a plain string and never gains a - // reading the token's own JSON type didn't give it. - if !h.checker().Matches(actual, requiredVal) { - h.logger.WarnContext(ctx, "check clause failed", "column", col, "expected", requiredVal, "actual", actual) - return false, &recordReject{ - Status: http.StatusForbidden, - Message: fmt.Sprintf("check failed for column %q", col), - }, nil - } - } else { - // Auto-inject the required value if not provided — as a plain - // string: a LiteralValue must not leak its named type into the - // published payload, where downstream type switches (timestamp - // canonicalization's `case string`) would silently miss it. - if lit, isLit := requiredVal.(policy.LiteralValue); isLit { - data[col] = string(lit) - } else { - data[col] = requiredVal - } - } +// verdictAt reads the verdict for the i-th record of a batch. A verdict the +// type layer did not return is a decline, never an acceptance: a missing answer +// must not publish a row nobody ruled on. +func verdictAt(batch typelayer.Batch, i int) typelayer.RowVerdict { + if i < len(batch.Rows) { + return batch.Rows[i] + } + return typelayer.RowVerdict{Declined: true, Message: "no verdict was returned for this record"} +} + +// verdictReject maps a verdict that is not an acceptance to the record's +// rejection. A refusal is ClickHouse's own answer about the data — 400, with +// its code. A decline is the validation engine failing to answer at all, which +// is never the caller's fault and must never be dressed as a 400: 422 says "we +// could not judge this", so a retry is meaningful and a client cannot learn +// from it that its payload was wrong. +func verdictReject(v typelayer.RowVerdict) *recordReject { + if v.Declined { + return &recordReject{ + Status: http.StatusUnprocessableEntity, + Message: "validation engine declined: " + v.Message, } } + return &recordReject{Status: http.StatusBadRequest, Message: v.Message, Code: v.Code} +} - // Canonicalize timestamps to RFC 3339 UTC (#372; fail-open — #381's row-filter - // enforces) after the permission checks: check clauses keep pre-#372 semantics. - h.validator().CanonicalizeTimestamps(schema, data) +// checkReject maps one check verdict that is not a definite true. "The data says +// no" is a 403; "we could not tell" is a 422, fail closed either way. +// +// The filter is AND-joined over every check clause, so a false verdict does not +// name which clause failed — with a single clause the column is unambiguous, and +// with several the message names the set that was tested rather than inventing +// an attribution. +func checkReject(reason string, cols []string) *recordReject { + if reason == typelayer.ReasonFilter { + return &recordReject{Status: http.StatusForbidden, Message: "check failed for " + columnList(cols)} + } + return &recordReject{ + Status: http.StatusUnprocessableEntity, + Message: "validation engine declined: the insert check for " + columnList(cols) + " could not be evaluated", + } +} + +// columnList renders a check's column set for a rejection message. +func columnList(cols []string) string { + quoted := make([]string, len(cols)) + for i, c := range cols { + quoted[i] = fmt.Sprintf("%q", c) + } + if len(cols) == 1 { + return "column " + quoted[0] + } + return "columns " + strings.Join(quoted, ", ") +} + +// logVerdict records a record ClickHouse would not take. A refusal carries its +// code so an operator can look it up without parsing the message; a decline is +// an ERROR because it is the engine, not the data, that failed. +func (h *IngestHandler) logVerdict(ctx context.Context, table string, v typelayer.RowVerdict) { + if v.Declined { + h.logger.ErrorContext(ctx, "validation engine declined a record", "reason", v.Message, "table", table) + return + } + h.logger.WarnContext(ctx, "schema validation failed", "error", v.Message, "code", v.Code, "table", table) +} +// publishAccepted runs the two steps reserved for a record the server itself +// accepted: dedupe, then the publish of the row ClickHouse's own writer +// produced. +// +// Dedupe runs AFTER validation deliberately — the idempotency key must mark +// only what is actually published, or a record ClickHouse refuses would burn +// its id and a corrected retry would be swallowed as a duplicate. +// +// line is the exported JSONCompactEachRow row, a sub-slice of the type layer's +// payload, so it is copied into the envelope before the handle is released. +func (h *IngestHandler) publishAccepted( + ctx context.Context, + st *ingestRun, + line []byte, + now time.Time, +) (duplicate bool, reject *recordReject, abort *requestAbort) { // Optional deduplication. enabled/id_field/require_id resolve per record // from one snapshot (table override → global; the settings directory // always states them, so no compiled fallback is needed), so a reload // lands at a record boundary. A Deduplicator without a settings source is // a wiring bug, not a mode — main wires both or neither. if h.Dedup != nil && h.DedupeSettings != nil { - if enabled, idField, requireID := h.DedupeSettings(table); enabled { - idVal, ok := data[idField] + if enabled, idField, requireID := h.DedupeSettings(st.table); enabled { + eventID, ok := eventIDAt(line, slices.Index(st.tbl.WireColumns, idField)) if !ok { - dedupeMissingIDCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("table", table))) + dedupeMissingIDCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("table", st.table))) if requireID { - h.logger.WarnContext(ctx, "dedupe id_field missing; rejecting", "id_field", idField, "table", table) + h.logger.WarnContext(ctx, "dedupe id_field missing; rejecting", "id_field", idField, "table", st.table) return false, &recordReject{ Status: http.StatusBadRequest, Message: fmt.Sprintf("missing dedupe id field %q", idField), }, nil } - h.logger.WarnContext(ctx, "dedupe id_field missing; publishing without idempotency", "id_field", idField, "table", table) + h.logger.WarnContext(ctx, "dedupe id_field missing; publishing without idempotency", "id_field", idField, "table", st.table) } else { - eventID := fmt.Sprint(idVal) dup, err := h.Dedup.CheckAndMark(ctx, eventID) switch { case errors.Is(err, dedupe.ErrDisabled): @@ -653,8 +898,8 @@ func (h *IngestHandler) processRecord( // carries the signal (a burst is a reload; a steady rate // is the store and settings out of step), so the line is // Debug rather than a WARN per record. - dedupeDisabledCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("table", table))) - h.logger.DebugContext(ctx, "dedupe switched off mid-reload; publishing without idempotency", "event_id", eventID, "table", table) + dedupeDisabledCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("table", st.table))) + h.logger.DebugContext(ctx, "dedupe switched off mid-reload; publishing without idempotency", "event_id", eventID, "table", st.table) case err != nil: h.logger.ErrorContext(ctx, "dedupe check failed", "error", err, "event_id", eventID) return false, nil, &requestAbort{Status: http.StatusInternalServerError, Message: "dedupe failed"} @@ -666,24 +911,20 @@ func (h *IngestHandler) processRecord( } } - // Render the record positionally against the table's declaration order. The - // column names ride alongside in the envelope rather than in the row, so a - // batch of rows for one table carries the names once — and the reader can - // tell a schema change mid-stream from a reordering. - cols := schema.InsertableColumns() - row, err := ingest.EncodeCompactRow(cols, data) - if err != nil { - h.logger.ErrorContext(ctx, "failed to encode compact row", "error", err, "table", table) - return false, nil, &requestAbort{Status: http.StatusInternalServerError, Message: "marshal failed"} - } - + // The row travels POSITIONALLY as the bytes ClickHouse's own writer + // produced, with the column names alongside rather than in the row: a batch + // of rows for one table carries the names once, and the reader can tell a + // schema change mid-stream from a reordering. WireColumns are the ROLE's + // columns, so a role that may not write every column publishes a shorter row + // with a matching name list — the worker already groups a flush by column + // signature, so that is one more group, not a new code path. evt := ingest.EventMessage{ - TableName: table, - Scope: scope, + TableName: st.table, + Scope: st.scope, ReceivedTimestamp: now.Format(time.RFC3339Nano), Format: ingest.FormatJSONCompactEachRow, - Columns: schema.InsertableColumnNames(), - Row: row, + Columns: st.tbl.WireColumns, + Row: json.RawMessage(line), } payload, err := json.Marshal(evt) @@ -692,12 +933,12 @@ func (h *IngestHandler) processRecord( return false, nil, &requestAbort{Status: http.StatusInternalServerError, Message: "marshal failed"} } - subject := "ingest." + query.SafeEncodeNATS(table) - if scope != "" { - subject += "." + query.SafeEncodeNATS(scope) + subject := "ingest." + query.SafeEncodeNATS(st.table) + if st.scope != "" { + subject += "." + query.SafeEncodeNATS(st.scope) } - h.logger.DebugContext(ctx, "publishing event to NATS", "subject", subject, "table", table, "scope", scope) + h.logger.DebugContext(ctx, "publishing event to NATS", "subject", subject, "table", st.table, "scope", st.scope) if err := h.Publisher.Publish(ctx, subject, payload); err != nil { if strings.Contains(err.Error(), "maximum bytes exceeded") { h.logger.WarnContext(ctx, "nats maximum bytes exceeded", "subject", subject) @@ -710,45 +951,36 @@ func (h *IngestHandler) processRecord( return false, nil, nil } -// checkValueMatches decides insert-check equality: the payload value must -// have a canonical scalar form (object/array/null match nothing) equal to the -// required value's canonical form. A policy.LiteralValue — and only that type, -// which Evaluate reserves for placeholder-free check values — also matches by -// its canonical numeric reading (policy.CanonicalNumericLiteral), so a static -// `_eq: "1.0"` accepts an inserted 1.0 and an inserted "1.0" alike. -func checkValueMatches(actual, required any) bool { - actualStr, hasForm := policy.CanonicalScalar(actual) - if !hasForm { - return false - } - if lit, isLit := required.(policy.LiteralValue); isLit { - if actualStr == string(lit) { - return true - } - n, ok := policy.CanonicalNumericLiteral(string(lit)) - return ok && actualStr == n - } - requiredStr, ok := policy.CanonicalScalar(required) - return ok && actualStr == requiredStr -} - -// valueInSet reports whether v matches any member of set, comparing by -// canonical string form (policy.CanonicalScalar) to mirror the scalar check's -// claim-derived equality — a JSON number in the insert body matches a -// claim-derived value by value, not spelling, and a v with no canonical form -// (object/array/null) is a member of no set. _in members never take the -// LiteralValue numeric reading: an _in set is claim-derived by design, and a -// placeholder-free _in template is a degenerate one-element set that keeps -// spelling equality. -func valueInSet(v any, set []any) bool { - vs, ok := policy.CanonicalScalar(v) - if !ok { - return false - } - for _, s := range set { - if ss, ok := policy.CanonicalScalar(s); ok && ss == vs { - return true +// eventIDAt reads the dedupe id out of the exported row by POSITION — the id +// column's index in the table's wire columns — so no record is decoded for it +// (AUDIT §A.4). idx is -1 when the column is not on the wire at all. +// +// Two consequences worth knowing, both documented: +// - the key is the STORED value, not the caller's spelling: `256` into a +// UInt8 keys on `0`, and a DateTime keys on ClickHouse's rendering. For the +// documented case — a string id — the two are identical. +// - "missing" now means "the row carries no value", which for an omitted +// column is its default. An empty id is therefore treated as absent, which +// is what an omitted `event_id String` produces and what the WARN, the +// counter and require_id have always been about. A numeric id column cannot +// distinguish an omitted 0 from a supplied one. +func eventIDAt(line []byte, idx int) (string, bool) { + if idx < 0 { + return "", false + } + cell, ok := cellAt(line, idx) + if !ok || len(cell) == 0 { + return "", false + } + if cell[0] == '"' { + // One scalar string, not the record: the cell is JSON-encoded by + // ClickHouse's own writer (it escapes "/" as "\/"), so Go's own + // string-literal unquoting would refuse it. + var s string + if err := json.Unmarshal(cell, &s); err != nil || s == "" { + return "", false } + return s, true } - return false + return string(cell), true } diff --git a/internal/api/ingest_formats_test.go b/internal/api/ingest_formats_test.go new file mode 100644 index 00000000..200a2850 --- /dev/null +++ b/internal/api/ingest_formats_test.go @@ -0,0 +1,410 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/testutil" +) + +// TestIngest_JSONArray_CompactWithOneBadRecord is the §0.2 regression guard, and +// the whole reason the depth-1 comma rewrite exists. +// +// Measured: a SINGLE-LINE array with one bad record makes chtypes answer +// Outcome=rejected with no exported bytes — the records that parsed perfectly +// are lost with it. That breaks #195's promise that one bad record never +// obscures the rest of the batch. Newline-framing the elements restores it. +func TestIngest_JSONArray_CompactWithOneBadRecord(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "application/json", + `[{"page":"/a"},{"page":"/b","nope":1},{"page":"/c"}]`)) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 3, resp.Total) + assert.Equal(t, 2, resp.Succeeded) + assert.Equal(t, 1, resp.Failed) + require.Len(t, resp.Results, 3) + assert.True(t, resultAt(t, resp, 1).Ok) + assert.Equal(t, 117, resultAt(t, resp, 2).Code) + assert.True(t, resultAt(t, resp, 3).Ok) + require.Len(t, pub.Messages, 2, "the siblings of a refused record still publish") + assert.Equal(t, "/a", publishedRow(t, pub.Messages[0].Data)["page"]) + assert.Equal(t, "/c", publishedRow(t, pub.Messages[1].Data)["page"]) +} + +// TestIngest_CSV: CSV is header-less and POSITIONAL in the table's declaration +// order — the wire columns, which is declaration order minus MATERIALIZED, ALIAS +// and EPHEMERAL. Every wire column must be present; an empty field takes the +// column's DEFAULT. +func TestIngest_CSV(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + // clicks is page, button, count, event_id, org_id. + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv", + "\"/a\",\"buy\",3,\"e1\",\"acme\"\n\"/b\",\"sell\",4,\"e2\",\"acme\"\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 2, resp.Total) + assert.Equal(t, 2, resp.Succeeded) + require.Len(t, pub.Messages, 2) + row := publishedRow(t, pub.Messages[0].Data) + assert.Equal(t, "/a", row["page"]) + assert.Equal(t, "buy", row["button"]) + assert.Equal(t, float64(3), row["count"]) + assert.Equal(t, "acme", row["org_id"]) +} + +// TestIngest_CSV_PositionalContract pins the three ways a producer gets the +// positional contract wrong, each with ClickHouse's own code, under +// `header=absent` (strictly positional, detection off). A header line there is +// not a header: it is one record that fails to parse, and the data rows around +// it still ingest. +func TestIngest_CSV_PositionalContract(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + body string + ok int + bad int + }{ + {"a short row is a per-record failure", "\"/a\",\"buy\"\n\"/b\",\"sell\",4,\"e2\",\"acme\"\n", 1, 1}, + {"an extra field is a per-record failure", "\"/a\",\"buy\",3,\"e1\",\"acme\",99\n", 0, 1}, + {"a header line is one failed record, not a header", "page,button,count,event_id,org_id\n\"/b\",\"sell\",4,\"e2\",\"acme\"\n", 1, 1}, + {"an empty field takes the column default", "\"/a\",\"buy\",3,\"e1\",\n", 1, 0}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv; header=absent", tt.body)) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, tt.ok, resp.Succeeded) + assert.Equal(t, tt.bad, resp.Failed) + assert.Len(t, pub.Messages, tt.ok) + for _, r := range resp.Results { + if r.Error != "" { + assert.NotZero(t, r.Code, "a parser refusal carries ClickHouse's code") + } + } + }) + } +} + +// TestIngest_BareCSV_AutoDetectsHeader: a `text/csv` with no header parameter +// is ClickHouse's default CSV, which consumes a first line that spells the +// column names as a header. The indices and total count only data rows; the +// same body under header=absent rejects that line at index 1 (code 27); a bare +// body with no header line ingests every row. +func TestIngest_BareCSV_AutoDetectsHeader(t *testing.T) { + t.Parallel() + const header = "page,button,count,event_id,org_id\n" + const rows = "\"/a\",\"buy\",3,\"e1\",\"acme\"\n\"/b\",\"sell\",4,\"e2\",\"acme\"\n" + + t.Run("bare consumes the header line", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv", header+rows)) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 2, resp.Total, "the detected header is not a record") + assert.Equal(t, 2, resp.Succeeded) + require.Len(t, pub.Messages, 2) + assert.Equal(t, "/a", publishedRow(t, pub.Messages[0].Data)["page"]) + }) + + t.Run("header=absent rejects the header line", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv; header=absent", header+rows)) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 3, resp.Total) + assert.Equal(t, 2, resp.Succeeded) + assert.Equal(t, 27, resultAt(t, resp, 1).Code, "the first line is a record ClickHouse's positional reader refuses") + assert.Len(t, pub.Messages, 2) + }) + + t.Run("bare with no header line ingests every row", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv", rows)) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 2, resp.Total) + assert.Equal(t, 2, resp.Succeeded) + assert.Len(t, pub.Messages, 2) + }) +} + +// TestIngest_TSV is CSV's tab-separated twin, with the same positional contract +// and ClickHouse's own \N for null. +func TestIngest_TSV(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/tab-separated-values", + "/a\tbuy\t3\te1\tacme\n/b\tsell\tnot-a-number\te2\tacme\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 2, resp.Total) + assert.Equal(t, 1, resp.Succeeded) + assert.Equal(t, 1, resp.Failed) + assert.Equal(t, 27, resultAt(t, resp, 2).Code) + require.Len(t, pub.Messages, 1) + assert.Equal(t, "/a", publishedRow(t, pub.Messages[0].Data)["page"]) +} + +// TestIngest_CSV_IsAlwaysABatch: the positional formats have no arity question — +// a one-row CSV still answers with the per-record envelope, never {"ok":true}. +func TestIngest_CSV_IsAlwaysABatch(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv", "\"/a\",\"buy\",3,\"e1\",\"acme\"\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 1, resp.Total) + assert.Equal(t, 1, resp.Succeeded) +} + +// TestIngest_CSV_EmptyBody names the format in the 400, like the NDJSON one. +func TestIngest_CSV_EmptyBody(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv", " \n ")) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, jsonErrorMessage(t, w), "empty csv body") + assert.Empty(t, pub.Messages) +} + +// TestIngest_CSVWithNames: `text/csv; header=present` reads the first line as +// the column names, in any order. The header is not a record, so indices +// count data lines; a column the header omits takes its DEFAULT. +func TestIngest_CSVWithNames(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv; header=present", + "org_id,page,count\nacme,/a,3\nacme,/b,not-a-number\nbeta,/c,5\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 3, resp.Total, "the header line is not a record") + assert.Equal(t, 2, resp.Succeeded) + assert.True(t, resultAt(t, resp, 1).Ok) + assert.False(t, resultAt(t, resp, 2).Ok) + assert.NotZero(t, resultAt(t, resp, 2).Code, "a parser refusal carries ClickHouse's code") + assert.True(t, resultAt(t, resp, 3).Ok) + require.Len(t, pub.Messages, 2) + row := publishedRow(t, pub.Messages[0].Data) + assert.Equal(t, "/a", row["page"]) + assert.Equal(t, float64(3), row["count"]) + assert.Equal(t, "acme", row["org_id"]) + assert.Equal(t, "", row["button"], "a column the header omits takes its DEFAULT") + assert.Equal(t, "/c", publishedRow(t, pub.Messages[1].Data)["page"]) +} + +// TestIngest_TSVWithNames is the tab-separated twin. +func TestIngest_TSVWithNames(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/tab-separated-values; header=present", + "count\tpage\n7\t/a\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.Equal(t, 1, resp.Total) + assert.Equal(t, 1, resp.Succeeded) + require.Len(t, pub.Messages, 1) + assert.Equal(t, float64(7), publishedRow(t, pub.Messages[0].Data)["count"]) +} + +// TestIngest_WithNames_HeaderRefusals: a header ClickHouse refuses is a +// verdict on the body, not on a record — a whole-request 400 with its own +// code, and nothing published. A header alone is zero records. +func TestIngest_WithNames_HeaderRefusals(t *testing.T) { + t.Parallel() + for name, tc := range map[string]struct { + ct, body, mention string + }{ + "unknown column": {"text/csv; header=present", "page,extra\n/a,1\n", "extra"}, + "repeated column": {"text/csv; header=present", "page,page\n/a,/b\n", "page"}, + "tsv unknown": {"text/tab-separated-values; header=present", "page\textra\n/a\t1\n", "extra"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", tc.ct, tc.body)) + + require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + msg, code := errorAndCode(t, w) + assert.Equal(t, 117, code) + assert.Contains(t, msg, tc.mention) + assert.Empty(t, pub.Messages) + }) + } + + t.Run("header only", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv; header=present", "page,count\n")) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, 0, decodeBatchResult(t, w).Total) + assert.Empty(t, pub.Messages) + }) + + t.Run("empty body names the format", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "text/csv; header=present", " \n")) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, jsonErrorMessage(t, w), "empty csvwithnames body") + }) +} + +// TestIngest_WithNames_RoleProjection: the header is read against the ROLE's +// compiled schema, so a denied column in it is ClickHouse's 117 for the whole +// body, an `_eq` check column the header omits is filled by its injected +// DEFAULT, and a record that supplies another value fails the check (403). +func TestIngest_WithNames_RoleProjection(t *testing.T) { + t.Parallel() + required := "acme" + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + h.PolicySource = policy.Static(&policy.Policy{Tables: map[string]policy.TablePolicy{ + "clicks": {"writer": {Insert: &policy.InsertPermissions{ + DenyColumns: []string{"count"}, + Check: map[string]policy.Filter{"org_id": {Eq: &required}}, + }}}, + }}) + send := func(body string) *httptest.ResponseRecorder { + req := rawIngestRequest(t, "clicks", "text/csv; header=present", body) + req = req.WithContext(auth.WithRole(req.Context(), "writer")) + w := httptest.NewRecorder() + h.Handle(w, req) + return w + } + + w := send("page,count\n/a,1\n") + require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + _, code := errorAndCode(t, w) + assert.Equal(t, 117, code) + assert.Empty(t, pub.Messages) + + w = send("page,org_id\n/a,acme\n/b,evil\n") + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + assert.True(t, resultAt(t, resp, 1).Ok) + assert.Contains(t, resultAt(t, resp, 2).Error, `check failed for column "org_id"`) + + w = send("page\n/c\n") + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.True(t, resultAt(t, decodeBatchResult(t, w), 1).Ok) + require.Len(t, pub.Messages, 2) + assert.Equal(t, "acme", publishedRow(t, pub.Messages[1].Data)["org_id"], "the omitted check column took its injected DEFAULT") +} + +// TestIngest_LargeBatch_IndicesStayContiguous guards the one thing removing the +// handler's 500-record chunking could plausibly break. Verdicts used to be +// gathered per chunk and stitched back together by a staging slice; they are +// now one array from one Ingest call, indexed directly. A batch that straddles +// the old boundary must still report 1-based indices in order, with each +// record's own outcome — an off-by-one there would attribute a refusal to the +// wrong record, which the response gives a caller no way to detect. +func TestIngest_LargeBatch_IndicesStayContiguous(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + const n = 600 + bad := map[int]bool{1: true, 250: true, 500: true, 501: true, n: true} // 1-based + var b strings.Builder + b.WriteByte('[') + for i := 1; i <= n; i++ { + if i > 1 { + b.WriteByte(',') + } + if bad[i] { + fmt.Fprintf(&b, `{"page":"/p%d","nope":%d}`, i, i) + continue + } + fmt.Fprintf(&b, `{"page":"/p%d"}`, i) + } + b.WriteByte(']') + + w := httptest.NewRecorder() + h.Handle(w, rawIngestRequest(t, "clicks", "application/json", b.String())) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + resp := decodeBatchResult(t, w) + require.Equal(t, n, resp.Total) + assert.Equal(t, len(bad), resp.Failed) + assert.Equal(t, n-len(bad), resp.Succeeded) + require.Len(t, resp.Results, n) + for i, r := range resp.Results { + require.Equal(t, i+1, r.Index, "results must be 1-based and in order") + if bad[i+1] { + assert.Equal(t, 117, r.Code, "record %d", i+1) + continue + } + assert.True(t, r.Ok, "record %d", i+1) + } + require.Len(t, pub.Messages, n-len(bad)) + // The published rows are the accepted records, in order, with the refused + // ones simply absent — so record 502 sits four slots earlier than its index + // (records 1, 250, 500 and 501 were refused before it). Spot-checking a row + // on the far side of the old chunk boundary is what would catch a verdict + // misattributed across it. + assert.Equal(t, "/p502", publishedRow(t, pub.Messages[502-1-4].Data)["page"]) + assert.Equal(t, "/p2", publishedRow(t, pub.Messages[0].Data)["page"], "record 1 was refused") + assert.Equal(t, "/p599", publishedRow(t, pub.Messages[len(pub.Messages)-1].Data)["page"], "record 600 was refused") +} diff --git a/internal/api/ingest_framing.go b/internal/api/ingest_framing.go new file mode 100644 index 00000000..81ae54bf --- /dev/null +++ b/internal/api/ingest_framing.go @@ -0,0 +1,154 @@ +package api + +// Framing: everything ingest reads out of the request body's own bytes. It is +// deliberately small — three single-pass scanners, no decoder, no allocation — +// because the whole point of the type layer is that ClickHouse's parser reads +// the records and Go does not. + +// reframeArray turns a top-level JSON array into the newline-framed body +// chtypes reads per record, IN PLACE, and reports how many elements it holds. +// +// It exists because of a measured cliff (AUDIT §0.2): a SINGLE-LINE array with +// one bad record loses the whole batch — chtypes answers Outcome=rejected with +// no exported bytes, so the records that parsed perfectly are lost too. The same +// records newline-separated skip the bad one and export the rest. Rewriting the +// depth-1 commas to newlines restores per-record salvage (#195's promise) for +// 0.7 ms per 617 KB, with no decode and no copy. +// +// The scan is string- and escape-aware, so a comma or a bracket inside a value +// is untouched, and it runs ONLY when the declared format is the JSON family and +// the first non-whitespace byte is '['. It must not run on anything else: a bare +// object's own commas are at depth 1 and rewriting them destroys the record +// (measured). +// +// Three substitutions, all in place and all the same length: +// +// - a depth-1 comma becomes a newline — the framing itself; +// - the OUTER '[' and ']' become spaces. Commas alone are not enough: +// measured, a bad LAST record still loses the whole batch, because the +// closing bracket shares that record's line and the reader cannot resync +// past it. JSONEachRow needs no brackets, so removing them costs nothing and +// makes every position salvageable, first and last included; +// - every other newline outside a string becomes a space. Not cosmetic: +// typelayer.Ingest pads its verdict list out to the body's newline count, so +// a pretty-printed array would come back with one phantom "no verdict" +// record per line of layout. This leaves exactly elements-1 newlines. +// +// A raw newline inside a string is illegal JSON, so leaving those alone costs +// nothing and keeps the caller's bytes the caller's. +// +// ok is false when the brackets do not balance — a truncated upload, or a +// structural syntax error — which is a whole-request 400. Nothing is published +// from a body we cannot frame. +func reframeArray(b []byte) (elements int, ok bool) { + depth, commas := 0, 0 + sawValue := false + inStr, esc := false, false + for i := range b { + c := b[i] + switch { + case esc: + esc = false + case inStr && c == '\\': + esc = true + case c == '"': + inStr = !inStr + sawValue = sawValue || depth >= 1 + case inStr: + case c == '[' || c == '{': + sawValue = sawValue || depth >= 1 + depth++ + if c == '[' && depth == 1 { + b[i] = ' ' + } + case c == ']' || c == '}': + depth-- + if c == ']' && depth == 0 { + b[i] = ' ' + } + case c == ',' && depth == 1: + b[i] = '\n' + commas++ + case c == '\n' || c == '\r': + b[i] = ' ' + case c == ' ' || c == '\t': + default: + sawValue = sawValue || depth >= 1 + } + } + if depth != 0 || inStr { + return 0, false + } + if !sawValue { + return 0, true // `[]`, possibly with whitespace inside + } + return commas + 1, true +} + +// cellAt returns the k-th top-level cell of one JSONCompactEachRow line — a +// `[v0, v1, …]` array as ClickHouse's own writer produced it — without decoding +// the row. Leading and trailing whitespace around the cell is trimmed; the cell +// itself is returned verbatim, still JSON-encoded. +// +// This is how the dedupe id is read (AUDIT §A.4): the id column's position in +// the table's wire columns is known, so the value is a byte span rather than a +// map lookup. The scanner is the same string- and escape-aware shape as +// reframeArray, so a comma or a bracket inside a value cannot end a cell. +func cellAt(line []byte, k int) ([]byte, bool) { + if k < 0 { + return nil, false + } + depth, idx, start := 0, 0, -1 + inStr, esc := false, false + for i := range line { + c := line[i] + switch { + case esc: + esc = false + continue + case inStr && c == '\\': + esc = true + continue + case c == '"': + inStr = !inStr + case inStr: + case c == '[' || c == '{': + depth++ + if depth == 1 { + start = i + 1 + continue + } + case c == ']' || c == '}': + depth-- + if depth == 0 { + if idx == k && start >= 0 { + return trimSpaceBytes(line[start:i]), true + } + return nil, false + } + case c == ',' && depth == 1: + if idx == k { + return trimSpaceBytes(line[start:i]), true + } + idx++ + start = i + 1 + continue + } + } + return nil, false +} + +// trimSpaceBytes drops ASCII layout around a cell. bytes.TrimSpace would also +// do it, but this stays byte-exact about which bytes count as layout in a +// JSONCompactEachRow line (the writer emits ", " between cells) and allocates +// nothing. +func trimSpaceBytes(b []byte) []byte { + i, j := 0, len(b) + for i < j && (b[i] == ' ' || b[i] == '\t' || b[i] == '\n' || b[i] == '\r') { + i++ + } + for j > i && (b[j-1] == ' ' || b[j-1] == '\t' || b[j-1] == '\n' || b[j-1] == '\r') { + j-- + } + return b[i:j] +} diff --git a/internal/api/ingest_framing_test.go b/internal/api/ingest_framing_test.go new file mode 100644 index 00000000..97a39863 --- /dev/null +++ b/internal/api/ingest_framing_test.go @@ -0,0 +1,168 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestReframeArray covers the one place ingest still looks at the body's own +// bytes. The rewrite is string- and escape-aware and runs ONLY on a declared +// JSON body whose first non-whitespace byte is '[' — the gate matters as much as +// the scan, because a bare object's commas are at depth 1 too and rewriting them +// destroys the record (measured, AUDIT §A.0a). +func TestReframeArray(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + body string + want string + count int + ok bool + }{ + { + name: "compact array becomes one record per line", + body: `[{"a":1},{"a":2},{"a":3}]`, + want: " {\"a\":1}\n{\"a\":2}\n{\"a\":3} ", + count: 3, ok: true, + }, + { + name: "a comma inside a string is data", + body: `[{"a":"x,y"},{"a":"z"}]`, + want: " {\"a\":\"x,y\"}\n{\"a\":\"z\"} ", + count: 2, ok: true, + }, + { + name: "brackets inside a string do not move the depth", + body: `[{"a":"]["},{"a":"[[["}]`, + want: " {\"a\":\"][\"}\n{\"a\":\"[[[\"} ", + count: 2, ok: true, + }, + { + name: "an escaped quote does not end the string", + body: `[{"a":"he said \"a,b\""},{"a":"z"}]`, + want: " {\"a\":\"he said \\\"a,b\\\"\"}\n{\"a\":\"z\"} ", + count: 2, ok: true, + }, + { + name: "nested arrays and objects keep their commas", + body: `[{"a":[1,2],"b":{"c":3,"d":4}},{"a":[5]}]`, + want: " {\"a\":[1,2],\"b\":{\"c\":3,\"d\":4}}\n{\"a\":[5]} ", + count: 2, ok: true, + }, + { + // A multi-line array keeps its meaning AND stops carrying layout + // newlines, which is what keeps the record count exact. + name: "a pretty-printed array is one record per line and nothing else", + body: "[\n {\"a\":1},\n {\"a\":2}\n]", + want: " {\"a\":1}\n {\"a\":2} ", + count: 2, ok: true, + }, + { + name: "an empty array holds no records", + body: `[]`, + want: ` `, + count: 0, ok: true, + }, + { + name: "whitespace inside an empty array is still no records", + body: "[\n ]", + want: " ", + count: 0, ok: true, + }, + { + name: "a one-element array is one record", + body: `[{"a":1}]`, + want: ` {"a":1} `, + count: 1, ok: true, + }, + { + name: "scalar elements are still elements", + body: `[1,"x",[2]]`, + want: " 1\n\"x\"\n[2] ", + // They will each be a per-record parse refusal; the framing's job is + // only to keep them separate so the objects around them survive. + count: 3, ok: true, + }, + {name: "a truncated array does not balance", body: `[{"a":1}`, ok: false}, + {name: "a trailing comma cut off does not balance", body: `[{"a":1},`, ok: false}, + {name: "a bare open bracket does not balance", body: `[`, ok: false}, + {name: "a cut-off element does not balance", body: `[{"a":1},{"b`, ok: false}, + {name: "a structural syntax error does not balance", body: `[{"a":1}, {bad]`, ok: false}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + b := []byte(tt.body) + count, ok := reframeArray(b) + require.Equal(t, tt.ok, ok, "balance") + if !tt.ok { + return + } + assert.Equal(t, tt.count, count, "element count") + assert.Equal(t, tt.want, string(b), "rewritten body") + }) + } +} + +// TestReframeArray_DestroysWhatItMustNotSee is the gate, not the scan: these +// bodies are the reason reframeArray runs only behind a leading '['. Running it +// on either would corrupt the record, so the test asserts the damage — if the +// handler ever stops gating, this is the shape of the bug. +func TestReframeArray_DestroysWhatItMustNotSee(t *testing.T) { + t.Parallel() + + obj := []byte(`{"a":1,"b":2}`) + reframeArray(obj) + assert.Equal(t, "{\"a\":1\n\"b\":2}", string(obj), + "a bare object's own commas are at depth 1 — the handler must never send one here") + + ndjson := []byte("{\"a\":1,\"b\":2}\n{\"a\":3,\"b\":4}\n") + reframeArray(ndjson) + assert.NotContains(t, string(ndjson), `{"a":1,"b":2}`, + "an NDJSON body is destroyed too — same reason, same gate") +} + +func TestCellAt(t *testing.T) { + t.Parallel() + const line = `["a, b", "c\"d", 42, null, ["x", "y"], {"k": 1}, "last"]` + for i, want := range []string{`"a, b"`, `"c\"d"`, `42`, `null`, `["x", "y"]`, `{"k": 1}`, `"last"`} { + got, ok := cellAt([]byte(line), i) + require.True(t, ok, "cell %d", i) + assert.Equal(t, want, string(got), "cell %d", i) + } + _, ok := cellAt([]byte(line), 7) + assert.False(t, ok, "past the end") + _, ok = cellAt([]byte(line), -1) + assert.False(t, ok, "before the start") + _, ok = cellAt([]byte(`[`), 0) + assert.False(t, ok, "an unterminated row yields nothing") +} + +// TestEventIDAt: the id is the STORED value, and a string cell is JSON-decoded +// because ClickHouse's writer escapes "/" as "\/" — Go's own string-literal +// unquoting refuses that. +func TestEventIDAt(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + line string + idx int + want string + ok bool + }{ + {"a quoted string is decoded", `["\/a", "evt-1", 0]`, 1, "evt-1", true}, + {"a slash escape survives", `["\/a\/b", 0]`, 0, "/a/b", true}, + {"a number is its digits", `["x", 18446744073709551615]`, 1, "18446744073709551615", true}, + {"an empty string is no id", `["x", ""]`, 1, "", false}, + {"a column that is not on the wire is no id", `["x", "y"]`, -1, "", false}, + {"past the end is no id", `["x"]`, 3, "", false}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, ok := eventIDAt([]byte(tt.line), tt.idx) + assert.Equal(t, tt.ok, ok) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/api/ingest_seams.go b/internal/api/ingest_seams.go deleted file mode 100644 index d95fdb58..00000000 --- a/internal/api/ingest_seams.go +++ /dev/null @@ -1,82 +0,0 @@ -package api - -import ( - "github.com/Wave-RF/WaveHouse/internal/discovery" -) - -// This file holds the per-record decision points a native type layer will take -// over: schema validation, timestamp canonicalization, and the insert-check -// comparison. Each is an interface with a default implementation that delegates -// to today's code unchanged, so the replacement is a wiring change rather than a -// rewrite of the ingest handler. Nothing here decides anything itself. - -// RecordValidator covers the two schema-driven steps of the record pipeline. -// Validate rejects a record the table's schema cannot accept; CanonicalizeTimestamps -// rewrites DateTime/DateTime64 values to the canonical wire form in place. -// -// CanonicalizeTimestamps returns nothing, matching discovery's function: it is -// deliberately fail-open (#372) — a value it cannot read is passed through for -// ClickHouse to judge, and the row filter is what enforces. Giving it an error -// return would invite a caller to change that. -// -// The two are one interface because they are one contract — "what this schema -// says about this record" — evaluated at two points in processRecord that must -// stay apart: the insert-check block sits between them deliberately, so checks -// keep pre-#372 semantics. -type RecordValidator interface { - Validate(schema *discovery.TableSchema, record map[string]any) error - CanonicalizeTimestamps(schema *discovery.TableSchema, record map[string]any) -} - -// discoveryValidator is the default RecordValidator, delegating to -// internal/discovery. -type discoveryValidator struct{} - -func (discoveryValidator) Validate(schema *discovery.TableSchema, record map[string]any) error { - return discovery.Validate(schema, record) -} - -func (discoveryValidator) CanonicalizeTimestamps(schema *discovery.TableSchema, record map[string]any) { - discovery.CanonicalizeTimestamps(schema, record) -} - -// validator returns the handler's RecordValidator, or the default when none is -// wired. Validator is an optional field set after construction, so nil is the -// ordinary case rather than a mistake — and nil must resolve to the enforcing -// default, never to skipping validation. That fail-closed direction is the -// point; see hub.go's rowEvaluator for the same shape on row visibility. -func (h *IngestHandler) validator() RecordValidator { - if h.Validator != nil { - return h.Validator - } - return discoveryValidator{} -} - -// InsertChecker decides whether a record's value satisfies a policy check -// clause. Matches answers the scalar `_eq` form (the required value), InSet the -// `_in` form (set membership). It never sees a record as a whole: the -// auto-injection of a missing check value stays in processRecord, where the -// ordering against validation and canonicalization is load-bearing. -type InsertChecker interface { - Matches(actual, required any) bool - InSet(v any, set []any) bool -} - -// canonicalChecker is the default InsertChecker: the canonical-scalar -// comparison in ingest.go, unchanged. -type canonicalChecker struct{} - -func (canonicalChecker) Matches(actual, required any) bool { - return checkValueMatches(actual, required) -} - -func (canonicalChecker) InSet(v any, set []any) bool { return valueInSet(v, set) } - -// checker returns the handler's InsertChecker, or the default when none is -// wired. Nil-safe for the same reason as validator. -func (h *IngestHandler) checker() InsertChecker { - if h.Checker != nil { - return h.Checker - } - return canonicalChecker{} -} diff --git a/internal/api/ingest_seams_test.go b/internal/api/ingest_seams_test.go deleted file mode 100644 index 683e9ff6..00000000 --- a/internal/api/ingest_seams_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package api - -import ( - "errors" - "net/http" - "net/http/httptest" - "testing" - - "github.com/golang-jwt/jwt/v5" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/Wave-RF/WaveHouse/internal/auth" - "github.com/Wave-RF/WaveHouse/internal/discovery" - "github.com/Wave-RF/WaveHouse/internal/policy" - "github.com/Wave-RF/WaveHouse/internal/testutil" -) - -// recordingValidator observes the two schema-driven steps and can fail -// validation on demand, so a test can prove the handler goes through the seam -// rather than calling discovery directly. -type recordingValidator struct { - validateErr error - validated int - canonicalized int - canonicalizeAs string // non-empty ⇒ stamp this into record["page"] -} - -func (v *recordingValidator) Validate(_ *discovery.TableSchema, _ map[string]any) error { - v.validated++ - return v.validateErr -} - -func (v *recordingValidator) CanonicalizeTimestamps(_ *discovery.TableSchema, record map[string]any) { - v.canonicalized++ - if v.canonicalizeAs != "" { - record["page"] = v.canonicalizeAs - } -} - -// TestIngest_RecordValidatorSeam_IsUsed: a wired RecordValidator replaces both -// steps — its rejection is the record's rejection, and its rewrite is what gets -// published. -func TestIngest_RecordValidatorSeam_IsUsed(t *testing.T) { - t.Parallel() - - t.Run("rejection surfaces as a 400", func(t *testing.T) { - t.Parallel() - pub := &testutil.MockPublisher{} - v := &recordingValidator{validateErr: errors.New("seam says no")} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.Validator = v - - w := httptest.NewRecorder() - h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/home"})) - - assert.Equal(t, http.StatusBadRequest, w.Code) - testutil.AssertJSONErrorResponse(t, w) - assert.Contains(t, w.Body.String(), "seam says no") - assert.Equal(t, 1, v.validated) - assert.Zero(t, v.canonicalized, "a rejected record never reaches canonicalization") - assert.Empty(t, pub.Messages) - }) - - t.Run("canonicalization rewrites the published record", func(t *testing.T) { - t.Parallel() - pub := &testutil.MockPublisher{} - v := &recordingValidator{canonicalizeAs: "/rewritten"} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.Validator = v - - w := httptest.NewRecorder() - h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/home"})) - - require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, 1, v.validated) - assert.Equal(t, 1, v.canonicalized) - require.Len(t, pub.Messages, 1) - assert.Contains(t, string(pub.Messages[0].Data), "/rewritten") - }) -} - -// TestIngest_DefaultValidator_WhenUnwired: a handler with no seam wired still -// validates against the schema — the nil case must not read as "allow". -func TestIngest_DefaultValidator_WhenUnwired(t *testing.T) { - t.Parallel() - pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - require.Nil(t, h.Validator) - assert.IsType(t, discoveryValidator{}, h.validator()) - - w := httptest.NewRecorder() - h.Handle(w, ingestRequest(t, "clicks", map[string]any{"nonexistent_field": 1})) - assert.Equal(t, http.StatusBadRequest, w.Code) - testutil.AssertJSONErrorResponse(t, w) - assert.Empty(t, pub.Messages) -} - -// alwaysChecker answers every insert check the same way, so a test can tell -// which arm the handler consulted. -type alwaysChecker struct { - matches bool - inSet bool -} - -func (c alwaysChecker) Matches(_, _ any) bool { return c.matches } -func (c alwaysChecker) InSet(_ any, _ []any) bool { return c.inSet } - -// TestIngest_InsertCheckerSeam_IsUsed: a wired InsertChecker decides the check -// clause. The record here would fail the canonical comparison, so a 200 proves -// the seam — not the default — answered. -func TestIngest_InsertCheckerSeam_IsUsed(t *testing.T) { - t.Parallel() - required := "org-allowed" - p := &policy.Policy{Tables: map[string]policy.TablePolicy{ - "clicks": {"viewer": {Insert: &policy.InsertPermissions{ - Check: map[string]policy.Filter{"org_id": {Eq: &required}}, - }}}, - }} - - for _, tt := range []struct { - name string - matches bool - want int - }{ - {"seam admits a value the canonical comparison would reject", true, http.StatusOK}, - {"seam rejects a value the canonical comparison would admit", false, http.StatusForbidden}, - } { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.PolicySource = policy.Static(p) - h.Checker = alwaysChecker{matches: tt.matches} - - value := "org-something-else" - if !tt.matches { - value = required // the default checker would accept this - } - w := httptest.NewRecorder() - h.Handle(w, viewerIngestRequest(t, "clicks", map[string]any{"page": "/a", "org_id": value})) - assert.Equal(t, tt.want, w.Code, "body=%s", w.Body.String()) - if tt.want == http.StatusForbidden { - testutil.AssertJSONErrorResponse(t, w) - } - }) - } -} - -// TestIngest_InsertCheckerSeam_InSet_IsUsed covers the _in arm, which reaches a -// DIFFERENT seam method (InSet, not Matches). Without this, replacing -// h.checker().InSet with the canonical valueInSet leaves the whole suite green — -// the existing _in tests exercise that arm only through the default checker. A -// swapped-in type-aware checker would then take effect for _eq and silently not -// for _in, inside insert-check authorization. -func TestIngest_InsertCheckerSeam_InSet_IsUsed(t *testing.T) { - t.Parallel() - - for _, tt := range []struct { - name string - inSet bool - value string - want int - }{ - {"seam admits a value the canonical comparison would reject", true, "org-z", http.StatusOK}, - {"seam rejects a value the canonical comparison would admit", false, "org-b", http.StatusForbidden}, - } { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.PolicySource = checkInStore() - h.Checker = alwaysChecker{inSet: tt.inSet} - - req := ingestRequest(t, "clicks", map[string]any{"page": "/a", "org_id": tt.value}) - ctx := auth.WithRole(req.Context(), "user") - ctx = auth.WithClaims(ctx, jwt.MapClaims{"orgs": []any{"org-a", "org-b"}}) - req = req.WithContext(ctx) - - w := httptest.NewRecorder() - h.Handle(w, req) - assert.Equal(t, tt.want, w.Code, "body=%s", w.Body.String()) - if tt.want == http.StatusForbidden { - testutil.AssertJSONErrorResponse(t, w) - } - }) - } -} - -// TestIngest_DefaultChecker_WhenUnwired: with no seam wired the canonical -// comparison decides, and a mismatched value is still a 403. -func TestIngest_DefaultChecker_WhenUnwired(t *testing.T) { - t.Parallel() - required := "org-allowed" - p := &policy.Policy{Tables: map[string]policy.TablePolicy{ - "clicks": {"viewer": {Insert: &policy.InsertPermissions{ - Check: map[string]policy.Filter{"org_id": {Eq: &required}}, - }}}, - }} - pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.PolicySource = policy.Static(p) - require.Nil(t, h.Checker) - assert.IsType(t, canonicalChecker{}, h.checker()) - - w := httptest.NewRecorder() - h.Handle(w, viewerIngestRequest(t, "clicks", map[string]any{"page": "/a", "org_id": "wrong"})) - assert.Equal(t, http.StatusForbidden, w.Code) - testutil.AssertJSONErrorResponse(t, w) - assert.Empty(t, pub.Messages) -} - -// TestIngest_SeamOrdering_ChecksSitBetweenValidateAndCanonicalize: the two -// RecordValidator calls stay at their current positions with the check-clause -// block between them. Merging them would move check clauses onto canonicalized -// values, silently changing pre-#372 check semantics. -func TestIngest_SeamOrdering_ChecksSitBetweenValidateAndCanonicalize(t *testing.T) { - t.Parallel() - required := "org-allowed" - p := &policy.Policy{Tables: map[string]policy.TablePolicy{ - "clicks": {"viewer": {Insert: &policy.InsertPermissions{ - Check: map[string]policy.Filter{"org_id": {Eq: &required}}, - }}}, - }} - pub := &testutil.MockPublisher{} - v := &recordingValidator{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - h.PolicySource = policy.Static(p) - h.Validator = v - - // A failing check must land AFTER Validate and BEFORE canonicalization. - w := httptest.NewRecorder() - h.Handle(w, viewerIngestRequest(t, "clicks", map[string]any{"page": "/a", "org_id": "wrong"})) - require.Equal(t, http.StatusForbidden, w.Code) - testutil.AssertJSONErrorResponse(t, w) - assert.Equal(t, 1, v.validated, "validation runs before the check clauses") - assert.Zero(t, v.canonicalized, "canonicalization runs after them, so a failed check never reaches it") -} - -// viewerIngestRequest is ingestRequest with the "viewer" role in context, for -// the policy-gated seam tests. -func viewerIngestRequest(t *testing.T, table string, body map[string]any) *http.Request { - t.Helper() - req := ingestRequest(t, table, body) - return req.WithContext(auth.WithRole(req.Context(), "viewer")) -} diff --git a/internal/api/ingest_test.go b/internal/api/ingest_test.go index b267036d..02b1f7b4 100644 --- a/internal/api/ingest_test.go +++ b/internal/api/ingest_test.go @@ -11,16 +11,18 @@ import ( "net/http/httptest" "net/url" "strings" + "sync" "testing" "testing/iotest" - "time" "github.com/Wave-RF/WaveHouse/internal/auth" "github.com/Wave-RF/WaveHouse/internal/dedupe" "github.com/Wave-RF/WaveHouse/internal/discovery" "github.com/Wave-RF/WaveHouse/internal/ingest" + "github.com/Wave-RF/WaveHouse/internal/mq" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/testutil" + "github.com/Wave-RF/WaveHouse/internal/typelayer" "github.com/golang-jwt/jwt/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -32,15 +34,36 @@ func testRegistry(t testing.TB) *discovery.SchemaRegistry { Name: "clicks", Columns: []discovery.Column{ {Name: "page", Type: "String"}, - {Name: "button", Type: "String", HasDefault: true}, - {Name: "count", Type: "UInt64", HasDefault: true}, - {Name: "event_id", Type: "String", HasDefault: true}, - {Name: "org_id", Type: "String", HasDefault: true}, + {Name: "button", Type: "String", HasDefault: true, DefaultExpression: "''"}, + {Name: "count", Type: "UInt64", HasDefault: true, DefaultExpression: "0"}, + {Name: "event_id", Type: "String", HasDefault: true, DefaultExpression: "''"}, + {Name: "org_id", Type: "String", HasDefault: true, DefaultExpression: "''"}, }, }, }) } +// engineMu serialises engine construction. typelayer.Engine.Bind sets +// chtypes.Timezone — a PACKAGE global in the SDK — under the engine's own lock, +// which does not order two engines against each other, so parallel tests each +// building one race on that write. Production has exactly one engine and never +// hits it; this is a test-only workaround for a typelayer defect, and the fix +// belongs there (guard the global with a package-level mutex). +var engineMu sync.Mutex + +// newTestIngestHandler wires a handler the way production does: over a real +// chtypes engine compiled from the registry's own schemas. There is no +// validation-free mode to test against — a nil Types is a 503 by design — so +// every handler test that reaches a record needs one. +func newTestIngestHandler(t testing.TB, reg *discovery.SchemaRegistry, pub mq.Publisher, logger *slog.Logger) *IngestHandler { + t.Helper() + engineMu.Lock() + defer engineMu.Unlock() + h := NewIngestHandler(reg, pub, logger) + h.Types = typelayer.TestEngine(t, reg.List()...) + return h +} + func ingestRequest(t *testing.T, table string, body any) *http.Request { t.Helper() data, err := json.Marshal(body) @@ -55,7 +78,7 @@ func ingestRequest(t *testing.T, table string, body any) *http.Request { func TestIngest_ValidPayload(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "clicks", map[string]any{"page": "/home", "count": 1}) w := httptest.NewRecorder() @@ -105,7 +128,7 @@ func TestIngest_MissingTable(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := httptest.NewRequestWithContext( context.Background(), @@ -128,7 +151,7 @@ func TestIngest_MissingTable(t *testing.T) { func TestIngest_UnknownTable(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "nonexistent", map[string]any{"x": 1}) w := httptest.NewRecorder() @@ -142,35 +165,118 @@ func TestIngest_UnknownTable(t *testing.T) { func TestIngest_InvalidJSON(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) r := rawIngestRequest(t, "clicks", "application/json", "not json") w := httptest.NewRecorder() h.Handle(w, r) assert.Equal(t, http.StatusBadRequest, w.Code) - assert.Contains(t, w.Body.String(), "invalid json") + // CONTRACT CHANGE: the message is ClickHouse's own, with its code, because + // nothing in Go reads the body any more. It used to be the flat + // "invalid json" the Go decoder produced. + msg, code := errorAndCode(t, w) + assert.Contains(t, msg, "expected '{'") + assert.Equal(t, 27, code) testutil.AssertJSONErrorResponse(t, w) + assert.Empty(t, pub.Messages) } +// TestIngest_SchemaValidation_UnknownField: a field the table does not have is +// a real ClickHouse rejection (code 117), not a gateway guess. The compile +// profile pins input_format_skip_unknown_fields=0 precisely so this is a +// verdict the caller hears about rather than silent data loss. func TestIngest_SchemaValidation_UnknownField(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "clicks", map[string]any{"page": "/home", "nonexistent_field": 42}) w := httptest.NewRecorder() h.Handle(w, req) assert.Equal(t, http.StatusBadRequest, w.Code) - assert.Contains(t, w.Body.String(), "error") + msg, code := errorAndCode(t, w) + assert.Contains(t, msg, "nonexistent_field") + assert.Equal(t, 117, code) + assert.Empty(t, pub.Messages) +} + +// TestIngest_MissingRequiredColumn_TakesTheDefault records a DELIBERATE +// behaviour change: WaveHouse used to answer 400 "missing required column" for +// a column that is neither nullable nor defaulted. ClickHouse does not — it +// reads an omitted field as the type's default — and the gateway now gives the +// server's answer rather than its own. Measured on 26.6.3.62: `{}` into +// `page String` stores the empty string. +func TestIngest_MissingRequiredColumn_TakesTheDefault(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"count": 1})) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "", publishedData(t, pub)["page"], "the omitted required column takes its type default") +} + +// TestIngest_ComputedColumns_AreNotOnTheWire: the envelope's Columns are the +// columns the exported row actually carries — declaration order minus +// MATERIALIZED, ALIAS and EPHEMERAL. The old envelope used InsertableColumns, +// which counts EPHEMERAL in, so a table with one announced a column the row did +// not have. Supplying one is the server's own 117. +func TestIngest_ComputedColumns_AreNotOnTheWire(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, computedRegistry(t), pub, testutil.NopLogger()) + + w := httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/a"})) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + + var evt ingest.EventMessage + require.NoError(t, json.Unmarshal(pub.LastMessage().Data, &evt)) + assert.Equal(t, []string{"page", "country"}, evt.Columns) + assert.JSONEq(t, `["/a", "US"]`, string(evt.Row), "the MATERIALIZED value is the server's to compute") + + w = httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/a", "raw": "x"})) + require.Equal(t, http.StatusBadRequest, w.Code) + _, code := errorAndCode(t, w) + assert.Equal(t, 117, code, "a record naming an EPHEMERAL column is refused per record, with ClickHouse's code") +} + +// TestIngest_Batch_PerRecordCodes: a batch reports each refused record's own +// ClickHouse code alongside its message, and one bad record does not cost its +// siblings. +func TestIngest_Batch_PerRecordCodes(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + + req := rawIngestRequest(t, "clicks", "application/json", + `[{"page":"/a"},{"page":"/b","nope":1},{"page":"/c","count":"x"},{"page":"/d"}]`) + w := httptest.NewRecorder() + h.Handle(w, req) + + require.Equal(t, http.StatusOK, w.Code) + resp := decodeBatchResult(t, w) + assert.Equal(t, 4, resp.Total) + assert.Equal(t, 2, resp.Succeeded) + assert.Equal(t, 2, resp.Failed) + require.Len(t, resp.Results, 4) + assert.True(t, resp.Results[0].Ok) + assert.Equal(t, 117, resp.Results[1].Code) + assert.Equal(t, 27, resp.Results[2].Code) + assert.True(t, resp.Results[3].Ok) + assert.Len(t, pub.Messages, 2, "the siblings of a refused record still publish") } func TestIngest_Dedup_FirstTime(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", false } @@ -186,7 +292,7 @@ func TestIngest_Dedup_Duplicate(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", false } @@ -212,7 +318,7 @@ func TestIngest_Dedup_Duplicate(t *testing.T) { func TestIngest_PublishError_503(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{Err: errors.New("maximum bytes exceeded")} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "clicks", map[string]any{"page": "/home"}) w := httptest.NewRecorder() @@ -226,7 +332,7 @@ func TestIngest_PublishError_503(t *testing.T) { func TestIngest_PublishError_500(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{Err: errors.New("some other error")} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "clicks", map[string]any{"page": "/home"}) w := httptest.NewRecorder() @@ -240,7 +346,7 @@ func TestIngest_PublishError_500(t *testing.T) { func TestIngest_Policy_Forbidden(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -265,7 +371,7 @@ func TestIngest_Policy_Forbidden(t *testing.T) { func TestIngest_Policy_ColumnDenied(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -282,14 +388,21 @@ func TestIngest_Policy_ColumnDenied(t *testing.T) { w := httptest.NewRecorder() h.Handle(w, req) - assert.Equal(t, http.StatusForbidden, w.Code) - assert.Contains(t, w.Body.String(), "not allowed for insert") + // CONTRACT CHANGE (AUDIT D1): column policy is answered by compiling the + // role's own schema WITHOUT the denied columns, so the refusal is + // ClickHouse's per-record code 117 — a 400, not the gateway's 403. It no + // longer confirms whether the column exists at all. + assert.Equal(t, http.StatusBadRequest, w.Code) + msg, code := errorAndCode(t, w) + assert.Contains(t, msg, "button") + assert.Equal(t, 117, code) + assert.Empty(t, pub.Messages) } func TestIngest_Policy_CheckClause_Mismatch(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -317,7 +430,7 @@ func TestIngest_Policy_CheckClause_Mismatch(t *testing.T) { func TestIngest_Policy_CheckClause_Match(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -343,15 +456,15 @@ func TestIngest_Policy_CheckClause_Match(t *testing.T) { assert.NotNil(t, pub.LastMessage(), "should have published") } -// TestIngest_Policy_CheckClause_NumericSpellingMatch: the check comparison is -// canonical on both sides (policy.CanonicalScalar), so a numeric claim and a -// numeric insert value match by value even when their JSON spellings differ — -// a claim spelled 1.0 accepts an inserted 1. The former string-form comparison -// ("1.0" != "1") rejected exactly this insert. +// TestIngest_Policy_CheckClause_NumericSpellingMatch: a numeric CLAIM still +// matches a numeric insert value whose JSON spelling differs — a claim spelled +// 1.0 accepts an inserted 1. Policy canonicalizes the claim to "1" on the way +// in, and the check then binds it as a String parameter that ClickHouse +// compares against the stored UInt64. Nothing in Go compares the two values. func TestIngest_Policy_CheckClause_NumericSpellingMatch(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) countTemplate := "{{ jwt.max_count }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -376,26 +489,41 @@ func TestIngest_Policy_CheckClause_NumericSpellingMatch(t *testing.T) { assert.NotNil(t, pub.LastMessage(), "should have published") } -// TestIngest_Policy_CheckClause_StaticNumericSpelling: a check value with no -// placeholder carries no JSON type, so the comparison accepts either reading -// of it — a static `_eq: "1.0"` accepts an inserted number 1 (its canonical -// numeric reading) and an inserted string "1.0" (its spelling, the pre-PR -// behavior) alike. Without the numeric reading, the payload side is canonical -// ("1") while the static side keeps its raw spelling ("1.0") and the check -// rejects every numeric insert it was written to allow. +// TestIngest_Policy_CheckClause_StaticNumericSpelling is a DOCUMENTED CONTRACT +// CHANGE. A placeholder-free check value used to carry a second, numeric +// reading in Go — a typed marker on the resolved clause plus a canonical +// numeric re-render of the literal, both since deleted — so a static +// `_eq: "1.0"` admitted an inserted number 1. The check is now ClickHouse's own +// comparison, and on an integer column the claim goes through the strict cast +// (chsql.StrictInt): "1.0" is not the canonical spelling of a UInt64, so it +// matches nothing and the record is refused as a failed check (403) — the +// same answer as a claim past the column's range. +// +// The value is also what would have been injected into a record that omitted +// the column, and `count UInt64 DEFAULT '1.0'` does not compile (code 6, +// measured). The handler retries the shape without its defaults rather than +// answering 503, so the request still gets a per-record verdict. +// +// The operator fix is to write the literal the column can read (`_eq: "1"`); +// the covering case below pins that it still works. func TestIngest_Policy_CheckClause_StaticNumericSpelling(t *testing.T) { t.Parallel() for _, tt := range []struct { - name string - body any + name string + body any + want int + checkFailed bool }{ - {"numeric reading", 1}, - {"literal spelling", "1.0"}, + {"numeric reading", 1, http.StatusForbidden, true}, + // ClickHouse refuses the record before the check is ever consulted: the + // column is a UInt64 and the string "1.0" is not one. Parse errors + // precede check errors now (AUDIT §A.2). + {"literal spelling", "1.0", http.StatusBadRequest, false}, } { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) staticCount := "1.0" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -415,18 +543,42 @@ func TestIngest_Policy_CheckClause_StaticNumericSpelling(t *testing.T) { w := httptest.NewRecorder() h.Handle(w, req) - assert.Equal(t, http.StatusOK, w.Code) - assert.NotNil(t, pub.LastMessage(), "should have published") + assert.Equal(t, tt.want, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, tt.checkFailed, strings.Contains(w.Body.String(), "check failed"), + "the claim that does not fit is the check saying no; the parse refusal comes first") + assert.Empty(t, pub.Messages) }) } + + // The covering case: a literal the column CAN read still admits the record, + // so the change above is about the spelling, not about static checks. + t.Run("a readable literal still admits the record", func(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + staticCount := "1" + h.PolicySource = policy.Static(&policy.Policy{ + Tables: map[string]policy.TablePolicy{ + "clicks": {"user": {Insert: &policy.InsertPermissions{Check: map[string]policy.Filter{ + "count": {Eq: &staticCount}, + }}}}, + }, + }) + req := ingestRequest(t, "clicks", map[string]any{"page": "/home", "count": 1}) + req = req.WithContext(auth.WithRole(req.Context(), "user")) + w := httptest.NewRecorder() + h.Handle(w, req) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Len(t, pub.Messages, 1) + }) } -// TestIngest_Policy_CheckClause_StringClaimStrictEquality: the numeric -// reading is reserved for placeholder-free literals (policy.LiteralValue). A -// claim-derived required value keeps strict canonical equality, so a writer -// whose claim is the STRING "1e3" cannot insert the number 1000 — its id is -// the three-character text, and accepting the numeric reading would let it -// store a row under the tenant whose String id is "1000". +// TestIngest_Policy_CheckClause_StringClaimStrictEquality: a writer whose claim +// is the STRING "1e3" cannot insert the number 1000 — its id is the +// three-character text, and a numeric reading would let it store a row under +// the tenant whose String id is "1000". ClickHouse enforces it now: the column +// is a String, so the comparison is between strings and no numeric reading +// exists to slip through. func TestIngest_Policy_CheckClause_StringClaimStrictEquality(t *testing.T) { t.Parallel() for _, tt := range []struct { @@ -440,7 +592,7 @@ func TestIngest_Policy_CheckClause_StringClaimStrictEquality(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -465,15 +617,25 @@ func TestIngest_Policy_CheckClause_StringClaimStrictEquality(t *testing.T) { } } -// TestIngest_Policy_CheckClause_NullValue_FailsClosed: the _eq twin of the -// _in null test. With the claim absent the required value is "", and -// CanonicalScalar(nil) also renders "" — only the hasForm guard separates -// them, so without it an explicit null in the payload would satisfy the check -// (pre-canonicalization, fmt.Sprint(nil) gave "" and this was impossible). -func TestIngest_Policy_CheckClause_NullValue_FailsClosed(t *testing.T) { +// TestIngest_Policy_CheckClause_NullValue_StoredValueIsWhatIsChecked is a +// DOCUMENTED CONTRACT CHANGE, and the reason is worth stating precisely because +// it reads as a loosening. +// +// The check is now evaluated against the row ClickHouse WOULD STORE, not against +// the caller's spelling. `input_format_null_as_default=1` — the setting the real +// INSERT already pins — turns an explicit JSON null on a non-nullable column +// into that column's default, so `org_id: null` stores "". The required value +// here is also "": policy deliberately resolves an unresolvable check claim to +// the empty string and auto-injects it (#463), so a record OMITTING org_id has +// always been accepted and stored "". The two cases now agree, where the Go-side +// rule ("null has no canonical form, so it matches nothing") made them differ. +// +// Nothing is admitted that the stored row does not satisfy — which is the +// property the check clause is for. +func TestIngest_Policy_CheckClause_NullValue_StoredValueIsWhatIsChecked(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -493,16 +655,47 @@ func TestIngest_Policy_CheckClause_NullValue_FailsClosed(t *testing.T) { w := httptest.NewRecorder() h.Handle(w, req) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "", publishedData(t, pub)["org_id"], "the stored value is what satisfied the check") + + // With a claim that DOES resolve, an explicit null takes the INJECTED value + // rather than the table's own default: null_as_default resolves it against + // the ROLE's compiled schema, whose DEFAULT is the claim. A null on a checked + // column therefore behaves exactly like omitting it, and can never carry + // another tenant's value — the property that matters. + pub2 := &testutil.MockPublisher{} + h2 := newTestIngestHandler(t, testRegistry(t), pub2, testutil.NopLogger()) + h2.PolicySource = h.PolicySource + req = ingestRequest(t, "clicks", map[string]any{"page": "/home", "org_id": nil}) + ctx = auth.WithRole(req.Context(), "user") + ctx = auth.WithClaims(ctx, jwt.MapClaims{"org_id": "real-org"}) + w = httptest.NewRecorder() + h2.Handle(w, req.WithContext(ctx)) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "real-org", publishedData(t, pub2)["org_id"]) + + // A null is not a way past the check: a value that is present and wrong is + // still refused. + pub3 := &testutil.MockPublisher{} + h3 := newTestIngestHandler(t, testRegistry(t), pub3, testutil.NopLogger()) + h3.PolicySource = h.PolicySource + req = ingestRequest(t, "clicks", map[string]any{"page": "/home", "org_id": "someone-else"}) + ctx = auth.WithRole(req.Context(), "user") + ctx = auth.WithClaims(ctx, jwt.MapClaims{"org_id": "real-org"}) + w = httptest.NewRecorder() + h3.Handle(w, req.WithContext(ctx)) + assert.Equal(t, http.StatusForbidden, w.Code) assert.Contains(t, w.Body.String(), "check failed") - assert.Nil(t, pub.LastMessage(), "a null payload value must not satisfy an unresolvable check") + assert.Empty(t, pub3.Messages) testutil.AssertJSONErrorResponse(t, w) } func TestIngest_Policy_CheckClause_AutoInject(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -548,7 +741,7 @@ func checkInStore() policy.Source { func TestIngest_Policy_CheckIn_InSet(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = checkInStore() // org_id is one of the token's allowed orgs — should pass. @@ -567,7 +760,7 @@ func TestIngest_Policy_CheckIn_InSet(t *testing.T) { func TestIngest_Policy_CheckIn_NotInSet(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = checkInStore() // org_id is NOT one of the token's allowed orgs — forging another tenant's row. @@ -584,15 +777,22 @@ func TestIngest_Policy_CheckIn_NotInSet(t *testing.T) { testutil.AssertJSONErrorResponse(t, w) } -// TestIngest_Policy_CheckIn_NullValue_FailsClosed: an explicit null passes -// schema validation for a defaulted column, but it has no canonical scalar -// form — so it is a member of NO set, even one carrying the empty string "" -// (the regression shape: an unguarded canonicalization would render null as -// "" and match an empty-string member). -func TestIngest_Policy_CheckIn_NullValue_FailsClosed(t *testing.T) { +// TestIngest_Policy_CheckIn_NullValue_ChecksTheStoredValue is a DOCUMENTED +// CONTRACT CHANGE, the _in twin of the _eq one above. An explicit null on a +// non-nullable column stores that column's default (input_format_null_as_default +// =1, the setting the real INSERT pins), and an _in check has no value to +// inject, so the stored "" is what the filter tests. A token whose allowed set +// LISTS "" therefore admits it — the row it stores really is one the token +// authorises. The old Go-side rule said null has no canonical form and matched +// nothing. +// +// The second half is the part that must not move: an allowed set WITHOUT "" is +// still a refusal, so this is not a way to write a row under a tenant the token +// does not carry. +func TestIngest_Policy_CheckIn_NullValue_ChecksTheStoredValue(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = checkInStore() req := ingestRequest(t, "clicks", map[string]any{"page": "/home", "org_id": nil}) @@ -603,15 +803,28 @@ func TestIngest_Policy_CheckIn_NullValue_FailsClosed(t *testing.T) { w := httptest.NewRecorder() h.Handle(w, req) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "", publishedData(t, pub)["org_id"], `the stored "" is a member of the token's own set`) + + pub2 := &testutil.MockPublisher{} + h2 := newTestIngestHandler(t, testRegistry(t), pub2, testutil.NopLogger()) + h2.PolicySource = checkInStore() + req = ingestRequest(t, "clicks", map[string]any{"page": "/home", "org_id": nil}) + ctx = auth.WithRole(req.Context(), "user") + ctx = auth.WithClaims(ctx, jwt.MapClaims{"orgs": []any{"org-a", "org-b"}}) + w = httptest.NewRecorder() + h2.Handle(w, req.WithContext(ctx)) + assert.Equal(t, http.StatusForbidden, w.Code) assert.Contains(t, w.Body.String(), "check failed") + assert.Empty(t, pub2.Messages, `a set without "" still refuses the null`) testutil.AssertJSONErrorResponse(t, w) } func TestIngest_Policy_CheckIn_Absent_FailsClosed(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = checkInStore() // org_id omitted — unlike _eq there's no single value to auto-inject, so the @@ -638,7 +851,7 @@ func TestIngest_Policy_CheckIn_Absent_FailsClosed(t *testing.T) { func TestIngest_Policy_CheckIn_AbsentClaim_FailsClosed(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = checkInStore() // The `orgs` claim is absent entirely, so the _in set resolves to a typed-nil @@ -662,7 +875,7 @@ func TestIngest_Dedup_MissingIDField(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", false } @@ -681,7 +894,7 @@ func TestIngest_Dedup_MissingIDField(t *testing.T) { func TestIngest_Dedup_RequireID_Rejects(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = testutil.NewMockDeduplicator() h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", true } @@ -703,7 +916,7 @@ func TestIngest_Dedup_RequireID_Rejects(t *testing.T) { func TestIngest_NDJSON_RequireID_Rejects(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = testutil.NewMockDeduplicator() h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", true } @@ -730,7 +943,7 @@ func TestIngest_NDJSON_RequireID_Rejects(t *testing.T) { func TestIngest_Policy_DenyColumns(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -746,14 +959,28 @@ func TestIngest_Policy_DenyColumns(t *testing.T) { w := httptest.NewRecorder() h.Handle(w, req) - assert.Equal(t, http.StatusForbidden, w.Code) - assert.Contains(t, w.Body.String(), "not allowed for insert") + // CONTRACT CHANGE (AUDIT D1), as for AllowColumns: a denied column is absent + // from the role's compiled schema, so naming it is ClickHouse's code 117. + assert.Equal(t, http.StatusBadRequest, w.Code) + msg, code := errorAndCode(t, w) + assert.Contains(t, msg, "count") + assert.Equal(t, 117, code) + assert.Empty(t, pub.Messages) + + // And the deny is real, not an artifact of the message: the same record + // without the denied column is accepted. + req = ingestRequest(t, "clicks", map[string]any{"page": "/home"}) + req = req.WithContext(auth.WithRole(req.Context(), "writer")) + w = httptest.NewRecorder() + h.Handle(w, req) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Len(t, pub.Messages, 1) } func TestIngest_AdminRole_NoPolicy(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": {}, @@ -817,7 +1044,7 @@ func resultAt(t *testing.T, resp batchResult, index int) recordResult { func TestIngest_NDJSON_AllValid(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a", "count": 1}), @@ -845,7 +1072,7 @@ func TestIngest_NDJSON_AllValid(t *testing.T) { func TestIngest_NDJSON_PartialFailure_Validation(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a"}), @@ -871,7 +1098,7 @@ func TestIngest_NDJSON_PartialFailure_Validation(t *testing.T) { func TestIngest_NDJSON_MalformedLine(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a"}), @@ -888,7 +1115,12 @@ func TestIngest_NDJSON_MalformedLine(t *testing.T) { assert.Equal(t, 1, resp.Failed) require.Len(t, resp.Results, 3) assert.True(t, resultAt(t, resp, 1).Ok) - assert.Contains(t, resultAt(t, resp, 2).Error, "invalid json") + // The message is ClickHouse's own parse refusal now, not Go's "invalid json", + // and the code is whichever one its reader raised (26 for a quoted string it + // cannot finish, 27 for a value it cannot read) — the point is that a code is + // attributed at all. + assert.NotZero(t, resultAt(t, resp, 2).Code) + assert.NotEmpty(t, resultAt(t, resp, 2).Error) assert.True(t, resultAt(t, resp, 3).Ok) assert.Len(t, pub.Messages, 2) } @@ -896,7 +1128,7 @@ func TestIngest_NDJSON_MalformedLine(t *testing.T) { func TestIngest_NDJSON_BlankLinesSkipped(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // Leading, interior, and whitespace-only lines are all skipped; only real // records are counted. @@ -933,7 +1165,7 @@ func TestIngest_NDJSON_EmptyBody(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", tt.lines...) w := httptest.NewRecorder() @@ -951,7 +1183,7 @@ func TestIngest_NDJSON_Dedup(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", false } @@ -980,7 +1212,7 @@ func TestIngest_NDJSON_Backpressure_503(t *testing.T) { // Publisher rejects every publish with the backpressure sentinel; the first // valid record aborts the whole batch with 503 + Retry-After. pub := &testutil.MockPublisher{Err: errors.New("maximum bytes exceeded")} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a"}), @@ -997,7 +1229,7 @@ func TestIngest_NDJSON_Backpressure_503(t *testing.T) { func TestIngest_NDJSON_PublishError_500(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{Err: errors.New("some other error")} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a"})) w := httptest.NewRecorder() @@ -1011,7 +1243,7 @@ func TestIngest_NDJSON_PublishError_500(t *testing.T) { func TestIngest_NDJSON_Policy_ColumnDenied_PerLine(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -1037,14 +1269,16 @@ func TestIngest_NDJSON_Policy_ColumnDenied_PerLine(t *testing.T) { assert.Equal(t, 1, resp.Failed) require.Len(t, resp.Results, 2) assert.True(t, resultAt(t, resp, 1).Ok) - assert.Contains(t, resultAt(t, resp, 2).Error, "not allowed for insert") + // CONTRACT CHANGE (AUDIT D1): the denied column is ClickHouse's code 117. + assert.Contains(t, resultAt(t, resp, 2).Error, "button") + assert.Equal(t, 117, resultAt(t, resp, 2).Code) assert.Len(t, pub.Messages, 1) } func TestIngest_NDJSON_Policy_TableForbidden(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -1071,7 +1305,7 @@ func TestIngest_NDJSON_Policy_TableForbidden(t *testing.T) { func TestIngest_NDJSON_Policy_CheckClause_PerLineAndAutoInject(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) orgTemplate := "{{ jwt.org_id }}" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ @@ -1111,7 +1345,7 @@ func TestIngest_NDJSON_Policy_CheckClause_PerLineAndAutoInject(t *testing.T) { func TestIngest_NDJSON_ContentTypeWithCharset(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ndjsonRequest(t, "clicks", jsonLine(t, map[string]any{"page": "/a"})) req.Header.Set("Content-Type", "application/x-ndjson; charset=utf-8") @@ -1128,7 +1362,7 @@ func TestIngest_NDJSON_ContentTypeWithCharset(t *testing.T) { func TestIngest_NDJSON_ErrorsTruncated(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) const total = maxReportedResults + 50 lines := make([]string, total) @@ -1159,7 +1393,7 @@ func TestIngest_NDJSON_ErrorsTruncated(t *testing.T) { // spell the whole list out, and always need editing: api.md's body/Content-Type // table, architecture.md's "the four NDJSON spellings" count, and the ingest // entry in CHANGELOG.md. -const wantAcceptedTypes = "application/json, application/x-ndjson, application/ndjson, application/jsonl, application/jsonlines" +const wantAcceptedTypes = "application/json, application/x-ndjson, application/ndjson, application/jsonl, application/jsonlines, text/csv, text/csv; header=present, text/csv; header=absent, text/tab-separated-values, text/tab-separated-values; header=present, text/tab-separated-values; header=absent" // TestAcceptedTypesAreAllResolvable pins that the advertised list never grows // beyond what the resolver accepts — an entry added to acceptedContentTypes but @@ -1237,8 +1471,35 @@ func TestIngestFormat(t *testing.T) { // Tracked in #563. {ct: `application/json; profile="a,b"; charset`, wantErr: true}, {ct: "APPLICATION/JSON", want: FormatJSON}, + {ct: "text/csv", want: FormatCSV}, + {ct: "text/csv; charset=utf-8", want: FormatCSV}, + {ct: "text/tab-separated-values", want: FormatTSV}, + // RFC 4180 §3's header parameter is the one parameter that decides a + // format, and only for the CSV/TSV pair: present, absent and no parameter + // are three different readings. The value is matched case-insensitively. + {ct: "text/csv; header=present", want: FormatCSVWithNames}, + {ct: "text/csv; charset=utf-8; header=present", want: FormatCSVWithNames}, + {ct: "text/csv; header=PRESENT", want: FormatCSVWithNames}, + {ct: "text/csv; header=absent", want: FormatCSVPositional}, + {ct: "text/csv; charset=utf-8; header=ABSENT", want: FormatCSVPositional}, + {ct: "text/tab-separated-values; header=present", want: FormatTSVWithNames}, + {ct: "text/tab-separated-values; header=absent", want: FormatTSVPositional}, + {ct: "application/json; header=present", want: FormatJSON}, + // A value that is neither is refused rather than guessed at, and so is a + // line whose parameters did not parse when it mentions a header: reading + // it as absent would ingest a declared header line as data. + {ct: "text/csv; header=yes", wantErr: true}, + {ct: "text/csv; header=", wantErr: true}, + {ct: "text/csv; charset; header=present", wantErr: true}, + {ct: "text/csv; header=present; header=absent", wantErr: true}, + {ct: "text/csv; charset", want: FormatCSV}, {ct: "text/plain", wantErr: true}, - {ct: "text/csv", wantErr: true}, + // Near misses for the positional pair, for the same reason as the JSON + // ones below: an exact-match lookup rewritten as a prefix test would + // start ingesting these with the suite green. + {ct: "text/csv2", wantErr: true}, + {ct: "text/tab-separated-value", wantErr: true}, + {ct: "text/tsv", wantErr: true}, {ct: "", wantErr: true}, {ct: " ", wantErr: true}, {ct: "???not-a-media-type", wantErr: true}, @@ -1305,14 +1566,14 @@ func TestIngest_UndeclaredOrUnsupportedContentType_415(t *testing.T) { }{ {"no content-type", "", "no Content-Type: "}, {"text/plain", "text/plain", `Content-Type "text/plain": `}, - {"text/csv", "text/csv", `Content-Type "text/csv": `}, + {"application/xml", "application/xml", `Content-Type "application/xml": `}, {"malformed media type", "???not-a-media-type", `Content-Type "???not-a-media-type": `}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, rawIngestRequest(t, "clicks", tt.ct, `{"page":"/a"}`)) @@ -1347,6 +1608,18 @@ func jsonErrorMessage(t *testing.T, w *httptest.ResponseRecorder) string { return body.Error } +// errorAndCode returns the decoded "error" message and ClickHouse's "code", +// which is absent (0) unless the server's own parser is what refused. +func errorAndCode(t *testing.T, w *httptest.ResponseRecorder) (string, int) { + t.Helper() + var body struct { + Error string `json:"error"` + Code int `json:"code"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body)) + return body.Error, body.Code +} + // TestIngest_ContentTypeRefusalBeatsEmptyBody: the PR's headline ordering claim // — "checked before the body is parsed" — is what lets a caller trust that a 415 // describes their header and not their payload. Nothing pinned it: every 415 case @@ -1362,7 +1635,7 @@ func TestIngest_ContentTypeRefusalBeatsEmptyBody(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, rawIngestRequest(t, "clicks", ct, "")) @@ -1378,18 +1651,41 @@ func TestIngest_ContentTypeRefusalBeatsEmptyBody(t *testing.T) { // TestIngest_DeclaredNDJSON_ArrayBodyIsNotReframed: the header is authoritative. // A JSON array sent as NDJSON is read as NDJSON — one line, not a JSON object — // so it fails as a per-record error instead of silently being re-read as a batch. +// TestIngest_DeclaredNDJSON_ArrayBodyIsNotReframed: the declared format is still +// authoritative — a declared-NDJSON body is never re-read as the JSON family, so +// the depth-1 comma rewrite (which is what makes a compact array salvageable per +// record) does not run on it. +// +// CONTRACT CHANGE: it used to be one unparseable NDJSON line, reported as a +// single per-record failure. ClickHouse's own JSONEachRow reader takes the +// surrounding brackets in its stride, so both objects now ingest and the batch +// reports two records. Nothing is silently dropped either way; what changed is +// that the mis-declaration now costs nothing instead of the whole body. func TestIngest_DeclaredNDJSON_ArrayBodyIsNotReframed(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, rawIngestRequest(t, "clicks", "application/x-ndjson", `[{"page":"/a"},{"page":"/b"}]`)) assert.Equal(t, http.StatusOK, w.Code) resp := decodeBatchResult(t, w) - assert.Equal(t, 1, resp.Total, "the array is one NDJSON line, not two records") - assert.Equal(t, 1, resp.Failed) - assert.Empty(t, pub.Messages) + assert.Equal(t, 2, resp.Total, "ClickHouse's reader frames the array's elements") + assert.Equal(t, 2, resp.Succeeded) + assert.Len(t, pub.Messages, 2) + + // The rewrite really is off for this declaration: a compact array with one + // bad record loses the whole batch here, which is exactly the cliff the + // rewrite exists to remove for a declared-JSON body (see + // TestIngest_JSONArray_CompactWithOneBadRecord). + pub2 := &testutil.MockPublisher{} + h2 := newTestIngestHandler(t, testRegistry(t), pub2, testutil.NopLogger()) + w = httptest.NewRecorder() + h2.Handle(w, rawIngestRequest(t, "clicks", "application/x-ndjson", + `[{"page":"/a"},{"page":"/b","nope":1},{"page":"/c"}]`)) + require.Equal(t, http.StatusOK, w.Code) + assert.Zero(t, decodeBatchResult(t, w).Succeeded, "the compact array is all-or-nothing without the rewrite") + assert.Empty(t, pub2.Messages) } // ── Multi-format ingest (JSON array, arity sniffing, body cap) ───────────── @@ -1428,7 +1724,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("disagreeing declarations are refused", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/json", ndjson) req.Header.Add("Content-Type", "application/x-ndjson") @@ -1453,7 +1749,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("a supported and an unsupported declaration are refused", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/json", ndjson) req.Header.Add("Content-Type", "text/csv") @@ -1472,7 +1768,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("different spellings of the same format are accepted", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/x-ndjson", ndjson) req.Header.Add("Content-Type", "application/ndjson; charset=utf-8") @@ -1502,7 +1798,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, rawIngestRequest(t, "clicks", ct, ndjson)) @@ -1547,7 +1843,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() wJ := httptest.NewRecorder() - NewIngestHandler(testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()). + newTestIngestHandler(t, testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()). Handle(wJ, rawIngestRequest(t, "clicks", tc.joined, `{"page":"/a"}`)) assert.Equal(t, tc.wJoined, wJ.Code, "joined") @@ -1556,7 +1852,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { req.Header.Add("Content-Type", v) } wR := httptest.NewRecorder() - NewIngestHandler(testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()). + newTestIngestHandler(t, testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()). Handle(wR, req) assert.Equal(t, tc.wRepeat, wR.Code, "repeated") }) @@ -1566,7 +1862,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("a quoted comma does not split a declaration", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, rawIngestRequest(t, "clicks", `application/json; profile="a,b"`, `{"page":"/a"}`)) @@ -1577,7 +1873,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("a third line that disagrees is refused", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/json", ndjson) req.Header.Add("Content-Type", "application/json") req.Header.Add("Content-Type", "application/x-ndjson") @@ -1599,7 +1895,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Parallel() for _, first := range []bool{false, true} { pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "", ndjson) if first { req.Header.Add("Content-Type", empty) @@ -1625,8 +1921,8 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("two unsupported lines name both", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) - req := rawIngestRequest(t, "clicks", "text/csv", ndjson) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + req := rawIngestRequest(t, "clicks", "application/xml", ndjson) req.Header.Add("Content-Type", "text/plain") w := httptest.NewRecorder() @@ -1635,7 +1931,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { assert.Equal(t, http.StatusUnsupportedMediaType, w.Code) testutil.AssertJSONErrorResponse(t, w) msg := jsonErrorMessage(t, w) - assert.Contains(t, msg, `"text/csv"`) + assert.Contains(t, msg, `"application/xml"`) assert.Contains(t, msg, `"text/plain"`, "a declaration the caller sent must not vanish from the message") assert.NotContains(t, msg, "conflicting", "agreeing-but-unsupported is not a conflict") assert.Empty(t, pub.Messages) @@ -1644,7 +1940,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { t.Run("an identical declaration repeated is not ambiguous", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/x-ndjson", ndjson) req.Header.Add("Content-Type", "application/x-ndjson") @@ -1659,7 +1955,7 @@ func TestIngest_DuplicateContentTypeHeaders(t *testing.T) { func TestIngest_JSONArray_AllValid(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // A JSON array declared as application/json is read as a batch — the body's // first byte picks arity within the family ingestRequest declares. @@ -1684,7 +1980,7 @@ func TestIngest_JSONArray_AllValid(t *testing.T) { func TestIngest_JSONArray_SingleElement(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // A one-element array is still a batch (returns the results envelope, not // the single-object {"ok":true}). @@ -1704,7 +2000,7 @@ func TestIngest_JSONArray_SingleElement(t *testing.T) { func TestIngest_JSONArray_PartialValidationFailure(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "clicks", []map[string]any{ {"page": "/a"}, @@ -1729,7 +2025,7 @@ func TestIngest_JSONArray_PartialValidationFailure(t *testing.T) { func TestIngest_JSONArray_ScalarElements(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // Non-object elements (number, string, nested array) are wrong-typed: the // decoder stays in sync, so each is a per-record error and the objects @@ -1760,11 +2056,12 @@ func TestIngest_JSONArray_ScalarElements(t *testing.T) { func TestIngest_JSONArray_SyntaxError_Fatal(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // A structural syntax error desyncs the decoder — the whole request fails - // (400), unlike a per-element type error. The leading good element may have - // already published (at-least-once on retry). + // (400), unlike a per-element type error. Records before it are published + // only if a chunk filled first (see chunkRecords); one leading record has + // not been validated yet when the abort comes, so nothing ships. req := rawIngestRequest(t, "clicks", "application/json", `[{"page":"/a"}, {bad]`) w := httptest.NewRecorder() h.Handle(w, req) @@ -1772,7 +2069,7 @@ func TestIngest_JSONArray_SyntaxError_Fatal(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) assert.Contains(t, w.Body.String(), "invalid json") testutil.AssertJSONErrorResponse(t, w) - assert.Len(t, pub.Messages, 1) // the leading record published before the abort + assert.Empty(t, pub.Messages, "the leading record was still staged when the body failed") } func TestIngest_JSONArray_Truncated_Fatal(t *testing.T) { @@ -1794,7 +2091,7 @@ func TestIngest_JSONArray_Truncated_Fatal(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/json", tt.body) w := httptest.NewRecorder() @@ -1810,7 +2107,7 @@ func TestIngest_JSONArray_Truncated_Fatal(t *testing.T) { func TestIngest_JSONArray_Empty(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // An explicit empty array is a valid, record-less batch → 200 with no rows. req := rawIngestRequest(t, "clicks", "application/json", `[]`) @@ -1827,7 +2124,7 @@ func TestIngest_JSONArray_Empty(t *testing.T) { func TestIngest_SingleObject_PrettyPrinted(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // A multi-line (pretty-printed) single object must not be mistaken for // NDJSON — it's one record on the single-object path. @@ -1845,7 +2142,7 @@ func TestIngest_SingleObject_PrettyPrinted(t *testing.T) { func TestIngest_DeclaredJSON_ConcatenatedObjects_FirstOnly(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) // Two concatenated objects declared as application/json take the // single-object path and ingest only the first (matching the historical @@ -1878,7 +2175,7 @@ func TestIngest_LeadingWhitespace_Sniff(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "application/json", tt.body) w := httptest.NewRecorder() @@ -1916,7 +2213,7 @@ func TestIngest_EmptyBody(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", tt.contentType, tt.body) w := httptest.NewRecorder() @@ -1947,7 +2244,7 @@ func TestIngest_BodyReadFailure_400(t *testing.T) { t.Run(ct, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ingest?table=clicks", iotest.ErrReader(errors.New("connection reset by peer"))) @@ -1999,7 +2296,7 @@ func TestIngest_BodyCap_413(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.maxRequestBytes = tt.cap // below the body req := rawIngestRequest(t, "clicks", tt.ct, tt.body) @@ -2033,11 +2330,11 @@ func TestIngest_ContentTypeResolvesBeforeTheBodyIsRead(t *testing.T) { t.Run("a body that cannot be read at all", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := httptest.NewRequestWithContext(context.Background(), http.MethodPost, "/v1/ingest?table=clicks", iotest.ErrReader(errors.New("connection reset by peer"))) - req.Header.Set("Content-Type", "text/csv") + req.Header.Set("Content-Type", "application/xml") w := httptest.NewRecorder() h.Handle(w, req) @@ -2051,10 +2348,10 @@ func TestIngest_ContentTypeResolvesBeforeTheBodyIsRead(t *testing.T) { t.Run("a body over the cap", func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.maxRequestBytes = 50 // below the body - req := rawIngestRequest(t, "clicks", "text/csv", + req := rawIngestRequest(t, "clicks", "application/xml", `{"page":"/`+strings.Repeat("a", 200)+`"}`) w := httptest.NewRecorder() h.Handle(w, req) @@ -2074,8 +2371,8 @@ func tsRegistry(t testing.TB) *discovery.SchemaRegistry { Name: "events", Columns: []discovery.Column{ {Name: "name", Type: "String"}, - {Name: "ts", Type: "DateTime('UTC')", HasDefault: true}, - {Name: "ts_ms", Type: "DateTime64(3, 'UTC')", HasDefault: true}, + {Name: "ts", Type: "DateTime('UTC')", HasDefault: true, DefaultExpression: "now()"}, + {Name: "ts_ms", Type: "DateTime64(3, 'UTC')", HasDefault: true, DefaultExpression: "now64(3)"}, }, }, }) @@ -2109,13 +2406,17 @@ func publishedRow(t *testing.T, payload []byte) map[string]any { return out } -// TestIngest_TimestampsCanonicalized is the #372 contract: whatever spelling a -// producer uses, the published payload — the one copy SSE subscribers, the -// ClickHouse insert, and the DLQ all consume — carries RFC 3339 UTC. +// TestIngest_TimestampsCanonicalized is the #372 contract, now satisfied by +// construction: whatever spelling a producer uses, the published payload — the +// one copy SSE subscribers, the ClickHouse insert and the DLQ all consume — +// carries the instant as ClickHouse's OWN writer renders it. That is +// "2026-06-21 04:00:00" in the column's zone, not RFC 3339 with a Z: the row is +// the server's rendering of the stored value, so the SSE frame and a +// /v1/query row cannot disagree. func TestIngest_TimestampsCanonicalized(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(tsRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, tsRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "events", map[string]any{ "name": "e", @@ -2127,24 +2428,24 @@ func TestIngest_TimestampsCanonicalized(t *testing.T) { require.Equal(t, http.StatusOK, w.Code) data := publishedData(t, pub) - assert.Equal(t, "2026-06-21T04:00:00Z", data["ts"]) - assert.Equal(t, "2026-06-21T04:00:00.5Z", data["ts_ms"]) + assert.Equal(t, "2026-06-21 04:00:00", data["ts"]) + // Sub-second digits are the column's precision, zeros and all — the server + // does not trim them the way the old canonicalizer did. + assert.Equal(t, "2026-06-21 04:00:00.500", data["ts_ms"]) assert.Equal(t, "e", data["name"], "non-timestamp columns untouched") } -// TestIngest_AutoInjectedLiteralTimestampCanonicalized pins the LiteralValue -// unwrap on the auto-inject path: a placeholder-free _eq check value is typed -// policy.LiteralValue for the comparison, but must enter the published data as -// a plain string — timestamp canonicalization switches on `case string`, so a -// leaked named type would silently skip the rewrite and publish the -// non-canonical spelling (the #372 fail-open that #381's row filter relies -// on). json.Marshal renders both identically, so only this assertion on the -// canonical form catches the leak. +// TestIngest_AutoInjectedLiteralTimestampCanonicalized: a value the policy +// auto-injects is not a shortcut past the parser. The check literal is written +// in a spelling ClickHouse does not store it in, so the published row proves +// the injected value went through the same parse every producer-supplied value +// does — an injected value published in its policy spelling would mean the +// server and the stream disagree about what the row holds. func TestIngest_AutoInjectedLiteralTimestampCanonicalized(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(tsRegistry(t), pub, testutil.NopLogger()) - staticTS := "2026-06-21 04:00:00" + h := newTestIngestHandler(t, tsRegistry(t), pub, testutil.NopLogger()) + staticTS := "2026-06-21T04:00:00Z" h.PolicySource = policy.Static(&policy.Policy{ Tables: map[string]policy.TablePolicy{ "events": { @@ -2166,31 +2467,38 @@ func TestIngest_AutoInjectedLiteralTimestampCanonicalized(t *testing.T) { h.Handle(w, req) require.Equal(t, http.StatusOK, w.Code) - assert.Equal(t, "2026-06-21T04:00:00Z", publishedData(t, pub)["ts"], - "auto-injected literal must be canonicalized, not published in its policy spelling") + assert.Equal(t, "2026-06-21 04:00:00", publishedData(t, pub)["ts"], + "auto-injected literal must be parsed, not published in its policy spelling") } -// TestIngest_TimestampGarbage_PassesThrough: fail-open — an unparseable value -// publishes verbatim; ClickHouse's own parser decides insertability (#372/#381). -func TestIngest_TimestampGarbage_PassesThrough(t *testing.T) { +// TestIngest_TimestampGarbage_Rejected: the old fail-open is gone. An +// unparseable timestamp used to publish verbatim and fail later at the worker's +// INSERT, where the caller could not see it; ClickHouse's own parser now +// answers at the edge, with its own code, and nothing is published. +func TestIngest_TimestampGarbage_Rejected(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(tsRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, tsRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "events", map[string]any{"name": "e", "ts": "banana"}) w := httptest.NewRecorder() h.Handle(w, req) - require.Equal(t, http.StatusOK, w.Code) - assert.Equal(t, "banana", publishedData(t, pub)["ts"], "unparseable value published verbatim") + require.Equal(t, http.StatusBadRequest, w.Code) + msg, code := errorAndCode(t, w) + assert.Contains(t, msg, "Cannot read DateTime") + assert.Equal(t, 41, code, "ClickHouse's own code rides with the message") + assert.Empty(t, pub.Messages) } -// TestIngest_Batch_MixedTimestampSpellings: parseable spellings canonicalize, -// the unparseable one passes through — no record fails on its timestamp. +// TestIngest_Batch_MixedTimestampSpellings: every parseable spelling lands on +// the same instant in the server's own rendering, and the unparseable one fails +// ALONE — one bad timestamp in a batch must not cost its siblings, which is +// what the compile profile's allow_errors_ratio buys. func TestIngest_Batch_MixedTimestampSpellings(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(tsRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, tsRegistry(t), pub, testutil.NopLogger()) req := ingestRequest(t, "events", []map[string]any{ {"name": "a", "ts": "2026-06-21T04:00:00Z"}, @@ -2201,25 +2509,21 @@ func TestIngest_Batch_MixedTimestampSpellings(t *testing.T) { h.Handle(w, req) require.Equal(t, http.StatusOK, w.Code) - var result struct { - Total int `json:"total"` - Succeeded int `json:"succeeded"` - Failed int `json:"failed"` - } - require.NoError(t, json.Unmarshal(w.Body.Bytes(), &result)) + result := decodeBatchResult(t, w) + require.Len(t, result.Results, 3) + assert.Equal(t, 41, result.Results[1].Code, "the failing record carries ClickHouse's code") assert.Equal(t, 3, result.Total) - assert.Equal(t, 3, result.Succeeded) - assert.Equal(t, 0, result.Failed) - require.Len(t, pub.Messages, 3, "every record published") + assert.Equal(t, 2, result.Succeeded) + assert.Equal(t, 1, result.Failed) + require.Len(t, pub.Messages, 2, "only the records ClickHouse accepted") var spellings []string for _, msg := range pub.Messages { spellings = append(spellings, publishedRow(t, msg.Data)["ts"].(string)) } assert.Equal(t, []string{ - "2026-06-21T04:00:00Z", // already canonical - "banana", // unparseable — passed through verbatim - "2026-06-21T04:00:00Z", // Unix seconds — canonicalized + "2026-06-21 04:00:00", // RFC 3339 in + "2026-06-21 04:00:00", // Unix seconds in — same instant, same rendering }, spellings) } @@ -2242,7 +2546,7 @@ func TestIngest_Dedup_DisabledBySettings(t *testing.T) { pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() dedup.Err = errors.New("must not be called while disabled") - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return false, "event_id", true } @@ -2262,7 +2566,7 @@ func TestIngest_Dedup_DisabledMidReload(t *testing.T) { pub := &testutil.MockPublisher{} dedup := testutil.NewMockDeduplicator() dedup.Err = dedupe.ErrDisabled - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.Dedup = dedup h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", true } @@ -2282,20 +2586,21 @@ func TestIngest_Dedup_DisabledMidReload(t *testing.T) { // // Second, the reachability claim at the CheckClauses call site. The column loop // above it iterates the RECORD's columns, so it runs zero times for `{}` — and -// discovery.Validate accepts `{}` here because every column is nullable or -// defaulted. I previously asserted this path was unreachable, having tested only -// against a schema with a required column; it is not. -func TestProcessRecord_UnresolvedInsertSideAborts(t *testing.T) { +// nothing now rejects an empty record before that point, because ClickHouse +// reads `{}` as every column taking its default. I previously asserted this +// path was unreachable, having tested only against a schema with a required +// column; it is not, and under the type layer it is reachable for EVERY table. +func TestCheckRecord_UnresolvedInsertSideAborts(t *testing.T) { t.Parallel() schema := &discovery.TableSchema{ Name: "loose", Columns: []discovery.Column{ {Name: "a", Type: "Nullable(String)", IsNullable: true}, - {Name: "b", Type: "String", HasDefault: true}, + {Name: "b", Type: "String", HasDefault: true, DefaultExpression: "''"}, }, } reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{schema}) - h := NewIngestHandler(reg, &testutil.MockPublisher{}, testutil.NopLogger()) + h := newTestIngestHandler(t, reg, &testutil.MockPublisher{}, testutil.NopLogger()) // A grant resolved for SELECT, reaching the insert path. selectResolved := policy.Evaluate(&policy.Policy{ @@ -2305,15 +2610,11 @@ func TestProcessRecord_UnresolvedInsertSideAborts(t *testing.T) { }, "viewer", "loose", "select", nil) require.True(t, selectResolved.Allowed) - // An empty record really does clear validation and the column loop. - require.NoError(t, discovery.Validate(schema, map[string]any{}), - "all-nullable/defaulted columns accept an empty record — this is what makes the read reachable") + _, preds, cols, abort := h.insertShape( + context.Background(), "loose", "viewer", schema, selectResolved, nil) - dup, reject, abort := h.processRecord( - context.Background(), "loose", "", schema, selectResolved, "viewer", map[string]any{}, time.Now(), nil) - - assert.False(t, dup) - assert.Nil(t, reject, "a request-scoped condition must not be reported per record") + assert.Nil(t, preds, "an unresolved insert side must produce no check predicates") + assert.Nil(t, cols) require.NotNil(t, abort, "an unresolved insert side must abort the request") assert.Equal(t, http.StatusForbidden, abort.Status) assert.Empty(t, abort.RetryAfter, "not a transient condition — retrying cannot help") @@ -2353,7 +2654,7 @@ func TestIngest_ContentTypeEchoIsBounded(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) w := httptest.NewRecorder() h.Handle(w, build(t)) @@ -2381,7 +2682,7 @@ func TestIngest_ContentTypeEchoIsBounded(t *testing.T) { req.Header.Add("Content-Type", fmt.Sprintf("application/%04d", i)+strings.Repeat("\xff", 112)) } w := httptest.NewRecorder() - NewIngestHandler(testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()).Handle(w, req) + newTestIngestHandler(t, testRegistry(t), &testutil.MockPublisher{}, testutil.NopLogger()).Handle(w, req) require.Equal(t, http.StatusUnsupportedMediaType, w.Code) return w.Body.Len() } @@ -2405,7 +2706,7 @@ func TestIngest_ContentTypeEchoIsBounded(t *testing.T) { func TestIngest_ConflictMessageNamesTheDisagreement(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "", "{\"page\":\"/a\"}\n{\"page\":\"/b\"}") for range 4 { req.Header.Add("Content-Type", "application/json") @@ -2435,7 +2736,7 @@ func TestIngest_ConflictMessageNamesTheDisagreement(t *testing.T) { func TestIngest_ConflictMessageNamesADifferentSpelling(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) req := rawIngestRequest(t, "clicks", "", `{"page":"/a"}`) for _, ct := range []string{ "application/json", @@ -2466,7 +2767,7 @@ func TestIngest_ConflictMessageNamesADifferentSpelling(t *testing.T) { func TestIngest_ConflictLogNamesTheDisagreement(t *testing.T) { t.Parallel() var buf bytes.Buffer - h := NewIngestHandler(testRegistry(t), &testutil.MockPublisher{}, + h := newTestIngestHandler(t, testRegistry(t), &testutil.MockPublisher{}, slog.New(slog.NewJSONHandler(&buf, nil))) req := rawIngestRequest(t, "clicks", "", `{"page":"/a"}`) @@ -2495,7 +2796,7 @@ func TestIngest_ConflictLogNamesTheDisagreement(t *testing.T) { func TestIngest_CheckColumnNotInSchema_Rejected(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(checkColumnPolicy(t, "tenant_id", "acme")) w := httptest.NewRecorder() @@ -2528,7 +2829,7 @@ func TestIngest_CheckOnComputedColumn_Rejected(t *testing.T) { }}}, }} pub := &testutil.MockPublisher{} - h := NewIngestHandler(computedRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, computedRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(p) w := httptest.NewRecorder() @@ -2551,7 +2852,7 @@ func TestIngest_CheckOnComputedColumn_Rejected(t *testing.T) { func TestIngest_CheckColumnInSchema_StillInjects(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(checkColumnPolicy(t, "org_id", "org-42")) w := httptest.NewRecorder() @@ -2577,7 +2878,7 @@ func TestIngest_CheckGuardLogsOncePerRequest(t *testing.T) { t.Parallel() var buf bytes.Buffer pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, slog.New(slog.NewJSONHandler(&buf, nil))) + h := newTestIngestHandler(t, testRegistry(t), pub, slog.New(slog.NewJSONHandler(&buf, nil))) h.PolicySource = policy.Static(checkColumnPolicy(t, "tenant_id", "acme")) const n = 25 @@ -2602,14 +2903,17 @@ func TestIngest_CheckGuardLogsOncePerRequest(t *testing.T) { // // Every record fails here, and that is not an artifact of the fixture: the // condition is a property of (table, role, policy), identical for every record -// in the request, so there is no sibling this guard could spare. A record -// SUPPLYING the missing column is rejected too, one step earlier, by schema -// validation — with its own message, which this pins so the two stay -// distinguishable. +// in the request, so there is no sibling this guard could spare. +// +// CONTRACT CHANGE: a record SUPPLYING the missing column used to get the +// guard's message too, because the gateway's key walk ran first. ClickHouse's +// parser now answers first, so that record reports its own code 117 — the same +// ordering change as everywhere else on this path (AUDIT §A.2). Both are still +// failures and nothing publishes. func TestIngest_CheckColumnNotInSchema_BatchRejectsPerRecord(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(checkColumnPolicy(t, "tenant_id", "acme")) req := rawIngestRequest(t, "clicks", "application/json", @@ -2625,9 +2929,13 @@ func TestIngest_CheckColumnNotInSchema_BatchRejectsPerRecord(t *testing.T) { assert.Equal(t, 3, resp.Failed) assert.Zero(t, resp.Succeeded) require.Len(t, resp.Results, 3) - assert.Contains(t, resp.Results[0].Error, "which table", "omitted ⇒ the new guard") - assert.Contains(t, resp.Results[1].Error, "unknown column", "supplied ⇒ schema validation, one step earlier") - assert.Contains(t, resp.Results[2].Error, "which table") + for _, i := range []int{0, 2} { + assert.Contains(t, resp.Results[i].Error, "which table", "record %d", i+1) + assert.Zero(t, resp.Results[i].Code, "a gateway rejection never carries a ClickHouse code") + } + assert.Contains(t, resp.Results[1].Error, "tenant_id") + assert.Equal(t, 117, resp.Results[1].Code, + "the record that SUPPLIES the column is refused by ClickHouse first") assert.Empty(t, pub.Messages) } @@ -2649,10 +2957,10 @@ func computedRegistry(t testing.TB) *discovery.SchemaRegistry { Name: "clicks", Columns: []discovery.Column{ {Name: "page", Type: "String"}, - {Name: "digest", Type: "String", DefaultKind: "MATERIALIZED", HasDefault: true}, - {Name: "country", Type: "String", HasDefault: true}, - {Name: "doubled", Type: "UInt64", DefaultKind: "ALIAS", HasDefault: true}, - {Name: "raw", Type: "String", DefaultKind: "EPHEMERAL", HasDefault: true}, + {Name: "digest", Type: "String", DefaultKind: "MATERIALIZED", HasDefault: true, DefaultExpression: "upper(page)"}, + {Name: "country", Type: "String", HasDefault: true, DefaultExpression: "'US'"}, + {Name: "doubled", Type: "UInt64", DefaultKind: "ALIAS", HasDefault: true, DefaultExpression: "length(page) * 2"}, + {Name: "raw", Type: "String", DefaultKind: "EPHEMERAL", HasDefault: true, DefaultExpression: "''"}, }, }, }) @@ -2668,7 +2976,7 @@ func computedRegistry(t testing.TB) *discovery.SchemaRegistry { func TestIngest_CheckOnEphemeralColumn_Rejected(t *testing.T) { t.Parallel() pub := &testutil.MockPublisher{} - h := NewIngestHandler(computedRegistry(t), pub, testutil.NopLogger()) + h := newTestIngestHandler(t, computedRegistry(t), pub, testutil.NopLogger()) h.PolicySource = policy.Static(checkColumnPolicy(t, "raw", "anything")) w := httptest.NewRecorder() diff --git a/internal/api/ingest_unavailable_test.go b/internal/api/ingest_unavailable_test.go new file mode 100644 index 00000000..f895bcea --- /dev/null +++ b/internal/api/ingest_unavailable_test.go @@ -0,0 +1,144 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/testutil" + "github.com/Wave-RF/WaveHouse/internal/typelayer" +) + +// This file replaces ingest_seams_test.go. That file covered the InsertChecker +// seam — the one per-record decision point the type layer did not take over. It +// has taken it over: a check clause is now a compiled chtypes filter over the +// rows ClickHouse already accepted, so there is no Go-side comparison left to +// swap out. What survives is the group of tests about the type layer being +// absent or unable to answer, which is the fail-closed property those seam tests +// were really pinning. + +// TestIngest_UnwiredTypeLayer_Refuses: a handler with no type layer must refuse +// the request, not wave it through. Nothing else on the ingest path looks at a +// value, so nil Types reading as "accept anything" would turn a wiring mistake +// into an open door — the same fail-closed direction the row evaluator takes. +func TestIngest_UnwiredTypeLayer_Refuses(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + h := NewIngestHandler(testRegistry(t), pub, testutil.NopLogger()) + require.Nil(t, h.Types) + + w := httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/home"})) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + assert.Equal(t, "30", w.Header().Get("Retry-After"), "an outage is retryable; the body is unchanged") + testutil.AssertJSONErrorResponse(t, w) + assert.Empty(t, pub.Messages) +} + +// TestIngest_UndiscoveredTable_Unavailable: a table the registry knows but the +// type layer has no compiled schema for is a 503 naming the cause, never a 400. +// The request is fine; the server cannot judge it. +func TestIngest_UndiscoveredTable_Unavailable(t *testing.T) { + t.Parallel() + reg := testRegistry(t) + pub := &testutil.MockPublisher{} + h := NewIngestHandler(reg, pub, testutil.NopLogger()) + // An engine bound to NO tables: every lookup is Unavailable. + engineMu.Lock() + h.Types = typelayer.TestEngine(t) + engineMu.Unlock() + + w := httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/home"})) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + assert.Contains(t, jsonErrorMessage(t, w), "not discovered") + assert.Empty(t, pub.Messages) +} + +// TestIngest_ParseErrorsPrecedeCheckErrors inverts the ordering this file's +// predecessor pinned, and is a DOCUMENTED contract change (AUDIT §A.2). Check +// clauses are now a filter over rows ClickHouse has already accepted, so a +// record that fails both reports the PARSE error, not the check. No enforcement +// is lost — nothing is published either way — but a client can no longer infer +// from a 403 that the rest of its payload was well formed. +func TestIngest_ParseErrorsPrecedeCheckErrors(t *testing.T) { + t.Parallel() + required := "org-allowed" + pub := &testutil.MockPublisher{} + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + h.PolicySource = policy.Static(&policy.Policy{Tables: map[string]policy.TablePolicy{ + "clicks": {"viewer": {Insert: &policy.InsertPermissions{ + Check: map[string]policy.Filter{"org_id": {Eq: &required}}, + }}}, + }}) + + w := httptest.NewRecorder() + h.Handle(w, viewerIngestRequest(t, "clicks", map[string]any{ + "page": "/a", + "org_id": "wrong", // fails the check clause + "count": "not-a-number", // and ClickHouse refuses it first + })) + + require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + _, code := errorAndCode(t, w) + assert.Equal(t, 27, code, "ClickHouse's own refusal, with its own code") + assert.Empty(t, pub.Messages) + + // The same record, parseable: NOW the check clause is what refuses it. + w = httptest.NewRecorder() + h.Handle(w, viewerIngestRequest(t, "clicks", map[string]any{ + "page": "/a", + "org_id": "wrong", + "count": 1, + })) + assert.Equal(t, http.StatusForbidden, w.Code, "body=%s", w.Body.String()) + assert.Contains(t, jsonErrorMessage(t, w), `check failed for column "org_id"`) + assert.Empty(t, pub.Messages) +} + +// TestIngest_DedupeMarksOnlyPublishedRecords: dedupe runs AFTER validation, so a +// record ClickHouse refuses does not burn its idempotency key — the caller can +// fix the record and resend it under the same id. Marking before validation +// would swallow the corrected retry as a duplicate. +func TestIngest_DedupeMarksOnlyPublishedRecords(t *testing.T) { + t.Parallel() + pub := &testutil.MockPublisher{} + dedup := testutil.NewMockDeduplicator() + h := newTestIngestHandler(t, testRegistry(t), pub, testutil.NopLogger()) + h.Dedup = dedup + h.DedupeSettings = func(string) (bool, string, bool) { return true, "event_id", false } + + // A record ClickHouse refuses (count is not a number), carrying an id. + w := httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/a", "event_id": "evt-1", "count": "nope"})) + require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + assert.Empty(t, pub.Messages) + + // The same id, corrected: it must publish, not report a duplicate. + w = httptest.NewRecorder() + h.Handle(w, ingestRequest(t, "clicks", map[string]any{"page": "/a", "event_id": "evt-1", "count": 1})) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Contains(t, w.Body.String(), `"ok":true`) + assert.Len(t, pub.Messages, 1) + + // And only NOW is the id spent. + seen, err := dedup.CheckAndMark(context.Background(), "evt-1") + require.NoError(t, err) + assert.True(t, seen, "the published record's id is marked") +} + +// viewerIngestRequest is ingestRequest with the "viewer" role in context, for +// the policy-gated tests. +func viewerIngestRequest(t *testing.T, table string, body map[string]any) *http.Request { + t.Helper() + req := ingestRequest(t, table, body) + return req.WithContext(auth.WithRole(req.Context(), "viewer")) +} diff --git a/internal/api/pipes.go b/internal/api/pipes.go index 2de2da76..e7ab1ef8 100644 --- a/internal/api/pipes.go +++ b/internal/api/pipes.go @@ -7,9 +7,9 @@ import ( "net/http" "time" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/Wave-RF/WaveHouse/internal/auth" "github.com/Wave-RF/WaveHouse/internal/cache" + "github.com/Wave-RF/WaveHouse/internal/chconn" "github.com/Wave-RF/WaveHouse/internal/pipes" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/go-chi/chi/v5" @@ -22,7 +22,7 @@ import ( type PipesHandler struct { Source pipes.Source PolicySource policy.Source // resolves empty role to default_role; may be nil - CHConn driver.Conn + CH *chReader Cache cache.Cache sf singleflight.Group // queryTimeout bounds each pipe execution, read per request @@ -39,8 +39,8 @@ type PipesHandler struct { maxRequestBytes int64 } -func NewPipesHandler(source pipes.Source, policySource policy.Source, conn driver.Conn, c cache.Cache, queryTimeout func() time.Duration, logger *slog.Logger) *PipesHandler { - return &PipesHandler{Source: source, PolicySource: policySource, CHConn: conn, Cache: c, queryTimeout: queryTimeout, logger: logger} +func NewPipesHandler(source pipes.Source, policySource policy.Source, target func() chconn.Target, c cache.Cache, queryTimeout func() time.Duration, logger *slog.Logger) *PipesHandler { + return &PipesHandler{Source: source, PolicySource: policySource, CH: newCHReader(target), Cache: c, queryTimeout: queryTimeout, logger: logger} } // List returns all named queries (admin endpoint). @@ -127,7 +127,11 @@ func (h *PipesHandler) Execute(w http.ResponseWriter, r *http.Request) { } } - sql, params, err := pipes.BindParams(q, supplied) + // BindParams inlines every value as an escaped SQL literal — a pipe's + // placeholders can sit anywhere in the statement, including positions + // (LIMIT, an identifier) where a bound parameter is not legal — so the + // rendered SQL carries no placeholders and nothing is bound here. + sql, err := pipes.BindParams(q, supplied) if err != nil { writeJSONError(w, http.StatusBadRequest, err.Error()) return @@ -138,7 +142,7 @@ func (h *PipesHandler) Execute(w http.ResponseWriter, r *http.Request) { // by sha alone (TTL-only) and the ingest worker cannot version-invalidate it. // TODO: once pipes expose their tables/scopes, pass them as deps here so writes // invalidate cached pipe results. - cacheKey := queryCacheKey(sql, params) + cacheKey := queryCacheKey(sql, nil) if h.Cache != nil { if data, _, err := h.Cache.Get(r.Context(), cacheKey, nil); err == nil && data != nil { w.Header().Set("Content-Type", "application/json") @@ -155,19 +159,16 @@ func (h *PipesHandler) Execute(w http.ResponseWriter, r *http.Request) { start := time.Now() - rows, err := executeCHQuery(queryCtx, h.CHConn, sql, params) + // ClickHouse's own JSON rendering of the rows, stored and served + // verbatim. A pipe carries no per-role resource caps (allowed_roles is + // its whole policy), so no settings are sent. + data, err := h.CH.query(queryCtx, sql, nil, nil) queryDuration := time.Since(start) if err != nil { // TODO: depending on the error, we may actually want to cache it return nil, err } - data, err := json.Marshal(rows) - if err != nil { - // TODO: eventually we want CSV support etc - return nil, err - } - ttl := cache.QueryTimeToTTL(queryDuration) if h.Cache != nil { diff --git a/internal/api/pipes_test.go b/internal/api/pipes_test.go index 35c63cd5..50652ade 100644 --- a/internal/api/pipes_test.go +++ b/internal/api/pipes_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "io" "net/http" "net/http/httptest" "strings" @@ -11,6 +12,7 @@ import ( "time" "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/chconn" "github.com/Wave-RF/WaveHouse/internal/pipes" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/testutil" @@ -38,9 +40,13 @@ func pipesRequest(t *testing.T, method, path, name string, body any) *http.Reque } // noTimeout is the pipe-execution deadline source for handler tests that -// never reach ClickHouse. +// never reach ClickHouse. It is an ALREADY-EXPIRED deadline, so a test whose +// pipe does reach ClickHouse must use shortTimeout instead. func noTimeout() time.Duration { return 0 } +// shortTimeout bounds the tests that execute against a stub ClickHouse. +func shortTimeout() time.Duration { return 5 * time.Second } + func TestPipesHandler_List(t *testing.T) { t.Parallel() store := pipes.Static( @@ -410,3 +416,70 @@ func TestPipesHandler_Execute_NoAllowedRoles_AdminAllowed(t *testing.T) { "admin bypasses the allowlist on a pipe with no allowed_roles") assert.NotEqual(t, http.StatusNotFound, w.Code) } + +// TestPipesHandler_Execute_ServesClickHouseBytes pins the pipe execution path +// now that it runs over ClickHouse's HTTP interface: the bound SQL is the POST +// body, the rows come back as ClickHouse rendered them, and the response is +// the JSON array with X-Cache: MISS. A pipe inlines its parameters (they can +// sit in a LIMIT, where a bound value is not legal SQL), so nothing is bound. +func TestPipesHandler_Execute_ServesClickHouseBytes(t *testing.T) { + t.Parallel() + var gotSQL string + var gotParams int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotSQL = string(body) + for k := range r.URL.Query() { + if strings.HasPrefix(k, "param_") { + gotParams++ + } + } + _, _ = io.WriteString(w, "{\"page\":\"/home\",\"n\":3}\n") + })) + t.Cleanup(srv.Close) + + store := pipes.Static(&pipes.NamedQuery{ + Name: "by_page", + SQL: "SELECT page, count() AS n FROM clicks WHERE page = {{page}} GROUP BY page LIMIT {{lim}}", + Parameters: []pipes.ParamDef{{Name: "page", Required: true}, {Name: "lim", Default: 5}}, + }) + h := NewPipesHandler(store, policy.Static(&policy.Policy{}), + func() chconn.Target { return chconn.Target{URL: srv.URL} }, + nil, shortTimeout, testutil.NopLogger()) + + w := httptest.NewRecorder() + r := pipesRequest(t, http.MethodPost, "/v1/pipes/by_page/execute", "by_page", map[string]any{"page": "/home"}) + r = r.WithContext(auth.WithRole(r.Context(), "admin")) + h.Execute(w, r) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "SELECT page, count() AS n FROM clicks WHERE page = '/home' GROUP BY page LIMIT 5", gotSQL) + assert.Zero(t, gotParams, "a pipe inlines its values; nothing should be bound") + assert.JSONEq(t, `[{"page":"/home","n":3}]`, w.Body.String()) + assert.Equal(t, "MISS", w.Header().Get("X-Cache")) +} + +// TestPipesHandler_Execute_ClickHouseErrorIs500 pins the error contract: a pipe +// whose SQL ClickHouse refuses is a 500 carrying ClickHouse's own message. +func TestPipesHandler_Execute_ClickHouseErrorIs500(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, "Code: 60. DB::Exception: Table default.nope does not exist. (UNKNOWN_TABLE)") + })) + t.Cleanup(srv.Close) + + store := pipes.Static(&pipes.NamedQuery{Name: "broken", SQL: "SELECT * FROM nope"}) + h := NewPipesHandler(store, policy.Static(&policy.Policy{}), + func() chconn.Target { return chconn.Target{URL: srv.URL} }, + nil, shortTimeout, testutil.NopLogger()) + + w := httptest.NewRecorder() + r := pipesRequest(t, http.MethodPost, "/v1/pipes/broken/execute", "broken", nil) + r = r.WithContext(auth.WithRole(r.Context(), "admin")) + h.Execute(w, r) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "Code: 60") + testutil.AssertJSONErrorResponse(t, w) +} diff --git a/internal/api/router_test.go b/internal/api/router_test.go index 4dd56240..3d36537b 100644 --- a/internal/api/router_test.go +++ b/internal/api/router_test.go @@ -328,7 +328,7 @@ func TestNewRouter_RoutesRegistered(t *testing.T) { t.Cleanup(func() { _ = emb.Close() }) deps := Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, @@ -486,7 +486,7 @@ func TestNewRouter_RawSQLAdminGate(t *testing.T) { hub := stream.NewHub(nil, nil, nil) router := NewRouter(Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, @@ -547,7 +547,7 @@ func TestNewRouter_OptionalDepsNil(t *testing.T) { hub := stream.NewHub(nil, nil, nil) deps := Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, @@ -625,7 +625,7 @@ func TestNewRouter_NotFoundEmitsJSON(t *testing.T) { pub := &testutil.MockPublisher{} hub := stream.NewHub(nil, nil, nil) deps := Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, @@ -650,7 +650,7 @@ func TestNewRouter_MethodNotAllowedEmitsJSON(t *testing.T) { pub := &testutil.MockPublisher{} hub := stream.NewHub(nil, nil, nil) deps := Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, @@ -750,7 +750,7 @@ func TestNewRouter_SchemaAdminOnly(t *testing.T) { hub := stream.NewHub(nil, nil, nil) router := NewRouter(Dependencies{ - Ingest: NewIngestHandler(reg, pub, testutil.NopLogger()), + Ingest: newTestIngestHandler(t, reg, pub, testutil.NopLogger()), Query: &QueryHandler{}, SSE: NewStreamHandler(hub, nil), Health: &HealthHandler{}, diff --git a/internal/api/structured_query.go b/internal/api/structured_query.go index 111cc072..1b5a53ab 100644 --- a/internal/api/structured_query.go +++ b/internal/api/structured_query.go @@ -8,10 +8,9 @@ import ( "net/http" "time" - "github.com/ClickHouse/clickhouse-go/v2" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/Wave-RF/WaveHouse/internal/auth" "github.com/Wave-RF/WaveHouse/internal/cache" + "github.com/Wave-RF/WaveHouse/internal/chconn" "github.com/Wave-RF/WaveHouse/internal/discovery" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/query" @@ -20,7 +19,7 @@ import ( // StructuredQueryHandler handles POST /v1/query?table={table} type StructuredQueryHandler struct { - CHConn driver.Conn + CH *chReader Cache cache.Cache Registry *discovery.SchemaRegistry PolicySource policy.Source @@ -50,7 +49,7 @@ type StructuredQueryHandler struct { } func NewStructuredQueryHandler( - conn driver.Conn, + target func() chconn.Target, c cache.Cache, registry *discovery.SchemaRegistry, policyStore policy.Source, @@ -60,7 +59,7 @@ func NewStructuredQueryHandler( logger *slog.Logger, ) *StructuredQueryHandler { return &StructuredQueryHandler{ - CHConn: conn, + CH: newCHReader(target), Cache: c, Registry: registry, PolicySource: policyStore, @@ -96,7 +95,12 @@ func (h *StructuredQueryHandler) Handle(w http.ResponseWriter, r *http.Request) r.Body = http.MaxBytesReader(w, r.Body, reqCap) var sq query.StructuredQuery - if err := json.NewDecoder(r.Body).Decode(&sq); err != nil { + // UseNumber so a filter value keeps the digits the caller wrote. Every + // value binds as a ClickHouse String parameter, so "12.50" and an integer + // past 2^53 reach the server intact instead of through a float64. + dec := json.NewDecoder(r.Body) + dec.UseNumber() + if err := dec.Decode(&sq); err != nil { if writeMaxBytesError(w, err, reqCap) { return } @@ -157,8 +161,21 @@ func (h *StructuredQueryHandler) Handle(w http.ResponseWriter, r *http.Request) return } - // Cache key. - cacheKey := queryCacheKey(result.SQL, result.Params) + // Bind the built SQL for ClickHouse's HTTP interface: positional `?` + // placeholders become {pN:String} / {pN:Array(String)} named parameters + // and each value becomes the text ClickHouse reads it back from. A value + // with no text form (a JSON null, an object) is a malformed query, not a + // server fault. + chSQL, chParams, err := result.NamedParams() + if err != nil { + writeJSONError(w, http.StatusBadRequest, err.Error()) + return + } + + // Cache key. Keyed on what actually reaches ClickHouse, so two requests + // that differ only in a spelling the binding erases still share an entry + // and two that differ in the bytes sent never do. + cacheKey := queryCacheKey(chSQL, chParams) // TODO: impl scope scope := "" @@ -194,13 +211,13 @@ func (h *StructuredQueryHandler) Handle(w http.ResponseWriter, r *http.Request) queryCtx, cancel := context.WithTimeout(r.Context(), timeout) defer cancel() - // Enforce the role's resource caps server-side, not just via the client - // context deadline (#316). The settings ride on the query context, so they - // reach ClickHouse for this query only. Server-wide backstops are - // ClickHouse's job (settings profiles / quotas); a role with no caps (e.g. - // admin) sends nothing here. An explicit max_execution_time is sent only - // when the role set a time cap; otherwise the context deadline (= - // query_timeout) is the time bound the driver derives. + // Enforce the role's resource caps server-side, not just via the request + // deadline (#316): the settings ride on this query's URL, so they reach + // ClickHouse for this query only. Server-wide backstops are ClickHouse's + // job (settings profiles / quotas); a role with no caps (e.g. admin) + // sends nothing here. An explicit max_execution_time is sent only when + // the role set a time cap; otherwise the context deadline (= query_timeout) + // is the only time bound. limits := chQueryLimits{ MaxResultRows: perms.Select.MaxRows, MaxRowsToRead: perms.Select.MaxRowsToRead, @@ -209,25 +226,18 @@ func (h *StructuredQueryHandler) Handle(w http.ResponseWriter, r *http.Request) if perms.Select.MaxExecutionTime > 0 { limits.ExecutionTime = timeout } - if settings := chReadSettings(limits); settings != nil { - queryCtx = clickhouse.Context(queryCtx, clickhouse.WithSettings(settings)) - } start := time.Now() - rows, err := executeCHQuery(queryCtx, h.CHConn, result.SQL, result.Params) + // ClickHouse's own JSON rendering of the rows, stored and served + // verbatim — no per-row scan, no re-marshal. + data, err := h.CH.query(queryCtx, chSQL, chParams, chReadSettings(limits)) queryDuration := time.Since(start) if err != nil { // TODO: depending on the error, we may actually want to cache it return nil, err } - data, err := json.Marshal(rows) - if err != nil { - // TODO: eventually we want CSV support etc - return nil, err - } - ttl := cache.QueryTimeToTTL(queryDuration) if h.Cache != nil { diff --git a/internal/api/structured_query_test.go b/internal/api/structured_query_test.go index 733140ed..9ca62c27 100644 --- a/internal/api/structured_query_test.go +++ b/internal/api/structured_query_test.go @@ -4,14 +4,18 @@ import ( "bytes" "context" "encoding/json" + "io" "net/http" "net/http/httptest" "net/url" + "strconv" + "sync" "testing" "time" - "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/cache" + "github.com/Wave-RF/WaveHouse/internal/chconn" "github.com/Wave-RF/WaveHouse/internal/discovery" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/query" @@ -40,7 +44,7 @@ func newStructuredQueryHandler(t testing.TB) *StructuredQueryHandler { }, }, }) - return NewStructuredQueryHandler(nil, nil, reg, nil, func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger()) + return NewStructuredQueryHandler(newCapturingCH(t).target, nil, reg, nil, func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger()) } func TestStructuredQuery_MissingTable(t *testing.T) { @@ -257,26 +261,67 @@ func TestStructuredQuery_NilPolicyFailsClosed(t *testing.T) { // ─── #223: column allowlist is a hard cap on every read, end-to-end ────────── -// sqlCapturingConn records the SQL (and bound args) the handler hands to -// ClickHouse so tests can assert the generated query without a live database. -// Query returns an empty result set (the handler marshals it to []); these -// tests assert on the SQL string, args, and HTTP status, not on rows. lastSQL -// stays empty when the request is rejected before execution — which is itself -// the assertion for denied paths. -type sqlCapturingConn struct { - driver.Conn - lastSQL string - lastArgs []any +// capturingCH stands in for ClickHouse's HTTP interface. It records the SQL +// the handler POSTs and the query parameters it binds, and answers with an +// empty JSONEachRow result (an empty body), which the handler frames as `[]`. +// These tests assert on the SQL string, the bound values and the HTTP status, +// not on rows. sql() stays empty when the request is rejected before +// execution — which is itself the assertion for the denied paths. +type capturingCH struct { + server *httptest.Server + mu sync.Mutex + lastSQL string + lastQ url.Values } -func (c *sqlCapturingConn) Query(_ context.Context, sql string, args ...any) (driver.Rows, error) { - c.lastSQL = sql - c.lastArgs = args - return &chainEmptyRows{}, nil +func newCapturingCH(t testing.TB) *capturingCH { + t.Helper() + c := &capturingCH{} + c.server = httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + c.mu.Lock() + defer c.mu.Unlock() + c.lastSQL = string(body) + c.lastQ = r.URL.Query() + })) + t.Cleanup(c.server.Close) + return c +} + +func (c *capturingCH) target() chconn.Target { + return chconn.Target{URL: c.server.URL} +} + +func (c *capturingCH) sql() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.lastSQL +} + +// params returns the bound values in placeholder order, i.e. what +// {p0:String}, {p1:String}, … received. +func (c *capturingCH) params() []string { + c.mu.Lock() + defer c.mu.Unlock() + var out []string + for i := 0; ; i++ { + v, ok := c.lastQ["param_p"+strconv.Itoa(i)] + if !ok { + return out + } + out = append(out, v[0]) + } +} + +// setting returns one per-query ClickHouse setting off the request URL. +func (c *capturingCH) setting(name string) string { + c.mu.Lock() + defer c.mu.Unlock() + return c.lastQ.Get(name) } // sensitiveSchema has a column (payload, user_id) that restrictive policies hide. -func newCapturingHandler(t *testing.T, conn driver.Conn, p *policy.Policy) *StructuredQueryHandler { +func newCapturingHandler(t *testing.T, ch *capturingCH, p *policy.Policy) *StructuredQueryHandler { t.Helper() reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ { @@ -289,7 +334,7 @@ func newCapturingHandler(t *testing.T, conn driver.Conn, p *policy.Policy) *Stru }, }, }) - return NewStructuredQueryHandler(conn, nil, reg, policy.Static(p), func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger()) + return NewStructuredQueryHandler(ch.target, nil, reg, policy.Static(p), func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger()) } func viewerRequest(t *testing.T, sq query.StructuredQuery) *http.Request { @@ -313,17 +358,17 @@ func policyWithViewer(perms policy.SelectPermissions) *policy.Policy { // payload/user_id never reach ClickHouse — let alone the client. func TestStructuredQuery_SelectAll_RestrictedRoleGetsAllowedProjection(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, "SELECT `page`, `ts` FROM `clicks` LIMIT 10000", conn.lastSQL) - assert.NotContains(t, conn.lastSQL, "*") - assert.NotContains(t, conn.lastSQL, "payload") - assert.NotContains(t, conn.lastSQL, "user_id") + assert.Equal(t, "SELECT `page`, `ts` FROM `clicks` LIMIT 10000", ch.sql()) + assert.NotContains(t, ch.sql(), "*") + assert.NotContains(t, ch.sql(), "payload") + assert.NotContains(t, ch.sql(), "user_id") } // TestStructuredQuery_RowFilterAndMaxRows_ReachClickHouse pins the handler seam @@ -336,8 +381,8 @@ func TestStructuredQuery_SelectAll_RestrictedRoleGetsAllowedProjection(t *testin func TestStructuredQuery_RowFilterAndMaxRows_ReachClickHouse(t *testing.T) { t.Parallel() eq := "{{ jwt.org_id }}" - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{ + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{ Filter: map[string]policy.Filter{"user_id": {Eq: &eq}}, MaxRows: 100, })) @@ -352,8 +397,8 @@ func TestStructuredQuery_RowFilterAndMaxRows_ReachClickHouse(t *testing.T) { h.Handle(w, r.WithContext(ctx)) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, "SELECT `page` FROM `clicks` WHERE (`user_id` = ?) AND `page` = ? LIMIT 100", conn.lastSQL) - assert.Equal(t, []any{"org-1", "/home"}, conn.lastArgs) + assert.Equal(t, "SELECT `page` FROM `clicks` WHERE (`user_id` = {p0:String}) AND `page` = {p1:String} LIMIT 100", ch.sql()) + assert.Equal(t, []string{"org-1", "/home"}, ch.params()) } // TestStructuredQuery_OmittedColumns_ReturnsNothing pins safe-by-default: a request @@ -362,30 +407,30 @@ func TestStructuredQuery_RowFilterAndMaxRows_ReachClickHouse(t *testing.T) { // simply leaving columns out. func TestStructuredQuery_OmittedColumns_ReturnsNothing(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{})) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) assert.JSONEq(t, "[]", w.Body.String()) - assert.Empty(t, conn.lastSQL, "an empty projection must not reach ClickHouse") + assert.Empty(t, ch.sql(), "an empty projection must not reach ClickHouse") } // TestStructuredQuery_SelectAll_DenyListExpands: select_all under a deny-list // (empty allow) expands to the non-denied columns, never a raw SELECT *. func TestStructuredQuery_SelectAll_DenyListExpands(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{DenyColumns: []string{"payload"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{DenyColumns: []string{"payload"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, "SELECT `page`, `user_id`, `ts` FROM `clicks` LIMIT 10000", conn.lastSQL) - assert.NotContains(t, conn.lastSQL, "payload") + assert.Equal(t, "SELECT `page`, `user_id`, `ts` FROM `clicks` LIMIT 10000", ch.sql()) + assert.NotContains(t, ch.sql(), "payload") } // TestStructuredQuery_LiteralStarColumn_Unknown: columns:["*"] is a literal column @@ -393,14 +438,14 @@ func TestStructuredQuery_SelectAll_DenyListExpands(t *testing.T) { // column — the all-columns wildcard is select_all. func TestStructuredQuery_LiteralStarColumn_Unknown(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{Columns: []string{"*"}})) require.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) - assert.Empty(t, conn.lastSQL) + assert.Empty(t, ch.sql()) } // TestStructuredQuery_UnrestrictedRoleKeepsSelectStar proves the common case is @@ -408,14 +453,14 @@ func TestStructuredQuery_LiteralStarColumn_Unknown(t *testing.T) { // columns and admin convenience preserved; no behavior change off the hot path). func TestStructuredQuery_UnrestrictedRoleKeepsSelectStar(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, "SELECT * FROM `clicks` LIMIT 10000", conn.lastSQL) + assert.Equal(t, "SELECT * FROM `clicks` LIMIT 10000", ch.sql()) } // TestStructuredQuery_DeniedColumnInAnyClause_Returns403 is the regression for @@ -454,15 +499,15 @@ func TestStructuredQuery_DeniedColumnInAnyClause_Returns403(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page", "ts"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, tt.sq)) assert.Equal(t, http.StatusForbidden, w.Code, "body=%s", w.Body.String()) assert.Contains(t, w.Body.String(), "not allowed") - assert.Empty(t, conn.lastSQL, "a denied query must never reach ClickHouse") + assert.Empty(t, ch.sql(), "a denied query must never reach ClickHouse") testutil.AssertJSONErrorResponse(t, w) }) } @@ -473,14 +518,14 @@ func TestStructuredQuery_DeniedColumnInAnyClause_Returns403(t *testing.T) { // a fail-open SELECT *. func TestStructuredQuery_NoReadableColumns_Returns403(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"nonexistent"}})) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"nonexistent"}})) w := httptest.NewRecorder() h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) assert.Equal(t, http.StatusForbidden, w.Code, "body=%s", w.Body.String()) - assert.Empty(t, conn.lastSQL) + assert.Empty(t, ch.sql()) testutil.AssertJSONErrorResponse(t, w) } @@ -489,10 +534,10 @@ func TestStructuredQuery_NoReadableColumns_Returns403(t *testing.T) { // and must get only that role's columns, not every column. func TestStructuredQuery_UnauthenticatedUsesDefaultRoleProjection(t *testing.T) { t.Parallel() - conn := &sqlCapturingConn{} + ch := newCapturingCH(t) p := policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"page"}}) p.DefaultRole = "viewer" // public access resolves to the restricted viewer role - h := newCapturingHandler(t, conn, p) + h := newCapturingHandler(t, ch, p) // No role on the context — a tokenless request. r := structuredQueryRequest(t, "clicks", query.StructuredQuery{SelectAll: true, Limit: 2}) @@ -500,8 +545,8 @@ func TestStructuredQuery_UnauthenticatedUsesDefaultRoleProjection(t *testing.T) h.Handle(w, r) require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) - assert.Equal(t, "SELECT `page` FROM `clicks` LIMIT 2", conn.lastSQL) - assert.NotContains(t, conn.lastSQL, "payload") + assert.Equal(t, "SELECT `page` FROM `clicks` LIMIT 2", ch.sql()) + assert.NotContains(t, ch.sql(), "payload") } // TestStructuredQuery_CacheKeyIsolatesColumnVisibility pins the cache-isolation @@ -519,17 +564,132 @@ func TestStructuredQuery_CacheKeyIsolatesColumnVisibility(t *testing.T) { }, }} sqlFor := func(role string) string { - conn := &sqlCapturingConn{} - h := newCapturingHandler(t, conn, p) + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, p) r := structuredQueryRequest(t, "clicks", query.StructuredQuery{SelectAll: true}) r = r.WithContext(auth.WithClaims(auth.WithRole(r.Context(), role), jwt.MapClaims{})) w := httptest.NewRecorder() h.Handle(w, r) require.Equal(t, http.StatusOK, w.Code, "role=%s body=%s", role, w.Body.String()) - return conn.lastSQL + return ch.sql() } viewerSQL, auditorSQL := sqlFor("viewer"), sqlFor("auditor") assert.NotEqual(t, viewerSQL, auditorSQL) assert.NotEqual(t, queryCacheKey(viewerSQL, nil), queryCacheKey(auditorSQL, nil), "roles with different column visibility must not share a cache key") } + +// TestStructuredQuery_ResourceCapsReachTheWire pins that the role's caps are +// still sent as ClickHouse settings now that the read goes over the HTTP +// interface rather than riding on a driver context (#316). The integration +// suite proves ClickHouse honours them; this proves they are sent at all, +// which is the half that silently regresses. +func TestStructuredQuery_ResourceCapsReachTheWire(t *testing.T) { + t.Parallel() + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{ + AllowColumns: []string{"page"}, + MaxRowsToRead: 1234, + })) + + w := httptest.NewRecorder() + h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) + + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "1234", ch.setting("max_rows_to_read")) + assert.Equal(t, "throw", ch.setting("read_overflow_mode")) +} + +// TestStructuredQuery_ServesClickHouseBytes pins the response path end to end: +// ClickHouse's own JSON rendering is what the caller gets and what the cache +// stores, with no re-marshal in between, and the second read is served from +// the cache byte-for-byte under X-Cache: HIT. +func TestStructuredQuery_ServesClickHouseBytes(t *testing.T) { + t.Parallel() + // A body only ClickHouse would produce: a Decimal as a bare number and a + // DateTime in ClickHouse's own spelling. A Go round-trip through + // map[string]any would reorder the keys. + const row = `{"page":"/home","amount":12.50,"ts":"2026-01-15 10:30:00"}` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, row+"\n") + })) + t.Cleanup(srv.Close) + + reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ + {Name: "clicks", Columns: []discovery.Column{{Name: "page", Type: "String"}}}, + }) + c, err := cache.NewLocal(1 << 20) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + + h := NewStructuredQueryHandler( + func() chconn.Target { return chconn.Target{URL: srv.URL} }, + c, reg, policy.Static(policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})), + func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger(), + ) + + w := httptest.NewRecorder() + h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) + require.Equal(t, http.StatusOK, w.Code, "body=%s", w.Body.String()) + assert.Equal(t, "MISS", w.Header().Get("X-Cache")) + assert.Equal(t, "["+row+"]", w.Body.String()) + + // Ristretto admits asynchronously, so the hit is eventual rather than + // immediate — the entry is not visible on the very next Get. + var hit *httptest.ResponseRecorder + require.Eventually(t, func() bool { + hit = httptest.NewRecorder() + h.Handle(hit, viewerRequest(t, query.StructuredQuery{SelectAll: true})) + return hit.Header().Get("X-Cache") == "HIT" + }, 5*time.Second, 20*time.Millisecond, "the result must become a cache hit") + assert.Equal(t, w.Body.String(), hit.Body.String(), "a cache hit must be byte-identical to the miss") +} + +// TestStructuredQuery_ClickHouseErrorIs500 pins the error contract: a query +// ClickHouse refuses is a 500 whose body carries ClickHouse's own message, +// unchanged from when the native driver raised it. +func TestStructuredQuery_ClickHouseErrorIs500(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, "Code: 47. DB::Exception: Unknown expression identifier. (UNKNOWN_IDENTIFIER)") + })) + t.Cleanup(srv.Close) + + reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ + {Name: "clicks", Columns: []discovery.Column{{Name: "page", Type: "String"}}}, + }) + h := NewStructuredQueryHandler( + func() chconn.Target { return chconn.Target{URL: srv.URL} }, + nil, reg, policy.Static(policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})), + func() int { return 60 }, func() time.Duration { return 5 * time.Second }, nil, testutil.NopLogger(), + ) + + w := httptest.NewRecorder() + h.Handle(w, viewerRequest(t, query.StructuredQuery{SelectAll: true})) + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "Code: 47") + testutil.AssertJSONErrorResponse(t, w) +} + +// TestStructuredQuery_NullFilterValueIs400 pins the one filter value that lost +// a meaning in the move to bound String parameters. `{"op":"eq","value":null}` +// used to become `col = NULL` — never true, so an empty result — and an empty +// String parameter would instead compare against the empty string, a different +// question. It is refused rather than answered wrongly. +func TestStructuredQuery_NullFilterValueIs400(t *testing.T) { + t.Parallel() + ch := newCapturingCH(t) + h := newCapturingHandler(t, ch, policyWithViewer(policy.SelectPermissions{AllowColumns: []string{"*"}})) + + w := httptest.NewRecorder() + h.Handle(w, viewerRequest(t, query.StructuredQuery{ + Columns: []string{"page"}, + Filters: []query.Filter{{Column: "page", Op: "eq", Value: nil}}, + })) + + assert.Equal(t, http.StatusBadRequest, w.Code, "body=%s", w.Body.String()) + assert.Contains(t, w.Body.String(), "must not be null") + assert.Empty(t, ch.sql(), "a query with no honest binding must never reach ClickHouse") + testutil.AssertJSONErrorResponse(t, w) +} diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go index 95988ca8..809b0595 100644 --- a/internal/auth/auth_test.go +++ b/internal/auth/auth_test.go @@ -129,8 +129,9 @@ func TestMiddleware_LargeIntegerClaim_ExactThroughPolicy(t *testing.T) { }} perms := policy.Evaluate(p, "viewer", "clicks", "select", c.claims) require.True(t, perms.Allowed) - assert.Equal(t, "`tenant_id` = ?", perms.Select.WhereClause) - assert.Equal(t, []any{"1234567890123456789"}, perms.Select.WhereParams) + clause, params := perms.Select.WhereSQL(nil) + assert.Equal(t, "`tenant_id` = ?", clause) + assert.Equal(t, []any{"1234567890123456789"}, params) } // TestMiddleware_NumericClaimSpelling_BindsCanonically: json.Number keeps the @@ -163,7 +164,8 @@ func TestMiddleware_NumericClaimSpelling_BindsCanonically(t *testing.T) { }} perms := policy.Evaluate(p, "viewer", "clicks", "select", c.claims) require.True(t, perms.Allowed) - assert.Equal(t, []any{tt.want}, perms.Select.WhereParams) + _, params := perms.Select.WhereSQL(nil) + assert.Equal(t, []any{tt.want}, params) }) } } diff --git a/internal/chsql/chsql.go b/internal/chsql/chsql.go index e91ba8b5..47d2cb47 100644 --- a/internal/chsql/chsql.go +++ b/internal/chsql/chsql.go @@ -1,16 +1,22 @@ // Package chsql holds ClickHouse SQL helpers shared across packages that build -// SQL — primarily safe identifier quoting. It is dependency-free so both -// internal/query and internal/policy can use it without an import cycle. +// SQL: safe identifier quoting, the encoding a value needs to survive a +// `{p:String}` query parameter, and the strict cast an integer column's claims +// are compared through. It is dependency-free so internal/query, +// internal/policy and internal/typelayer can all use it without an import +// cycle. package chsql import "strings" -// identEscaper escapes a ClickHouse identifier's special characters exactly as -// ClickHouse's own backQuote() does (confirmed against SHOW CREATE TABLE on a -// live server): a backslash becomes `\\` and a backtick becomes “ \` “. The -// two replacements run in a single left-to-right pass, so neither re-processes -// the other's output. -var identEscaper = strings.NewReplacer(`\`, `\\`, "`", "\\`") +// identEscaper is ClickHouse's backQuote() escaping, byte for byte as the +// chtypes library's QuoteIdentifier returns it (measured over all 256 bytes; +// typelayer's parity test pins it): `\\`, `\“, and `\0 \b \t \n \f \r` +// for NUL and those control characters. One left-to-right pass, so no +// replacement re-processes another's output. +var identEscaper = strings.NewReplacer( + `\`, `\\`, "`", "\\`", + "\x00", `\0`, "\b", `\b`, "\t", `\t`, "\n", `\n`, "\f", `\f`, "\r", `\r`, +) // QuoteIdent renders any value as a backtick-quoted ClickHouse identifier // (column, table, or alias). It is the single place an identifier becomes SQL @@ -25,19 +31,125 @@ var identEscaper = strings.NewReplacer(`\`, `\\`, "`", "\\`") // (verified on a live server). Always quoting is unconditionally correct and // needs no keyword table. // -// Values are never quoted here; they remain positional `?` parameters bound by -// the driver. +// A value is never rendered into SQL text here; it binds as a query parameter +// and is encoded by EscapeStringParam. func QuoteIdent(name string) string { return "`" + identEscaper.Replace(name) + "`" } -// BindUnsafe reports whether an identifier contains a character that -// clickhouse-go's positional binder miscounts. Today that is just '?': the -// driver counts every '?' in the query text — even inside a backtick-quoted -// identifier — so a name containing '?' would shift the value parameters that -// follow it. Callers refuse such names (fail closed) rather than risk mis-binding -// a value (including a row-level-security filter value). Pathological; no real -// schema names a column '?'. Tracked in Wave-RF/WaveHouse#279. +// BindUnsafe reports whether an identifier contains a character that the +// positional-placeholder scan miscounts. Today that is just '?': the scan that +// rewrites a built query's `?` placeholders into named `{pN:…}` parameters +// walks the SQL text left to right and cannot tell a placeholder from a '?' +// inside a backtick-quoted identifier, so a name containing '?' would shift +// every value that follows it. Callers refuse such names (fail closed) rather +// than risk mis-binding a value (including a row-level-security filter value). +// Pathological; no real schema names a column '?'. Tracked in +// Wave-RF/WaveHouse#279. func BindUnsafe(name string) bool { return strings.ContainsRune(name, '?') } + +// EscapeStringParam encodes one value for a ClickHouse `{p:String}` query +// parameter — the SINGLE encoding both read surfaces use, so the SQL path +// (internal/query, over the HTTP interface) and the row-filter path +// (internal/typelayer, over the chtypes artifact) cannot disagree about what a +// claim value is. +// +// ClickHouse reads a scalar parameter with its ESCAPED-TEXT reader, so a raw +// backslash starts an escape sequence and a raw tab or newline ends the field. +// Measured on 26.6.3.62 over the HTTP interface: `param_p0=a\b` came back +// holding a backspace with no error, and a raw tab or newline was a hard code +// 457 parse error (a 500 for the caller). Measured on the 26.6 chtypes +// artifact through typelayer's compiled filters: the same stored values were +// answered false (the backslash case) or refused at compile time (tab, +// newline, a trailing backslash), and the same encoding made all of them +// compare equal. Encoding `\` → `\\`, tab → `\t`, newline → `\n`, CR → `\r` +// round-trips every value byte for byte on both surfaces, including an +// embedded NUL. +// +// An Array(String) parameter takes a DIFFERENT rule and must NOT be run +// through this one: its elements are read as QUOTED values, where a raw tab or +// newline rides through untouched and only `'` and `\` need escaping. +// Applying both encodings is corruption — `a\b` becomes `a\\b`. See +// quoteCHElement in internal/query. +var EscapeStringParam = strings.NewReplacer( + `\`, `\\`, + "\t", `\t`, + "\n", `\n`, + "\r", `\r`, +).Replace + +// IntType is one of ClickHouse's integer type names, as IntegerType returns +// it. It is a distinct type so StrictInt can only be handed a name from that +// closed set, never text read off a schema. +type IntType string + +// intTypes is the closed set IntegerType answers from. Bool is UInt8 +// underneath but compares as a boolean, and Enum/Decimal are not integers, so +// none of them is here. +var intTypes = map[string]IntType{ + "UInt8": "UInt8", "UInt16": "UInt16", "UInt32": "UInt32", "UInt64": "UInt64", + "UInt128": "UInt128", "UInt256": "UInt256", + "Int8": "Int8", "Int16": "Int16", "Int32": "Int32", "Int64": "Int64", + "Int128": "Int128", "Int256": "Int256", +} + +// IntegerType reports whether colType — a ClickHouse type as system.columns +// and the chtypes library spell it — is an integer type, possibly wrapped in +// Nullable(...) and/or LowCardinality(...), and returns the bare integer type. +// +// It only picks which expression form a policy claim is compared through +// (StrictInt for an integer column, a plain {p:String} for everything else); +// it models no ClickHouse semantics. The wrappers are stripped because +// accurateCastOrNull to a LowCardinality type is refused by the server (code +// 455) and the bare type answers identically on a Nullable column (measured +// on 26.6.3.62 and the 26.6 chtypes artifact). A type it does not recognise +// keeps the {p:String} form. +func IntegerType(colType string) (IntType, bool) { + t := colType + for { + inner, ok := unwrap(t, "Nullable(") + if !ok { + inner, ok = unwrap(t, "LowCardinality(") + } + if !ok { + break + } + t = inner + } + it, ok := intTypes[t] + return it, ok +} + +func unwrap(t, prefix string) (string, bool) { + if strings.HasPrefix(t, prefix) && strings.HasSuffix(t, ")") { + return t[len(prefix) : len(t)-1], true + } + return "", false +} + +// StrictInt renders the round-trip strict cast an integer column compares a +// claim bound as {param:String} against: +// +// if(toString(accurateCastOrNull({p:String}, 'T')) = {p:String}, accurateCastOrNull({p:String}, 'T'), NULL) +// +// A bare {p:String} wraps a value at or past 2^64 on every integer column (and +// a 128/256-bit column at its own width), and accurateCastOrNull alone still +// wraps on [U]Int128/[U]Int256. The round trip turns every value that is not +// the canonical spelling of an in-range integer into NULL, which no operator +// admits, while an in-range canonical value compares exactly as before and +// keeps the primary key in use. Measured identical on ClickHouse 26.6.3.62 and +// the 26.6/25.8 chtypes artifacts. +func StrictInt(param string, t IntType) string { + cast := "accurateCastOrNull({" + param + ":String}, '" + string(t) + "')" + return "if(toString(" + cast + ") = {" + param + ":String}, " + cast + ", NULL)" +} + +// IntParam is a positional value the query builder binds through StrictInt +// instead of as a bare {pN:String}: one parameter, referenced from the +// expression the placeholder expands to. +type IntParam struct { + Value string + Type IntType +} diff --git a/internal/chsql/chsql_test.go b/internal/chsql/chsql_test.go index df121cc6..b732a943 100644 --- a/internal/chsql/chsql_test.go +++ b/internal/chsql/chsql_test.go @@ -19,6 +19,7 @@ func TestQuoteIdent(t *testing.T) { {"embedded backslash", `back\slash`, "`back\\\\slash`"}, {"backslash then backtick", "x\\`y", "`x\\\\\\`y`"}, {"star is just a name here", "*", "`*`"}, + {"control characters as backQuote spells them", "a\x00\b\t\n\f\rb", "`a" + `\0\b\t\n\f\r` + "b`"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -43,3 +44,108 @@ func TestBindUnsafe(t *testing.T) { } } } + +// TestEscapeStringParam pins the encoding both {p:String} surfaces depend on. +// The single quote and '%'/'_' are deliberately NOT escaped: the value is read +// as an escaped FIELD, not as a quoted literal and not as a LIKE pattern, so +// encoding them would bind characters the caller never wrote. +func TestEscapeStringParam(t *testing.T) { + t.Parallel() + tests := []struct { + name string + in string + want string + }{ + {"plain", "acme", "acme"}, + {"empty", "", ""}, + {"backslash", `a\b`, `a\\b`}, + {"trailing backslash", `trail\`, `trail\\`}, + {"tab", "a\tb", `a\tb`}, + {"newline", "a\nb", `a\nb`}, + {"carriage return", "a\rb", `a\rb`}, + {"single quote is untouched", "O'Brien", "O'Brien"}, + {"like metacharacters are untouched", "50%_off", "50%_off"}, + {"nul is untouched", "a\x00b", "a\x00b"}, + {"backslash then t is not re-read as a tab", `a\tb`, `a\\tb`}, + {"every byte at once", "a\\\tb\nc\rd", `a\\\tb\nc\rd`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := EscapeStringParam(tt.in); got != tt.want { + t.Errorf("EscapeStringParam(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestIntegerType(t *testing.T) { + t.Parallel() + tests := []struct { + in string + want IntType + wantOK bool + }{ + {"UInt8", "UInt8", true}, + {"UInt16", "UInt16", true}, + {"UInt32", "UInt32", true}, + {"UInt64", "UInt64", true}, + {"UInt128", "UInt128", true}, + {"UInt256", "UInt256", true}, + {"Int8", "Int8", true}, + {"Int16", "Int16", true}, + {"Int32", "Int32", true}, + {"Int64", "Int64", true}, + {"Int128", "Int128", true}, + {"Int256", "Int256", true}, + {"Nullable(UInt64)", "UInt64", true}, + {"LowCardinality(UInt32)", "UInt32", true}, + {"LowCardinality(Nullable(Int64))", "Int64", true}, + {"Nullable(Int256)", "Int256", true}, + + {"String", "", false}, + {"LowCardinality(String)", "", false}, + {"Nullable(String)", "", false}, + {"Bool", "", false}, + {"Nullable(Bool)", "", false}, + {"Decimal(18, 4)", "", false}, + {"Decimal64(4)", "", false}, + {"Float64", "", false}, + {"Enum8('a' = 1, 'b' = 2)", "", false}, + {"Enum16('x' = 1)", "", false}, + {"UUID", "", false}, + {"Date", "", false}, + {"DateTime", "", false}, + {"DateTime64(3, 'UTC')", "", false}, + {"IPv4", "", false}, + {"Array(UInt64)", "", false}, + {"Map(String, UInt64)", "", false}, + {"Tuple(UInt64)", "", false}, + {"FixedString(8)", "", false}, + // Not ClickHouse's canonical spelling, so not recognised: the column + // keeps the plain {p:String} form rather than a guessed one. + {"uint64", "", false}, + {"BIGINT UNSIGNED", "", false}, + {"Nullable(UInt64", "", false}, + {"Nullable()", "", false}, + {"", "", false}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + got, ok := IntegerType(tt.in) + if got != tt.want || ok != tt.wantOK { + t.Errorf("IntegerType(%q) = (%q, %v), want (%q, %v)", tt.in, got, ok, tt.want, tt.wantOK) + } + }) + } +} + +func TestStrictInt(t *testing.T) { + t.Parallel() + got := StrictInt("p3", "UInt64") + want := "if(toString(accurateCastOrNull({p3:String}, 'UInt64')) = {p3:String}, accurateCastOrNull({p3:String}, 'UInt64'), NULL)" + if got != want { + t.Errorf("StrictInt = %q\nwant %q", got, want) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index f02bb5b3..acc1856a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -116,13 +116,26 @@ type Server struct { ShutdownTimeout int `yaml:"shutdown_timeout" env:"WH_SERVER_SHUTDOWN_TIMEOUT" env-default:"10"` } -// ClickHouse holds only the password. The wiring — address, HTTP port and -// scheme, database, username, query timeout — is the settings directory's -// `clickhouse` block (hot-reloadable: a change swaps the connection). The password stays here because secrets +// ClickHouse holds the password plus the chtypes registry directory. The +// wiring — address, HTTP port and scheme, database, username, query +// timeout — is the settings directory's `clickhouse` block (hot-reloadable: +// a change swaps the connection). The password stays here because secrets // don't belong in a tracked JSON file; it is combined with the adopted -// wiring on every (re)connect. +// wiring on every (re)connect. ChtypesRegistry is boot-tier like the +// password (typelayer.NewEngine opens the registry once at boot, not +// hot-reloadable) but isn't a secret — it's grouped here because it is the +// other ClickHouse-adjacent boot input, not because it needs the same +// protection. type ClickHouse struct { Password string `yaml:"password" env:"WH_CH_PASSWORD"` + // ChtypesRegistry names the chtypes artifact registry directory (see + // typelayer.Config.RegistryDir). Empty (the default) defers to the SDK's + // own search path — $CHTYPES_REGISTRY, the per-user cache, then the + // system dirs; an explicit directory is searched first, then the rest of + // that path. Either way a library is opened lazily, on first use of its + // line. deployments/Dockerfile sets CHTYPES_REGISTRY rather than this + // field. + ChtypesRegistry string `yaml:"chtypes_registry" env:"WH_CHTYPES_REGISTRY"` } // Cache sizes the in-process L1 cache. The time-range bucket structured diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go index 9c134c71..67518a0d 100644 --- a/internal/discovery/discovery.go +++ b/internal/discovery/discovery.go @@ -34,12 +34,6 @@ type Column struct { // carries the ordinal itself so a caller holding a lone Column still knows // where it sits. Position uint64 `json:"position"` - - // tsSpec is a DateTime/DateTime64 column's canonicalization spec, resolved - // once at schema build (Refresh). nil for non-timestamp columns, hand-built - // Column literals, and unresolvable zones — CanonicalizeTimestamps passes - // those through untouched (fail-open, #372). - tsSpec *timestampSpec } // TableSchema holds the discovered schema for one ClickHouse table. Columns is @@ -187,6 +181,13 @@ type SchemaRegistry struct { // serverVersion is the ClickHouse version string from the last successful // Refresh, guarded by mu alongside tables. serverVersion string + // serverTZ is the server's default time zone name from the same Refresh. + // ClickHouse reads zone-less timestamps in it, so the type layer has to + // parse in the same zone or it answers about a different instant. + serverTZ string + // onRefresh is notified after every successful Refresh, with the same + // version/zone/tables the swap published. + onRefresh []func(serverVersion, serverTZ string, tables []*TableSchema) } // NewSchemaRegistry creates a registry that discovers schemas from system.columns. @@ -200,16 +201,28 @@ func NewSchemaRegistry(conn driver.Conn, database func() string, refreshInterval } } +// OnRefresh registers a hook invoked after every successful Refresh, once the +// new schemas are published. Hooks run synchronously on the refresh goroutine — +// they are expected to be cheap (the type layer's rebind compiles only the +// tables whose columns changed) — and must be registered before the first +// Refresh. +func (sr *SchemaRegistry) OnRefresh(hook func(serverVersion, serverTZ string, tables []*TableSchema)) { + sr.mu.Lock() + defer sr.mu.Unlock() + sr.onRefresh = append(sr.onRefresh, hook) +} + // Refresh rebuilds the in-memory schema cache: it discovers the server's default -// time zone and version, queries system.columns, attaches each table's DDL from -// system.tables, and precomputes timestamp column specs. +// time zone and version, queries system.columns, and attaches each table's DDL +// from system.tables. func (sr *SchemaRegistry) Refresh(ctx context.Context) error { tracer := otel.GetTracerProvider().Tracer("wavehouse-discovery") ctx, span := tracer.Start(ctx, "SchemaRegistry.Refresh") defer span.End() // ClickHouse interprets zone-less timestamp strings in the server's default - // zone; canonicalization applies the same rule so the instant never changes (#372). + // zone; the type layer parses in the same zone so the instant never + // changes (#372). var tzName string if err := sr.conn.QueryRow(ctx, "SELECT timezone()").Scan(&tzName); err != nil { return fmt.Errorf("query server timezone: %w", err) @@ -231,16 +244,6 @@ func (sr *SchemaRegistry) Refresh(ctx context.Context) error { return fmt.Errorf("query server version: %w", err) } - var serverTZ *time.Location - if loc, err := loadLocation(tzName); err == nil { - serverTZ = loc - } else { - // Unresolvable — warn, not fatal, and no UTC fallback (that could move - // instants). A nil server zone means zone-less values pass through. - sr.logger.Warn("cannot resolve server timezone; zone-less timestamps will pass through un-canonicalized", - "timezone", tzName, "error", err) - } - // One database per refresh. `database` is a live getter so a ClickHouse // reconfigure is honored on the NEXT refresh — reading it twice would let a // reconfigure land between the system.columns and system.tables queries and @@ -295,16 +298,33 @@ func (sr *SchemaRegistry) Refresh(ctx context.Context) error { return err } + var noDDL []string + published := make([]*TableSchema, 0, len(tables)) for _, ts := range tables { - resolveTimestampSpecs(ts, serverTZ, sr.logger) ts.cacheInsertable() + published = append(published, ts) + if ts.DDL == "" { + noDDL = append(noDDL, ts.Name) + } } sr.mu.Lock() sr.tables = tables sr.serverVersion = serverVersion + sr.serverTZ = tzName + hooks := sr.onRefresh sr.mu.Unlock() sr.logger.Info("schema registry refreshed", "tables", len(tables), "server_tz", tzName, "server_version", serverVersion) + if len(noDDL) > 0 { + // The two scans are not one snapshot, so a table can be missing its + // CREATE statement without being missing (#558). One line per refresh, + // not per table. + sr.logger.Warn("tables discovered without DDL", "tables", noDDL) + } + + for _, hook := range hooks { + hook(serverVersion, tzName, published) + } return nil } @@ -353,6 +373,16 @@ func (sr *SchemaRegistry) ServerVersion() string { return sr.serverVersion } +// ServerTimezone returns the server's default time zone name captured by the +// last successful Refresh, or "" before the first one. It is the name +// ClickHouse reported, not a resolved location: nothing here interprets it, the +// type layer hands it straight to the parser that will read the rows. +func (sr *SchemaRegistry) ServerTimezone() string { + sr.mu.RLock() + defer sr.mu.RUnlock() + return sr.serverTZ +} + // Get returns the schema for a table, or nil if not found. func (sr *SchemaRegistry) Get(name string) *TableSchema { sr.mu.RLock() diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go index 6ba50a3c..372c9b5a 100644 --- a/internal/discovery/discovery_test.go +++ b/internal/discovery/discovery_test.go @@ -360,13 +360,68 @@ func newFakeRegistry(t *testing.T, errs []error) (*SchemaRegistry, *fakeConn) { return NewSchemaRegistry(conn, func() string { return "test" }, func() time.Duration { return time.Hour }, logger), conn } -// TestRefresh_UnresolvableServerTimezone_NotFatal: an unresolvable server zone -// degrades to pass-through canonicalization (#372), never a failed refresh. +// TestRefresh_UnresolvableServerTimezone_NotFatal: the registry reports the +// zone name verbatim and never interprets it, so a name Go cannot resolve is +// still published rather than failing the refresh. Whoever consumes it decides +// what an unusable zone means (#372). func TestRefresh_UnresolvableServerTimezone_NotFatal(t *testing.T) { t.Parallel() conn := &fakeConn{tz: "Not/AZone"} sr := NewSchemaRegistry(conn, func() string { return "test" }, func() time.Duration { return time.Hour }, discardLogger()) require.NoError(t, sr.Refresh(context.Background())) + assert.Equal(t, "Not/AZone", sr.ServerTimezone()) +} + +// TestServerTimezone_EmptyBeforeRefresh: nothing is published until a refresh +// succeeds, so a consumer cannot mistake "not probed yet" for UTC. +func TestServerTimezone_EmptyBeforeRefresh(t *testing.T) { + t.Parallel() + sr, _ := newFakeRegistry(t, nil) + assert.Empty(t, sr.ServerTimezone()) + require.NoError(t, sr.Refresh(context.Background())) + assert.Equal(t, "UTC", sr.ServerTimezone()) +} + +// TestOnRefresh_FiresAfterSwapWithPublishedSchemas: the hook is what binds the +// type layer, so it must see the version, the zone and the same schemas Get() +// now returns — not the ones from before the swap. +func TestOnRefresh_FiresAfterSwapWithPublishedSchemas(t *testing.T) { + t.Parallel() + conn := &fakeConn{ + tz: "Europe/Berlin", + version: "26.6.3.62", + columns: []fakeColumn{{table: "events", name: "id", chType: "UInt64", position: 1}}, + } + sr := NewSchemaRegistry(conn, func() string { return "test" }, func() time.Duration { return time.Hour }, discardLogger()) + + var gotVersion, gotTZ string + var gotTables []*TableSchema + calls := 0 + sr.OnRefresh(func(version, tz string, tables []*TableSchema) { + calls++ + gotVersion, gotTZ, gotTables = version, tz, tables + assert.NotNil(t, sr.Get("events"), "hook must run after the swap") + }) + + require.NoError(t, sr.Refresh(context.Background())) + assert.Equal(t, 1, calls) + assert.Equal(t, "26.6.3.62", gotVersion) + assert.Equal(t, "Europe/Berlin", gotTZ) + require.Len(t, gotTables, 1) + assert.Equal(t, "events", gotTables[0].Name) + assert.Equal(t, []string{"id"}, gotTables[0].InsertableColumnNames(), "hook sees the memoized schema") +} + +// TestOnRefresh_NotFiredOnFailure: a failed refresh keeps the previous cache, +// so rebinding off a half-read registry would compile the wrong thing. +func TestOnRefresh_NotFiredOnFailure(t *testing.T) { + t.Parallel() + conn := &fakeConn{versionErr: errors.New("server gone")} + sr := NewSchemaRegistry(conn, func() string { return "test" }, func() time.Duration { return time.Hour }, discardLogger()) + fired := false + sr.OnRefresh(func(string, string, []*TableSchema) { fired = true }) + require.Error(t, sr.Refresh(context.Background())) + assert.False(t, fired) } // TestRefresh_RowsIterationError_Fails: rows.Next() returns false on a @@ -809,3 +864,9 @@ func TestInsertableColumns_CachedAndUncachedAgree(t *testing.T) { require.NotEmpty(t, first) assert.Same(t, &first[0], &second[0], "callers share the memoized backing array") } + +// discardLogger is the registries' test logger: refresh warnings are asserted +// via behavior, not log output. +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} diff --git a/internal/discovery/timestamp.go b/internal/discovery/timestamp.go deleted file mode 100644 index 072096c4..00000000 --- a/internal/discovery/timestamp.go +++ /dev/null @@ -1,323 +0,0 @@ -package discovery - -import ( - "encoding/json" - "fmt" - "log/slog" - "math" - "strconv" - "strings" - "time" -) - -// Ingest canonicalizes every DateTime/DateTime64 column value to one wire form — -// RFC 3339 UTC (`2026-06-21T04:00:00Z`, fraction per column precision) — before -// the event is published (#372), so SSE subscribers see the same spelling -// /v1/query renders; fail-open, with fail-closed enforcement left to the stream -// row-filter (#381). The grammar is a strict subset of what ClickHouse's insert -// reads — mirrored per column kind and differential-tested against a live server -// (tests/integration/timestamp_canonicalization_test.go) — and everything else -// passes through verbatim, so ClickHouse is never second-guessed. - -// isTimestampType reports whether chType is a ClickHouse DateTime or DateTime64 -// (unwrapping Nullable/LowCardinality). Date/Date32 are excluded — day-precision, -// no zone or spelling ambiguity. -func isTimestampType(chType string) bool { - return strings.HasPrefix(unwrapType(chType), "DateTime") -} - -// CanonicalizeTimestamps rewrites every DateTime/DateTime64 column value in data -// to the canonical RFC 3339 UTC form, in place: zone-less values are read in the -// column's zone, else the server default — ClickHouse's own rule, so only the -// spelling changes, never the instant — and the fraction truncates to the -// column's precision to byte-match /v1/query. Everything else passes through -// verbatim (fail-open): absent/null values, unparseable values, columns without -// a spec, and instants outside the column kind's range (see rewritable). -func CanonicalizeTimestamps(schema *TableSchema, data map[string]any) { - for _, col := range schema.Columns { - spec := col.tsSpec - if spec == nil { - continue - } - v, ok := data[col.Name] - if !ok || v == nil { - continue - } - t, err := parseTimestamp(v, spec) - if err != nil || !spec.rewritable(t) { - continue - } - data[col.Name] = canonicalTimestamp(t, spec.precision) - } -} - -// TimeParser returns the mapping from one rendering of this DateTime/DateTime64 -// column's value to the instant ClickHouse would store: the same grammar and -// zone rule ingest canonicalization applies (parseTimestamp), the same range -// guard (rewritable — an out-of-range operand, which insert-time saturation -// would move, is refused), truncated to the column's precision exactly like the -// canonical wire form. nil when the column isn't a timestamp column with a -// resolved spec; such columns keep byte-equality semantics on the stream. The -// stream row-filter uses it (policy.ColumnSpec.ParseTime) so a filter constant -// in any accepted spelling — zone-less read in the column's zone, RFC 3339, -// Unix seconds — and the canonicalized payload compare as instants (#381), -// through one grammar that can't drift from ingest's. -func (c *Column) TimeParser() func(v any) (time.Time, bool) { - spec := c.tsSpec - if spec == nil { - return nil - } - unit := time.Second - for range spec.precision { - unit /= 10 - } - return func(v any) (time.Time, bool) { - t, err := parseTimestamp(v, spec) - if err != nil || !spec.rewritable(t) { - return time.Time{}, false - } - return t.Truncate(unit), true - } -} - -// timestampSpec is a timestamp column's precomputed canonicalization inputs: -// sub-second precision (0 for DateTime), the column kind (ClickHouse reads -// numbers and Unix-string fractions differently for DateTime64), and the zone -// for zone-less values (nil = unknown ⇒ only zone-explicit values canonicalize). -type timestampSpec struct { - precision int - isDT64 bool - loc *time.Location -} - -// resolveTimestampSpecs precomputes the timestampSpec of every DateTime/DateTime64 -// column in ts once per schema build, so the per-record ingest path parses no -// type strings and loads no zones. An unresolvable zone (no embedded tzdata — -// resolution needs the runtime's zone database) keeps a nil spec — warned, not -// fatal: those values pass through un-canonicalized. -func resolveTimestampSpecs(ts *TableSchema, serverTZ *time.Location, logger *slog.Logger) { - for i := range ts.Columns { - col := &ts.Columns[i] - if !isTimestampType(col.Type) { - continue - } - spec, err := resolveTimestampSpec(col.Type, serverTZ) - if err != nil { - logger.Warn("cannot resolve timestamp column spec; its ingest values will pass through un-canonicalized", - "table", ts.Name, "column", col.Name, "type", col.Type, "error", err) - continue - } - col.tsSpec = &spec - } -} - -// resolveTimestampSpec extracts a DateTime/DateTime64 type's sub-second precision -// and time zone: `DateTime` / `DateTime('TZ')` / `DateTime64(P)` / -// `DateTime64(P, 'TZ')`. A type without an explicit zone takes serverTZ (possibly -// nil = unknown) — ClickHouse's own rule for zone-less strings. -func resolveTimestampSpec(chType string, serverTZ *time.Location) (timestampSpec, error) { - t := unwrapType(chType) - - var args []string - if open := strings.IndexByte(t, '('); open != -1 && strings.HasSuffix(t, ")") { - for arg := range strings.SplitSeq(t[open+1:len(t)-1], ",") { - args = append(args, strings.TrimSpace(arg)) - } - t = t[:open] - } - - isDT64 := t == "DateTime64" - var precision int - if isDT64 { - if len(args) == 0 { - return timestampSpec{}, fmt.Errorf("malformed type %q: DateTime64 requires a precision", chType) - } - p, err := strconv.Atoi(args[0]) - if err != nil || p < 0 || p > 9 { - return timestampSpec{}, fmt.Errorf("malformed type %q: bad precision %q", chType, args[0]) - } - precision = p - args = args[1:] - } - - if len(args) == 0 { - return timestampSpec{precision: precision, isDT64: isDT64, loc: serverTZ}, nil - } - name := strings.Trim(args[0], "'") - loc, err := loadLocation(name) - if err != nil { - return timestampSpec{}, fmt.Errorf("unknown time zone %q in type %q: %w", name, chType, err) - } - return timestampSpec{precision: precision, isDT64: isDT64, loc: loc}, nil -} - -func loadLocation(name string) (*time.Location, error) { - switch name { - case "Etc/UTC": - return time.UTC, nil - case "", "Local": - // time.LoadLocation reads "" as UTC and "Local" from the process - // environment — neither is a zone ClickHouse would declare. Treat both - // as unresolvable (nil spec, pass-through) instead of guessing. - return nil, fmt.Errorf("not an IANA zone name: %q", name) - } - return time.LoadLocation(name) -} - -// Rewrite bounds. ClickHouse saturates out-of-range instants spelling-dependently -// (the date clamps in the column zone keeping local time-of-day; DateTime64(9) -// even fails the insert past the Int64-nanosecond ceiling, a bound applied here -// to precision ≥7 conservatively), so no rewrite out there is safe — the -// producer's own spelling passes through and saturates as it did before #372. -// One-day margins keep zone arithmetic from straddling a bound. -var ( - dtMin = time.Unix(0, 0) - dtMax = time.Unix(4294967295-86400, 0) // UInt32 seconds ceiling, one day of margin - dt64Min = time.Date(1900, 1, 2, 0, 0, 0, 0, time.UTC) - dt64Max = time.Date(2299, 12, 30, 23, 59, 59, 0, time.UTC) - ns64Max = time.Date(2262, 4, 10, 23, 59, 59, 0, time.UTC) // Int64 ns ceiling (binds at precision 9; applied to ≥ 7) -) - -// rewritable reports whether t is safely inside the column kind's range — -// outside it the value passes through, so ClickHouse's saturation applies to -// the producer's spelling, never a rewritten one. -func (s *timestampSpec) rewritable(t time.Time) bool { - lo, hi := dtMin, dtMax - if s.isDT64 { - lo, hi = dt64Min, dt64Max - if s.precision >= 7 { - hi = ns64Max - } - } - return !t.Before(lo) && !t.After(hi) -} - -// parseTimestamp converts one ingested value into a time.Time: RFC 3339 -// ('.'-fraction only), zone-less `YYYY-MM-DD[ T]HH:MM:SS[.fraction]` / -// `YYYY-MM-DD` in the spec's zone (skipped when nil), and Unix seconds per -// parseUnixDigits and unixNumber — anything else errors and the caller passes -// it through. The grammar is deliberately a subset of ClickHouse best_effort's, -// instant-identical on that subset: when unsure what ClickHouse would read, -// this parser must fail, never guess. -func parseTimestamp(v any, spec *timestampSpec) (time.Time, error) { - switch x := v.(type) { - case string: - // ClickHouse has no ',' decimal separator (it would fight CSV), while - // Go's RFC3339Nano accepts one per ISO 8601. Reject up front so a - // ",999" fraction can't widen insertability. - if strings.ContainsRune(x, ',') { - return time.Time{}, fmt.Errorf( - "unrecognized timestamp %.64q (',' is not a fraction separator to ClickHouse)", x) - } - if t, err := time.Parse(time.RFC3339Nano, x); err == nil { - return t, nil - } - // Zone-less forms; Go's Parse accepts an input fraction after the seconds - // even when the layout carries none. - if spec.loc != nil { - for _, layout := range []string{"2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} { - if t, err := time.ParseInLocation(layout, x, spec.loc); err == nil { - return t, nil - } - } - } - if t, ok := parseUnixDigits(x, spec.isDT64); ok { - return t, nil - } - return time.Time{}, fmt.Errorf( - "unrecognized timestamp %.64q (accepted: RFC 3339, 'YYYY-MM-DD[ T]HH:MM:SS[.fff]', 'YYYY-MM-DD', or 9-10 digit Unix seconds)", x) - case float64: - // Without json.Decoder.UseNumber a producer's integer is exact only - // through 2^53; past that the digits ClickHouse would read are already - // lost. Non-integers pass through — ClickHouse rejects them everywhere. - if x != math.Trunc(x) || math.Abs(x) > 1<<53 { - return time.Time{}, fmt.Errorf("timestamp number %v is not an exact integer", x) - } - return unixNumber(int64(x), spec) - case json.Number: - s := x.String() - if !isDigits(s) { - // A sign, '.', or exponent: not the integer epoch shape ClickHouse - // reads for DateTime columns — pass through for its verdict. - return time.Time{}, fmt.Errorf("timestamp number %q is not a non-negative integer", s) - } - n, err := strconv.ParseInt(s, 10, 64) - if err != nil { - return time.Time{}, fmt.Errorf("timestamp number %q: %w", s, err) - } - return unixNumber(n, spec) - default: - return time.Time{}, fmt.Errorf("timestamp must be a string or Unix-seconds number, got %T", v) - } -} - -// unixNumber mirrors how ClickHouse reads an integer JSON number: Unix seconds -// for DateTime, ticks at the column's scale for DateTime64 — reading a -// DateTime64 number as seconds would change the stored instant (1750478400 is -// a 1970 instant on a DateTime64(3); the ms epoch 1750478400500 is a valid -// 2025 one). Negative numbers pass through, and out-of-range results are -// caught by rewritable. -func unixNumber(n int64, spec *timestampSpec) (time.Time, error) { - if n < 0 { - return time.Time{}, fmt.Errorf("negative timestamp number %d", n) - } - if !spec.isDT64 { - return time.Unix(n, 0), nil - } - scale := int64(1) - for range spec.precision { - scale *= 10 - } - return time.Unix(n/scale, (n%scale)*(1_000_000_000/scale)), nil -} - -// parseUnixDigits reads Unix seconds in the one string shape ClickHouse -// best_effort does — 9–10 integer digits; other run lengths are its calendar -// forms (8 ⇒ YYYYMMDD, 14 ⇒ YYYYMMDDhhmmss) or its ms/µs/ns epochs (13/16/19) -// and pass through — with a '.' fraction honored only for DateTime64 targets -// (plain DateTime fails the row on it). The fraction is an exact decimal -// truncated at nine digits: ClickHouse truncates, never rounds, and a float64 -// round-trip would corrupt nanoseconds or roll across the second. -func parseUnixDigits(s string, isDT64 bool) (time.Time, bool) { - intPart, frac, hasFrac := strings.Cut(s, ".") - if len(intPart) < 9 || len(intPart) > 10 || !isDigits(intPart) { - return time.Time{}, false - } - if hasFrac && (!isDT64 || frac == "" || !isDigits(frac)) { - return time.Time{}, false - } - sec, _ := strconv.ParseInt(intPart, 10, 64) // 9-10 digits by construction - var ns int64 - if hasFrac { - f := frac - if len(f) > 9 { - f = f[:9] - } - ns, _ = strconv.ParseInt(f, 10, 64) // all digits by construction - // Zero-fill the fraction out to nanosecond scale. - for range 9 - len(f) { - ns *= 10 - } - } - return time.Unix(sec, ns), true -} - -func isDigits(s string) bool { - for i := 0; i < len(s); i++ { - if s[i] < '0' || s[i] > '9' { - return false - } - } - return true -} - -// canonicalTimestamp renders t in the canonical wire form: UTC RFC 3339, fraction -// truncated to the column's precision. RFC3339Nano trims trailing zeros exactly -// like /v1/query's transformRow, keeping the two read paths byte-identical. -func canonicalTimestamp(t time.Time, precision int) string { - unit := time.Second - for range precision { - unit /= 10 - } - return t.UTC().Truncate(unit).Format(time.RFC3339Nano) -} diff --git a/internal/discovery/timestamp_test.go b/internal/discovery/timestamp_test.go deleted file mode 100644 index aa1f1382..00000000 --- a/internal/discovery/timestamp_test.go +++ /dev/null @@ -1,315 +0,0 @@ -package discovery - -import ( - "context" - "encoding/json" - "io" - "log/slog" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestIsTimestampType(t *testing.T) { - t.Parallel() - - tests := []struct { - chType string - want bool - }{ - {"DateTime", true}, - {"DateTime('UTC')", true}, - {"DateTime64(3)", true}, - {"DateTime64(6, 'America/New_York')", true}, - {"Nullable(DateTime)", true}, - {"LowCardinality(Nullable(DateTime('UTC')))", true}, - {"Date", false}, // day-precision, no spelling ambiguity — deliberately excluded - {"Date32", false}, // as above - {"String", false}, - {"UInt64", false}, - } - - for _, tt := range tests { - t.Run(tt.chType, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, isTimestampType(tt.chType)) - }) - } -} - -// tsSchema builds a one-column schema of the given ClickHouse type, so each case -// exercises exactly one column's canonicalization. -func tsSchema(colType string) *TableSchema { - return &TableSchema{Name: "t", Columns: []Column{{Name: "ts", Type: colType}}} -} - -func TestCanonicalizeTimestamps(t *testing.T) { - t.Parallel() - nyc, err := time.LoadLocation("America/New_York") - require.NoError(t, err) - - tests := []struct { - name string - colType string - serverTZ *time.Location - value any - want any - }{ - {"canonical passes through", "DateTime('UTC')", nil, "2026-06-21T04:00:00Z", "2026-06-21T04:00:00Z"}, - {"offset converts to Z", "DateTime('UTC')", nil, "2026-06-21T06:30:00+02:30", "2026-06-21T04:00:00Z"}, - {"naive space form, column zone", "DateTime('America/New_York')", nil, "2026-06-21 00:00:00", "2026-06-21T04:00:00Z"}, - {"naive T form, column zone", "DateTime('America/New_York')", nil, "2026-06-21T00:00:00", "2026-06-21T04:00:00Z"}, - {"naive form, Etc/UTC column zone", "DateTime('Etc/UTC')", nil, "2026-06-21 04:00:00", "2026-06-21T04:00:00Z"}, - {"naive form, server zone", "DateTime", nyc, "2026-06-21 00:00:00", "2026-06-21T04:00:00Z"}, - {"naive form, unknown server zone ⇒ passes through", "DateTime", nil, "2026-06-21 04:00:00", "2026-06-21 04:00:00"}, - {"offset form, unknown server zone still canonicalizes", "DateTime", nil, "2026-06-21T06:00:00+02:00", "2026-06-21T04:00:00Z"}, - {"date-only ⇒ midnight in zone", "DateTime('America/New_York')", nil, "2026-06-21", "2026-06-21T04:00:00Z"}, - {"unix seconds number", "DateTime('UTC')", nil, float64(1782014400), "2026-06-21T04:00:00Z"}, - {"unix seconds json.Number", "DateTime('UTC')", nil, json.Number("1782014400"), "2026-06-21T04:00:00Z"}, - {"unix seconds digit-string", "DateTime('UTC')", nil, "1782014400", "2026-06-21T04:00:00Z"}, - {"9-digit unix string", "DateTime('UTC')", nil, "999999999", "2001-09-09T01:46:39Z"}, - {"fractional unix digit-string", "DateTime64(3, 'UTC')", nil, "1782014400.5", "2026-06-21T04:00:00.5Z"}, - {"nanosecond fraction parsed exactly, not via float64", "DateTime64(9, 'UTC')", nil, "1782014400.123456789", "2026-06-21T04:00:00.123456789Z"}, - {"fraction digits beyond nine truncate, never round", "DateTime64(9, 'UTC')", nil, "1782014400.9999999995", "2026-06-21T04:00:00.999999999Z"}, - // Integer numbers are ClickHouse *ticks* at the column scale on DateTime64 - // — the ms epoch is the natural producer shape there, and an epoch-seconds - // number really is a 1970 instant (what the insert stores either way). - {"epoch-ms number is ticks on DateTime64(3)", "DateTime64(3, 'UTC')", nil, float64(1782014400000), "2026-06-21T04:00:00Z"}, - {"epoch-ms json.Number with sub-second ticks", "DateTime64(3, 'UTC')", nil, json.Number("1782014400123"), "2026-06-21T04:00:00.123Z"}, - {"epoch-seconds number on DateTime64(3) is a 1970 instant", "DateTime64(3, 'UTC')", nil, float64(1782014400), "1970-01-21T15:00:14.4Z"}, - {"number on DateTime64(0) is seconds (scale 1)", "DateTime64(0, 'UTC')", nil, float64(1782014400), "2026-06-21T04:00:00Z"}, - {"fraction truncated to column precision", "DateTime64(3, 'UTC')", nil, "2026-06-21T04:00:00.123456Z", "2026-06-21T04:00:00.123Z"}, - {"fraction truncated off a second-precision column", "DateTime('UTC')", nil, "2026-06-21 04:00:00.999", "2026-06-21T04:00:00Z"}, - {"trailing fractional zeros trimmed, like /v1/query", "DateTime64(3, 'UTC')", nil, "2026-06-21T04:00:00.120Z", "2026-06-21T04:00:00.12Z"}, - {"Nullable unwraps", "Nullable(DateTime('UTC'))", nil, "2026-06-21 04:00:00", "2026-06-21T04:00:00Z"}, - {"null left for ClickHouse", "Nullable(DateTime)", nil, nil, nil}, - {"String column untouched", "String", nil, "2026-06-21 04:00:00", "2026-06-21 04:00:00"}, - {"Date column untouched (excluded)", "Date", nil, "2026-06-21", "2026-06-21"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - // The production path: specs resolved once at schema-build time. - schema := tsSchema(tt.colType) - resolveTimestampSpecs(schema, tt.serverTZ, discardLogger()) - data := map[string]any{"ts": tt.value} - CanonicalizeTimestamps(schema, data) - assert.Equal(t, tt.want, data["ts"]) - }) - } -} - -// TestColumnTimeParser: the stream row-filter's per-column parser (#381) is the -// canonicalization grammar exactly — same spellings, zone rule, and Unix forms — -// truncated to the column's precision and bounded by the rewrite range, so a -// filter constant and a canonicalized payload always meet on the instant -// ClickHouse stores. -func TestColumnTimeParser(t *testing.T) { - t.Parallel() - utc4 := time.Date(2026, 6, 21, 4, 0, 0, 0, time.UTC) - - tests := []struct { - name string - colType string - value any - want time.Time - ok bool - }{ - {"canonical RFC 3339", "DateTime('UTC')", "2026-06-21T04:00:00Z", utc4, true}, - {"zone-less read in column zone", "DateTime('UTC')", "2026-06-21 04:00:00", utc4, true}, - {"explicit offset, same instant", "DateTime('UTC')", "2026-06-21T06:00:00+02:00", utc4, true}, - {"unix seconds string", "DateTime('UTC')", "1782014400", utc4, true}, - {"unix seconds number", "DateTime('UTC')", json.Number("1782014400"), utc4, true}, - {"fraction truncated to column precision", "DateTime64(1, 'UTC')", "2026-06-21T04:00:00.19Z", utc4.Add(100 * time.Millisecond), true}, - {"junk refused", "DateTime('UTC')", "not a timestamp", time.Time{}, false}, - {"out of range refused (insert-time saturation would move it)", "DateTime('UTC')", "2400-01-01T00:00:00Z", time.Time{}, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - schema := tsSchema(tt.colType) - resolveTimestampSpecs(schema, nil, discardLogger()) - parse := schema.Columns[0].TimeParser() - require.NotNil(t, parse) - got, ok := parse(tt.value) - assert.Equal(t, tt.ok, ok) - if tt.ok { - assert.True(t, got.Equal(tt.want), "got %v, want %v", got, tt.want) - } - }) - } -} - -// TestColumnTimeParser_NilOrZoneLimited: only DateTime/DateTime64 columns with a -// resolved spec carry a parser — String/Date columns and hand-built literals -// return nil (byte-equality semantics on the stream). A timestamp column whose -// zone is unknown still parses zone-explicit forms but refuses zone-less strings: -// the zone would be a guess, and a guessed instant could move a row across a -// filter boundary. -func TestColumnTimeParser_NilOrZoneLimited(t *testing.T) { - t.Parallel() - schema := &TableSchema{Name: "t", Columns: []Column{ - {Name: "s", Type: "String"}, - {Name: "d", Type: "Date"}, - {Name: "ts", Type: "DateTime"}, - }} - resolveTimestampSpecs(schema, nil, discardLogger()) - assert.Nil(t, schema.Columns[0].TimeParser(), "String column: no parser") - assert.Nil(t, schema.Columns[1].TimeParser(), "Date column: excluded from timestamp handling") - assert.Nil(t, tsSchema("DateTime").Columns[0].TimeParser(), "hand-built literal without spec resolution: no parser") - - unknownZone := schema.Columns[2].TimeParser() - require.NotNil(t, unknownZone, "zone-less DateTime with unknown server zone still has a (zone-explicit-only) parser") - _, ok := unknownZone("2026-06-21 04:00:00") - assert.False(t, ok, "zone-less string with unknown column zone: refused, never guessed") - got, ok := unknownZone("2026-06-21T04:00:00Z") - assert.True(t, ok) - assert.True(t, got.Equal(time.Date(2026, 6, 21, 4, 0, 0, 0, time.UTC))) -} - -// TestCanonicalizeTimestamps_NoPrecomputedSpec: a schema that skipped spec -// resolution (hand-built literals) passes through untouched. -func TestCanonicalizeTimestamps_NoPrecomputedSpec(t *testing.T) { - t.Parallel() - data := map[string]any{"ts": "2026-06-21 04:00:00"} - CanonicalizeTimestamps(tsSchema("DateTime"), data) - assert.Equal(t, "2026-06-21 04:00:00", data["ts"]) -} - -// TestCanonicalizeTimestamps_AbsentColumn: a column not in the payload (DEFAULT- -// filled by ClickHouse) is left absent, never invented. -func TestCanonicalizeTimestamps_AbsentColumn(t *testing.T) { - t.Parallel() - schema := &TableSchema{Name: "t", Columns: []Column{ - {Name: "ts", Type: "DateTime", HasDefault: true}, - {Name: "page", Type: "String"}, - }} - resolveTimestampSpecs(schema, time.UTC, discardLogger()) - data := map[string]any{"page": "/home"} - CanonicalizeTimestamps(schema, data) - assert.Equal(t, map[string]any{"page": "/home"}, data) -} - -// TestCanonicalizeTimestamps_Unparseable_PassThrough: fail-open — unparseable -// values and unresolvable column specs pass through verbatim. -func TestCanonicalizeTimestamps_Unparseable_PassThrough(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - colType string - value any - }{ - {"unrecognized string", "DateTime('UTC')", "banana"}, - {"non-finite numeric string is not an instant", "DateTime('UTC')", "NaN"}, - {"wrong value type", "DateTime('UTC')", true}, - {"unknown zone in type", "DateTime('Not/AZone')", "2026-06-21 04:00:00"}, - {"malformed DateTime64 precision", "DateTime64(x)", "2026-06-21 04:00:00"}, - // Digit-strings outside the 9–10 digit Unix shape mean calendar forms - // (or nothing) to ClickHouse best_effort — parsing them as Unix seconds - // would store a different instant than the insert (PR #402 review). - {"8-digit string is YYYYMMDD to ClickHouse", "DateTime('UTC')", "20260711"}, - {"12-digit string (ClickHouse rejects)", "DateTime('UTC')", "202607111500"}, - {"14-digit string is YYYYMMDDhhmmss to ClickHouse", "DateTime('UTC')", "20260711150000"}, - {"4-digit string is a year to ClickHouse", "DateTime('UTC')", "2026"}, - {"11-digit string", "DateTime('UTC')", "17504784000"}, - {"13-digit string is ClickHouse's ms epoch, not ours", "DateTime('UTC')", "1752278400000"}, - {"16-digit string is ClickHouse's µs epoch, not ours", "DateTime('UTC')", "1750478400123456"}, - {"scientific notation is not a timestamp", "DateTime('UTC')", "1e9"}, - {"negative digit-string", "DateTime('UTC')", "-100"}, - {"empty fraction", "DateTime('UTC')", "1750478400."}, - // ClickHouse consumes a fraction after a Unix epoch only for DateTime64 - // targets; on plain DateTime the leftover fraction fails the row. - {"fractional unix string on DateTime", "DateTime('UTC')", "1782014400.5"}, - // ClickHouse has no ',' decimal separator (Go's RFC3339Nano accepts one - // per ISO 8601) — rewriting would insert a row ClickHouse rejects raw. - {"comma fraction", "DateTime('UTC')", "2026-06-21T04:00:00,999Z"}, - {"comma fraction on DateTime64", "DateTime64(3, 'UTC')", "2026-06-21T04:00:00,9Z"}, - // ClickHouse rejects non-integer JSON numbers for every DateTime kind. - {"non-integer number", "DateTime64(3, 'UTC')", 1782014400.5}, - {"non-integer number on DateTime", "DateTime('UTC')", 1782014400.5}, - {"negative number", "DateTime('UTC')", float64(-100)}, - {"json.Number with exponent", "DateTime('UTC')", json.Number("1.5e9")}, - // Out of the column kind's range: ClickHouse saturates, and saturation is - // spelling-dependent (local time-of-day is kept while the date clamps), so - // no rewrite is safe — the raw spelling must be the one that saturates. - {"pre-epoch instant on DateTime", "DateTime('UTC')", "1960-01-01T00:00:00Z"}, - {"beyond UInt32 seconds on DateTime", "DateTime('UTC')", "2107-01-01T00:00:00Z"}, - {"number beyond UInt32 seconds on DateTime", "DateTime('UTC')", float64(4294967296)}, - {"beyond 2299 on DateTime64", "DateTime64(3, 'UTC')", "2300-06-30 12:30:00"}, - {"beyond the Int64-ns ceiling on DateTime64(9)", "DateTime64(9, 'UTC')", "2280-01-01T00:00:00Z"}, - // Valid RFC 3339 the subset deliberately omits (Go rejects :60). - {"leap-second spelling", "DateTime('UTC')", "2016-12-31T23:59:60Z"}, - // "" and "Local" are Go LoadLocation quirks (UTC / process env), not - // zone declarations — strict: unresolvable, pass through. - {"empty zone name in type", "DateTime('')", "2026-06-21 04:00:00"}, - {"Local zone in type", "DateTime('Local')", "2026-06-21 04:00:00"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - schema := tsSchema(tt.colType) - resolveTimestampSpecs(schema, time.UTC, discardLogger()) - data := map[string]any{"ts": tt.value} - CanonicalizeTimestamps(schema, data) - assert.Equal(t, tt.value, data["ts"]) - }) - } -} - -// TestResolveTimestampSpecs: timestamp columns get a spec (own zone, else server -// default); others don't; an unresolvable zone degrades to nil, not a failure. -func TestResolveTimestampSpecs(t *testing.T) { - t.Parallel() - nyc, err := time.LoadLocation("America/New_York") - require.NoError(t, err) - - schema := &TableSchema{Name: "t", Columns: []Column{ - {Name: "plain", Type: "DateTime"}, - {Name: "zoned", Type: "DateTime64(3, 'America/New_York')"}, - {Name: "page", Type: "String"}, - {Name: "broken", Type: "DateTime('Not/AZone')"}, - {Name: "etc_utc", Type: "DateTime('Etc/UTC')"}, - }} - resolveTimestampSpecs(schema, nyc, discardLogger()) - - require.NotNil(t, schema.Columns[0].tsSpec) - assert.Equal(t, nyc, schema.Columns[0].tsSpec.loc, "zone-less column takes the server zone") - assert.False(t, schema.Columns[0].tsSpec.isDT64, "DateTime is not kind DateTime64") - require.NotNil(t, schema.Columns[1].tsSpec) - assert.Equal(t, nyc, schema.Columns[1].tsSpec.loc) - assert.Equal(t, 3, schema.Columns[1].tsSpec.precision) - assert.True(t, schema.Columns[1].tsSpec.isDT64, "numbers and unix fractions follow the DateTime64 rules") - assert.Nil(t, schema.Columns[2].tsSpec, "non-timestamp column gets no spec") - assert.Nil(t, schema.Columns[3].tsSpec, "unresolvable zone degrades to nil, not a failed build") - require.NotNil(t, schema.Columns[4].tsSpec) - assert.Same(t, time.UTC, schema.Columns[4].tsSpec.loc, "Etc/UTC maps to UTC without a tzdata lookup") - - // The degraded column passes through untouched — fail-open, never a rejection. - data := map[string]any{"broken": "2026-06-21 04:00:00"} - CanonicalizeTimestamps(schema, data) - assert.Equal(t, "2026-06-21 04:00:00", data["broken"]) -} - -// TestRefresh_PrecomputesSpecs: schema builds resolve timestamp specs with the -// server zone from SELECT timezone(), so registry consumers — production and -// the testutil mock-conn path alike — exercise the same precomputed path. -func TestRefresh_PrecomputesSpecs(t *testing.T) { - t.Parallel() - conn := &fakeConn{columns: []fakeColumn{{table: "t", name: "ts", chType: "DateTime", position: 1}}} - reg := NewSchemaRegistry(conn, func() string { return "test" }, func() time.Duration { return time.Hour }, discardLogger()) - require.NoError(t, reg.Refresh(context.Background())) - col := reg.Get("t").Columns[0] - require.NotNil(t, col.tsSpec) - assert.Equal(t, time.UTC, col.tsSpec.loc) -} - -// discardLogger mirrors the registries' test logger: spec-resolution warnings are -// asserted via behavior (nil specs), not log output. -func discardLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} diff --git a/internal/discovery/validation.go b/internal/discovery/validation.go deleted file mode 100644 index 6fa1c556..00000000 --- a/internal/discovery/validation.go +++ /dev/null @@ -1,308 +0,0 @@ -package discovery - -import ( - "encoding/json" - "fmt" - "strconv" - "strings" -) - -// Validate checks that the given data matches the table schema. -// It rejects unknown fields and checks type compatibility. -func Validate(schema *TableSchema, data map[string]any) error { - colMap := make(map[string]Column, len(schema.Columns)) - for _, col := range schema.Columns { - colMap[col.Name] = col - } - - // TODO: the unknown field rejection can technically be controlled with `input_format_skip_unknown_fields = 1` I think, which would mean this is a false negative in some cases... - - // Reject unknown fields. - for key := range data { - col, ok := colMap[key] - if !ok { - return fmt.Errorf("unknown column %q for table %q", key, schema.Name) - } - // A computed column has no writable storage: ClickHouse refuses to - // insert a MATERIALIZED column and does not resolve an ALIAS one at - // all. The published row carries only insertable columns, so a value - // supplied for one of these would otherwise be dropped on the way out - // and the record would insert as though it had never been sent. - // Refuse it here instead, where the caller still hears about it. - if !col.IsInsertable() { - return fmt.Errorf("column %q of table %q is %s and cannot be inserted", - key, schema.Name, strings.ToLower(col.DefaultKind)) - } - } - - // TODO: I think clickhouse actually implicitly has defaults for strings, numbers etc like "" and 0 – so (if true) then omitting that column, even if the schema doesn't set that column to nullable or have a default, clickhouse will still set the implicit value if one – so technically we then shouldn't reject missing columns and this is a false negative? But that get's quite a bit messier... - - // Check type compatibility and required columns. - for _, col := range schema.Columns { - val, provided := data[col.Name] - if !provided { - if !col.IsNullable && !col.HasDefault { - return fmt.Errorf("missing required column %q for table %q", col.Name, schema.Name) - } - continue - } - if val == nil { - if !col.IsNullable && !col.HasDefault { - return fmt.Errorf("null value for non-nullable column %q", col.Name) - } - continue - } - if !isTypeCompatible(col.Type, val) { - return fmt.Errorf("type mismatch for column %q: cannot store %T in %s", col.Name, val, col.Type) - } - } - - return nil -} - -// unwrapType strips Nullable(...) and LowCardinality(...) modifiers (nested in any -// order, e.g. LowCardinality(Nullable(String))) down to the base ClickHouse type. -func unwrapType(chType string) string { - for { - if strings.HasPrefix(chType, "Nullable(") && strings.HasSuffix(chType, ")") { - chType = chType[9 : len(chType)-1] - continue - } - if strings.HasPrefix(chType, "LowCardinality(") && strings.HasSuffix(chType, ")") { - chType = chType[15 : len(chType)-1] - continue - } - return chType - } -} - -// isTypeCompatible checks whether a Go/JSON value can be stored in the given ClickHouse type. -func isTypeCompatible(chType string, val any) bool { - chType = unwrapType(chType) - - switch { - // String-compatible types accept Strings, Numbers (coerced), and Bools - case chType == "String", - strings.HasPrefix(chType, "FixedString("), - chType == "UUID": - switch val.(type) { - case string, float64, json.Number, bool: - return true - default: - return false - } - - // Dates/Times accept Strings and Numbers (Unix timestamps) - case strings.HasPrefix(chType, "DateTime"), - strings.HasPrefix(chType, "Date"): - switch val.(type) { - case string, float64, json.Number: - return true - default: - return false - } - - // Enums accept Strings (names) and Numbers (integer mappings) - case strings.HasPrefix(chType, "Enum8("), - strings.HasPrefix(chType, "Enum16("): - switch val.(type) { - case string, float64, json.Number: - return true - default: - return false - } - - // IPs accept Strings and Numbers (UInt32 representations) - case chType == "IPv4", chType == "IPv6": - switch val.(type) { - case string, float64, json.Number: - return true - default: - return false - } - - // Bools accept actual bools, numbers (0/1), and strings ("true"/"false") - case chType == "Bool": - switch val.(type) { - case bool, float64, json.Number, string: - return true - default: - return false - } - - // Numerics accept Numbers and Strings (to prevent JS precision loss) - case isNumericType(chType): - switch val.(type) { - case float64, json.Number, string: - return true - default: - return false - } - - // Array types accept JSON arrays. - case strings.HasPrefix(chType, "Array("): - _, ok := val.([]any) - return ok - - // Map types accept JSON objects. - case strings.HasPrefix(chType, "Map("): - _, ok := val.(map[string]any) - return ok - - // Tuple types accept JSON arrays or objects. - case strings.HasPrefix(chType, "Tuple("): - _, okArr := val.([]any) - _, okMap := val.(map[string]any) - return okArr || okMap - - default: - // Unknown type — accept any value and let ClickHouse validate. - return true - } -} - -// IsNumericType reports whether chType is a ClickHouse numeric type (integer, float, -// or decimal), unwrapping Nullable/LowCardinality modifiers first. The stream -// row-filter evaluator classifies such columns numeric-comparable, so ordering -// predicates (>, <) on numbers match ClickHouse (9 < 100). -func IsNumericType(chType string) bool { - return isNumericType(unwrapType(chType)) -} - -// IsStringType reports whether chType is a ClickHouse String (unwrapping -// Nullable/LowCardinality). For String columns, byte comparison IS ClickHouse -// comparison — equality and lexicographic order alike — so the stream row-filter -// evaluator can compare them exactly. FixedString is deliberately excluded: its -// stored values are zero-padded to the declared width, so a byte comparison of an -// ingested value against a filter constant would not match ClickHouse. -func IsStringType(chType string) bool { - return unwrapType(chType) == "String" -} - -// NumericStorage describes how a ClickHouse numeric column stores a value — -// the narrowing AND range the stream row-filter must apply to BOTH comparison -// operands so its verdicts match the query path, where ClickHouse narrows the -// stored value at insert and the bound constant at compare, and errors the -// query outright on a constant outside the column's range. Each family carries -// its parameters: Integer + IntBits/Unsigned for Int*/UInt* (exact within the -// width's range), FloatBits (32/64) for Float*, Precision+Scale for Decimal*. -type NumericStorage struct { - Integer bool - IntBits int - Unsigned bool - FloatBits int - Precision int - Scale int -} - -// NumericStorageOf classifies chType's numeric storage model, unwrapping -// Nullable/LowCardinality. ok=false for non-numeric types AND for a Decimal -// whose precision/scale cannot be parsed — the caller must then refuse numeric -// comparison rather than compare under a guessed model (fail closed). -// system.columns always reports the two-argument canonical Decimal(P, S) form -// (Decimal32(4) is stored as Decimal(9, 4)); the shorthand widths are handled -// anyway for robustness, and a bare single-argument Decimal(P) is refused -// rather than misread. -func NumericStorageOf(chType string) (NumericStorage, bool) { - chType = unwrapType(chType) - switch { - case !isNumericType(chType): - return NumericStorage{}, false - case chType == "Float32": - return NumericStorage{FloatBits: 32}, true - case chType == "Float64": - return NumericStorage{FloatBits: 64}, true - case strings.HasPrefix(chType, "Decimal"): - p, s, ok := decimalParams(chType) - if !ok { - return NumericStorage{}, false - } - return NumericStorage{Precision: p, Scale: s}, true - default: // isNumericType admits only Int*/UInt* beyond the cases above - bits, unsigned, ok := integerWidth(chType) - if !ok { - return NumericStorage{}, false - } - return NumericStorage{Integer: true, IntBits: bits, Unsigned: unsigned}, true - } -} - -// integerWidth reads the bit width and signedness from an Int*/UInt* type name. -func integerWidth(chType string) (bits int, unsigned bool, ok bool) { - rest, found := strings.CutPrefix(chType, "UInt") - if found { - unsigned = true - } else { - rest, found = strings.CutPrefix(chType, "Int") - if !found { - return 0, false, false - } - } - bits, err := strconv.Atoi(rest) - if err != nil { - return 0, false, false - } - switch bits { - case 8, 16, 32, 64, 128, 256: - return bits, unsigned, true - default: - return 0, false, false - } -} - -// decimalParams extracts (P, S) from Decimal(P, S) and the DecimalN(S) -// shorthands (Decimal32/64/128/256, whose precisions are fixed at 9/18/38/76). -// ClickHouse bounds them to 1 ≤ P ≤ 76 and 0 ≤ S ≤ P; anything outside that, -// malformed, or a single-argument Decimal(P) — whose lone number is a -// precision, not a scale — reports ok=false. -func decimalParams(chType string) (precision, scale int, ok bool) { - open := strings.IndexByte(chType, '(') - if open < 0 || !strings.HasSuffix(chType, ")") { - return 0, 0, false - } - args := strings.Split(chType[open+1:len(chType)-1], ",") - last, err := strconv.Atoi(strings.TrimSpace(args[len(args)-1])) - if err != nil { - return 0, 0, false - } - switch prefix := chType[:open]; prefix { - case "Decimal32": - precision = 9 - case "Decimal64": - precision = 18 - case "Decimal128": - precision = 38 - case "Decimal256": - precision = 76 - case "Decimal": - if len(args) != 2 { - return 0, 0, false - } - if precision, err = strconv.Atoi(strings.TrimSpace(args[0])); err != nil { - return 0, 0, false - } - default: - return 0, 0, false - } - scale = last - if precision < 1 || precision > 76 || scale < 0 || scale > precision { - return 0, 0, false - } - return precision, scale, true -} - -// isNumericType returns true for ClickHouse integer, float, and decimal types. -func isNumericType(chType string) bool { - switch { - case chType == "UInt8", chType == "UInt16", chType == "UInt32", chType == "UInt64", chType == "UInt128", chType == "UInt256": - return true - case chType == "Int8", chType == "Int16", chType == "Int32", chType == "Int64", chType == "Int128", chType == "Int256": - return true - case chType == "Float32", chType == "Float64": - return true - case strings.HasPrefix(chType, "Decimal"): - return true - default: - return false - } -} diff --git a/internal/discovery/validation_test.go b/internal/discovery/validation_test.go deleted file mode 100644 index 6028d232..00000000 --- a/internal/discovery/validation_test.go +++ /dev/null @@ -1,358 +0,0 @@ -package discovery - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValidate(t *testing.T) { - t.Parallel() - - // A shared schema used across multiple tests - baseSchema := &TableSchema{ - Name: "clicks", - Columns: []Column{ - {Name: "user_id", Type: "String", IsNullable: false}, - {Name: "amount", Type: "Float64", IsNullable: false}, - {Name: "notes", Type: "Nullable(String)", IsNullable: true}, - {Name: "created_at", Type: "DateTime64(3, 'UTC')", HasDefault: true}, - }, - } - - tests := []struct { - name string - schema *TableSchema - data map[string]any - wantErr string // Substring to match in error; empty means success expected - }{ - { - name: "valid data all fields", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - "notes": "hello", - "created_at": "2025-01-01T00:00:00Z", - }, - }, - { - name: "unknown field rejected", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - "unknown": "value", - }, - wantErr: "unknown column", - }, - { - name: "type mismatch", - schema: baseSchema, - data: map[string]any{ - "user_id": json.Number("123"), // json.Number is valid for String - "amount": []any{"arrays", "fail"}, // invalid for Float64 - }, - wantErr: "type mismatch", - }, - { - name: "missing required column", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - // amount is missing, not nullable, no default - }, - wantErr: "missing required column", - }, - { - name: "missing nullable column is allowed", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - // notes is omitted - }, - }, - { - name: "missing default column is allowed", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - // created_at is omitted - }, - }, - { - name: "nil for non-nullable rejected", - schema: baseSchema, - data: map[string]any{ - "user_id": nil, // non-nullable - "amount": json.Number("42.5"), - }, - wantErr: "null value for non-nullable", - }, - { - name: "nil for nullable allowed", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - "notes": nil, - }, - }, - { - name: "nil for non-nullable with default allowed", - schema: baseSchema, - data: map[string]any{ - "user_id": "alice", - "amount": json.Number("42.5"), - "created_at": nil, - }, - }, - { - name: "nil data triggers missing column", - schema: baseSchema, - data: nil, - wantErr: "missing required column", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := Validate(tt.schema, tt.data) - if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestIsTypeCompatible(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - chType string - val any - want bool - }{ - // String and Date types - {"String accepts string", "String", "hello", true}, - {"String accepts float64", "String", 42.0, true}, - {"String accepts json.Number", "String", json.Number("42"), true}, - {"String accepts bool", "String", true, true}, - {"String rejects array", "String", []any{}, false}, - {"DateTime accepts string", "DateTime64(3, 'UTC')", "2025-01-01", true}, - {"DateTime accepts number", "DateTime64", json.Number("1716570889"), true}, - - // Numeric types - {"UInt64 accepts float64", "UInt64", 42.0, true}, - {"UInt64 accepts json.Number", "UInt64", json.Number("1234567890"), true}, - {"UInt64 accepts string", "UInt64", "1234567890", true}, - {"Decimal accepts string", "Decimal(18,4)", "42.5000", true}, - {"Float64 rejects bool", "Float64", true, false}, - - // Bool - {"Bool accepts bool", "Bool", true, true}, - {"Bool accepts float64", "Bool", 1.0, true}, - {"Bool accepts json.Number", "Bool", json.Number("0"), true}, - {"Bool accepts string", "Bool", "true", true}, - {"Bool rejects array", "Bool", []any{}, false}, - - // Enums & IPs - {"Enum accepts string", "Enum16('a'=1,'b'=2)", "a", true}, - {"Enum accepts number", "Enum8('a'=1)", json.Number("1"), true}, - {"IPv4 accepts string", "IPv4", "192.168.1.1", true}, - {"IPv4 accepts number", "IPv4", json.Number("3232235777"), true}, - - // Complex Types - {"Array accepts slice", "Array(String)", []any{"a", "b"}, true}, - {"Array rejects string", "Array(String)", "not-an-array", false}, - {"Map accepts map", "Map(String, String)", map[string]any{"k": "v"}, true}, - {"Tuple accepts slice", "Tuple(String, Int32)", []any{"a", 1.0}, true}, - {"Tuple accepts map", "Tuple(a String, b Int32)", map[string]any{"a": "x", "b": 1.0}, true}, - - // Modifiers (Nullable / LowCardinality) - {"Nullable accepts valid", "Nullable(String)", "hello", true}, - {"LowCardinality accepts valid", "LowCardinality(String)", "hello", true}, - {"Nested modifiers unwrapped 1", "LowCardinality(Nullable(String))", "hello", true}, - {"Nested modifiers unwrapped 2", "Nullable(LowCardinality(String))", "hello", true}, - {"Nested modifiers reject invalid", "LowCardinality(Nullable(UInt64))", []any{}, false}, - - // Fallback - {"Unknown type accepts everything", "SomeFutureType", []any{"sure"}, true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - got := isTypeCompatible(tt.chType, tt.val) - assert.Equal(t, tt.want, got, "chType=%q, val=%T(%v)", tt.chType, tt.val, tt.val) - }) - } -} - -func TestIsNumericType(t *testing.T) { - t.Parallel() - - tests := []struct { - chType string - want bool - }{ - {"UInt8", true}, - {"UInt256", true}, - {"Int16", true}, - {"Int128", true}, - {"Float32", true}, - {"Float64", true}, - {"Decimal(10,2)", true}, - {"Decimal128(5)", true}, - {"String", false}, - {"Bool", false}, - } - - for _, tt := range tests { - t.Run(tt.chType, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, isNumericType(tt.chType)) - }) - } -} - -// TestIsNumericType_Exported checks the exported wrapper the stream row-filter uses: -// it must unwrap Nullable/LowCardinality (in any nesting) before classifying, so a -// Nullable(UInt64) column still compares numerically. -func TestIsNumericType_Exported(t *testing.T) { - t.Parallel() - - tests := []struct { - chType string - want bool - }{ - {"UInt64", true}, - {"Nullable(UInt64)", true}, - {"LowCardinality(Int32)", true}, - {"LowCardinality(Nullable(Float64))", true}, - {"Decimal(10,2)", true}, - {"String", false}, - {"Nullable(String)", false}, - {"LowCardinality(String)", false}, - {"DateTime", false}, - } - - for _, tt := range tests { - t.Run(tt.chType, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, IsNumericType(tt.chType)) - }) - } -} - -// TestIsStringType: only String (under any Nullable/LowCardinality wrapping) -// qualifies — byte comparison is ClickHouse comparison for it. FixedString is -// excluded on purpose (zero-padded storage), as is everything whose text form is -// not canonical (UUID, Enum, DateTime, Bool). -func TestIsStringType(t *testing.T) { - t.Parallel() - - tests := []struct { - chType string - want bool - }{ - {"String", true}, - {"Nullable(String)", true}, - {"LowCardinality(String)", true}, - {"LowCardinality(Nullable(String))", true}, - {"FixedString(16)", false}, - {"UUID", false}, - {"Enum8('a' = 1)", false}, - {"DateTime", false}, - {"Bool", false}, - {"UInt64", false}, - } - - for _, tt := range tests { - t.Run(tt.chType, func(t *testing.T) { - t.Parallel() - assert.Equal(t, tt.want, IsStringType(tt.chType)) - }) - } -} - -// TestNumericStorageOf pins the storage classification the stream row-filter -// narrows comparisons with: integer family exact at any width, float bit -// widths, Decimal scale extraction across every declaration form, wrappers -// unwrapped, and ok=false for non-numerics and for a Decimal whose scale can't -// be parsed — the caller must refuse comparison rather than guess a model. -func TestNumericStorageOf(t *testing.T) { - t.Parallel() - tests := []struct { - chType string - want NumericStorage - ok bool - }{ - {"UInt64", NumericStorage{Integer: true, IntBits: 64, Unsigned: true}, true}, - {"Int256", NumericStorage{Integer: true, IntBits: 256}, true}, - {"Nullable(UInt32)", NumericStorage{Integer: true, IntBits: 32, Unsigned: true}, true}, - {"Float32", NumericStorage{FloatBits: 32}, true}, - {"LowCardinality(Nullable(Float64))", NumericStorage{FloatBits: 64}, true}, - {"Decimal(10, 2)", NumericStorage{Precision: 10, Scale: 2}, true}, - {"Decimal(10,2)", NumericStorage{Precision: 10, Scale: 2}, true}, - {"Decimal(2, 2)", NumericStorage{Precision: 2, Scale: 2}, true}, - {"Decimal32(4)", NumericStorage{Precision: 9, Scale: 4}, true}, - {"Decimal64(0)", NumericStorage{Precision: 18, Scale: 0}, true}, - {"Decimal256(76)", NumericStorage{Precision: 76, Scale: 76}, true}, - {"Decimal", NumericStorage{}, false}, - {"Decimal(10)", NumericStorage{}, false}, - {"Decimal(10, -1)", NumericStorage{}, false}, - {"Decimal(10, 77)", NumericStorage{}, false}, - {"String", NumericStorage{}, false}, - {"DateTime", NumericStorage{}, false}, - {"Bool", NumericStorage{}, false}, - } - for _, tt := range tests { - t.Run(tt.chType, func(t *testing.T) { - t.Parallel() - got, ok := NumericStorageOf(tt.chType) - assert.Equal(t, tt.ok, ok) - assert.Equal(t, tt.want, got) - }) - } -} - -// TestValidate_RejectsSuppliedComputedColumn: a record naming a MATERIALIZED or -// ALIAS column must be refused where the caller still hears about it. The -// published row carries only insertable columns, so without this the value -// would be silently dropped and the record would insert as though it had never -// been sent — the failure mode that made this class invisible. -func TestValidate_RejectsSuppliedComputedColumn(t *testing.T) { - t.Parallel() - schema := &TableSchema{Name: "clicks", Columns: []Column{ - {Name: "id", Type: "UInt64"}, - {Name: "mat", Type: "String", DefaultKind: "MATERIALIZED", HasDefault: true}, - {Name: "ali", Type: "UInt64", DefaultKind: "ALIAS", HasDefault: true}, - }} - - for _, tt := range []struct{ name, col, want string }{ - {"materialized", "mat", "materialized and cannot be inserted"}, - {"alias", "ali", "alias and cannot be inserted"}, - } { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - err := Validate(schema, map[string]any{"id": float64(1), tt.col: "x"}) - require.Error(t, err) - assert.Contains(t, err.Error(), tt.col) - assert.Contains(t, err.Error(), tt.want) - }) - } - - // Omitting them is the normal case and must still pass: they are not - // "missing required columns", they are computed by the server. - require.NoError(t, Validate(schema, map[string]any{"id": float64(1)})) -} diff --git a/internal/ingest/compact.go b/internal/ingest/compact.go deleted file mode 100644 index e3ff6e75..00000000 --- a/internal/ingest/compact.go +++ /dev/null @@ -1,52 +0,0 @@ -package ingest - -import ( - "bytes" - "encoding/json" - "fmt" - - "github.com/Wave-RF/WaveHouse/internal/discovery" -) - -// EncodeCompactRow renders one record as a single JSONCompactEachRow line: a -// JSON array carrying exactly one value per INSERTABLE column, in declaration order -// (discovery orders TableSchema.Columns by system.columns.position). A column -// the record does not carry encodes as null. For a NON-nullable column with a -// default the insert turns that back into the default -// (input_format_null_as_default); on a NULLABLE column ClickHouse stores the -// NULL, because only an ABSENT key ever took the default and a positional row -// cannot express absence. worker.go pins -// input_format_null_as_default=1 explicitly, alongside date_time_input_format, -// so a server-default change cannot silently alter either. -// The result has no trailing newline — the caller joins lines. -// -// This is serialization ONLY. It performs no validation and makes no decision -// about a value: schema validation upstream has already rejected unknown keys -// and unacceptable types, so there is nothing here to reject and nothing to -// coerce. Record values arrive json.Number-preserving (every decoder on the -// ingest path sets UseNumber), and json.Marshal writes a json.Number as its -// exact digits, so a 64-bit id past 2^53 keeps every one of them. -// -// Transitional: replaced by chtypes RowsExport; serialization only, never add -// rules here. -func EncodeCompactRow(orderedCols []discovery.Column, record map[string]any) (json.RawMessage, error) { - var buf bytes.Buffer - buf.WriteByte('[') - for i, col := range orderedCols { - if i > 0 { - buf.WriteByte(',') - } - v, ok := record[col.Name] - if !ok { - buf.WriteString("null") - continue - } - b, err := json.Marshal(v) - if err != nil { - return nil, fmt.Errorf("encode column %q: %w", col.Name, err) - } - buf.Write(b) - } - buf.WriteByte(']') - return json.RawMessage(buf.Bytes()), nil -} diff --git a/internal/ingest/compact_test.go b/internal/ingest/compact_test.go deleted file mode 100644 index 40ac676e..00000000 --- a/internal/ingest/compact_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package ingest - -import ( - "encoding/json" - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/Wave-RF/WaveHouse/internal/discovery" -) - -func cols(names ...string) []discovery.Column { - out := make([]discovery.Column, 0, len(names)) - for i, n := range names { - out = append(out, discovery.Column{Name: n, Type: "String", Position: uint64(i + 1)}) - } - return out -} - -// TestEncodeCompactRow_DeclarationOrder: the array follows the SCHEMA's order, -// not the record's — a Go map has none, which is the whole reason the column -// list travels separately from the row. -func TestEncodeCompactRow_DeclarationOrder(t *testing.T) { - t.Parallel() - line, err := EncodeCompactRow(cols("page", "button", "country"), map[string]any{ - "country": "US", - "page": "/home", - "button": "signup", - }) - require.NoError(t, err) - assert.JSONEq(t, `["/home","signup","US"]`, string(line)) - assert.Equal(t, `["/home","signup","US"]`, string(line), "positional output must be byte-exact, not merely equivalent") -} - -// TestEncodeCompactRow_MissingColumnIsNull: a column the record omits holds a -// position in the array — dropping it would shift every value after it. -func TestEncodeCompactRow_MissingColumnIsNull(t *testing.T) { - t.Parallel() - line, err := EncodeCompactRow(cols("page", "button", "country"), map[string]any{ - "page": "/home", - "country": "US", - }) - require.NoError(t, err) - assert.Equal(t, `["/home",null,"US"]`, string(line)) -} - -// TestEncodeCompactRow_ExplicitNullAndMissingAgree: a column present as JSON -// null encodes the same as one that is absent, so the insert path treats them -// alike. -func TestEncodeCompactRow_ExplicitNullAndMissingAgree(t *testing.T) { - t.Parallel() - explicit, err := EncodeCompactRow(cols("page", "button"), map[string]any{"page": "/a", "button": nil}) - require.NoError(t, err) - absent, err := EncodeCompactRow(cols("page", "button"), map[string]any{"page": "/a"}) - require.NoError(t, err) - assert.Equal(t, string(absent), string(explicit)) -} - -// TestEncodeCompactRow_NumberFidelity: a json.Number keeps its exact digits, so -// a 64-bit id past 2^53 survives the round trip a float64 would round off. -func TestEncodeCompactRow_NumberFidelity(t *testing.T) { - t.Parallel() - const bigID = "9007199254740993" // 2^53 + 1: not representable as a float64 - line, err := EncodeCompactRow(cols("id", "ratio"), map[string]any{ - "id": json.Number(bigID), - "ratio": json.Number("1.500"), - }) - require.NoError(t, err) - assert.Equal(t, `[9007199254740993,1.500]`, string(line), - "json.Number writes its exact digits, trailing zeros and all") - - // The same value decoded as a float64 would not survive — the guard the - // UseNumber decoders on the ingest path exist for. - lossy, err := EncodeCompactRow(cols("id"), map[string]any{"id": float64(9007199254740993)}) - require.NoError(t, err) - assert.NotEqual(t, "["+bigID+"]", string(lossy)) -} - -// TestEncodeCompactRow_EmptySchema: no columns is an empty array, never a bare -// or malformed line. -func TestEncodeCompactRow_EmptySchema(t *testing.T) { - t.Parallel() - for _, record := range []map[string]any{nil, {}, {"stray": 1}} { - line, err := EncodeCompactRow(nil, record) - require.NoError(t, err) - assert.Equal(t, `[]`, string(line)) - } -} - -// TestEncodeCompactRow_NoTrailingNewline: the caller joins lines, so a line -// must not carry its own terminator. -func TestEncodeCompactRow_NoTrailingNewline(t *testing.T) { - t.Parallel() - line, err := EncodeCompactRow(cols("page"), map[string]any{"page": "/a"}) - require.NoError(t, err) - assert.NotContains(t, string(line), "\n") - assert.Equal(t, byte(']'), line[len(line)-1]) -} - -// TestEncodeCompactRow_StructuredAndUnicodeValues: arrays, maps, and non-ASCII -// text pass through as the JSON they are — the encoder judges no value. -func TestEncodeCompactRow_StructuredAndUnicodeValues(t *testing.T) { - t.Parallel() - line, err := EncodeCompactRow(cols("tags", "attrs", "label"), map[string]any{ - "tags": []any{"a", "b"}, - "attrs": map[string]any{"k": "v"}, - "label": "héllo · 世界", - }) - require.NoError(t, err) - - var decoded []any - require.NoError(t, json.Unmarshal(line, &decoded)) - require.Len(t, decoded, 3) - assert.Equal(t, []any{"a", "b"}, decoded[0]) - assert.Equal(t, map[string]any{"k": "v"}, decoded[1]) - assert.Equal(t, "héllo · 世界", decoded[2]) -} - -// TestEncodeCompactRow_UnmarshalableValue: a value encoding/json cannot render -// is an error naming the column, not a silently dropped or shifted field. -func TestEncodeCompactRow_UnmarshalableValue(t *testing.T) { - t.Parallel() - _, err := EncodeCompactRow(cols("page", "score"), map[string]any{ - "page": "/a", - "score": math.Inf(1), // JSON has no infinity - }) - require.Error(t, err) - assert.Contains(t, err.Error(), `"score"`) -} diff --git a/internal/ingest/worker.go b/internal/ingest/worker.go index 59600aa8..83baa4fa 100644 --- a/internal/ingest/worker.go +++ b/internal/ingest/worker.go @@ -19,6 +19,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/chsql" "github.com/Wave-RF/WaveHouse/internal/mq" "github.com/Wave-RF/WaveHouse/internal/query" + "github.com/Wave-RF/WaveHouse/internal/typelayer" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "go.opentelemetry.io/otel" @@ -579,20 +580,33 @@ func (w *IngestWorker) insertToClickHouse(ctx context.Context, tableName string, q.Set("database", t.Database) q.Set("param_target_table", tableName) q.Set("query", fmt.Sprintf("INSERT INTO {target_table:Identifier} (%s) FORMAT JSONCompactEachRow", strings.Join(quoted, ", "))) - q.Set("date_time_input_format", "best_effort") - // A field the record omitted rides as an explicit null in its column's slot, - // because a positional row has one value per column and no way to say - // "absent". For a NON-nullable column with a default this setting turns that - // null back into the default, matching what omitting the key did under - // JSONEachRow. It is already the server default (verified on 26.6.3), so - // this is belt-and-braces for a server configured otherwise. + + // The PARSING settings come from typelayer, not from literals here: chtypes + // ruled on these rows under exactly this map at ingest time, and a setting + // the two do not share is a setting whose verdict was answered for a + // question the server is not being asked. One definition, both sides. // - // TRANSITIONAL DIVERGENCE, and it is NOT what this setting controls: on a - // NULLABLE column an explicit null is stored as NULL whatever the setting - // says — only an ABSENT key ever took the default. So a `Nullable(T) DEFAULT - // …` column now stores NULL where it previously took its default. Verified - // on 26.6.3: omitted key → default; explicit null → NULL at both settings. - q.Set("input_format_null_as_default", "1") + // date_time_input_format=best_effort: the row already carries ClickHouse's + // own rendering, but a DateTime column still parses it under this setting. + // input_format_null_as_default=1: a field the record omitted rides as an + // explicit null in its column's slot, because a positional row has one value + // per column and no way to say "absent"; for a NON-nullable column with a + // default this turns that null back into the default. It is already the + // server default (verified on 26.6.3), so this is belt-and-braces for a + // server configured otherwise. NOT what it controls: on a NULLABLE column an + // explicit null is stored as NULL whatever the setting says. + for k, v := range typelayer.InsertSettings() { + q.Set(k, v) + } + // Synchronous insert: the worker owns batching and acks a message only once + // the rows are in, so an async buffer would ack data that is still in + // flight. Not a parsing setting, so chtypes never sees it. + q.Set("async_insert", "0") + // insert_deduplicate is deliberately left at the SERVER default (it flipped + // in 26.2). WaveHouse's idempotency is app-level (internal/dedupe, keyed on + // the caller's id field) and a Replicated engine's block-hash dedupe answers + // a different question; pinning either value here would override an + // operator's own choice for their engine. req, err := http.NewRequestWithContext(ctx, "POST", t.URL+"?"+q.Encode(), &buf) if err != nil { diff --git a/internal/ingest/worker_test.go b/internal/ingest/worker_test.go index be4f6212..5e508aec 100644 --- a/internal/ingest/worker_test.go +++ b/internal/ingest/worker_test.go @@ -25,7 +25,6 @@ import ( "github.com/Wave-RF/WaveHouse/internal/cache" "github.com/Wave-RF/WaveHouse/internal/chconn" - "github.com/Wave-RF/WaveHouse/internal/discovery" "github.com/Wave-RF/WaveHouse/internal/mq" "github.com/Wave-RF/WaveHouse/internal/query" "github.com/Wave-RF/WaveHouse/internal/testutil" @@ -67,24 +66,43 @@ func makeEnvelope(t *testing.T, tableName, scope string, data map[string]any) [] // that publish two different schemas for one table. func makeEnvelopeCols(t *testing.T, tableName, scope string, cols []string, data map[string]any) []byte { t.Helper() - schema := make([]discovery.Column, len(cols)) - for i, c := range cols { - schema[i] = discovery.Column{Name: c, Position: uint64(i + 1)} - } - row, err := EncodeCompactRow(schema, data) - require.NoError(t, err) out, err := json.Marshal(EventMessage{ TableName: tableName, Scope: scope, ReceivedTimestamp: "2026-01-01T00:00:00Z", Format: FormatJSONCompactEachRow, Columns: cols, - Row: row, + Row: compactRow(t, cols, data), }) require.NoError(t, err) return out } +// compactRow renders data as one JSONCompactEachRow row in cols order, a +// column the record omits encoding as null. Production rows come from +// ClickHouse's own writer via internal/typelayer; the worker only forwards +// bytes, so a hand-built row is exactly the right fixture here. +func compactRow(t *testing.T, cols []string, data map[string]any) json.RawMessage { + t.Helper() + var buf bytes.Buffer + buf.WriteByte('[') + for i, c := range cols { + if i > 0 { + buf.WriteByte(',') + } + v, ok := data[c] + if !ok { + buf.WriteString("null") + continue + } + b, err := json.Marshal(v) + require.NoError(t, err) + buf.Write(b) + } + buf.WriteByte(']') + return json.RawMessage(buf.Bytes()) +} + // newIngestMsg builds a MockJetStreamMsg shaped exactly the way the // /v1/ingest producer (internal/api/ingest.go) publishes events: // diff --git a/internal/pipes/pipes.go b/internal/pipes/pipes.go index e146a201..59456908 100644 --- a/internal/pipes/pipes.go +++ b/internal/pipes/pipes.go @@ -82,9 +82,11 @@ func isNumericLiteral(s string) bool { // parameter definitions. // // Values are inlined directly into the SQL string (scalars are escaped, arrays -// render as a parenthesized list — see formatParamValue). This avoids -// driver-level positional parameter limitations (e.g. LIMIT position). -func BindParams(q *NamedQuery, supplied map[string]any) (string, []any, error) { +// render as a parenthesized list — see formatParamValue). This avoids the +// positional-parameter limitations a pipe would otherwise hit: a placeholder +// may sit in a LIMIT, a FORMAT clause or an identifier, where a bound value is +// not legal SQL. Nothing is left for the caller to bind. +func BindParams(q *NamedQuery, supplied map[string]any) (string, error) { // Build lookup from formal parameter definitions. formal := make(map[string]*ParamDef, len(q.Parameters)) for i := range q.Parameters { @@ -95,7 +97,7 @@ func BindParams(q *NamedQuery, supplied map[string]any) (string, []any, error) { for _, p := range q.Parameters { if p.Required { if _, ok := supplied[p.Name]; !ok { - return "", nil, fmt.Errorf("missing required parameter: %s", p.Name) + return "", fmt.Errorf("missing required parameter: %s", p.Name) } } } @@ -132,9 +134,9 @@ func BindParams(q *NamedQuery, supplied map[string]any) (string, []any, error) { }) if bindErr != nil { - return "", nil, bindErr + return "", bindErr } - return sql, nil, nil + return sql, nil } // formatParamValue converts a Go value to a safe SQL literal for inline diff --git a/internal/pipes/pipes_test.go b/internal/pipes/pipes_test.go index d5314d9e..d35e2335 100644 --- a/internal/pipes/pipes_test.go +++ b/internal/pipes/pipes_test.go @@ -16,10 +16,9 @@ func TestBindParams_AllSupplied(t *testing.T) { {Name: "min_count", Type: "number", Required: true}, }, } - sql, params, err := BindParams(q, map[string]any{"page": "/home", "min_count": 10}) + sql, err := BindParams(q, map[string]any{"page": "/home", "min_count": 10}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks WHERE page = '/home' AND count > 10", sql) - assert.Nil(t, params) } func TestBindParams_MissingRequired(t *testing.T) { @@ -28,7 +27,7 @@ func TestBindParams_MissingRequired(t *testing.T) { SQL: "SELECT * FROM clicks WHERE page = {{page}}", Parameters: []ParamDef{{Name: "page", Type: "string", Required: true}}, } - _, _, err := BindParams(q, map[string]any{}) + _, err := BindParams(q, map[string]any{}) assert.Error(t, err) assert.Contains(t, err.Error(), "missing required parameter: page") } @@ -39,10 +38,9 @@ func TestBindParams_DefaultApplied(t *testing.T) { SQL: "SELECT * FROM clicks LIMIT {{limit}}", Parameters: []ParamDef{{Name: "limit", Type: "number", Default: 100}}, } - sql, params, err := BindParams(q, map[string]any{}) + sql, err := BindParams(q, map[string]any{}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks LIMIT 100", sql) - assert.Nil(t, params) } func TestBindParams_MultipleOccurrences(t *testing.T) { @@ -51,19 +49,17 @@ func TestBindParams_MultipleOccurrences(t *testing.T) { SQL: "SELECT * FROM t WHERE a = {{val}} OR b = {{val}}", Parameters: []ParamDef{{Name: "val", Type: "string", Required: true}}, } - sql, params, err := BindParams(q, map[string]any{"val": "x"}) + sql, err := BindParams(q, map[string]any{"val": "x"}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE a = 'x' OR b = 'x'", sql) - assert.Nil(t, params) } func TestBindParams_NoParameters(t *testing.T) { t.Parallel() q := &NamedQuery{SQL: "SELECT count(*) FROM clicks"} - sql, params, err := BindParams(q, map[string]any{}) + sql, err := BindParams(q, map[string]any{}) require.NoError(t, err) assert.Equal(t, "SELECT count(*) FROM clicks", sql) - assert.Empty(t, params) } func TestBindParams_OptionalWithDefault_Supplied(t *testing.T) { @@ -72,10 +68,9 @@ func TestBindParams_OptionalWithDefault_Supplied(t *testing.T) { SQL: "SELECT * FROM clicks LIMIT {{limit}}", Parameters: []ParamDef{{Name: "limit", Type: "number", Default: 100}}, } - sql, params, err := BindParams(q, map[string]any{"limit": 50}) + sql, err := BindParams(q, map[string]any{"limit": 50}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks LIMIT 50", sql) - assert.Nil(t, params) } func TestBindParams_PlaceholderNotInSQL(t *testing.T) { @@ -84,10 +79,9 @@ func TestBindParams_PlaceholderNotInSQL(t *testing.T) { SQL: "SELECT * FROM clicks", Parameters: []ParamDef{{Name: "unused", Type: "string"}}, } - sql, params, err := BindParams(q, map[string]any{"unused": "val"}) + sql, err := BindParams(q, map[string]any{"unused": "val"}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks", sql) - assert.Empty(t, params, "unused param should not generate positional args") } func TestBindParams_InlineDefault_NoFormalParam(t *testing.T) { @@ -95,10 +89,9 @@ func TestBindParams_InlineDefault_NoFormalParam(t *testing.T) { q := &NamedQuery{ SQL: "SELECT page, count() FROM clicks GROUP BY page LIMIT {{limit:10}}", } - sql, params, err := BindParams(q, map[string]any{}) + sql, err := BindParams(q, map[string]any{}) require.NoError(t, err) assert.Equal(t, "SELECT page, count() FROM clicks GROUP BY page LIMIT 10", sql) - assert.Nil(t, params) } func TestBindParams_InlineDefault_SuppliedOverrides(t *testing.T) { @@ -106,10 +99,9 @@ func TestBindParams_InlineDefault_SuppliedOverrides(t *testing.T) { q := &NamedQuery{ SQL: "SELECT * FROM clicks LIMIT {{limit:10}}", } - sql, params, err := BindParams(q, map[string]any{"limit": float64(5)}) + sql, err := BindParams(q, map[string]any{"limit": float64(5)}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks LIMIT 5", sql) - assert.Nil(t, params) } func TestBindParams_InlineNoDefault_MissingRequired(t *testing.T) { @@ -117,7 +109,7 @@ func TestBindParams_InlineNoDefault_MissingRequired(t *testing.T) { q := &NamedQuery{ SQL: "SELECT * FROM clicks WHERE page = {{page}}", } - _, _, err := BindParams(q, map[string]any{}) + _, err := BindParams(q, map[string]any{}) assert.Error(t, err) assert.Contains(t, err.Error(), "missing required parameter: page") } @@ -127,10 +119,9 @@ func TestBindParams_InlineMultipleParams(t *testing.T) { q := &NamedQuery{ SQL: "SELECT * FROM clicks WHERE country = {{country:US}} LIMIT {{limit:10}}", } - sql, params, err := BindParams(q, map[string]any{}) + sql, err := BindParams(q, map[string]any{}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM clicks WHERE country = 'US' LIMIT 10", sql) - assert.Nil(t, params) } func TestBindParams_StringEscaping(t *testing.T) { @@ -139,7 +130,7 @@ func TestBindParams_StringEscaping(t *testing.T) { SQL: "SELECT * FROM t WHERE name = {{name}}", Parameters: []ParamDef{{Name: "name", Type: "string", Required: true}}, } - sql, _, err := BindParams(q, map[string]any{"name": "O'Brien"}) + sql, err := BindParams(q, map[string]any{"name": "O'Brien"}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE name = 'O''Brien'", sql) } @@ -150,7 +141,7 @@ func TestBindParams_BooleanParam(t *testing.T) { SQL: "SELECT * FROM t WHERE active = {{active}}", Parameters: []ParamDef{{Name: "active", Type: "boolean", Required: true}}, } - sql, _, err := BindParams(q, map[string]any{"active": true}) + sql, err := BindParams(q, map[string]any{"active": true}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE active = 1", sql) } @@ -161,7 +152,7 @@ func TestBindParams_NilParam(t *testing.T) { SQL: "SELECT * FROM t WHERE col = {{val}}", Parameters: []ParamDef{{Name: "val", Type: "string", Default: nil}}, } - sql, _, err := BindParams(q, map[string]any{}) + sql, err := BindParams(q, map[string]any{}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE col = NULL", sql) } @@ -231,17 +222,16 @@ func TestBindParams_ArrayInClause(t *testing.T) { SQL: "SELECT * FROM t WHERE id IN {{ids}}", Parameters: []ParamDef{{Name: "ids", Type: "array", Required: true}}, } - sql, params, err := BindParams(q, map[string]any{"ids": []any{"a", "b", "c"}}) + sql, err := BindParams(q, map[string]any{"ids": []any{"a", "b", "c"}}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE id IN ('a', 'b', 'c')", sql) - assert.Nil(t, params) } func TestBindParams_ArrayWorksWithoutDeclaredType(t *testing.T) { t.Parallel() // An undeclared (inline) parameter still renders an array safely. q := &NamedQuery{SQL: "SELECT * FROM t WHERE id IN {{ids}}"} - sql, _, err := BindParams(q, map[string]any{"ids": []any{float64(1), float64(2)}}) + sql, err := BindParams(q, map[string]any{"ids": []any{float64(1), float64(2)}}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE id IN (1, 2)", sql) } @@ -252,7 +242,7 @@ func TestBindParams_ArrayMixedNumericAndStringElements(t *testing.T) { // quoted on its own: a numeric-looking string renders bare, a non-numeric one // is quoted (and escaped). q := &NamedQuery{SQL: "SELECT * FROM t WHERE id IN {{ids}}"} - sql, _, err := BindParams(q, map[string]any{"ids": []any{"100", "abc"}}) + sql, err := BindParams(q, map[string]any{"ids": []any{"100", "abc"}}) require.NoError(t, err) assert.Equal(t, "SELECT * FROM t WHERE id IN (100, 'abc')", sql) } @@ -263,7 +253,7 @@ func TestBindParams_ArrayMixedNumericAndStringElements(t *testing.T) { func TestBindParams_ArrayNeutralizesInjection(t *testing.T) { t.Parallel() q := &NamedQuery{SQL: "SELECT secret FROM t WHERE id IN {{ids}}"} - sql, _, err := BindParams(q, map[string]any{ + sql, err := BindParams(q, map[string]any{ "ids": []any{"' UNION SELECT pw FROM users -- "}, }) require.NoError(t, err) @@ -273,7 +263,7 @@ func TestBindParams_ArrayNeutralizesInjection(t *testing.T) { func TestBindParams_ObjectRejected(t *testing.T) { t.Parallel() q := &NamedQuery{SQL: "SELECT * FROM t WHERE col = {{p}}"} - _, _, err := BindParams(q, map[string]any{"p": map[string]any{"k": "v"}}) + _, err := BindParams(q, map[string]any{"p": map[string]any{"k": "v"}}) require.Error(t, err) assert.Contains(t, err.Error(), `parameter "p"`) assert.Contains(t, err.Error(), "unsupported parameter type object") @@ -282,7 +272,7 @@ func TestBindParams_ObjectRejected(t *testing.T) { func TestBindParams_EmptyArrayRejected(t *testing.T) { t.Parallel() q := &NamedQuery{SQL: "SELECT * FROM t WHERE id IN {{ids}}"} - _, _, err := BindParams(q, map[string]any{"ids": []any{}}) + _, err := BindParams(q, map[string]any{"ids": []any{}}) require.Error(t, err) assert.Contains(t, err.Error(), "array parameter must not be empty") } diff --git a/internal/policy/canonical.go b/internal/policy/canonical.go index 1f91776f..c75b9505 100644 --- a/internal/policy/canonical.go +++ b/internal/policy/canonical.go @@ -10,46 +10,28 @@ import ( ) // This file is the policy engine's ONE rendering layer for comparison operands: -// every value a row-filter or insert-check compares — a JWT claim, a -// policy-authored literal, an ingested payload value — is rendered here before -// any comparison or SQL bind, so the two read surfaces can't disagree on what a -// value "is". Three entry points, one per operand source: +// every claim value a row-filter or insert-check compares is rendered here +// before any comparison or SQL bind, so the two read surfaces can't disagree on +// what a value "is". One entry point: // -// CanonicalScalar — decoded claim values (Evaluate's template resolution) -// and the insert-check comparison's two sides (internal/api) -// CanonicalNumericLiteral — policy-authored literals ("1.0" → "1"), behind the -// json.Valid grammar gate -// numericCanonical — ingested payload values on the stream's row-filter -// path (string / json.Number / float64), routed through -// the two above +// CanonicalScalar — decoded claim values (Evaluate's template resolution) // -// All three converge on one canonical decimal form (canonicalDecimal, bounded -// by maxCanonicalDigits): exact at any width, positional (never an exponent — +// It converges on one canonical decimal form (canonicalDecimal, bounded by +// maxCanonicalDigits): exact at any width, positional (never an exponent — // "1e-3" renders as "0.001"), no leading/trailing zeros, "-0" folded to "0". -// Those invariants are what numeric.go's digit-string comparison -// (compareCanonicalDecimals) and storage-domain narrowing (NumericSpec.compare) -// rely on. scalarString is the deliberate exception: the raw byte rendering for -// ColumnText/ColumnOpaque comparison, where the payload's own spelling IS the -// compared value. +// Comparing a rendered value against a STORED row is not done here at all: both +// surfaces hand the predicate to ClickHouse's own engine — the type layer for +// the stream and the insert check, the server for /v1/query — rather than to a +// Go re-derivation of its coercion rules. // maxCanonicalDigits bounds both a numeric literal's digit count and its // exact decimal expansion. Big-integer parsing is superlinear in digit count -// and the ingest check path hands CanonicalScalar client-controlled literals -// (CWE-400), and a short exponent literal can hide a wide expansion ("1e-150" -// is six characters with a 152-character exact form). 100 digits is far past -// any real id — uint256 is 78. +// and a claim is attacker-influenced input (CWE-400), and a short exponent +// literal can hide a wide expansion ("1e-150" is six characters with a +// 152-character exact form). 100 digits is far past any real id — uint256 +// is 78. const maxCanonicalDigits = 100 -// maxNumericOperandChars is the O(1) length pre-gate on numeric comparison -// operands, checked before ANY scan of the value. It is verdict-preserving: a -// JSON number literal carries at most four non-digit bytes (a sign, a decimal -// point, an exponent marker and its sign), so anything longer already fails -// the canonical digit bound (maxCanonicalDigits) — but proving that inside the -// canonical gate costs a full json.Valid pass plus a digit count over a -// client-controlled value, per subscriber per event on the fan-out goroutine. -// The gate refuses first, without reading the bytes. -const maxNumericOperandChars = maxCanonicalDigits + 4 - // CanonicalScalar renders a decoded JSON value as the canonical string the // policy layer binds and compares, reporting ok=false for values with no such // form: null, objects, and arrays. A structured value is never a sensible @@ -66,10 +48,10 @@ const maxNumericOperandChars = maxCanonicalDigits + 4 // canonicalDecimal, never a float64 round-trip that could bind a value the // token doesn't carry ("1e-400" fails closed rather than collapsing to "0"). // A literal, or an exact form, past maxCanonicalDigits likewise has no -// canonical form and fails closed (1e400, 1e-400). Claim resolution and the -// insert-check comparison's two sides (internal/api) all route through this -// one function, so what a read filter binds and what a write check accepts -// can't drift. +// canonical form and fails closed (1e400, 1e-400). Read-filter claim +// resolution and insert-check claim resolution both route through this one +// function, so what a read filter binds and what a write check requires can't +// drift. func CanonicalScalar(v any) (string, bool) { switch val := v.(type) { case nil, map[string]any, []any: @@ -112,30 +94,6 @@ func CanonicalScalar(v any) (string, bool) { } } -// LiteralValue marks an insert-check required value the policy author wrote -// as a placeholder-free literal (Evaluate). A literal carries no JSON type — -// "1.0" means the number 1 to a numeric column and the three-character text -// to a String column — so the check comparison (internal/api) accepts its -// numeric reading as well as its spelling. The type is the gate: a -// claim-derived value is never wrapped, so a string-typed claim keeps strict -// canonical equality and can't gain a numeric reading it didn't have. Only -// CheckClauses carries this type; read filters bind plain strings. -type LiteralValue string - -// CanonicalNumericLiteral renders a policy-authored literal that spells a -// JSON number in canonical decimal form ("1.0" → "1"), reporting ok=false for -// everything else. The json.Valid gate keeps this to spellings JSON itself -// can produce: big.Int would also take "+5" or "007", readings no decoded -// claim or payload value ever has. It canonicalizes nothing at resolve time — -// the literal still binds and auto-injects exactly as written; only the check -// comparison consults this second reading. -func CanonicalNumericLiteral(s string) (string, bool) { - if !json.Valid([]byte(s)) { - return "", false - } - return CanonicalScalar(json.Number(s)) -} - // canonicalDecimal renders a non-integer JSON number literal (one carrying a // fraction or exponent) as its exact canonical decimal string: "1.0" → "1", // "2.50" → "2.5", "1e3" → "1000", "25e-4" → "0.0025", every digit preserved @@ -209,65 +167,3 @@ func canonicalDecimal(lit string) (string, bool) { } return sign + out, true } - -// numericCanonical renders a payload value as the canonical decimal form the -// numeric comparison consumes, ok=false for anything that is not a number a -// ClickHouse numeric column could have stored: booleans, structured values and -// null, spellings outside the JSON number grammar ("Inf", "NaN", "0x1f", -// "007"), values whose exact digits were lost upstream (float64 at/past 2^53), -// and anything past the canonical digit bound. String and json.Number inputs -// take the claim side's own gates (CanonicalNumericLiteral / CanonicalScalar), -// so the payload and constant sides can never disagree on what counts as a -// number or how it is spelled. -func numericCanonical(v any) (string, bool) { - switch x := v.(type) { - case string: - if len(x) > maxNumericOperandChars { - return "", false - } - return CanonicalNumericLiteral(x) - case json.Number: - if len(x) > maxNumericOperandChars { - return "", false - } - return CanonicalScalar(x) - case float64: - // CanonicalScalar applies the 2^53 exactness guard and renders - // positionally; the literal gate then re-canonicalizes the one - // rendering FormatFloat emits that canonical form forbids ("-0"). - s, ok := CanonicalScalar(x) - if !ok { - return "", false - } - return CanonicalNumericLiteral(s) - default: - return "", false - } -} - -// scalarString renders a JSON-decoded scalar as the exact BYTES compared under -// ColumnText and ColumnOpaque — deliberately the payload's raw spelling, never -// a canonical form: a String column stores the payload text verbatim, so byte -// comparison against it must use that spelling (canonicalizing "1.0" to "1" -// here would move equality away from what ClickHouse stores). Numeric coercion -// deliberately does NOT live here — the ColumnNumeric arm routes both operands -// through the claim side's canonical machinery (numericCanonical). Non-scalars -// (arrays, objects, null) return ok=false so the predicate fails closed rather -// than guessing. The float64 case serves callers that decoded without -// UseNumber (the stream itself always does); -1 precision emits the shortest -// round-trip form without an exponent, so integer IDs read back as "123", not -// "1.23e+02". -func scalarString(v any) (string, bool) { - switch x := v.(type) { - case string: - return x, true - case json.Number: - return string(x), true - case float64: - return strconv.FormatFloat(x, 'f', -1, 64), true - case bool: - return strconv.FormatBool(x), true - default: - return "", false - } -} diff --git a/internal/policy/numeric.go b/internal/policy/numeric.go deleted file mode 100644 index 1a27017b..00000000 --- a/internal/policy/numeric.go +++ /dev/null @@ -1,260 +0,0 @@ -package policy - -import ( - "math/big" - "strconv" - "strings" -) - -// This file compares canonical decimal forms (canonical.go's output) the way -// the column that stores them would: compareCanonicalDecimals is the exact -// digit-string ordering, and NumericSpec narrows both operands into the -// column's STORAGE domain first — the same narrowing ClickHouse applies to the -// stored value at insert and to the filter constant at compare — so the -// stream's in-memory verdict can't drift from the query path's SQL verdict. - -// NumericFamily classifies how a ClickHouse numeric column stores a value — -// the narrowing the row-filter comparison must apply to BOTH operands so its -// verdict matches the query path, where ClickHouse narrows the stored value at -// insert AND the filter constant at compare. The zero value is NumericNone: no -// storage model, every comparison refused — the same fail-closed zero-value -// contract as ColumnOpaque, so a future numeric type nobody classified can -// never be compared under the wrong model. -type NumericFamily uint8 - -const ( - NumericNone NumericFamily = iota // unclassified: refuse, fail closed - NumericInteger // Int*/UInt*: exact at any width - NumericFloat // Float32/Float64: IEEE rounding at Bits - NumericDecimal // Decimal*: truncation at Scale -) - -// NumericSpec is a numeric column's storage model. Bits is the bit width -// (float width for NumericFloat, integer width for NumericInteger); Unsigned -// marks UInt* (NumericInteger only); Precision and Scale are the stored total -// and fractional digit counts (NumericDecimal only, 1 ≤ Precision ≤ 76). -type NumericSpec struct { - Family NumericFamily - Bits int - Unsigned bool - Precision int - Scale int -} - -// intBounds holds each ClickHouse integer width's inclusive decimal bounds as -// canonical-form strings, computed once. A constant outside the width is not -// reliably modelable — on one and the same release, ClickHouse was measured to -// ERROR the comparison (a negative literal against an unsigned column: the -// role reads no rows), to PROMOTE and compare mathematically ('256' against a -// UInt8), and to WRAP at a width boundary ('9223372036854775808' against an -// Int64 compares as −2^63, where exact-precision comparison would ADMIT the -// −2^63 rows SQL hides under !=). Refusing out-of-range operands is the one -// rule safe under all three behaviors; the cost is availability on bounds no -// in-range data could ever satisfy differently. -var intBounds = func() map[int]struct{ sMin, sMax, uMax string } { - m := make(map[int]struct{ sMin, sMax, uMax string }, 6) - for _, bits := range []int{8, 16, 32, 64, 128, 256} { - // big.Int because Int128/Int256 exceed every native width; rendered - // once to decimal strings so range checks are compareCanonicalDecimals - // calls. For bits=8 the three bounds are −128, 127, and 255. - pow := new(big.Int).Lsh(big.NewInt(1), uint(bits-1)) // 2^(bits−1) - sMin := new(big.Int).Neg(pow) // signed min: −2^(bits−1) - sMax := new(big.Int).Sub(pow, big.NewInt(1)) // signed max: 2^(bits−1) − 1 - uMax := new(big.Int).Sub(new(big.Int).Lsh(pow, 1), big.NewInt(1)) // unsigned max: 2^bits − 1 - m[bits] = struct{ sMin, sMax, uMax string }{sMin.String(), sMax.String(), uMax.String()} - } - return m -}() - -// integerInRange reports whether a canonical integer form lies within the -// column's width. An unknown width refuses — fail closed, never a guessed -// range. Integers alone need this explicit bounds table because their -// comparison is digit-string arithmetic with no inherent width; the float -// family's range gate is narrowFloat's ParseFloat-overflow refusal, and the -// decimal family's is decimalInPrecision. -func (n NumericSpec) integerInRange(c string) bool { - b, ok := intBounds[n.Bits] - if !ok { - return false - } - if n.Unsigned { - return compareCanonicalDecimals(c, "0") >= 0 && compareCanonicalDecimals(c, b.uMax) <= 0 - } - return compareCanonicalDecimals(c, b.sMin) >= 0 && compareCanonicalDecimals(c, b.sMax) <= 0 -} - -// decimalInPrecision reports whether a canonical form's integer digits fit the -// column's Precision−Scale budget. A payload past it is never storable -// (DECIMAL_OVERFLOW rejects the insert); a constant past it was measured to -// promote and compare mathematically on the query path — but the integer -// widths' wrap behavior (intBounds) shows the same class is not reliably -// modelable across pairs, so the refusal keeps one rule for every family at an -// availability-only cost. A lone "0" integer part spends no digits -// (Decimal(2,2) legally stores 0.99). -func (n NumericSpec) decimalInPrecision(c string) bool { - if n.Precision < 1 || n.Scale < 0 || n.Scale > n.Precision { - // No coherent model: refuse, fail closed. The Scale bounds also protect - // truncateScale's slicing — a hand-built spec must degrade to refusal, - // never a panic on the fan-out goroutine. - return false - } - intPart, _, _ := strings.Cut(strings.TrimPrefix(c, "-"), ".") - digits := len(intPart) - if intPart == "0" { - digits = 0 - } - return digits <= n.Precision-n.Scale -} - -// compare orders two canonical decimal operands (numericCanonical / -// CanonicalNumericLiteral output) in the column's storage domain, ok=false when -// the model refuses the pair. Narrowing BOTH sides is what ClickHouse itself -// does — it narrows the payload at insert and the bound constant at compare -// (verified: Float32 stores 16777217 as 16777216 and `= '16777217'` still -// matches; Decimal(10,2) stores 1.005 as 1.00 and `= '1.005'` still matches) — -// so a threshold filter can no longer admit an event whose stored row lands on -// the other side of the comparison (the ordering fail-open raised on #381). -func (n NumericSpec) compare(a, b string) (int, bool) { - switch n.Family { - case NumericInteger: - // A fractional CONSTANT against an integer column is a per-query type - // error on the SQL path (the role reads no rows, loudly); a fractional - // PAYLOAD was never storable in the column. Refuse both — fail closed. - if strings.Contains(a, ".") || strings.Contains(b, ".") { - return 0, false - } - // Same rule for the column's range: ClickHouse's reading of an - // out-of-range constant varies by pair (error, mathematical promotion, - // or a width-boundary wrap that compares against a DIFFERENT value - // than written — see intBounds), and an out-of-range payload was never - // storable. Refuse both sides rather than model any one behavior. - if !n.integerInRange(a) || !n.integerInRange(b) { - return 0, false - } - return compareCanonicalDecimals(a, b), true - case NumericFloat: - fa, ok := narrowFloat(a, n.Bits) - if !ok { - return 0, false - } - fb, ok := narrowFloat(b, n.Bits) - if !ok { - return 0, false - } - // Both operands are now exact values of the column's float domain, so - // direct comparison IS the domain comparison — no ties left to break. - switch { - case fa < fb: - return -1, true - case fa > fb: - return 1, true - default: - return 0, true - } - case NumericDecimal: - // Precision is the range gate of the decimal family: a payload with - // integer digits past Precision−Scale is never storable (the insert is - // rejected), while a constant past it was measured to PROMOTE on the - // query path and compare mathematically — the refusal there is an - // accepted availability cost, taken because the integer widths' wrap - // behavior proves this class has no reliable single model (see - // decimalInPrecision — one story across the three sites). Scale - // truncation below cannot change integer digits, so gating - // pre-truncation is exact. - if !n.decimalInPrecision(a) || !n.decimalInPrecision(b) { - return 0, false - } - return compareCanonicalDecimals(truncateScale(a, n.Scale), truncateScale(b, n.Scale)), true - case NumericNone: - return 0, false // no storage model: refuse, fail closed - default: - return 0, false // future family nobody taught this switch: same refusal - } -} - -// narrowFloat converts a canonical decimal form to the column's float domain -// with a single correct rounding (ParseFloat at the exact bit width — never a -// float64 detour, whose double rounding can land Float32 values one ULP off). -// A magnitude the domain can't hold refuses the comparison (ParseFloat reports -// the overflow as an error — the float family's range gate): ClickHouse would -// store ±Inf there, and matching infinities is a verdict this evaluator can't -// prove cheaply, so the row is withheld — availability, never exposure. An -// unknown width refuses too: ParseFloat silently treats any other bitSize as -// 64, which would compare a Float32 column in the wrong (wider) domain — the -// fail-open direction — instead of the zero-value-refuses contract the -// integer and decimal families keep. -func narrowFloat(canonical string, bits int) (float64, bool) { - if bits != 32 && bits != 64 { - return 0, false - } - f, err := strconv.ParseFloat(canonical, bits) - if err != nil { - return 0, false - } - return f, true -} - -// truncateScale narrows a canonical decimal form to scale fractional digits, -// truncating toward zero — ClickHouse's Decimal cast (1.005, 1.006 and 1.009 -// all store as 1.00 in a Decimal(10,2); rounding would predict 1.01). The -// result is re-canonicalized (trailing zeros trimmed, bare "-0" folded) so it -// stays valid compareCanonicalDecimals input. -func truncateScale(canonical string, scale int) string { - intPart, frac, hasFrac := strings.Cut(canonical, ".") - if !hasFrac { - return canonical - } - if len(frac) > scale { - frac = frac[:scale] - } - for len(frac) > 0 && frac[len(frac)-1] == '0' { - frac = frac[:len(frac)-1] - } - if len(frac) > 0 { - return intPart + "." + frac - } - if intPart == "-0" { - return "0" - } - return intPart -} - -// compareCanonicalDecimals orders two canonical decimal forms (CanonicalScalar -// output) as numbers, by digit-string arithmetic alone — the comparison twin of -// canonicalDecimal, sharing its invariants: an optional leading '-' (never on -// zero), no leading integer zeros except a lone "0", no trailing fraction -// zeros, no exponent. Those invariants are what make the string operations -// sound: with no leading zeros a longer integer part IS the larger magnitude, -// and with no trailing zeros a fraction that is a proper prefix of another IS -// the smaller. Never a float round-trip, so 64-bit-plus IDs order exactly. -func compareCanonicalDecimals(a, b string) int { - if a == b { - return 0 - } - na, nb := strings.HasPrefix(a, "-"), strings.HasPrefix(b, "-") - switch { - case na && !nb: - return -1 - case !na && nb: - return 1 - case na && nb: - return -compareCanonicalMagnitudes(a[1:], b[1:]) - } - return compareCanonicalMagnitudes(a, b) -} - -// compareCanonicalMagnitudes orders two unsigned canonical forms. -func compareCanonicalMagnitudes(a, b string) int { - ai, af, _ := strings.Cut(a, ".") - bi, bf, _ := strings.Cut(b, ".") - if len(ai) != len(bi) { - if len(ai) < len(bi) { - return -1 - } - return 1 - } - if c := strings.Compare(ai, bi); c != 0 { - return c - } - return strings.Compare(af, bf) -} diff --git a/internal/policy/policy.go b/internal/policy/policy.go index 3ad4c9c1..b48ce6be 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -115,13 +115,11 @@ type ResolvedPermissions struct { type ResolvedSelect struct { AllowColumns []string DenyColumns []string - WhereClause string - WhereParams []any - // rowFilter is the same row-level-security predicate as WhereClause/WhereParams, - // kept in resolved form so the stream path can evaluate it in memory (RowVisible) - // while the query path renders it to SQL. Both derive from one resolvePredicates - // call in Evaluate, so the two read surfaces can't drift. See rowfilter.go. - rowFilter []resolvedPredicate + // rowFilter is the row-level-security predicate in resolved form: the query + // path renders it to SQL (WhereSQL) and the stream path evaluates it against + // the stored row (Predicates, handed to the type layer). Both read this one + // resolvePredicates result, so the two read surfaces can't drift (#457). + rowFilter []Predicate AllowedAggregations []string DeniedAggregations []string MaxRows int @@ -255,8 +253,8 @@ func evaluateSelect(perms *SelectPermissions, claims map[string]any) *ResolvedPe } // Resolve filters into WHERE clause. A bind-unsafe filter column can't be - // emitted safely — a '?' in it would shift clickhouse-go's positional value - // binding, including this RLS filter's own bound value — so deny the role + // emitted safely — a '?' in it would shift the positional-to-named parameter + // rewrite, including this RLS filter's own bound value — so deny the role // fail-closed rather than drop the predicate (which would widen row access) // or emit a mis-bound query. validateSelectPerms rejects such a policy at // write time; this guards the query path as defense-in-depth (Evaluate does @@ -267,17 +265,12 @@ func evaluateSelect(perms *SelectPermissions, claims map[string]any) *ResolvedPe return &ResolvedPermissions{Allowed: false} } } - // Resolve the row-filter once into predicates, then render both read surfaces - // from that single source so they can't drift: the query path binds them into - // a SQL WHERE here; the stream path evaluates the same predicates in memory - // (ResolvedPermissions.RowVisible). - preds := resolvePredicates(perms.Filter, claims) - resolved.Select.rowFilter = preds - clauses, params := predicatesToSQL(preds) - if len(clauses) > 0 { - resolved.Select.WhereClause = strings.Join(clauses, " AND ") - resolved.Select.WhereParams = params - } + // Resolve the row-filter once into predicates; both read surfaces render + // from that single source so they can't drift: the query path binds them + // into a SQL WHERE (WhereSQL, which needs the column types only the + // builder has); the stream path compiles the same predicates through the + // type layer (ResolvedPermissions.Predicates). + resolved.Select.rowFilter = resolvePredicates(perms.Filter, claims) } return resolved @@ -320,15 +313,8 @@ func evaluateInsert(perms *InsertPermissions, claims map[string]any) *ResolvedPe case f.Eq != nil: // Deliberate asymmetry with the read path: an unresolvable check claim // still resolves to "" and is auto-injected as the required value (#463). - // A placeholder-free value is marked LiteralValue so the check - // comparison can accept its numeric reading; a claim-derived value - // stays a plain string and keeps strict canonical equality. v, _ := resolveTemplate(*f.Eq, claims) - if !claimTemplateRe.MatchString(*f.Eq) { - resolved.Insert.CheckClauses[col] = LiteralValue(v) - } else { - resolved.Insert.CheckClauses[col] = v - } + resolved.Insert.CheckClauses[col] = v case f.In != nil: // A []any value marks a set-membership check (vs a scalar required // value); ingest enforces "inserted value must be one of these". @@ -340,15 +326,20 @@ func evaluateInsert(perms *InsertPermissions, claims map[string]any) *ResolvedPe return resolved } -// resolvedPredicate is one row-filter or check comparison with its claim templates -// already resolved to concrete string values — the shared, render-agnostic form the query -// path turns into SQL (predicatesToSQL) and the stream path evaluates in memory -// (RowVisible). Op is one of "=", "!=", ">", "<", "in". Values holds one element -// for the scalar operators and zero-or-more for "in"; an EMPTY Values matches no -// rows on either surface — an empty/unresolvable "in" set, or a scalar whose -// constant was unresolvable (an absent/null claim, a structured value, or one with -// no canonical form — see resolveTemplate/CanonicalScalar). -type resolvedPredicate struct { +// Predicate is one row-filter comparison with its claim templates already +// resolved to concrete string values — the shared, render-agnostic form the query +// path turns into SQL (predicatesToSQL) and the stream path hands to the type +// layer to evaluate against the stored row. Op is one of "=", "!=", ">", "<", +// "in". Values holds one element for the scalar operators and zero-or-more for +// "in"; an EMPTY Values matches no rows on either surface — an empty/unresolvable +// "in" set, or a scalar whose constant was unresolvable (an absent/null claim, a +// structured value, or one with no canonical form — see +// resolveTemplate/CanonicalScalar). +// +// It is exported because the stream path evaluates it outside this package. The +// values are bound as typed parameters there, exactly as they are bound as query +// parameters here — neither surface ever splices one into expression text. +type Predicate struct { Column string Op string Values []string @@ -357,17 +348,17 @@ type resolvedPredicate struct { // resolvePredicates resolves each filter's claim templates once into predicates. // Both read surfaces derive from this single result so they can't drift; the // operator order within a column (=, !=, >, <, in) mirrors the former inline SQL. -func resolvePredicates(filters map[string]Filter, claims map[string]any) []resolvedPredicate { - var preds []resolvedPredicate +func resolvePredicates(filters map[string]Filter, claims map[string]any) []Predicate { + var preds []Predicate // An unresolvable constant (ok=false from resolveTemplate) yields a predicate // with NO values, which matches no rows on either surface (#385) — never a // synthesized stand-in that could match some other principal's rows. - scalar := func(col, op, tmpl string) resolvedPredicate { + scalar := func(col, op, tmpl string) Predicate { v, ok := resolveTemplate(tmpl, claims) if !ok { - return resolvedPredicate{Column: col, Op: op} + return Predicate{Column: col, Op: op} } - return resolvedPredicate{Column: col, Op: op, Values: []string{v}} + return Predicate{Column: col, Op: op, Values: []string{v}} } for col, f := range filters { if f.Eq != nil { @@ -383,14 +374,28 @@ func resolvePredicates(filters map[string]Filter, claims map[string]any) []resol preds = append(preds, scalar(col, "<", *f.Lt)) } if f.In != nil { - preds = append(preds, resolvedPredicate{col, "in", toStrings(resolveInValues(*f.In, claims))}) + preds = append(preds, Predicate{col, "in", toStrings(resolveInValues(*f.In, claims))}) } } return preds } +// WhereSQL renders the row filter for the query path: the AND-joined clause +// ("" when the role has no row filter) and its positional `?` params, ready to +// splice into the builder's WHERE. colType returns a column's ClickHouse type +// ("" when unknown, or colType nil): a claim compared against an integer +// column binds as a chsql.IntParam — the strict cast, so a claim that does not +// fit the column matches nothing instead of wrapping — and every other value +// binds as a plain string. +// +// A nil receiver panics, deliberately: see ResolvedPermissions. +func (s *ResolvedSelect) WhereSQL(colType func(column string) string) (string, []any) { + clauses, params := predicatesToSQL(s.rowFilter, colType) + return strings.Join(clauses, " AND "), params +} + // predicatesToSQL renders resolved predicates into WHERE clauses and bound params. -func predicatesToSQL(preds []resolvedPredicate) ([]string, []any) { +func predicatesToSQL(preds []Predicate, colType func(string) string) ([]string, []any) { var clauses []string var params []any for _, p := range preds { @@ -398,6 +403,12 @@ func predicatesToSQL(preds []resolvedPredicate) ([]string, []any) { // caller columns, so a row-filter on a weird-but-legal column name (dots, // spaces, keywords) is emitted safely. qcol := chsql.QuoteIdent(p.Column) + bind := func(v string) any { return v } + if colType != nil { + if it, ok := chsql.IntegerType(colType(p.Column)); ok { + bind = func(v string) any { return chsql.IntParam{Value: v, Type: it} } + } + } switch p.Op { case "in": if len(p.Values) == 0 { @@ -406,10 +417,14 @@ func predicatesToSQL(preds []resolvedPredicate) ([]string, []any) { // fail-open). `IN ()` is not valid SQL, so emit a constant false. clauses = append(clauses, "1 = 0") } else { + // One scalar parameter per element, not one Array(String): the + // strict cast applies per element, and `c IN (E(p0), E(p1))` keeps + // the primary key where `c IN arrayMap(…)` reads every granule + // (measured on 26.6.3.62). placeholders := strings.TrimSuffix(strings.Repeat("?,", len(p.Values)), ",") clauses = append(clauses, fmt.Sprintf("%s IN (%s)", qcol, placeholders)) for _, v := range p.Values { - params = append(params, v) + params = append(params, bind(v)) } } default: @@ -423,20 +438,14 @@ func predicatesToSQL(preds []resolvedPredicate) ([]string, []any) { continue } clauses = append(clauses, fmt.Sprintf("%s %s ?", qcol, p.Op)) - params = append(params, p.Values[0]) + params = append(params, bind(p.Values[0])) } } return clauses, params } -// resolveFilters converts filter definitions with claim templates into SQL WHERE -// clauses. Retained as the predicates→SQL composition the query-path tests target. -func resolveFilters(filters map[string]Filter, claims map[string]any) ([]string, []any) { - return predicatesToSQL(resolvePredicates(filters, claims)) -} - // toStrings normalizes resolveInValues' []any (already canonical strings) to the -// []string a resolvedPredicate carries. +// []string a Predicate carries. func toStrings(vals []any) []string { if len(vals) == 0 { return nil @@ -457,9 +466,9 @@ func toStrings(vals []any) []string { // with no placeholders — including a literal "" — is always ok, and binds // exactly as written: canonicalizing a numeric-spelled literal here would // silently move read filters on String columns (`_neq: "1.0"` on a version -// column is a different predicate than `_neq: "1"`); the insert-check -// comparison instead accepts a literal's numeric reading at compare time -// (CanonicalNumericLiteral). +// column is a different predicate than `_neq: "1"`). A literal a numeric +// column cannot read is ClickHouse's own code 53 at evaluation time, on both +// surfaces, not a second Go reading of the literal. func resolveTemplate(tmpl string, claims map[string]any) (string, bool) { ok := true resolved := claimTemplateRe.ReplaceAllStringFunc(tmpl, func(match string) string { @@ -603,6 +612,50 @@ func (rp *ResolvedPermissions) IsColumnAllowed(col string, insert bool) bool { return false } +// HasRowFilter reports whether this role/table entry carries a row-level-security +// predicate. The stream fan-out uses it to decide whether an event can be projected +// once for a whole role bucket (no filter) or must be checked per subscriber against +// that subscriber's claims (filter present). A nil receiver (no policy applies) has +// no filter. +// +// It answers YES for a denied grant and for one whose read side was never +// resolved, neither of which has a predicate to speak of. That is deliberate: +// this is the GATE in front of the per-row evaluation, and a "no filter" answer +// sends the caller down the deliver-to-the-whole-bucket fast path where no row is +// ever checked. Saying yes forces the per-subscriber path, where Predicates +// denies. Same shape and same reason as RestrictsColumns, which guards the +// builder's SELECT * expansion. +func (rp *ResolvedPermissions) HasRowFilter() bool { + if rp == nil { + return false + } + if !rp.Allowed || rp.Select == nil { + return true + } + return len(rp.Select.rowFilter) > 0 +} + +// Predicates returns the resolved row-filter predicates for the read side — the +// SAME slice WhereSQL renders into the query's WHERE clause, so the +// stream and the query answer off one resolution and cannot drift (#457). The +// caller evaluates them against the stored row; every value is bound, never +// spliced. +// +// ok is false when no row may be admitted at all: a denied grant, or one +// resolved for INSERT whose empty read side would otherwise read as "no +// predicates, everything visible" — the same fail-closed shape as CheckClauses. +// A nil receiver means no policy applies, so there is nothing to filter by and +// ok is TRUE with no predicates. +func (rp *ResolvedPermissions) Predicates() ([]Predicate, bool) { + if rp == nil { + return nil, true + } + if !rp.Allowed || rp.Select == nil { + return nil, false + } + return rp.Select.rowFilter, true +} + // CheckClauses returns the insert side's check clauses, and false when the // insert side was never resolved. // @@ -806,8 +859,8 @@ func validateSelectPerms(table, role string, perms *SelectPermissions) error { return fmt.Errorf("table %q, op %q, role %q: max_memory_usage must be non-negative", table, op, role) } // Filter column names are interpolated into SQL (backtick-quoted) at query - // time, so a '?' in one would shift clickhouse-go's positional value - // binding. Refuse such a policy at write time, mirroring the query builder's + // time, so a '?' in one would shift the positional-to-named parameter + // rewrite. Refuse such a policy at write time, mirroring the query builder's // chsql.BindUnsafe guard on caller-supplied columns. for col, f := range perms.Filter { if chsql.BindUnsafe(col) { @@ -815,7 +868,7 @@ func validateSelectPerms(table, role string, perms *SelectPermissions) error { } // An entry naming no operator resolves to no predicate, so the row-level // restriction the author declared would silently not apply — Evaluate would - // answer HasRowFilter() false and RowVisible true for every row. Refuse it + // answer HasRowFilter() false and admit every row on the stream. Refuse it // here; the resolver's matching deny is defense-in-depth. if !f.hasOperator() { return fmt.Errorf("table %q, op %q, role %q: filter column %q sets no operator — use _eq, _neq, _gt, _lt, or _in", table, op, role, col) diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index f45c4e20..2839e9eb 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/Wave-RF/WaveHouse/internal/chsql" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -125,9 +126,10 @@ func TestEvaluate_FilterWithClaimTemplate(t *testing.T) { claims := map[string]any{"org_id": "org-123"} perms := Evaluate(p, "user", "clicks", "select", claims) assert.True(t, perms.Allowed) - assert.Contains(t, perms.Select.WhereClause, "`org_id` = ?") - require.Len(t, perms.Select.WhereParams, 1) - assert.Equal(t, "org-123", perms.Select.WhereParams[0]) + clause, params := perms.Select.WhereSQL(nil) + assert.Contains(t, clause, "`org_id` = ?") + require.Len(t, params, 1) + assert.Equal(t, "org-123", params[0]) } func TestEvaluate_CheckClauses(t *testing.T) { @@ -149,28 +151,6 @@ func TestEvaluate_CheckClauses(t *testing.T) { assert.Equal(t, "org-456", perms.Insert.CheckClauses["org_id"]) } -// TestEvaluate_CheckClauses_StaticLiteralTyped: a placeholder-free check -// value is wrapped as LiteralValue — the marker that lets the ingest -// comparison accept its numeric reading — while a claim-derived value (above) -// stays a plain string, so a string-typed claim can never gain that reading. -func TestEvaluate_CheckClauses_StaticLiteralTyped(t *testing.T) { - t.Parallel() - eqVal := "1.0" - p := &Policy{ - Tables: map[string]TablePolicy{ - "clicks": { - "user": {Insert: &InsertPermissions{Check: map[string]Filter{ - "count": {Eq: &eqVal}, - }}}, - }, - }, - } - perms := Evaluate(p, "user", "clicks", "insert", map[string]any{}) - assert.True(t, perms.Allowed) - require.Contains(t, perms.Insert.CheckClauses, "count") - assert.Equal(t, LiteralValue("1.0"), perms.Insert.CheckClauses["count"]) -} - func TestEvaluate_AggregationLimits(t *testing.T) { t.Parallel() p := &Policy{ @@ -469,8 +449,8 @@ func TestResolveTemplate(t *testing.T) { // A static literal binds exactly as written even when it spells a JSON // number: canonicalizing it here would move read filters on String // columns (`_neq: "1.0"` on a version column would stop excluding rows - // storing "1.0"). The insert-check comparison accepts the numeric - // reading at compare time instead (CanonicalNumericLiteral). + // storing "1.0"). A literal a numeric column cannot read is + // ClickHouse's own code 53 at evaluation time, on both surfaces. {"numeric-spelled literal binds as written", "1.0", "1.0", true}, {"exponent-spelled literal binds as written", "1e400", "1e400", true}, } @@ -576,41 +556,6 @@ func TestCanonicalScalar(t *testing.T) { } } -// TestCanonicalNumericLiteral pins the numeric reading of a policy-authored -// check literal: only spellings JSON itself can produce canonicalize — the -// json.Valid gate rejects big.Int-acceptable forms like "+5" and "007" that -// no decoded claim or payload value ever carries, so the check comparison's -// second reading can't accept a spelling the first side can't produce. -func TestCanonicalNumericLiteral(t *testing.T) { - t.Parallel() - tests := []struct { - name string - lit string - want string - ok bool - }{ - {"float spelling of an integer", "1.0", "1", true}, - {"exponent spelling", "25e-4", "0.0025", true}, - {"negative fraction", "-2.50", "-2.5", true}, - {"integer passes through", "7", "7", true}, - {"leading plus is not JSON", "+5", "", false}, - {"leading zero is not JSON", "007", "", false}, - {"whitespace-padded number is not a bare literal", " 5", "", false}, - {"non-numeric literal", "org-123", "", false}, - {"boolean literal is valid JSON but not a number", "true", "", false}, - {"empty literal", "", "", false}, - {"no canonical form past the bound", "1e400", "", false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - got, ok := CanonicalNumericLiteral(tt.lit) - assert.Equal(t, tt.want, got) - assert.Equal(t, tt.ok, ok) - }) - } -} - func TestValidate(t *testing.T) { t.Parallel() tests := []struct { @@ -845,7 +790,12 @@ func TestEvaluate_WildcardRoleKeyDoesNotGrant(t *testing.T) { "a \"*\" role key must not grant a roleless request") } -func TestResolveFilters_MultipleOperators(t *testing.T) { +// The TestFiltersToSQL_* tests below target the query path's rendering as +// evaluateSelect performs it: resolvePredicates, then predicatesToSQL over the +// result. They spell the pair out rather than going through a helper, so the +// composition under test is the one production runs. + +func TestFiltersToSQL_MultipleOperators(t *testing.T) { t.Parallel() neqVal := "deleted" gtVal := "0" @@ -853,29 +803,29 @@ func TestResolveFilters_MultipleOperators(t *testing.T) { "status": {Neq: &neqVal}, "count": {Gt: >Val}, } - clauses, params := resolveFilters(filters, nil) + clauses, params := predicatesToSQL(resolvePredicates(filters, nil), nil) assert.Len(t, clauses, 2) assert.Len(t, params, 2) } -func TestResolveFilters_LtOperator(t *testing.T) { +func TestFiltersToSQL_LtOperator(t *testing.T) { t.Parallel() ltVal := "100" filters := map[string]Filter{ "price": {Lt: <Val}, } - clauses, params := resolveFilters(filters, nil) + clauses, params := predicatesToSQL(resolvePredicates(filters, nil), nil) require.Len(t, clauses, 1) assert.Contains(t, clauses[0], "`price` < ?") assert.Equal(t, "100", params[0]) } -// TestResolveFilters_UnresolvableClaim_FailsClosed: the #385 fix — a row-filter +// TestFiltersToSQL_UnresolvableClaim_FailsClosed: the #385 fix — a row-filter // template referencing a claim the token doesn't carry emits a constant-false // predicate for EVERY operator, never a real comparison against the empty // string it renders to (where not-equals / greater-than on a string column // would match essentially all rows). -func TestResolveFilters_UnresolvableClaim_FailsClosed(t *testing.T) { +func TestFiltersToSQL_UnresolvableClaim_FailsClosed(t *testing.T) { t.Parallel() tmpl := "{{ jwt.tenant_id }}" partial := "t-{{ jwt.tenant_id }}" @@ -894,7 +844,7 @@ func TestResolveFilters_UnresolvableClaim_FailsClosed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - clauses, params := resolveFilters(map[string]Filter{"tenant_id": tt.filter}, claims) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": tt.filter}, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "1 = 0", clauses[0]) assert.Empty(t, params) @@ -902,14 +852,14 @@ func TestResolveFilters_UnresolvableClaim_FailsClosed(t *testing.T) { } } -// TestResolveFilters_StructuredClaim_FailsClosed: a claim that resolves to a +// TestFiltersToSQL_StructuredClaim_FailsClosed: a claim that resolves to a // JSON object or array — usually a policy typo that dropped the final path // segment ({{ jwt.meta }} for {{ jwt.meta.tenant_id }}) — fails closed on every // operator instead of binding its "map[…]"/"[…]" stringification, which _neq // would match against essentially every row. The one legitimate structured // shape is unaffected: a bare-claim _in against an ARRAY binds its elements -// (TestResolveFilters_InArrayClaim). -func TestResolveFilters_StructuredClaim_FailsClosed(t *testing.T) { +// (TestFiltersToSQL_InArrayClaim). +func TestFiltersToSQL_StructuredClaim_FailsClosed(t *testing.T) { t.Parallel() obj := "{{ jwt.meta }}" arr := "{{ jwt.tids }}" @@ -931,7 +881,7 @@ func TestResolveFilters_StructuredClaim_FailsClosed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - clauses, params := resolveFilters(map[string]Filter{"tenant_id": tt.filter}, claims) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": tt.filter}, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "1 = 0", clauses[0]) assert.Empty(t, params) @@ -939,59 +889,59 @@ func TestResolveFilters_StructuredClaim_FailsClosed(t *testing.T) { } } -// TestResolveFilters_LiteralEmptyValue_Binds: a template-free literal "" is the +// TestFiltersToSQL_LiteralEmptyValue_Binds: a template-free literal "" is the // policy author's chosen value, not a resolution failure — it must keep binding // an equality against the empty string rather than be mistaken for the #385 // fail-closed case. -func TestResolveFilters_LiteralEmptyValue_Binds(t *testing.T) { +func TestFiltersToSQL_LiteralEmptyValue_Binds(t *testing.T) { t.Parallel() empty := "" - clauses, params := resolveFilters(map[string]Filter{"status": {Eq: &empty}}, nil) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"status": {Eq: &empty}}, nil), nil) require.Len(t, clauses, 1) assert.Equal(t, "`status` = ?", clauses[0]) assert.Equal(t, []any{""}, params) } -// TestResolveFilters_EmptyStringClaim_Binds: the boundary of the #385 +// TestFiltersToSQL_EmptyStringClaim_Binds: the boundary of the #385 // fail-closed rule — only an ABSENT or null claim fails closed. A claim present // as an empty string resolves and binds normally, so `_neq` against an // empty-string claim still emits a real `col != ?` predicate bound to the // empty string, never a constant-false predicate. -func TestResolveFilters_EmptyStringClaim_Binds(t *testing.T) { +func TestFiltersToSQL_EmptyStringClaim_Binds(t *testing.T) { t.Parallel() neq := "{{ jwt.tenant_id }}" claims := map[string]any{"tenant_id": ""} - clauses, params := resolveFilters(map[string]Filter{"tenant_id": {Neq: &neq}}, claims) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": {Neq: &neq}}, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` != ?", clauses[0]) assert.Equal(t, []any{""}, params) } -// TestResolveFilters_InEmptyStringClaim_Binds: an _in whose claim is present as +// TestFiltersToSQL_InEmptyStringClaim_Binds: an _in whose claim is present as // an empty STRING is a scalar, not an empty set — it binds as the one-element // set `IN (?)` bound to the empty string. Failing closed is reserved for -// absent/null claims and empty arrays (TestResolveFilters_InEmptyClaim_FailsClosed). -func TestResolveFilters_InEmptyStringClaim_Binds(t *testing.T) { +// absent/null claims and empty arrays (TestFiltersToSQL_InEmptyClaim_FailsClosed). +func TestFiltersToSQL_InEmptyStringClaim_Binds(t *testing.T) { t.Parallel() in := "{{ jwt.tenant_id }}" claims := map[string]any{"tenant_id": ""} - clauses, params := resolveFilters(map[string]Filter{"tenant_id": {In: &in}}, claims) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": {In: &in}}, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` IN (?)", clauses[0]) assert.Equal(t, []any{""}, params) } -// TestResolveFilters_InTemplateWithText_Binds: the resolvable half of the _in +// TestFiltersToSQL_InTemplateWithText_Binds: the resolvable half of the _in // surrounding-text branch — a template with literal text AND a claim the token // carries binds the rendered one-element set `IN (?)`. Its fail-closed twin -// (unresolvable claim → 1 = 0) is TestResolveFilters_UnresolvableClaim_FailsClosed; +// (unresolvable claim → 1 = 0) is TestFiltersToSQL_UnresolvableClaim_FailsClosed; // this guards against a regression to an unconditional nil, which would deny every // row for a policy of this shape while the whole suite still passed. -func TestResolveFilters_InTemplateWithText_Binds(t *testing.T) { +func TestFiltersToSQL_InTemplateWithText_Binds(t *testing.T) { t.Parallel() in := "t-{{ jwt.tenant_id }}" claims := map[string]any{"tenant_id": "x"} - clauses, params := resolveFilters(map[string]Filter{"tenant_id": {In: &in}}, claims) + clauses, params := predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": {In: &in}}, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` IN (?)", clauses[0]) assert.Equal(t, []any{"t-x"}, params) @@ -1027,13 +977,14 @@ func TestEvaluate_FilterUnresolvableClaim_FailsClosed(t *testing.T) { }} perms := Evaluate(p, "user", "clicks", "select", map[string]any{"role": "user"}) require.True(t, perms.Allowed) - assert.Equal(t, "1 = 0", perms.Select.WhereClause) - assert.Empty(t, perms.Select.WhereParams) + clause, params := perms.Select.WhereSQL(nil) + assert.Equal(t, "1 = 0", clause) + assert.Empty(t, params) } // TestValidate_RejectsBindUnsafeFilterColumn: a policy whose row-filter column -// contains '?' is refused at write time — it would shift clickhouse-go's -// positional value binding when interpolated into the WHERE clause. +// contains '?' is refused at write time — it would shift the +// positional-to-named parameter rewrite when interpolated into the WHERE clause. func TestValidate_RejectsBindUnsafeFilterColumn(t *testing.T) { t.Parallel() eq := "{{ jwt.org }}" @@ -1100,42 +1051,42 @@ func TestValidate_AcceptsWellFormedTemplates(t *testing.T) { } } -// TestResolveFilters_InArrayClaim: the headline #224 fix — an _in filter whose +// TestFiltersToSQL_InArrayClaim: the headline #224 fix — an _in filter whose // value is a single array-valued claim expands to `col IN (?, …)` with one bound // param per element, scoping the role to that set instead of producing no // predicate (the former fail-open). -func TestResolveFilters_InArrayClaim(t *testing.T) { +func TestFiltersToSQL_InArrayClaim(t *testing.T) { t.Parallel() in := "{{ jwt.tenants }}" filters := map[string]Filter{"tenant_id": {In: &in}} claims := map[string]any{"tenants": []any{"a", "b", "c"}} - clauses, params := resolveFilters(filters, claims) + clauses, params := predicatesToSQL(resolvePredicates(filters, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` IN (?,?,?)", clauses[0]) assert.Equal(t, []any{"a", "b", "c"}, params) } -// TestResolveFilters_InScalarClaim: a non-array claim yields a single-element IN, +// TestFiltersToSQL_InScalarClaim: a non-array claim yields a single-element IN, // so _in degrades gracefully to the _eq case rather than erroring. -func TestResolveFilters_InScalarClaim(t *testing.T) { +func TestFiltersToSQL_InScalarClaim(t *testing.T) { t.Parallel() in := "{{ jwt.tenant }}" filters := map[string]Filter{"tenant_id": {In: &in}} claims := map[string]any{"tenant": "solo"} - clauses, params := resolveFilters(filters, claims) + clauses, params := predicatesToSQL(resolvePredicates(filters, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` IN (?)", clauses[0]) assert.Equal(t, []any{"solo"}, params) } -// TestResolveFilters_InNonScalarElement_FailsClosed: the structured-claim rule -// (TestResolveFilters_StructuredClaim_FailsClosed) extends INSIDE a bare-claim +// TestFiltersToSQL_InNonScalarElement_FailsClosed: the structured-claim rule +// (TestFiltersToSQL_StructuredClaim_FailsClosed) extends INSIDE a bare-claim // _in array — one object, null, nested-array, or canonical-form-less numeric // element fails the WHOLE set closed. Binding such an element's // "map[…]"/"" rendering would bind a value no row legitimately carries, // and binding only the clean remainder would silently shrink the set the // policy author declared. -func TestResolveFilters_InNonScalarElement_FailsClosed(t *testing.T) { +func TestFiltersToSQL_InNonScalarElement_FailsClosed(t *testing.T) { t.Parallel() in := "{{ jwt.tenants }}" filters := map[string]Filter{"tenant_id": {In: &in}} @@ -1151,7 +1102,7 @@ func TestResolveFilters_InNonScalarElement_FailsClosed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - clauses, params := resolveFilters(filters, map[string]any{"tenants": tt.tenants}) + clauses, params := predicatesToSQL(resolvePredicates(filters, map[string]any{"tenants": tt.tenants}), nil) require.Len(t, clauses, 1) assert.Equal(t, "1 = 0", clauses[0]) assert.Empty(t, params) @@ -1159,89 +1110,131 @@ func TestResolveFilters_InNonScalarElement_FailsClosed(t *testing.T) { } } -// TestResolveFilters_InNumericElements_BindCanonically: scalar elements of a +// TestFiltersToSQL_InNumericElements_BindCanonically: scalar elements of a // bare-claim _in array bind through the same CanonicalScalar rule as every // other operator — numeric spellings canonicalize, large integers keep exact // digits, strings pass through. -func TestResolveFilters_InNumericElements_BindCanonically(t *testing.T) { +func TestFiltersToSQL_InNumericElements_BindCanonically(t *testing.T) { t.Parallel() in := "{{ jwt.tenants }}" filters := map[string]Filter{"tenant_id": {In: &in}} claims := map[string]any{"tenants": []any{json.Number("1.0"), json.Number("12345678901234567890"), "b"}} - clauses, params := resolveFilters(filters, claims) + clauses, params := predicatesToSQL(resolvePredicates(filters, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "`tenant_id` IN (?,?,?)", clauses[0]) assert.Equal(t, []any{"1", "12345678901234567890", "b"}, params) } -// TestResolveFilters_NumericClaimBinding pins the SQL surface of numeric claim +// TestFiltersToSQL_NumericClaimBinding pins the SQL surface of numeric claim // rendering for a hand-built claims map: a json.Number claim binds its canonical // exact digits; a float64 below 2^53 binds positionally (never the "1e+06" // spelling ClickHouse integer columns reject); a float64 at or past 2^53 lost // its digits at decode, so the predicate renders `1 = 0` — matching no rows, the -// same verdict RowVisible reaches in memory — alone or as one _in element. -func TestResolveFilters_NumericClaimBinding(t *testing.T) { +// same verdict the stream reaches on its own surface — alone or as one _in element. +func TestFiltersToSQL_NumericClaimBinding(t *testing.T) { t.Parallel() tmpl := "{{ jwt.tenant }}" eq := map[string]Filter{"tenant_id": {Eq: &tmpl}} - clauses, params := resolveFilters(eq, map[string]any{"tenant": json.Number("10000000000000001")}) + clauses, params := predicatesToSQL(resolvePredicates(eq, map[string]any{"tenant": json.Number("10000000000000001")}), nil) require.Equal(t, []string{"`tenant_id` = ?"}, clauses) assert.Equal(t, []any{"10000000000000001"}, params, "json.Number binds exact digits") - clauses, params = resolveFilters(eq, map[string]any{"tenant": float64(1_000_000)}) + clauses, params = predicatesToSQL(resolvePredicates(eq, map[string]any{"tenant": float64(1_000_000)}), nil) require.Equal(t, []string{"`tenant_id` = ?"}, clauses) assert.Equal(t, []any{"1000000"}, params, "small float binds positionally, not 1e+06") - clauses, params = resolveFilters(eq, map[string]any{"tenant": float64(10000000000000001)}) + clauses, params = predicatesToSQL(resolvePredicates(eq, map[string]any{"tenant": float64(10000000000000001)}), nil) assert.Equal(t, []string{"1 = 0"}, clauses, "lossy float64 claim matches no rows") assert.Empty(t, params) in := "{{ jwt.tenants }}" - clauses, params = resolveFilters(map[string]Filter{"tenant_id": {In: &in}}, - map[string]any{"tenants": []any{"a", float64(1 << 60)}}) + clauses, params = predicatesToSQL(resolvePredicates(map[string]Filter{"tenant_id": {In: &in}}, + map[string]any{"tenants": []any{"a", float64(1 << 60)}}), nil) assert.Equal(t, []string{"1 = 0"}, clauses, "one poisoned element resolves the whole set empty") assert.Empty(t, params) } -// TestCompareCanonicalDecimals pins the digit-string ordering over canonical -// forms — the comparison twin of canonicalDecimal, exact at any width, never a -// float round-trip. Each pair is asserted in both directions. -func TestCompareCanonicalDecimals(t *testing.T) { +// TestRowFilter_UnresolvableClaim_NoRowsOnBothPaths pins the #457 fail-closed +// rule on BOTH read surfaces at once: a filter template whose claim the token +// doesn't carry renders the constant-false predicate on the query path AND +// yields a predicate with NO values on the stream path, which the type layer +// refuses without compiling anything. One Evaluate resolution drives both, so a +// claim-less token can never see zero rows on /v1/query yet every row on +// /v1/stream. HasRowFilter must stay true for the failed predicate — dropping it +// would put the role back on the unfiltered once-per-role fast path, the exact +// fail-open this test exists to prevent. +func TestRowFilter_UnresolvableClaim_NoRowsOnBothPaths(t *testing.T) { t.Parallel() + noTenant := map[string]any{"role": "user"} // validly signed token, no tenant claim tests := []struct { - a, b string - want int + name string + filter map[string]Filter + claims map[string]any }{ - {"0", "0", 0}, - {"1", "2", -1}, - {"9", "100", -1}, - {"-1", "1", -1}, - {"-2", "-1", -1}, - {"-100", "-9", -1}, - {"1.5", "1.5", 0}, - {"1.05", "1.5", -1}, - {"0.5", "0.55", -1}, - {"2", "2.5", -1}, - {"-1.5", "-1", -1}, - {"0.0025", "0.003", -1}, - {"12345678901234567890", "12345678901234567891", -1}, - {"9007199254740992", "9007199254740993", -1}, + {"_eq", map[string]Filter{"tenant_id": {Eq: new("{{ jwt.tenant }}")}}, noTenant}, + {"_neq, the leak direction", map[string]Filter{"tenant_id": {Neq: new("{{ jwt.tenant }}")}}, noTenant}, + {"_gt", map[string]Filter{"tenant_id": {Gt: new("{{ jwt.tenant }}")}}, noTenant}, + {"_in with surrounding text", map[string]Filter{"tenant_id": {In: new("t-{{ jwt.tenant }}")}}, noTenant}, + { + "object claim in a scalar slot", + map[string]Filter{"tenant_id": {Eq: new("{{ jwt.meta }}")}}, + map[string]any{"meta": map[string]any{"tenant": "acme"}}, + }, } for _, tt := range tests { - assert.Equal(t, tt.want, compareCanonicalDecimals(tt.a, tt.b), "%s vs %s", tt.a, tt.b) - assert.Equal(t, -tt.want, compareCanonicalDecimals(tt.b, tt.a), "%s vs %s reversed", tt.b, tt.a) + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p := &Policy{Tables: map[string]TablePolicy{ + "t": {"r": {Select: &SelectPermissions{Filter: tt.filter}}}, + }} + perms := Evaluate(p, "r", "t", "select", tt.claims) + + clause, params := perms.Select.WhereSQL(nil) + assert.Equal(t, "1 = 0", clause, "query path: constant-false predicate") + assert.Empty(t, params) + assert.True(t, perms.HasRowFilter(), "failed predicate must keep the stream on the per-subscriber path") + + preds, ok := perms.Predicates() + require.True(t, ok, "a resolved read side answers with its predicates") + require.Len(t, preds, 1, "the failed predicate is present, not dropped") + assert.Equal(t, "tenant_id", preds[0].Column) + assert.Empty(t, preds[0].Values, + "stream path: no values to bind, which matches no row without compiling anything") + }) } } -// TestResolveFilters_InEmptyClaim_FailsClosed: an empty set makes the predicate +// TestPredicates_IsTheSameResolutionAsTheWhereClause: the two read surfaces are +// rendered from ONE resolvePredicates call, so the predicates handed to the +// stream carry exactly the values bound into the query's WHERE — in the same +// order. A second resolution, even of the same policy, is what #457 was about. +func TestPredicates_IsTheSameResolutionAsTheWhereClause(t *testing.T) { + t.Parallel() + p := &Policy{Tables: map[string]TablePolicy{ + "t": {"r": {Select: &SelectPermissions{Filter: map[string]Filter{ + "tenant_id": {Eq: new("{{ jwt.tenant }}")}, + }}}}, + }} + perms := Evaluate(p, "r", "t", "select", map[string]any{"tenant": "acme"}) + + clause, params := perms.Select.WhereSQL(nil) + assert.Equal(t, "`tenant_id` = ?", clause) + assert.Equal(t, []any{"acme"}, params) + + preds, ok := perms.Predicates() + require.True(t, ok) + assert.Equal(t, []Predicate{{Column: "tenant_id", Op: "=", Values: []string{"acme"}}}, preds) +} + +// TestFiltersToSQL_InEmptyClaim_FailsClosed: an empty set makes the predicate // match no rows (a constant-false predicate) rather than widen to all rows — the // fail-closed direction. `IN ()` is invalid SQL. Two distinct branches of // resolveInValues reach this: an absent claim (navigateClaims returns nil, which // CanonicalScalar rejects in its nil/object/array case, so resolveInValues' // `default` branch fails the set closed) and a present-but-empty array (the // `case []any` branch with zero elements). Both must fail closed. -func TestResolveFilters_InEmptyClaim_FailsClosed(t *testing.T) { +func TestFiltersToSQL_InEmptyClaim_FailsClosed(t *testing.T) { t.Parallel() in := "{{ jwt.tenants }}" filters := map[string]Filter{"tenant_id": {In: &in}} @@ -1252,7 +1245,7 @@ func TestResolveFilters_InEmptyClaim_FailsClosed(t *testing.T) { for name, claims := range cases { t.Run(name, func(t *testing.T) { t.Parallel() - clauses, params := resolveFilters(filters, claims) + clauses, params := predicatesToSQL(resolvePredicates(filters, claims), nil) require.Len(t, clauses, 1) assert.Equal(t, "1 = 0", clauses[0]) assert.Empty(t, params) @@ -1261,8 +1254,8 @@ func TestResolveFilters_InEmptyClaim_FailsClosed(t *testing.T) { } // TestEvaluate_FilterInClause: end-to-end through Evaluate, an _in filter lands -// in the role's WhereClause/WhereParams (exercising the bind-safe guard + IN -// assembly), not just the resolveFilters unit. +// in the role's WhereSQL (exercising the bind-safe guard + IN assembly), not +// just the resolvePredicates+predicatesToSQL pair below it. func TestEvaluate_FilterInClause(t *testing.T) { t.Parallel() in := "{{ jwt.app_metadata.tenant_ids }}" @@ -1272,8 +1265,9 @@ func TestEvaluate_FilterInClause(t *testing.T) { claims := map[string]any{"app_metadata": map[string]any{"tenant_ids": []any{"t1", "t2"}}} perms := Evaluate(p, "user", "clicks", "select", claims) require.True(t, perms.Allowed) - assert.Contains(t, perms.Select.WhereClause, "`tenant_id` IN (?,?)") - assert.Equal(t, []any{"t1", "t2"}, perms.Select.WhereParams) + clause, params := perms.Select.WhereSQL(nil) + assert.Contains(t, clause, "`tenant_id` IN (?,?)") + assert.Equal(t, []any{"t1", "t2"}, params) } // TestEvaluate_CheckInResolvesToSet: an _in check resolves to a []any set in @@ -1548,14 +1542,16 @@ func TestEvaluate_UnresolvedSideFailsClosed(t *testing.T) { assert.False(t, ins.IsAggregationAllowed("count")) assert.True(t, ins.HasRowFilter(), "the GATE must not report 'no filter' — that sends the caller down the "+ - "whole-bucket fast path where RowVisible is never consulted") - assert.False(t, ins.RowVisible(map[string]any{"tenant_id": "acme"}, nil), - "an unresolved read side must not admit every row") + "whole-bucket fast path where no row is ever checked") + _, insRows := ins.Predicates() + assert.False(t, insRows, + "an unresolved read side must refuse the row question, not answer 'no predicates'") - // The select-resolved grant still evaluates its row filter normally. + // The select-resolved grant still hands out its row filter normally. assert.True(t, sel.HasRowFilter()) - assert.True(t, sel.RowVisible(map[string]any{"tenant_id": "acme"}, nil)) - assert.False(t, sel.RowVisible(map[string]any{"tenant_id": "globex"}, nil)) + selPreds, selRows := sel.Predicates() + require.True(t, selRows) + assert.Equal(t, []Predicate{{Column: "tenant_id", Op: "=", Values: []string{"acme"}}}, selPreds) } // TestHandBuiltPermissions_PresentSidesKeepPlainReading: a value assembled by @@ -1569,7 +1565,9 @@ func TestHandBuiltPermissions_PresentSidesKeepPlainReading(t *testing.T) { assert.True(t, rp.IsColumnAllowed("anything", true)) assert.True(t, rp.IsAggregationAllowed("count")) assert.False(t, rp.RestrictsColumns()) - assert.True(t, rp.RowVisible(map[string]any{"a": 1}, nil)) + preds, rows := rp.Predicates() + assert.True(t, rows) + assert.Empty(t, preds, "a resolved but unfiltered read side admits every row") // An EMPTY insert side has no checks and is resolved — not the same answer as // a nil one below. A slip to `rp.Insert != nil && len(...) > 0` would break // exactly here. @@ -1591,10 +1589,11 @@ func TestHandBuiltPermissions_NilSideDenies(t *testing.T) { assert.False(t, insertOnly.IsAggregationAllowed("count")) assert.True(t, insertOnly.RestrictsColumns(), "an unresolved read side restricts everything") // HasRowFilter says YES on an unresolved read side on purpose: it routes the - // hub onto the per-subscriber path where RowVisible denies, instead of the - // no-filter fast path that never consults RowVisible at all. + // hub onto the per-subscriber path where Predicates denies, instead of the + // no-filter fast path that never asks about a row at all. assert.True(t, insertOnly.HasRowFilter(), "must not take the no-filter fast path") - assert.False(t, insertOnly.RowVisible(map[string]any{"a": 1}, nil), "and the per-row check denies") + _, insertOnlyRows := insertOnly.Predicates() + assert.False(t, insertOnlyRows, "and the per-row question is refused") assert.Empty(t, insertOnly.AllowedProjection([]string{"a", "b"})) _, insertChecksOK := insertOnly.CheckClauses() @@ -1672,7 +1671,7 @@ func TestEvaluate_OperatorLessFilterAndCheckDenyFailClosed(t *testing.T) { // `"tenant_id": {}` survives a strict decode — every Filter operator is // omitempty — and then matches no case in either resolver, so the declared // restriction resolves to nothing. Before this was refused, the policy below - // validated clean and RowVisible answered true for every tenant. + // validated clean and the row filter admitted every tenant. sel := &Policy{Tables: map[string]TablePolicy{ "clicks": {"viewer": {Select: &SelectPermissions{ AllowColumns: []string{"*"}, @@ -1698,8 +1697,48 @@ func TestEvaluate_OperatorLessFilterAndCheckDenyFailClosed(t *testing.T) { selPerms := Evaluate(sel, "viewer", "clicks", "select", nil) assert.False(t, selPerms.Allowed, "an operator-less filter must deny, not read as unrestricted") assert.True(t, selPerms.HasRowFilter(), "a denied grant gates every row") - assert.False(t, selPerms.RowVisible(map[string]any{"tenant_id": "someone-else"}, nil)) + _, selRows := selPerms.Predicates() + assert.False(t, selRows, "and refuses the row question rather than reading as unfiltered") insPerms := Evaluate(ins, "writer", "clicks", "insert", nil) assert.False(t, insPerms.Allowed, "an operator-less check must deny, not drop the rule") } + +// TestWhereSQL_IntegerColumnsBindThroughTheStrictCast: given the column types, +// a claim on an integer column binds as a chsql.IntParam carrying the bare +// integer type (the query builder expands it to chsql.StrictInt), and a claim +// on any other column binds as the plain string it always did. Without types +// (nil) every claim is a plain string. +func TestWhereSQL_IntegerColumnsBindThroughTheStrictCast(t *testing.T) { + t.Parallel() + types := map[string]string{"tenant": "Nullable(UInt64)", "org": "String", "n": "Int128"} + p := &Policy{Tables: map[string]TablePolicy{ + "t": {"r": {Select: &SelectPermissions{Filter: map[string]Filter{ + "tenant": {Eq: new("{{ jwt.tenant }}")}, + "org": {Neq: new("x")}, + "n": {In: new("{{ jwt.ns }}")}, + }}}}, + }} + claims := map[string]any{"tenant": "18446744073709551621", "ns": []any{"1", "-2"}} + perms := Evaluate(p, "r", "t", "select", claims) + require.True(t, perms.Allowed) + + clause, params := perms.Select.WhereSQL(func(c string) string { return types[c] }) + byClause := map[string][]any{} + i := 0 + for part := range strings.SplitSeq(clause, " AND ") { + n := strings.Count(part, "?") + byClause[part] = params[i : i+n] + i += n + } + require.Equal(t, len(params), i, "every ? has exactly one param") + assert.Equal(t, []any{chsql.IntParam{Value: "18446744073709551621", Type: "UInt64"}}, byClause["`tenant` = ?"]) + assert.Equal(t, []any{"x"}, byClause["`org` != ?"]) + assert.Equal(t, []any{chsql.IntParam{Value: "1", Type: "Int128"}, chsql.IntParam{Value: "-2", Type: "Int128"}}, + byClause["`n` IN (?,?)"]) + + _, untyped := perms.Select.WhereSQL(nil) + for _, v := range untyped { + assert.IsType(t, "", v, "no column types: every claim is a plain string") + } +} diff --git a/internal/policy/rowfilter.go b/internal/policy/rowfilter.go deleted file mode 100644 index a318e4d4..00000000 --- a/internal/policy/rowfilter.go +++ /dev/null @@ -1,294 +0,0 @@ -package policy - -import ( - "encoding/json" - "strings" - "time" -) - -// HasRowFilter reports whether this role/table entry carries a row-level-security -// predicate. The stream fan-out uses it to decide whether an event can be projected -// once for a whole role bucket (no filter) or must be checked per subscriber against -// that subscriber's claims (filter present). A nil receiver (no policy applies) has -// no filter. -// -// It answers YES for a denied grant and for one whose read side was never -// resolved, neither of which has a predicate to speak of. That is deliberate: -// this is the GATE in front of RowVisible, and a "no filter" answer sends the -// caller down the deliver-to-the-whole-bucket fast path where RowVisible is -// never consulted. Saying yes forces the per-subscriber path, where RowVisible -// denies. Same shape and same reason as RestrictsColumns, which guards the -// builder's SELECT * expansion. -func (p *ResolvedPermissions) HasRowFilter() bool { - if p == nil { - return false - } - if !p.Allowed || p.Select == nil { - return true - } - return len(p.Select.rowFilter) > 0 -} - -// maxTimeOperandChars is the same O(1) pre-gate for timestamp operands: the -// ingest grammar's longest accepted spelling (RFC 3339 with nanoseconds and a -// numeric offset) is 35 bytes, so 64 is generous slack — and the parser scans -// its input, which without the gate a megabyte "timestamp" would make a -// per-subscriber-per-event cost. -const maxTimeOperandChars = 64 - -// ColumnKind classifies a column's ClickHouse type for the in-memory row-filter -// comparison. The zero value is ColumnOpaque, so a nil map, a column absent from -// the map, and a column the schema doesn't know all land on the most conservative -// class — the three "no type knowledge" states are indistinguishable and equally -// closed, never a silent downgrade to a laxer comparison. -type ColumnKind uint8 - -const ( - // ColumnOpaque: no usable type knowledge (no schema, unknown column) or a type - // whose text rendering is not canonical — UUID (case), Enum (name vs number), - // Bool (true vs 1), Date/Date32 (producer spelling), IPv4/IPv6, … For these only - // byte-equality is trustworthy: identical strings parse to identical ClickHouse - // values, but differing strings prove nothing. So = and in admit exactly the - // event's own rendering, while !=, > and < fail closed (the row is withheld). - ColumnOpaque ColumnKind = iota - // ColumnNumeric (Int*/UInt*/Float*/Decimal*): both operands render to exact - // canonical decimal form through the claim side's #457 machinery, then - // compare in the column's STORAGE domain (ColumnSpec.Numeric): integers at - // any width exactly, floats after IEEE narrowing to the column's bit width, - // decimals after truncation to the column's scale — the same narrowing - // ClickHouse applies to the stored value and the bound constant, so stream - // and query verdicts agree even on narrowing columns. - ColumnNumeric - // ColumnText (String, incl. Nullable/LowCardinality): byte comparison is - // ClickHouse comparison — equality AND lexicographic order — so every operator - // is exact. FixedString is NOT ColumnText (zero-padded storage). - ColumnText - // ColumnTime (DateTime/DateTime64): operands parse as instants through the - // caller-supplied ColumnSpec.ParseTime — the same grammar, zone rule, and - // range guard ingest canonicalization applies — and compare chronologically, - // so every operator is exact across spellings: a zone-less filter constant - // matches the canonicalized RFC 3339 payload denoting the same instant. A - // side that can't be read as a provable instant fails closed. - ColumnTime -) - -// ColumnSpec is one column's comparison contract for the in-memory row filter: -// the ColumnKind classification plus the kind's parameters — ColumnTime's -// instant parser, ColumnNumeric's storage model. The zero value is ColumnOpaque -// with neither, so a nil map, an absent column, and an unknown type all land on -// the most conservative class — never a silent downgrade to a laxer comparison. -type ColumnSpec struct { - Kind ColumnKind - // ParseTime converts one rendering of this timestamp column's value — an - // ingested payload value (string / json.Number / float64) or a resolved - // filter constant (always a string) — to the instant ClickHouse would store, - // truncated to the column's precision. ok=false (unparseable, or outside the - // column type's range, which insert-time saturation would move) fails the - // comparison closed. Set iff Kind is ColumnTime; the stream supplies it from - // the schema registry (discovery's Column.TimeParser) so the filter and - // ingest canonicalization can never disagree on the grammar. - ParseTime func(v any) (t time.Time, ok bool) - // Numeric is the column's storage model, set iff Kind is ColumnNumeric - // (from discovery.NumericStorageOf via the stream's columnSpecs). Its zero - // value refuses every comparison, so a ColumnNumeric spec built without a - // model fails closed rather than comparing under the wrong semantics. - Numeric NumericSpec -} - -// RowVisible reports whether row satisfies every resolved row-filter predicate — the -// in-memory twin of the query path's WHERE clause, evaluated against a decoded event -// so the stream applies the same row-level security the query path does. Predicates -// are ANDed; the query path joins them with AND too. -// -// cols maps column name → ColumnSpec, supplied by the caller from the table -// schema (see stream.Hub's columnSpecs). Numeric columns compare numerically in -// the column's storage domain (9 < 100, as ClickHouse would; Float/Decimal -// operands narrowed the way insert and constant binding narrow them), String -// columns compare bytewise (exactly ClickHouse's String collation), -// DateTime/DateTime64 columns compare as instants (both operands parsed through -// the spec's ParseTime, the same grammar ingest canonicalizes with), and -// everything else — including every column when no schema is available — admits -// only byte-equality (= / in) and fails !=, > and < closed: the evaluator cannot -// mirror ClickHouse's per-type coercion, and text comparison there could admit rows -// the query path excludes ("9" > "100" as text, an uppercase UUID under !=). Every -// ambiguous or uncomparable case fails closed — the row is hidden, never leaked — -// so the boundary costs availability, not confidentiality. -// -// That guarantee is about the INGESTED PAYLOAD value, which is what the stream -// evaluates; the query path evaluates the stored row. Storage-domain narrowing -// keeps the two verdicts aligned for values ClickHouse stores; the residual -// asymmetry is an event whose INSERT later fails entirely (out-of-range value, -// batch error → DLQ): it was already streamed to whoever the filter admitted, -// and the row never becomes queryable. Documented in access-control.mdx's -// enforcement caution. -// -// A nil receiver (no policy applies) makes every row visible. -func (p *ResolvedPermissions) RowVisible(row map[string]any, cols map[string]ColumnSpec) bool { - if p == nil { - return true - } - // A denied role sees no rows — fail closed, mirroring the !Allowed guard on - // IsColumnAllowed, so a denied receiver never reads as "no filter ⇒ all visible". - // A grant resolved for INSERT is refused here for the same reason: its empty - // read side would otherwise read as "no filter", admitting every row. - if !p.Allowed || p.Select == nil { - return false - } - for _, pred := range p.Select.rowFilter { - if !pred.matches(row, cols[pred.Column]) { - return false - } - } - return true -} - -// matches evaluates one predicate against the row, failing closed (false) whenever -// the value is absent or can't be compared as required. -func (pred resolvedPredicate) matches(row map[string]any, spec ColumnSpec) bool { - // No values ⇒ matches nothing: an empty/unresolvable "in" set, or a scalar - // whose constant was unrenderable — the in-memory twin of the `1 = 0` - // predicatesToSQL emits for the same cases. - if len(pred.Values) == 0 { - return false - } - raw, ok := row[pred.Column] - if !ok { - return false // column not in the event ⇒ can't prove the row is allowed - } - switch pred.Op { - case "=": - c, ok := compareScalar(raw, pred.Values[0], spec) - return ok && c == 0 - case "!=": - c, ok := compareScalar(raw, pred.Values[0], spec) - return ok && c != 0 - case ">": - c, ok := compareScalar(raw, pred.Values[0], spec) - return ok && c > 0 - case "<": - c, ok := compareScalar(raw, pred.Values[0], spec) - return ok && c < 0 - case "in": - for _, v := range pred.Values { - if c, ok := compareScalar(raw, v, spec); ok && c == 0 { - return true - } - } - return false - default: - return false - } -} - -// compareScalar compares an event value against a resolved filter value, returning -// -1/0/+1 and ok=false when the comparison can't be made: a non-scalar event value, -// a numeric comparison whose operands don't parse as numbers, or two unequal values -// of a ColumnOpaque column (where inequality and order are unprovable). Every -// ok=false fails the enclosing predicate closed — for != that is what keeps a mere -// representation difference (an uppercase UUID, 1 for a Bool true) from being -// mistaken for a real inequality and admitting a row the query path excludes. -// filterVal is the raw resolved spelling — what byte-comparison arms compare -// and predicatesToSQL binds; the numeric arm derives its canonical reading per -// comparison, behind an O(1) length gate (see maxNumericOperandChars — folding -// the re-derivation into a memoized resolution is part of #435's scope). -func compareScalar(rowVal any, filterVal string, spec ColumnSpec) (int, bool) { - switch spec.Kind { - case ColumnTime: - // The RAW value goes to the parser — a payload timestamp may legitimately - // be a number (Unix seconds/ticks), which the spec's parser reads the same - // way ingest does; scalarString's rendering would be a detour. Both sides - // must parse; either failing (or a spec missing its parser) refuses the - // comparison, fail closed. - if spec.ParseTime == nil { - return 0, false - } - // O(1) length gate before the parser scans either operand (see - // maxTimeOperandChars). The payload arrives as a string OR a - // json.Number (a Unix-epoch timestamp) — both are client-controlled and - // must be gated; a bare-number payload that skipped this would leave the - // parser's digit scan (and its %q error rendering) unbounded on the - // fan-out path. Verdict-preserving for numbers (>19 digits never parsed) - // and, for strings, refuses only past 64 bytes, where Go's parser would - // have truncated sub-second digits rather than matched a real instant. - switch v := rowVal.(type) { - case string: - if len(v) > maxTimeOperandChars { - return 0, false - } - case json.Number: - if len(v) > maxTimeOperandChars { - return 0, false - } - } - if len(filterVal) > maxTimeOperandChars { - return 0, false - } - a, ok := spec.ParseTime(rowVal) - if !ok { - return 0, false - } - b, ok := spec.ParseTime(filterVal) - if !ok { - return 0, false - } - return a.Compare(b), true - case ColumnNumeric: - // Both operands route through the ONE canonical numeric gate the claim - // side already uses (#457's CanonicalScalar machinery): exact decimal - // form at any width, digit-bounded (the superlinear-parse guard lives - // there — CWE-400, the row operand is client-controlled), with "NaN", - // any Inf spelling, and every non-JSON-number rendering refused by the - // grammar rather than by ad-hoc checks. Then the comparison itself runs - // in the column's storage domain (NumericSpec.compare), narrowing both - // sides the way ClickHouse narrows the stored value and the constant. - if len(filterVal) > maxNumericOperandChars { - return 0, false - } - a, ok := numericCanonical(rowVal) - if !ok { - return 0, false - } - b, ok := CanonicalNumericLiteral(filterVal) - if !ok { - return 0, false - } - // Spelling fidelity, integer family only: predicatesToSQL binds the - // constant AS WRITTEN, and ClickHouse's integer cast in a WHERE rejects - // non-plain spellings ("1e3", "1.5", "007") with a per-query type - // error — the role reads no rows there, so comparing the canonical - // reading here would ADMIT rows SQL never returns. A constant that - // isn't its own canonical form refuses the comparison instead - // (withhold — matching SQL's nothing, just quietly; the one measured - // over-refusal is "-0", which ClickHouse accepts in a WHERE but the - // canonical fold rewrites to "0" — availability, never exposure). - // Claim-derived constants are canonical by construction and - // unaffected. Float AND Decimal casts accept every JSON-number - // spelling ('1e3' casts to Decimal as 1000 — verified), so neither - // family gets a gate. - if spec.Numeric.Family == NumericInteger && b != filterVal { - return 0, false - } - return spec.Numeric.compare(a, b) - case ColumnText: - s, ok := scalarString(rowVal) - if !ok { - return 0, false - } - return strings.Compare(s, filterVal), true - case ColumnOpaque: - // Byte-equality is the only relation provable without type knowledge - // (identical strings always parse to the same ClickHouse value); unequal - // bytes prove nothing, so the comparison is refused and the predicate - // fails closed. - s, ok := scalarString(rowVal) - if !ok { - return 0, false - } - if s == filterVal { - return 0, true - } - return 0, false - default: - return 0, false // unknown future kind: refuse to compare, fail closed - } -} diff --git a/internal/policy/rowfilter_test.go b/internal/policy/rowfilter_test.go deleted file mode 100644 index bda90f4d..00000000 --- a/internal/policy/rowfilter_test.go +++ /dev/null @@ -1,543 +0,0 @@ -package policy - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -// intNumeric is the Int64 numeric spec most tests compare under — exact within -// the width's range, the storage model of the default integer id column. Float -// and Decimal narrowing, other widths, and range gating have dedicated tests. -func intNumeric() ColumnSpec { - return ColumnSpec{Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericInteger, Bits: 64}} -} - -// evalRowFilter builds a one-role/one-table policy carrying filter and returns the -// permissions resolved against claims, so tests exercise the full -// resolvePredicates → RowVisible path the stream fan-out uses. -func evalRowFilter(t *testing.T, filter map[string]Filter, claims map[string]any) *ResolvedPermissions { - t.Helper() - p := &Policy{Tables: map[string]TablePolicy{ - "t": {"r": {Select: &SelectPermissions{Filter: filter}}}, - }} - return Evaluate(p, "r", "t", "select", claims) -} - -// TestRowVisible_EqualityScoping covers the canonical row-level-security shape — -// tenant_id = {{ jwt.tenant }} — including the fail-closed on a missing column that -// stops an event without the filtered field from slipping through. -func TestRowVisible_EqualityScoping(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"tenant_id": {Eq: new("{{ jwt.tenant }}")}}, map[string]any{"tenant": "acme"}) - is := assert.New(t) - is.True(perms.HasRowFilter()) - is.True(perms.RowVisible(map[string]any{"tenant_id": "acme"}, nil)) - is.False(perms.RowVisible(map[string]any{"tenant_id": "globex"}, nil)) - is.False(perms.RowVisible(map[string]any{"other": "x"}, nil), "missing filtered column ⇒ fail closed") -} - -// TestRowVisible_InSet covers _in against an array claim (multi-tenant scoping) and -// the fail-closed empty-set case (absent claim never widens to all rows). -func TestRowVisible_InSet(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"tenant_id": {In: new("{{ jwt.tenants }}")}}, map[string]any{"tenants": []any{"a", "b"}}) - assert.True(t, perms.RowVisible(map[string]any{"tenant_id": "b"}, nil)) - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": "c"}, nil)) - - empty := evalRowFilter(t, map[string]Filter{"tenant_id": {In: new("{{ jwt.tenants }}")}}, nil) - assert.True(t, empty.HasRowFilter()) - assert.False(t, empty.RowVisible(map[string]any{"tenant_id": "a"}, nil), "absent claim ⇒ empty set ⇒ no row matches") -} - -// TestRowVisible_Neq: != admits a row only when inequality is PROVABLE — a numeric -// column comparing numerically or a String column comparing bytewise. On a column -// with no usable type (no schema, or a type like UUID/Bool whose text rendering -// isn't canonical) a byte difference may be pure representation — 'ABC-def' vs -// 'abc-def' for a UUID ClickHouse would treat as equal — so != fails closed rather -// than admit a row the query path excludes. -func TestRowVisible_Neq(t *testing.T) { - t.Parallel() - text := map[string]ColumnSpec{"status": {Kind: ColumnText}} - perms := evalRowFilter(t, map[string]Filter{"status": {Neq: new("deleted")}}, nil) - assert.True(t, perms.RowVisible(map[string]any{"status": "active"}, text), "String column: byte inequality is real inequality") - assert.False(t, perms.RowVisible(map[string]any{"status": "deleted"}, text)) - - assert.False(t, perms.RowVisible(map[string]any{"status": "active"}, nil), - "no schema: byte inequality proves nothing, fail closed") - - uuid := evalRowFilter(t, map[string]Filter{"device": {Neq: new("ABC-DEF")}}, nil) - assert.False(t, uuid.RowVisible(map[string]any{"device": "abc-def"}, map[string]ColumnSpec{"device": {Kind: ColumnOpaque}}), - "opaque column (e.g. UUID): a case difference is not proof of inequality — fail closed") - - num := evalRowFilter(t, map[string]Filter{"amount": {Neq: new("100")}}, nil) - kinds := map[string]ColumnSpec{"amount": intNumeric()} - assert.True(t, num.RowVisible(map[string]any{"amount": float64(250)}, kinds), "numeric column: 250 ≠ 100 is provable") - assert.False(t, num.RowVisible(map[string]any{"amount": float64(100)}, kinds)) -} - -// TestRowVisible_Ordering_SchemaInformed: ordering needs type knowledge. An event -// value of 9 is numerically LESS than 100 but lexicographically GREATER ("9" > -// "100") — so a numeric column compares numerically (matching ClickHouse), a String -// column compares bytewise (which IS ClickHouse's String order), and a column with -// no usable type fails closed: no schema, a column the schema doesn't know, and a -// non-numeric non-String type (Enum order follows enum values, not names; Date -// formats vary) must all withhold rather than fall back to a text comparison that -// can admit rows the query path excludes. -func TestRowVisible_Ordering_SchemaInformed(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"amount": {Gt: new("100")}}, nil) - small := map[string]any{"amount": float64(9)} - big := map[string]any{"amount": float64(250)} - numeric := map[string]ColumnSpec{"amount": intNumeric()} - - assert.False(t, perms.RowVisible(small, numeric), "numeric: 9 is not > 100") - assert.True(t, perms.RowVisible(big, numeric)) - - assert.False(t, perms.RowVisible(small, nil), `no schema: fail closed — never the "9" > "100" text leak`) - assert.False(t, perms.RowVisible(big, nil), "no schema: fail closed even when the numbers would pass") - assert.False(t, perms.RowVisible(small, map[string]ColumnSpec{"other": intNumeric()}), - "column absent from a known schema: fail closed") - assert.False(t, perms.RowVisible(small, map[string]ColumnSpec{"amount": {Kind: ColumnOpaque}}), - "opaque type (Enum/Date/UUID/…): order is unprovable, fail closed") - - // String columns order bytewise in ClickHouse, so ordering there is exact. - page := evalRowFilter(t, map[string]Filter{"page": {Gt: new("/m")}}, nil) - text := map[string]ColumnSpec{"page": {Kind: ColumnText}} - assert.True(t, page.RowVisible(map[string]any{"page": "/z"}, text)) - assert.False(t, page.RowVisible(map[string]any{"page": "/a"}, text)) -} - -// TestRowVisible_NumericEquality_FloatFormatting: a JSON float64(100) equals the -// string filter value "100" under numeric comparison, so integer-valued numeric -// columns aren't tripped up by float formatting. -func TestRowVisible_NumericEquality_FloatFormatting(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"amount": {Eq: new("100")}}, nil) - num := map[string]ColumnSpec{"amount": intNumeric()} - assert.True(t, perms.RowVisible(map[string]any{"amount": float64(100)}, num)) - assert.False(t, perms.RowVisible(map[string]any{"amount": float64(101)}, num)) -} - -// TestRowVisible_NaN_FailsClosed: strconv.ParseFloat accepts "NaN", and NaN's -// three-way comparison would otherwise read as "equal to everything" — a fail-open -// that delivers a row the query path's WHERE excludes. Either operand parsing to -// NaN must withhold the row instead. -func TestRowVisible_NaN_FailsClosed(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"amount": intNumeric()} - - byRow := evalRowFilter(t, map[string]Filter{"amount": {Eq: new("100")}}, nil) - assert.False(t, byRow.RowVisible(map[string]any{"amount": "NaN"}, num), "NaN row value must not equal 100") - - byClaim := evalRowFilter(t, map[string]Filter{"amount": {Eq: new("{{ jwt.cap }}")}}, map[string]any{"cap": "NaN"}) - assert.False(t, byClaim.RowVisible(map[string]any{"amount": float64(100)}, num), "NaN claim must not match any row") - - neq := evalRowFilter(t, map[string]Filter{"amount": {Neq: new("NaN")}}, nil) - assert.False(t, neq.RowVisible(map[string]any{"amount": float64(100)}, num), "NaN is uncomparable, so even != fails closed") -} - -// TestRowVisible_Inf_FailsClosed: ParseFloat also accepts "Inf"/"+Inf"/"-Inf"/ -// "Infinity" (any case), and an infinite bound would make _gt/_lt admit every -// finite row — a fail-open the query path can't reproduce (ClickHouse rejects -// binding an Inf-spelled string to an integer column). Either operand parsing -// to ±Inf must withhold the row instead. -func TestRowVisible_Inf_FailsClosed(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"amount": intNumeric()} - - byClaim := evalRowFilter(t, map[string]Filter{"amount": {Gt: new("{{ jwt.min }}")}}, map[string]any{"min": "-Inf"}) - assert.False(t, byClaim.RowVisible(map[string]any{"amount": float64(5)}, num), "-Inf lower bound must not admit finite rows") - - lt := evalRowFilter(t, map[string]Filter{"amount": {Lt: new("Infinity")}}, nil) - assert.False(t, lt.RowVisible(map[string]any{"amount": float64(5)}, num), "Infinity upper bound must not admit finite rows") - - byRow := evalRowFilter(t, map[string]Filter{"amount": {Gt: new("100")}}, nil) - assert.False(t, byRow.RowVisible(map[string]any{"amount": "+Inf"}, num), "Inf row value is uncomparable, fail closed") -} - -// TestRowVisible_NumericEquality_ExactBeyondFloat64: ingest accepts string-encoded -// numerics precisely so 64-bit IDs survive JS precision loss; equality must not -// collapse distinct IDs that round to the same float64 (adjacent values past 2^53). -func TestRowVisible_NumericEquality_ExactBeyondFloat64(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"id": intNumeric()} - - perms := evalRowFilter(t, map[string]Filter{"id": {Eq: new("9007199254740993")}}, nil) - assert.False(t, perms.RowVisible(map[string]any{"id": "9007199254740992"}, num), "float64-equal neighbors are not equal") - assert.True(t, perms.RowVisible(map[string]any{"id": "9007199254740993"}, num)) - assert.True(t, perms.RowVisible(map[string]any{"id": "9007199254740993.0"}, num), "same value in a different rendering still matches") - - // Bare JSON numbers reach RowVisible as json.Number (the stream decodes with - // UseNumber), so the same exactness holds without string-encoding: a lossy - // float64 decode would have collapsed these neighbors and delivered another - // tenant's row. - assert.False(t, perms.RowVisible(map[string]any{"id": json.Number("9007199254740992")}, num), "json.Number neighbor is not equal") - assert.True(t, perms.RowVisible(map[string]any{"id": json.Number("9007199254740993")}, num)) - - neq := evalRowFilter(t, map[string]Filter{"id": {Neq: new("9007199254740993")}}, nil) - assert.True(t, neq.RowVisible(map[string]any{"id": "9007199254740992"}, num), "the exact comparison keeps distinct IDs unequal for !=") -} - -// TestRowVisible_OverlongNumericOperand_FailsClosed: an over-long operand is -// refused by the O(1) length pre-gate (maxNumericOperandChars) before ANY scan -// of its bytes — the row operand is client-controlled and the comparison runs -// per subscriber per event on the fan-out goroutine, so even a linear -// json.Valid pass over it is a cost an attacker controls. The gate is -// verdict-preserving: anything longer already fails the canonical digit bound. -// Withheld, never read; real-width values unaffected. -func TestRowVisible_OverlongNumericOperand_FailsClosed(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"amount": intNumeric()} - perms := evalRowFilter(t, map[string]Filter{"amount": {Eq: new("100")}}, nil) - - long := "100." + strings.Repeat("0", 200_000) + "1" // far past the operand length gate - assert.False(t, perms.RowVisible(map[string]any{"amount": json.Number(long)}, num)) - assert.False(t, perms.RowVisible(map[string]any{"amount": long}, num), "string-encoded operand is bounded too") - assert.True(t, perms.RowVisible(map[string]any{"amount": json.Number("100.0")}, num), "real-width values still compare") -} - -// TestRowVisible_LossyFloatClaim_FailsClosed: a claim that arrives as a float64 -// at or past 2^53 has already collapsed onto its float neighbors — rendering -// digits for it could name another tenant (the fail-open caught in #381 review: -// the claim rendered as "1e+16" and matched the neighbor's rows). CanonicalScalar -// now refuses such a float64 outright, so the predicate matches NOTHING: not the -// neighbor the float equals, and not even the row whose exact ID the claim -// originally carried — availability, never confidentiality. -func TestRowVisible_LossyFloatClaim_FailsClosed(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"tenant_id": intNumeric()} - filter := map[string]Filter{"tenant_id": {Eq: new("{{ jwt.tenant }}")}} - - lossy := evalRowFilter(t, filter, map[string]any{"tenant": float64(10000000000000001)}) // already 1e16 - assert.False(t, lossy.RowVisible(map[string]any{"tenant_id": json.Number("10000000000000000")}, num), - "the float64-equal neighbor tenant's rows must not be delivered") - assert.False(t, lossy.RowVisible(map[string]any{"tenant_id": json.Number("10000000000000001")}, num), - "the original tenant's own rows are withheld too — the digits are unrecoverable") - - // The exact-digit path — a json.Number claim, which is what jwt.Parse yields - // since WithJSONNumber — scopes to precisely one tenant. - exact := evalRowFilter(t, filter, map[string]any{"tenant": json.Number("10000000000000001")}) - assert.True(t, exact.RowVisible(map[string]any{"tenant_id": json.Number("10000000000000001")}, num)) - assert.False(t, exact.RowVisible(map[string]any{"tenant_id": json.Number("10000000000000000")}, num)) -} - -// rfc3339Spec is a ColumnTime spec whose parser reads RFC 3339 strings — a -// hermetic stand-in for discovery's grammar, which is exercised in -// internal/discovery (Column.TimeParser) and end-to-end in the hub tests. -func rfc3339Spec() ColumnSpec { - return ColumnSpec{Kind: ColumnTime, ParseTime: func(v any) (time.Time, bool) { - s, ok := v.(string) - if !ok { - return time.Time{}, false - } - ts, err := time.Parse(time.RFC3339, s) - return ts, err == nil - }} -} - -// TestRowVisible_TimeColumn: DateTime/DateTime64 operands compare as instants, -// so equality holds across spellings of the same instant, ordering works (the -// time-window policy shape), and any operand the parser refuses — junk on either -// side, or a ColumnTime spec missing its parser — withholds the row. -func TestRowVisible_TimeColumn(t *testing.T) { - t.Parallel() - cols := map[string]ColumnSpec{"created_at": rfc3339Spec()} - - eq := evalRowFilter(t, map[string]Filter{"created_at": {Eq: new("2026-06-21T06:00:00+02:00")}}, nil) - assert.True(t, eq.RowVisible(map[string]any{"created_at": "2026-06-21T04:00:00Z"}, cols), - "different spelling, same instant ⇒ equal") - assert.False(t, eq.RowVisible(map[string]any{"created_at": "2026-06-21T04:00:01Z"}, cols)) - assert.False(t, eq.RowVisible(map[string]any{"created_at": "junk"}, cols), "unparseable payload withholds") - - gt := evalRowFilter(t, map[string]Filter{"created_at": {Gt: new("2026-06-21T00:00:00Z")}}, nil) - assert.True(t, gt.RowVisible(map[string]any{"created_at": "2026-06-21T04:00:00Z"}, cols)) - assert.False(t, gt.RowVisible(map[string]any{"created_at": "2026-06-20T04:00:00Z"}, cols)) - - bad := evalRowFilter(t, map[string]Filter{"created_at": {Eq: new("not-a-time")}}, nil) - assert.False(t, bad.RowVisible(map[string]any{"created_at": "2026-06-21T04:00:00Z"}, cols), - "unparseable constant withholds") - - noParser := map[string]ColumnSpec{"created_at": {Kind: ColumnTime}} - assert.False(t, eq.RowVisible(map[string]any{"created_at": "2026-06-21T04:00:00Z"}, noParser), - "ColumnTime without a parser refuses the comparison — never a text fallback") -} - -// TestRowVisible_TimeColumn_OverlongOperand_Gated: the O(1) length gate refuses -// an over-long timestamp operand — string OR json.Number (a Unix-epoch payload -// shape) — BEFORE the parser scans it, so a client-controlled megabyte "value" -// can't stall the per-subscriber fan-out. The parser here counts every call, so -// a gated operand must produce zero calls. -func TestRowVisible_TimeColumn_OverlongOperand_Gated(t *testing.T) { - t.Parallel() - var calls int - counting := ColumnSpec{Kind: ColumnTime, ParseTime: func(v any) (time.Time, bool) { - calls++ - return time.Time{}, false - }} - cols := map[string]ColumnSpec{"created_at": counting} - perms := evalRowFilter(t, map[string]Filter{"created_at": {Eq: new("2026-06-21T04:00:00Z")}}, nil) - - huge := strings.Repeat("9", maxTimeOperandChars+1) - assert.False(t, perms.RowVisible(map[string]any{"created_at": huge}, cols)) - assert.False(t, perms.RowVisible(map[string]any{"created_at": json.Number(huge)}, cols)) - assert.Zero(t, calls, "an over-long operand must be refused before the parser is called") -} - -func TestRowVisible_NilReceiver_AllVisible(t *testing.T) { - t.Parallel() - var perms *ResolvedPermissions - assert.True(t, perms.RowVisible(map[string]any{"x": "y"}, nil)) - assert.False(t, perms.HasRowFilter()) -} - -// TestRowVisible_NonScalar_FailsClosed: a nested/array/null event value can't be -// compared to a scalar filter value, so the predicate fails closed rather than guess. -func TestRowVisible_NonScalar_FailsClosed(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"tenant_id": {Eq: new("acme")}}, nil) - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": []any{"acme"}}, nil)) - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": nil}, nil)) -} - -// TestRowVisible_MultiplePredicates_AllMustPass: predicates are ANDed, matching the -// query path's "AND"-joined WHERE clause. -func TestRowVisible_MultiplePredicates_AllMustPass(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{ - "tenant_id": {Eq: new("{{ jwt.tenant }}")}, - "amount": {Gt: new("100")}, - }, map[string]any{"tenant": "acme"}) - num := map[string]ColumnSpec{"amount": intNumeric()} - assert.True(t, perms.RowVisible(map[string]any{"tenant_id": "acme", "amount": float64(250)}, num)) - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": "acme", "amount": float64(9)}, num), "amount fails") - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": "globex", "amount": float64(250)}, num), "tenant fails") -} - -// TestRowFilter_UnresolvableClaim_NoRowsOnBothPaths pins the #457 fail-closed -// rule on BOTH read surfaces at once: a filter template whose claim the token -// doesn't carry renders the constant-false predicate on the query path AND -// withholds every row in the stream's in-memory evaluation. One Evaluate -// resolution drives both, so a claim-less token can never see zero rows on -// /v1/query yet every row on /v1/stream. HasRowFilter must stay true for the -// failed predicate — dropping it would put the role back on the unfiltered -// once-per-role fast path, the exact fail-open this test exists to prevent. -func TestRowFilter_UnresolvableClaim_NoRowsOnBothPaths(t *testing.T) { - t.Parallel() - noTenant := map[string]any{"role": "user"} // validly signed token, no tenant claim - tests := []struct { - name string - filter map[string]Filter - claims map[string]any - }{ - {"_eq", map[string]Filter{"tenant_id": {Eq: new("{{ jwt.tenant }}")}}, noTenant}, - {"_neq, the leak direction", map[string]Filter{"tenant_id": {Neq: new("{{ jwt.tenant }}")}}, noTenant}, - {"_gt", map[string]Filter{"tenant_id": {Gt: new("{{ jwt.tenant }}")}}, noTenant}, - {"_in with surrounding text", map[string]Filter{"tenant_id": {In: new("t-{{ jwt.tenant }}")}}, noTenant}, - { - "object claim in a scalar slot", - map[string]Filter{"tenant_id": {Eq: new("{{ jwt.meta }}")}}, - map[string]any{"meta": map[string]any{"tenant": "acme"}}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, tt.filter, tt.claims) - assert.Equal(t, "1 = 0", perms.Select.WhereClause, "query path: constant-false predicate") - assert.Empty(t, perms.Select.WhereParams) - assert.True(t, perms.HasRowFilter(), "failed predicate must keep the stream on the per-subscriber path") - assert.False(t, perms.RowVisible(map[string]any{"tenant_id": "acme"}, nil), "stream path: every row withheld") - }) - } -} - -// TestRowVisible_FloatNarrowing: Float32/Float64 columns compare in the -// column's float domain — BOTH operands narrowed, exactly as ClickHouse -// narrows the stored value at insert and the bound constant at compare. The -// Float32 case is the #381 review repro: payload 16777217 stores as 16777216, -// so `_gt: "16777216"` must NOT admit the event (the SQL predicate over the -// stored row is false), while equality against the same spelling matches on -// both surfaces because the constant narrows too. The integer family, by -// contrast, keeps such neighbors distinct. -func TestRowVisible_FloatNarrowing(t *testing.T) { - t.Parallel() - f32 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericFloat, Bits: 32}}} - f64 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericFloat, Bits: 64}}} - intCol := map[string]ColumnSpec{"v": intNumeric()} - - gt := evalRowFilter(t, map[string]Filter{"v": {Gt: new("16777216")}}, nil) - assert.False(t, gt.RowVisible(map[string]any{"v": json.Number("16777217")}, f32), - "stored Float32(16777217) is 16777216, not > 16777216 — the ordering fail-open, closed") - assert.True(t, gt.RowVisible(map[string]any{"v": json.Number("16777218")}, f32), - "16777218 is Float32-representable and greater on both surfaces") - assert.True(t, gt.RowVisible(map[string]any{"v": json.Number("16777217")}, intCol), - "an integer column stores 16777217 exactly, so the same event IS greater there") - - eq := evalRowFilter(t, map[string]Filter{"v": {Eq: new("16777217")}}, nil) - assert.True(t, eq.RowVisible(map[string]any{"v": json.Number("16777217")}, f32), - "the constant narrows like the stored value — ClickHouse matches `= '16777217'` too") - assert.True(t, eq.RowVisible(map[string]any{"v": json.Number("16777216")}, f32), - "the Float32-equal neighbor matches on both surfaces — the column type gave that distinction away") - assert.False(t, eq.RowVisible(map[string]any{"v": json.Number("16777216")}, intCol), - "integer storage keeps the neighbors distinct") - - eq64 := evalRowFilter(t, map[string]Filter{"v": {Eq: new("9007199254740993")}}, nil) - assert.True(t, eq64.RowVisible(map[string]any{"v": json.Number("9007199254740992")}, f64), - "Float64 column: 2^53 neighbors collapse in the storage domain, matching SQL") - assert.False(t, eq64.RowVisible(map[string]any{"v": json.Number("9007199254740992")}, intCol)) - - // A magnitude the float domain can't hold (ClickHouse would store ±Inf) - // refuses the comparison — withheld, never a guessed verdict. - overflow := evalRowFilter(t, map[string]Filter{"v": {Gt: new("0")}}, nil) - assert.False(t, overflow.RowVisible(map[string]any{"v": json.Number("1e39")}, f32), - "beyond Float32 range ⇒ withhold") -} - -// TestRowVisible_DecimalScaleTruncation: Decimal columns compare after -// truncating BOTH operands to the column's scale — ClickHouse's cast semantics -// (1.005, 1.006 and 1.009 all store as 1.00 in a Decimal(10,2), and a bound -// constant '1.005' truncates the same way, so `= '1.005'` matches a stored -// 1.00 while `> '1.004'` matches nothing; verified on 25.5 and 26.6). -func TestRowVisible_DecimalScaleTruncation(t *testing.T) { - t.Parallel() - dec2 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericDecimal, Precision: 10, Scale: 2}}} - - gt := evalRowFilter(t, map[string]Filter{"v": {Gt: new("1.004")}}, nil) - assert.False(t, gt.RowVisible(map[string]any{"v": json.Number("1.005")}, dec2), - "stored 1.00 vs constant 1.00: not greater — the pre-narrowing payload must not leak through") - assert.True(t, gt.RowVisible(map[string]any{"v": json.Number("1.02")}, dec2)) - - eq := evalRowFilter(t, map[string]Filter{"v": {Eq: new("1.005")}}, nil) - assert.True(t, eq.RowVisible(map[string]any{"v": json.Number("1.006")}, dec2), - "both operands truncate to 1.00 — ClickHouse matches this pair too") - assert.False(t, eq.RowVisible(map[string]any{"v": json.Number("1.02")}, dec2)) - - lt := evalRowFilter(t, map[string]Filter{"v": {Lt: new("-1")}}, nil) - assert.False(t, lt.RowVisible(map[string]any{"v": json.Number("-1.005")}, dec2), - "truncation is toward zero: -1.005 stores as -1.00, which is not < -1") -} - -// TestRowVisible_IntegerFractionalOperand_FailsClosed: an integer column -// refuses fractional operands on either side — a fractional constant is a -// per-query type error on the SQL path (the role reads no rows there), and a -// fractional payload was never storable in the column — so the stream -// withholds rather than inventing a verdict SQL cannot produce. An integral -// value in fractional SPELLING is a different thing entirely: it -// canonicalizes to its integer and compares normally. -func TestRowVisible_IntegerFractionalOperand_FailsClosed(t *testing.T) { - t.Parallel() - num := map[string]ColumnSpec{"v": intNumeric()} - - frac := evalRowFilter(t, map[string]Filter{"v": {Neq: new("1.5")}}, nil) - assert.False(t, frac.RowVisible(map[string]any{"v": json.Number("2")}, num), - "fractional constant: SQL errors the query, the stream withholds — neither returns rows") - - pay := evalRowFilter(t, map[string]Filter{"v": {Gt: new("1")}}, nil) - assert.False(t, pay.RowVisible(map[string]any{"v": json.Number("2.5")}, num), - "fractional payload was never storable in an integer column") - assert.True(t, pay.RowVisible(map[string]any{"v": json.Number("2.0")}, num), - "integral value in fractional spelling canonicalizes to 2 and compares") -} - -// TestRowVisible_ConstantSpellingFidelity: on an integer column a literal -// constant that is not its own canonical form ("1e3", "1.5") refuses the -// comparison — the SQL path binds the spelling as written and ClickHouse's -// integer cast errors the query there, so the role reads no rows; admitting -// the canonical reading here would deliver rows SQL never returns. Float and -// Decimal casts accept every JSON-number spelling ('1e3' casts to -// Decimal(10,2) as 1000 — verified against ClickHouse), so those families -// compare such constants normally. -func TestRowVisible_ConstantSpellingFidelity(t *testing.T) { - t.Parallel() - intCol := map[string]ColumnSpec{"v": intNumeric()} - dec2 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericDecimal, Precision: 10, Scale: 2}}} - f32 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericFloat, Bits: 32}}} - - exp := evalRowFilter(t, map[string]Filter{"v": {Eq: new("1e3")}}, nil) - assert.False(t, exp.RowVisible(map[string]any{"v": json.Number("1000")}, intCol), - "integer column: SQL errors on '1e3', so the stream must not admit its canonical reading") - assert.True(t, exp.RowVisible(map[string]any{"v": json.Number("1000")}, f32), - "float column: ClickHouse casts '1e3' fine, so the stream compares it") - assert.True(t, exp.RowVisible(map[string]any{"v": json.Number("1000")}, dec2), - "Decimal column: ClickHouse casts '1e3' fine too, so the stream compares it") - - trailing := evalRowFilter(t, map[string]Filter{"v": {Gt: new("1.50")}}, nil) - assert.True(t, trailing.RowVisible(map[string]any{"v": json.Number("2")}, dec2), - "a trailing-zero Decimal spelling casts on both surfaces and compares by value") -} - -// TestRowVisible_OutOfRangeOperand_FailsClosed: an operand outside the -// column's numeric range refuses the comparison on either side. ClickHouse's -// reading of such a CONSTANT was measured to vary by pair on one release — -// error (negative vs unsigned: the role reads no rows), mathematical promotion -// ('256' vs UInt8), or a width-boundary wrap that compares against a DIFFERENT -// value than written (2^63 vs Int64 reads as −2^63, where exact-precision -// comparison would admit the −2^63 rows SQL hides under !=) — so no single -// model is safe to reproduce, and refusal is. A PAYLOAD out of range was never -// storable (the insert is rejected), so withholding matches the stored world. -func TestRowVisible_OutOfRangeOperand_FailsClosed(t *testing.T) { - t.Parallel() - u64 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericInteger, Bits: 64, Unsigned: true}}} - u8 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericInteger, Bits: 8, Unsigned: true}}} - i8 := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericInteger, Bits: 8}}} - dec := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericDecimal, Precision: 10, Scale: 2}}} - - neg := evalRowFilter(t, map[string]Filter{"v": {Gt: new("-5")}}, nil) - assert.False(t, neg.RowVisible(map[string]any{"v": json.Number("7")}, u64), - "negative constant on an unsigned column: SQL errors, the stream must not admit everything") - - wrap := evalRowFilter(t, map[string]Filter{"v": {Neq: new("18446744073709551616")}}, nil) - assert.False(t, wrap.RowVisible(map[string]any{"v": json.Number("7")}, u64), - "a width-boundary constant may wrap on the SQL side — comparing it as written risks admitting rows SQL hides") - - wide := evalRowFilter(t, map[string]Filter{"v": {Lt: new("99999999999999999999999")}}, nil) - assert.False(t, wide.RowVisible(map[string]any{"v": json.Number("7")}, u64), - "a constant past the width has no reliable SQL reading; the stream withholds") - - over8 := evalRowFilter(t, map[string]Filter{"v": {Neq: new("256")}}, nil) - assert.False(t, over8.RowVisible(map[string]any{"v": json.Number("7")}, u8)) - assert.False(t, over8.RowVisible(map[string]any{"v": json.Number("300")}, u8), - "an out-of-range payload was never storable — withheld") - ok8 := evalRowFilter(t, map[string]Filter{"v": {Neq: new("254")}}, nil) - assert.True(t, ok8.RowVisible(map[string]any{"v": json.Number("7")}, u8), "in-range operands still compare") - - i8lo := evalRowFilter(t, map[string]Filter{"v": {Gt: new("-129")}}, nil) - assert.False(t, i8lo.RowVisible(map[string]any{"v": json.Number("0")}, i8)) - i8ok := evalRowFilter(t, map[string]Filter{"v": {Gt: new("-128")}}, nil) - assert.True(t, i8ok.RowVisible(map[string]any{"v": json.Number("0")}, i8), "the signed minimum itself is in range") - - prec := evalRowFilter(t, map[string]Filter{"v": {Eq: new("999999999")}}, nil) - assert.False(t, prec.RowVisible(map[string]any{"v": json.Number("5")}, dec), - "9 integer digits exceed Decimal(10,2)'s 8-digit budget: unmodelable on the SQL side, withheld here") - precOK := evalRowFilter(t, map[string]Filter{"v": {Eq: new("99999999")}}, nil) - assert.True(t, precOK.RowVisible(map[string]any{"v": json.Number("99999999")}, dec), "the budget's edge is in range") - - // A hand-built spec with an incoherent Scale must degrade to refusal — - // never a panic on the fan-out goroutine (truncateScale slices by Scale). - badScale := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericDecimal, Precision: 10, Scale: -1}}} - assert.NotPanics(t, func() { - assert.False(t, precOK.RowVisible(map[string]any{"v": json.Number("1.5")}, badScale)) - }) -} - -// TestRowVisible_NumericWithoutStorageModel_FailsClosed pins the NumericSpec -// zero value: a ColumnNumeric spec carrying no storage model must refuse every -// comparison, so a numeric type the classifier doesn't recognize (or a caller -// that forgot to set the model) can never compare under guessed semantics. -func TestRowVisible_NumericWithoutStorageModel_FailsClosed(t *testing.T) { - t.Parallel() - perms := evalRowFilter(t, map[string]Filter{"v": {Eq: new("1")}}, nil) - - unmodeled := map[string]ColumnSpec{"v": {Kind: ColumnNumeric}} - assert.False(t, perms.RowVisible(map[string]any{"v": json.Number("1")}, unmodeled)) - - // A float family with no (or a bogus) bit width must refuse too: - // ParseFloat would silently treat any other bitSize as 64 and compare a - // Float32 column in the wider domain — the fail-open direction. - widthless := map[string]ColumnSpec{"v": {Kind: ColumnNumeric, Numeric: NumericSpec{Family: NumericFloat}}} - assert.False(t, perms.RowVisible(map[string]any{"v": json.Number("1")}, widthless)) -} diff --git a/internal/policy/scalars.go b/internal/policy/scalars.go index 14393bbd..d691986a 100644 --- a/internal/policy/scalars.go +++ b/internal/policy/scalars.go @@ -22,6 +22,18 @@ import ( // the humanization. The canonical units are milliseconds (time) and bytes // (memory); ClickHouse receives those numbers directly, so its own size/duration // syntax never leaks to policy authors. +// +// They stay WaveHouse's own vocabulary rather than becoming a passthrough of +// ClickHouse's setting syntax, because there is no such syntax to pass through. +// Measured on a live 26.3.28.5 server: `max_memory_usage='4GiB'` and +// `max_execution_time='10s'` are both refused with code 27 ("Cannot parse +// input: expected 'eof' before: 'B'" / "before: 's'"), as is `'1m30s'`; only +// `'4G'` and `'10'` parse. Forwarding the documented spellings would break every +// policy value already written AND turn a config typo into a per-query 500 +// instead of a load-time refusal. Millis is also not only a setting: it is read +// as a Go number (api/structured_query.go bounds the client context with +// Duration(), and ch_settings.go emits fractional seconds so a sub-second cap +// survives), which an opaque duration string could not supply. // Millis is a duration stored as whole milliseconds. Input accepts a Go duration // string ("10s", "500ms", "1m30s") or a bare integer count of milliseconds. diff --git a/internal/query/builder.go b/internal/query/builder.go index 9851e655..de8cb0bb 100644 --- a/internal/query/builder.go +++ b/internal/query/builder.go @@ -1,6 +1,7 @@ package query import ( + "encoding/json" "fmt" "regexp" "strconv" @@ -20,7 +21,10 @@ import ( // misconfigured to 0. const DefaultMaxRows = 10000 -// BuildResult holds the generated SQL and bound parameters. +// BuildResult holds the generated SQL and its bound values. The SQL carries +// positional `?` placeholders, one per entry in Params, in left-to-right +// order. NamedParams turns that pair into the named-parameter form +// ClickHouse's HTTP interface takes. type BuildResult struct { SQL string Params []any @@ -43,7 +47,7 @@ type BuildResult struct { // Every identifier that reaches the SQL — columns, the table, aggregation // aliases — is backtick-quoted via chsql.QuoteIdent, so the builder accepts any // name ClickHouse accepts while remaining injection-safe. Values stay positional -// `?` parameters bound by the driver. +// `?` parameters, which NamedParams turns into ClickHouse named parameters. // // Projection rules: SelectAll requests every readable column (expanded to the // role's allow/deny set); an explicit Columns list projects exactly those (where @@ -115,13 +119,15 @@ func Build(table string, q *StructuredQuery, schema *discovery.TableSchema, perm // - aggregations only → validateAndAuthorizeColumns → IsAggregationAllowed // // All three fail closed on a nil Select; all three are pinned by - // TestBuild_InsertResolvedGrantIsRejected. The bare read is the backstop if + // TestBuild_InsertResolvedGrantIsRejected. The bare call is the backstop if // that ordering changes — it would panic rather than skip the filter. Do NOT // add a `perms.Select != nil` guard: it would emit an unfiltered query, the // silent widening the pointer shape exists to prevent. - if perms != nil && perms.Select.WhereClause != "" { - whereParts = append([]string{"(" + perms.Select.WhereClause + ")"}, whereParts...) - params = append(params, perms.Select.WhereParams...) + if perms != nil { + if clause, rowParams := perms.Select.WhereSQL(columnType(schema)); clause != "" { + whereParts = append([]string{"(" + clause + ")"}, whereParts...) + params = append(params, rowParams...) + } } params = append(params, whereParams...) if len(whereParts) > 0 { @@ -234,7 +240,7 @@ func validateAndAuthorizeColumns(q *StructuredQuery, colSet map[string]bool, per } // The alias is backtick-quoted by aggregationExpr, so any legal ClickHouse // name is safe against injection. The lone refusal is a '?', which would - // break clickhouse-go's positional value binding. + // shift the positional-to-named parameter rewrite (see NamedParams). if chsql.BindUnsafe(a.Alias) { return fmt.Errorf("unsupported aggregation alias (contains '?'): %s", a.Alias) } @@ -255,7 +261,7 @@ func validateAndAuthorizeColumns(q *StructuredQuery, colSet map[string]bool, per // (ORDER BY an aggregation's AS name). Aliases carry no column policy; // the aggregation that defines them was authorized above. The name is // backtick-quoted when emitted, so any content is safe except a '?' - // (would break value binding, as for aliases). + // (would shift the parameter rewrite, as for aliases). if chsql.BindUnsafe(o.Column) { return fmt.Errorf("unsupported order column (contains '?'): %s", o.Column) } @@ -348,7 +354,14 @@ func buildWhere(filters []Filter, timeRange *TimeRange, bucketSeconds int) ([]st func filterToSQL(f Filter) (string, []any, error) { col := chsql.QuoteIdent(f.Column) - val := coerceFilterValue(f.Value) + // The value is bound, never rendered, so it reaches ClickHouse as the + // caller wrote it — including a timestamp. WaveHouse used to rewrite an + // RFC3339 value into ClickHouse's own spelling here; ClickHouse has + // accepted the RFC3339 spelling itself since 26.5 (cast_string_to_date_ + // time_mode defaults to best_effort), and every surface that hands a + // caller a timestamp to filter on — /v1/query, the SSE wire — already + // emits ClickHouse's spelling, which is what the rewrite existed for. + val := f.Value switch strings.ToLower(f.Op) { case "eq": return col + " = ?", []any{val}, nil @@ -365,10 +378,15 @@ func filterToSQL(f Filter) (string, []any, error) { case "like": return col + " LIKE ?", []any{val}, nil case "in": + // One placeholder for the whole list, bound as an Array(String) — not + // one placeholder per element. Both spellings answer identically, but + // ClickHouse's HTTP interface caps a request at 999 query-string + // fields (measured on 26.6.3.62: 999 ok, 1000 → "Too many form + // fields"), so a per-element binding would fail an `in` list that the + // 1 MiB request cap otherwise allows. One field per list raises the + // ceiling to the ~64 KiB per-field cap instead. if vals, ok := f.Value.([]any); ok && len(vals) > 0 { - placeholders := strings.Repeat("?,", len(vals)) - placeholders = placeholders[:len(placeholders)-1] - return fmt.Sprintf("%s IN (%s)", col, placeholders), vals, nil + return col + " IN ?", []any{vals}, nil } return "", nil, fmt.Errorf("invalid value for 'in' operator") default: @@ -376,27 +394,6 @@ func filterToSQL(f Filter) (string, []any, error) { } } -// coerceFilterValue converts string values that look like RFC3339 timestamps -// to ClickHouse-compatible DateTime strings preserving sub-second precision. -// The clickhouse-go driver's time.Time formatting uses toDateTime() (second -// precision), which loses milliseconds needed for DateTime64 cursor comparisons. -// Returning a formatted string lets ClickHouse parse it with full precision. -// -// A value that isn't a timestamp (a plain string, a number, etc.) is a valid -// non-temporal filter value, so the parse "failure" is just the expected -// non-timestamp case — pass it through unchanged rather than treat it as an error. -func coerceFilterValue(v any) any { - s, ok := v.(string) - if !ok { - return v - } - // RFC3339Nano parses both fractional and whole-second RFC3339 input. - if t, err := time.Parse(time.RFC3339Nano, s); err == nil { - return formatClickHouseTime(t) - } - return v -} - // clickHouseDateTimeLayout renders a time in ClickHouse's native DateTime text // format. The fractional ".999999999" preserves sub-second precision when // present and drops trailing zeros, so a whole-second time has no decimal point. @@ -442,8 +439,8 @@ func expandDayWeek(s string) string { // (which the builder surfaces as a 400) rather than returned unchanged: passing // a raw string to ClickHouse surfaces as an opaque DateTime parse error (#285). // -// The output deliberately matches coerceFilterValue's format rather than RFC3339: -// a bare "…T…Z" string is rejected by DateTime64 columns. +// The output is ClickHouse's own DateTime spelling rather than RFC3339 so the +// bound value is unambiguous on every server version. func resolveTimeValue(val string, bucketSeconds int) (string, error) { // Try a relative duration first (e.g., "1h", "30m", "7d", "2w"). Go's // time.ParseDuration only understands units up to hours, so day/week @@ -468,6 +465,15 @@ func bucketTime(t time.Time, bucketSeconds int) time.Time { return t.Truncate(d) } +// columnType looks a column's ClickHouse type up in the discovered schema, so +// the row filter can bind an integer column's claims through the strict cast. +func columnType(schema *discovery.TableSchema) func(string) string { + return func(col string) string { + c, _ := schema.Lookup(col) + return c.Type + } +} + func schemaColumnSet(schema *discovery.TableSchema) map[string]bool { m := make(map[string]bool, len(schema.Columns)) for _, c := range schema.Columns { @@ -514,3 +520,155 @@ func isValidAggFn(fn string) bool { } return false } + +// ─── ClickHouse named-parameter binding ───────────────────────────────────── + +// NamedParams rewrites the positional `?` placeholders in the built SQL into +// ClickHouse named parameters and renders each bound value as the text +// ClickHouse will read it back from. It returns the rewritten SQL and the +// values for `param_p0` … `param_pN-1`, positionally. +// +// Every scalar binds as `{pN:String}` and every list as `{pN:Array(String)}` +// — one rule for the whole product, shared with the row filters the type +// layer compiles. String is not a weaker binding than the column's own type: +// ClickHouse parses the parameter against the column on both sides of the +// comparison, so `UInt8 = {p:String}` with "256" is false and with "1.5" is a +// type error, matching what the server answers for the same literal (AUDIT +// §C.1, measured against 26.3 and 26.6). The exception is a policy claim on +// an integer column (a chsql.IntParam): its placeholder expands to +// chsql.StrictInt over the one `{pN:String}` parameter, because the plain form +// wraps a value at or past 2^64. +// +// The rewrite is a left-to-right scan for `?`, which is exact for this SQL and +// only for this SQL: Build never renders a value or a string literal, and +// chsql.BindUnsafe rejects a `?` in any identifier it quotes — the same +// invariant positional binding relied on. +func (r *BuildResult) NamedParams() (string, []string, error) { + if len(r.Params) == 0 { + if strings.Contains(r.SQL, "?") { + return "", nil, fmt.Errorf("query has placeholders but no bound values") + } + return r.SQL, nil, nil + } + + params := make([]string, 0, len(r.Params)) + var b strings.Builder + b.Grow(len(r.SQL) + len(r.Params)*12) + + rest := r.SQL + for i, v := range r.Params { + q := strings.IndexByte(rest, '?') + if q < 0 { + return "", nil, fmt.Errorf("query has %d bound values but only %d placeholders", len(r.Params), i) + } + name := "p" + strconv.Itoa(i) + text, placeholder, err := chParamValue(name, v) + if err != nil { + return "", nil, err + } + b.WriteString(rest[:q]) + b.WriteString(placeholder) + params = append(params, text) + rest = rest[q+1:] + } + if strings.Contains(rest, "?") { + return "", nil, fmt.Errorf("query has more placeholders than the %d bound values", len(r.Params)) + } + b.WriteString(rest) + return b.String(), params, nil +} + +// chParamValue renders one bound value as its ClickHouse query-parameter text +// and the SQL its placeholder becomes, for the parameter called name. +func chParamValue(name string, v any) (text, placeholder string, err error) { + switch val := v.(type) { + case []any: + lit, err := chArrayLiteral(val) + if err != nil { + return "", "", err + } + return lit, "{" + name + ":Array(String)}", nil + case chsql.IntParam: + return chsql.EscapeStringParam(val.Value), chsql.StrictInt(name, val.Type), nil + } + raw, err := chScalarText(v) + if err != nil { + return "", "", err + } + return chsql.EscapeStringParam(raw), "{" + name + ":String}", nil +} + +// chScalarText is one scalar's value as plain text, before any encoding — +// what the caller means, not what the wire needs. +func chScalarText(v any) (string, error) { + switch val := v.(type) { + case string: + return val, nil + case json.Number: + // The caller's own digits, not a float64 round-trip: 12.50 stays + // "12.50" and an integer past 2^53 keeps every digit. + return val.String(), nil + case bool: + return strconv.FormatBool(val), nil + case float64: + return strconv.FormatFloat(val, 'f', -1, 64), nil + case int: + return strconv.Itoa(val), nil + case int64: + return strconv.FormatInt(val, 10), nil + case uint64: + return strconv.FormatUint(val, 10), nil + case nil: + // `col = NULL` is never true in SQL, so the driver quietly answered + // "no rows"; an empty String parameter would instead compare against + // the empty string, which is a different question. Refuse it (→ 400) + // rather than answer a question the caller did not ask. + return "", fmt.Errorf("filter value must not be null") + default: + return "", fmt.Errorf("unsupported filter value type %T", v) + } +} + +// chArrayLiteral renders a list as the `['a','b']` text an Array(String) +// query parameter is parsed from. A nested list has no place inside an `in` +// list, and the elements take quoteCHElement's encoding INSTEAD of +// chsql.EscapeStringParam's, not on top of it. +func chArrayLiteral(vals []any) (string, error) { + var b strings.Builder + b.WriteByte('[') + for i, v := range vals { + if _, isList := v.([]any); isList { + return "", fmt.Errorf("nested list in an 'in' value") + } + text, err := chScalarText(v) + if err != nil { + return "", err + } + if i > 0 { + b.WriteByte(',') + } + b.WriteString(quoteCHElement(text)) + } + b.WriteByte(']') + return b.String(), nil +} + +// quoteCHElement wraps one already-rendered value as a single-quoted element +// of an Array(String) parameter literal. That literal is read as a quoted +// value rather than an escaped field — measured on 26.6.3.62, a raw tab or +// newline inside the quotes round-trips untouched — so only the quote and the +// backslash need encoding, and chsql.EscapeStringParam's encoding must NOT be +// applied on top of it. +func quoteCHElement(s string) string { + var b strings.Builder + b.Grow(len(s) + 2) + b.WriteByte('\'') + for i := 0; i < len(s); i++ { + if s[i] == '\\' || s[i] == '\'' { + b.WriteByte('\\') + } + b.WriteByte(s[i]) + } + b.WriteByte('\'') + return b.String() +} diff --git a/internal/query/builder_test.go b/internal/query/builder_test.go index 5c31945d..2fca66f8 100644 --- a/internal/query/builder_test.go +++ b/internal/query/builder_test.go @@ -1,6 +1,7 @@ package query import ( + "encoding/json" "fmt" "strings" "testing" @@ -136,8 +137,16 @@ func TestBuild_InFilter(t *testing.T) { } result, err := Build("clicks", sq, testSchema(), nil, 0, DefaultMaxRows) require.NoError(t, err) - assert.Contains(t, result.SQL, "`page` IN (?,?)") - assert.Len(t, result.Params, 2) + // One placeholder for the whole list — it binds as an Array(String), which + // is what keeps a long list under ClickHouse's 999-query-string-field cap. + assert.Contains(t, result.SQL, "`page` IN ?") + require.Len(t, result.Params, 1) + assert.Equal(t, []any{"/home", "/about"}, result.Params[0]) + + sql, params, err := result.NamedParams() + require.NoError(t, err) + assert.Contains(t, sql, "`page` IN {p0:Array(String)}") + assert.Equal(t, []string{`['/home','/about']`}, params) } func TestBuild_OrderBy(t *testing.T) { @@ -178,10 +187,18 @@ func TestBuild_TimeRange(t *testing.T) { assert.Len(t, result.Params, 1) } -// permsWithFilter returns resolved permissions carrying a row-filter predicate, -// shaped exactly as policy.Evaluate emits one (quoted column, positional '?'). +// permsWithFilter returns resolved permissions carrying a row-filter predicate +// on org_id (a String column), resolved by policy.Evaluate itself. func permsWithFilter() *policy.ResolvedPermissions { - return &policy.ResolvedPermissions{Allowed: true, Select: &policy.ResolvedSelect{WhereClause: "`org_id` = ?", WhereParams: []any{"org-1"}}} + return permsFiltering(map[string]policy.Filter{"org_id": {Eq: new("org-1")}}, nil) +} + +// permsFiltering resolves a read grant carrying filter for role "r" on clicks. +func permsFiltering(filter map[string]policy.Filter, claims map[string]any) *policy.ResolvedPermissions { + p := &policy.Policy{Tables: map[string]policy.TablePolicy{ + "clicks": {"r": {Select: &policy.SelectPermissions{Filter: filter}}}, + }} + return policy.Evaluate(p, "r", "clicks", "select", claims) } // TestBuild_PolicyPredicate pins the structural emission of the row-level- @@ -259,6 +276,147 @@ func TestBuild_PolicyPredicate_SurvivesCraftedIdentifiers(t *testing.T) { } } +// TestBuild_PolicyPredicate_IntegerColumnsBindThroughTheStrictCast pins the +// query path's half of the integer-claim rule: a policy claim compared against +// an integer column (any width, Nullable or LowCardinality) renders as +// chsql.StrictInt over ONE {pN:String} parameter, on every operator and on +// each element of an _in list, while every other column keeps the plain +// {pN:String} form. The typelayer renders the same expression for the stream +// and the insert check. +func TestBuild_PolicyPredicate_IntegerColumnsBindThroughTheStrictCast(t *testing.T) { + t.Parallel() + schema := &discovery.TableSchema{Name: "clicks", Columns: []discovery.Column{ + {Name: "page", Type: "String"}, + {Name: "u64", Type: "UInt64"}, + {Name: "i8", Type: "Int8"}, + {Name: "nu256", Type: "Nullable(UInt256)"}, + {Name: "lci32", Type: "LowCardinality(Nullable(Int32))"}, + {Name: "org_id", Type: "String"}, + {Name: "amount", Type: "Decimal(18, 4)"}, + {Name: "flag", Type: "Bool"}, + }} + e := func(p, typ string) string { + c := "accurateCastOrNull({" + p + ":String}, '" + typ + "')" + return "if(toString(" + c + ") = {" + p + ":String}, " + c + ", NULL)" + } + tests := []struct { + name string + column string + filter policy.Filter + claims map[string]any + wantWhere string + wantParams []string + }{ + { + "eq on UInt64", "u64", + policy.Filter{Eq: new("{{ jwt.t }}")}, + map[string]any{"t": "5"}, + "`u64` = " + e("p0", "UInt64"), + []string{"5"}, + }, + { + "neq on Int8", "i8", + policy.Filter{Neq: new("-3")}, + nil, + "`i8` != " + e("p0", "Int8"), + []string{"-3"}, + }, + { + "gt on Nullable(UInt256) casts to the bare type", "nu256", + policy.Filter{Gt: new("7")}, + nil, + "`nu256` > " + e("p0", "UInt256"), + []string{"7"}, + }, + { + "lt on LowCardinality(Nullable(Int32)) casts to the bare type", "lci32", + policy.Filter{Lt: new("9")}, + nil, + "`lci32` < " + e("p0", "Int32"), + []string{"9"}, + }, + { + "in on UInt64 casts each element", "u64", + policy.Filter{In: new("{{ jwt.ts }}")}, + map[string]any{"ts": []any{"5", "18446744073709551621", "007"}}, + "`u64` IN (" + e("p0", "UInt64") + "," + e("p1", "UInt64") + "," + e("p2", "UInt64") + ")", + []string{"5", "18446744073709551621", "007"}, + }, + { + "a claim needing escape is encoded once", "u64", + policy.Filter{Eq: new("{{ jwt.t }}")}, + map[string]any{"t": `a\b`}, + "`u64` = " + e("p0", "UInt64"), + []string{`a\\b`}, + }, + { + "eq on String keeps the plain form", "org_id", + policy.Filter{Eq: new("acme")}, + nil, + "`org_id` = {p0:String}", + []string{"acme"}, + }, + { + "in on String keeps the plain form", "org_id", + policy.Filter{In: new("{{ jwt.ts }}")}, + map[string]any{"ts": []any{"a", "b"}}, + "`org_id` IN ({p0:String},{p1:String})", + []string{"a", "b"}, + }, + { + "Decimal keeps the plain form", "amount", + policy.Filter{Gt: new("1.50")}, + nil, + "`amount` > {p0:String}", + []string{"1.50"}, + }, + { + "Bool keeps the plain form", "flag", + policy.Filter{Eq: new("true")}, + nil, + "`flag` = {p0:String}", + []string{"true"}, + }, + { + "an unresolvable claim still fails closed", "u64", + policy.Filter{Eq: new("{{ jwt.absent }}")}, + nil, + "1 = 0", nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + perms := permsFiltering(map[string]policy.Filter{tt.column: tt.filter}, tt.claims) + res, err := Build("clicks", &StructuredQuery{Columns: []string{"page"}}, schema, perms, 0, DefaultMaxRows) + require.NoError(t, err) + sql, params, err := res.NamedParams() + require.NoError(t, err) + assert.Equal(t, "SELECT `page` FROM `clicks` WHERE ("+tt.wantWhere+") LIMIT 10000", sql) + assert.Equal(t, tt.wantParams, params) + }) + } +} + +// TestBuild_PolicyPredicate_CallerFiltersKeepThePlainForm: the strict cast is +// for policy claims only. A caller's own filter on an integer column binds as +// before — it can only narrow what the policy already admits. +func TestBuild_PolicyPredicate_CallerFiltersKeepThePlainForm(t *testing.T) { + t.Parallel() + perms := permsFiltering(map[string]policy.Filter{"count": {Eq: new("5")}}, nil) + sq := &StructuredQuery{Columns: []string{"page"}, Filters: []Filter{ + {Column: "count", Op: "gt", Value: json.Number("1")}, + {Column: "count", Op: "in", Value: []any{json.Number("1"), json.Number("2")}}, + }} + res, err := Build("clicks", sq, testSchema(), perms, 0, DefaultMaxRows) + require.NoError(t, err) + sql, params, err := res.NamedParams() + require.NoError(t, err) + assert.Equal(t, "SELECT `page` FROM `clicks` WHERE (`count` = "+chsql.StrictInt("p0", "UInt64")+ + ") AND `count` > {p1:String} AND `count` IN {p2:Array(String)} LIMIT 10000", sql) + assert.Equal(t, []string{"5", "1", "['1','2']"}, params) +} + // TestBuild_PolicyMaxRows pins the role's max_rows cap folded into Build's LIMIT // computation (#322): the emitted LIMIT is min(caller limit, default cap, policy // cap), with non-positive values meaning "no cap from that source". Includes a @@ -490,34 +648,6 @@ func TestBucketTime_ZeroBucket(t *testing.T) { assert.Equal(t, ts, got, "zero bucket should not truncate") } -func TestCoerceFilterValue(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - input any - wantTyp string - wantVal any - }{ - {"RFC3339", "2026-04-02T16:02:07Z", "string", "2026-04-02 16:02:07"}, - {"RFC3339Nano", "2026-04-02T16:02:07.666Z", "string", "2026-04-02 16:02:07.666"}, - {"RFC3339Nano_short", "2026-04-02T16:02:07.15Z", "string", "2026-04-02 16:02:07.15"}, - {"plain_string", "hello", "string", "hello"}, - {"number", 42, "int", 42}, - {"nil", nil, "", nil}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - got := coerceFilterValue(tt.input) - assert.Equal(t, tt.wantTyp, fmt.Sprintf("%T", got)) - if tt.wantVal != nil { - assert.Equal(t, tt.wantVal, got) - } - }) - } -} - func TestBuild_FilterWithTimestampValue(t *testing.T) { t.Parallel() schema := &discovery.TableSchema{ @@ -539,9 +669,17 @@ func TestBuild_FilterWithTimestampValue(t *testing.T) { require.NoError(t, err) assert.Contains(t, result.SQL, "`received_timestamp` < ?") require.Len(t, result.Params, 1) - strVal, isString := result.Params[0].(string) - assert.True(t, isString, "timestamp filter value should be coerced to formatted string, got %T", result.Params[0]) - assert.Equal(t, "2026-04-02 16:02:07.666", strVal) + // The value is bound, not rendered, so it reaches ClickHouse exactly as + // the caller wrote it. WaveHouse used to rewrite an RFC3339 value into + // ClickHouse's own spelling; ClickHouse has parsed the RFC3339 spelling + // itself since 26.5, and the surfaces that hand a caller a timestamp to + // filter on already emit ClickHouse's spelling. + assert.Equal(t, "2026-04-02T16:02:07.666Z", result.Params[0]) + + sql, params, err := result.NamedParams() + require.NoError(t, err) + assert.Contains(t, sql, "`received_timestamp` < {p0:String}") + assert.Equal(t, []string{"2026-04-02T16:02:07.666Z"}, params) } func TestBuild_TableNameWithBacktick(t *testing.T) { @@ -573,8 +711,8 @@ func TestBuild_InvalidColumns(t *testing.T) { sq: &StructuredQuery{ Columns: []string{"page"}, // A non-schema order column is allowed as an alias reference and - // backtick-quoted; only a '?' (which clickhouse-go's binder would - // miscount) is rejected. + // backtick-quoted; only a '?' (which the positional-to-named rewrite + // would miscount) is rejected. OrderBy: []OrderClause{{Column: "we?ird", Dir: "asc"}}, }, wantErr: "unsupported order column", @@ -855,7 +993,7 @@ func TestBuild_AggregationAliasQuotedAndContained(t *testing.T) { } // TestBuild_RejectsBindUnsafeAlias keeps the one alias rejection that remains: a -// '?' would be miscounted by clickhouse-go's positional value binder. +// '?' would be miscounted by the positional-to-named parameter rewrite. func TestBuild_RejectsBindUnsafeAlias(t *testing.T) { t.Parallel() sq := &StructuredQuery{Aggregations: []Aggregation{{Fn: "count", Column: "*", Alias: "we?ird"}}} @@ -1002,3 +1140,128 @@ func TestBuild_InsertResolvedGrantIsRejected(t *testing.T) { require.NoError(t, err) assert.NotNil(t, res) } + +// TestNamedParams pins the ClickHouse binding rule: every positional `?` +// becomes a named parameter, every scalar binds as String and every list as +// Array(String), in the order the WHERE assembly emitted them. +func TestNamedParams(t *testing.T) { + t.Parallel() + tests := []struct { + name string + sql string + params []any + wantSQL string + wantParams []string + }{ + { + name: "no parameters", + sql: "SELECT `page` FROM `clicks` LIMIT 10", + wantSQL: "SELECT `page` FROM `clicks` LIMIT 10", + wantParams: nil, + }, + { + name: "policy predicate keeps its leading position", + sql: "SELECT `page` FROM `clicks` WHERE (`org_id` = ?) AND `page` = ? LIMIT 100", + params: []any{"org-1", "/home"}, + wantSQL: "SELECT `page` FROM `clicks` WHERE (`org_id` = {p0:String}) AND `page` = {p1:String} LIMIT 100", + wantParams: []string{"org-1", "/home"}, + }, + { + name: "list binds as one Array(String)", + sql: "SELECT * FROM `t` WHERE `page` IN ? LIMIT 10", + params: []any{[]any{"/a", "/b"}}, + wantSQL: "SELECT * FROM `t` WHERE `page` IN {p0:Array(String)} LIMIT 10", + wantParams: []string{`['/a','/b']`}, + }, + { + name: "numbers keep the caller's own digits", + sql: "SELECT * FROM `t` WHERE `a` = ? AND `b` = ? AND `c` = ? LIMIT 10", + params: []any{json.Number("12.50"), json.Number("9007199254740993"), true}, + wantSQL: "SELECT * FROM `t` WHERE `a` = {p0:String} AND `b` = {p1:String} " + + "AND `c` = {p2:String} LIMIT 10", + wantParams: []string{"12.50", "9007199254740993", "true"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + sql, params, err := (&BuildResult{SQL: tt.sql, Params: tt.params}).NamedParams() + require.NoError(t, err) + assert.Equal(t, tt.wantSQL, sql) + assert.Equal(t, tt.wantParams, params) + }) + } +} + +// TestNamedParams_Encoding pins the two escapings a ClickHouse query parameter +// needs, which are NOT the same and must not be applied to each other's +// values. Measured on 26.6.3.62 (the version the integration suite pins): +// +// - a scalar `{p:String}` is read by the escaped-text reader, so a raw +// backslash is taken as the start of an escape sequence ("a\b" came back +// holding a backspace) and a raw tab or newline ends the field outright +// (code 457, a 500 for the caller); +// - an `Array(String)` value is an array literal whose elements are quoted, +// so a raw tab or newline rides through untouched and only the quote and +// the backslash need encoding. +// +// Both encodings round-trip every case below byte for byte against a live +// server; getting either wrong is silent data loss, not an error. +func TestNamedParams_Encoding(t *testing.T) { + t.Parallel() + tests := []struct { + name string + value any + wantParam string + }{ + {"plain", "hello", "hello"}, + {"single quote needs nothing", "it's", "it's"}, + {"backslash", `a\b`, `a\\b`}, + {"windows path", `C:\Users\x`, `C:\\Users\\x`}, + {"tab", "a\tb", `a\tb`}, + {"newline", "a\nb", `a\nb`}, + {"carriage return", "a\rb", `a\rb`}, + {"a literal backslash-n", `a\nb`, `a\\nb`}, + {"like pattern", "%foo%", "%foo%"}, + {"list quotes and backslashes", []any{`it's`, `a\b`, "a\tb"}, "['it\\'s','a\\\\b','a\tb']"}, + {"list containment attempt", []any{`']) OR 1=1 --`}, `['\']) OR 1=1 --']`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, params, err := (&BuildResult{SQL: "SELECT ?", Params: []any{tt.value}}).NamedParams() + require.NoError(t, err) + require.Len(t, params, 1) + assert.Equal(t, tt.wantParam, params[0]) + }) + } +} + +// TestNamedParams_Rejects covers the values and shapes that have no honest +// binding. A JSON null is the notable one: the driver quietly turned it into +// `col = NULL` (never true), where an empty String parameter would compare +// against the empty string — a different question, so it is refused (→ 400). +func TestNamedParams_Rejects(t *testing.T) { + t.Parallel() + tests := []struct { + name string + sql string + params []any + wantErr string + }{ + {"null value", "SELECT ?", []any{nil}, "must not be null"}, + {"object value", "SELECT ?", []any{map[string]any{"k": "v"}}, "unsupported filter value type"}, + {"nested list", "SELECT ?", []any{[]any{[]any{"a"}}}, "nested list"}, + {"more values than placeholders", "SELECT 1", []any{"a"}, "only 0 placeholders"}, + {"more placeholders than values", "SELECT ?, ?", []any{"a"}, "more placeholders"}, + {"placeholder with no values", "SELECT ?", nil, "no bound values"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, _, err := (&BuildResult{SQL: tt.sql, Params: tt.params}).NamedParams() + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + }) + } +} diff --git a/internal/stream/hub.go b/internal/stream/hub.go index f9527660..3e99a7c1 100644 --- a/internal/stream/hub.go +++ b/internal/stream/hub.go @@ -21,52 +21,43 @@ import ( // resolved against each subscriber's claims, so two subscribers of the same role can // be entitled to different rows. For a role that carries a row-filter, Broadcast // therefore keeps the shared column projection but evaluates row visibility PER -// subscriber (ResolvedPermissions.RowVisible) before delivering — closing the -// query/stream RLS drift in #319. Roles without a row-filter keep the pure -// once-per-role fast path unchanged. See projectColumns. +// subscriber (RowView.Visible over the event parsed once) before delivering — +// closing the query/stream RLS drift in #319. Roles without a row-filter keep the +// pure once-per-role fast path unchanged, and never reach the type layer at all. +// See projectColumns. type Hub struct { mu sync.RWMutex topics map[string]*topicRoutes policy policy.Source // nil ⇒ policy filtering not configured (legacy passthrough) - registry *discovery.SchemaRegistry // nil ⇒ no column types; row-filter comparison degrades fail-closed (see columnSpecs) + registry *discovery.SchemaRegistry // nil ⇒ no schema frame on subscribe; row filtering is unaffected (the type layer owns it) metric *Metrics // nil-safe - // RowEvaluator is the seam a native type layer will take over: the one - // place a row's visibility under a role's row-filter is decided. nil means - // the default implementation, which delegates to ResolvedPermissions.RowVisible - // — today's behavior unchanged. Wired once before the Hub serves traffic and - // not safe to mutate afterwards: rowAdmitted reads it from the consumer - // goroutine and from SSE handler goroutines without holding h.mu. Every - // delivery path reaches it through rowAdmitted, never directly. + // RowEvaluator decides a row's visibility under a role's row-filter — the + // one place that decision is taken, for live fan-out and replay alike. + // Production wires NewRowEvaluator (the type layer); nil means the + // fail-closed default below, which withholds every row of a row-filtered + // role, because an unwired evaluator must never read as "everything + // visible". Wired once before the Hub serves traffic and not safe to mutate + // afterwards: rowAdmitted reads it from the consumer goroutine and from SSE + // handler goroutines without holding h.mu. Every delivery path reaches it + // through Prepare + rowAdmitted, never directly. RowEvaluator RowEvaluator -} - -// RowEvaluator decides whether one decoded event row is visible to a subscriber -// under their resolved permissions. specs classifies each column for the -// comparison (see Hub.columnSpecs); a nil map means no type knowledge, which the -// default implementation treats as the fail-closed floor. -type RowEvaluator interface { - Visible(perms *policy.ResolvedPermissions, row map[string]any, specs map[string]policy.ColumnSpec) bool -} - -// policyRowEvaluator is the default RowEvaluator, delegating to the policy -// package's in-memory row-filter evaluation. -type policyRowEvaluator struct{} -// Visible delegates to the policy package's in-memory row-filter evaluation, -// the same resolution the query path renders to SQL (#319). -func (policyRowEvaluator) Visible(perms *policy.ResolvedPermissions, row map[string]any, specs map[string]policy.ColumnSpec) bool { - return perms.RowVisible(row, specs) + // unwired is that fail-closed default: an engineEvaluator with no Engine, + // which withholds and logs once. It is a field rather than a fresh value per + // call so the "no engine" report really is once per Hub. + unwired engineEvaluator } -// rowEvaluator returns the Hub's RowEvaluator, or the default when none is -// wired. Nil-safe rather than constructor-enforced, so a zero Hub still -// evaluates row-level security instead of panicking past it. +// rowEvaluator returns the Hub's RowEvaluator, or the fail-closed default when +// none is wired. Nil-safe rather than constructor-enforced, so a zero Hub still +// enforces row-level security instead of panicking past it — or, worse, reading +// an absent evaluator as an absent restriction. func (h *Hub) rowEvaluator() RowEvaluator { if h.RowEvaluator != nil { return h.RowEvaluator } - return policyRowEvaluator{} + return &h.unwired } // topicRoutes holds the per-role buckets subscribed to one topic. @@ -76,10 +67,10 @@ type topicRoutes struct { // NewHub builds an event hub. A nil policy store passes every event through // unfiltered (the unwired-tests case); a non-nil store whose Get returns nil is a -// total lockout (a deleted/absent policy denies everyone). A nil registry leaves -// every column's type unknown, so row-filter comparison degrades FAIL-CLOSED: -// equality/set predicates admit only a byte-identical value and ordering/!= admit -// nothing (see policy.ColumnKind); metric may be nil. +// total lockout (a deleted/absent policy denies everyone). A nil registry only +// costs the on-subscribe schema frame — row filtering reads its types from the +// type layer behind RowEvaluator, which the caller wires separately; metric may +// be nil. func NewHub(policyStore policy.Source, registry *discovery.SchemaRegistry, metric *Metrics) *Hub { return &Hub{topics: make(map[string]*topicRoutes), policy: policyStore, registry: registry, metric: metric} } @@ -172,11 +163,18 @@ func (h *Hub) Broadcast(topic string, raw []byte) { ev := newEventView(raw) p, filter := h.snapshotPolicy() - // Column specs for type-aware row-filter comparison — resolved lazily at most - // once per event, only when some role actually carries a row-filter, and reused - // across every filtered role and subscriber. - var colSpecs map[string]policy.ColumnSpec - specsResolved := false + // The row is parsed at most ONCE per event — only when some role actually + // carries a row-filter — and the parsed view is reused by every filtered role + // and subscriber. Parsing is the expensive half; evaluating a compiled + // predicate over an already-parsed row is not. + var view RowView + var prepErr error + prepared := false + defer func() { + if view != nil { + view.Close() + } + }() for _, rb := range roleBuckets { plan, ok := planForRole(p, filter, rb.role, ev, KindEvent) @@ -201,12 +199,12 @@ func (h *Hub) Broadcast(topic string, raw []byte) { // shared, but whether each subscriber may see THIS row depends on its claims, so // evaluate visibility per subscriber. Predicates read the full event row (a // filter may key on a column the role can't SELECT), not the projected columns. - if !specsResolved { - colSpecs = h.columnSpecs(ev.evt.TableName) - specsResolved = true + if !prepared { + view, prepErr = h.rowEvaluator().Prepare(ev.evt.TableName, ev.evt.Columns, ev.evt.Row) + prepared = true } for _, sub := range rb.bucket.Snapshot() { - if h.rowAdmitted(p, rb.role, ev, sub.claims, colSpecs) { + if h.rowAdmitted(p, rb.role, ev, sub.claims, view, prepErr) { deliver(sub, plan) } } @@ -239,81 +237,36 @@ func deliver(sub *Subscriber, plan rolePlan) { } // rowAdmitted reports whether claims admit this event's row under the role's -// row-filter, counting a withheld row when they don't. It is the one admission -// step shared by the live fan-out (per subscriber) and replay (per connection), -// so the two delivery paths can't drift on how row-level security is evaluated. -func (h *Hub) rowAdmitted(p *policy.Policy, role string, ev *eventView, claims map[string]any, colSpecs map[string]policy.ColumnSpec) bool { +// row-filter, counting a withheld row (with the reason) when they don't. It is +// the one admission step shared by the live fan-out (per subscriber) and replay +// (per connection), so the two delivery paths can't drift on how row-level +// security is evaluated. +// +// view is the event parsed once for the whole fan-out; prepErr is why there is +// none. An event that could not be prepared — no compiled schema for the table, +// a column list that is not the compiled generation's, an unparseable row — +// withholds from every row-filtered subscriber, never from the unfiltered roles +// that never asked the type layer anything. +func (h *Hub) rowAdmitted(p *policy.Policy, role string, ev *eventView, claims map[string]any, view RowView, prepErr error) bool { + if prepErr != nil || view == nil { + // view == nil with no error is a broken evaluator, not a verdict: withhold. + h.metric.RowWithheld(ev.evt.TableName, role, WithheldReason(prepErr)) + return false + } perms := policy.Evaluate(p, role, ev.evt.TableName, "select", claims) - if !h.rowEvaluator().Visible(perms, ev.row, colSpecs) { - h.metric.RowWithheld(ev.evt.TableName, role) + visible, reason := view.Visible(perms) + if !visible { + h.metric.RowWithheld(ev.evt.TableName, role, reason) return false } return true } -// columnSpecs classifies each of the table's columns for the row-filter evaluator: -// DateTime/DateTime64 columns compare as instants (through discovery's -// Column.TimeParser — the same grammar ingest canonicalization applies, so a -// zone-less filter constant matches the canonical RFC 3339 payload), numeric types -// compare numerically (9 < 100, matching ClickHouse), String compares bytewise -// (exactly ClickHouse's String collation), and any other type is omitted — -// policy.ColumnOpaque, the map's zero value — admitting byte-equality only. nil when -// no schema is available (unknown table, or a Hub built without a registry), which -// reads as every column Opaque: the fail-closed floor, never a lexicographic -// fallback that could admit rows the query path excludes ("9" > "100" as text). -func (h *Hub) columnSpecs(table string) map[string]policy.ColumnSpec { - if h.registry == nil { - return nil - } - schema := h.registry.Get(table) - if schema == nil { - return nil - } - m := make(map[string]policy.ColumnSpec, len(schema.Columns)) - for _, c := range schema.Columns { - if pt := c.TimeParser(); pt != nil { - m[c.Name] = policy.ColumnSpec{Kind: policy.ColumnTime, ParseTime: pt} - continue - } - switch { - case discovery.IsNumericType(c.Type): - // The storage model narrows both comparison operands the way - // ClickHouse narrows the stored value and the bound constant. A - // numeric type whose model can't be classified keeps the zero - // NumericSpec, which refuses every comparison — fail closed, - // never a comparison under guessed semantics. - spec := policy.ColumnSpec{Kind: policy.ColumnNumeric} - if st, ok := discovery.NumericStorageOf(c.Type); ok { - spec.Numeric = NumericSpecOf(st) - } - m[c.Name] = spec - case discovery.IsStringType(c.Type): - m[c.Name] = policy.ColumnSpec{Kind: policy.ColumnText} - } - } - return m -} - -// NumericSpecOf renders discovery's storage classification as the policy -// evaluator's storage model. Exported so the tests/integration differential -// oracle builds specs through the very mapping production uses — one source, -// so the oracle can't keep validating a mapping the Hub no longer applies. -func NumericSpecOf(st discovery.NumericStorage) policy.NumericSpec { - switch { - case st.Integer: - return policy.NumericSpec{Family: policy.NumericInteger, Bits: st.IntBits, Unsigned: st.Unsigned} - case st.FloatBits != 0: - return policy.NumericSpec{Family: policy.NumericFloat, Bits: st.FloatBits} - default: - return policy.NumericSpec{Family: policy.NumericDecimal, Precision: st.Precision, Scale: st.Scale} - } -} - -// eventView is one published event decoded once per Broadcast, in the two forms -// the delivery paths need: cells, the raw JSON value at each envelope column -// position (sliced positionally into the outgoing frame, so a value's bytes are -// never re-encoded), and row, the same values keyed by column name for the -// row-filter evaluator. +// eventView is one published event decoded once per Broadcast: cells, the raw +// JSON value at each envelope column position, sliced positionally into the +// outgoing frame so a value's bytes are never re-encoded. The row-filter reads +// the ORIGINAL positional bytes (ev.evt.Row) through the type layer rather than +// any decoding done here — the parse that decides visibility is ClickHouse's own. // // raw and decoded carry the legacy no-policy passthrough: a payload that is not // an EventMessage at all is forwarded verbatim when no policy store is wired, @@ -324,16 +277,13 @@ type eventView struct { decoded bool // raw parsed as an EventMessage usable bool // ...and it declares a known format whose columns and row pair cells []json.RawMessage - row map[string]any } -// newEventView decodes raw once for the whole fan-out. Numbers decode as -// json.Number — exact digit strings, not float64 — because the row-filter -// comparison must see the same value ClickHouse stores: ingest decodes with -// UseNumber and forwards the row verbatim, so a bare 64-bit ID past 2^53 keeps -// its exact digits on the query path, and a lossy float64 decode here would -// collapse neighboring IDs into one value and deliver another tenant's row. The -// outgoing frame reuses the raw cell bytes, so it stays byte-faithful regardless. +// newEventView decodes raw once for the whole fan-out. The envelope's own fields +// decode with UseNumber so nothing in it is rounded, and the row's cells are kept +// as raw bytes: the outgoing frame reuses them verbatim, so a 64-bit id past 2^53 +// keeps every digit on the wire, and the visibility decision never sees a Go +// float at all. func newEventView(raw []byte) *eventView { ev := &eventView{raw: raw} if !decodeEvent(raw, &ev.evt) { @@ -349,23 +299,28 @@ func newEventView(raw []byte) *eventView { if ev.evt.Format != ingest.FormatJSONCompactEachRow { return ev } - ev.cells, ev.row, ev.usable = pairRow(ev.evt.Columns, ev.evt.Row) + ev.cells, ev.usable = pairRow(ev.evt.Columns, ev.evt.Row) return ev } -// pairRow splits a compact row into its cells and zips them with the column -// names. ok is false when the two cannot be paired — an undecodable row, a +// pairRow splits a compact row into its cells and checks that they pair with the +// column names. ok is false when the two cannot be paired — an undecodable row, a // length that disagrees with the column list, a repeated column name, or an // empty column list (which no length check catches, since a zero-length row // agrees with it) — because there is then no way to say which value belongs to -// which column, and a row-filter that cannot read its column must withhold -// rather than guess. -func pairRow(cols []string, row json.RawMessage) (cells []json.RawMessage, byName map[string]any, ok bool) { +// which column, and neither the announced schema frame nor a row-filter may +// guess. +// +// This is ALL that survives of the old name-keyed decode: the arity and drift +// check the schema frame needs. The values themselves are never decoded here — +// they go to the type layer as the bytes they arrived as, and out to the client +// as the same bytes. +func pairRow(cols []string, row json.RawMessage) (cells []json.RawMessage, ok bool) { if len(row) == 0 { - return nil, nil, false + return nil, false } if err := json.Unmarshal(row, &cells); err != nil { - return nil, nil, false + return nil, false } // A zero-column envelope pairs with anything of length zero — both `null`, // which unmarshals to a nil slice, and `[]` — and would then be announced as @@ -374,32 +329,26 @@ func pairRow(cols []string, row json.RawMessage) (cells []json.RawMessage, byNam // (parseMsg's len(envelope.Columns) == 0), so refuse it here too rather than // let the two consumers disagree about an envelope neither can read. if len(cols) == 0 { - return nil, nil, false + return nil, false } if len(cells) != len(cols) { - return nil, nil, false + return nil, false } - byName = make(map[string]any, len(cols)) - for i, c := range cols { - // A repeated name has no single meaning: the map would keep the last - // value and silently drop the first, and this map is what a row-level - // filter is evaluated against — so a duplicate could decide visibility - // on a value the row never really carried. Unpairable, like a length - // mismatch. Cannot arise from our own producer (the envelope's columns - // come from system.columns, where ClickHouse forbids two columns of one - // name), so this is defence for an envelope we did not write. - if _, dup := byName[c]; dup { - return nil, nil, false - } - dec := json.NewDecoder(bytes.NewReader(cells[i])) - dec.UseNumber() - var v any - if err := dec.Decode(&v); err != nil { - return nil, nil, false + // A repeated name has no single meaning: the client zips the announced list + // against the positional row, so a duplicate makes two positions + // indistinguishable to it, and the type layer would read the row against a + // column list the table cannot have. Unpairable, like a length mismatch. + // Cannot arise from our own producer (the envelope's columns come from + // system.columns, where ClickHouse forbids two columns of one name), so this + // is defence for an envelope we did not write. + seen := make(map[string]struct{}, len(cols)) + for _, c := range cols { + if _, dup := seen[c]; dup { + return nil, false } - byName[c] = v + seen[c] = struct{}{} } - return cells, byName, true + return cells, true } // decodeEvent parses raw as a published EventMessage, reporting whether it is one @@ -438,15 +387,12 @@ func (h *Hub) snapshotPolicy() (p *policy.Policy, filter bool) { // policy. Replay is already per-connection, so row-level security evaluates against // this connection's claims directly; the returned closure holds one policy snapshot // for the whole gap-fill (matching Broadcast's one-snapshot-per-event — a reload -// landing mid-replay applies from the first live event) and caches the per-table -// column-kind lookup across the replay loop, so a large Last-Event-ID gap-fill -// doesn't pay a store read-lock plus a registry lookup and map build per event. +// landing mid-replay applies from the first live event), so a large Last-Event-ID +// gap-fill doesn't pay a store read-lock per event. // The closure is for a single goroutine — each connection makes its own. The live // path uses Broadcast. func (h *Hub) ReplayProjector(role string, sub *Subscriber) func(raw []byte) []Frame { p, filter := h.snapshotPolicy() - var colSpecs map[string]policy.ColumnSpec - specsFor := "" // table name colSpecs was resolved for ("" ⇒ not yet resolved) // Schema-drift state is LOCAL to this gap-fill, not the connection's shared // lastSchema. Replay writes straight to the socket while live events queue // behind it, so sharing the state would let a live event's announcement @@ -475,13 +421,15 @@ func (h *Hub) ReplayProjector(role string, sub *Subscriber) func(raw []byte) []F return nil } if plan.perms.HasRowFilter() { - // One topic ⇒ one table, so this resolves once per replay in practice; the - // guard re-resolves if a stream ever mixes tables rather than going stale. - if specsFor != ev.evt.TableName { - colSpecs = h.columnSpecs(ev.evt.TableName) - specsFor = ev.evt.TableName + // Replay is per connection, so one prepared row serves exactly one + // visibility question — but it is still closed immediately, because the + // gap-fill loop can run for thousands of events. + view, err := h.rowEvaluator().Prepare(ev.evt.TableName, ev.evt.Columns, ev.evt.Row) + admitted := h.rowAdmitted(p, role, ev, sub.claims, view, err) + if view != nil { + view.Close() } - if !h.rowAdmitted(p, role, ev, sub.claims, colSpecs) { + if !admitted { return nil // this row is filtered out for these claims } } diff --git a/internal/stream/hub_test.go b/internal/stream/hub_test.go index 0affb513..1efe864f 100644 --- a/internal/stream/hub_test.go +++ b/internal/stream/hub_test.go @@ -1,6 +1,7 @@ package stream import ( + "bytes" "context" "encoding/json" "fmt" @@ -18,6 +19,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/ingest" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/testutil" + "github.com/Wave-RF/WaveHouse/internal/typelayer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" @@ -166,10 +168,12 @@ func TestEventView_UnknownFormatWithheld(t *testing.T) { } // TestPairRow_Verdict enumerates the shapes that cannot be paired. Each one has -// to fail closed: a row-filter is evaluated against the name-keyed map, so a -// value read under the wrong name decides visibility on data the row never -// carried. The pairable case is the control that keeps the other seven honest — -// a pairRow that rejected everything would satisfy them alone. +// to fail closed: the announced column list is what the client zips the +// positional row against, and it is also the list the type layer reads the row +// under, so a shape where the two cannot be lined up would decide visibility — +// and label values — on data the row never carried. The pairable case is the +// control that keeps the other seven honest: a pairRow that rejected everything +// would satisfy them alone. func TestPairRow_Verdict(t *testing.T) { t.Parallel() for _, tt := range []struct { @@ -193,11 +197,10 @@ func TestPairRow_Verdict(t *testing.T) { } { t.Run(tt.name, func(t *testing.T) { t.Parallel() - cells, byName, ok := pairRow(tt.cols, json.RawMessage(tt.row)) + cells, ok := pairRow(tt.cols, json.RawMessage(tt.row)) assert.Equal(t, tt.ok, ok) if !tt.ok { assert.Nil(t, cells, "an unpairable envelope yields no cells to splice into a frame") - assert.Nil(t, byName, "and no map for a row-filter to read") } }) } @@ -207,12 +210,26 @@ func TestPairRow_Verdict(t *testing.T) { // declaration order or publish a column the record omits. func rawEventCols(tb testing.TB, table, ts string, cols []string, data map[string]any) []byte { tb.Helper() - schema := make([]discovery.Column, len(cols)) + // The positional line the ingest path publishes: one cell per column, in + // order, a column the record omits as null. Built here rather than through a + // production encoder because the wire shape is what these tests pin. + var buf bytes.Buffer + buf.WriteByte('[') for i, c := range cols { - schema[i] = discovery.Column{Name: c, Position: uint64(i + 1)} + if i > 0 { + buf.WriteByte(',') + } + v, ok := data[c] + if !ok { + buf.WriteString("null") + continue + } + b, err := json.Marshal(v) + require.NoError(tb, err) + buf.Write(b) } - row, err := ingest.EncodeCompactRow(schema, data) - require.NoError(tb, err) + buf.WriteByte(']') + row := json.RawMessage(buf.Bytes()) raw, err := json.Marshal(ingest.EventMessage{ TableName: table, ReceivedTimestamp: ts, @@ -455,6 +472,55 @@ func TestHub_ProjectsPerRole_DistinctRolesGetDistinctFrames(t *testing.T) { assert.NotEqual(t, fv.Data, fe.Data, "distinct role projections produce distinct bytes") } +// chtypesTable declares a table the way discovery would, numbering the columns +// in the order given. That order IS the wire order: an event's envelope carries +// the same list, and a row published under any other one is drift. +func chtypesTable(name string, cols ...discovery.Column) *discovery.TableSchema { + for i := range cols { + cols[i].Position = uint64(i + 1) + } + return &discovery.TableSchema{Name: name, Columns: cols} +} + +// col is chtypesTable's shorthand; every column here is an ordinary stored one. +func col(name, chType string) discovery.Column { + return discovery.Column{Name: name, Type: chType} +} + +// chtypesHub is a Hub whose row filtering is decided the way production decides +// it: by the type layer, against the real ClickHouse 26.6 artifact. Every +// row-filter test goes through this rather than a stub, because the verdicts +// under test ARE ClickHouse's — storage-domain narrowing, instant equality +// across spellings, exactness past 2^53 — and a stub could only restate what +// the test already believes. +func chtypesHub(tb testing.TB, store policy.Source, metric *Metrics, tables ...*discovery.TableSchema) *Hub { + tb.Helper() + hub := NewHub(store, nil, metric) + hub.RowEvaluator = NewRowEvaluator(newTestEngine(tb, tables...), nil) + return hub +} + +// engineBuild serializes Engine construction. typelayer.Engine.Bind sets +// chtypes' process-global Timezone under its OWN lock, so two Engines opened +// concurrently write it at once — harmless (both write "UTC") but a -race +// report, and these tests are parallel. Production has exactly one Engine and +// never hits it; the durable fix belongs in typelayer, not here. +var engineBuild sync.Mutex + +func newTestEngine(tb testing.TB, tables ...*discovery.TableSchema) *typelayer.Engine { + tb.Helper() + engineBuild.Lock() + defer engineBuild.Unlock() + return typelayer.TestEngine(tb, tables...) +} + +// clicksTable is the table the tenant-scoping tests publish into: the filtered +// column, the readable one, and one the role may not select. Declaration order +// matches the order rawEvent publishes (sorted by name). +func clicksTable() *discovery.TableSchema { + return chtypesTable("clicks", col("page", "String"), col("secret", "String"), col("tenant_id", "String")) +} + // rowFilterPolicy scopes role "viewer" to column "page" only, and to rows whose // tenant_id equals the caller's {{ jwt.tenant }} claim. The filter keys on tenant_id // — a column viewer may NOT select — so it also exercises the rule that row @@ -480,7 +546,7 @@ func rowFilterPolicy() *policy.Policy { // matching the constant-false predicate the query path binds for it. func TestHub_RowFilter_PerSubscriberIsolation(t *testing.T) { t.Parallel() - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), nil, clicksTable()) const topic = "ingest.clicks" acme := NewSubscriber(jwtClaims(t, map[string]any{"tenant": "acme"}), nil) @@ -502,9 +568,10 @@ func TestHub_RowFilter_PerSubscriberIsolation(t *testing.T) { // Unresolvable claim ⇒ no rows on the stream, matching the query path (#457). assertNoFrame(t, noTenant) - // A globex row reaches only the globex subscriber. + // A globex row reaches only the globex subscriber. Every event on a table + // carries that table's full column list, so "secret" is present here too. hub.Broadcast(topic, rawEvent(t, "clicks", "2026-06-26T00:00:01Z", - map[string]any{"tenant_id": "globex", "page": "/g"})) + map[string]any{"tenant_id": "globex", "page": "/g", "secret": "y"})) _, _, grow := recvEvent(t, globex) assert.Equal(t, "/g", grow["page"]) assertNoFrame(t, acme) @@ -524,7 +591,8 @@ func TestHub_RowFilter_ClaimsSnapshotImmuneToCallerMutation(t *testing.T) { "clicks": {"viewer": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"tenant_id": {Eq: new("{{ jwt.org.tenant }}")}}}}}, }, } - hub := NewHub(policy.Static(p), nil, nil) + tenantTable := chtypesTable("clicks", col("page", "String"), col("tenant_id", "String")) + hub := chtypesHub(t, policy.Static(p), nil, tenantTable) const topic = "ingest.clicks" org := map[string]any{"tenant": "globex"} @@ -550,7 +618,7 @@ func TestHub_RowFilter_ClaimsSnapshotImmuneToCallerMutation(t *testing.T) { "clicks": {"viewer": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"tenant_id": {In: new("{{ jwt.tenants }}")}}}}}, }, } - inHub := NewHub(policy.Static(inPolicy), nil, nil) + inHub := chtypesHub(t, policy.Static(inPolicy), nil, tenantTable) tenants := []any{"globex"} inSub := NewSubscriber(map[string]any{"tenants": tenants}, nil) inHub.Add(topic, "viewer", inSub) @@ -566,19 +634,29 @@ func TestHub_RowFilter_ClaimsSnapshotImmuneToCallerMutation(t *testing.T) { "the array snapshot keeps admitting the tenant list the connection authenticated with") } -// TestHub_RowFilter_MissingColumn_FailsClosed: an event that lacks the filtered -// column can't be proven visible, so it is withheld rather than leaked. -func TestHub_RowFilter_MissingColumn_FailsClosed(t *testing.T) { +// TestHub_RowFilter_ColumnsDrift_FailsClosed: an event whose column list is not +// the one the compiled generation exports cannot be read positionally at all — +// the filtered column could be at any offset, or absent — so it is withheld +// rather than guessed at. With positional rows this subsumes the old "event +// lacks the filtered column" case: a row missing a column IS a different column +// list. The control proves the withholding is the drift's doing and not a +// permanently silent hub. +func TestHub_RowFilter_ColumnsDrift_FailsClosed(t *testing.T) { t.Parallel() - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), nil, clicksTable()) const topic = "ingest.clicks" acme := NewSubscriber(map[string]any{"tenant": "acme"}, nil) hub.Add(topic, "viewer", acme) hub.Broadcast(topic, rawEvent(t, "clicks", "2026-06-26T00:00:00Z", - map[string]any{"page": "/a"})) // no tenant_id + map[string]any{"page": "/a"})) // a one-column envelope: not this generation's assertNoFrame(t, acme) + + hub.Broadcast(topic, rawEvent(t, "clicks", "2026-06-26T00:00:01Z", + map[string]any{"page": "/a", "secret": "x", "tenant_id": "acme"})) + _, _, row := recvEvent(t, acme) + assert.Equal(t, "/a", row["page"], "the generation's own column list is delivered") } // TestHub_RowFilter_SharedProjectionAcrossSameClaims: the column projection is still @@ -587,7 +665,7 @@ func TestHub_RowFilter_MissingColumn_FailsClosed(t *testing.T) { // per-subscriber, not the serialization. func TestHub_RowFilter_SharedProjectionAcrossSameClaims(t *testing.T) { t.Parallel() - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), nil, clicksTable()) const topic = "ingest.clicks" a := NewSubscriber(map[string]any{"tenant": "acme"}, nil) @@ -596,7 +674,7 @@ func TestHub_RowFilter_SharedProjectionAcrossSameClaims(t *testing.T) { hub.Add(topic, "viewer", b) hub.Broadcast(topic, rawEvent(t, "clicks", "2026-06-26T00:00:00Z", - map[string]any{"tenant_id": "acme", "page": "/a"})) + map[string]any{"tenant_id": "acme", "page": "/a", "secret": "x"})) fa, _, _ := recvEvent(t, a) fb, _, _ := recvEvent(t, b) @@ -605,27 +683,22 @@ func TestHub_RowFilter_SharedProjectionAcrossSameClaims(t *testing.T) { assert.Same(t, &fa.Data[0], &fb.Data[0], "one serialization shared across same-role subscribers") } -// TestHub_RowFilter_NumericOrdering_SchemaInformed drives the registry-backed path: -// with a numeric column type in the schema, an `amount > 100` filter compares -// numerically, so amount=9 is withheld (a lexicographic "9" > "100" comparison would -// have leaked it) and amount=250 is delivered. Without a registry the same ordering -// filter has no type to trust and withholds every row — fail closed, never the -// lexicographic leak (the schemaless window is real: boot-time discovery failure -// retries in the background while the server serves). -func TestHub_RowFilter_NumericOrdering_SchemaInformed(t *testing.T) { +// TestHub_RowFilter_NumericOrdering drives the type-layer path: on a UInt64 +// column an `amount > 100` filter compares the way ClickHouse compares, so +// amount=9 is withheld (a lexicographic "9" > "100" would have leaked it) and +// amount=250 is delivered. With no type layer wired there is nothing that can +// answer the question at all, so every row is withheld — fail closed, never the +// lexicographic leak (the engineless window is real: a boot that cannot open an +// engine still serves). +func TestHub_RowFilter_NumericOrdering(t *testing.T) { t.Parallel() - reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ - {Name: "clicks", Columns: []discovery.Column{ - {Name: "amount", Type: "UInt64"}, - {Name: "page", Type: "String"}, - }}, - }) p := &policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": {"viewer": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"amount": {Gt: new("100")}}}}}, }, } - hub := NewHub(policy.Static(p), reg, nil) + hub := chtypesHub(t, policy.Static(p), nil, + chtypesTable("clicks", col("amount", "UInt64"), col("page", "String"))) const topic = "ingest.clicks" sub := NewSubscriber(nil, nil) // constant filter value ⇒ no claims needed @@ -638,34 +711,32 @@ func TestHub_RowFilter_NumericOrdering_SchemaInformed(t *testing.T) { _, _, row := recvEvent(t, sub) assert.Equal(t, float64(250), row["amount"]) - // Same policy, no schema registry: an ordering predicate can't be proven either - // way, so both rows are withheld — including the one the schema-informed path - // delivers above. - noSchema := NewHub(policy.Static(p), nil, nil) + // Same policy, no type layer: an ordering predicate can't be answered either + // way, so both rows are withheld — including the one the wired path delivers. + noEngine := NewHub(policy.Static(p), nil, nil) blind := NewSubscriber(nil, nil) - noSchema.Add(topic, "viewer", blind) - noSchema.Broadcast(topic, rawEvent(t, "clicks", "t1", map[string]any{"amount": float64(9), "page": "/a"})) - noSchema.Broadcast(topic, rawEvent(t, "clicks", "t2", map[string]any{"amount": float64(250), "page": "/b"})) + noEngine.Add(topic, "viewer", blind) + noEngine.Broadcast(topic, rawEvent(t, "clicks", "t1", map[string]any{"amount": float64(9), "page": "/a"})) + noEngine.Broadcast(topic, rawEvent(t, "clicks", "t2", map[string]any{"amount": float64(250), "page": "/b"})) assertNoFrame(t, blind) } -// TestHub_RowFilter_FloatNarrowing_SchemaInformed drives storage-domain -// narrowing end-to-end through the registry: on a Float32 column, payload -// 16777217 stores as 16777216, so a `_gt: "16777216"` filter must withhold the -// event — the query path's WHERE over the stored row is false, and delivering -// the pre-narrowing payload was the ordering fail-open raised in review. A -// Float32-representable greater value still delivers. -func TestHub_RowFilter_FloatNarrowing_SchemaInformed(t *testing.T) { +// TestHub_RowFilter_FloatNarrowing drives storage-domain narrowing end-to-end: +// on a Float32 column, payload 16777217 stores as 16777216, so a +// `_gt: "16777216"` filter must withhold the event — the query path's WHERE over +// the stored row is false, and delivering the pre-narrowing payload was the +// ordering fail-open raised in review. A Float32-representable greater value +// still delivers. The narrowing is no longer a Go re-derivation of ClickHouse's +// rule: the row is parsed into the column's real storage before anything is +// compared. +func TestHub_RowFilter_FloatNarrowing(t *testing.T) { t.Parallel() - reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ - {Name: "clicks", Columns: []discovery.Column{{Name: "score", Type: "Float32"}}}, - }) p := &policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": {"viewer": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"score": {Gt: new("16777216")}}}}}, }, } - hub := NewHub(policy.Static(p), reg, nil) + hub := chtypesHub(t, policy.Static(p), nil, chtypesTable("clicks", col("score", "Float32"))) const topic = "ingest.clicks" sub := NewSubscriber(nil, nil) hub.Add(topic, "viewer", sub) @@ -678,6 +749,45 @@ func TestHub_RowFilter_FloatNarrowing_SchemaInformed(t *testing.T) { assert.Equal(t, float64(16777218), row["score"]) } +// TestHub_RowFilter_Float32EqualityBindsInTheColumnsWidth: the constant has to +// be read in the COLUMN's float domain, not the widest one. A Float32 column +// stores 0.1 as 0.100000001490116…, which is not Float64's 0.1 — so binding the +// constant as Float64 made `= "0.1"` withhold the row and `!= "0.1"` ADMIT it, +// on a row /v1/query returns. Measured against a real 26.6 server in +// tests/integration/rowfilter_stream_test.go; pinned here so the regression +// costs a unit test rather than a Docker run. +func TestHub_RowFilter_Float32EqualityBindsInTheColumnsWidth(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + filter policy.Filter + delivered bool + }{ + {"equality matches the stored Float32", policy.Filter{Eq: new("0.1")}, true}, + {"inequality does not", policy.Filter{Neq: new("0.1")}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p := &policy.Policy{Tables: map[string]policy.TablePolicy{ + "clicks": {"viewer": {Select: &policy.SelectPermissions{ + Filter: map[string]policy.Filter{"score": tt.filter}, + }}}, + }} + hub := chtypesHub(t, policy.Static(p), nil, chtypesTable("clicks", col("score", "Float32"))) + sub := NewSubscriber(nil, nil) + hub.Add("ingest.clicks", "viewer", sub) + + hub.Broadcast("ingest.clicks", rawEvent(t, "clicks", "t", map[string]any{"score": json.Number("0.1")})) + if tt.delivered { + f, _, _ := recvEvent(t, sub) + assert.NotEmpty(t, f.Data) + } else { + assertNoFrame(t, sub) + } + }) + } +} + func TestHub_TopicIsolation(t *testing.T) { t.Parallel() hub := NewHub(nil, nil, nil) @@ -866,7 +976,7 @@ func TestHub_ReplayProjector(t *testing.T) { // gap-fill event is projected only when the connection's claims satisfy the filter. func TestHub_ReplayProjector_RowFilter(t *testing.T) { t.Parallel() - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), nil, clicksTable()) raw := rawEvent(t, "clicks", "2026-06-26T00:00:00Z", map[string]any{"tenant_id": "acme", "page": "/a", "secret": "x"}) @@ -882,8 +992,8 @@ func TestHub_ReplayProjector_RowFilter(t *testing.T) { assert.NotContains(t, row, "secret", "denied column stripped on replay too") // The projector is reusable across a replay loop: a second event through the - // same closure (cached column kinds) projects identically — and does NOT - // re-announce a column list the connection already has. + // same closure projects identically — and does NOT re-announce a column + // list the connection already has. again := project(raw) require.Len(t, again, 1, "the column list is announced once per connection") assert.Equal(t, frames[1].Data, again[0].Data) @@ -929,9 +1039,9 @@ func TestHub_ConcurrentAddRemoveBroadcast_Race(t *testing.T) { // racing silently on a security decision. func TestHub_ConcurrentRowFilteredBroadcast_Race(t *testing.T) { t.Parallel() - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), nil, clicksTable()) const topic = "ingest.clicks" - raw := rawEvent(t, "clicks", "t", map[string]any{"tenant_id": "acme", "page": "/a"}) + raw := rawEvent(t, "clicks", "t", map[string]any{"tenant_id": "acme", "page": "/a", "secret": "x"}) var wg sync.WaitGroup for range 4 { // broadcasters: per-subscriber claims evaluation on every event @@ -959,26 +1069,23 @@ func TestHub_ConcurrentRowFilteredBroadcast_Race(t *testing.T) { } // TestHub_RowFilter_BigIntegerExact: a bare JSON integer past 2^53 must keep its -// exact digits through the hub's decode (UseNumber), or the row filter compares a +// exact digits all the way to the comparison, or the row filter compares a // lossily-rounded value: tenant 10000000000000001's row would falsely equal a // tenant claim of 10000000000000000 — float64 collapses the neighbors — and be // delivered cross-tenant on the stream while the query path (ClickHouse stores the -// exact digits ingest forwarded) excludes it. The raw payload is hand-built — -// marshaling a Go float64 would already have destroyed the value this test is about. +// exact digits ingest forwarded) excludes it. The row bytes now go to ClickHouse's +// own parser untouched and the claim binds as a UInt64 parameter, so no Go float +// is on the path at all. The raw payload is hand-built — marshaling a Go float64 +// would already have destroyed the value this test is about. func TestHub_RowFilter_BigIntegerExact(t *testing.T) { t.Parallel() - reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ - {Name: "clicks", Columns: []discovery.Column{ - {Name: "tenant_id", Type: "UInt64"}, - {Name: "page", Type: "String"}, - }}, - }) p := &policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": {"viewer": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"tenant_id": {Eq: new("{{ jwt.tenant }}")}}}}}, }, } - hub := NewHub(policy.Static(p), reg, nil) + hub := chtypesHub(t, policy.Static(p), nil, + chtypesTable("clicks", col("page", "String"), col("tenant_id", "UInt64"))) const topic = "ingest.clicks" // Claims come from real signed tokens through the production middleware, so a @@ -1000,20 +1107,14 @@ func TestHub_RowFilter_BigIntegerExact(t *testing.T) { "the wire frame carries the exact digits, not a float64 rounding") } -// TestHub_RowFilter_TimestampInstantMatch: since #402, ingest canonicalizes -// DateTime/DateTime64 payload values to RFC 3339 UTC before publish, while policy -// authors write the ClickHouse-friendly zone-less spelling the query path wants. -// The row filter compares the two as instants through the discovery grammar (one -// parser shared with canonicalization), so the spellings agree; an operand the -// grammar can't read withholds the row. +// TestHub_RowFilter_TimestampInstantMatch: the wire now carries ClickHouse's own +// rendering of a DateTime, and policy authors write the same zone-less spelling +// the query path wants. The filter compares them as instants because the row is +// parsed into the column's real storage before the predicate runs, so a +// different spelling of the same instant still matches; an operand the parser +// can't read withholds the row rather than guessing at it. func TestHub_RowFilter_TimestampInstantMatch(t *testing.T) { t.Parallel() - reg := testutil.NewTestSchemaRegistry(t, []*discovery.TableSchema{ - {Name: "clicks", Columns: []discovery.Column{ - {Name: "created_at", Type: "DateTime"}, - {Name: "page", Type: "String"}, - }}, - }) p := &policy.Policy{ Tables: map[string]policy.TablePolicy{ "clicks": { @@ -1023,21 +1124,27 @@ func TestHub_RowFilter_TimestampInstantMatch(t *testing.T) { }, }, } - hub := NewHub(policy.Static(p), reg, nil) + hub := chtypesHub(t, policy.Static(p), nil, + chtypesTable("clicks", col("created_at", "DateTime"), col("page", "String"))) const topic = "ingest.clicks" sub := NewSubscriber(nil, nil) hub.Add(topic, "viewer", sub) - // The canonical wire spelling ingest publishes: same instant, different bytes. - hub.Broadcast(topic, rawEvent(t, "clicks", "t1", map[string]any{"created_at": "2026-06-21T04:00:00Z", "page": "/a"})) - f, _, _ := recvEvent(t, sub) - assert.NotEmpty(t, f.Data, "canonical payload matches the zone-less constant as an instant") + hub.Broadcast(topic, rawEvent(t, "clicks", "t1", map[string]any{"created_at": "2026-06-21 04:00:00", "page": "/a"})) + f, cols, _ := recvEvent(t, sub) + assert.NotEmpty(t, f.Data, "the wire rendering matches the zone-less constant") + + // A different spelling of the same instant matches too: the comparison is + // between parsed instants, not between bytes. + hub.Broadcast(topic, rawEvent(t, "clicks", "t2", map[string]any{"created_at": "2026-06-21T04:00:00Z", "page": "/a"})) + f, _ = recvEventCols(t, sub, cols) + assert.NotEmpty(t, f.Data) - hub.Broadcast(topic, rawEvent(t, "clicks", "t2", map[string]any{"created_at": "2026-06-21T04:00:01Z", "page": "/a"})) + hub.Broadcast(topic, rawEvent(t, "clicks", "t3", map[string]any{"created_at": "2026-06-21 04:00:01", "page": "/a"})) assertNoFrame(t, sub) - hub.Broadcast(topic, rawEvent(t, "clicks", "t3", map[string]any{"created_at": "not a timestamp", "page": "/a"})) + hub.Broadcast(topic, rawEvent(t, "clicks", "t4", map[string]any{"created_at": "not a timestamp", "page": "/a"})) assertNoFrame(t, sub) } @@ -1057,34 +1164,44 @@ func TestHub_RowFilterWithheldIncrementsMetric(t *testing.T) { otel.SetMeterProvider(savedMP) }) - hub := NewHub(policy.Static(rowFilterPolicy()), nil, NewMetrics()) + hub := chtypesHub(t, policy.Static(rowFilterPolicy()), NewMetrics(), clicksTable()) const topic = "ingest.clicks" acme := NewSubscriber(map[string]any{"tenant": "acme"}, nil) globex := NewSubscriber(map[string]any{"tenant": "globex"}, nil) hub.Add(topic, "viewer", acme) hub.Add(topic, "viewer", globex) - raw := rawEvent(t, "clicks", "t", map[string]any{"tenant_id": "acme", "page": "/a"}) + raw := rawEvent(t, "clicks", "t", map[string]any{"tenant_id": "acme", "page": "/a", "secret": "x"}) hub.Broadcast(topic, raw) // delivered to acme, withheld from globex → 1 frames := hub.ReplayProjector("viewer", NewSubscriber(map[string]any{"tenant": "globex"}, nil))(raw) require.Empty(t, frames) // replay withhold → 2 + // A column list the compiled generation does not export: a FAULT, not a + // filter verdict, and the label is the only thing that says so. It withholds + // from BOTH subscribers — nothing about this row is readable — → 4. + hub.Broadcast(topic, rawEvent(t, "clicks", "t", map[string]any{"page": "/a"})) + var rm metricdata.ResourceMetrics require.NoError(t, reader.Collect(context.Background(), &rm)) - assert.Equal(t, int64(2), sumByName(rm, "wavehouse_sse_rows_withheld_total")) + assert.Equal(t, int64(4), sumByName(rm, "wavehouse_sse_rows_withheld_total")) + assert.Equal(t, int64(2), sumByNameAttr(rm, "wavehouse_sse_rows_withheld_total", "reason", ReasonFilter), + "the two claim mismatches are ordinary filtering") + assert.Equal(t, int64(2), sumByNameAttr(rm, "wavehouse_sse_rows_withheld_total", "reason", ReasonDrift), + "drift must be distinguishable from a policy decision") f, _, _ := recvEvent(t, acme) assert.NotEmpty(t, f.Data, "the entitled subscriber still gets the event") assertNoFrame(t, globex) } // BenchmarkBroadcast_RowFilteredFanout measures the per-subscriber cost a -// row-filtered role pays on the delivery hot path (#294/#353 vs #319): each -// subscriber's claims run through policy.Evaluate + RowVisible per event, where an -// unfiltered role shares one projection bucket-wide. Half the subscribers share the -// event's tenant (row visible), half don't (row withheld); either way each pays the -// per-subscriber evaluation, which is the cost under measurement. See #435 for the -// memoization follow-up this benchmark exists to arbitrate. +// row-filtered role pays on the delivery hot path (#294/#353 vs #319): the row is +// parsed once per event, then each subscriber's claims run through +// policy.Evaluate and one compiled-predicate evaluation, where an unfiltered role +// shares one projection bucket-wide. Half the subscribers share the event's +// tenant (row visible), half don't (row withheld); either way each pays the +// per-subscriber evaluation, which is the cost under measurement. See #435 for +// the memoization follow-up this benchmark exists to arbitrate. func BenchmarkBroadcast_RowFilteredFanout(b *testing.B) { const topic = "ingest.clicks" raw := rawEvent(b, "clicks", "2026-06-26T00:00:00Z", @@ -1092,7 +1209,7 @@ func BenchmarkBroadcast_RowFilteredFanout(b *testing.B) { for _, n := range []int{100, 1_000, 10_000} { b.Run(fmt.Sprintf("subscribers=%d", n), func(b *testing.B) { - hub := NewHub(policy.Static(rowFilterPolicy()), nil, nil) + hub := chtypesHub(b, policy.Static(rowFilterPolicy()), nil, clicksTable()) subs := make([]*Subscriber, n) for i := range n { tenant := "acme" @@ -1197,6 +1314,30 @@ func sumByNameKind(rm metricdata.ResourceMetrics, name, kind string) int64 { } // sumByName totals all datapoints of an Int64 sum instrument across kinds. +// sumByNameAttr sums one counter's data points restricted to a single attribute +// value — the withheld counter's "reason" is a closed set, and the point of the +// label is that a fault does not read as a filter verdict. +func sumByNameAttr(rm metricdata.ResourceMetrics, name, key, want string) int64 { + var total int64 + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != name { + continue + } + sum, ok := m.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + for _, dp := range sum.DataPoints { + if v, found := dp.Attributes.Value(attribute.Key(key)); found && v.AsString() == want { + total += dp.Value + } + } + } + } + return total +} + func sumByName(rm metricdata.ResourceMetrics, name string) int64 { for _, sm := range rm.ScopeMetrics { for _, md := range sm.Metrics { diff --git a/internal/stream/metrics.go b/internal/stream/metrics.go index c49c4bd1..0cd58845 100644 --- a/internal/stream/metrics.go +++ b/internal/stream/metrics.go @@ -45,7 +45,7 @@ func NewMetrics() *Metrics { dropped, _ := meter.Int64Counter("wavehouse_sse_dropped_frames_total", metric.WithDescription("SSE frames dropped to a full subscriber queue (slow consumer)")) withheld, _ := meter.Int64Counter("wavehouse_sse_rows_withheld_total", - metric.WithDescription("Event rows withheld from a subscriber by the role's row-level-security filter (including fail-closed evaluations)")) + metric.WithDescription("Event rows withheld from a subscriber by the role's row-level-security filter, labelled by why (including fail-closed evaluations)")) return &Metrics{active: active, duration: duration, frames: frames, bytes: bytes, dropped: dropped, withheld: withheld} } @@ -89,11 +89,23 @@ func (m *Metrics) FrameDropped(kind string) { // RowWithheld records one event row withheld from one subscriber (live or replay) // by the role's row-level-security filter, including fail-closed evaluations. // Labeled by table and role (policy-bounded, not data-bounded) so an operator can -// tell "no matching rows" from "a misconfigured filter withholding everything". -func (m *Metrics) RowWithheld(table, role string) { +// tell "no matching rows" from "a misconfigured filter withholding everything", +// and by reason (a Reason* constant, a closed set) so the cases that are a FAULT +// rather than a filter verdict are visible on their own: `unavailable` means the +// type layer has no compiled schema for the table — every row-filtered +// subscriber is dark until it does; `drift` means events are arriving under a +// column list the compiled generation does not export; and `error` means the +// row could not be parsed, or ClickHouse raised evaluating the predicate over +// it — which since the filter params became String includes a filter CONSTANT +// the column's type cannot read ("abc" or "1.5" against a numeric column), a +// case that used to count as `decline`. None of the three is a policy decision, +// and all three read as ordinary filtering without the label. +func (m *Metrics) RowWithheld(table, role, reason string) { if m == nil { return } - m.withheld.Add(context.Background(), 1, - metric.WithAttributes(attribute.String("table", table), attribute.String("role", role))) + m.withheld.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("table", table), + attribute.String("role", role), + attribute.String("reason", reason))) } diff --git a/internal/stream/roweval.go b/internal/stream/roweval.go new file mode 100644 index 00000000..ed10f215 --- /dev/null +++ b/internal/stream/roweval.go @@ -0,0 +1,190 @@ +package stream + +import ( + "errors" + "log/slog" + "sync" + + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/typelayer" +) + +// Reasons a row was withheld, the "reason" label on +// wavehouse_sse_rows_withheld_total. The first three are the type layer's own +// verdict classes, aliased so the two packages cannot drift on the spelling; the +// last two are this package's, for the cases where no verdict was ever reached. +const ( + // ReasonFilter: the role's predicate answered false — the ordinary case, and + // the only one that is not a symptom of something being wrong. It is also + // the answer for a predicate whose claim was unresolvable, which matches no + // row without the type layer compiling anything (the query path's `1 = 0`). + ReasonFilter = typelayer.ReasonFilter + // ReasonError: ClickHouse evaluated the predicate over this row and raised. + // Either side of the comparison can cause it, and both are the policy + // author's to fix: a stored value the expression cannot read, and a filter + // CONSTANT the column's type cannot read — a claim rendering as "abc", "-1", + // "1.5" or "007" against a numeric column now compiles and answers code 53 + // per row, where before the String-parameter binding it failed to compile + // and counted as ReasonDecline. Nothing hidden became visible; the label + // moved. + ReasonError = typelayer.ReasonError + // ReasonDecline: no verdict was reached at all — the expression would not + // compile for this generation, or chtypes would not answer for it. Withheld, + // like every answer that is not a definite true. + ReasonDecline = typelayer.ReasonDecline + // ReasonUnavailable: no compiled schema can answer for this table — no + // artifact for the server's version line, a compile refusal, or no engine + // wired at all. Nothing about the row; every row-filtered subscriber of the + // table is affected until it is fixed. + ReasonUnavailable = "unavailable" + // ReasonDrift: the event's column list is not the one the current compiled + // generation exports, so its positional row cannot be read at all. Expected + // briefly after a schema change, alarming if it persists. + ReasonDrift = "drift" +) + +// RowEvaluator is the one place a row's visibility under a role's row-filter is +// decided. Prepare is called ONCE per event — parsing the positional row is the +// expensive half — and the returned view answers for each subscriber's resolved +// permissions. The interface exists because that ratio (one parse, K visibility +// questions) is the whole shape of the hot path, and because it lets the tests +// drive admission from a stub instead of a compiled schema; the production +// implementation is engineEvaluator below. +// +// A nil RowEvaluator on the Hub means the fail-closed default (see +// Hub.rowEvaluator), never "everything visible". +type RowEvaluator interface { + // Prepare reads one event's row. columns is the envelope's column list, row + // the positional JSON array as published. An error means no subscriber of a + // row-filtered role may see this row; the Hub labels the withhold with + // WithheldReason(err). + Prepare(table string, columns []string, row []byte) (RowView, error) +} + +// RowView is one prepared event row, reusable across every subscriber of every +// row-filtered role on that event. Close must be called once the fan-out is +// done: the parsed row holds native memory that is not reference-counted. +type RowView interface { + // Visible reports whether perms admit this row, and when they do not, the + // Reason* label the withheld metric wants. reason is "" when visible. + Visible(perms *policy.ResolvedPermissions) (visible bool, reason string) + Close() +} + +// NewRowEvaluator builds the production evaluator: the row is parsed by +// ClickHouse's own parser and the role's predicate is evaluated by ClickHouse's +// own expression engine, so the stream's verdict is the one the query path's +// WHERE would reach over the stored row. +// +// A nil engine is a valid argument and withholds every row-filtered row (see +// engineEvaluator.Prepare) — the same answer the Hub's unwired default gives, +// so a boot path that forgets to build an Engine fails closed rather than +// silently unfiltered. +func NewRowEvaluator(types *typelayer.Engine, logger *slog.Logger) RowEvaluator { + return &engineEvaluator{types: types, logger: logger} +} + +// engineEvaluator is the RowEvaluator every production path uses: a +// typelayer.Engine parsing the row and evaluating the role's predicates on it. +type engineEvaluator struct { + types *typelayer.Engine + logger *slog.Logger + // unwired reports the missing engine once rather than once per event. The + // condition is a boot-time wiring mistake, so the first line says everything + // the next million would. + unwired sync.Once +} + +func (e *engineEvaluator) log() *slog.Logger { + if e.logger == nil { + return slog.Default() + } + return e.logger +} + +// errNoEngine is the cause behind an unwired evaluator's withholds. It is not a +// verdict about the row: no row of any row-filtered role can be evaluated. +var errNoEngine = errors.New("no type engine is wired into the stream hub") + +func (e *engineEvaluator) Prepare(table string, columns []string, row []byte) (RowView, error) { + if e.types == nil { + e.unwired.Do(func() { + e.log().Error("row-level security cannot be evaluated: no type engine is wired into the stream hub; "+ + "every row is withheld from every subscriber of a row-filtered role until one is", + "table", table) + }) + return nil, &withheldError{reason: ReasonUnavailable, err: errNoEngine} + } + tbl, err := e.types.Table(table) + if err != nil { + return nil, classifyPrepare(err) + } + parsed, err := tbl.ParseRow(columns, row) + if err != nil { + tbl.Release() + return nil, classifyPrepare(err) + } + return &engineRowView{tbl: tbl, row: parsed}, nil +} + +// engineRowView holds the table handle for as long as the parsed row lives: the +// row is owned by the compiled schema, so releasing the handle first would leave +// a rebind free of memory the fan-out is still reading. +type engineRowView struct { + tbl *typelayer.Table + row *typelayer.Row +} + +func (v *engineRowView) Visible(perms *policy.ResolvedPermissions) (bool, string) { + preds, ok := perms.Predicates() + if !ok { + // A denied grant, or one resolved for INSERT: no row is admissible. Same + // answer the query path gives by never running the SELECT at all. + return false, ReasonFilter + } + if len(preds) == 0 { + return true, "" + } + return v.row.VisibleWithReason(preds) +} + +func (v *engineRowView) Close() { + v.row.Close() + v.tbl.Release() +} + +// withheldError carries the metric label alongside the cause, so the Hub does +// not have to re-derive from an error string what the evaluator already knew. +type withheldError struct { + reason string + err error +} + +func (e *withheldError) Error() string { return e.err.Error() } +func (e *withheldError) Unwrap() error { return e.err } + +// classifyPrepare names WHY no view could be prepared. The three causes are +// operationally different — a missing artifact is an estate problem, drift is a +// schema change in flight, a parse failure is one bad event — and they are +// indistinguishable in the metric without the label. +func classifyPrepare(err error) error { + switch { + case typelayer.IsUnavailable(err): + return &withheldError{reason: ReasonUnavailable, err: err} + case errors.Is(err, typelayer.ErrColumnsDrift): + return &withheldError{reason: ReasonDrift, err: err} + default: + return &withheldError{reason: ReasonError, err: err} + } +} + +// WithheldReason maps a Prepare error to its metric label. An evaluator that +// returns a plain error (a test double, a future implementation) reads as +// "error" rather than losing the withhold. +func WithheldReason(err error) string { + var w *withheldError + if errors.As(err, &w) { + return w.reason + } + return ReasonError +} diff --git a/internal/stream/roweval_test.go b/internal/stream/roweval_test.go index 4dd85014..50febc01 100644 --- a/internal/stream/roweval_test.go +++ b/internal/stream/roweval_test.go @@ -1,26 +1,48 @@ package stream import ( + "errors" + "fmt" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/typelayer" ) -// recordingEvaluator answers every row the same way and counts the calls, so a -// test can prove both delivery paths reach row-level security through the seam. +// recordingEvaluator answers every row the same way and counts what the Hub +// does, so a test can prove both delivery paths reach row-level security through +// the seam — and that each path parses once and closes what it parsed. type recordingEvaluator struct { - visible bool - calls int + visible bool + prepErr error + prepares int + calls int + closes int } -func (e *recordingEvaluator) Visible(*policy.ResolvedPermissions, map[string]any, map[string]policy.ColumnSpec) bool { - e.calls++ - return e.visible +func (e *recordingEvaluator) Prepare(string, []string, []byte) (RowView, error) { + e.prepares++ + if e.prepErr != nil { + return nil, e.prepErr + } + return &recordingView{e: e}, nil +} + +type recordingView struct{ e *recordingEvaluator } + +func (v *recordingView) Visible(*policy.ResolvedPermissions) (bool, string) { + v.e.calls++ + if v.e.visible { + return true, "" + } + return false, ReasonFilter } +func (v *recordingView) Close() { v.e.closes++ } + // filteredPolicy grants "viewer" a row-filter, which is what puts the Hub on // the per-subscriber admission path in the first place. func filteredPolicy() *policy.Policy { @@ -44,8 +66,8 @@ func TestHub_RowEvaluatorSeam_LiveBroadcast(t *testing.T) { t.Parallel() // The tenant is chosen per case so each subtest builds the scenario its name // describes. With both cases sending a row the predicate withholds, the - // withhold case proved nothing — `seam.Visible(…) || perms.RowVisible(…)` - // would have passed it, since the policy withheld the row anyway. + // withhold case proved nothing — a seam consulted only as a veto would have + // passed it, since the policy withheld the row anyway. for _, tt := range []struct { name string visible bool @@ -70,6 +92,8 @@ func TestHub_RowEvaluatorSeam_LiveBroadcast(t *testing.T) { map[string]any{"tenant_id": tt.tenantID, "page": "/a"})) assert.Equal(t, 1, eval.calls, "the live path must consult the seam") + assert.Equal(t, 1, eval.prepares, "the row is parsed once for the event") + assert.Equal(t, 1, eval.closes, "and released after the fan-out") if tt.visible { f, _, _ := recvEvent(t, sub) assert.NotEmpty(t, f.Data) @@ -80,6 +104,64 @@ func TestHub_RowEvaluatorSeam_LiveBroadcast(t *testing.T) { } } +// TestHub_RowEvaluatorSeam_PreparesOncePerEvent: parsing the row is the +// expensive half of a row-filter decision, so it happens once per EVENT, not +// once per subscriber or once per role — that ratio is the whole reason the seam +// is split into Prepare and Visible. +func TestHub_RowEvaluatorSeam_PreparesOncePerEvent(t *testing.T) { + t.Parallel() + p := filteredPolicy() + // A second filtered role on the same table: the parse must be shared across + // roles too, not just across a role's subscribers. + tmpl := "{{ jwt.tenant }}" + p.Tables["clicks"]["editor"] = policy.RolePermissions{Select: &policy.SelectPermissions{ + Filter: map[string]policy.Filter{"tenant_id": {Eq: &tmpl}}, + }} + + eval := &recordingEvaluator{visible: true} + hub := NewHub(policy.Static(p), nil, nil) + hub.RowEvaluator = eval + for _, role := range []string{"viewer", "viewer", "viewer", "editor"} { + hub.Add("ingest.clicks", role, NewSubscriber(map[string]any{"tenant": "t1"}, nil)) + } + + hub.Broadcast("ingest.clicks", rawEvent(t, "clicks", "2026-06-26T00:00:00Z", + map[string]any{"tenant_id": "t1", "page": "/a"})) + + assert.Equal(t, 1, eval.prepares, "one parse serves every role and subscriber") + assert.Equal(t, 4, eval.calls, "visibility is still decided per subscriber") + assert.Equal(t, 1, eval.closes) +} + +// TestHub_RowEvaluatorSeam_PrepareError_WithholdsFilteredRolesOnly: when the row +// cannot be prepared at all — no compiled schema, a column list that is not the +// compiled generation's, an unparseable row — every row-filtered subscriber is +// withheld. Roles WITHOUT a row-filter never asked the type layer anything, so +// they must be unaffected: a table whose schema handle is briefly unavailable +// must not black out the streams that do not depend on it. +func TestHub_RowEvaluatorSeam_PrepareError_WithholdsFilteredRolesOnly(t *testing.T) { + t.Parallel() + p := filteredPolicy() + p.Tables["clicks"]["public"] = policy.RolePermissions{Select: &policy.SelectPermissions{}} + + eval := &recordingEvaluator{visible: true, prepErr: errors.New("boom")} + hub := NewHub(policy.Static(p), nil, nil) + hub.RowEvaluator = eval + + filtered := NewSubscriber(map[string]any{"tenant": "t1"}, nil) + unfiltered := NewSubscriber(nil, nil) + hub.Add("ingest.clicks", "viewer", filtered) + hub.Add("ingest.clicks", "public", unfiltered) + + hub.Broadcast("ingest.clicks", rawEvent(t, "clicks", "2026-06-26T00:00:00Z", + map[string]any{"tenant_id": "t1", "page": "/a"})) + + assertNoFrame(t, filtered) + f, _, _ := recvEvent(t, unfiltered) + assert.NotEmpty(t, f.Data, "an unfiltered role never consults the type layer, so it is unaffected") + assert.Zero(t, eval.calls, "a row that could not be prepared is never asked about") +} + // TestHub_RowEvaluatorSeam_Replay: the gap-fill path goes through the same seam // as the live path, so the two can't drift on how row visibility is decided. func TestHub_RowEvaluatorSeam_Replay(t *testing.T) { @@ -95,24 +177,62 @@ func TestHub_RowEvaluatorSeam_Replay(t *testing.T) { assert.Empty(t, frames, "the seam's verdict decides replay too") assert.Equal(t, 1, eval.calls) + assert.Equal(t, 1, eval.prepares) + assert.Equal(t, 1, eval.closes, "a gap-fill of thousands of events must not accumulate parsed rows") } -// TestHub_DefaultRowEvaluator_WhenUnwired: an un-wired Hub still enforces -// row-level security. A nil seam must never read as "everything is visible". +// TestHub_DefaultRowEvaluator_WhenUnwired: an un-wired Hub must never read as +// "everything is visible". With the type layer behind the seam there is nothing +// left in-process that can evaluate a filter, so the only safe default is to +// withhold every row of every row-filtered role — including the rows the +// predicate would have admitted, which is what makes this a real assertion +// rather than a restatement of the policy. func TestHub_DefaultRowEvaluator_WhenUnwired(t *testing.T) { t.Parallel() hub := NewHub(policy.Static(filteredPolicy()), nil, nil) require.Nil(t, hub.RowEvaluator) - assert.IsType(t, policyRowEvaluator{}, hub.rowEvaluator()) + assert.IsType(t, &engineEvaluator{}, hub.rowEvaluator(), + "the default must be the engine-backed evaluator with no engine, not a permissive stub") sub := NewSubscriber(map[string]any{"tenant": "t1"}, nil) hub.Add("ingest.clicks", "viewer", sub) + hub.Broadcast("ingest.clicks", rawEvent(t, "clicks", "2026-06-26T00:00:00Z", map[string]any{"tenant_id": "t2", "page": "/a"})) assertNoFrame(t, sub) + // The row the filter WOULD admit is withheld too: no engine, no verdict. hub.Broadcast("ingest.clicks", rawEvent(t, "clicks", "2026-06-26T00:00:01Z", map[string]any{"tenant_id": "t1", "page": "/a"})) - f, _, _ := recvEvent(t, sub) - assert.NotEmpty(t, f.Data) + assertNoFrame(t, sub) + + assert.Empty(t, hub.ReplayProjector("viewer", sub)(rawEvent(t, "clicks", "2026-06-26T00:00:02Z", + map[string]any{"tenant_id": "t1", "page": "/a"})), "replay fails closed on the same grounds") +} + +// TestNewRowEvaluator_NilEngineFailsClosed: the constructor accepts a nil Engine +// (a boot path that could not open one still has to build a Hub) and answers the +// same way the unwired default does, with the reason an operator needs. The +// reason matters: `unavailable` says the estate is broken, where `filter` would +// say the policy is working. +func TestNewRowEvaluator_NilEngineFailsClosed(t *testing.T) { + t.Parallel() + view, err := NewRowEvaluator(nil, nil).Prepare("clicks", []string{"page"}, []byte(`["/a"]`)) + require.Error(t, err) + assert.Nil(t, view) + assert.Equal(t, ReasonUnavailable, WithheldReason(err)) +} + +// TestWithheldReason_ClassifiesTypeLayerErrors: the three fault causes are +// operationally different and must not collapse into one label. An error from +// another evaluator that names no reason reads as "error" rather than being +// lost. +func TestWithheldReason_ClassifiesTypeLayerErrors(t *testing.T) { + t.Parallel() + assert.Equal(t, ReasonUnavailable, + WithheldReason(classifyPrepare(&typelayer.Unavailable{Table: "clicks", Cause: "no artifact"}))) + assert.Equal(t, ReasonDrift, + WithheldReason(classifyPrepare(fmt.Errorf("wrapped: %w", typelayer.ErrColumnsDrift)))) + assert.Equal(t, ReasonError, WithheldReason(classifyPrepare(errors.New("unparseable row")))) + assert.Equal(t, ReasonError, WithheldReason(errors.New("an evaluator that names no reason"))) } diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index eb7ab654..ecf1533a 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -183,6 +183,7 @@ func (m *MockJetStreamMsg) DoubleAck(_ context.Context) error { func (m *MockJetStreamMsg) NakWithDelay(_ time.Duration) error { panic("MockJetStreamMsg.NakWithDelay not implemented") } + func (m *MockJetStreamMsg) InProgress() error { panic("MockJetStreamMsg.InProgress not implemented") } func (m *MockJetStreamMsg) Term() error { panic("MockJetStreamMsg.Term not implemented") } func (m *MockJetStreamMsg) TermWithReason(string) error { diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 63f9a4c5..65947b35 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -27,8 +27,8 @@ func NopLogger() *slog.Logger { // NewTestSchemaRegistry creates a SchemaRegistry pre-loaded with the given // table schemas, without a real ClickHouse: a mock connection serves the // schemas as system.columns rows (UTC as the server zone) and the registry is -// built by the real discovery path — NewSchemaRegistry + Refresh — so -// timestamp column specs are precomputed exactly as in production. +// built by the real discovery path — NewSchemaRegistry + Refresh — so the +// derived fields and the refresh hooks behave exactly as in production. // // The registry holds schemas rebuilt from those rows (Name, Type, HasDefault, // DefaultKind, DefaultExpression, Position, DDL; IsNullable derived from the diff --git a/internal/typelayer/checks_test.go b/internal/typelayer/checks_test.go new file mode 100644 index 00000000..9f2480b8 --- /dev/null +++ b/internal/typelayer/checks_test.go @@ -0,0 +1,502 @@ +package typelayer + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wave-rf/chtypes/go/chtypes" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +func checksTable() *discovery.TableSchema { + return &discovery.TableSchema{ + Name: "checks", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt32", Position: 1}, + {Name: "tenant", Type: "String", Position: 2}, + {Name: "kind", Type: "String", Position: 3}, + }, + } +} + +// checksBody is four records whose verdicts under the cases below are the +// measured table (AUDIT §A.2). +const checksBody = `{"id":1,"tenant":"acme","kind":"a"}` + "\n" + + `{"id":2,"tenant":"acme","kind":"a"}` + "\n" + + `{"id":3,"tenant":"evil","kind":"a"}` + "\n" + + `{"id":4,"tenant":"acme","kind":"z"}` + "\n" + +func checksHandle(t *testing.T) *Table { + t.Helper() + eng := TestEngine(t, checksTable()) + tbl, err := eng.Table("checks") + require.NoError(t, err) + t.Cleanup(tbl.Release) + return tbl +} + +// checkReasons is each record's CheckReason, after asserting every record +// parsed and that exactly the admitted ones carry exported bytes. +func checkReasons(t *testing.T, b Batch) []string { + t.Helper() + out := make([]string, len(b.Rows)) + for i, r := range b.Rows { + require.True(t, r.Accepted, "record %d: %s", i, r.Message) + assert.Equal(t, r.CheckReason == "", r.Line != nil, + "record %d: bytes are exported for exactly the admitted records", i) + out[i] = r.CheckReason + } + return out +} + +// TestIngestChecks_MatchesTheMeasuredTable: one AND-joined filter attached to +// the one parse, a verdict per record, and bytes only for the records it +// admits. +func TestIngestChecks_MatchesTheMeasuredTable(t *testing.T) { + tbl := checksHandle(t) + + const f = ReasonFilter + cases := []struct { + name string + preds []Predicate + want []string + }{ + { + "tenant equals", + []Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}}, + []string{"", "", f, ""}, + }, + { + "kind in", + []Predicate{{Column: "kind", Op: "in", Values: []string{"a", "b"}}}, + []string{"", "", "", f}, + }, + { + "both, AND-joined", + []Predicate{ + {Column: "tenant", Op: "=", Values: []string{"acme"}}, + {Column: "kind", Op: "in", Values: []string{"a"}}, + }, + []string{"", "", f, f}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), tc.preds...) + require.NoError(t, err) + assert.Equal(t, tc.want, checkReasons(t, batch)) + }) + } + + // A record the filter cuts from the middle does not stop the reader: the + // admitted record after it still carries its own bytes. + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), + Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}) + require.NoError(t, err) + assert.Equal(t, `[4, "acme", "z"]`, string(batch.Rows[3].Line)) +} + +// TestIngestChecks_NoPredicatesPassesEveryRow: a role with no check clauses +// pays nothing and hides nothing. +func TestIngestChecks_NoPredicatesPassesEveryRow(t *testing.T) { + tbl := checksHandle(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody)) + require.NoError(t, err) + assert.Equal(t, []string{"", "", "", ""}, checkReasons(t, batch)) + assert.Equal(t, 0, tbl.slots[0].filters.len(), "no filter is compiled without checks") +} + +func TestIngestChecks_EmptyBody(t *testing.T) { + tbl := checksHandle(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, nil, Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}) + require.NoError(t, err) + assert.Empty(t, batch.Rows) +} + +// TestIngestChecks_FailsClosed: everything that is not a definite true +// withholds, and an unresolvable claim never reaches the compiler. +func TestIngestChecks_FailsClosed(t *testing.T) { + tbl := checksHandle(t) + compiled := func() int { return tbl.slots[0].filters.len() } + const f, e = ReasonFilter, ReasonError + + t.Run("empty values", func(t *testing.T) { + before := compiled() + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), + Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}, + Predicate{Column: "kind", Op: "in", Values: nil}) + require.NoError(t, err) + assert.Equal(t, []string{f, f, f, f}, checkReasons(t, batch)) + assert.Equal(t, before, compiled(), "an unresolvable claim must not reach the compiler") + }) + + t.Run("unknown column", func(t *testing.T) { + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), + Predicate{Column: "nosuch", Op: "=", Values: []string{"x"}}) + require.NoError(t, err) + assert.Equal(t, []string{f, f, f, f}, checkReasons(t, batch)) + }) + + t.Run("value the column cannot read", func(t *testing.T) { + // On an integer column the strict cast answers it: no record fits. + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), + Predicate{Column: "id", Op: "=", Values: []string{"abc"}}) + require.NoError(t, err) + assert.Equal(t, []string{f, f, f, f}, checkReasons(t, batch), + "an integer claim that does not fit the column is 'the data says no' (403)") + + // On any other column the server's own reader throws. + eng := TestEngine(t, &discovery.TableSchema{Name: "ratios", Columns: []discovery.Column{ + {Name: "id", Type: "UInt32", Position: 1}, + {Name: "ratio", Type: "Float64", Position: 2}, + }}) + ratios, err := eng.Table("ratios") + require.NoError(t, err) + t.Cleanup(ratios.Release) + batch, err = ratios.Ingest(FormatJSONEachRow, []byte(`{"id":1,"ratio":0.5}`+"\n"+`{"id":2,"ratio":1}`+"\n"), + Predicate{Column: "ratio", Op: "=", Values: []string{"abc"}}) + require.NoError(t, err) + assert.Equal(t, []string{e, e}, checkReasons(t, batch), + "a thrown predicate is 'we could not tell' (422), not 'the data says no' (403)") + assert.Contains(t, batch.Rows[0].Message, "abc", "the predicate's own error rides along for the log") + }) + + t.Run("a filter closed between lookup and use", func(t *testing.T) { + preds := []Predicate{{Column: "tenant", Op: "=", Values: []string{"closed"}}} + expr, params, ok := tbl.render(preds) + require.True(t, ok) + for _, s := range tbl.slots { // whichever slot the request lands on + f := tbl.filterOn(s, expr, params) + require.NotNil(t, f) + f.Close() // what an eviction racing this request does + } + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(checksBody), preds...) + require.NoError(t, err, "a closed filter fails the checks, not the request") + assert.Equal(t, []string{ReasonDecline, ReasonDecline, ReasonDecline, ReasonDecline}, checkReasons(t, batch)) + }) +} + +// TestIngestChecks_IntegerClaimThatDoesNotFitIsRefused: the insert check +// compares an integer claim through the strict cast, so a claim the column +// cannot hold refuses every record — including one whose value the plain +// String binding would have wrapped the claim onto (2^64+5 → 5), and one filled +// from the injected DEFAULT, which the compiler may itself wrap. A claim that +// fits admits exactly as before. +func TestIngestChecks_IntegerClaimThatDoesNotFitIsRefused(t *testing.T) { + eng := TestEngine(t, ordersTable()) + const over = "18446744073709551621" // 2^64+5 + body := []byte(`{"id":1}` + "\n" + `{"id":2,"amount":5}` + "\n" + `{"id":3,"amount":6}` + "\n") + + t.Run("a claim past the column's range", func(t *testing.T) { + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"amount": over}}) + batch, err := tbl.Ingest(FormatJSONEachRow, body, Predicate{Column: "amount", Op: "=", Values: []string{over}}) + require.NoError(t, err) + assert.Equal(t, []string{ReasonFilter, ReasonFilter, ReasonFilter}, checkReasons(t, batch)) + }) + + t.Run("a claim that fits", func(t *testing.T) { + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"amount": "5"}}) + batch, err := tbl.Ingest(FormatJSONEachRow, body, Predicate{Column: "amount", Op: "=", Values: []string{"5"}}) + require.NoError(t, err) + assert.Equal(t, []string{"", "", ReasonFilter}, checkReasons(t, batch)) + assert.Equal(t, `[1, "", "", 5]`, string(batch.Rows[0].Line)) + }) + + t.Run("an _in set keeps only the elements that fit", func(t *testing.T) { + tbl := checksHandleFor(t, eng, "orders") + in := []byte(`{"id":1,"amount":5}` + "\n" + `{"id":2,"amount":0}` + "\n" + + `{"id":3,"amount":"18446744073709551615"}` + "\n" + `{"id":4,"amount":7}` + "\n") + batch, err := tbl.Ingest(FormatJSONEachRow, in, Predicate{ + Column: "amount", Op: "in", + Values: []string{over, "007", "0", "115792089237316195423570985008687907853269984665640564039457584007913129639941", "+7"}, + }) + require.NoError(t, err) + assert.Equal(t, []string{ReasonFilter, "", ReasonFilter, ReasonFilter}, checkReasons(t, batch)) + }) +} + +func checksHandleFor(t *testing.T, eng *Engine, table string) *Table { + t.Helper() + tbl, err := eng.Table(table) + require.NoError(t, err) + t.Cleanup(tbl.Release) + return tbl +} + +// TestIngestChecks_ParseOutcomeDecidesFirst pins a measured trap: under the +// compile profile's allow_errors_ratio a record that does not parse is +// skipped, and chtypes answers it 'd' beside that outcome — with the verdict's +// own code and message EMPTY on 26.6. It must report its parse error (a 400 +// with code 27), never a check decline (a 422), and never shift a neighbour +// onto its answer. +func TestIngestChecks_ParseOutcomeDecidesFirst(t *testing.T) { + tbl := checksHandle(t) + body := []byte(strings.Join([]string{ + `{"id":1,"tenant":"acme","kind":"a"}`, + `{"id":"not-a-number","tenant":"acme","kind":"a"}`, + `{"id":3,"tenant":"evil","kind":"a"}`, + `{"id":4,"tenant":"acme","kind":"a"}`, + }, "\n") + "\n") + preds := []Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}} + + // The trap itself, at the SDK: a skipped row's verdict is 'd' with no + // verdict code, and its error is only in ErrCode/ErrMsg. + s := tbl.slots[0] + expr, params, ok := tbl.render(preds) + require.True(t, ok) + res, err := s.schema.RowsExportWith(FormatJSONEachRow, body, InsertSettings(), chtypes.JSONCompactEachRow, + chtypes.WithRowFilter(tbl.filterOn(s, expr, params))) + require.NoError(t, err) + require.Len(t, res.Rows, 4) + require.Equal(t, chtypes.Skipped, res.Rows[1].Outcome) + require.NotNil(t, res.Rows[1].Verdict) + assert.Equal(t, chtypes.VerdictDecline, *res.Rows[1].Verdict) + assert.Equal(t, 27, res.Rows[1].ErrCode) + assert.Equal(t, 2, res.RowsPassed) + assert.Equal(t, 1, res.RowsCut, "a skipped row is 'd' but not counted as cut") + + batch, err := tbl.Ingest(FormatJSONEachRow, body, preds...) + require.NoError(t, err) + require.Len(t, batch.Rows, 4) + assert.True(t, batch.Rows[0].Accepted) + assert.Empty(t, batch.Rows[0].CheckReason) + assert.Equal(t, `[1, "acme", "a"]`, string(batch.Rows[0].Line)) + + assert.False(t, batch.Rows[1].Accepted) + assert.False(t, batch.Rows[1].Declined, "a parse refusal is a verdict about the data") + assert.Equal(t, 27, batch.Rows[1].Code) + assert.Empty(t, batch.Rows[1].CheckReason) + + assert.Equal(t, ReasonFilter, batch.Rows[2].CheckReason) + assert.Nil(t, batch.Rows[2].Line) + assert.Equal(t, `[4, "acme", "a"]`, string(batch.Rows[3].Line)) +} + +// TestIngestChecks_RejectedBatchExportsNothing pins the other measured trap: +// RowsPassed counts the admitted rows of a batch whose own outcome is +// rejected, and such a batch exports no bytes. It is reachable here — an +// NDJSON-declared single-line array is not reframed (only the JSON family's +// arrays are), and one bad element rejects it whole. Every record must be +// declined; none may be published on the strength of RowsPassed. +func TestIngestChecks_RejectedBatchExportsNothing(t *testing.T) { + tbl := checksHandle(t) + body := []byte(`[{"id":1,"tenant":"acme","kind":"a"},{"id":"x","tenant":"acme","kind":"a"},{"id":3,"tenant":"acme","kind":"a"}]`) + preds := []Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}} + + s := tbl.slots[0] + expr, params, ok := tbl.render(preds) + require.True(t, ok) + res, err := s.schema.RowsExportWith(FormatJSONEachRow, body, InsertSettings(), chtypes.JSONCompactEachRow, + chtypes.WithRowFilter(tbl.filterOn(s, expr, params))) + require.NoError(t, err) + require.Equal(t, chtypes.Rejected, res.Outcome) + require.Positive(t, res.RowsPassed, "the trap: an admitted row is counted in a rejected batch") + require.Empty(t, res.Payload) + + batch, err := tbl.Ingest(FormatJSONEachRow, body, preds...) + require.NoError(t, err) + require.NotEmpty(t, batch.Rows) + for i, r := range batch.Rows { + assert.False(t, r.Accepted, "record %d", i) + assert.True(t, r.Declined, "record %d", i) + assert.Nil(t, r.Line, "record %d", i) + } +} + +// TestIngestChecks_OnTheWithNamesFormats: the header is not a record, so the +// verdicts line up with the data lines whatever order the header names the +// columns in. +func TestIngestChecks_OnTheWithNamesFormats(t *testing.T) { + tbl := checksHandle(t) + + batch, err := tbl.Ingest(FormatCSVWithNames, []byte("tenant,id,kind\nacme,1,a\nevil,2,a\nacme,x,a\nacme,4,a\n"), + Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}) + require.NoError(t, err) + require.Len(t, batch.Rows, 4) + assert.Equal(t, `[1, "acme", "a"]`, string(batch.Rows[0].Line)) + assert.Equal(t, ReasonFilter, batch.Rows[1].CheckReason) + assert.Equal(t, 27, batch.Rows[2].Code) + assert.Equal(t, `[4, "acme", "a"]`, string(batch.Rows[3].Line)) +} + +func TestIngest_UnavailableTable(t *testing.T) { + eng := TestEngine(t, checksTable()) + eng.Bind(TestServerVersion, "UTC", nil) + + _, err := eng.Table("checks") + require.Error(t, err) + assert.True(t, IsUnavailable(err)) +} + +func TestIngest_RefusesAFormatItCannotParse(t *testing.T) { + tbl := checksHandle(t) + + for _, f := range []Format{chtypes.JSONCompactEachRow, chtypes.RowBinary, chtypes.Native, Format(99)} { + _, err := tbl.Ingest(f, []byte(`{"id":1}`+"\n")) + require.Error(t, err, "format %d", int(f)) + assert.False(t, IsUnavailable(err), "a bad format is a programming error, not an outage") + assert.Contains(t, err.Error(), "FormatJSONEachRow") + } +} + +// TestIngest_PositionalFormats: CSV and TSV are declaration-order positional; +// header detection is ClickHouse's default and StrictPositional switches it off. +func TestIngest_PositionalFormats(t *testing.T) { + tbl := checksHandle(t) + + csv, err := tbl.Ingest(FormatCSV, []byte("1,acme,a\n2,evil,z\n")) + require.NoError(t, err) + require.Len(t, csv.Rows, 2) + require.True(t, csv.Rows[0].Accepted, csv.Rows[0].Message) + assert.Equal(t, `[1, "acme", "a"]`, string(csv.Rows[0].Line)) + + tsv, err := tbl.Ingest(FormatTSV, []byte("1\tacme\ta\n")) + require.NoError(t, err) + require.Len(t, tsv.Rows, 1) + require.True(t, tsv.Rows[0].Accepted, tsv.Rows[0].Message) + assert.Equal(t, `[1, "acme", "a"]`, string(tsv.Rows[0].Line)) + + // With StrictPositional a header line is one failed record with ClickHouse's + // own code 27, even one that names every column; by default ClickHouse + // consumes it as a header (input_format_*_detect_header is on by default), + // so the same body is one data row. + for name, body := range map[string][]byte{ + "csv": []byte("id,tenant,kind\n1,acme,a\n"), + "tsv": []byte("id\ttenant\tkind\n1\tacme\ta\n"), + } { + format := FormatCSV + if name == "tsv" { + format = FormatTSV + } + strict, err := tbl.IngestWith(format, IngestOptions{StrictPositional: true}, body) + require.NoError(t, err) + require.Len(t, strict.Rows, 2, name) + assert.False(t, strict.Rows[0].Accepted, name) + assert.Equal(t, 27, strict.Rows[0].Code, name) + assert.True(t, strict.Rows[1].Accepted, name) + + detected, err := tbl.Ingest(format, body) + require.NoError(t, err) + require.Len(t, detected.Rows, 1, name) + assert.True(t, detected.Rows[0].Accepted, name) + } +} + +// TestIngest_WithNamesFormats pins the header formats as measured on the 26.6 +// artifact: the header line is not a record (Rows index the data lines), it +// names the columns in any order, a column it omits takes its DEFAULT, and a +// name the schema lacks or a repeated name refuses the body whole with +// ClickHouse's own 117. +func TestIngest_WithNamesFormats(t *testing.T) { + tbl := checksHandle(t) + + for _, tc := range []struct { + format Format + sep string + }{{FormatCSVWithNames, ","}, {FormatTSVWithNames, "\t"}} { + body := func(lines ...string) []byte { + for i, l := range lines { + lines[i] = strings.ReplaceAll(l, ",", tc.sep) + } + return []byte(strings.Join(lines, "\n") + "\n") + } + name := fmt.Sprintf("format %d", int(tc.format)) + + b, err := tbl.Ingest(tc.format, body("kind,id,tenant", "a,1,acme", "z,x,acme", "b,3,evil")) + require.NoError(t, err, name) + require.Len(t, b.Rows, 3, "%s: the header is not a record", name) + assert.Equal(t, `[1, "acme", "a"]`, string(b.Rows[0].Line), name) + assert.Equal(t, 27, b.Rows[1].Code, name) + assert.Equal(t, `[3, "evil", "b"]`, string(b.Rows[2].Line), name) + + b, err = tbl.Ingest(tc.format, body("id,tenant", "1,acme")) + require.NoError(t, err, name) + require.Len(t, b.Rows, 1, name) + assert.Equal(t, `[1, "acme", ""]`, string(b.Rows[0].Line), "%s: an omitted column takes its DEFAULT", name) + + b, err = tbl.Ingest(tc.format, body("id,tenant,kind")) + require.NoError(t, err, name) + assert.Empty(t, b.Rows, "%s: a header alone is zero records", name) + assert.Nil(t, b.Refused, name) + + for _, header := range []string{"id,tenant,kind,extra", "id,id,kind"} { + b, err = tbl.Ingest(tc.format, body(header, "1,acme,a,z")) + require.NoError(t, err, name) + require.NotNil(t, b.Refused, "%s: %s", name, header) + assert.Equal(t, 117, b.Refused.Code, "%s: %s", name, header) + assert.NotEmpty(t, b.Refused.Message) + assert.Empty(t, b.Rows) + } + } + + // Header names match case-insensitively from 26.5, as the server does. + b, err := tbl.Ingest(FormatCSVWithNames, []byte("ID,Tenant,KIND\n1,acme,a\n")) + require.NoError(t, err) + require.Len(t, b.Rows, 1) + assert.True(t, b.Rows[0].Accepted, b.Rows[0].Message) +} + +// BenchmarkIngest_HandlePool is the standing evidence behind maxPoolSize (re-run it on deployment hardware). +// Three arms, identical work, only the concurrency and the handle differ: +// +// - serial: one goroutine, one handle — the cost of a call with no contention +// - parallel-shared: GOMAXPROCS goroutines, ONE handle +// - parallel-pooled: GOMAXPROCS goroutines, the whole pool +// +// Read it this way: if parallel-shared's ns/op is not meaningfully better than +// serial's, RowsExport is not parallelizing at all and a pool of handles +// cannot help — which is what darwin shows, while Linux scales (see maxPoolSize). +// Only when parallel-shared is ~GOMAXPROCS× worse than serial does a pool have +// anything to win, and parallel-pooled is then the size of the win. +func BenchmarkIngest_HandlePool(b *testing.B) { + eng := TestEngine(b, checksTable()) + tbl, err := eng.Table("checks") + require.NoError(b, err) + defer tbl.Release() + + var body strings.Builder + for i := range 500 { + body.WriteString(`{"id":` + strconv.Itoa(i) + `,"tenant":"acme","kind":"a"}` + "\n") + } + raw := []byte(body.String()) + + // Identical work on both arms — only the handle choice differs. + export := func(b *testing.B, pick func() *schemaSlot) { + b.Helper() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := pick().schema.RowsExport( + FormatJSONEachRow, raw, InsertSettings(), chtypes.JSONCompactEachRow); err != nil { + b.Fatal(err) + } + } + }) + } + + b.Run("serial", func(b *testing.B) { + s := tbl.slots[0] + b.ResetTimer() + for range b.N { + if _, err := s.schema.RowsExport( + FormatJSONEachRow, raw, InsertSettings(), chtypes.JSONCompactEachRow); err != nil { + b.Fatal(err) + } + } + }) + b.Run("parallel-shared", func(b *testing.B) { + export(b, func() *schemaSlot { return tbl.slots[0] }) + }) + b.Run("parallel-pooled", func(b *testing.B) { + export(b, tbl.slot) + }) +} diff --git a/internal/typelayer/errors.go b/internal/typelayer/errors.go new file mode 100644 index 00000000..a70a5891 --- /dev/null +++ b/internal/typelayer/errors.go @@ -0,0 +1,40 @@ +package typelayer + +import ( + "errors" + "fmt" +) + +// Unavailable reports that no compiled schema can answer for a table right +// now. It is never a verdict about data: callers map it to HTTP 503 on ingest +// and to "withhold" on the stream, so it must stay distinguishable from a +// ClickHouse rejection. +// +// Table is "" when the cause is process-wide (no artifact for the server's +// version line, or a timezone the process cannot adopt). +type Unavailable struct { + Table string + // Cause is the operator-facing reason, already carrying the SDK's own + // wording where there is one (an artifact-missing error lists the + // directories it searched, which is the whole diagnostic). + Cause string +} + +func (e *Unavailable) Error() string { + if e.Table == "" { + return "chtypes unavailable: " + e.Cause + } + return fmt.Sprintf("chtypes unavailable for table %q: %s", e.Table, e.Cause) +} + +// IsUnavailable reports whether err is an *Unavailable anywhere in its chain. +func IsUnavailable(err error) bool { + var u *Unavailable + return errors.As(err, &u) +} + +// ErrColumnsDrift is returned by ParseRow when the envelope's column list is +// not the one the current compiled handle exports. A positional row is only +// interpretable against the generation that produced it, so a mismatch means +// the event predates a schema change and must be withheld rather than guessed. +var ErrColumnsDrift = errors.New("row columns do not match the table's wire columns") diff --git a/internal/typelayer/filter.go b/internal/typelayer/filter.go new file mode 100644 index 00000000..b66b4dbc --- /dev/null +++ b/internal/typelayer/filter.go @@ -0,0 +1,290 @@ +package typelayer + +import ( + "container/list" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + + "github.com/wave-rf/chtypes/go/chtypes" + + "github.com/Wave-RF/WaveHouse/internal/chsql" + "github.com/Wave-RF/WaveHouse/internal/policy" +) + +// filterCacheSize bounds the compiled filters held per table. A filter handle +// is identified by (expression, bound values), and the values come from tenant +// claims, so an unbounded cache is a memory/CPU denial of service. The budget +// is split across the handle pool (see compileDDL), so this is the table's +// total, not each slot's. +const filterCacheSize = 4096 + +// Predicate is one resolved row-filter clause. Values are the canonical strings +// policy already resolved for the SQL path, so the stream and the query answer +// off one resolution; len(Values)==0 means "matches nothing" and never widens. +type Predicate = policy.Predicate + +// Reason names why a row was not visible, for the withheld-reason metric. +const ( + ReasonFilter = "filter" // the predicate answered false + ReasonError = "error" // the predicate threw on this row's values + ReasonDecline = "decline" // chtypes would not answer +) + +// Row is one parsed event, reusable across every subscriber's filter. Parsing +// is the expensive half, so the hub parses once per event and evaluates K +// filters against the result. +// +// The Table it came from must stay held (not Released) until Close. +type Row struct { + table *Table + slot *schemaSlot + block *chtypes.LoadedBlock +} + +// ParseRow parses one JSONCompactEachRow line, with or without its trailing +// newline. columns must be the generation's wire columns exactly: a positional +// row is uninterpretable against any other order, so a mismatch is +// ErrColumnsDrift rather than a guess. +func (t *Table) ParseRow(columns []string, row []byte) (*Row, error) { + if len(t.slots) == 0 { + return nil, &Unavailable{Table: t.Name, Cause: t.cause} + } + if !slices.Equal(columns, t.WireColumns) { + return nil, fmt.Errorf("%w: event carries %v, generation %d exports %v", + ErrColumnsDrift, columns, t.Generation, t.WireColumns) + } + body := row + if n := len(body); n == 0 || body[n-1] != '\n' { + body = append(append(make([]byte, 0, n+1), body...), '\n') + } + // The block and every filter evaluated against it stay on ONE handle: a + // cross-handle Eval takes both handles' locks and hands back exactly the + // serialization the pool exists to avoid. + s := t.slot() + block, err := s.schema.ParseBlock(chtypes.JSONCompactEachRow, body, InsertSettings()) + if err != nil { + return nil, err + } + return &Row{table: t, slot: s, block: block}, nil +} + +// Close frees the parsed block. Required: the C layer does not refcount. +func (r *Row) Close() { + if r.block != nil { + r.block.Close() + r.block = nil + } +} + +// Visible reports whether the row satisfies every predicate. Only a definite +// true is visible — false, a predicate that threw, and a decline all withhold. +func (r *Row) Visible(preds []Predicate) bool { + ok, _ := r.VisibleWithReason(preds) + return ok +} + +// VisibleWithReason is Visible plus the label the withheld-reason metric wants. +// The reason is "" when the row is visible. +func (r *Row) VisibleWithReason(preds []Predicate) (bool, string) { + if len(preds) == 0 { + return true, "" + } + if r.block == nil { + return false, ReasonDecline + } + expr, params, ok := r.table.render(preds) + if !ok { + // An unresolvable predicate matches nothing, exactly as the SQL path's + // `1 = 0` does — the two surfaces must not disagree (#457). + return false, ReasonFilter + } + + filter := r.table.filterOn(r.slot, expr, params) + if filter == nil { + return false, ReasonDecline + } + res, err := filter.Eval(r.block) + if err != nil || res.Outcome != chtypes.FilterOK || len(res.Verdicts) == 0 { + return false, ReasonDecline + } + return verdictBool(res.Verdicts[0]) +} + +// verdictBool maps one chtypes verdict onto (visible, reason) — for a stored +// row's visibility and for an ingested record's insert check alike. Only +// VerdictTrue is true, and the zero value is Decline, so an answer nobody set +// withholds. +func verdictBool(v chtypes.Verdict) (bool, string) { + switch v { + case chtypes.VerdictTrue: + return true, "" + case chtypes.VerdictFalse: + return false, ReasonFilter + case chtypes.VerdictError: + return false, ReasonError + case chtypes.VerdictDecline: + return false, ReasonDecline + default: + return false, ReasonDecline + } +} + +// render builds the AND-joined expression and the parameter map. +// +// Every value binds as a {pN:String} parameter, whatever the column's declared +// type — the same binding the SQL path uses, so both surfaces read a claim with +// ClickHouse's own comparison-time coercion (AUDIT §C.1): a spelling the column +// cannot read (`-1` on an unsigned column, `1.5`) is the server's own code 53 +// at evaluation, which withholds the row. A bare String binding still wraps an +// integer value at or past 2^64 before comparing (and a 128/256-bit column at +// its own width), so on an integer column the parameter is compared through +// chsql.StrictInt instead: a claim that is not the canonical spelling of a +// value the column can hold is NULL and matches nothing on any operator, and +// an in-range claim answers exactly as the plain binding does. The query path +// renders the same expression (policy.ResolvedSelect.WhereSQL). +// +// Every value is encoded with chsql.EscapeStringParam, the SAME encoding the +// SQL path uses for its `{p:String}` parameters. The artifact reads a filter +// parameter with ClickHouse's escaped-text reader, exactly as the server reads +// one off the HTTP interface: measured on the 26.6 artifact, a stored `a\b` +// compared false against the raw value (the `\b` read as a backspace), and a +// stored tab, newline or trailing backslash would not compile at all (a +// decline — every row withheld). With the encoding applied all of them compare +// equal, on `=` and on `in` alike. Before this, a claim carrying any of those +// bytes silently withheld rows the SQL path returned. +// +// Identifiers are the library's own QuoteIdentifier spelling (see +// declaredColumns). Values are never interpolated, so a hostile claim is inert +// by construction. Reports false when a predicate cannot be expressed, which +// fails closed without compiling anything. +func (t *Table) render(preds []Predicate) (string, map[string]string, bool) { + var b strings.Builder + params := make(map[string]string, len(preds)) + n := 0 + bind := func(col filterColumn, v string) { + name := fmt.Sprintf("p%d", n) + n++ + params[name] = chsql.EscapeStringParam(v) + if col.intType != "" { + b.WriteString(chsql.StrictInt(name, col.intType)) + return + } + fmt.Fprintf(&b, "{%s:String}", name) + } + for i, p := range preds { + col, known := t.cols[p.Column] + if !known || len(p.Values) == 0 { + return "", nil, false + } + if i > 0 { + b.WriteString(" AND ") + } + b.WriteString(col.ident) + switch p.Op { + case "=", "!=", ">", "<": + if len(p.Values) != 1 { + return "", nil, false + } + fmt.Fprintf(&b, " %s ", p.Op) + bind(col, p.Values[0]) + case "in": + b.WriteString(" IN (") + for j, v := range p.Values { + if j > 0 { + b.WriteString(", ") + } + bind(col, v) + } + b.WriteByte(')') + default: + return "", nil, false + } + } + return b.String(), params, true +} + +// filterOn returns the compiled handle for this expression and parameter set +// on ONE slot, compiling it at most once per (slot, generation). A compile +// failure is cached as a negative entry so a broken policy costs one compile +// and one log line, not one per event. +func (t *Table) filterOn(s *schemaSlot, expr string, params map[string]string) *chtypes.LoadedFilter { + key := cacheKey(t.Generation, expr, params) + c := s.filters + + c.mu.Lock() + defer c.mu.Unlock() + if el, hit := c.index[key]; hit { + c.order.MoveToFront(el) + return el.Value.(*filterEntry).filter + } + + f, err := s.schema.CompileFilter(expr, chtypes.WithFilterParams(params)) + if err != nil { + f = nil + t.log.Error("row filter will not compile; withholding every row for it", + "table", t.Name, "generation", t.Generation, "expr", expr, "error", err) + } + el := c.order.PushFront(&filterEntry{key: key, filter: f}) + c.index[key] = el + if c.order.Len() > c.cap { + c.evictOldestLocked() + } + return f +} + +// cacheKey identifies a compiled handle. Values are baked into the handle at +// compile time, so they belong in the key alongside the generation that owns +// the schema. +func cacheKey(generation uint64, expr string, params map[string]string) string { + // The parameter names are positional (p0, p1 …) and generated from the same + // expression, so a JSON object over them is stable without sorting. + enc, _ := json.Marshal(params) + return fmt.Sprintf("%d\x00%s\x00%s", generation, expr, enc) +} + +type filterEntry struct { + key string + filter *chtypes.LoadedFilter // nil: this expression does not compile +} + +type filterCache struct { + mu sync.Mutex + cap int + order *list.List // front = most recently used + index map[string]*list.Element +} + +func newFilterCache(capacity int) *filterCache { + return &filterCache{cap: capacity, order: list.New(), index: make(map[string]*list.Element)} +} + +func (c *filterCache) evictOldestLocked() { + el := c.order.Back() + if el == nil { + return + } + c.order.Remove(el) + e := el.Value.(*filterEntry) + delete(c.index, e.key) + if e.filter != nil { + e.filter.Close() + } +} + +// closeAll drops every handle. Called before the schema is closed so no freed +// pointer survives in the index; the schema would close them anyway, but not +// the map holding them. +func (c *filterCache) closeAll() { + c.mu.Lock() + defer c.mu.Unlock() + for el := c.order.Front(); el != nil; el = el.Next() { + if f := el.Value.(*filterEntry).filter; f != nil { + f.Close() + } + } + c.order.Init() + c.index = make(map[string]*list.Element) +} diff --git a/internal/typelayer/filter_test.go b/internal/typelayer/filter_test.go new file mode 100644 index 00000000..78f52d2a --- /dev/null +++ b/internal/typelayer/filter_test.go @@ -0,0 +1,611 @@ +package typelayer + +import ( + "encoding/json" + "fmt" + "math/big" + "strconv" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// rowsTable carries one column of each family the row filter has to compare, +// including the three (UInt8, Int64, Float32) where a typed parameter used to +// disagree with the server and a String parameter does not. +func rowsTable() *discovery.TableSchema { + return &discovery.TableSchema{ + Name: "rows", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt8", Position: 1}, + {Name: "tenant", Type: "String", Position: 2}, + {Name: "ts", Type: "DateTime", Position: 3}, + {Name: "amt", Type: "Decimal(10,2)", Position: 4}, + {Name: "big", Type: "Int64", Position: 5}, + {Name: "ratio", Type: "Float32", Position: 6}, + {Name: "tags", Type: "Array(String)", Position: 7}, + }, + } +} + +const sampleRow = `[7, "acme", "2026-01-15 10:30:00", "12.50", -5, 0.1, ["a"]]` + +func parsedRow(t *testing.T) (*Table, *Row) { + t.Helper() + eng := TestEngine(t, rowsTable()) + tbl, err := eng.Table("rows") + require.NoError(t, err) + t.Cleanup(tbl.Release) + + row, err := tbl.ParseRow(tbl.WireColumns, []byte(sampleRow)) + require.NoError(t, err) + t.Cleanup(row.Close) + return tbl, row +} + +// TestVisible_StringBindingAcrossColumnFamilies is the pin for the whole +// §C.1 decision: every value binds as {pN:String} and the answer still matches +// the SQL path on every operator and every column family. The row is +// id=7, tenant="acme", ts=2026-01-15 10:30:00, amt=12.50, big=-5, ratio=0.1. +func TestVisible_StringBindingAcrossColumnFamilies(t *testing.T) { + _, row := parsedRow(t) + + cases := []struct { + name string + pred Predicate + want bool + }{ + {"string equal", Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}, true}, + {"string equal miss", Predicate{Column: "tenant", Op: "=", Values: []string{"beta"}}, false}, + {"string not equal", Predicate{Column: "tenant", Op: "!=", Values: []string{"beta"}}, true}, + {"string not equal self", Predicate{Column: "tenant", Op: "!=", Values: []string{"acme"}}, false}, + {"string greater", Predicate{Column: "tenant", Op: ">", Values: []string{"aaa"}}, true}, + {"string less", Predicate{Column: "tenant", Op: "<", Values: []string{"aaa"}}, false}, + {"string in", Predicate{Column: "tenant", Op: "in", Values: []string{"acme", "beta"}}, true}, + {"string in miss", Predicate{Column: "tenant", Op: "in", Values: []string{"beta", "gamma"}}, false}, + + {"uint equal", Predicate{Column: "id", Op: "=", Values: []string{"7"}}, true}, + {"uint not equal", Predicate{Column: "id", Op: "!=", Values: []string{"8"}}, true}, + {"uint greater", Predicate{Column: "id", Op: ">", Values: []string{"3"}}, true}, + {"uint less", Predicate{Column: "id", Op: "<", Values: []string{"3"}}, false}, + {"uint in", Predicate{Column: "id", Op: "in", Values: []string{"1", "7"}}, true}, + + {"int64 equal", Predicate{Column: "big", Op: "=", Values: []string{"-5"}}, true}, + {"int64 not equal", Predicate{Column: "big", Op: "!=", Values: []string{"-5"}}, false}, + {"int64 greater", Predicate{Column: "big", Op: ">", Values: []string{"-6"}}, true}, + {"int64 less", Predicate{Column: "big", Op: "<", Values: []string{"-6"}}, false}, + {"int64 in", Predicate{Column: "big", Op: "in", Values: []string{"-5", "0"}}, true}, + + // The #381 storage-narrowing case: a Float32 column's 0.1 is + // 0.100000001490116…, so a constant widened to Float64 would NOT equal + // it and the stream would then admit `!= '0.1'` on a row /v1/query + // hides. A String parameter is read in the column's own domain. + {"float32 equal", Predicate{Column: "ratio", Op: "=", Values: []string{"0.1"}}, true}, + {"float32 not equal", Predicate{Column: "ratio", Op: "!=", Values: []string{"0.1"}}, false}, + {"float32 greater", Predicate{Column: "ratio", Op: ">", Values: []string{"0.05"}}, true}, + {"float32 less", Predicate{Column: "ratio", Op: "<", Values: []string{"0.05"}}, false}, + {"float32 in", Predicate{Column: "ratio", Op: "in", Values: []string{"0.1", "2"}}, true}, + + {"decimal equal", Predicate{Column: "amt", Op: "=", Values: []string{"12.50"}}, true}, + {"decimal equal shorter spelling", Predicate{Column: "amt", Op: "=", Values: []string{"12.5"}}, true}, + {"decimal not equal", Predicate{Column: "amt", Op: "!=", Values: []string{"12.49"}}, true}, + {"decimal greater", Predicate{Column: "amt", Op: ">", Values: []string{"12.49"}}, true}, + {"decimal less", Predicate{Column: "amt", Op: "<", Values: []string{"12.49"}}, false}, + {"decimal in", Predicate{Column: "amt", Op: "in", Values: []string{"12.50", "1"}}, true}, + + {"datetime equal", Predicate{Column: "ts", Op: "=", Values: []string{"2026-01-15 10:30:00"}}, true}, + {"datetime not equal", Predicate{Column: "ts", Op: "!=", Values: []string{"2026-01-15 10:30:00"}}, false}, + {"datetime greater", Predicate{Column: "ts", Op: ">", Values: []string{"2026-01-01 00:00:00"}}, true}, + {"datetime less", Predicate{Column: "ts", Op: "<", Values: []string{"2026-01-01 00:00:00"}}, false}, + {"datetime in", Predicate{Column: "ts", Op: "in", Values: []string{"2026-01-15 10:30:00"}}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, row.Visible([]Predicate{tc.pred})) + }) + } +} + +// TestVisible_HostileSpellingsMatchNothing: on an integer column a claim is +// compared through the strict cast, so a value outside the column's domain, or +// a spelling that is not its canonical form, matches nothing on EVERY operator +// — `!=` included — and is answered false rather than thrown. On a +// non-integer column the String binding is unchanged: a spelling the column's +// reader refuses is still the server's own code 53, which withholds as an +// error. +func TestVisible_HostileSpellingsMatchNothing(t *testing.T) { + _, row := parsedRow(t) + + for _, v := range []string{"256", "300", "-1", "1.5", "007", "+7", "7.0", "1e3", "abc", "", " 7", "18446744073709551623"} { + for _, op := range []string{"=", "!=", "<", ">", "in"} { + got, reason := row.VisibleWithReason([]Predicate{{Column: "id", Op: op, Values: []string{v}}}) + assert.False(t, got, "id %s %q", op, v) + assert.Equal(t, ReasonFilter, reason, "id %s %q: answered, not thrown", op, v) + } + } + + for _, v := range []string{"abc", "1.5.5"} { + got, reason := row.VisibleWithReason([]Predicate{{Column: "ratio", Op: "=", Values: []string{v}}}) + assert.False(t, got, "ratio = %q", v) + assert.Equal(t, ReasonError, reason, "ratio = %q", v) + } +} + +// intsTable holds one column per integer width the strict cast has to cover. +func intsTable() *discovery.TableSchema { + return &discovery.TableSchema{ + Name: "ints", + Columns: []discovery.Column{ + {Name: "u8", Type: "UInt8", Position: 1}, + {Name: "u32", Type: "UInt32", Position: 2}, + {Name: "u64", Type: "UInt64", Position: 3}, + {Name: "i64", Type: "Int64", Position: 4}, + {Name: "u128", Type: "UInt128", Position: 5}, + {Name: "i128", Type: "Int128", Position: 6}, + {Name: "u256", Type: "UInt256", Position: 7}, + {Name: "i256", Type: "Int256", Position: 8}, + {Name: "nu64", Type: "Nullable(UInt64)", IsNullable: true, Position: 9}, + }, + } +} + +// intDomain is a column's [min, max]. +func intDomain(typ string) (*big.Int, *big.Int) { + bits := map[string]uint{"8": 8, "32": 32, "64": 64, "128": 128, "256": 256} + pow := func(n uint) *big.Int { return new(big.Int).Lsh(big.NewInt(1), n) } + signed := strings.HasPrefix(typ, "Int") + n := bits[strings.TrimPrefix(strings.TrimPrefix(typ, "U"), "Int")] + if signed { + return new(big.Int).Neg(pow(n - 1)), new(big.Int).Sub(pow(n-1), big.NewInt(1)) + } + return big.NewInt(0), new(big.Int).Sub(pow(n), big.NewInt(1)) +} + +// TestVisible_IntegerClaimsMatchExactlyWhatFits drives every integer width +// with the boundary claims that used to wrap (2^63, 2^64, 2^64+5, 2^127, +// 2^128, 2^255, 2^256, 2^256+5, their negatives) and the non-canonical +// spellings, on every operator, against rows holding 0, 5, the column's MIN +// and MAX (and NULL). The answer must be the mathematical one when the claim +// is the canonical spelling of a value the column can hold, and false +// otherwise — never an over-admit, and never a thrown row. +func TestVisible_IntegerClaimsMatchExactlyWhatFits(t *testing.T) { + eng := TestEngine(t, intsTable()) + tbl, err := eng.Table("ints") + require.NoError(t, err) + t.Cleanup(tbl.Release) + + pow := func(n uint) *big.Int { return new(big.Int).Lsh(big.NewInt(1), n) } + add := func(a *big.Int, d int64) *big.Int { return new(big.Int).Add(a, big.NewInt(d)) } + neg := func(a *big.Int) *big.Int { return new(big.Int).Neg(a) } + claims := []string{ + "0", "5", "-1", "255", "256", "4294967295", "4294967296", + "007", "+5", "1.5", "5.0", "1e3", "abc", "", "-0", + } + for _, n := range []*big.Int{ + pow(63), add(neg(pow(63)), -1), neg(pow(63)), add(pow(63), -1), + pow(64), add(pow(64), -1), add(pow(64), 5), pow(127), add(neg(pow(127)), -1), add(pow(127), -1), + pow(128), add(pow(128), -1), pow(255), add(pow(255), -1), neg(pow(255)), pow(256), add(pow(256), -1), add(pow(256), 5), + } { + claims = append(claims, n.String()) + } + + cols := intsTable().Columns + type stored struct { + vals map[string]*big.Int // nil value: NULL + row *Row + } + var rows []stored + for _, label := range []string{"0", "5", "MIN", "MAX"} { + vals := map[string]*big.Int{} + line := make([]any, len(cols)) + for i, c := range cols { + base := strings.TrimSuffix(strings.TrimPrefix(c.Type, "Nullable("), ")") + lo, hi := intDomain(base) + var v *big.Int + switch label { + case "0": + v = big.NewInt(0) + case "5": + v = big.NewInt(5) + case "MIN": + v = lo + case "MAX": + v = hi + } + if c.IsNullable && label == "MIN" { + line[i], vals[c.Name] = nil, nil + continue + } + line[i], vals[c.Name] = v.String(), v + } + b, err := json.Marshal(line) + require.NoError(t, err) + row, err := tbl.ParseRow(tbl.WireColumns, b) + require.NoError(t, err) + t.Cleanup(row.Close) + rows = append(rows, stored{vals: vals, row: row}) + } + + cells := 0 + for _, c := range cols { + base := strings.TrimSuffix(strings.TrimPrefix(c.Type, "Nullable("), ")") + lo, hi := intDomain(base) + for _, claim := range claims { + v, isInt := new(big.Int).SetString(claim, 10) + fits := isInt && v.String() == claim && v.Cmp(lo) >= 0 && v.Cmp(hi) <= 0 + for _, op := range []string{"=", "!=", "<", ">", "in"} { + for _, r := range rows { + cells++ + stored := r.vals[c.Name] + want := false + if fits && stored != nil { + cmp := stored.Cmp(v) + want = map[string]bool{"=": cmp == 0, "in": cmp == 0, "!=": cmp != 0, "<": cmp < 0, ">": cmp > 0}[op] + } + got, reason := r.row.VisibleWithReason([]Predicate{{Column: c.Name, Op: op, Values: []string{claim}}}) + if got != want || (!got && reason != ReasonFilter) { + t.Errorf("%s(%v) %s %q: got %v (%s), want %v", c.Type, stored, op, claim, got, reason, want) + } + } + } + } + } + + // A multi-element _in list keeps the elements that fit and drops the rest, + // element by element: 2^64+5 must not wrap onto the row holding 5. + for _, c := range cols { + for _, r := range rows { + stored := r.vals[c.Name] + want := stored != nil && stored.Sign() == 0 + got := r.row.Visible([]Predicate{{ + Column: c.Name, Op: "in", + Values: []string{add(pow(64), 5).String(), "007", "0", add(pow(256), 5).String(), "abc"}, + }}) + assert.Equal(t, want, got, "%s(%v) IN (2^64+5, 007, 0, 2^256+5, abc)", c.Type, stored) + } + } + t.Logf("%d cells", cells) +} + +// storedRow parses one row whose `tenant` column holds the given value, with +// every other column at sampleRow's value. +func storedRow(t *testing.T, tbl *Table, tenant string) *Row { + t.Helper() + line, err := json.Marshal([]any{7, tenant, "2026-01-15 10:30:00", "12.50", -5, 0.1, []string{"a"}}) + require.NoError(t, err) + row, err := tbl.ParseRow(tbl.WireColumns, line) + require.NoError(t, err) + t.Cleanup(row.Close) + return row +} + +// TestVisible_EscapedStringParamsMatchTheStoredValue pins the shared +// {p:String} encoding (chsql.EscapeStringParam) on the artifact. +// +// Measured on the 26.6 artifact BEFORE the encoding was applied: a stored +// `a\b` compared FALSE against the raw claim value (the artifact's parameter +// reader is ClickHouse's escaped-text reader, so `\b` arrived as a backspace), +// and a stored tab, newline or trailing backslash made the filter fail to +// compile at all — a decline, every row withheld. The SQL path answered true +// for all of them, so the two read surfaces disagreed. With the encoding they +// agree. +func TestVisible_EscapedStringParamsMatchTheStoredValue(t *testing.T) { + eng := TestEngine(t, rowsTable()) + tbl, err := eng.Table("rows") + require.NoError(t, err) + t.Cleanup(tbl.Release) + + cases := []struct { + name string + stored string + }{ + {"embedded backslash", `a\b`}, + {"tab", "a\tb"}, + {"newline", "a\nb"}, + {"carriage return", "a\rb"}, + {"single quote", "O'Brien"}, + {"trailing backslash", `trail\`}, + {"backslash then quote", `a\'b`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + row := storedRow(t, tbl, tc.stored) + + got, reason := row.VisibleWithReason([]Predicate{{Column: "tenant", Op: "=", Values: []string{tc.stored}}}) + assert.True(t, got, "tenant = %q withheld (reason %q)", tc.stored, reason) + + got, reason = row.VisibleWithReason([]Predicate{{Column: "tenant", Op: "in", Values: []string{"other", tc.stored}}}) + assert.True(t, got, "tenant in (…, %q) withheld (reason %q)", tc.stored, reason) + + got, _ = row.VisibleWithReason([]Predicate{{Column: "tenant", Op: "=", Values: []string{tc.stored + "x"}}}) + assert.False(t, got, "tenant = %q must not match", tc.stored+"x") + }) + } + + // The encoding has to DISTINGUISH, not merely admit: the three characters + // `a\tb` and the three bytes "ab" are different stored values, and each + // must match only its own filter. Unencoded, both filters read as the tab. + t.Run("a literal backslash-t is not a tab", func(t *testing.T) { + literal := storedRow(t, tbl, `a\tb`) + assert.True(t, literal.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{`a\tb`}}})) + assert.False(t, literal.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"a\tb"}}})) + + tab := storedRow(t, tbl, "a\tb") + assert.True(t, tab.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"a\tb"}}})) + assert.False(t, tab.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{`a\tb`}}})) + }) +} + +// TestVisible_PredicatesAreANDed matches the SQL path's AND-joined WHERE. +func TestVisible_PredicatesAreANDed(t *testing.T) { + _, row := parsedRow(t) + + assert.True(t, row.Visible([]Predicate{ + {Column: "tenant", Op: "=", Values: []string{"acme"}}, + {Column: "id", Op: "=", Values: []string{"7"}}, + })) + assert.False(t, row.Visible([]Predicate{ + {Column: "tenant", Op: "=", Values: []string{"acme"}}, + {Column: "id", Op: "=", Values: []string{"8"}}, + })) +} + +func TestVisible_NoPredicatesIsVisible(t *testing.T) { + _, row := parsedRow(t) + assert.True(t, row.Visible(nil)) +} + +// TestVisible_FailsClosed: every shape typelayer cannot answer for must hide +// the row. A row filter that widens on a bad input is the leak class this +// package exists to remove. +func TestVisible_FailsClosed(t *testing.T) { + _, row := parsedRow(t) + + t.Run("unknown column", func(t *testing.T) { + before := row.slot.filters.len() + assert.False(t, row.Visible([]Predicate{{Column: "nosuch", Op: "=", Values: []string{"x"}}})) + assert.Equal(t, before, row.slot.filters.len(), "an unknown column must not reach the compiler") + }) + + t.Run("empty values", func(t *testing.T) { + before := row.slot.filters.len() + assert.False(t, row.Visible([]Predicate{{Column: "tenant", Op: "in", Values: nil}})) + assert.Equal(t, before, row.slot.filters.len(), "an unresolvable claim must not reach the compiler") + }) + + t.Run("unsupported operator", func(t *testing.T) { + assert.False(t, row.Visible([]Predicate{{Column: "tenant", Op: "LIKE", Values: []string{"ac%"}}})) + }) + + t.Run("value the column cannot read", func(t *testing.T) { + // "abc" is not a UInt8 (the strict cast makes it NULL) nor a Float32 + // (the server's own per-row error). Neither is a compile failure. + assert.False(t, row.Visible([]Predicate{{Column: "id", Op: "=", Values: []string{"abc"}}})) + assert.False(t, row.Visible([]Predicate{{Column: "ratio", Op: "=", Values: []string{"abc"}}})) + }) + + t.Run("type mismatch against the column", func(t *testing.T) { + // A String parameter compared with an Array column has no supertype. + assert.False(t, row.Visible([]Predicate{{Column: "tags", Op: "=", Values: []string{"a"}}})) + }) +} + +func TestVisibleWithReason_LabelsTheCause(t *testing.T) { + tbl, row := parsedRow(t) + + ok, reason := row.VisibleWithReason([]Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}}) + assert.True(t, ok) + assert.Empty(t, reason) + + ok, reason = row.VisibleWithReason([]Predicate{{Column: "tenant", Op: "=", Values: []string{"beta"}}}) + assert.False(t, ok) + assert.Equal(t, ReasonFilter, reason) + + ok, reason = row.VisibleWithReason([]Predicate{{Column: "ratio", Op: "=", Values: []string{"abc"}}}) + assert.False(t, ok) + assert.Equal(t, ReasonError, reason, "a value the column's reader refuses throws on the row") + + ok, reason = row.VisibleWithReason([]Predicate{{Column: "id", Op: "=", Values: []string{"abc"}}}) + assert.False(t, ok) + assert.Equal(t, ReasonFilter, reason, "an integer claim that does not fit is answered false") + + ok, reason = row.VisibleWithReason([]Predicate{{Column: "nosuch", Op: "=", Values: []string{"x"}}}) + assert.False(t, ok) + assert.Equal(t, ReasonFilter, reason, "an unexpressible predicate matches nothing, like the SQL path's 1 = 0") + + // A filter that will not compile is the decline case; render never emits + // one, so it is reached through the compiler directly. + assert.Nil(t, tbl.filterOn(row.slot, "notAFunction(`tenant`) = {p0:String}", map[string]string{"p0": "x"})) +} + +// TestParseRow_ColumnsDriftIsAnError: a positional row is only interpretable +// against the generation that produced it. +func TestParseRow_ColumnsDriftIsAnError(t *testing.T) { + eng := TestEngine(t, rowsTable()) + tbl, err := eng.Table("rows") + require.NoError(t, err) + defer tbl.Release() + + _, err = tbl.ParseRow([]string{"id", "tenant"}, []byte(sampleRow)) + require.ErrorIs(t, err, ErrColumnsDrift) + + reordered := append([]string(nil), tbl.WireColumns...) + reordered[0], reordered[1] = reordered[1], reordered[0] + _, err = tbl.ParseRow(reordered, []byte(sampleRow)) + require.ErrorIs(t, err, ErrColumnsDrift, "same names in a different order is still drift") +} + +func TestParseRow_AcceptsALineWithOrWithoutNewline(t *testing.T) { + eng := TestEngine(t, rowsTable()) + tbl, err := eng.Table("rows") + require.NoError(t, err) + defer tbl.Release() + + for _, body := range []string{sampleRow, sampleRow + "\n"} { + row, err := tbl.ParseRow(tbl.WireColumns, []byte(body)) + require.NoError(t, err) + assert.True(t, row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}})) + row.Close() + } +} + +// TestFilterCache_CompilesOncePerPredicateSet: the cache is what keeps a +// per-subscriber fan-out from recompiling on every event. One Row stays on one +// handle, so the count is that slot's. +func TestFilterCache_CompilesOncePerPredicateSet(t *testing.T) { + _, row := parsedRow(t) + require.Equal(t, 0, row.slot.filters.len()) + + pred := []Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}} + for range 5 { + assert.True(t, row.Visible(pred)) + } + assert.Equal(t, 1, row.slot.filters.len()) + + // A different bound value is a different compiled handle: values are baked + // in at compile time. + assert.False(t, row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"beta"}}})) + assert.Equal(t, 2, row.slot.filters.len()) +} + +// TestFilterCache_NegativeEntryStopsRecompiling: a broken expression must cost +// one compile and one log line, not one per event. +func TestFilterCache_NegativeEntryStopsRecompiling(t *testing.T) { + tbl, row := parsedRow(t) + for range 3 { + assert.Nil(t, tbl.filterOn(row.slot, "notAFunction(`tenant`) = {p0:String}", map[string]string{"p0": "x"})) + } + assert.Equal(t, 1, row.slot.filters.len()) +} + +// TestFilterCache_BoundedUnderTenantValueChurn: the bound values come from +// tenant claims, so an unbounded cache is a denial of service. Eviction closes +// the handle it drops, against the real library. +func TestFilterCache_BoundedUnderTenantValueChurn(t *testing.T) { + _, row := parsedRow(t) + row.slot.filters = newFilterCache(8) + + for i := range 40 { + pred := []Predicate{{Column: "tenant", Op: "=", Values: []string{strconv.Itoa(i)}}} + assert.False(t, row.Visible(pred)) + } + assert.Equal(t, 8, row.slot.filters.len()) + + // The surviving handles still answer after their neighbours were closed. + assert.True(t, row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}})) +} + +// TestFilterCache_BudgetIsSplitAcrossTheHandlePool: the 4096 bound is what +// makes the cache not a DoS, and it must be a bound on the TABLE — a pool of +// handles must not multiply it. +func TestFilterCache_BudgetIsSplitAcrossTheHandlePool(t *testing.T) { + eng := TestEngine(t, rowsTable()) + tbl, err := eng.Table("rows") + require.NoError(t, err) + defer tbl.Release() + + total := 0 + for _, s := range tbl.slots { + total += s.filters.cap + } + assert.LessOrEqual(t, total, filterCacheSize) + assert.Equal(t, poolSize(), len(tbl.slots)) +} + +// TestFilterCache_GenerationInvalidatesEntries: a filter only answers for the +// schema handle it was compiled against, so a rebind must not reuse one. +func TestFilterCache_GenerationInvalidatesEntries(t *testing.T) { + eng := TestEngine(t, rowsTable()) + + visible := func() bool { + tbl, err := eng.Table("rows") + require.NoError(t, err) + defer tbl.Release() + row, err := tbl.ParseRow(tbl.WireColumns, []byte(sampleRow)) + require.NoError(t, err) + defer row.Close() + return row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}}) + } + assert.True(t, visible()) + + // Widen a column: the signature changes, so the handle and every filter + // over it are rebuilt, while the wire arity stays the same. + changed := rowsTable() + changed.Columns[0].Type = "UInt16" + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{changed}) + assert.True(t, visible(), "a rebind recompiles rather than reusing a freed handle") +} + +func (c *filterCache) len() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.order.Len() +} + +// TestVisible_ConcurrentSubscribers: the hub evaluates K filters against one +// parsed block; one LoadedSchema serializes them internally, so this must be +// safe rather than merely fast. +func TestVisible_ConcurrentSubscribers(t *testing.T) { + _, row := parsedRow(t) + + done := make(chan bool, 16) + for i := range 16 { + go func() { + done <- row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{fmt.Sprintf("t%d", i%4)}}}) + }() + } + for range 16 { + assert.False(t, <-done) + } +} + +// TestTable_ConcurrentAcrossThePool exercises every entry point on one table +// from many goroutines at once. Under -race it is the pin that the pool's +// round-robin, the per-slot filter caches and the shared block parsing are +// safe; a slot chosen per call rather than per Row would show up here as a +// filter and a block on different handles. +func TestTable_ConcurrentAcrossThePool(t *testing.T) { + eng := TestEngine(t, rowsTable()) + + const goroutines = 16 + var wg sync.WaitGroup + for g := range goroutines { + wg.Add(1) + go func() { + defer wg.Done() + for i := range 20 { + tbl, err := eng.Table("rows") + if !assert.NoError(t, err) { + return + } + + row, err := tbl.ParseRow(tbl.WireColumns, []byte(sampleRow)) + if assert.NoError(t, err) { + assert.True(t, row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{"acme"}}})) + assert.False(t, row.Visible([]Predicate{{Column: "tenant", Op: "=", Values: []string{fmt.Sprintf("t%d", (g+i)%4)}}})) + row.Close() + } + + record := []byte(`{"id":7,"tenant":"acme","ts":"2026-01-15 10:30:00","amt":"12.50","big":-5,"ratio":0.1,"tags":["a"]}` + "\n") + batch, err := tbl.Ingest(FormatJSONEachRow, record) + assert.NoError(t, err) + assert.Len(t, batch.Rows, 1) + + checked, err := tbl.Ingest(FormatJSONEachRow, record, + Predicate{Column: "tenant", Op: "=", Values: []string{"acme"}}) + if assert.NoError(t, err) && assert.Len(t, checked.Rows, 1) { + assert.Empty(t, checked.Rows[0].CheckReason) + assert.NotNil(t, checked.Rows[0].Line) + } + + tbl.Release() + } + }() + } + wg.Wait() +} diff --git a/internal/typelayer/ingest.go b/internal/typelayer/ingest.go new file mode 100644 index 00000000..ed1f531b --- /dev/null +++ b/internal/typelayer/ingest.go @@ -0,0 +1,326 @@ +package typelayer + +import ( + "bytes" + "fmt" + + "github.com/wave-rf/chtypes/go/chtypes" +) + +// InsertSettings are the parsing settings the ingest worker pins on the real +// INSERT, and which chtypes must therefore see — otherwise it answers a +// different question than the server will be asked. Returned fresh so a caller +// may add its own non-parsing pins (async_insert=0) without mutating ours. +func InsertSettings() map[string]string { + return map[string]string{ + "date_time_input_format": "best_effort", + "input_format_null_as_default": "1", + } +} + +// IngestOptions adjusts how IngestWith reads a body. The zero value is +// ClickHouse's own behaviour. +type IngestOptions struct { + // StrictPositional switches header auto-detection off for FormatCSV and + // FormatTSV (input_format_csv_detect_header / input_format_tsv_detect_header + // = 0), so every line is a record. By default ClickHouse consumes a first + // line that names the columns as a header (both settings are on by default; + // measured on the 26.6 server and artifact). Ignored for other formats. + StrictPositional bool +} + +// parseSettings is what a body is parsed under: the insert pins, plus header +// detection off when the caller asked for strict positional CSV/TSV. The worker +// inserts JSONCompactEachRow, so the detect_header settings have no real-INSERT +// twin to keep in step. ok is false for a format Ingest does not parse. +func parseSettings(format Format, opts IngestOptions) (settings map[string]string, ok bool) { + settings = InsertSettings() + switch format { + case FormatJSONEachRow, FormatCSVWithNames, FormatTSVWithNames: + case FormatCSV: + if opts.StrictPositional { + settings["input_format_csv_detect_header"] = "0" + } + case FormatTSV: + if opts.StrictPositional { + settings["input_format_tsv_detect_header"] = "0" + } + default: + return nil, false + } + return settings, true +} + +// RowVerdict is one input record's answer, index-aligned with the records the +// caller wrote into the body. +type RowVerdict struct { + Accepted bool + // Code is ClickHouse's own error code when the record was refused (27, 117, + // 6 …) and 0 otherwise. Declined verdicts carry no code: chtypes did not + // answer, so there is nothing to attribute to the data. + Code int + Message string + // Declined marks "the validation engine could not answer", never "the data + // is bad". A caller must not turn it into a 400. + Declined bool + // CheckReason is why the insert checks given to Ingest did not admit an + // accepted record: ReasonFilter (the data says no), ReasonError or + // ReasonDecline (we could not tell). "" when they admitted it or there were + // none. A record with a CheckReason has no Line — only the rows the filter + // admits are exported — and Message may carry the predicate's own error. + CheckReason string + // Line is the record as ClickHouse's own JSONCompactEachRow writer + // serialized it, without the trailing newline. nil unless Accepted with no + // CheckReason. It is a sub-slice of the batch's exported bytes, so a caller + // that outlives the request must copy it. + Line []byte +} + +// Batch holds one verdict per input record, in input order. +type Batch struct { + // Rows holds one verdict per record chtypes read, in input order. When + // Answered is false chtypes gave no per-record detail (the whole batch was + // declined) and Rows is padded to the body's line count so a caller still + // has something index-shaped to report. + Rows []RowVerdict + Answered bool + // Refused is ClickHouse's own refusal of a WithNames body as a whole — a + // header naming a column the schema does not have, or naming one twice — + // before any record was read. Rows is then empty; nil otherwise. + Refused *Refusal +} + +// Refusal is ClickHouse's verdict on a body rather than on any one record. +type Refusal struct { + Code int + Message string +} + +// Ingest asks ClickHouse's own parser whether each record in body would +// insert, in ONE call, and exports the accepted rows as JSONCompactEachRow. +// +// format is how body is spelled: +// +// - FormatJSONEachRow: newline-separated and name-addressed (an NDJSON body +// is byte-identical to this); +// - FormatCSV, FormatTSV: positional in declaration order. ClickHouse's +// header auto-detection applies (a first line naming the columns is +// consumed, not a record) unless IngestWith sets StrictPositional, where a +// header line is one failed record; +// - FormatCSVWithNames, FormatTSVWithNames: a first line naming the columns +// in any order. It is not a record, so Rows index the data lines; a column +// it omits takes its DEFAULT, and one it names that the schema lacks (or +// names twice) refuses the whole body (Batch.Refused, code 117). +// +// Any other format is a programming error and returns an error without +// touching the data — a binding must never declare a format it has not asked +// the artifact about. +// +// checks are a role's insert check clauses. They compile to ONE filter, +// AND-joined, attached to the same parse (RowsExportWith, docs/guides/ +// filters.md "Exporting only the rows a filter admits"), so each record's +// parse outcome and check answer come from one read of the body and only the +// admitted records are exported. The parse outcome decides first: a record +// chtypes did not accept reports its own error whatever the filter says (it +// answers such a row 'd'). A predicate with no Values matches nothing and +// never reaches the compiler; a filter that will not compile declines every +// accepted record. The compiled filter is cached per (generation, expression, +// values) on the handle it runs against. +// +// Document flags stay lean (verdicts and exported bytes only). The per-value +// provenance DocValues would give costs 2.65× on this path and nothing here +// reads it. +// +// The returned error is for a Go-level failure only — every data verdict is in +// the batch. +func (t *Table) Ingest(format Format, body []byte, checks ...Predicate) (Batch, error) { + return t.IngestWith(format, IngestOptions{}, body, checks...) +} + +// IngestWith is Ingest with options; see IngestOptions. +func (t *Table) IngestWith(format Format, opts IngestOptions, body []byte, checks ...Predicate) (Batch, error) { + settings, ok := parseSettings(format, opts) + if !ok { + return Batch{}, fmt.Errorf( + "typelayer: Ingest cannot parse format %d; use FormatJSONEachRow, FormatCSV, FormatTSV, FormatCSVWithNames or FormatTSVWithNames", + int(format)) + } + if len(t.slots) == 0 { + return Batch{}, &Unavailable{Table: t.Name, Cause: t.cause} + } + // The filter must be compiled on the handle the parse runs on: one from + // another handle rejects the whole call. + s := t.slot() + filter, uniform := t.checkFilter(s, checks) + res, err := export(s, format, body, settings, filter) + if err != nil && filter != nil { + // The cached filter was evicted and closed between lookup and use. Fail + // the checks closed, as an evaluation error would, not the request. + filter, uniform = nil, ReasonDecline + res, err = export(s, format, body, settings, nil) + } + if err != nil { + return Batch{}, err + } + + // Only a fully accepted batch exports bytes. Gate on the outcome, never on + // RowsPassed, which counts the admitted rows of a rejected batch too. + if res.Outcome != chtypes.Accepted { + if len(res.Rows) == 0 && res.Outcome == chtypes.Rejected && res.ErrCode != 0 && + (format == FormatCSVWithNames || format == FormatTSVWithNames) { + return Batch{Answered: true, Refused: &Refusal{Code: res.ErrCode, Message: res.ErrMsg}}, nil + } + return declineAll(countRecords(body, len(res.Rows)), firstNonEmpty(res.ExportDeclined, res.ErrMsg, res.Outcome.String())), nil + } + // Accepted but withheld (the full-arity guard, a serialization failure): + // nothing can be forwarded, whatever the per-row detail says. + if res.ExportDeclined != "" { + return declineAll(countRecords(body, len(res.Rows)), res.ExportDeclined), nil + } + + // chtypes answered per record, so its count is the record count: the + // verdicts are index-aligned with the records it read, and padding to the + // body's newline count would invent declined records out of blank lines + // and pretty-printed framing. + out := Batch{Rows: make([]RowVerdict, len(res.Rows)), Answered: true} + for i := range out.Rows { + v := rowVerdict(res.Rows[i], span(res, i), filter != nil) + if v.Accepted && uniform != "" { + v.Line, v.CheckReason = nil, uniform + } + out.Rows[i] = v + } + return out, nil +} + +// checkFilter resolves the check clauses to the filter attached to the parse, +// or to the one answer every accepted record gets when no filter runs: +// ReasonFilter for a predicate render cannot express (it matches nothing, like +// the SQL path's `1 = 0`), ReasonDecline for one that will not compile. +func (t *Table) checkFilter(s *schemaSlot, checks []Predicate) (*chtypes.LoadedFilter, string) { + if len(checks) == 0 { + return nil, "" + } + expr, params, ok := t.render(checks) + if !ok { + return nil, ReasonFilter + } + if f := t.filterOn(s, expr, params); f != nil { + return f, "" + } + return nil, ReasonDecline +} + +// export is the one parse. With no filter RowsExportWith is RowsExport. +func export(s *schemaSlot, format Format, body []byte, settings map[string]string, f *chtypes.LoadedFilter) (chtypes.BatchResult, error) { + if f == nil { + return s.schema.RowsExportWith(format, body, settings, chtypes.JSONCompactEachRow) + } + return s.schema.RowsExportWith(format, body, settings, chtypes.JSONCompactEachRow, chtypes.WithRowFilter(f)) +} + +// rowVerdict maps one chtypes RowResult. An unsupported setting is the engine +// declining even when the row itself parsed, so it is checked before the +// outcome, and the outcome before the filter's verdict. +func rowVerdict(r chtypes.RowResult, line []byte, filtered bool) RowVerdict { + if len(r.UnsupportedSettings) > 0 { + return RowVerdict{Declined: true, Message: "chtypes does not support setting(s) " + joinQuoted(r.UnsupportedSettings)} + } + switch r.Outcome { + case chtypes.Accepted: + if filtered { + // A nil verdict is a row the filter never answered: it must not pass. + if r.Verdict == nil { + return RowVerdict{Accepted: true, CheckReason: ReasonDecline} + } + if ok, reason := verdictBool(*r.Verdict); !ok { + return RowVerdict{Accepted: true, CheckReason: reason, Message: r.VerdictErr} + } + } + if line == nil { + return RowVerdict{Declined: true, Message: "accepted but no bytes were exported for this row"} + } + return RowVerdict{Accepted: true, Line: line} + case chtypes.Skipped, chtypes.Rejected: + // ErrCode/ErrMsg, not VerdictCode/VerdictErr: chtypes answers such a row + // 'd', and on 26.6 leaves the verdict's own code and message empty. + return RowVerdict{Code: r.ErrCode, Message: r.ErrMsg} + case chtypes.Unsupported, chtypes.AcceptedPoisoned: + // AcceptedPoisoned holds a value no writer can honestly serialize, so + // like Unsupported it yields no bytes and is not a data verdict. + return declinedVerdict(r) + default: + return declinedVerdict(r) + } +} + +func firstNonEmpty(s ...string) string { + for _, v := range s { + if v != "" { + return v + } + } + return "" +} + +// declinedVerdict reports an outcome that is not a verdict about the data, +// keeping the engine's own wording when it supplied any. +func declinedVerdict(r chtypes.RowResult) RowVerdict { + msg := r.ErrMsg + if msg == "" { + msg = r.Outcome.String() + } + return RowVerdict{Declined: true, Message: msg} +} + +// span slices row i's exported line out of the payload, dropping the trailing +// newline the writer emits. Returns nil when the row contributed no bytes. +func span(res chtypes.BatchResult, i int) []byte { + if i >= len(res.Spans) { + return nil + } + s := res.Spans[i] + if s.Len <= 0 || s.Off < 0 || s.Off+s.Len > len(res.Payload) { + return nil + } + return bytes.TrimSuffix(res.Payload[s.Off:s.Off+s.Len], []byte("\n")) +} + +func declineAll(n int, msg string) Batch { + b := Batch{Rows: make([]RowVerdict, n)} + for i := range b.Rows { + b.Rows[i] = RowVerdict{Declined: true, Message: msg} + } + return b +} + +// countRecords recovers the input record count when chtypes returned no +// per-row detail, so the caller still gets an index-aligned answer. JSONEachRow +// records are newline-separated and json.Marshal escapes any newline inside a +// value, so counting lines is exact for the bodies this package is handed. A +// CSV field may legally contain a raw newline, and a WithNames header is a +// line but not a record, so for those formats the fallback can OVER-count, +// which produces extra declined verdicts — never an extra acceptance. +func countRecords(body []byte, known int) int { + if known > 0 { + return known + } + n := bytes.Count(body, []byte("\n")) + if len(body) > 0 && !bytes.HasSuffix(body, []byte("\n")) { + n++ + } + return n +} + +func joinQuoted(names []string) string { + var b bytes.Buffer + for i, n := range names { + if i > 0 { + b.WriteString(", ") + } + b.WriteByte('"') + b.WriteString(n) + b.WriteByte('"') + } + return b.String() +} diff --git a/internal/typelayer/ingest_test.go b/internal/typelayer/ingest_test.go new file mode 100644 index 00000000..a043ada0 --- /dev/null +++ b/internal/typelayer/ingest_test.go @@ -0,0 +1,211 @@ +package typelayer + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +func ingestTable(t *testing.T) *Table { + t.Helper() + eng := TestEngine(t, eventsTable()) + tbl, err := eng.Table("events") + require.NoError(t, err) + t.Cleanup(tbl.Release) + return tbl +} + +// TestIngest_PerRecordVerdicts pins the measured behaviour of the whole +// compile profile at once: a bad value is one record's rejection rather than +// the batch's, the codes are ClickHouse's own, and the accepted records come +// back as the bytes the server itself would store. +func TestIngest_PerRecordVerdicts(t *testing.T) { + tbl := ingestTable(t) + + body := strings.Join([]string{ + `{"id":1,"name":"a","ts":"2026-01-15 10:30:00.000","tags":["x"],"score":null}`, + `{"id":"bad","name":"b","ts":"2026-01-15 10:30:00.000","tags":[],"score":1}`, + `{"id":3,"name":"c","ts":"2026-01-15 10:30:00.000","tags":[],"score":null,"extra":1}`, + `{"id":4,"name":"d","ts":"2026-01-15 10:30:00.000","tags":[],"score":null}`, + }, "\n") + "\n" + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(body)) + require.NoError(t, err) + require.Len(t, batch.Rows, 4, "one verdict per input record, index-aligned") + + assert.True(t, batch.Rows[0].Accepted) + assert.Equal(t, 0, batch.Rows[0].Code) + + // An unparseable value: ClickHouse's own CANNOT_PARSE_TEXT. + assert.False(t, batch.Rows[1].Accepted) + assert.False(t, batch.Rows[1].Declined) + assert.Equal(t, 27, batch.Rows[1].Code) + assert.NotEmpty(t, batch.Rows[1].Message) + + // input_format_skip_unknown_fields=0 turns an unknown field into a real + // per-row rejection instead of silent data loss. + assert.False(t, batch.Rows[2].Accepted) + assert.Equal(t, 117, batch.Rows[2].Code) + assert.Contains(t, batch.Rows[2].Message, "extra") + + assert.True(t, batch.Rows[3].Accepted) +} + +// TestIngest_AcceptedLineIsOneWireRow: the published line must be exactly one +// JSONCompactEachRow row with no trailing newline, with the MATERIALIZED column +// absent and the volatile DEFAULT already baked in. +func TestIngest_AcceptedLineIsOneWireRow(t *testing.T) { + tbl := ingestTable(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":7,"name":"a","ts":"2026-01-15 10:30:00.000","tags":["x"],"score":null}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + require.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) + + line := string(batch.Rows[0].Line) + assert.NotContains(t, line, "\n") + assert.True(t, strings.HasPrefix(line, "["), line) + assert.True(t, strings.HasSuffix(line, "]"), line) + assert.Equal(t, len(tbl.WireColumns), strings.Count(line, ",")+1, "one cell per wire column: %s", line) + // now() was substituted at parse time, so the worker inserts the bytes + // verbatim and the server never re-evaluates the clock. + assert.NotContains(t, line, "now()") +} + +// TestIngest_OverflowIsStoredTruth, not the producer's spelling: 256 into a +// UInt8 wraps to 0, which is what the table will hold and therefore what the +// stream must show (#372's payload-vs-stored asymmetry, closed by construction). +func TestIngest_OverflowIsStoredTruth(t *testing.T) { + tbl := ingestTable(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":256,"name":"a","ts":"2026-01-15 10:30:00.000","tags":[],"score":null}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + require.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) + assert.True(t, strings.HasPrefix(string(batch.Rows[0].Line), "[0,"), string(batch.Rows[0].Line)) +} + +// TestIngest_ComputedColumnsAreRejectedPerRecord: a record naming a +// MATERIALIZED, ALIAS or EPHEMERAL column gets ClickHouse's own 117 and the +// rest of the batch still gets verdicts — no WaveHouse-side guard needed. +func TestIngest_ComputedColumnsAreRejectedPerRecord(t *testing.T) { + schema := &discovery.TableSchema{ + Name: "computed", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt32", Position: 1}, + {Name: "e", Type: "UInt8", DefaultKind: "EPHEMERAL", HasDefault: true, Position: 2}, + {Name: "d", Type: "UInt8", DefaultKind: "DEFAULT", DefaultExpression: "e + 1", HasDefault: true, Position: 3}, + {Name: "a", Type: "UInt8", DefaultKind: "ALIAS", DefaultExpression: "id + 2", HasDefault: true, Position: 4}, + }, + } + eng := TestEngine(t, schema) + tbl, err := eng.Table("computed") + require.NoError(t, err) + defer tbl.Release() + + assert.Equal(t, []string{"id", "d"}, tbl.WireColumns) + + body := strings.Join([]string{ + `{"id":1}`, + `{"id":2,"e":5}`, + `{"id":3,"a":9}`, + `{"id":4,"d":7}`, + }, "\n") + "\n" + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(body)) + require.NoError(t, err) + require.Len(t, batch.Rows, 4) + + assert.True(t, batch.Rows[0].Accepted) + assert.Equal(t, 117, batch.Rows[1].Code) + assert.Contains(t, batch.Rows[1].Message, "e") + assert.Equal(t, 117, batch.Rows[2].Code) + assert.Contains(t, batch.Rows[2].Message, "a") + assert.True(t, batch.Rows[3].Accepted) + // ClickHouse's writer separates cells with ", " — the worker inserts these + // bytes verbatim, so nothing may re-render them. + assert.Equal(t, "[4, 7]", string(batch.Rows[3].Line)) +} + +// TestInsertSettings_AreAllSupported: every setting typelayer passes must be one +// chtypes understands. An unsupported one turns a real verdict into a decline, +// which is a silent availability regression on the ingest path. +func TestInsertSettings_AreAllSupported(t *testing.T) { + tbl := ingestTable(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":1,"name":"a","ts":"2026-01-15 10:30:00.000","tags":[],"score":null}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + require.False(t, batch.Rows[0].Declined, "InsertSettings must not contain a setting chtypes rejects: %s", batch.Rows[0].Message) + assert.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) +} + +// TestInsertSettings_NullAsDefaultApplies: the worker pins it on the real +// INSERT, so chtypes has to see the same rule — an explicit null in a +// non-nullable column becomes the type's default rather than a rejection. +func TestInsertSettings_NullAsDefaultApplies(t *testing.T) { + tbl := ingestTable(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":null,"name":"a","ts":"2026-01-15 10:30:00.000","tags":[],"score":null}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + assert.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) + assert.True(t, strings.HasPrefix(string(batch.Rows[0].Line), "[0,"), string(batch.Rows[0].Line)) +} + +// TestInsertSettings_BestEffortDateTime: without the worker's own +// date_time_input_format pin, chtypes would reject an RFC3339 value the real +// server accepts — a manufactured over-reject. +func TestInsertSettings_BestEffortDateTime(t *testing.T) { + tbl := ingestTable(t) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":1,"name":"a","ts":"2026-01-15T10:30:00Z","tags":[],"score":null}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + assert.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) +} + +func TestInsertSettings_ReturnsAFreshMap(t *testing.T) { + t.Parallel() + a := InsertSettings() + a["async_insert"] = "0" + assert.NotContains(t, InsertSettings(), "async_insert") + assert.Equal(t, map[string]string{ + "date_time_input_format": "best_effort", + "input_format_null_as_default": "1", + }, InsertSettings()) +} + +// TestIngest_CountIsChtypesOwn: blank lines and pretty-printed framing are +// not records. chtypes' per-record answer is the record count; nothing is +// padded on top of it. +func TestIngest_CountIsChtypesOwn(t *testing.T) { + tbl := ingestTable(t) + + ndjson := "{\"id\":1,\"name\":\"a\",\"ts\":\"2026-01-15 10:30:00.000\",\"tags\":[],\"score\":null}\n\n" + + "{\"id\":2,\"name\":\"b\",\"ts\":\"2026-01-15 10:30:00.000\",\"tags\":[],\"score\":null}\n" + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(ndjson)) + require.NoError(t, err) + assert.True(t, batch.Answered) + require.Len(t, batch.Rows, 2, "a blank line is not a record") + assert.True(t, batch.Rows[0].Accepted) + assert.True(t, batch.Rows[1].Accepted) + + pretty := "{\n \"id\": 3,\n \"name\": \"c\",\n \"ts\": \"2026-01-15 10:30:00.000\",\n \"tags\": [],\n \"score\": null\n}\n" + batch, err = tbl.Ingest(FormatJSONEachRow, []byte(pretty)) + require.NoError(t, err) + require.Len(t, batch.Rows, 1, "a pretty-printed object is one record, not one per line") + assert.True(t, batch.Rows[0].Accepted) +} + +func TestCountRecords(t *testing.T) { + t.Parallel() + assert.Equal(t, 0, countRecords(nil, 0)) + assert.Equal(t, 1, countRecords([]byte(`{"a":1}`), 0)) + assert.Equal(t, 2, countRecords([]byte("{}\n{}\n"), 0)) + assert.Equal(t, 2, countRecords([]byte("{}\n{}"), 0)) + assert.Equal(t, 5, countRecords([]byte("{}\n{}"), 5), "chtypes' own count wins when it has one") +} diff --git a/internal/typelayer/roletable.go b/internal/typelayer/roletable.go new file mode 100644 index 00000000..496acc34 --- /dev/null +++ b/internal/typelayer/roletable.go @@ -0,0 +1,301 @@ +package typelayer + +import ( + "container/list" + "crypto/sha256" + "fmt" + "maps" + "slices" + "strings" + "sync" + + "github.com/wave-rf/chtypes/go/chtypes" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// roleCacheSize bounds the per-role shapes held per table. A shape's Defaults +// values come from tenant claims and are baked into the compiled handle, so an +// unbounded cache is a memory and CPU denial of service — the same reason +// filterCache is bounded. Each entry costs roleHandles (1) compiled handle. +const roleCacheSize = 256 + +// RoleShape is the projection of a table a role may insert through. It is the +// whole input to Engine.RoleTable and the whole cache key, so two roles with +// the same shape share one compiled handle. +// +// Columns is the set of columns the role may write. nil means "every column" +// and is the identity shape; a non-nil, EMPTY slice means "no column", which +// compiles to nothing and fails closed. Order is irrelevant — the DDL always +// follows the table's own declaration order — and a name the table does not +// have is ignored. Columns the role cannot supply anyway (MATERIALIZED, ALIAS, +// EPHEMERAL) are always kept: dropping one would change what the server +// computes, and a MATERIALIZED expression over a dropped column would not +// compile at all. +// +// Defaults maps a column to a literal value injected when a record omits it, +// rendered into the column's DEFAULT clause. A value the record DOES supply +// still wins (measured, AUDIT §A.3). Every key must name a column the shape +// keeps and must be an ordinary column (no DEFAULT, or a plain DEFAULT) — +// anything else is a programming error and returns an error rather than +// silently reshaping the table. +type RoleShape struct { + Columns []string + Defaults map[string]string +} + +// identity reports whether the shape asks for nothing the base table does not +// already answer, in which case no second handle is compiled. +func (s RoleShape) identity() bool { + return s.Columns == nil && len(s.Defaults) == 0 +} + +// key is the cache key: the generation that owns the schema plus a canonical +// hash of the shape. Every component is length-prefixed, so no column name or +// claim value can spell another shape's encoding. +func (s RoleShape) key(generation uint64) string { + var b strings.Builder + fmt.Fprintf(&b, "g%d\x1e", generation) + if s.Columns == nil { + b.WriteString("*\x1e") + } else { + cols := slices.Clone(s.Columns) + slices.Sort(cols) + cols = slices.Compact(cols) + for _, c := range cols { + fmt.Fprintf(&b, "%d:%s", len(c), c) + } + b.WriteByte(0x1e) + } + for _, k := range slices.Sorted(maps.Keys(s.Defaults)) { + v := s.Defaults[k] + fmt.Fprintf(&b, "%d:%s%d:%s", len(k), k, len(v), v) + } + // Hashed rather than kept verbatim: the Defaults values are tenant claims, + // so the key's size must not be the caller's to choose. + sum := sha256.Sum256([]byte(b.String())) + return string(sum[:]) +} + +// RoleTable returns the compiled handle for one role's projection of a table, +// read-locked exactly like Engine.Table: the caller must Release it, and the +// handle stays alive and stable until it does. +// +// The shape is answered by ClickHouse's own parser rather than by a Go walk +// over the record's keys (AUDIT §A.1): a column the role may not write is +// simply absent from the compiled DDL, so a record naming it is refused +// per-row with ClickHouse's own code 117 "Unknown field found while parsing +// JSONEachRow format: x"; a Defaults column is declared DEFAULT '', +// quoted by the library's own QuoteLiteral, so an absent value is filled and a +// supplied one still wins. +// +// The identity shape (no column restriction, no defaults) returns the base +// table itself — no second handle, no cache entry. +// +// A shape that does not compile is cached as a negative entry and reported as +// *Unavailable, so a broken policy costs one compile and one log line per +// generation rather than one per request. +func (e *Engine) RoleTable(table string, shape RoleShape) (*Table, error) { + base, err := e.Table(table) + if err != nil { + return nil, err + } + if shape.identity() { + return base, nil // still read-locked; the caller's Release covers it + } + rt, evicted, err := base.roleTable(shape) + base.Release() + // Closed after the cache lock and the base read lock are both gone: it + // waits for the evicted shape's own readers, which are requests in flight. + if evicted != nil { + evicted.close() + } + if err != nil { + return nil, err + } + return rt, nil +} + +// roleTable is RoleTable's cache half, run under the base table's read lock. +// It returns the projection read-locked, plus the entry its insertion evicted +// (to be closed by the caller, outside the cache lock). +func (t *Table) roleTable(shape RoleShape) (*Table, *Table, error) { + key := shape.key(t.Generation) + c := t.roles + c.mu.Lock() + defer c.mu.Unlock() + + if el, hit := c.index[key]; hit { + c.order.MoveToFront(el) + e := el.Value.(*roleEntry) + if e.table == nil { + return nil, nil, &Unavailable{Table: t.Name, Cause: e.cause} + } + // Taken while the base read lock is still held, so a rebind cannot be + // closing this projection underneath us. + e.table.mu.RLock() + return e.table, nil, nil + } + + rt, cause := t.compileRole(shape) + if cause != "" { + t.log.Error("chtypes could not compile a per-role schema; every insert for this role fails closed", + "table", t.Name, "generation", t.Generation, + "allowed_columns", shape.Columns, "default_columns", slices.Sorted(maps.Keys(shape.Defaults)), + "cause", cause) + } + el := c.order.PushFront(&roleEntry{key: key, table: rt, cause: cause}) + c.index[key] = el + var evicted *Table + if c.order.Len() > c.cap { + evicted = c.evictOldestLocked() + } + if rt == nil { + return nil, evicted, &Unavailable{Table: t.Name, Cause: cause} + } + rt.mu.RLock() + return rt, evicted, nil +} + +// compileRole builds and compiles the role's declaration list. It returns +// (nil, cause) for every refusal. +func (t *Table) compileRole(shape RoleShape) (*Table, string) { + cols, wire, err := roleColumns(t.lib, t.discovered, shape) + if err != nil { + return nil, err.Error() + } + ddl, rerr := t.lib.ReconstructDDL(cols) + if rerr != nil { + return nil, "cannot reconstruct role column declarations: " + rerr.Error() + } + slots, cause := compileDDL(t.lib, ddl, roleHandles) + if cause != "" { + return nil, cause + } + declared, cause := declaredColumns(t.lib, slots[0].schema, nil) + if cause != "" { + closeSlots(slots) + return nil, cause + } + + return &Table{ + Name: t.Name, + Generation: t.Generation, + WireColumns: deriveWireColumns(slots[0].schema, wire), + log: t.log, + slots: slots, + cols: declared, + lib: t.lib, + }, "" +} + +// roleColumns projects the table's discovered columns onto a shape. It returns +// the declaration list and the wire column names (the declaration list minus +// the three kinds a positional INSERT never carries). An injected value is +// quoted by lib.QuoteLiteral, ClickHouse's own quoteString, so it reaches the +// compiler as one string literal whatever bytes it holds. +func roleColumns(lib *chtypes.Library, src []discovery.Column, shape RoleShape) ([]chtypes.DiscoveredColumn, []string, error) { + var allowed map[string]struct{} + if shape.Columns != nil { + allowed = make(map[string]struct{}, len(shape.Columns)) + for _, c := range shape.Columns { + allowed[c] = struct{}{} + } + } + + cols := make([]chtypes.DiscoveredColumn, 0, len(src)) + wire := make([]string, 0, len(src)) + kept := make(map[string]discovery.Column, len(src)) + for _, c := range src { + computed := c.DefaultKind == "MATERIALIZED" || c.DefaultKind == "ALIAS" || c.DefaultKind == "EPHEMERAL" + if allowed != nil && !computed { + if _, ok := allowed[c.Name]; !ok { + continue + } + } + dc := chtypes.DiscoveredColumn{ + Name: c.Name, + Type: c.Type, + DefaultKind: c.DefaultKind, + DefaultExpression: c.DefaultExpression, + Position: c.Position, + } + if v, inject := shape.Defaults[c.Name]; inject { + if computed { + return nil, nil, fmt.Errorf( + "cannot inject a default into column %q: it is %s", c.Name, c.DefaultKind) + } + lit, err := lib.QuoteLiteral(v) + if err != nil { + return nil, nil, fmt.Errorf("cannot quote the default for column %q: %w", c.Name, err) + } + dc.DefaultKind, dc.DefaultExpression = "DEFAULT", lit + } + cols = append(cols, dc) + if !computed { + wire = append(wire, c.Name) + } + kept[c.Name] = c + } + + // A default for a column this shape does not carry cannot be expressed, + // and silently dropping it would turn "force this value" into "whatever + // the caller sent". Fail loudly instead. + for _, name := range slices.Sorted(maps.Keys(shape.Defaults)) { + if _, ok := kept[name]; !ok { + return nil, nil, fmt.Errorf( + "cannot inject a default into column %q: the role's schema does not carry it", name) + } + } + return cols, wire, nil +} + +type roleEntry struct { + key string + table *Table // nil: this shape does not compile + cause string +} + +// roleCache is an LRU of per-role projections, with the same discipline as +// filterCache: bounded, negative entries cached, and every evicted handle +// closed. +type roleCache struct { + mu sync.Mutex + cap int + order *list.List // front = most recently used + index map[string]*list.Element +} + +func newRoleCache(capacity int) *roleCache { + return &roleCache{cap: capacity, order: list.New(), index: make(map[string]*list.Element)} +} + +// evictOldestLocked drops the least recently used entry and returns the handle +// the caller must close. Closing is the caller's job because it waits for that +// projection's readers, and waiting under the cache lock would stall every +// other role on the table. +func (c *roleCache) evictOldestLocked() *Table { + el := c.order.Back() + if el == nil { + return nil + } + c.order.Remove(el) + e := el.Value.(*roleEntry) + delete(c.index, e.key) + return e.table +} + +// closeAll drops every projection. Called with the base table's write lock +// held, so no new lookup can be in flight; each projection's own write lock +// waits for the requests already holding it. +func (c *roleCache) closeAll() { + c.mu.Lock() + defer c.mu.Unlock() + for el := c.order.Front(); el != nil; el = el.Next() { + if rt := el.Value.(*roleEntry).table; rt != nil { + rt.close() + } + } + c.order.Init() + c.index = make(map[string]*list.Element) +} diff --git a/internal/typelayer/roletable_test.go b/internal/typelayer/roletable_test.go new file mode 100644 index 00000000..88bbb44e --- /dev/null +++ b/internal/typelayer/roletable_test.go @@ -0,0 +1,299 @@ +package typelayer + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// ordersTable is the per-role fixture: an ordinary column a role may be denied +// (secret), one a check clause injects into (tenant), and a numeric column +// whose reader refuses a text literal (amount). +func ordersTable() *discovery.TableSchema { + return &discovery.TableSchema{ + Name: "orders", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt32", Position: 1}, + {Name: "tenant", Type: "String", Position: 2}, + {Name: "secret", Type: "String", Position: 3}, + {Name: "amount", Type: "UInt64", Position: 4}, + }, + } +} + +func roleTableFor(t *testing.T, eng *Engine, shape RoleShape) *Table { + t.Helper() + tbl, err := eng.RoleTable("orders", shape) + require.NoError(t, err) + t.Cleanup(tbl.Release) + return tbl +} + +// TestRoleTable_IdentityShapeIsTheBaseTable: a role that may write everything +// and injects nothing costs no second compile and no cache entry. +func TestRoleTable_IdentityShapeIsTheBaseTable(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + base, err := eng.Table("orders") + require.NoError(t, err) + base.Release() + + role, err := eng.RoleTable("orders", RoleShape{}) + require.NoError(t, err) + defer role.Release() + + assert.Same(t, base, role) + assert.Equal(t, 0, base.roles.len()) +} + +// TestRoleTable_DeniedColumnIsAbsentFromTheSchema is the §A.1 contract: a +// column the role may not write is simply not in the compiled DDL, so a record +// naming it is ClickHouse's own per-row code 117 rather than a Go key walk's +// 403 — and the exported row carries the ROLE's column list. +func TestRoleTable_DeniedColumnIsAbsentFromTheSchema(t *testing.T) { + eng := TestEngine(t, ordersTable()) + tbl := roleTableFor(t, eng, RoleShape{Columns: []string{"id", "tenant", "amount"}}) + + assert.Equal(t, []string{"id", "tenant", "amount"}, tbl.WireColumns) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte( + `{"id":1,"tenant":"acme","amount":5}`+"\n"+ + `{"id":2,"tenant":"acme","secret":"x","amount":5}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 2) + + assert.True(t, batch.Rows[0].Accepted) + assert.Equal(t, `[1, "acme", 5]`, string(batch.Rows[0].Line)) + + assert.False(t, batch.Rows[1].Accepted) + assert.False(t, batch.Rows[1].Declined, "a denied column is a verdict about the data, not a decline") + assert.Equal(t, 117, batch.Rows[1].Code) + assert.Contains(t, batch.Rows[1].Message, "secret") +} + +// TestRoleTable_DefaultInjectsWhenAbsentAndLosesToASuppliedValue is the §A.3 +// contract, measured: DEFAULT '' fills a column the record omits, and a +// value the record DOES supply still wins. +func TestRoleTable_DefaultInjectsWhenAbsentAndLosesToASuppliedValue(t *testing.T) { + eng := TestEngine(t, ordersTable()) + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"tenant": "acme"}}) + + assert.Equal(t, []string{"id", "tenant", "secret", "amount"}, tbl.WireColumns) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte( + `{"id":1,"amount":5}`+"\n"+ + `{"id":2,"tenant":"other","amount":5}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 2) + + require.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) + assert.Equal(t, `[1, "acme", "", 5]`, string(batch.Rows[0].Line)) + require.True(t, batch.Rows[1].Accepted, batch.Rows[1].Message) + assert.Equal(t, `[2, "other", "", 5]`, string(batch.Rows[1].Line)) +} + +// TestRoleTable_LiteralEscaping: the injected value is the one thing in this +// DDL that is SQL TEXT, spelled by the library's QuoteLiteral, so every +// spelling that could close the literal early has to survive as data. The +// breakout attempt is the case that matters — it must not add, remove or +// retype a single column. +func TestRoleTable_LiteralEscaping(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + for name, value := range map[string]string{ + "apostrophe": "O'Brien", + "backslash": `back\slash`, + "backslash quote": `x\'y`, + "trailing escape": `ends with \`, + "ddl breakout": `', x UInt8 DEFAULT '`, + "comment breakout": `' --`, + "empty": "", + "nul": "a\x00b", + "newline and tab": "a\nb\tc", + "unicode": "Ünï 日本", + } { + t.Run(name, func(t *testing.T) { + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"tenant": value}}) + + // An escape that closed the literal early would show up here, as an + // extra or missing column, not as a mangled value. + assert.Equal(t, []string{"id", "tenant", "secret", "amount"}, tbl.WireColumns) + assert.Equal(t, []string{"id", "tenant", "secret", "amount"}, + compiledColumnNames(tbl.slots[0].schema)) + + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":1,"amount":5}`+"\n")) + require.NoError(t, err) + require.Len(t, batch.Rows, 1) + require.True(t, batch.Rows[0].Accepted, batch.Rows[0].Message) + + // The exported line is ClickHouse's own writer on the stored value, + // so it is the ground truth for what the DEFAULT actually holds. + want, err := json.Marshal(value) + require.NoError(t, err) + assert.Equal(t, `[1, `+string(want)+`, "", 5]`, string(batch.Rows[0].Line)) + }) + } +} + +// TestRoleTable_UnparseableLiteralFailsClosed: a literal the column's reader +// cannot read is a compile refusal (ClickHouse code 6). It must be Unavailable +// — a 503 — never a handle that silently drops the injection. +func TestRoleTable_UnparseableLiteralFailsClosed(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + _, err := eng.RoleTable("orders", RoleShape{Defaults: map[string]string{"amount": "abc"}}) + require.Error(t, err) + assert.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), "orders") +} + +// TestRoleTable_ContradictoryShapeIsAnError: a default for a column the shape +// does not carry cannot be expressed. Dropping it silently would turn "force +// this value" into "whatever the caller sent". +func TestRoleTable_ContradictoryShapeIsAnError(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + _, err := eng.RoleTable("orders", RoleShape{ + Columns: []string{"id", "amount"}, + Defaults: map[string]string{"tenant": "acme"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "tenant") + + _, err = eng.RoleTable("orders", RoleShape{Defaults: map[string]string{"nosuch": "acme"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "nosuch") +} + +// TestRoleTable_DefaultIntoAComputedColumnIsRefused: MATERIALIZED/ALIAS/ +// EPHEMERAL are never supplied by a record, and rewriting one into a plain +// DEFAULT would change what the server stores. +func TestRoleTable_DefaultIntoAComputedColumnIsRefused(t *testing.T) { + ts := ordersTable() + ts.Columns = append(ts.Columns, discovery.Column{ + Name: "double", Type: "UInt64", HasDefault: true, + DefaultKind: "MATERIALIZED", DefaultExpression: "amount * 2", Position: 5, + }) + eng := TestEngine(t, ts) + + _, err := eng.RoleTable("orders", RoleShape{Defaults: map[string]string{"double": "1"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "MATERIALIZED") + + // A computed column is kept whatever the allow-list says: dropping it would + // change what the server computes. + tbl := roleTableFor(t, eng, RoleShape{Columns: []string{"id", "amount"}}) + assert.Equal(t, []string{"id", "amount"}, tbl.WireColumns) + assert.Equal(t, []string{"id", "amount", "double"}, compiledColumnNames(tbl.slots[0].schema)) +} + +// TestRoleTable_CachedPerShapeAndGeneration: the same shape must reuse the +// handle (a compile per request is the thing this cache exists to stop), a +// different shape must not, and a rebind must invalidate both. +func TestRoleTable_CachedPerShapeAndGeneration(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + shape := RoleShape{Columns: []string{"tenant", "id", "amount"}, Defaults: map[string]string{"tenant": "acme"}} + role := func(s RoleShape) *Table { + tbl, err := eng.RoleTable("orders", s) + require.NoError(t, err) + // Released immediately: a rebind waits for every projection's readers, + // so a held handle would block Bind rather than be invalidated by it. + tbl.Release() + return tbl + } + + first := role(shape) + // Column ORDER is not part of the shape — the DDL always follows the + // table's declaration order. + assert.Same(t, first, role(RoleShape{ + Columns: []string{"amount", "tenant", "id"}, + Defaults: map[string]string{"tenant": "acme"}, + })) + assert.NotSame(t, first, role(RoleShape{ + Columns: []string{"tenant", "id", "amount"}, + Defaults: map[string]string{"tenant": "beta"}, + }), "the injected value is baked into the handle") + + base, err := eng.Table("orders") + require.NoError(t, err) + assert.Equal(t, 2, base.roles.len()) + base.Release() + + // A rebind closes every projection; the next lookup compiles a fresh one. + changed := ordersTable() + changed.Columns[3].Type = "UInt32" + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{changed}) + + after := role(shape) + assert.NotSame(t, first, after) + assert.Equal(t, uint64(2), after.Generation) +} + +// TestRoleTable_NegativeEntryStopsRecompiling: a shape that will not compile +// costs one compile and one log line per generation, like filterCache. +func TestRoleTable_NegativeEntryStopsRecompiling(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + for range 3 { + _, err := eng.RoleTable("orders", RoleShape{Defaults: map[string]string{"amount": "abc"}}) + require.Error(t, err) + } + base, err := eng.Table("orders") + require.NoError(t, err) + defer base.Release() + assert.Equal(t, 1, base.roles.len()) +} + +// TestRoleTable_BoundedUnderTenantValueChurn: the injected values come from +// tenant claims and are baked into compiled handles, so the cache must be +// bounded exactly like filterCache. +func TestRoleTable_BoundedUnderTenantValueChurn(t *testing.T) { + eng := TestEngine(t, ordersTable()) + + base, err := eng.Table("orders") + require.NoError(t, err) + base.Release() + base.mu.Lock() + base.roles = newRoleCache(4) + base.mu.Unlock() + + for i := range 20 { + tbl, err := eng.RoleTable("orders", RoleShape{ + Defaults: map[string]string{"tenant": string(rune('a' + i))}, + }) + require.NoError(t, err) + tbl.Release() + } + + base, err = eng.Table("orders") + require.NoError(t, err) + defer base.Release() + assert.Equal(t, 4, base.roles.len()) + + // The survivors still answer after their neighbours were closed. + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"tenant": "acme"}}) + batch, err := tbl.Ingest(FormatJSONEachRow, []byte(`{"id":1,"amount":5}`+"\n")) + require.NoError(t, err) + require.True(t, batch.Rows[0].Accepted) + assert.Equal(t, `[1, "acme", "", 5]`, string(batch.Rows[0].Line)) +} + +// TestRoleTable_HasItsOwnHandlePool: a role shape holds its own single handle, +// not the base table's pool (256 shapes x the pool would be unbounded memory). +func TestRoleTable_HasItsOwnHandlePool(t *testing.T) { + eng := TestEngine(t, ordersTable()) + tbl := roleTableFor(t, eng, RoleShape{Defaults: map[string]string{"tenant": "acme"}}) + assert.Equal(t, roleHandles, len(tbl.slots)) + assert.Nil(t, tbl.roles, "a projection is never itself projected") +} + +func (c *roleCache) len() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.order.Len() +} diff --git a/internal/typelayer/testing.go b/internal/typelayer/testing.go new file mode 100644 index 00000000..21d47e14 --- /dev/null +++ b/internal/typelayer/testing.go @@ -0,0 +1,50 @@ +package typelayer + +import ( + "io" + "log/slog" + "os" + "testing" + + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// TestServerVersion is the ClickHouse version TestEngine binds to — a real +// 26.6 patch release, so the registry resolves the 26.6 artifact line. +const TestServerVersion = "26.6.3.62" + +// missingArtifact is what a developer without the artifact needs to read: the +// exact command that installs it, and where it lands. +const missingArtifact = "chtypes artifact for 26.6 (ABI revision 6) not installed: run " + + "`go run github.com/wave-rf/chtypes/go/cmd/chtypes@v0.4.0 fetch 26.6`, " + + "which installs it under ~/.cache/chtypes/artifacts/abi6/-" + +// TestEngine opens an Engine on the SDK's default search path and binds the +// given tables at TestServerVersion in UTC. It skips the test when the artifact +// is absent — unless WAVEHOUSE_TEST_REQUIRE_CHTYPES=1, which CI sets, so a runner with a +// broken artifact cache fails loudly instead of quietly testing nothing. +func TestEngine(t testing.TB, tables ...*discovery.TableSchema) *Engine { + t.Helper() + eng, err := NewEngine(Config{}, slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + skipUnlessRequired(t, err.Error()) + } + t.Cleanup(eng.Close) + + eng.Bind(TestServerVersion, "UTC", tables) + eng.mu.RLock() + global := eng.global + eng.mu.RUnlock() + if global != "" { + skipUnlessRequired(t, global) + } + return eng +} + +func skipUnlessRequired(t testing.TB, cause string) { + t.Helper() + if os.Getenv("WAVEHOUSE_TEST_REQUIRE_CHTYPES") == "1" { + t.Fatalf("%s\n%s", missingArtifact, cause) + } + t.Skipf("%s\n%s", missingArtifact, cause) +} diff --git a/internal/typelayer/typelayer.go b/internal/typelayer/typelayer.go new file mode 100644 index 00000000..c08ea43d --- /dev/null +++ b/internal/typelayer/typelayer.go @@ -0,0 +1,582 @@ +// Package typelayer owns every call into the chtypes library. It compiles one +// ClickHouse schema handle per discovered table against the artifact for the +// server's own version line, and answers the questions the gateway would +// otherwise have to re-derive in Go, with the server's own parser: +// +// - "would this record insert, and does it satisfy a role's insert check +// clauses?" — Table.Ingest +// - "does this stored row match a role's row filter?" — Table.ParseRow / +// Row.Visible +// - "what would this record insert for a role that may not write every +// column, and whose absent check columns must be filled?" — +// Engine.RoleTable +// +// Nothing outside this package imports github.com/wave-rf/chtypes/go/chtypes. +package typelayer + +import ( + "errors" + "fmt" + "log/slog" + "runtime" + "strings" + "sync" + "sync/atomic" + + "github.com/wave-rf/chtypes/go/chtypes" + + "github.com/Wave-RF/WaveHouse/internal/chsql" + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// Format is the wire format a body is parsed as. It is chtypes' own enum, +// aliased so no other package has to import the SDK to name one; only the +// constants below are formats Ingest accepts. +type Format = chtypes.Format + +// The formats Ingest accepts. JSONEachRow is name-addressed (an NDJSON body is +// the same format, byte for byte); CSV and TSV are positional in declaration +// order, with ClickHouse's header auto-detection unless IngestOptions turns it +// off; the WithNames pair open with a header line that names the columns, in +// any order. +const ( + FormatJSONEachRow = chtypes.JSONEachRow + FormatCSV = chtypes.CSV + FormatTSV = chtypes.TSV + FormatCSVWithNames = chtypes.CSVWithNames + FormatTSVWithNames = chtypes.TSVWithNames +) + +// processTZ guards the chtypes.Timezone package global across Engines. +var processTZ struct { + mu sync.Mutex + zone string + set bool +} + +// compileSettings is the fixed parsing profile every table handle is compiled +// with. allow_errors_ratio turns "first bad row ends the batch" into +// skip-and-continue so every record gets its own verdict; skip_unknown_fields=0 +// makes an unknown field a real per-row rejection (ClickHouse code 117) instead +// of silent data loss. Neither is ever forwarded to the real INSERT. +var compileSettings = map[string]string{ + "input_format_allow_errors_ratio": "1", + "input_format_skip_unknown_fields": "0", +} + +// maxPoolSize caps the identically-compiled handles a base table holds. +// +// A LoadedSchema serializes its own calls, so one handle per table is a +// ceiling. Measured on a Linux arm64 VM (BenchmarkIngest_HandlePool, artifacts +// 26.6 and 25.8), own handles scale 4-6x at 8 goroutines, while a +// process-wide serialization gate never beats one thread (0.83-1.0x). A +// compiled handle costs ~40 KiB (~96 KiB warm), so a pool of 8 is cheap. +// Role tables keep one handle (roleHandles): hot tenants already spread over +// distinct role handles, and 256 shapes x 8 warm handles would be ~200 MiB. +// An earlier darwin run showed a flat curve, so treat the size as +// hardware-dependent and re-measure it on the deployment hardware. +const maxPoolSize = 8 + +// roleHandles is the handle count of a role-shape table. +const roleHandles = 1 + +// poolSize is how many identical handles one base-table shape gets. +func poolSize() int { + n := runtime.GOMAXPROCS(0) + if n > maxPoolSize { + n = maxPoolSize + } + if n < 1 { + n = 1 + } + return n +} + +// schemaSlot is one compiled handle plus the filters compiled against it. A +// filter is bound to the schema it was compiled from — evaluating it against a +// block from another handle takes BOTH handles' locks — so each slot owns its +// own cache and one evaluation stays on one slot. +type schemaSlot struct { + schema *chtypes.LoadedSchema + filters *filterCache +} + +// closeSlots tears down a pool. Filters go before the schema, which the C +// layer requires, and the cache index is emptied so no freed pointer survives. +func closeSlots(slots []*schemaSlot) { + for _, s := range slots { + if s == nil { + continue + } + if s.filters != nil { + s.filters.closeAll() + } + if s.schema != nil { + s.schema.Close() + } + } +} + +// Config is boot-tier: the registry directory is read once at process start. +// "" means the SDK's own search path ($CHTYPES_REGISTRY, the per-user cache, +// then the system directories); an explicit directory is searched first, then +// the rest of that path. Either way a library is opened lazily, by the first +// Bind for its line (~120 MB resident each). +type Config struct { + RegistryDir string +} + +// Engine wraps one chtypes.Registry for the process. Opened once at boot and +// rebound by discovery after every successful refresh. +type Engine struct { + reg *chtypes.Registry + logger *slog.Logger + + // bindMu serializes Bind. Refresh normally runs on one goroutine, but the + // manual refresh endpoint can overlap the auto-refresh loop, and two binds + // racing would leak a handle neither of them installed. + bindMu sync.Mutex + + mu sync.RWMutex + // lib is the library the current handles were compiled against; a change + // invalidates every handle, because a filter or block only answers for the + // library its schema came from. + lib *chtypes.Library + // tz is the zone chtypes.Timezone was initialised with. chtypes reads that + // package global once per dlopen, so it cannot be changed afterwards. + tz string + tzSet bool + global string // non-empty: the cause every Table() reports + tables map[string]*Table +} + +// NewEngine opens the registry, which reads manifests and opens no library. It +// fails only when an explicit directory cannot be read or nothing on the search +// path holds an artifact, and returns the SDK's own message, which names the +// directories it looked in. Anything an artifact itself can be wrong about — a +// missing line, a refused ABI revision, a truncated library — surfaces at the +// first Bind for that line, as a global Unavailable. +// +// WithPreload is deliberately not used: it opens a library at construction, +// and chtypes.Timezone must be set before that from the server's own zone, +// which only discovery knows (see resolve). +func NewEngine(cfg Config, logger *slog.Logger) (*Engine, error) { + reg, err := chtypes.NewRegistry(cfg.RegistryDir, chtypes.WithAutoFetch(false)) + if err != nil { + return nil, err + } + return &Engine{reg: reg, logger: logger, tables: make(map[string]*Table)}, nil +} + +// Table returns the current compiled handle for a table, read-locked. The +// caller must Release it when done; the handle stays alive and stable for the +// whole time it is held, so a concurrent rebind waits rather than pulling the +// schema out from under a request. +func (e *Engine) Table(name string) (*Table, error) { + e.mu.RLock() + global, t := e.global, e.tables[name] + e.mu.RUnlock() + + if global != "" { + return nil, &Unavailable{Table: name, Cause: global} + } + if t == nil { + return nil, &Unavailable{Table: name, Cause: "no compiled schema (table not discovered)"} + } + t.mu.RLock() + if len(t.slots) == 0 { + cause := t.cause + t.mu.RUnlock() + return nil, &Unavailable{Table: name, Cause: cause} + } + return t, nil +} + +// Close releases every compiled handle. Libraries are never unloaded — chtypes +// deliberately has no dlclose path — so this is only for tests and shutdown. +func (e *Engine) Close() { + e.mu.Lock() + tables := e.tables + e.tables = make(map[string]*Table) + e.mu.Unlock() + for _, t := range tables { + t.close() + } +} + +// Table is one compiled shape: a pool of identical schema handles plus the +// caches built over them. Fields are written only under the exclusive lock +// Bind takes, so a holder of a Release-pending read lock sees a consistent set. +// +// A Table is either the discovered table's own schema (Engine.Table) or a +// per-role projection of it (Engine.RoleTable). The two have the same method +// set; a role Table's WireColumns are the ROLE's columns, which is what makes +// the exported row per-role. +type Table struct { + Name string + Generation uint64 + // WireColumns is declaration order minus MATERIALIZED/ALIAS/EPHEMERAL — + // exactly the columns RowsExport emits, and therefore exactly what the NATS + // envelope's Columns must carry. It is read off the COMPILED handle + // (LoadedSchema.Columns), not recomputed from discovery, so it is a + // property of the thing that produced the bytes. + WireColumns []string + + log *slog.Logger + mu sync.RWMutex + // slots holds the identically-compiled handles; empty means unavailable. + slots []*schemaSlot + next atomic.Uint64 + // cols maps every column the compiled schema declares, of every kind, to + // how render writes a predicate over it — what render tests a predicate's + // column against. + cols map[string]filterColumn + cause string // why slots is empty + sig string + // lib and discovered are what a per-role recompile needs: the library the + // handles came from, and the column list the DDL was built from. + lib *chtypes.Library + discovered []discovery.Column + // roles caches per-role projections of this table; nil on a role Table, + // which is never itself projected. + roles *roleCache +} + +// Release drops the read lock taken by Engine.Table or Engine.RoleTable. +func (t *Table) Release() { t.mu.RUnlock() } + +// slot picks the handle this call runs on. Round-robin rather than a real +// sync.Pool: a LoadedSchema is safe for concurrent use (it serializes +// internally), so there is nothing to check out and return, and a stable slot +// keeps a parsed block and every filter evaluated against it on ONE handle. +func (t *Table) slot() *schemaSlot { + n := len(t.slots) + if n == 1 { + return t.slots[0] + } + return t.slots[t.next.Add(1)%uint64(n)] +} + +// close tears every handle down, waiting for this shape's readers first. +func (t *Table) close() { + t.mu.Lock() + defer t.mu.Unlock() + t.closeLocked() +} + +func (t *Table) closeLocked() { + // Role projections go first, and each waits for its own readers: a role + // handle outlives a slot swap only for as long as a request holds it. + if t.roles != nil { + t.roles.closeAll() + } + closeSlots(t.slots) + t.slots = nil +} + +// Bind resolves the library for serverVersion and (re)compiles a handle per +// table. It is called synchronously from discovery's refresh hook, so it must +// never be fatal: a failure is recorded as a cause — per table for a compile +// refusal, process-wide for a missing artifact or a timezone mismatch — and +// surfaces as *Unavailable from Table. +func (e *Engine) Bind(serverVersion, serverTZ string, tables []*discovery.TableSchema) { + e.bindMu.Lock() + defer e.bindMu.Unlock() + + tz := serverTZ + if tz == "" { + tz = "UTC" // chtypes' own default; never leak the host's zone + } + + lib, ok := e.resolve(serverVersion, tz) + if !ok { + return + } + + // Compile outside every lock: a handle costs milliseconds and Table() + // readers are on the request path. + type pending struct { + name string + sig string + slots []*schemaSlot + cause string + wire []string + cols map[string]filterColumn + discovered []discovery.Column + } + + e.mu.RLock() + current := make(map[string]*Table, len(e.tables)) + for k, v := range e.tables { + current[k] = v + } + libChanged := e.lib != lib + e.mu.RUnlock() + + fresh := make([]pending, 0, len(tables)) + keep := make(map[string]struct{}, len(tables)) + for _, ts := range tables { + keep[ts.Name] = struct{}{} + sig := signature(ts) + if !libChanged { + if old, exists := current[ts.Name]; exists && old.sig == sig && len(old.slots) > 0 { + continue // same columns, same library: the handle still answers + } + } + p := pending{name: ts.Name, sig: sig, discovered: ts.Columns} + p.slots, p.cause = compile(lib, ts) + if p.cause == "" { + p.wire = deriveWireColumns(p.slots[0].schema, wireColumns(ts)) + if p.cols, p.cause = declaredColumns(lib, p.slots[0].schema, ts.Columns); p.cause != "" { + closeSlots(p.slots) + p.slots = nil + } + } + if p.cause != "" { + e.logger.Error("chtypes could not compile table schema", "table", ts.Name, "cause", p.cause) + } + fresh = append(fresh, p) + } + + // Swap each slot under its own lock and NOT under e.mu: taking a slot's + // write lock waits for that table's in-flight requests, and holding the + // engine lock through that wait would stall lookups for every other table. + added := make(map[string]*Table, len(fresh)) + for _, p := range fresh { + t, exists := current[p.name] + if !exists { + // Nobody holds a pointer to a new slot yet, so it is filled before + // it is published rather than swapped. + t = &Table{Name: p.name, log: e.logger, roles: newRoleCache(roleCacheSize), Generation: 1} + t.WireColumns, t.cols, t.sig = p.wire, p.cols, p.sig + t.lib, t.discovered = lib, p.discovered + t.slots, t.cause = p.slots, p.cause + added[p.name] = t + continue + } + t.mu.Lock() + t.closeLocked() + t.Generation++ + t.WireColumns, t.cols, t.sig = p.wire, p.cols, p.sig + t.lib, t.discovered = lib, p.discovered + t.slots, t.cause = p.slots, p.cause + t.mu.Unlock() + } + + e.mu.Lock() + e.lib = lib + for name, t := range added { + e.tables[name] = t + } + var dropped []*Table + for name, t := range e.tables { + if _, still := keep[name]; !still { + dropped = append(dropped, t) + delete(e.tables, name) + } + } + e.mu.Unlock() + + for _, t := range dropped { + t.close() + } +} + +// resolve sets the process timezone on the first bind and looks up the library +// for the server's version line. It reports false when the engine is now +// globally unavailable. +func (e *Engine) resolve(serverVersion, tz string) (*chtypes.Library, bool) { + e.mu.Lock() + // chtypes.Timezone is read once per dlopen, so it is a fact about the + // process, not this Engine: the first Engine to bind sets it under a + // package lock and every later bind (any Engine) must agree with it. + processTZ.mu.Lock() + if !processTZ.set { + chtypes.Timezone = tz + processTZ.zone, processTZ.set = tz, true + } + e.tz, e.tzSet = processTZ.zone, true + processTZ.mu.Unlock() + if e.tz != tz { + e.global = fmt.Sprintf( + "ClickHouse reports timezone %q but this process initialised chtypes with %q; restart wavehouse to adopt the new zone", + tz, e.tz) + e.mu.Unlock() + e.logger.Error("chtypes timezone mismatch", "process_tz", e.tz, "server_tz", tz) + return nil, false + } + e.mu.Unlock() + + lib, err := e.reg.For(chtypes.Version(serverVersion)) + if err != nil { + e.mu.Lock() + e.global = err.Error() + e.mu.Unlock() + e.logger.Error("no chtypes artifact for this ClickHouse version", "server_version", serverVersion, "error", err) + return nil, false + } + + e.mu.Lock() + e.global = "" + e.mu.Unlock() + return lib, true +} + +// compile reconstructs the column-declaration list chtypes wants (not a CREATE +// TABLE) and compiles the table's pool of handles. The engine and TTL clauses +// are deliberately not declared: chtypes declines engines it cannot model, and +// neither affects the insert verdicts or filter semantics this package asks +// for. +func compile(lib *chtypes.Library, ts *discovery.TableSchema) ([]*schemaSlot, string) { + cols := make([]chtypes.DiscoveredColumn, 0, len(ts.Columns)) + for _, c := range ts.Columns { + cols = append(cols, chtypes.DiscoveredColumn{ + Name: c.Name, + Type: c.Type, + DefaultKind: c.DefaultKind, + DefaultExpression: c.DefaultExpression, + Position: c.Position, + }) + } + ddl, err := lib.ReconstructDDL(cols) + if err != nil { + return nil, "cannot reconstruct column declarations: " + err.Error() + } + return compileDDL(lib, ddl, poolSize()) +} + +// compileDDL compiles n identical handles for one declaration list. A +// refusal on any of them is a refusal for the whole shape: the handles are +// interchangeable by construction, so half a pool would be a table that +// answers differently depending on which slot a request landed on. +// +// The per-table filter budget is SPLIT across the pool rather than multiplied +// by it: values are tenant-controlled and baked into a compiled handle, so the +// bound that makes the cache not-a-DoS has to be a bound on the table. +func compileDDL(lib *chtypes.Library, ddl string, n int) ([]*schemaSlot, string) { + perSlot := max(filterCacheSize/n, 1) + slots := make([]*schemaSlot, 0, n) + for range n { + schema, err := lib.CompileDDL(ddl, chtypes.WithCompileSettings(compileSettings)) + if err != nil { + closeSlots(slots) + return nil, describeCompileError(err) + } + slots = append(slots, &schemaSlot{schema: schema, filters: newFilterCache(perSlot)}) + } + return slots, "" +} + +// describeCompileError keeps ClickHouse's own refusal (a real code) distinct +// from chtypes declining to answer — the two mean different things to an +// operator and to the HTTP status a caller picks. +func describeCompileError(err error) string { + var se *chtypes.SchemaError + if errors.As(err, &se) { + return fmt.Sprintf("compile refused with ClickHouse code %d: %s", se.Code, se.Msg) + } + var ue *chtypes.UnsupportedError + if errors.As(err, &ue) { + return "chtypes declined: " + ue.Msg + } + return err.Error() +} + +// signature is the column shape a handle was compiled from. An unchanged +// signature keeps the handle and the generation, so a refresh that discovers +// nothing new costs no compiles and invalidates no cached filter. +func signature(ts *discovery.TableSchema) string { + var b strings.Builder + for _, c := range ts.Columns { + fmt.Fprintf(&b, "%d\x1f%s\x1f%s\x1f%s\x1f%s\x1e", + c.Position, c.Name, c.Type, c.DefaultKind, c.DefaultExpression) + } + return b.String() +} + +// deriveWireColumns is what RowsExport emits, read off the compiled handle: +// declaration order minus the three kinds a positional INSERT never carries. +// MATERIALIZED and ALIAS are computed by the server; EPHEMERAL is insert-only +// and never stored. +// +// Taking it from LoadedSchema.Columns rather than recomputing it from +// discovery makes the wire list a property of the handle that produced the +// bytes, which is what ParseRow's drift check actually wants. An artifact +// without column introspection reports no Columns at all; the fallback is then +// the discovery-side computation, which answers identically on every artifact +// measured (AUDIT §C.2). +func deriveWireColumns(schema *chtypes.LoadedSchema, fallback []string) []string { + if schema == nil || len(schema.Columns) == 0 { + return fallback + } + out := make([]string, 0, len(schema.Columns)) + for _, c := range schema.Columns { + switch c.DefaultKind { + case chtypes.KindMaterialized, chtypes.KindAlias, chtypes.KindEphemeral: + // Never serialized by RowsExport. + case chtypes.KindNone, chtypes.KindDefault: + out = append(out, c.Name) + default: + // A kind this build does not know is treated as ordinary, matching + // the discovery-side fallback: dropping it would silently change + // the arity of every exported line. + out = append(out, c.Name) + } + } + return out +} + +// wireColumns is deriveWireColumns' discovery-side fallback. +func wireColumns(ts *discovery.TableSchema) []string { + out := make([]string, 0, len(ts.Columns)) + for _, c := range ts.Columns { + switch c.DefaultKind { + case "MATERIALIZED", "ALIAS", "EPHEMERAL": + default: + out = append(out, c.Name) + } + } + return out +} + +// filterColumn is how render writes a predicate over one column. +type filterColumn struct { + ident string // lib.QuoteIdentifier's spelling + intType chsql.IntType // "" unless the claim binds through chsql.StrictInt +} + +// declaredColumns maps every column name the compiled schema knows, of every +// kind, to its identifier as lib.QuoteIdentifier spells it (ClickHouse's own +// backQuote, always quoted) and, for an integer column, the type its claims +// are strictly cast to. It is the set render tests a predicate's column +// against: answering "no such column" here keeps a misspelled policy from +// costing a compile and a log line per generation, and on a ROLE table it is +// what makes a filter over a denied column fail closed instead of compiling +// against a column that is not there. Quoting once per compile keeps a C call +// off render's per-event path. A non-empty second return is the cause. +func declaredColumns(lib *chtypes.Library, schema *chtypes.LoadedSchema, fallback []discovery.Column) (map[string]filterColumn, string) { + type named struct{ name, typ string } + var cols []named + if schema != nil && len(schema.Columns) > 0 { + for _, c := range schema.Columns { + cols = append(cols, named{c.Name, c.Type}) + } + } else { + for _, c := range fallback { + cols = append(cols, named{c.Name, c.Type}) + } + } + m := make(map[string]filterColumn, len(cols)) + for _, c := range cols { + q, err := lib.QuoteIdentifier(c.name) + if err != nil { + return nil, fmt.Sprintf("cannot quote column %q: %s", c.name, err) + } + col := filterColumn{ident: q} + if it, ok := chsql.IntegerType(c.typ); ok { + col.intType = it + } + m[c.name] = col + } + return m, "" +} diff --git a/internal/typelayer/typelayer_test.go b/internal/typelayer/typelayer_test.go new file mode 100644 index 00000000..68ec2857 --- /dev/null +++ b/internal/typelayer/typelayer_test.go @@ -0,0 +1,416 @@ +package typelayer + +import ( + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/wave-rf/chtypes/go/chtypes" + + "github.com/Wave-RF/WaveHouse/internal/chsql" + "github.com/Wave-RF/WaveHouse/internal/discovery" +) + +// eventsTable reproduces M1's measured table: every column kind the ingest path +// has to answer for, including the two that never reach the wire. +func eventsTable() *discovery.TableSchema { + return &discovery.TableSchema{ + Name: "events", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt8", Position: 1}, + {Name: "name", Type: "String", Position: 2}, + {Name: "ts", Type: "DateTime64(3)", Position: 3}, + {Name: "tags", Type: "Array(String)", Position: 4}, + {Name: "score", Type: "Nullable(Int32)", IsNullable: true, Position: 5}, + {Name: "created", Type: "DateTime", HasDefault: true, DefaultKind: "DEFAULT", DefaultExpression: "now()", Position: 6}, + {Name: "id_plus", Type: "UInt16", HasDefault: true, DefaultKind: "MATERIALIZED", DefaultExpression: "id + 1", Position: 7}, + }, + } +} + +func TestBind_CompilesAndExposesWireColumns(t *testing.T) { + eng := TestEngine(t, eventsTable()) + + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + + assert.Equal(t, uint64(1), tbl.Generation) + // MATERIALIZED never crosses the wire: RowsExport does not serialize it and + // the envelope's column list must match the exported line's arity. + assert.Equal(t, []string{"id", "name", "ts", "tags", "score", "created"}, tbl.WireColumns) +} + +func TestTable_UnknownTableIsUnavailable(t *testing.T) { + eng := TestEngine(t, eventsTable()) + _, err := eng.Table("nosuch") + require.Error(t, err) + assert.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), "nosuch") +} + +// TestBind_GenerationBumpsOnlyOnSignatureChange: a refresh that discovers the +// same columns must not invalidate handles or cached filters, and one that +// discovers a new column must. +func TestBind_GenerationBumpsOnlyOnSignatureChange(t *testing.T) { + eng := TestEngine(t, eventsTable()) + + generation := func() uint64 { + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + return tbl.Generation + } + require.Equal(t, uint64(1), generation()) + + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{eventsTable()}) + assert.Equal(t, uint64(1), generation(), "identical columns keep the handle") + + changed := eventsTable() + changed.Columns = append(changed.Columns, discovery.Column{Name: "extra", Type: "String", Position: 8}) + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{changed}) + + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + assert.Equal(t, uint64(2), tbl.Generation) + assert.Contains(t, tbl.WireColumns, "extra") +} + +// TestBind_DroppedTableBecomesUnavailable: a table that leaves the database +// must stop answering rather than serve a handle for a schema that is gone. +func TestBind_DroppedTableBecomesUnavailable(t *testing.T) { + eng := TestEngine(t, eventsTable()) + eng.Bind(TestServerVersion, "UTC", nil) + + _, err := eng.Table("events") + require.Error(t, err) + assert.True(t, IsUnavailable(err)) +} + +// TestBind_MissingArtifact_UnavailableWithSDKMessage: the SDK's own text names +// every directory it searched and the command that installs the artifact — +// that is the whole diagnostic, so it is passed through verbatim. +func TestBind_MissingArtifact_UnavailableWithSDKMessage(t *testing.T) { + eng := TestEngine(t, eventsTable()) + + eng.Bind("1.2.3.4", "UTC", []*discovery.TableSchema{eventsTable()}) + _, err := eng.Table("events") + require.Error(t, err) + require.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), "no artifact for ClickHouse 1.2") + assert.Contains(t, err.Error(), "Looked in:") + + // Rebinding a version that does resolve clears the global cause. + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{eventsTable()}) + tbl, err := eng.Table("events") + require.NoError(t, err) + tbl.Release() +} + +// TestBind_TimezoneMismatchIsGlobalAndNamesBothZones: chtypes reads its +// Timezone global once per dlopen, so a server that changes zone cannot be +// adopted in-process. Every table must stop answering, loudly. +func TestBind_TimezoneMismatchIsGlobalAndNamesBothZones(t *testing.T) { + eng := TestEngine(t, eventsTable()) + + eng.Bind(TestServerVersion, "Europe/Berlin", []*discovery.TableSchema{eventsTable()}) + _, err := eng.Table("events") + require.Error(t, err) + require.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), "Europe/Berlin") + assert.Contains(t, err.Error(), "UTC") + assert.Contains(t, err.Error(), "restart") +} + +// TestBind_CompileRefusalIsPerTable: one undeclarable table must not take the +// rest of the database down with it. +func TestBind_CompileRefusalIsPerTable(t *testing.T) { + broken := &discovery.TableSchema{ + Name: "broken", + Columns: []discovery.Column{{Name: "x", Type: "NotAType(9)", Position: 1}}, + } + eng := TestEngine(t, eventsTable(), broken) + + good, err := eng.Table("events") + require.NoError(t, err) + good.Release() + + _, err = eng.Table("broken") + require.Error(t, err) + require.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), "broken") +} + +func TestSignatureDistinguishesEveryField(t *testing.T) { + t.Parallel() + base := &discovery.TableSchema{Columns: []discovery.Column{ + {Name: "a", Type: "UInt8", DefaultKind: "DEFAULT", DefaultExpression: "1", Position: 1}, + }} + sig := signature(base) + for _, mutate := range []func(c *discovery.Column){ + func(c *discovery.Column) { c.Name = "b" }, + func(c *discovery.Column) { c.Type = "UInt16" }, + func(c *discovery.Column) { c.DefaultKind = "MATERIALIZED" }, + func(c *discovery.Column) { c.DefaultExpression = "2" }, + func(c *discovery.Column) { c.Position = 2 }, + } { + other := &discovery.TableSchema{Columns: []discovery.Column{base.Columns[0]}} + mutate(&other.Columns[0]) + assert.NotEqual(t, sig, signature(other)) + } +} + +// renderExpr is what a test asserts against: the expression typelayer hands +// chtypes, so a change in quoting or parameter naming is visible. +func renderExpr(t *testing.T, tbl *Table, preds ...Predicate) (string, map[string]string) { + t.Helper() + expr, params, ok := tbl.render(preds) + require.True(t, ok) + return expr, params +} + +func TestRender_QuotesIdentifiersAndBindsEveryValue(t *testing.T) { + eng := TestEngine(t, eventsTable()) + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + + expr, params := renderExpr(t, tbl, + Predicate{Column: "name", Op: "=", Values: []string{"acme"}}, + Predicate{Column: "id", Op: "in", Values: []string{"1", "7"}}, + ) + // Every identifier is backticked by the library's own QuoteIdentifier, + // which always quotes. Every value is a bound {pN:String} parameter, never + // text; on an integer column (id is UInt8 here) each one is compared + // through the strict cast, element by element. + assert.Equal(t, "`name` = {p0:String} AND `id` IN ("+ + chsql.StrictInt("p1", "UInt8")+", "+chsql.StrictInt("p2", "UInt8")+")", expr) + assert.Equal(t, map[string]string{"p0": "acme", "p1": "1", "p2": "7"}, params) + + hostile := Predicate{Column: "name", Op: "=", Values: []string{"' OR 1=1 --"}} + expr, params = renderExpr(t, tbl, hostile) + assert.Equal(t, "`name` = {p0:String}", expr) + assert.Equal(t, "' OR 1=1 --", params["p0"]) + assert.NotContains(t, expr, "OR 1=1") +} + +// TestRender_IntegerColumnsBindThroughTheStrictCast: every operator on an +// integer column (Nullable included, cast to the bare type) renders the same +// chsql.StrictInt expression the query path renders; every other column keeps +// the plain {pN:String} form. +func TestRender_IntegerColumnsBindThroughTheStrictCast(t *testing.T) { + eng := TestEngine(t, eventsTable()) + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + + for _, op := range []string{"=", "!=", ">", "<"} { + expr, params := renderExpr(t, tbl, Predicate{Column: "score", Op: op, Values: []string{"-4"}}) + assert.Equal(t, "`score` "+op+" "+chsql.StrictInt("p0", "Int32"), expr, "Nullable(Int32) %s", op) + assert.Equal(t, map[string]string{"p0": "-4"}, params) + + expr, _ = renderExpr(t, tbl, Predicate{Column: "id", Op: op, Values: []string{"7"}}) + assert.Equal(t, "`id` "+op+" "+chsql.StrictInt("p0", "UInt8"), expr, "UInt8 %s", op) + + for _, col := range []string{"name", "ts", "created"} { + expr, _ = renderExpr(t, tbl, Predicate{Column: col, Op: op, Values: []string{"x"}}) + assert.Equal(t, "`"+col+"` "+op+" {p0:String}", expr, "%s %s", col, op) + } + } + expr, params := renderExpr(t, tbl, Predicate{Column: "score", Op: "in", Values: []string{"1", "2", "3"}}) + assert.Equal(t, "`score` IN ("+chsql.StrictInt("p0", "Int32")+", "+chsql.StrictInt("p1", "Int32")+", "+ + chsql.StrictInt("p2", "Int32")+")", expr) + assert.Equal(t, map[string]string{"p0": "1", "p1": "2", "p2": "3"}, params) + + // The MATERIALIZED UInt16 column is declared too, so a filter over it is + // rendered the same way. + expr, _ = renderExpr(t, tbl, Predicate{Column: "id_plus", Op: "=", Values: []string{"8"}}) + assert.Equal(t, "`id_plus` = "+chsql.StrictInt("p0", "UInt16"), expr) +} + +// TestRender_QuotesEveryIdentifier: a column whose name is a reserved word is +// a syntax error unquoted, so the always-quoting spelling is the one render +// uses. +func TestRender_QuotesEveryIdentifier(t *testing.T) { + reserved := &discovery.TableSchema{ + Name: "reserved", + Columns: []discovery.Column{{Name: "all", Type: "String", Position: 1}}, + } + eng := TestEngine(t, reserved) + tbl, err := eng.Table("reserved") + require.NoError(t, err) + defer tbl.Release() + + expr, _ := renderExpr(t, tbl, Predicate{Column: "all", Op: "=", Values: []string{"x"}}) + assert.Equal(t, "`all` = {p0:String}", expr) +} + +func TestRender_BackticksAnIdentifierThatNeedsIt(t *testing.T) { + odd := &discovery.TableSchema{ + Name: "odd", + Columns: []discovery.Column{{Name: "weird name", Type: "String", Position: 1}}, + } + eng := TestEngine(t, odd) + tbl, err := eng.Table("odd") + require.NoError(t, err) + defer tbl.Release() + + expr, _ := renderExpr(t, tbl, Predicate{Column: "weird name", Op: "=", Values: []string{"x"}}) + assert.Equal(t, "`weird name` = {p0:String}", expr) +} + +func TestRender_RefusesWhatItCannotExpress(t *testing.T) { + eng := TestEngine(t, eventsTable()) + tbl, err := eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + + for name, pred := range map[string]Predicate{ + "unknown column": {Column: "nosuch", Op: "=", Values: []string{"x"}}, + "empty values": {Column: "name", Op: "=", Values: nil}, + "unknown operator": {Column: "name", Op: "like", Values: []string{"x"}}, + "multi-value equal": {Column: "name", Op: "=", Values: []string{"a", "b"}}, + } { + _, _, ok := tbl.render([]Predicate{pred}) + assert.False(t, ok, name) + } +} + +func TestFilterCache_EvictsAndClosesOldest(t *testing.T) { + t.Parallel() + c := newFilterCache(2) + for i := range 3 { + key := fmt.Sprintf("k%d", i) + c.index[key] = c.order.PushFront(&filterEntry{key: key}) + if c.order.Len() > c.cap { + c.evictOldestLocked() + } + } + assert.Equal(t, 2, c.order.Len()) + assert.NotContains(t, c.index, "k0") +} + +// TestBind_WireColumnsComeFromTheCompiledHandle: the wire list is read off +// LoadedSchema.Columns, so it is a property of the handle that produced the +// bytes rather than a second derivation from discovery that could drift from +// it. All four default kinds are present so the filter is exercised whole. +func TestBind_WireColumnsComeFromTheCompiledHandle(t *testing.T) { + kinds := &discovery.TableSchema{ + Name: "kinds", + Columns: []discovery.Column{ + {Name: "id", Type: "UInt32", Position: 1}, + {Name: "plain", Type: "String", HasDefault: true, DefaultKind: "DEFAULT", DefaultExpression: "'zzz'", Position: 2}, + {Name: "mat", Type: "UInt32", HasDefault: true, DefaultKind: "MATERIALIZED", DefaultExpression: "id + 1", Position: 3}, + {Name: "ali", Type: "UInt32", HasDefault: true, DefaultKind: "ALIAS", DefaultExpression: "id + 2", Position: 4}, + {Name: "eph", Type: "UInt8", DefaultKind: "EPHEMERAL", Position: 5}, + }, + } + eng := TestEngine(t, kinds) + tbl, err := eng.Table("kinds") + require.NoError(t, err) + defer tbl.Release() + + assert.Equal(t, []string{"id", "plain"}, tbl.WireColumns) + assert.Equal(t, []string{"id", "plain", "mat", "ali", "eph"}, compiledColumnNames(tbl.slots[0].schema), + "every declared column is known to the handle, of every kind") + // render tests against the handle's own column set, not the wire list: a + // filter may name a MATERIALIZED column. + _, _, ok := tbl.render([]Predicate{{Column: "mat", Op: "=", Values: []string{"2"}}}) + assert.True(t, ok) + _, _, ok = tbl.render([]Predicate{{Column: "nosuch", Op: "=", Values: []string{"2"}}}) + assert.False(t, ok) +} + +// TestBind_HandlePoolPerTable: every table gets the pool, and a rebind +// replaces all of it. +func TestBind_HandlePoolPerTable(t *testing.T) { + eng := TestEngine(t, eventsTable()) + tbl, err := eng.Table("events") + require.NoError(t, err) + assert.Equal(t, poolSize(), len(tbl.slots)) + first := tbl.slots[0] + tbl.Release() + + changed := eventsTable() + changed.Columns[0].Type = "UInt16" + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{changed}) + + tbl, err = eng.Table("events") + require.NoError(t, err) + defer tbl.Release() + assert.Equal(t, poolSize(), len(tbl.slots)) + assert.NotSame(t, first, tbl.slots[0]) +} + +// compiledColumnNames is the column list a handle compiled to, in declaration +// order. +func compiledColumnNames(schema *chtypes.LoadedSchema) []string { + out := make([]string, 0, len(schema.Columns)) + for _, c := range schema.Columns { + out = append(out, c.Name) + } + return out +} + +// TestQuoteIdentifier_AgreesWithChsql: the stream (render, through the +// library's QuoteIdentifier) and the SQL path (chsql.QuoteIdent, which has no +// library to ask) must spell every name byte for byte alike, so a divergence +// fails here rather than in a customer's query. +func TestQuoteIdentifier_AgreesWithChsql(t *testing.T) { + eng := TestEngine(t, eventsTable()) + lib := eng.lib + require.NotNil(t, lib) + + corpus := []string{ + "x", "a`b", `a\b`, "`", `\`, "\\`", "a\\`b", "it's", `"q"`, "", "null", "NULL", "all", + "select", "from", "where", "weird name", "n.a", "Ünï", "日本", "\xff\xfe", "1abc", "?", "--", "/*", + "a\x00b", "\x00", "\\\n", "\n\\", + } + for c := range 256 { + corpus = append(corpus, string([]byte{byte(c)}), "a"+string([]byte{byte(c)})+"b") + } + for _, name := range corpus { + ours, err := lib.QuoteIdentifier(name) + require.NoError(t, err, "%q", name) + assert.Equal(t, ours, chsql.QuoteIdent(name), "name %q", name) + } +} + +// TestNewEngine_OpensNoLibraryAtConstruction: the registry is lazy, so an +// artifact that cannot load is not a boot failure; the first Bind for its line +// reports the SDK's own error as a global Unavailable. +func TestNewEngine_OpensNoLibraryAtConstruction(t *testing.T) { + TestEngine(t) // skips (or fails under WAVEHOUSE_TEST_REQUIRE_CHTYPES) without the real artifact + + dir := t.TempDir() + line := filepath.Join(dir, "26.6") + require.NoError(t, os.MkdirAll(line, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(line, "manifest.json"), + []byte(`{"library":"libchtypes.so","clickhouse_version":"26.6.8.7-stable","clickhouse_minor":"26.6"}`), 0o600)) + + eng, err := NewEngine(Config{RegistryDir: dir}, slog.New(slog.NewTextHandler(io.Discard, nil))) + require.NoError(t, err, "a broken artifact is not a construction error") + t.Cleanup(eng.Close) + + eng.Bind(TestServerVersion, "UTC", []*discovery.TableSchema{eventsTable()}) + _, err = eng.Table("events") + require.Error(t, err) + require.True(t, IsUnavailable(err)) + assert.Contains(t, err.Error(), line, "the SDK's message names the directory that failed") +} + +// TestNewEngine_UnreadableDirectoryFailsAtConstruction: a directory somebody +// named and that does not exist is a typo, reported at boot. +func TestNewEngine_UnreadableDirectoryFailsAtConstruction(t *testing.T) { + t.Parallel() + _, err := NewEngine(Config{RegistryDir: filepath.Join(t.TempDir(), "nosuch")}, slog.New(slog.NewTextHandler(io.Discard, nil))) + require.Error(t, err) + assert.Contains(t, err.Error(), "nosuch") +} diff --git a/scripts/build.sh b/scripts/build.sh index f628f23b..2f4aa743 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -64,7 +64,10 @@ printf '%s==> Building %s...%s\n' "$CYAN" "$label" "$RESET" start=$(date +%s) # build_flags may be empty; use the ${arr[@]+"${arr[@]}"} idiom so the empty # expansion is silent under `set -u` on bash 3.2 (macOS default). -CGO_ENABLED=0 go build \ +# +# CGO_ENABLED=1 is unconditional: the chtypes SDK's dlopen path needs cgo for +# dlfcn (no C library to link against, no header). +CGO_ENABLED=1 go build \ -tags="${TAGS:-}" \ ${build_flags[@]+"${build_flags[@]}"} \ -ldflags="$ldflags" \ diff --git a/scripts/fetch-chtypes.sh b/scripts/fetch-chtypes.sh new file mode 100755 index 00000000..fbaae906 --- /dev/null +++ b/scripts/fetch-chtypes.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Fetch the chtypes artifact(s) this repo pins in chtypes.lock, via the +# published SDK CLI. One wrapper, three callers: +# - deployments/Dockerfile / Dockerfile.goreleaser (bakes the artifact +# into the runtime image at build time) +# - .github/actions/setup-env (CI cache-miss fetch, before Go test jobs) +# - developers (`scripts/fetch-chtypes.sh` with no args pulls the host +# platform's build of the e2e harness's pinned line) +# +# Always --frozen --lock chtypes.lock: this repo's lock is the only source +# of truth for which exact build gets installed — never the rolling +# `artifacts` release (see R3 §7 / docs/guides/artifacts.md upstream: "a +# chtypes.lock... refreshed deliberately, never regenerated implicitly"). +# A line not in the lock, or a lock/registry mismatch, is a hard failure. +# +# Usage: scripts/fetch-chtypes.sh [ ...] [--platform ] [--dest ] +# ClickHouse minor line(s) to fetch, e.g. 26.6. Defaults to +# every line this repo needs today (LOCK_LINES below). +# --platform os-arch pair to fetch for (default: host platform, chosen +# by the SDK's own HostPlatform()). Pass linux-amd64 / +# linux-arm64 to fetch a foreign platform's artifact — used +# when baking a Linux container image from a non-Linux host. +# --dest registry directory to fetch into (default: the SDK's own +# default per-platform cache dir — see `chtypes where`). +# +# Exit codes are the SDK CLI's own: 0 ok, 1 verification/lock mismatch, +# 2 usage, 3 source unreachable, 4 not published. +set -euo pipefail + +# shellcheck source=scripts/_colors.sh +. "$(dirname "$0")/_colors.sh" + +# The SDK's own package is cgo-gated, so building the CLI below without a C +# compiler on PATH fails with a pile of "undefined: Registry" / "undefined: +# minorOf" errors from files the build tags excluded — which reads like a +# broken dependency, not a missing toolchain. Say so instead. (Measured in a +# bare ubuntu:24.04; every CI runner and golang:*-bookworm already ship one.) +if [ "$(go env CGO_ENABLED 2>/dev/null || echo 0)" != "1" ]; then + printf '%sno C compiler: go env CGO_ENABLED is not 1%s\n' "${RED}" "${RESET}" >&2 + printf ' The chtypes SDK is cgo-only. Install a C toolchain (Debian/Ubuntu:\n' >&2 + printf ' apt-get install gcc; macOS: xcode-select --install) and retry.\n' >&2 + exit 2 +fi + +# Bump together with go.mod's `require github.com/wave-rf/chtypes/go` line. +CHTYPES_SDK_VERSION="v0.4.0" +CHTYPES_CLI="github.com/wave-rf/chtypes/go/cmd/chtypes@${CHTYPES_SDK_VERSION}" + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +LOCK_FILE="${REPO_ROOT}/chtypes.lock" + +# Lines every deployment of this repo needs today. The e2e harness +# (tests/integration/setup_test.go, scripts/orchestrator) and dev compose +# both pin ClickHouse 26.6.3.62; chtypes resolves by MINOR line (26.6), +# never nearest (R3 §6), so this is "26.6", not the exact patch. Widening +# this list is how a new line gets adopted: fetch it, add it here, commit +# the updated lock. +LOCK_LINES=(26.6) + +platform="" +dest="" +lines=() + +while [ $# -gt 0 ]; do + case "$1" in + --platform) + platform=${2:?--platform requires a value} + shift 2 + ;; + --dest) + dest=${2:?--dest requires a value} + shift 2 + ;; + -h | --help) + printf 'Usage: %s [ ...] [--platform ] [--dest ]\n' "$0" + exit 0 + ;; + -*) + printf '%sunknown flag: %s%s\n' "${RED}" "$1" "${RESET}" >&2 + exit 2 + ;; + *) + lines+=("$1") + shift + ;; + esac +done + +if [ ${#lines[@]} -eq 0 ]; then + lines=("${LOCK_LINES[@]}") +fi + +args=(fetch "${lines[@]}" --frozen --lock "$LOCK_FILE") +[ -n "$platform" ] && args+=(--platform "$platform") +[ -n "$dest" ] && args+=(--dest "$dest") + +printf '%s==> chtypes fetch --frozen%s %s (lock: %s)\n' "${CYAN}" "${RESET}" "${lines[*]}" "$LOCK_FILE" +exec go run "$CHTYPES_CLI" "${args[@]}" diff --git a/scripts/size.sh b/scripts/size.sh index 91b73363..518123c8 100755 --- a/scripts/size.sh +++ b/scripts/size.sh @@ -47,8 +47,10 @@ human() { # Heads-up on a single common point of confusion in the gsa output. # Section-name reference lives in docs/, not reprinted every run. printf '%s==> Reading the gsa output:%s\n' "$CYAN" "$RESET" -printf ' %s"CGO" rows are mostly Go reflection metadata, not C code%s — this build is CGO_ENABLED=0.\n' \ +printf ' %s"CGO" rows are mostly Go reflection metadata, not linked C code%s — chtypes needs cgo for\n' \ "$YELLOW" "$RESET" +printf ' dlfcn, but it dlopens its artifact at runtime rather than linking a C library, so it adds\n' +printf ' only a thin shim to this bucket.\n' printf ' Focus on large NAMED packages (vendor / std); treat CGO/Unknown rows as noise.\n\n' # ── Side-by-side debug / release comparison. diff --git a/tests/e2e/sdk/batching.test.ts b/tests/e2e/sdk/batching.test.ts index bc2e8fc0..5049928d 100644 --- a/tests/e2e/sdk/batching.test.ts +++ b/tests/e2e/sdk/batching.test.ts @@ -2,6 +2,12 @@ import { describe, expect, it } from "vitest"; import { chQuery, dataClient, testId, waitForCondition } from "./helpers.js"; import { suiteTables } from "./tables.js"; +// These trigger the INGEST WORKER's flush (500 rows or a 5s linger), which is +// the only batching left on this path: the API handler's own 500-record +// chunking is gone. A body is now one type-layer call whatever its size, so the +// 500 here is the worker's number and nothing in the handler shares it — a +// batch of any size comes back as one contiguous, 1-based result set +// (TestIngest_LargeBatch_IndicesStayContiguous pins the indexing cheaply). describe("Ingest Batching Triggers", () => { const wh = dataClient(); const T = suiteTables("batching"); diff --git a/tests/e2e/sdk/dlq.test.ts b/tests/e2e/sdk/dlq.test.ts index 1421b894..cc23ace9 100644 --- a/tests/e2e/sdk/dlq.test.ts +++ b/tests/e2e/sdk/dlq.test.ts @@ -7,25 +7,31 @@ describe("Dead Letter Queue (DLQ) & Failures", () => { const admin = adminClient(); const T = suiteTables("dlq"); - it("routes only the failed row to DLQ while valid rows are inserted", async () => { + // This case used to assert the opposite: a row that "bypasses API validation + // but fails database insertion" landing in the DLQ. That gap is what the type + // layer closed — ingest now asks ClickHouse's own parser before publishing, + // so an unparseable value is refused at the gateway with the server's real + // code and never enters the queue. The DLQ still exists for failures that + // only surface at INSERT time (covered by tests/integration/dlq_test.go and + // internal/ingest/worker_test.go); what this pins is that bad data no longer + // gets that far. + it("refuses the unparseable row at ingest, so the DLQ never sees it", async () => { const runId = testId(); - // Get baseline DLQ stats before we pollute them. DLQ stats are keyed by - // table name, and this suite owns T.clicks exclusively, so the count is - // isolated from every other test file. + // Baseline before we touch it. DLQ stats are keyed by table name and this + // suite owns T.clicks exclusively, so the count is isolated from every + // other test file. const initialDlq = await admin.dlq.list(); const initialClicksDlq = (initialDlq.data?.tables as any)?.[T.clicks] || 0; - // We are going to send 9 perfectly valid rows, and 1 critically malformed row. + // Nine valid rows and one whose duration_ms no ClickHouse parser can read. const rows = Array.from({ length: 10 }).map((_, i) => { if (i === 9) { return { event_id: `${runId}-bad`, - page: "/bag-page", + page: "/bad-page", session_id: `session-${runId}`, user_id: `user-${runId}`, - // Go accepts strings for numerics, but ClickHouse cannot parse this into an Int/Float. - // This successfully bypasses API validation but fails database insertion. duration_ms: "definitely-not-a-number", }; } @@ -38,15 +44,17 @@ describe("Dead Letter Queue (DLQ) & Failures", () => { }); const res = await wh.from(T.clicks).insert(rows as any); - expect(res.error).toBeNull(); // API accepts it (schema validation is loose by design) + // A per-record rejection is not a request failure: the batch is a 200 whose + // body carries one verdict per record. + expect(res.error).toBeNull(); + expect(res.data?.succeeded).toBe(9); + expect(res.data?.failed).toBe(1); + const refused = res.data?.results?.find((r) => r.error); + expect(refused?.index).toBe(10); + // A `code` means ClickHouse answered — a gateway rejection carries none. + expect(refused?.code).toBeGreaterThan(0); - // 9 good rows must land. Budget is 10s (the suite norm), not the 6s a plain - // timer-flush uses: the bad row forces the worker into 1-by-1 isolation — - // ~11 sequential CH round-trips after the 5s maxWait timer — which is far - // more post-timer work than a single-insert flush, so 6s sat right at the - // edge under CI load. The structural fix is a lower e2e maxWait (deferred - // config PR), which drops the 5s-timer dependency; this budget can shrink - // back once that lands. + // The nine good rows still land: one bad record never blocks the batch. await waitForCondition(async (signal) => { const chRows = await chQuery( `SELECT count() as cnt FROM default.${T.clicks} WHERE user_id = 'user-${runId}'`, @@ -55,17 +63,10 @@ describe("Dead Letter Queue (DLQ) & Failures", () => { return Number((chRows[0] as any).cnt) === 9; }, 10_000); - // Verify exactly 1 message was added to the DLQ for this suite's clicks table - await waitForCondition(async () => { - const dlqRes = await admin.dlq.list(); - const currentClicksDlq = (dlqRes.data?.tables as any)?.[T.clicks] || 0; - return currentClicksDlq === initialClicksDlq + 1; - }, 5_000); - + // The refused row was never published, so nothing can have reached the DLQ. + // The wait above already proves the worker drained this batch. const finalDlq = await admin.dlq.list(); const finalClicksDlq = (finalDlq.data?.tables as any)?.[T.clicks] || 0; - - // Only 1 was rejected and routed to the DLQ - expect(finalClicksDlq).toBe(initialClicksDlq + 1); + expect(finalClicksDlq).toBe(initialClicksDlq); }, 20_000); }); diff --git a/tests/e2e/sdk/ingest.test.ts b/tests/e2e/sdk/ingest.test.ts index d540147a..0a960add 100644 --- a/tests/e2e/sdk/ingest.test.ts +++ b/tests/e2e/sdk/ingest.test.ts @@ -72,6 +72,9 @@ describe("Ingest", () => { expect(inCH).toHaveLength(3); }); + // The unknown field is refused by ClickHouse's own parser (code 117): the + // compile profile pins input_format_skip_unknown_fields=0 precisely so this + // stays a verdict the caller hears rather than silent data loss. it("rejects unknown fields with a validation error", async () => { const result = await wh.from(T.clicks).insert({ event_id: testId(), @@ -265,10 +268,41 @@ describe("Ingest", () => { await chQuery(`DROP TABLE IF EXISTS \`${maliciousName}\``); }); - it("rejects ingest containing the reserved received_timestamp field", async () => { + it("accepts an explicit received_timestamp and stores the caller's instant", async () => { + // This case used to assert a 400 and pass for the wrong reason: the payload + // omitted three columns that are neither nullable nor defaulted, and + // WaveHouse's own validator called that "missing required column". + // ClickHouse does not — an omitted JSONEachRow field takes the type's + // default — and the gateway now gives the server's answer. Nothing is + // reserved about received_timestamp on the way in: it is an ordinary column + // with a DEFAULT, and a caller that supplies one keeps it. + const id = testId(); + const result = await wh.from(T.clicks).insert({ + event_id: id, + received_timestamp: "2026-01-01T00:00:00.000", + } as any); + expect(result.error).toBeNull(); + + await waitForCondition(async () => { + const rows = await chQuery( + `SELECT page, toString(received_timestamp) AS ts FROM default.${T.clicks} WHERE event_id = '${id}'`, + ); + if (rows.length !== 1) return false; + expect(rows[0].ts).toBe("2026-01-01 00:00:00.000"); + expect(rows[0].page).toBe(""); + return true; + }, 15_000); + }); + + it("rejects a value ClickHouse cannot parse, with ClickHouse's own code", async () => { + // The replacement for the "reserved field" case above: a real per-record + // refusal, carrying the server's error code rather than a gateway guess. const result = await wh.from(T.clicks).insert({ event_id: testId(), - received_timestamp: "2026-01-01T00:00:00Z", + page: "/bad-duration", + user_id: "u1", + session_id: "s1", + duration_ms: "not-a-number", } as any); expect(result.error).not.toBeNull(); @@ -288,6 +322,10 @@ describe("Ingest", () => { body: "{ bad json", }); expect(res.status).toBe(400); + // The refusal is ClickHouse's own parser now, so the body carries its code + // rather than a flat gateway "invalid json". + const body = (await res.json()) as { error?: string; code?: number }; + expect(typeof body.code).toBe("number"); }); it("rejects an ingest that declares no readable Content-Type", async () => { @@ -296,7 +334,9 @@ describe("Ingest", () => { { Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}` }, { Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, - "Content-Type": "text/csv", + // text/csv USED to sit here; it is an accepted format now, so the + // refusal case needs a type ingest genuinely does not read. + "Content-Type": "application/xml", }, ]) { const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { @@ -312,7 +352,7 @@ describe("Ingest", () => { // `application/jsonlines`. const body = (await res.json()) as { error?: string }; expect(body.error).toContain( - "application/json, application/x-ndjson, application/ndjson, application/jsonl, application/jsonlines", + "application/json, application/x-ndjson, application/ndjson, application/jsonl, application/jsonlines, text/csv, text/csv; header=present, text/csv; header=absent, text/tab-separated-values, text/tab-separated-values; header=present, text/tab-separated-values; header=absent", ); } }); @@ -373,6 +413,291 @@ describe("Ingest", () => { await setPolicy(currentPolicy); }); + // CONTRACT CHANGE (AUDIT D1): a column the caller's role may not write is no + // longer a gateway 403 `column "x" not allowed for insert`. Column policy is + // enforced by compiling the role's own schema WITHOUT the denied columns, so + // naming one is ClickHouse's own per-record UNKNOWN_FIELD — a 400 with + // `code: 117`, whose message does not say whether the column exists. + it("refuses a denied insert column with ClickHouse's code 117", async () => { + const currentPolicy = readPolicyFile(); + await setPolicy({ + tables: { + ...currentPolicy.tables, + [T.clicks]: { + ...(currentPolicy.tables[T.clicks] || {}), + viewer: { + ...(currentPolicy.tables[T.clicks]?.viewer || {}), + insert: { allow_columns: ["*"], deny_columns: ["country"] }, + }, + }, + }, + }); + + try { + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + event_id: testId(), + page: "/denied-column", + user_id: "u1", + session_id: "s1", + country: "GB", + }), + }); + expect(res.status).toBe(400); + const body = (await res.json()) as { error?: string; code?: number }; + expect(body.code).toBe(117); + expect(body.error).toContain("country"); + + // The same role writing only permitted columns still succeeds, and the + // denied column takes the server's own default. + const okId = testId(); + const okRes = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + event_id: okId, + page: "/denied-column-ok", + user_id: "u1", + session_id: "s1", + }), + }); + expect(okRes.status).toBe(200); + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT country FROM default.${T.clicks} WHERE event_id = '${okId}'`, + signal, + ); + return r.length === 1 && r[0].country === "US"; + }, 10_000); + } finally { + await setPolicy(currentPolicy); + } + }); + + // CSV and TSV are new accepted formats. They are positional + // in the table's declaration order — every wire column, in that order, with + // an empty field meaning "take the DEFAULT". An end-to-end assertion is the + // only one that catches a column-order bug: a mis-ordered body still answers + // 200. + it("ingests a complete positional CSV row end to end", async () => { + const id = testId(); + // Every wire column, in declaration order. An empty field takes the + // column's DEFAULT — which is how received_timestamp gets now64(3). + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "text/csv", + }, + body: `"${id}","/csv-full","u-csv","s-csv","","GB",7,\n`, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { succeeded: number; results?: unknown[] }; + expect(body.succeeded).toBe(1); + + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT page, country, duration_ms FROM default.${T.clicks} WHERE event_id = '${id}'`, + signal, + ); + if (r.length !== 1) return false; + expect(r[0].page).toBe("/csv-full"); + expect(r[0].country).toBe("GB"); + expect(Number(r[0].duration_ms)).toBe(7); + return true; + }, 10_000); + }); + + // A bare text/csv is ClickHouse's default CSV: a first line that spells the + // column names is auto-detected as a header, so only the data row counts. + it("auto-detects a header line in bare CSV", async () => { + const id = testId(); + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "text/csv", + }, + body: + "event_id,page,user_id,session_id,referrer,country,duration_ms,received_timestamp\n" + + `"${id}","/csv-detect","u-d","s-d","","GB",7,\n`, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { total: number; succeeded: number }; + expect(body.total).toBe(1); + expect(body.succeeded).toBe(1); + }); + + it("ingests a complete positional TSV row, and a short row is code 27", async () => { + const id = testId(); + // TSV spells "take the DEFAULT" as ClickHouse's own \\N (null, which + // input_format_null_as_default turns into the column default). An EMPTY TSV + // field is the empty string, which a DateTime64 cannot read — unlike CSV, + // where an empty field IS the default. + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "text/tab-separated-values", + }, + // Row 1 is complete; row 2 is short, which is a PER-RECORD refusal with + // ClickHouse's code 27 — not a whole-request failure. + body: `${id}\t/tsv\tu-tsv\ts-tsv\t\tGB\t7\t\\N\nshort\t/tsv\n`, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + total: number; + succeeded: number; + failed: number; + results: Array<{ index: number; code?: number }>; + }; + expect(body.total).toBe(2); + expect(body.succeeded).toBe(1); + expect(body.failed).toBe(1); + expect(body.results[1].code).toBe(27); + + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT page, country FROM default.${T.clicks} WHERE event_id = '${id}'`, + signal, + ); + return r.length === 1 && r[0].page === "/tsv" && r[0].country === "GB"; + }, 10_000); + }); + + // `header=present` (RFC 4180's parameter, mirrored for TSV) selects the + // header formats: the first line names the columns in any order, it is not + // a record, and a column it omits takes its DEFAULT. + it("ingests CSV whose header names the columns in any order", async () => { + const id = testId(); + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}`, + "Content-Type": "text/csv; header=present", + }, + body: `page,event_id,user_id,session_id\n/csv-names,${id},u-n,s-n\n`, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { total: number; succeeded: number }; + expect(body.total).toBe(1); + expect(body.succeeded).toBe(1); + + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT page, user_id, country FROM default.${T.clicks} WHERE event_id = '${id}'`, + signal, + ); + return ( + r.length === 1 && + r[0].page === "/csv-names" && + r[0].user_id === "u-n" && + r[0].country === "US" + ); + }, 10_000); + }); + + it("ingests TSV with a header, and refuses an unknown header column with code 117", async () => { + const id = testId(); + const auth = { Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer" })}` }; + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { ...auth, "Content-Type": "text/tab-separated-values; header=present" }, + body: `session_id\tevent_id\tpage\tuser_id\ns-t\t${id}\t/tsv-names\tu-t\n`, + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { succeeded: number }).succeeded).toBe(1); + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT page FROM default.${T.clicks} WHERE event_id = '${id}'`, + signal, + ); + return r.length === 1 && r[0].page === "/tsv-names"; + }, 10_000); + + // A header ClickHouse refuses is a verdict on the body, not on a record. + const bad = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { ...auth, "Content-Type": "text/csv; header=present" }, + body: `event_id,nosuch\n${testId()},1\n`, + }); + expect(bad.status).toBe(400); + const err = (await bad.json()) as { error?: string; code?: number }; + expect(err.code).toBe(117); + expect(err.error).toContain("nosuch"); + }); + + // CONTRACT CHANGE (AUDIT D3): an `_in` check has no single value to inject, + // so a record omitting the column is judged on the TABLE's own default rather + // than failing closed on absence. country DEFAULTs to 'US' here, so a token + // whose allowed set contains 'US' admits the record and one without it does + // not — the old behaviour refused both. + it("tests the table default when an _in check column is absent", async () => { + const currentPolicy = readPolicyFile(); + // `_in` takes a claim TEMPLATE, not a literal set: the allowed values come + // from the token, which is the multi-tenant case the check exists for. + await setPolicy({ + tables: { + ...currentPolicy.tables, + [T.clicks]: { + ...(currentPolicy.tables[T.clicks] || {}), + viewer: { + ...(currentPolicy.tables[T.clicks]?.viewer || {}), + insert: { allow_columns: ["*"], check: { country: { _in: "{{ jwt.countries }}" } } }, + }, + }, + }, + }); + + const post = async (id: string, countries: string[]) => + fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + Authorization: `Bearer ${makeJWT({ sub: "test", role: "viewer", countries })}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + event_id: id, + page: "/in-default", + user_id: "u-in", + session_id: "s-in", + }), + }); + + try { + // country DEFAULTs to 'US'; the token allows it, so the record is admitted + // and stored with that default — the behaviour change. + const okId = testId(); + const ok = await post(okId, ["US", "CA"]); + expect(ok.status).toBe(200); + await waitForCondition(async (signal) => { + const r = await chQuery( + `SELECT country FROM default.${T.clicks} WHERE event_id = '${okId}'`, + signal, + ); + return r.length === 1 && r[0].country === "US"; + }, 10_000); + + // The same absent column with a token that does NOT allow the default is + // refused — the check is really being evaluated, not skipped. + const badId = testId(); + const bad = await post(badId, ["CA", "MX"]); + expect(bad.status).toBe(403); + const body = (await bad.json()) as { error?: string }; + expect(body.error).toContain("check failed"); + } finally { + await setPolicy(currentPolicy); + } + }); + it("rejects invalid JSON queries", async () => { const res = await fetch(`${WH_URL}/v1/query?table=${T.clicks}`, { method: "POST", diff --git a/tests/e2e/sdk/ndjson.test.ts b/tests/e2e/sdk/ndjson.test.ts index 9ed8cf00..28f541f1 100644 --- a/tests/e2e/sdk/ndjson.test.ts +++ b/tests/e2e/sdk/ndjson.test.ts @@ -122,7 +122,10 @@ describe("NDJSON ingest", () => { expect(result.data?.failed).toBe(1); const failed = result.data?.results?.find((r) => r.error); expect(failed?.index).toBe(2); - expect(failed?.error).toContain("invalid json"); + // CONTRACT CHANGE: the message is ClickHouse's own parse refusal, carrying + // its code, where it used to be the Go decoder's flat "invalid json". + expect(typeof failed?.code).toBe("number"); + expect(failed?.error).toBeTruthy(); await waitForCondition(async (signal) => { const r = await chQuery( @@ -133,6 +136,65 @@ describe("NDJSON ingest", () => { }, 10_000); }); + // The §0.2 regression guard on the wire: a SINGLE-LINE JSON array with one + // bad record used to lose the whole batch — chtypes rejects it outright and + // exports no bytes, so the records that parsed perfectly went with it. + // Ingest rewrites the array's depth-1 commas to newlines in place, which + // restores #195's promise that one bad record never obscures the rest. + it("salvages the good records of a compact JSON array with one bad record", async () => { + const runId = testId(); + const good = [`${runId}-a`, `${runId}-c`]; + const body = + "[" + + [ + { event_id: good[0], page: "/a", user_id: `user-${runId}`, session_id: `s-${runId}` }, + { + event_id: `${runId}-b`, + page: "/b", + user_id: `user-${runId}`, + session_id: `s-${runId}`, + totally_fake_field: "nope", + }, + { event_id: good[1], page: "/c", user_id: `user-${runId}`, session_id: `s-${runId}` }, + ] + .map((r) => JSON.stringify(r)) + .join(",") + + "]"; + // Deliberately one line: JSON.stringify of the array would be too, but + // spelling it out is what makes the framing the subject of the test. + expect(body.includes("\n")).toBe(false); + + const res = await fetch(`${WH_URL}/v1/ingest?table=${T.clicks}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${makeJWT({ sub: "test-viewer", role: "viewer", tenant_id: "acme" })}`, + }, + body, + }); + expect(res.status).toBe(200); + const parsed = (await res.json()) as { + total: number; + succeeded: number; + failed: number; + results: Array<{ index: number; code?: number }>; + }; + expect(parsed).toMatchObject({ total: 3, succeeded: 2, failed: 1 }); + expect(parsed.results[1].code).toBe(117); + + await waitForCondition(async (signal) => { + const r = await chQuery<{ event_id: string }>( + `SELECT event_id FROM default.${T.clicks} WHERE user_id = 'user-${runId}'`, + signal, + ); + return r.length === 2; + }, 10_000); + const inCH = await chQuery<{ event_id: string }>( + `SELECT event_id FROM default.${T.clicks} WHERE user_id = 'user-${runId}'`, + ); + expect(inCH.map((r) => r.event_id).sort()).toEqual([...good].sort()); + }); + it("accepts a raw JSON array body (Content-Type: application/json) and lands every row", async () => { const runId = testId(); const rows = [1, 2].map((n) => ({ diff --git a/tests/e2e/sdk/query.test.ts b/tests/e2e/sdk/query.test.ts index 3e0b27b4..b89eeadf 100644 --- a/tests/e2e/sdk/query.test.ts +++ b/tests/e2e/sdk/query.test.ts @@ -236,6 +236,44 @@ describe("Query", () => { } }); + // The value SPELLING is part of the SDK contract, and none of the suite + // tables can show it: they are all String/UInt32/DateTime64. A Decimal + // arrives as a JSON number (`wavehouse codegen` types it as one), and a + // DateTime64 keeps ClickHouse's own `YYYY-MM-DD HH:MM:SS.fff` spelling + // rather than ISO-8601 — the same bytes the stream carries (#372). The + // byte-exact pin for every type family lives in the Go integration suite + // (tests/integration/query_types_test.go); this is the consumer-side half. + it("renders a Decimal as a number and a DateTime64 in ClickHouse's spelling", async () => { + const admin = adminClient(); + const t = `types_${testId().replace(/-/g, "_")}`; + + await chQuery( + `CREATE TABLE IF NOT EXISTS default.\`${t}\` (id String, amount Decimal(10, 2), at DateTime64(3)) ENGINE = Memory`, + ); + await chQuery(`INSERT INTO default.\`${t}\` VALUES ('r1', 12.50, '2026-01-15 10:30:00.123')`); + await admin.schema.refresh(); + + const currentPolicy = readPolicyFile(); + await setPolicy({ + tables: { + ...currentPolicy.tables, + [t]: { viewer: { select: { allow_columns: ["*"] } } }, + }, + }); + + try { + const result = await wh.from(t).selectAll().fetch(); + expect(result.error).toBeNull(); + expect(result.data).toHaveLength(1); + const row = result.data![0] as Record; + expect(typeof row.amount).toBe("number"); + expect(row.amount).toBe(12.5); + expect(row.at).toBe("2026-01-15 10:30:00.123"); + } finally { + await chQuery(`DROP TABLE IF EXISTS default.\`${t}\``); + } + }); + it("rejects queries to unauthorized tables (403)", async () => { const admin = adminClient(); diff --git a/tests/e2e/sdk/streaming.test.ts b/tests/e2e/sdk/streaming.test.ts index 45d4e38b..b74ca693 100644 --- a/tests/e2e/sdk/streaming.test.ts +++ b/tests/e2e/sdk/streaming.test.ts @@ -20,7 +20,7 @@ describe("Streaming", () => { // Explicitly allow the 'anon' role to SELECT (stream) from this suite's tables. // 'scoped' additionally carries a per-subscriber row filter — streamed rows are // limited to the caller's own country claim — so the SSE fan-out exercises the - // row-level-security path (ResolvedPermissions.RowVisible) end to end, not just + // row-level-security path (the type layer's compiled filters) end to end, not just // column projection. Object.assign(publicPolicy.tables[T.clicks], { anon: { select: { allow_columns: ["*"] } }, @@ -105,9 +105,11 @@ describe("Streaming", () => { it("row DateTime columns arrive canonicalized, matching /v1/query (#372)", async () => { // Ingest spells the row timestamp with an offset; the wire form everywhere - // downstream must be canonical RFC 3339 UTC, so the SSE frame and the - // /v1/query rendering of the same stored instant are byte-identical — the - // query/stream clock drift #372 reported. + // downstream is ClickHouse's OWN rendering of the stored instant, so the + // SSE frame and the /v1/query rendering are byte-identical — the + // query/stream clock drift #372 reported. Since the row is produced by the + // server's writer at validation time, that identity now holds by + // construction rather than by a canonicalizer agreeing with the server. const whPublic = publicClient(); const whAuth = dataClient(); const receivedEvents: any[] = []; @@ -133,7 +135,9 @@ describe("Streaming", () => { await waitForCondition(() => receivedEvents.some((e) => e.data?.event_id === id), 10_000); const frame = receivedEvents.find((e) => e.data?.event_id === id); - expect(frame?.data.received_timestamp).toBe("2026-06-21T04:00:00.123Z"); + // The column's own rendering: "YYYY-MM-DD hh:mm:ss.SSS" in its zone + // (DateTime64(3) here), not RFC 3339 with a Z. + expect(frame?.data.received_timestamp).toBe("2026-06-21 04:00:00.123"); // The ClickHouse insert is async behind the stream event — poll the query // path until the row lands, then compare the two renderings. diff --git a/tests/integration/ingest_test.go b/tests/integration/ingest_test.go index 752b48ae..1a5f8ef6 100644 --- a/tests/integration/ingest_test.go +++ b/tests/integration/ingest_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/policy" ) // TestIngest_FlowsToClickHouseWithoutDLQ exercises the happy path: POST @@ -123,6 +125,11 @@ func TestIngest_ComputedColumns_FlowToClickHouse(t *testing.T) { // TestIngest_SuppliedComputedColumn_Rejected: the other half — a record that // names a computed column is refused at the API with a 400 naming it, rather // than having the value silently dropped by the positional encoder. +// +// The refusal is now ClickHouse's own, not WaveHouse's phrasing of it: the +// column is not one a JSONEachRow record may carry, so the parser answers +// UNKNOWN_FIELD (117) and the code rides back with the message. The old +// "is materialized and cannot be inserted" wording is gone deliberately. func TestIngest_SuppliedComputedColumn_Rejected(t *testing.T) { e := env(t) @@ -143,5 +150,315 @@ func TestIngest_SuppliedComputedColumn_Rejected(t *testing.T) { var body map[string]any require.NoError(t, json.NewDecoder(resp.Body).Decode(&body)) assert.Contains(t, body["error"], "digest") - assert.Contains(t, body["error"], "cannot be inserted") + assert.EqualValues(t, 117, body["code"], "ClickHouse's UNKNOWN_FIELD, not a gateway guess") +} + +// postIngest is the shared HTTP call for the format/policy cases below: one +// POST to /v1/ingest with a verbatim body, an explicit Content-Type, and the +// optional test role/claims headers (see setup_test.go). +func postIngest(t *testing.T, table, contentType, body, role, claims string) (int, map[string]any) { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + env(t).server.URL+"/v1/ingest?table="+url.QueryEscape(table), strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", contentType) + if role != "" { + req.Header.Set(testRoleHeader, role) + } + if claims != "" { + req.Header.Set(testClaimsHeader, claims) + } + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + var decoded map[string]any + require.NoError(t, json.NewDecoder(resp.Body).Decode(&decoded)) + return resp.StatusCode, decoded +} + +// eventuallyRows waits for the ingest worker's batch window and reports the row +// count matching a predicate. +func eventuallyRows(t *testing.T, table, where string, want uint64) { + t.Helper() + ctx := context.Background() + assert.Eventually(t, func() bool { + var count uint64 + err := env(t).chConn.QueryRow(ctx, + fmt.Sprintf("SELECT count() FROM %s WHERE %s", table, where)).Scan(&count) + return err == nil && count == want + }, 30*time.Second, 250*time.Millisecond, "expected %d row(s) in %s WHERE %s", want, table, where) +} + +// TestIngest_CompactArray_OneBadRecord_TheOthersStillLand is the §0.2 +// regression guard end to end, against a real ClickHouse. +// +// Measured on the artifact: a single-line JSON array with one bad record makes +// chtypes reject the WHOLE batch and export nothing — the two good records are +// lost with the bad one, which breaks #195's promise. Ingest rewrites the +// array's depth-1 commas to newlines in place, which restores per-record +// salvage. This asserts the two survivors actually reach the table, not merely +// that the response said so. +func TestIngest_CompactArray_OneBadRecord_TheOthersStillLand(t *testing.T) { + table := createTable(t, "user_id String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "application/json", + `[{"user_id":"a1","value":1},{"user_id":"a2","value":"not-a-number"},{"user_id":"a3","value":3}]`, + "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 3, body["total"]) + assert.EqualValues(t, 2, body["succeeded"]) + assert.EqualValues(t, 1, body["failed"]) + + eventuallyRows(t, table, "user_id IN ('a1','a3')", 2) + eventuallyRows(t, table, "user_id = 'a2'", 0) +} + +// TestIngest_CSVBody_LandsInClickHouse: CSV is positional in +// the table's declaration order. The end-to-end assertion is what makes the +// positional contract real — a column-order bug would still answer 200. +func TestIngest_CSVBody_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/csv", + "\"c1\",\"click\",7\n\"c2\",\"view\",9\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 2, body["succeeded"]) + + eventuallyRows(t, table, "user_id = 'c1' AND event_type = 'click' AND value = 7", 1) + eventuallyRows(t, table, "user_id = 'c2' AND event_type = 'view' AND value = 9", 1) +} + +// TestIngest_BareCSVHeader_LandsInClickHouse: a bare `text/csv` is ClickHouse's +// default CSV, so a first line spelling the column names is consumed as a +// header and only the data rows land. +func TestIngest_BareCSVHeader_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/csv", + "user_id,event_type,value\n\"d1\",\"click\",7\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 1, body["total"], "the detected header is not a record") + assert.EqualValues(t, 1, body["succeeded"]) + + eventuallyRows(t, table, "user_id = 'd1' AND value = 7", 1) + eventuallyRows(t, table, "user_id = 'user_id'", 0) +} + +// TestIngest_CSVHeaderAbsent_LandsInClickHouse: `header=absent` is strictly +// positional, so the same header line is one refused record (code 27) and +// never reaches the table. +func TestIngest_CSVHeaderAbsent_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/csv; header=absent", + "user_id,event_type,value\n\"a1\",\"click\",7\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 2, body["total"]) + assert.EqualValues(t, 1, body["succeeded"]) + assert.EqualValues(t, 1, body["failed"]) + + eventuallyRows(t, table, "user_id = 'a1' AND value = 7", 1) + eventuallyRows(t, table, "user_id = 'user_id'", 0) +} + +// TestIngest_TSVBody_LandsInClickHouse is CSV's tab-separated twin, with a bad +// row alongside a good one so the per-record salvage is covered for the +// positional formats too. +func TestIngest_TSVBody_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/tab-separated-values", + "t1\tclick\t5\nt2\tview\tnope\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 1, body["succeeded"]) + assert.EqualValues(t, 1, body["failed"]) + + eventuallyRows(t, table, "user_id = 't1' AND value = 5", 1) + eventuallyRows(t, table, "user_id = 't2'", 0) +} + +// TestIngest_CSVWithNamesBody_LandsInClickHouse: `text/csv; header=present` +// addresses the columns by the header, in any order. The header is not a +// record, a column it omits takes the table's own DEFAULT, and a bad row is +// salvaged like any other. Asserted in the table, not only in the response. +func TestIngest_CSVWithNamesBody_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32 DEFAULT 42", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/csv; header=present", + "event_type,user_id\nclick,h1\nview,h2\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 2, body["total"], "the header line is not a record") + assert.EqualValues(t, 2, body["succeeded"]) + + eventuallyRows(t, table, "user_id = 'h1' AND event_type = 'click' AND value = 42", 1) + eventuallyRows(t, table, "user_id = 'h2' AND event_type = 'view' AND value = 42", 1) +} + +// TestIngest_TSVWithNamesBody_LandsInClickHouse is the tab-separated twin, with +// a bad row beside a good one. +func TestIngest_TSVWithNamesBody_LandsInClickHouse(t *testing.T) { + table := createTable(t, "user_id String, event_type String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/tab-separated-values; header=present", + "value\tuser_id\tevent_type\n5\tn1\tclick\nnope\tn2\tview\n", "", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + assert.EqualValues(t, 1, body["succeeded"]) + assert.EqualValues(t, 1, body["failed"]) + + eventuallyRows(t, table, "user_id = 'n1' AND value = 5", 1) + eventuallyRows(t, table, "user_id = 'n2'", 0) +} + +// TestIngest_WithNamesUnknownHeader_Is400WithCode117: a header naming a column +// the table does not have is ClickHouse's own refusal of the body, before any +// record — a whole-request 400 carrying its code, and nothing stored. +func TestIngest_WithNamesUnknownHeader_Is400WithCode117(t *testing.T) { + table := createTable(t, "user_id String, value UInt32", "ORDER BY user_id") + + status, body := postIngest(t, table, "text/csv; header=present", "user_id,nosuch\nu1,1\n", "", "") + require.Equal(t, http.StatusBadRequest, status, "body=%v", body) + assert.EqualValues(t, 117, body["code"]) + assert.Contains(t, body["error"], "nosuch") + eventuallyRows(t, table, "1", 0) +} + +// TestIngest_DeniedColumn_IsClickHouseCode117 is decision D1 end to end: column +// policy is answered by compiling the ROLE's own schema without the denied +// columns, so a record naming one is ClickHouse's per-record UNKNOWN_FIELD — +// a 400 with code 117, where it used to be the gateway's 403 +// `column "x" not allowed for insert`. +func TestIngest_DeniedColumn_IsClickHouseCode117(t *testing.T) { + table := createTable(t, "user_id String, secret String", "ORDER BY user_id") + withIngestPolicy(t, &policy.Policy{ + AdminRole: "admin", + Tables: map[string]policy.TablePolicy{ + table: {"writer": {Insert: &policy.InsertPermissions{DenyColumns: []string{"secret"}}}}, + }, + }) + + status, body := postIngest(t, table, "application/json", + `{"user_id":"d1","secret":"leak"}`, "writer", "") + require.Equal(t, http.StatusBadRequest, status, "body=%v", body) + assert.Contains(t, body["error"], "secret") + assert.EqualValues(t, 117, body["code"]) + + // The same role WITHOUT the denied column still writes, and the column takes + // the server's default rather than the caller's value. + status, body = postIngest(t, table, "application/json", `{"user_id":"d2"}`, "writer", "") + require.Equal(t, http.StatusOK, status, "body=%v", body) + eventuallyRows(t, table, "user_id = 'd2' AND secret = ''", 1) + eventuallyRows(t, table, "user_id = 'd1'", 0) +} + +// TestIngest_AutoInject_FillsAnAbsentCheckColumn covers both halves of the +// per-role DEFAULT mechanism (AUDIT §A.3): a record that omits the checked +// column is filled from the claim, and a record that supplies a value keeps its +// own — the caller's value wins over the injected default, measured. +func TestIngest_AutoInject_FillsAnAbsentCheckColumn(t *testing.T) { + table := createTable(t, "user_id String, tenant String", "ORDER BY user_id") + tmpl := "{{ jwt.tenant }}" + withIngestPolicy(t, &policy.Policy{ + AdminRole: "admin", + Tables: map[string]policy.TablePolicy{ + table: {"writer": {Insert: &policy.InsertPermissions{ + Check: map[string]policy.Filter{"tenant": {Eq: &tmpl}}, + }}}, + }, + }) + const claims = `{"tenant":"acme"}` + + // Absent → injected. + status, body := postIngest(t, table, "application/json", `{"user_id":"i1"}`, "writer", claims) + require.Equal(t, http.StatusOK, status, "body=%v", body) + + // Supplied and matching → the caller's own value rides through. + status, body = postIngest(t, table, "application/json", + `{"user_id":"i2","tenant":"acme"}`, "writer", claims) + require.Equal(t, http.StatusOK, status, "body=%v", body) + + // Supplied and NOT matching → the check refuses it, 403, nothing stored. + status, body = postIngest(t, table, "application/json", + `{"user_id":"i3","tenant":"other"}`, "writer", claims) + require.Equal(t, http.StatusForbidden, status, "body=%v", body) + assert.Contains(t, body["error"], "check failed") + + eventuallyRows(t, table, "user_id = 'i1' AND tenant = 'acme'", 1) + eventuallyRows(t, table, "user_id = 'i2' AND tenant = 'acme'", 1) + eventuallyRows(t, table, "user_id = 'i3'", 0) +} + +// TestIngest_CheckIn_AbsentColumn_TestsTheTableDefault is decision D3, the one +// behaviour change auto-inject cannot cover: an `_in` check has no single value +// to inject, so a record omitting the column is judged on the TABLE's own +// default rather than failing closed on absence. Both directions are asserted, +// because the difference is entirely in what the column's default happens to be. +func TestIngest_CheckIn_AbsentColumn_TestsTheTableDefault(t *testing.T) { + tmpl := "{{ jwt.tenants }}" + perms := func(table string) *policy.Policy { + return &policy.Policy{ + AdminRole: "admin", + Tables: map[string]policy.TablePolicy{ + table: {"writer": {Insert: &policy.InsertPermissions{ + Check: map[string]policy.Filter{"tenant": {In: &tmpl}}, + }}}, + }, + } + } + + t.Run("a default outside the set is refused", func(t *testing.T) { + table := createTable(t, "user_id String, tenant String", "ORDER BY user_id") + withIngestPolicy(t, perms(table)) + status, body := postIngest(t, table, "application/json", + `{"user_id":"n1"}`, "writer", `{"tenants":["acme","globex"]}`) + require.Equal(t, http.StatusForbidden, status, "body=%v", body) + assert.Contains(t, body["error"], "check failed") + eventuallyRows(t, table, "user_id = 'n1'", 0) + }) + + t.Run("a default inside the set is admitted", func(t *testing.T) { + table := createTable(t, "user_id String, tenant String DEFAULT 'acme'", "ORDER BY user_id") + withIngestPolicy(t, perms(table)) + status, body := postIngest(t, table, "application/json", + `{"user_id":"n2"}`, "writer", `{"tenants":["acme","globex"]}`) + require.Equal(t, http.StatusOK, status, "body=%v", body) + eventuallyRows(t, table, "user_id = 'n2' AND tenant = 'acme'", 1) + }) +} + +// TestIngest_IntegerCheckClaimThatDoesNotFit_IsRefused: an _eq check on an +// integer column compares the claim through the strict cast. 2^64+5 does not +// fit a UInt64, yet the plain String binding wrapped it onto 5 in the check — +// and the injected DEFAULT wraps it onto 5 too — so a record used to land under +// tenant 5. Now the check refuses it (403) and nothing is published, whether +// the record omits the column or supplies the wrapped value itself. The same +// policy with a claim that fits lands as before. +func TestIngest_IntegerCheckClaimThatDoesNotFit_IsRefused(t *testing.T) { + table := createTable(t, "user_id String, tenant UInt64", "ORDER BY user_id") + tmpl := "{{ jwt.tenant }}" + withIngestPolicy(t, &policy.Policy{ + AdminRole: "admin", + Tables: map[string]policy.TablePolicy{ + table: {"writer": {Insert: &policy.InsertPermissions{ + Check: map[string]policy.Filter{"tenant": {Eq: &tmpl}}, + }}}, + }, + }) + const over = `{"tenant":"18446744073709551621"}` + + for _, rec := range []string{`{"user_id":"o1"}`, `{"user_id":"o2","tenant":5}`, `{"user_id":"o3","tenant":"18446744073709551621"}`} { + status, body := postIngest(t, table, "application/json", rec, "writer", over) + require.Equal(t, http.StatusForbidden, status, "%s: body=%v", rec, body) + assert.Contains(t, body["error"], "check failed", rec) + } + + status, body := postIngest(t, table, "application/json", `{"user_id":"i1"}`, "writer", `{"tenant":"5"}`) + require.Equal(t, http.StatusOK, status, "body=%v", body) + status, body = postIngest(t, table, "application/json", `{"user_id":"i2","tenant":5}`, "writer", `{"tenant":"5"}`) + require.Equal(t, http.StatusOK, status, "body=%v", body) + + // The admitted records were posted after the refused ones, so once they + // have landed a published refusal would have landed too. + eventuallyRows(t, table, "user_id IN ('i1', 'i2') AND tenant = 5", 2) + eventuallyRows(t, table, "user_id IN ('o1', 'o2', 'o3')", 0) + eventuallyRows(t, table, "1 = 1", 2) } diff --git a/tests/integration/query_binding_test.go b/tests/integration/query_binding_test.go new file mode 100644 index 00000000..b1c9d250 --- /dev/null +++ b/tests/integration/query_binding_test.go @@ -0,0 +1,113 @@ +//go:build integration + +package tests + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/api" + "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/chconn" + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/testutil" +) + +// TestStructuredQuery_FilterValuesRoundTrip drives every filter value shape +// through the real server, for both the `eq` (scalar `{pN:String}`) and `in` +// (`{pN:Array(String)}`) bindings. +// +// It exists because the two bindings need DIFFERENT encodings and a Go-side +// unit test can only assert what the author believed: a scalar parameter is +// read by ClickHouse's escaped-text reader (an unencoded backslash silently +// becomes an escape sequence, an unencoded tab or newline is a hard code-457 +// parse error), while an Array(String) element is read as a quoted literal (a +// raw tab rides through, but a quote or backslash must be escaped). Applying +// either encoding to the other's value is silent data loss, so the server is +// the oracle: seed the value, filter for it, and require exactly the one row. +func TestStructuredQuery_FilterValuesRoundTrip(t *testing.T) { + e := env(t) + table := createTable(t, "id String, v String", "ORDER BY id") + + values := map[string]string{ + "plain": "hello", + "single-quote": "it's", + "backslash": `a\b`, + "windows-path": `C:\Users\x`, + "tab": "a\tb", + "newline": "a\nb", + "carriage-return": "a\rb", + "literal-bs-n": `a\nb`, + "double-backslash": `a\\b`, + "percent": "100%", + "ampersand": "a&b", + "unicode": "héllo→", + "quote-and-slash": `it's a\b`, + "sql-ish": `') OR 1=1 --`, + "empty": "", + } + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + for id, v := range values { + require.NoError(t, e.chConn.Exec(ctx, + fmt.Sprintf("INSERT INTO `%s` (id, v) VALUES (?, ?)", table), id, v), "seed %q", id) + } + + store := policy.Static(&policy.Policy{AdminRole: "admin"}) + h := api.NewStructuredQueryHandler( + func() chconn.Target { + return chconn.Target{URL: e.chHTTPURL, Username: testCHUser, Password: testCHPassword, Database: testCHDatabase} + }, + nil, e.registry, store, + func() int { return 60 }, + func() time.Duration { return 30 * time.Second }, + nil, testutil.NopLogger(), + ) + + query := func(t *testing.T, body string) []map[string]any { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/v1/query?table="+table, strings.NewReader(body)) + req = req.WithContext(auth.WithRole(req.Context(), "admin")) + rec := httptest.NewRecorder() + h.Handle(rec, req) + require.Equal(t, http.StatusOK, rec.Code, "body: %s", rec.Body.String()) + var rows []map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rows)) + return rows + } + + for id, v := range values { + t.Run("eq/"+id, func(t *testing.T) { + filter, err := json.Marshal(map[string]any{ + "columns": []string{"id"}, + "filters": []any{map[string]any{"column": "v", "op": "eq", "value": v}}, + }) + require.NoError(t, err) + rows := query(t, string(filter)) + require.Len(t, rows, 1, "eq on %q must match exactly the row that holds it", v) + require.Equal(t, id, rows[0]["id"]) + }) + + t.Run("in/"+id, func(t *testing.T) { + // A second element that is nowhere in the table keeps the list a + // real list, so a bad separator would show up as a wrong count. + filter, err := json.Marshal(map[string]any{ + "columns": []string{"id"}, + "filters": []any{map[string]any{"column": "v", "op": "in", "value": []string{v, "\x00absent\x00"}}}, + }) + require.NoError(t, err) + rows := query(t, string(filter)) + require.Len(t, rows, 1, "in on %q must match exactly the row that holds it", v) + require.Equal(t, id, rows[0]["id"]) + }) + } +} diff --git a/tests/integration/query_limits_test.go b/tests/integration/query_limits_test.go index f629644c..72e705a1 100644 --- a/tests/integration/query_limits_test.go +++ b/tests/integration/query_limits_test.go @@ -16,6 +16,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/api" "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/chconn" "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/testutil" ) @@ -67,12 +68,11 @@ func TestStructuredQuery_ResourceCapsEnforcedServerSide(t *testing.T) { { // max_rows_to_read bounds rows SCANNED — the lever that stops a // full-table scan. A 25-row scan blows past a cap of 1. ClickHouse - // error code 158 == TOO_MANY_ROWS (the native driver surfaces the - // numeric code, not the HTTP interface's symbolic suffix). + // error code 158 == TOO_MANY_ROWS. name: "per-role max_rows_to_read is enforced (code 158 TOO_MANY_ROWS)", perms: policy.SelectPermissions{AllowColumns: []string{"*"}, MaxRowsToRead: 1}, wantStatus: http.StatusInternalServerError, - wantBodyHas: "code: 158", + wantBodyHas: "Code: 158", }, { // max_memory_usage bounds peak query memory — the lever that stops @@ -82,7 +82,7 @@ func TestStructuredQuery_ResourceCapsEnforcedServerSide(t *testing.T) { name: "per-role max_memory_usage is enforced (code 241 MEMORY_LIMIT_EXCEEDED)", perms: policy.SelectPermissions{AllowColumns: []string{"*"}, MaxMemoryUsage: 1}, wantStatus: http.StatusInternalServerError, - wantBodyHas: "code: 241", + wantBodyHas: "Code: 241", }, } @@ -99,7 +99,10 @@ func TestStructuredQuery_ResourceCapsEnforcedServerSide(t *testing.T) { }, }) h := api.NewStructuredQueryHandler( - e.chConn, nil, e.registry, store, func() int { return 60 }, func() time.Duration { return 30 * time.Second }, nil, testutil.NopLogger(), + func() chconn.Target { + return chconn.Target{URL: e.chHTTPURL, Username: testCHUser, Password: testCHPassword, Database: testCHDatabase} + }, + nil, e.registry, store, func() int { return 60 }, func() time.Duration { return 30 * time.Second }, nil, testutil.NopLogger(), ) req := httptest.NewRequest(http.MethodPost, diff --git a/tests/integration/query_types_test.go b/tests/integration/query_types_test.go new file mode 100644 index 00000000..13268edf --- /dev/null +++ b/tests/integration/query_types_test.go @@ -0,0 +1,133 @@ +//go:build integration + +package tests + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/api" + "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/chconn" + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/testutil" +) + +// queryTypesGolden is the pinned `/v1/query` response body for one row of +// every ClickHouse type family. It is a byte-for-byte pin, not a semantic +// one: the JSON *rendering* of a value is the endpoint's public contract, +// so a change from `"12.5"` to `12.5` — or a reordering of the keys — must +// show up as a failing test and be re-recorded deliberately. +// +// Regenerate with `WAVEHOUSE_UPDATE_PIN=1 make test-integration` (or +// `-run TestQuery_TypeRendering_Pin`) after an intentional contract change, +// and put the diff in the PR description. +const queryTypesGolden = "testdata/query_types_pin.json" + +// queryTypesDDL is one column per rendering family. Order matters: it is the +// order `SELECT *` projects, which some marshallers preserve and others do +// not — part of what the pin records. +const queryTypesDDL = ` + i8 Int8, i16 Int16, i32 Int32, i64 Int64, + u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, + f32 Float32, f64 Float64, + dec Decimal(10, 2), dec64 Decimal64(3), + s String, ls LowCardinality(String), fs FixedString(4), + uu UUID, en Enum8('a' = 1, 'b' = 2), bl Bool, + ip4 IPv4, ip6 IPv6, + d Date, dt DateTime('UTC'), dt64 DateTime64(3, 'UTC'), + arr Array(String), m Map(String, UInt8), + nn Nullable(Int32), nnull Nullable(String)` + +// queryTypesRow is the single row, written as SQL literals so the values +// reach ClickHouse without passing through a driver's own type mapping — +// the pin must describe ClickHouse's storage, not clickhouse-go's encoder. +// i64/u64 sit past 2^53 so the pin also records how 64-bit integers are +// spelled; fs is shorter than its FixedString(4) so the NUL padding shows. +const queryTypesRow = `( + -8, -16, -32, -9007199254740993, + 8, 16, 32, 18446744073709551615, + 0.1, 0.1, + 12.50, 1.500, + 'hello', 'lc', 'ab', + toUUID('11111111-2222-3333-4444-555555555555'), 'a', true, + '10.0.0.1', '::1', + '2026-01-15', '2026-01-15 10:30:00', '2026-01-15 10:30:00.123', + ['a', 'b'], map('k', 7), + 42, NULL)` + +// TestQuery_TypeRendering_Pin snapshots the exact JSON `/v1/query` returns for +// one row of every ClickHouse type family, against the real server the suite +// pins (26.6.3.62). +// +// It exists because the structured-query response is the SDK's data contract +// and nothing else asserts on the *spelling* of a value: the e2e tables are +// all String/UInt32/DateTime64, so a Decimal silently changing from a JSON +// string to a JSON number, or a FixedString from a byte array to a string, +// would reach consumers with a green suite. Any diff here is a breaking API +// change and belongs in the CHANGELOG. +// +// The pin was recorded once against the old native-driver path and re-recorded +// when the endpoint moved onto ClickHouse's own renderer. Exactly two things +// moved, and NOTHING else did — every other family is byte-identical: +// - Decimal* is a JSON number (12.5) where it was a JSON string ("12.5"), +// which is what the SDK codegen already claimed it was; +// - the object keys are in SELECT order rather than alphabetical, because +// Go's map marshaller sorted them and ClickHouse does not. +func TestQuery_TypeRendering_Pin(t *testing.T) { + e := env(t) + + table := createTable(t, queryTypesDDL, "ORDER BY i8") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + require.NoError(t, e.chConn.Exec(ctx, + fmt.Sprintf("INSERT INTO `%s` VALUES %s", table, queryTypesRow)), + "seed the one typed row") + + // Admin resolves to an unrestricted grant, so select_all stays a bare + // SELECT * and the projection is the DDL order above. Cache is nil so the + // body is always freshly rendered. + store := policy.Static(&policy.Policy{AdminRole: "admin"}) + h := api.NewStructuredQueryHandler( + func() chconn.Target { + return chconn.Target{URL: e.chHTTPURL, Username: testCHUser, Password: testCHPassword, Database: testCHDatabase} + }, + nil, e.registry, store, + func() int { return 60 }, + func() time.Duration { return 30 * time.Second }, + nil, testutil.NopLogger(), + ) + + req := httptest.NewRequest(http.MethodPost, + "/v1/query?table="+table, strings.NewReader(`{"select_all":true}`)) + req = req.WithContext(auth.WithRole(req.Context(), "admin")) + rec := httptest.NewRecorder() + + h.Handle(rec, req) + + body := rec.Body.String() + require.Equal(t, http.StatusOK, rec.Code, "body: %s", body) + + if os.Getenv("WAVEHOUSE_UPDATE_PIN") == "1" { + require.NoError(t, os.MkdirAll(filepath.Dir(queryTypesGolden), 0o755)) + require.NoError(t, os.WriteFile(queryTypesGolden, []byte(body+"\n"), 0o600)) + t.Logf("pin updated: %s", queryTypesGolden) + return + } + + want, err := os.ReadFile(queryTypesGolden) + require.NoError(t, err, "read pin; regenerate with WAVEHOUSE_UPDATE_PIN=1") + require.Equal(t, strings.TrimRight(string(want), "\n"), body, + "the /v1/query type-rendering contract changed — re-record deliberately "+ + "with WAVEHOUSE_UPDATE_PIN=1 and document the diff") +} diff --git a/tests/integration/rowfilter_narrowing_test.go b/tests/integration/rowfilter_narrowing_test.go deleted file mode 100644 index 6186d559..00000000 --- a/tests/integration/rowfilter_narrowing_test.go +++ /dev/null @@ -1,224 +0,0 @@ -//go:build integration - -package tests - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/Wave-RF/WaveHouse/internal/discovery" - "github.com/Wave-RF/WaveHouse/internal/policy" - "github.com/Wave-RF/WaveHouse/internal/stream" -) - -// TestRowFilterNumeric_DifferentialAgainstClickHouse pins the stream/query -// row-visibility agreement with ClickHouse itself as the oracle (the #381 -// review's storage-narrowing fail-open): for every numeric column shape × -// insertable payload × filter constant × operator, the in-memory verdict -// (policy.Evaluate → RowVisible over the pre-insert payload, specs built the -// way stream.Hub builds them) must equal what a structured query returns over -// the STORED row — `WHERE v ?` with the constant bound exactly as -// predicatesToSQL binds it. A constant ClickHouse rejects with a type error -// means the role reads no rows on the query path, so the stream must withhold -// too. Inserts go through the worker's exact HTTP surface (JSONEachRow), so -// storage narrowing — Float32/Float64 rounding, Decimal scale truncation — is -// ClickHouse's own, not a lookalike. -func TestRowFilterNumeric_DifferentialAgainstClickHouse(t *testing.T) { - shapes := []struct { - name string - ddl string - payloads []any - constants []string - // looseConstants are out-of-range spellings whose ClickHouse reading - // was measured to vary by pair on one release — mathematical promotion - // ('256' vs UInt8), or a width-boundary WRAP that compares against a - // different value than written (2^63 vs Int64 reads as −2^63). The - // stream refuses them all (the range gate), so verdicts legitimately - // diverge in the withholding direction; only the subset half of the - // guarantee is asserted here: the stream must never admit where SQL - // hides. - looseConstants []string - }{ - { - name: "uint64", - ddl: "UInt64", - payloads: []any{ - json.Number("16777217"), - json.Number("9007199254740992"), - json.Number("9007199254740993"), // 2^53+1: float64 would collapse it onto its neighbor - "12345678901234567890", // string-encoded (the JS-precision-loss escape hatch), > 2^63 - json.Number("0"), - }, - constants: []string{ - "16777216", "16777217", - "9007199254740992", "9007199254740993", - "12345678901234567890", - "1e3", // exponent spelling: ClickHouse's integer cast errors the query - "1.5", // fractional constant: same - "-5", // negative vs unsigned: measured cast error, role reads no rows — strict parity holds - }, - // Wide or width-boundary constants: ClickHouse's reading varies by - // pair (promotion vs wrap); the stream refuses — subset assertion. - looseConstants: []string{"18446744073709551616", "99999999999999999999999"}, - }, - { - name: "int64", - ddl: "Int64", - payloads: []any{json.Number("-5"), json.Number("9007199254740993")}, - constants: []string{ - "-4", "-5", "9007199254740992", - }, - // 2^63 vs Int64 was measured to WRAP (compares as −2^63) — the - // exact case the range gate exists for; subset assertion only. - looseConstants: []string{"9223372036854775808"}, - }, - { - name: "uint8", - ddl: "UInt8", - payloads: []any{json.Number("0"), json.Number("5"), json.Number("255")}, - constants: []string{ - "5", "255", - "-1", // negative vs unsigned: measured cast error — strict parity holds - }, - // Past the width, ClickHouse promotes and compares mathematically - // while the stream refuses — subset assertion only. - looseConstants: []string{"256", "300"}, - }, - { - name: "float32", - ddl: "Float32", - payloads: []any{ - json.Number("16777217"), // stores as 16777216 — the review repro - json.Number("16777218"), - json.Number("0.1"), - json.Number("1.5"), - }, - constants: []string{"16777216", "16777217", "0.1", "1.5", "2", "1e3"}, - }, - { - name: "float64", - ddl: "Float64", - payloads: []any{ - json.Number("9007199254740992"), - json.Number("9007199254740993"), // stores rounded: the storage domain collapses it - json.Number("0.1"), - }, - constants: []string{"9007199254740992", "9007199254740993", "0.1"}, - }, - { - name: "decimal_10_2", - ddl: "Decimal(10, 2)", - payloads: []any{ - json.Number("1.005"), // stores as 1.00 (truncation, not rounding) - json.Number("1.006"), - json.Number("1.02"), - json.Number("-1.005"), - }, - constants: []string{"1.005", "1.004", "1", "1.5", "-1", "1.50", "1e3"}, - looseConstants: []string{"999999999"}, // past Precision−Scale: promoted on the SQL side, refused here - }, - } - - ops := []string{"=", "!=", ">", "<"} - - for _, sh := range shapes { - t.Run(sh.name, func(t *testing.T) { - t.Parallel() - table := createTable(t, "id UInt32, v "+sh.ddl, "ORDER BY id") - spec := numericColumnSpec(t, sh.ddl) - - inserted := make(map[int]any, len(sh.payloads)) - for i, payload := range sh.payloads { - if err := insertBestEffort(t, table, map[string]any{"id": uint32(i), "v": payload}); err != nil { - // Un-storable payloads are the documented transient (DLQ) - // class, out of the parity claim — skip, on the record. - t.Logf("payload %v not insertable into %s (%v); skipping", payload, sh.ddl, err) - continue - } - inserted[i] = payload - } - require.NotEmpty(t, inserted, "corpus must contain insertable payloads") - - for id, payload := range inserted { - for _, constant := range sh.constants { - for _, op := range ops { - stream := streamVerdict(t, table, op, constant, payload, spec) - sql, sqlErr := storedVerdict(t, table, uint32(id), op, constant) - if stream != sql { - t.Errorf("%s: payload %v %s %q — stream says %v, ClickHouse says %v (query err: %v)", - sh.ddl, payload, op, constant, stream, sql, sqlErr) - } - } - } - for _, constant := range sh.looseConstants { - for _, op := range ops { - stream := streamVerdict(t, table, op, constant, payload, spec) - sql, sqlErr := storedVerdict(t, table, uint32(id), op, constant) - if stream && !sql { - t.Errorf("%s: payload %v %s %q — stream admits where ClickHouse hides (query err: %v)", - sh.ddl, payload, op, constant, sqlErr) - } - } - } - } - }) - } -} - -// numericColumnSpec builds the policy.ColumnSpec for a numeric ClickHouse type -// through the very mapping production uses (stream.NumericSpecOf, the same -// classifier and spec builder as stream.Hub's columnSpecs), so this oracle can -// never validate a mapping the Hub no longer applies. -func numericColumnSpec(t *testing.T, chType string) policy.ColumnSpec { - t.Helper() - st, ok := discovery.NumericStorageOf(chType) - require.True(t, ok, "corpus types must classify: %s", chType) - return policy.ColumnSpec{Kind: policy.ColumnNumeric, Numeric: stream.NumericSpecOf(st)} -} - -// streamVerdict resolves a one-operator literal filter through the full -// production path (Evaluate → RowVisible) and reports whether the stream -// would deliver the payload's event. -func streamVerdict(t *testing.T, table, op, constant string, payload any, spec policy.ColumnSpec) bool { - t.Helper() - f := policy.Filter{} - switch op { - case "=": - f.Eq = &constant - case "!=": - f.Neq = &constant - case ">": - f.Gt = &constant - case "<": - f.Lt = &constant - default: - t.Fatalf("unknown op %q", op) - } - p := &policy.Policy{Tables: map[string]policy.TablePolicy{ - table: {"r": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"v": f}}}}, - }} - perms := policy.Evaluate(p, "r", table, "select", nil) - require.True(t, perms.Allowed) - return perms.RowVisible(map[string]any{"v": payload}, map[string]policy.ColumnSpec{"v": spec}) -} - -// storedVerdict asks ClickHouse whether the stored row satisfies the predicate, -// with the constant bound as a positional parameter exactly like -// predicatesToSQL emits it. A query error (an exact-domain cast rejecting the -// constant's spelling) means the role reads no rows on that path. -func storedVerdict(t *testing.T, table string, id uint32, op, constant string) (bool, error) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - var cnt uint64 - q := fmt.Sprintf("SELECT count() FROM %s WHERE id = ? AND v %s ?", table, op) - if err := sharedEnv.chConn.QueryRow(ctx, q, id, constant).Scan(&cnt); err != nil { - return false, err - } - return cnt == 1, nil -} diff --git a/tests/integration/rowfilter_stream_test.go b/tests/integration/rowfilter_stream_test.go new file mode 100644 index 00000000..1456dcc8 --- /dev/null +++ b/tests/integration/rowfilter_stream_test.go @@ -0,0 +1,571 @@ +//go:build integration + +package tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/require" + + "github.com/Wave-RF/WaveHouse/internal/api" + "github.com/Wave-RF/WaveHouse/internal/auth" + "github.com/Wave-RF/WaveHouse/internal/chconn" + "github.com/Wave-RF/WaveHouse/internal/policy" + "github.com/Wave-RF/WaveHouse/internal/stream" + "github.com/Wave-RF/WaveHouse/internal/testutil" + "github.com/Wave-RF/WaveHouse/internal/typelayer" +) + +// TestRowFilterStream_DifferentialAgainstClickHouse pins the stream/query +// row-visibility agreement with ClickHouse itself as the oracle (the #381 +// review's storage-narrowing fail-open): for every column shape × insertable +// payload × filter constant × operator, the stream's verdict must equal what +// the production /v1/query handler returns over the SAME stored row, with the +// role's row filter rendered and bound exactly as production renders it. A +// query ClickHouse rejects means the role reads no rows on the query path, so +// the stream must withhold too. +// +// On an integer column both surfaces compare a claim through the strict cast +// (chsql.StrictInt), so there is a second oracle as well: the admitted set +// must be the mathematically correct one — the comparison itself when the +// constant is the canonical spelling of a value the column can hold, and +// nothing at all otherwise. Parity alone could not catch an over-admit both +// surfaces share, and the plain String binding had one: a constant at or past +// 2^64 wrapped on every integer width. +// +// Both sides now read the STORED row: the payload goes in over the worker's own +// HTTP surface, comes back out as the positional JSONCompactEachRow line the +// ingest path publishes, and the stream's evaluator parses that with the same +// ClickHouse build the server is running. So this no longer checks a Go +// re-derivation of storage narrowing against ClickHouse — it checks the WIRING, +// which is the part that can still be wrong. +// +// Parity is STRICT on every constant and every operator: there is no "the +// stream may be stricter" bucket and no excluded case. There used to be both, +// because a filter value bound in a TYPED parameter disagreed with the server — +// a sub-64-bit column's out-of-domain constant ("256" against UInt8) admitted +// where the server hides it, and a constant past a Decimal's precision could +// not be bound at all. Binding every value as {p:String} (AUDIT §C.1) closed +// both, so a divergence appearing here is a regression, not a known gap. +func TestRowFilterStream_DifferentialAgainstClickHouse(t *testing.T) { + shapes := []diffShape{ + { + name: "uint64", + ddl: "UInt64", + intType: "UInt64", + payloads: []any{ + json.Number("16777217"), + json.Number("9007199254740993"), // 2^53+1: float64 would collapse it onto its neighbor + "12345678901234567890", // string-encoded (the JS-precision-loss escape hatch), > 2^63 + json.Number("0"), + }, + constants: []string{ + "16777216", "16777217", + "9007199254740992", "9007199254740993", + "12345678901234567890", + "0", + // Spellings and magnitudes the old Go comparator refused outright + // (it could not reproduce ClickHouse's own inconsistency): an + // exponent form, a fractional constant, a negative against an + // unsigned column, and two values past the column's width. Both + // sides now go through ClickHouse, so these are strict-parity + // cases rather than a "never admit where SQL hides" subset. + "1e3", "1.5", "-5", + "18446744073709551616", "99999999999999999999999", + }, + }, + { + name: "int64", + ddl: "Int64", + intType: "Int64", + payloads: []any{json.Number("-5"), json.Number("9007199254740993")}, + constants: []string{"-4", "-5", "9007199254740992", "9223372036854775808"}, + }, + { + name: "uint8", + ddl: "UInt8", + intType: "UInt8", + payloads: []any{json.Number("0"), json.Number("5"), json.Number("255")}, + // "256"/"300" were excluded while the stream bound integers through + // the widest integer type and answered `5 < '256'` true where the + // server answers false. String binding closed that (AUDIT §C.1), so + // they are strict-parity constants — this is the pin for it. + constants: []string{"5", "255", "0", "-1", "256", "300"}, + }, + // The integer widths, with every boundary that used to wrap. + intShape("uint8_bounds", "UInt8", "UInt8"), + intShape("uint32_bounds", "UInt32", "UInt32"), + intShape("uint64_bounds", "UInt64", "UInt64"), + intShape("int64_bounds", "Int64", "Int64"), + intShape("uint128_bounds", "UInt128", "UInt128"), + intShape("int128_bounds", "Int128", "Int128"), + intShape("uint256_bounds", "UInt256", "UInt256"), + intShape("int256_bounds", "Int256", "Int256"), + intShape("nullable_uint64_bounds", "Nullable(UInt64)", "UInt64"), + { + name: "float32", + ddl: "Float32", + payloads: []any{ + json.Number("16777217"), // stores as 16777216 — the review repro + json.Number("16777218"), + json.Number("0.1"), + json.Number("1.5"), + }, + constants: []string{"16777216", "16777217", "0.1", "1.5", "2", "1e3"}, + }, + { + name: "float64", + ddl: "Float64", + payloads: []any{ + json.Number("9007199254740992"), + json.Number("9007199254740993"), // stores rounded: the storage domain collapses it + json.Number("0.1"), + }, + constants: []string{"9007199254740992", "9007199254740993", "0.1"}, + }, + { + name: "decimal_10_2", + ddl: "Decimal(10, 2)", + payloads: []any{ + json.Number("1.005"), // stores as 1.00 (truncation, not rounding) + json.Number("1.02"), + json.Number("-1.005"), + }, + // "999999999" is past Precision−Scale. It used to be a + // "never admit where SQL hides" case because the constant could not + // be bound as that Decimal at all; as a String parameter it is read + // in the column's domain and agrees outright. + constants: []string{"1.005", "1.004", "1", "1.5", "-1", "1.50", "1e3", "999999999"}, + }, + { + name: "string", + ddl: "String", + // Byte ordering, not collation: "9" sorts after "100", which is + // exactly the leak a text fallback would cause on a numeric column + // and exactly the right answer on this one. + // + // The last five are the {p:String} escaping cases. A ClickHouse + // query parameter is read by an escaped-text reader on BOTH + // surfaces — the server's HTTP interface and the chtypes artifact's + // filter params — so an unencoded backslash arrives as an escape + // sequence and an unencoded tab or newline does not parse at all. + // Measured before chsql.EscapeStringParam was applied in + // typelayer.render(): the backslash rows answered false and the + // tab/newline/trailing-backslash rows declined, while this oracle + // (a natively-bound literal) answered true — the stream hid rows + // the query path returns. Strict parity here is the pin for that. + payloads: []any{"acme", "9", "100", "", `a\b`, "a\tb", "a\nb", `trail\`, "O'Brien"}, + constants: []string{"acme", "beta", "9", "100", "", `a\b`, "a\tb", "a\nb", `trail\`, "O'Brien"}, + }, + { + name: "datetime", + ddl: "DateTime", + // The policy author's zone-less spelling against the stored instant. + payloads: []any{"2026-06-21 04:00:00", "2026-06-21T04:00:01Z"}, + constants: []string{"2026-06-21 04:00:00", "2026-06-21 04:00:01"}, + }, + } + + ops := []string{"=", "!=", ">", "<", "in"} + + // One row under test per (shape, payload): the table it lives in, its id, and + // the positional line the ingest path would publish for it. + type storedRow struct { + shape string + intType string + table string + id uint32 + payload any + columns []string + line []byte + } + var rows []storedRow + + for _, sh := range shapes { + table := createTable(t, "id UInt32, v "+sh.ddl, "ORDER BY id") + any_ := false + for i, payload := range sh.payloads { + if err := rowFilterInsert(t, table, map[string]any{"id": uint32(i), "v": payload}); err != nil { + // Un-storable payloads are the documented transient (DLQ) class, + // out of the parity claim — skip, on the record. + t.Logf("payload %v not insertable into %s (%v); skipping", payload, sh.ddl, err) + continue + } + any_ = true + rows = append(rows, storedRow{ + shape: sh.name, intType: sh.intType, table: table, id: uint32(i), payload: payload, + columns: []string{"id", "v"}, + line: rowFilterStoredLine(t, table, uint32(i)), + }) + } + require.True(t, any_, "corpus for %s must contain insertable payloads", sh.name) + } + + // ONE engine for every shape, bound to the registry createTable already + // refreshed — the same discovery output production binds. + eval := rowFilterEvaluator(t) + + byShape := map[string][]string{} + for _, sh := range shapes { + byShape[sh.name] = sh.constants + } + cells, mathCells := 0, 0 + for _, r := range rows { + stored := rowFilterStoredInt(t, r.intType, r.line) + for _, constant := range byShape[r.shape] { + for _, op := range ops { + cells++ + f := rowFilterLiteral(t, op, constant) + got := rowFilterStreamVerdict(t, eval, r.table, r.columns, r.line, f, nil) + want, sqlErr := rowFilterQueryVerdict(t, r.table, r.id, f, nil) + if got != want { + t.Errorf("%s: stored %v %s %q — stream says %v, /v1/query says %v (query err: %v)", + r.shape, r.payload, op, constant, got, want, sqlErr) + } + if exact, ok := intExpected(r.intType, stored, op, constant); ok { + mathCells++ + if want != exact || got != exact { + t.Errorf("%s: stored %v %s %q — admitted stream=%v query=%v, the mathematically correct answer is %v", + r.shape, r.payload, op, constant, got, want, exact) + } + } + } + } + + // A multi-element _in from a claim array: each element is cast on its + // own, so the elements that fit decide and the rest drop out — 2^64+5 + // must not wrap onto a row holding 5. + if r.intType != "" { + for _, set := range intInSets { + cells++ + mathCells++ + f := policy.Filter{In: new("{{ jwt.ids }}")} + claims := map[string]any{"ids": toAnys(set)} + got := rowFilterStreamVerdict(t, eval, r.table, r.columns, r.line, f, claims) + want, sqlErr := rowFilterQueryVerdict(t, r.table, r.id, f, claims) + exact := false + for _, c := range set { + if eq, ok := intExpected(r.intType, stored, "=", c); ok && eq { + exact = true + } + } + if got != want || want != exact { + t.Errorf("%s: stored %v IN %v — stream says %v, /v1/query says %v (query err: %v), correct is %v", + r.shape, r.payload, set, got, want, sqlErr, exact) + } + } + } + } + t.Logf("%d cells compared stream against /v1/query, %d of them also against the exact answer", cells, mathCells) +} + +// Boundary constants for the integer shapes: each width's own edges, the +// values that wrapped under the plain String binding, and spellings that are +// not canonical. +var intBoundaryConstants = func() []string { + p := func(n uint) *big.Int { return new(big.Int).Lsh(big.NewInt(1), n) } + add := func(a *big.Int, d int64) string { return new(big.Int).Add(a, big.NewInt(d)).String() } + neg := func(a *big.Int) *big.Int { return new(big.Int).Neg(a) } + return []string{ + "0", "1", "5", "-1", "-5", "255", "256", "4294967295", "4294967296", + add(p(63), 0), add(p(63), -1), add(neg(p(63)), -1), add(neg(p(63)), 0), + add(p(64), 0), add(p(64), -1), add(p(64), 5), + add(p(127), 0), add(p(127), -1), add(neg(p(127)), -1), add(neg(p(127)), 0), + add(p(128), 0), add(p(128), -1), + add(p(255), 0), add(p(255), -1), add(neg(p(255)), -1), add(neg(p(255)), 0), + add(p(256), 0), add(p(256), -1), add(p(256), 5), + "007", "+5", "5.0", "1e3", "abc", "", + } +}() + +// intInSets are claim arrays for the multi-element _in cases. +var intInSets = [][]string{ + {"18446744073709551621", "0"}, // 2^64+5 must not wrap onto 5 + {"5", "007", "abc"}, // the junk elements drop out, 5 still decides + {"-1", "340282366920938463463374607431768211456", "1"}, // 2^128 + {"115792089237316195423570985008687907853269984665640564039457584007913129639941", "+5", "5.0"}, // 2^256+5 +} + +// diffShape is one column type under the differential. intType names the +// bare integer type the column holds, "" for a non-integer column; it is +// declared here rather than derived with chsql.IntegerType, so a regression in +// that derivation shows up as a wrong answer instead of a skipped oracle. +type diffShape struct { + name string + ddl string + intType string + payloads []any + constants []string +} + +// intShape is a differential shape for one integer type: a row at each edge of +// its domain (and a NULL for a Nullable column), filtered by every boundary +// constant. +func intShape(name, ddl, intType string) diffShape { + lo, hi := intDomain(intType) + payloads := []any{"0", "1", "5", hi.String()} + if lo.Sign() < 0 { + payloads = append(payloads, lo.String(), "-5") + } + if strings.HasPrefix(ddl, "Nullable(") { + payloads = append(payloads, nil) + } + return diffShape{name: name, ddl: ddl, intType: intType, payloads: payloads, constants: intBoundaryConstants} +} + +// intDomain is an integer type's [min, max]. +func intDomain(name string) (*big.Int, *big.Int) { + bits, err := strconv.Atoi(strings.TrimPrefix(strings.TrimPrefix(name, "U"), "Int")) + if err != nil { + panic(name) + } + one := big.NewInt(1) + if strings.HasPrefix(name, "U") { + return big.NewInt(0), new(big.Int).Sub(new(big.Int).Lsh(one, uint(bits)), one) + } + half := new(big.Int).Lsh(one, uint(bits-1)) + return new(big.Int).Neg(half), new(big.Int).Sub(half, one) +} + +// rowFilterStoredInt reads the stored value back off the published line for an +// integer column; nil for NULL or a non-integer column. +func rowFilterStoredInt(t *testing.T, intType string, line []byte) *big.Int { + t.Helper() + if intType == "" { + return nil + } + dec := json.NewDecoder(bytes.NewReader(line)) + dec.UseNumber() + var cols []any + require.NoError(t, dec.Decode(&cols)) + require.Len(t, cols, 2) + var text string + switch v := cols[1].(type) { + case nil: + return nil + case json.Number: + text = v.String() + case string: + text = v + default: + t.Fatalf("stored integer came back as %T", v) + } + n, ok := new(big.Int).SetString(text, 10) + require.True(t, ok, "stored integer %q", text) + return n +} + +// intExpected is the mathematically correct verdict for an integer column: a +// constant that is not the canonical spelling of a value the column can hold +// admits nothing, and a NULL row is never admitted. ok is false for a +// non-integer column, which has no such oracle here. +func intExpected(intType string, stored *big.Int, op, constant string) (bool, bool) { + if intType == "" { + return false, false + } + lo, hi := intDomain(intType) + v, ok := new(big.Int).SetString(constant, 10) + if !ok || v.String() != constant || v.Cmp(lo) < 0 || v.Cmp(hi) > 0 || stored == nil { + return false, true + } + cmp := stored.Cmp(v) + switch op { + case "=", "in": + return cmp == 0, true + case "!=": + return cmp != 0, true + case "<": + return cmp < 0, true + case ">": + return cmp > 0, true + } + panic(op) +} + +func toAnys(ss []string) []any { + out := make([]any, len(ss)) + for i, s := range ss { + out[i] = s + } + return out +} + +// rowFilterEvaluator builds the production row evaluator over a type-layer +// Engine bound to the integration registry's own discovery — server version, +// server timezone and table list all as discovered, never hand-set, so the test +// cannot pass against a binding production would not make. +func rowFilterEvaluator(t *testing.T) stream.RowEvaluator { + t.Helper() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + eng, err := typelayer.NewEngine(typelayer.Config{}, logger) + require.NoError(t, err, "the 26.6 chtypes artifact must be installed") + t.Cleanup(eng.Close) + reg := env(t).registry + eng.Bind(reg.ServerVersion(), reg.ServerTimezone(), reg.List()) + return stream.NewRowEvaluator(eng, logger) +} + +// rowFilterLiteral is a one-operator filter on v with a literal constant. +func rowFilterLiteral(t *testing.T, op, constant string) policy.Filter { + t.Helper() + f := policy.Filter{} + switch op { + case "=": + f.Eq = &constant + case "!=": + f.Neq = &constant + case ">": + f.Gt = &constant + case "<": + f.Lt = &constant + case "in": + // A placeholder-free _in template resolves to a one-element set, so this + // is the IN (…) renderer on both surfaces with one bound value. + f.In = &constant + default: + t.Fatalf("unknown op %q", op) + } + return f +} + +// rowFilterPolicy grants role "r" a read of table filtered by f on v. +func rowFilterPolicy(table string, f policy.Filter) *policy.Policy { + return &policy.Policy{AdminRole: "admin", Tables: map[string]policy.TablePolicy{ + table: {"r": {Select: &policy.SelectPermissions{Filter: map[string]policy.Filter{"v": f}}}}, + }} +} + +// rowFilterStreamVerdict resolves the filter through the full production path +// (Evaluate → Prepare → Visible) and reports whether the stream would deliver +// this stored row. +func rowFilterStreamVerdict(t *testing.T, eval stream.RowEvaluator, table string, columns []string, line []byte, f policy.Filter, claims map[string]any) bool { + t.Helper() + perms := policy.Evaluate(rowFilterPolicy(table, f), "r", table, "select", claims) + require.True(t, perms.Allowed) + + view, err := eval.Prepare(table, columns, line) + if err != nil { + return false // withheld: no view, no row + } + defer view.Close() + visible, _ := view.Visible(perms) + return visible +} + +// rowFilterQueryVerdict asks the production /v1/query handler, as role "r" +// with claims, whether it returns the stored row: the row filter is rendered, +// bound and sent to ClickHouse exactly as production does it. A query the +// server rejects means the role reads no rows on that path. +func rowFilterQueryVerdict(t *testing.T, table string, id uint32, f policy.Filter, claims map[string]any) (bool, error) { + t.Helper() + e := env(t) + h := api.NewStructuredQueryHandler( + func() chconn.Target { + return chconn.Target{URL: e.chHTTPURL, Username: testCHUser, Password: testCHPassword, Database: testCHDatabase} + }, + nil, e.registry, policy.Static(rowFilterPolicy(table, f)), + func() int { return 60 }, + func() time.Duration { return 10 * time.Second }, + nil, testutil.NopLogger(), + ) + body, err := json.Marshal(map[string]any{ + "columns": []string{"id"}, + "filters": []any{map[string]any{"column": "id", "op": "eq", "value": id}}, + }) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/v1/query?table="+table, bytes.NewReader(body)) + ctx := auth.WithRole(req.Context(), "r") + if claims != nil { + ctx = auth.WithClaims(ctx, jwt.MapClaims(claims)) + } + rec := httptest.NewRecorder() + h.Handle(rec, req.WithContext(ctx)) + if rec.Code != http.StatusOK { + return false, fmt.Errorf("HTTP %d: %s", rec.Code, rec.Body.String()) + } + var out []map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + require.LessOrEqual(t, len(out), 1, "id is unique per table") + return len(out) == 1, nil +} + +// rowFilterStoredLine reads one stored row back as the positional +// JSONCompactEachRow line the ingest path publishes for it — the exact bytes the +// stream evaluates, produced by the server rather than reconstructed here. +func rowFilterStoredLine(t *testing.T, table string, id uint32) []byte { + t.Helper() + q := url.Values{} + q.Set("database", testCHDatabase) + q.Set("param_target_table", table) + q.Set("param_id", fmt.Sprint(id)) + q.Set("query", "SELECT * FROM {target_table:Identifier} WHERE id = {id:UInt32} FORMAT JSONCompactEachRow") + body := rowFilterCH(t, http.MethodGet, q, nil) + line := strings.TrimRight(string(body), "\n") + require.NotEmpty(t, line, "stored row must come back") + require.NotContains(t, line, "\n", "exactly one row per id") + return []byte(line) +} + +// rowFilterInsert inserts one JSONEachRow row over ClickHouse HTTP with the same +// parsing settings the ingest worker pins, and returns ClickHouse's verdict as +// an error (nil on 2xx). +func rowFilterInsert(t *testing.T, table string, row map[string]any) error { + t.Helper() + body, err := json.Marshal(row) + require.NoError(t, err) + + q := url.Values{} + q.Set("database", testCHDatabase) + q.Set("param_target_table", table) + q.Set("query", "INSERT INTO {target_table:Identifier} FORMAT JSONEachRow") + for k, v := range typelayer.InsertSettings() { + q.Set(k, v) + } + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, + env(t).chHTTPURL+"?"+q.Encode(), bytes.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-ClickHouse-User", testCHUser) + req.Header.Set("X-ClickHouse-Key", testCHPassword) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, msg) + } + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +// rowFilterCH runs one ClickHouse HTTP request and returns the body, failing the +// test on anything but 2xx. +func rowFilterCH(t *testing.T, method string, q url.Values, body io.Reader) []byte { + t.Helper() + req, err := http.NewRequestWithContext(context.Background(), method, env(t).chHTTPURL+"?"+q.Encode(), body) + require.NoError(t, err) + req.Header.Set("X-ClickHouse-User", testCHUser) + req.Header.Set("X-ClickHouse-Key", testCHPassword) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + out, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Less(t, resp.StatusCode, 300, "clickhouse: %s", out) + return out +} diff --git a/tests/integration/setup_test.go b/tests/integration/setup_test.go index 344fadca..38e373a6 100644 --- a/tests/integration/setup_test.go +++ b/tests/integration/setup_test.go @@ -10,6 +10,7 @@ package tests import ( "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -36,6 +37,7 @@ import ( "github.com/Wave-RF/WaveHouse/internal/policy" "github.com/Wave-RF/WaveHouse/internal/stream" "github.com/Wave-RF/WaveHouse/internal/testutil" + "github.com/Wave-RF/WaveHouse/internal/typelayer" ) const ( @@ -51,8 +53,32 @@ type testEnv struct { embeddedMQ *mq.EmbeddedNATS server *httptest.Server registry *discovery.SchemaRegistry + // ingest is the live handler, so a test can install its own policy for the + // ingest path (see withIngestPolicy). Nothing in this package runs in + // parallel, so swapping it for the length of one test is safe. + ingest *api.IngestHandler } +// withIngestPolicy installs an ingest-side policy for the calling test and +// restores the default (none — an unrestricted path) afterwards. Paired with +// testRoleHeader / testClaimsHeader, it is how the column-policy and +// insert-check cases reach the real HTTP path. +func withIngestPolicy(t *testing.T, p *policy.Policy) { + t.Helper() + prev := sharedEnv.ingest.PolicySource + sharedEnv.ingest.PolicySource = policy.Static(p) + t.Cleanup(func() { sharedEnv.ingest.PolicySource = prev }) +} + +// Test-only request headers honoured by the suite's AuthMW. Production derives +// the role and claims from a JWT; the integration suite needs to drive a +// NON-admin role and real claims through the actual handler without standing up +// a signer, and these two headers are the whole of that seam. +const ( + testRoleHeader = "X-Test-Role" + testClaimsHeader = "X-Test-Claims" // a JSON object +) + var sharedEnv *testEnv // env returns the package-shared environment. Tests must call this rather @@ -155,6 +181,20 @@ func setup() (int, func()) { } registry := discovery.NewSchemaRegistry(ch.conn, func() string { return testCHDatabase }, func() time.Duration { return time.Minute }, logger) + + // The type layer is bound from the refresh hook, exactly as main.go wires + // it, so a table created mid-suite is compiled by the createTable refresh + // rather than needing its own step. Failing here is fatal on purpose: the + // artifact missing would otherwise turn every ingest assertion into a 503 + // that reads like a product bug. + types, err := typelayer.NewEngine(typelayer.Config{}, logger) + if err != nil { + fmt.Fprintf(os.Stderr, "integration setup: chtypes engine: %v\n", err) + return 1, cleanup + } + cleanups.push(types.Close) + registry.OnRefresh(types.Bind) + if err := registry.Refresh(ctx); err != nil { fmt.Fprintf(os.Stderr, "integration setup: schema refresh: %v\n", err) return 1, cleanup @@ -181,7 +221,7 @@ func setup() (int, func()) { return 1, cleanup } - server, err := buildServer(ch, embeddedMQ, registry, logger) + server, ingestHandler, err := buildServer(ch, embeddedMQ, registry, types, logger) if err != nil { fmt.Fprintf(os.Stderr, "integration setup: build server: %v\n", err) return 1, cleanup @@ -194,6 +234,7 @@ func setup() (int, func()) { embeddedMQ: embeddedMQ, server: server, registry: registry, + ingest: ingestHandler, } return 0, cleanup } @@ -224,10 +265,9 @@ func (c *chInstance) httpURL() string { return fmt.Sprintf("http://%s:%s", c. // race; the dominant flake mode tracked in #70. func startClickHouse(ctx context.Context) (*chInstance, error) { chReq := testcontainers.ContainerRequest{ - // Pinned: 26.8 reads bare numbers in DateTime64 columns as epoch seconds, - // not ticks at column precision — CanonicalizeTimestamps still models the - // pre-26.8 rule (TestTimestampCanonicalization_DifferentialAgainstClickHouse - // catches the divergence). Bump the pin together with the canonicalizer (#536). + // Pinned to a line the chtypes artifact set covers; the type layer answers + // with the artifact matching the server's own version, so bumping this pin + // means fetching that line into chtypes.lock too (#536). Image: "clickhouse/clickhouse-server:26.6.3.62", ExposedPorts: []string{"9000/tcp", "8123/tcp"}, Env: map[string]string{"CLICKHOUSE_PASSWORD": testCHPassword}, @@ -312,14 +352,17 @@ func waitForNativeReady(ctx context.Context, conn driver.Conn, timeout time.Dura // The RequireAdmin gate resolves that stamped role against PolicySource, so the // server is wired with an in-memory policy whose admin_role is "admin". A nil // store would deny every admin-gated route (IsAdmin(nil) is false by design). -func buildServer(ch *chInstance, embeddedMQ *mq.EmbeddedNATS, registry *discovery.SchemaRegistry, logger *slog.Logger) (*httptest.Server, error) { +func buildServer(ch *chInstance, embeddedMQ *mq.EmbeddedNATS, registry *discovery.SchemaRegistry, types *typelayer.Engine, logger *slog.Logger) (*httptest.Server, *api.IngestHandler, error) { js := embeddedMQ.JetStream() policyStore := policy.Static(&policy.Policy{AdminRole: "admin"}) streamHub := stream.NewHub(policyStore, registry, nil) + ingestHandler := api.NewIngestHandler(registry, embeddedMQ, logger) + ingestHandler.Types = types + deps := api.Dependencies{ - Ingest: api.NewIngestHandler(registry, embeddedMQ, logger), + Ingest: ingestHandler, // /v1/ops/query proxies straight to ClickHouse's HTTP interface, // so the handler needs the HTTP URL + creds rather than the // native-protocol driver.Conn other handlers use. @@ -333,7 +376,20 @@ func buildServer(ch *chInstance, embeddedMQ *mq.EmbeddedNATS, registry *discover PolicySource: policyStore, AuthMW: func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - next.ServeHTTP(w, r.WithContext(auth.WithRole(r.Context(), "admin"))) + role := r.Header.Get(testRoleHeader) + if role == "" { + role = "admin" + } + ctx := auth.WithRole(r.Context(), role) + if raw := r.Header.Get(testClaimsHeader); raw != "" { + var claims map[string]any + if err := json.Unmarshal([]byte(raw), &claims); err != nil { + http.Error(w, "bad "+testClaimsHeader, http.StatusBadRequest) + return + } + ctx = auth.WithClaims(ctx, claims) + } + next.ServeHTTP(w, r.WithContext(ctx)) }) }, JS: js, @@ -341,7 +397,7 @@ func buildServer(ch *chInstance, embeddedMQ *mq.EmbeddedNATS, registry *discover } server := httptest.NewServer(api.NewRouter(deps)) - return server, nil + return server, ingestHandler, nil } func mustTempDir() string { diff --git a/tests/integration/testdata/query_types_pin.json b/tests/integration/testdata/query_types_pin.json new file mode 100644 index 00000000..58f2f03f --- /dev/null +++ b/tests/integration/testdata/query_types_pin.json @@ -0,0 +1 @@ +[{"i8":-8,"i16":-16,"i32":-32,"i64":-9007199254740993,"u8":8,"u16":16,"u32":32,"u64":18446744073709551615,"f32":0.1,"f64":0.1,"dec":12.5,"dec64":1.5,"s":"hello","ls":"lc","fs":"ab\u0000\u0000","uu":"11111111-2222-3333-4444-555555555555","en":"a","bl":true,"ip4":"10.0.0.1","ip6":"::1","d":"2026-01-15","dt":"2026-01-15 10:30:00","dt64":"2026-01-15 10:30:00.123","arr":["a","b"],"m":{"k":7},"nn":42,"nnull":null}] diff --git a/tests/integration/timestamp_canonicalization_test.go b/tests/integration/timestamp_canonicalization_test.go deleted file mode 100644 index c0463ed7..00000000 --- a/tests/integration/timestamp_canonicalization_test.go +++ /dev/null @@ -1,163 +0,0 @@ -//go:build integration - -package tests - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/Wave-RF/WaveHouse/internal/discovery" -) - -// TestTimestampCanonicalization_DifferentialAgainstClickHouse pins the #372 -// invariant with ClickHouse itself as the oracle (PR #402 review): for every -// corpus input × timestamp column shape, inserting the raw producer value and -// inserting CanonicalizeTimestamps' output must both succeed or both fail, and -// when both succeed store the same instant — anything else means the rewrite -// changed what ClickHouse stores or accepts. Inserts go through the worker's -// exact HTTP surface (JSONEachRow, date_time_input_format=best_effort), so the -// oracle is the production parse, not a lookalike. -func TestTimestampCanonicalization_DifferentialAgainstClickHouse(t *testing.T) { - colTypes := []struct { - name string - ddl string - }{ - {"datetime", "DateTime"}, - {"datetime_nyc", "DateTime('America/New_York')"}, - {"datetime64_3", "DateTime64(3)"}, - {"datetime64_6_tokyo", "DateTime64(6, 'Asia/Tokyo')"}, - // Precision 9 has its own ceiling (Int64 nanoseconds end 2262-04-11, past - // which an insert fails outright instead of saturating), and precision 0 - // pins the DateTime64-kind rules on a second-granular column. - {"datetime64_9", "DateTime64(9)"}, - {"datetime64_0_utc", "DateTime64(0, 'UTC')"}, - } - - // One entry per spelling family, each found or pinned by differentially - // fuzzing the canonicalizer against a live ClickHouse (PR #402 review). The - // digit runs, number shapes, out-of-range instants, and garbage document the - // pass-through side: they must reach ClickHouse verbatim and get its - // verdict, never a rewrite. - corpus := []any{ - "2026-06-21T04:00:00Z", // canonical already - "2026-06-21T06:30:00+02:30", // offset form - "2026-06-21 04:00:00", // zone-less space form - "2026-06-21T04:00:00", // zone-less T form - "2026-06-21", // date-only - "2026-06-21 04:00:00.123456", // zone-less with fraction - "2026-06-21T04:00:00.9999Z", // fraction beyond column precision - "2026-06-21T04:00:00,999Z", // comma fraction: ISO 8601 yes, ClickHouse no - "1750478400", // 10-digit Unix string - "999999999", // 9-digit Unix string - "1750478400.5", // fractional Unix string: DateTime64-only to ClickHouse - "1750478400.123456789", // ns-exact fraction (float64 would corrupt it) - "1750478400.9999999995", // >9 fraction digits: ClickHouse truncates, never rounds - float64(1750478400), // integer number: seconds to DateTime, *ticks* to DateTime64 - float64(1750478400.5), // non-integer number: ClickHouse rejects for every kind - float64(1750478400500), // epoch-ms number: DateTime64(3)'s natural ticks shape - json.Number("1750478400"), // integer seconds as the production-decoded type - json.Number("1750478400.5"), // non-integer json.Number: pass-through, ClickHouse rejects - json.Number("1750478400123456789"), // 19-digit ns epoch > 2^53: rewritten only on DateTime64(9) - "20260711", // 8 digits: YYYYMMDD to ClickHouse - "20260711150000", // 14 digits: YYYYMMDDhhmmss - "202607111500", // 12 digits: ClickHouse rejects - "1752278400000", // 13 digits: epoch milliseconds to ClickHouse - "1750478400123456", // 16 digits: epoch microseconds to ClickHouse - "2026", // 4 digits: a year to ClickHouse - "1e9", // not a timestamp - "-100", // not a timestamp - "banana", // garbage - "2026-11-01 01:30:00", // DST fall-back: ambiguous local time - "2026-03-08 02:30:00", // DST spring-forward: nonexistent local time - "1960-01-01T00:00:00Z", // pre-range: ClickHouse saturates, spelling-dependently - "2300-06-30 12:30:00", // beyond DateTime64's ceiling, zone-less - } - - for _, ct := range colTypes { - t.Run(ct.name, func(t *testing.T) { - // Independent tables per shape; parallel keeps the six shapes from - // serializing ~380 single-row inserts against the suite timeout. - t.Parallel() - rawTable := createTable(t, "id UInt32, v "+ct.ddl, "ORDER BY id") - canonTable := createTable(t, "id UInt32, v "+ct.ddl, "ORDER BY id") - schema := env(t).registry.Get(rawTable) - require.NotNil(t, schema, "registry must discover the raw table") - - for i, input := range corpus { - id := uint32(i) - rawErr := insertBestEffort(t, rawTable, map[string]any{"id": id, "v": input}) - - canonData := map[string]any{"id": id, "v": input} - discovery.CanonicalizeTimestamps(schema, canonData) - canonErr := insertBestEffort(t, canonTable, canonData) - - if (rawErr == nil) != (canonErr == nil) { - t.Errorf("input %v: asymmetric insertability — raw err=%v, canonicalized (%v) err=%v", - input, rawErr, canonData["v"], canonErr) - continue - } - if rawErr != nil { - continue // both rejected — consistent - } - rawStored := selectInstant(t, rawTable, id) - canonStored := selectInstant(t, canonTable, id) - if !rawStored.Equal(canonStored) { - t.Errorf("input %v: stored instants differ — raw %s vs canonicalized (%v) %s", - input, rawStored.UTC().Format(time.RFC3339Nano), - canonData["v"], canonStored.UTC().Format(time.RFC3339Nano)) - } - } - }) - } -} - -// insertBestEffort inserts one JSONEachRow row over ClickHouse HTTP with -// date_time_input_format=best_effort — the exact settings the ingest worker -// uses (see IngestWorker.insertToClickHouse) — and returns ClickHouse's -// verdict as an error (nil on 2xx). -func insertBestEffort(t *testing.T, table string, row map[string]any) error { - t.Helper() - body, err := json.Marshal(row) - require.NoError(t, err) - - q := url.Values{} - q.Set("database", testCHDatabase) - q.Set("param_target_table", table) - q.Set("query", "INSERT INTO {target_table:Identifier} FORMAT JSONEachRow") - q.Set("date_time_input_format", "best_effort") - - req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, - env(t).chHTTPURL+"?"+q.Encode(), bytes.NewReader(body)) - require.NoError(t, err) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("X-ClickHouse-User", testCHUser) - req.Header.Set("X-ClickHouse-Key", testCHPassword) - - resp, err := http.DefaultClient.Do(req) - require.NoError(t, err) - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode >= 300 { - msg, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return fmt.Errorf("HTTP %d: %s", resp.StatusCode, msg) - } - _, _ = io.Copy(io.Discard, resp.Body) - return nil -} - -// selectInstant reads back the single stored timestamp as a time.Time instant. -func selectInstant(t *testing.T, table string, id uint32) time.Time { - t.Helper() - var v time.Time - require.NoError(t, env(t).chConn.QueryRow(context.Background(), - fmt.Sprintf("SELECT v FROM %s WHERE id = ?", table), id).Scan(&v)) - return v -} diff --git a/tests/integration/typelayer_wire_test.go b/tests/integration/typelayer_wire_test.go new file mode 100644 index 00000000..9d31e84a --- /dev/null +++ b/tests/integration/typelayer_wire_test.go @@ -0,0 +1,167 @@ +//go:build integration + +package tests + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestTypelayerWire_PublishedRowIsTheStoredRow is the one claim the whole +// migration rests on: the bytes WaveHouse publishes — to NATS, to the SSE +// stream, to the DLQ — are the row ClickHouse itself will hold. They are +// produced by the server's own writer at validation time, so a subscriber +// reading the stream and a client reading /v1/query cannot disagree about a +// value, and the worker's INSERT re-parses bytes the server already wrote. +// +// This replaces the timestamp differential oracle (a corpus of ~30 producer +// spellings × 6 column shapes, asserting WaveHouse's canonicalizer landed on +// the same instant as the raw value). That oracle existed to prove a +// hand-written parser matched ClickHouse; there is no longer a second parser to +// disagree, so one end-to-end identity check is the whole remaining claim. +// Deliberately one test, not a corpus. +func TestTypelayerWire_PublishedRowIsTheStoredRow(t *testing.T) { + e := env(t) + ctx := context.Background() + before := time.Now().UTC().Add(-time.Minute) + + table := createTable(t, + "user_id String, "+ + "ts DateTime('UTC'), "+ + "ts_ms DateTime64(3, 'UTC'), "+ + "amount Decimal(10, 2), "+ + "small UInt8, "+ + "tags Array(String), "+ + "maybe Nullable(String)", + "ORDER BY user_id", + ) + + // Every value in a spelling ClickHouse does NOT store it in: an offset + // timestamp, an epoch-tick DateTime64, a string decimal, and an integer past + // the column's width (which wraps — the stored truth, not the producer's). + body := `{"user_id":"erin",` + + `"ts":"2026-06-21T06:00:00+02:00",` + + `"ts_ms":1782014400123,` + + `"amount":"12.50",` + + `"small":256,` + + `"tags":["a","b"],` + + `"maybe":null}` + + resp, err := http.Post( + e.server.URL+"/v1/ingest?table="+url.QueryEscape(table), + "application/json", + strings.NewReader(body), + ) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // 30s upper bound for the worker's 5s batch window plus loaded-runner slack. + require.Eventually(t, func() bool { + var count uint64 + err := e.chConn.QueryRow(ctx, + fmt.Sprintf("SELECT count() FROM %s WHERE user_id = 'erin'", table), + ).Scan(&count) + return err == nil && count == 1 + }, 30*time.Second, 250*time.Millisecond, "row never landed") + + // Read the stored row back in the same format the wire carries, so the two + // are comparable as bytes rather than through two different renderings. + stored := selectJSONCompactRow(t, e.chHTTPURL, + fmt.Sprintf("SELECT user_id, ts, ts_ms, amount, small, tags, maybe FROM %s WHERE user_id = 'erin'", table)) + + wire := publishedWireRow(t, e.server.URL, table, before) + assert.Equal(t, stored, wire, + "the published row and the stored row must be the same values: wire=%v stored=%v", wire, stored) + + // And spot-check that these really are the coerced values, not the + // producer's spellings passed through. + require.Len(t, wire, 7) + assert.Equal(t, "2026-06-21 04:00:00", wire[1], "offset applied, rendered in the column's zone") + assert.Equal(t, "2026-06-21 04:00:00.123", wire[2], "ticks at the column's precision") + assert.EqualValues(t, 0, wire[4], "256 into a UInt8 wraps — the stored truth") +} + +// publishedWireRow replays the table's SSE stream from before the insert, so +// the test reads the published bytes without racing the publish. +func publishedWireRow(t *testing.T, serverURL, table string, since time.Time) []any { + t.Helper() + req, err := http.NewRequest(http.MethodGet, + serverURL+"/v1/stream?table="+url.QueryEscape(table)+ + "&since="+url.QueryEscape(since.Format(time.RFC3339Nano)), nil) + require.NoError(t, err) + req.Header.Set("Accept", "text/event-stream") + + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + cells, ok := firstDataRow(t, resp.Body) + require.True(t, ok, "no data frame arrived on the replay stream") + return cells +} + +// firstDataRow reads SSE frames until one carries a row, and returns its cells. +func firstDataRow(t *testing.T, body interface{ Read([]byte) (int, error) }) ([]any, bool) { + t.Helper() + buf := make([]byte, 0, 8192) + chunk := make([]byte, 4096) + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + n, err := body.Read(chunk) + if n > 0 { + buf = append(buf, chunk[:n]...) + for _, line := range strings.Split(string(buf), "\n") { + payload, found := strings.CutPrefix(line, "data: ") + if !found { + continue + } + var frame struct { + Row []any `json:"row"` + } + if json.Unmarshal([]byte(payload), &frame) == nil && frame.Row != nil { + return frame.Row, true + } + } + } + if err != nil { + return nil, false + } + } + return nil, false +} + +// selectJSONCompactRow reads one row back through ClickHouse's HTTP interface in +// JSONCompactEachRow — the same writer that produced the published row, so the +// two renderings are comparable without a second interpretation step. +func selectJSONCompactRow(t *testing.T, chHTTPURL, query string) []any { + t.Helper() + q := url.Values{} + q.Set("database", testCHDatabase) + q.Set("query", query+" FORMAT JSONCompactEachRow") + + req, err := http.NewRequest(http.MethodGet, chHTTPURL+"?"+q.Encode(), nil) + require.NoError(t, err) + req.Header.Set("X-ClickHouse-User", testCHUser) + req.Header.Set("X-ClickHouse-Key", testCHPassword) + + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + var cells []any + require.NoError(t, json.NewDecoder(resp.Body).Decode(&cells)) + return cells +}