From 2a4e2c7ddafe85d7899873dbdd3fdd7b876deca8 Mon Sep 17 00:00:00 2001 From: Juniper Bevensee Date: Tue, 20 Oct 2026 16:19:40 +1300 Subject: [PATCH 1/4] refactor(sanitization): call the canonical gate The vendored check_sanitization.py had drifted from the other repos' copies. CI now calls the canonical reusable workflow in cyborg-garden/swarm-map, pinned to sanitization-gate-v1.0.0, and this repo keeps only its sanitize.config.json. scripts/check_sanitization.py becomes a launcher that fetches the canonical script at the same tag, so local runs and docs keep working. The gate's self-tests move to swarm-map; a thin test here keeps the two pins in step. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/sanitization.yml | 64 +++---- CONTRIBUTING.md | 16 +- README.md | 4 +- SETUP.md | 38 ++-- docs/promotion-and-upstream.md | 11 +- sanitize.config.json | 4 +- scripts/check_sanitization.py | 285 +++++------------------------ tests/test_check_sanitization.py | 154 ---------------- tests/test_sanitization_pin.py | 50 +++++ 9 files changed, 158 insertions(+), 468 deletions(-) delete mode 100644 tests/test_check_sanitization.py create mode 100644 tests/test_sanitization_pin.py diff --git a/.github/workflows/sanitization.yml b/.github/workflows/sanitization.yml index 434057f..6a96488 100644 --- a/.github/workflows/sanitization.yml +++ b/.github/workflows/sanitization.yml @@ -1,46 +1,36 @@ name: sanitization -# The commons gate. Runs on every PR. Scans ONLY the diff (fast, never -# flaky-blocks a contributor on unrelated pre-existing content) — the full-tree -# audit belongs in a scheduled/gardener lane, not per-PR. +# The sanitization gate (the Alembic gate) is CANONICAL: one script, one reusable +# workflow, both in cyborg-garden/swarm-map. This repo keeps only what is its own: +# sanitize.config.json (what a "particular" means here, which paths carry +# content) and this caller. Do not vendor the script back in — the per-repo +# copies drifted apart, each missing a protection another had. # -# SECURITY NOTE for when you open this to untrusted forks: switch the trigger to -# `pull_request_target` and check out the PR head explicitly, so the build-time -# ANTHROPIC_API_KEY is never exposed to injected PR code. Until then, keep -# `pull_request` (key only runs on trusted branches). +# Runs on every PR, scanning ONLY the diff (the full PR range, three-dot). The +# full-tree audit is a separate lane (run the canonical gate with --all locally, or add a caller with mode: full-tree). +# +# The reusable workflow runs, as separate steps so a red check names its cause: +# the gate self-tests, a config check, the deterministic secrets/PII layer +# (always, including fork PRs), and the semantic particulars layer with +# --require-semantic (fails closed on a missing or broken OPENROUTER_API_KEY). +# Fork PRs get no secrets, so they get the deterministic layer plus a loud +# "partial coverage" warning: a maintainer reviews particulars by hand. Making +# the semantic layer run on forks needs pull_request_target, which is a +# deliberate maintainer decision and intentionally NOT made here. +# +# Pinned to a release tag, never @main. Bump this pin and GATE_TAG in +# scripts/check_sanitization.py together (tests/test_sanitization_pin.py). on: pull_request: +permissions: + contents: read + jobs: sanitize: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install deps (semantic layer + self-tests) - run: pip install anthropic pytest - - - name: Scanner self-tests (deterministic, no key needed) - run: python -m pytest tests/test_check_sanitization.py -q - - - name: Sanitization gate (diff) - # The semantic (LLM) layer runs automatically when ANTHROPIC_API_KEY is set - # as a repo Actions secret — strongly recommended, since it's the layer that - # catches engagement particulars the deterministic regexes can't. Without the - # key it skips with a loud warning (deterministic layer still hard-fails on - # secrets/PII). To make the semantic layer MANDATORY once your key is wired, - # append `--require-semantic` below: the gate then fails closed if the key is - # ever missing instead of silently passing deterministic-only. - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: | - BASE="origin/${{ github.base_ref }}" - git diff --name-only "$BASE...HEAD" \ - | xargs -r python scripts/check_sanitization.py + uses: cyborg-garden/swarm-map/.github/workflows/sanitization-gate.yml@sanitization-gate-v1.0.0 + with: + mode: pr + secrets: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c76c183..42751e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,14 +60,14 @@ flag is **not** an automatic rejection — it routes the PR to a human maintaine makes the call. If your PR is flagged and you believe it's clean, say so in the PR description; a maintainer reviews the flagged content and decides. -The semantic layer needs `ANTHROPIC_API_KEY`. Set it as a repo Actions secret (this is -on the onboarding checklist) so the layer runs in CI — it's the part that catches -engagement particulars the regexes can't. Without the key it skips with a loud warning -(the deterministic layer still hard-fails on secrets/PII), so local/offline runs work. -**Once your key is wired, make the semantic layer mandatory** by appending -`--require-semantic` to the gate step in `.github/workflows/sanitization.yml`: the gate -then fails closed if the key is ever missing, instead of silently passing -deterministic-only. +The semantic layer needs `OPENROUTER_API_KEY`, set as a repo Actions secret (this is +on the onboarding checklist) — it's the part that catches engagement particulars the +regexes can't. In CI the layer is **mandatory**: the canonical gate (a reusable workflow +in `cyborg-garden/swarm-map`, called from `.github/workflows/sanitization.yml`) runs it +with `--require-semantic`, so a missing or broken key fails closed instead of silently +passing deterministic-only. Fork PRs get no secrets, so they get the deterministic layer +plus a loud partial-coverage warning. Locally, without the key, the semantic layer skips +with a loud warning so offline runs still work. To tune what counts as a particular for this domain, edit `sanitize.config.json` — that is the single configuration surface for the gate. See the inline comments in that diff --git a/README.md b/README.md index 5c16cbf..9ca9274 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ but the scaffolding — base image, instance bootstrap, HSM consumption — assu Click **"Use this template"** on GitHub to create your private repo, then follow the instantiation checklist in [SETUP.md](SETUP.md). -The checklist covers: renaming placeholders, filling `sanitize.config.json`, wiring `ANTHROPIC_API_KEY` in CI, and verifying the gate runs clean before your first real commit. +The checklist covers: renaming placeholders, filling `sanitize.config.json`, wiring `OPENROUTER_API_KEY` in CI, and verifying the gate runs clean before your first real commit. --- @@ -64,7 +64,7 @@ You build under real conditions in your private instance. When a capability prov | `docker/Dockerfile.{{SLUG}}` | System-level deps (binaries, libraries) for the agent image | Only if you need system deps | | `engagements/` | Per-engagement working data — gitignored, stays local | N/A — ignored by construction | | `instance/` `.overlay/` | Operator-local overrides, instance-specific config | N/A — ignored by construction | -| `scripts/check_sanitization.py` | The sanitization gate — runs in CI and via gardener | No — leave alone | +| `scripts/check_sanitization.py` | Local launcher for the canonical sanitization gate (in `cyborg-garden/swarm-map`; CI calls it as a reusable workflow) | No — leave alone | | `.github/workflows/sanitization.yml` | CI gate | No — leave alone | | `.gitignore` | Instance-overlay privacy model | No — leave alone | | `sanitize.config.json` | **The one tuning surface** — teach the gate your domain | Yes — required on setup | diff --git a/SETUP.md b/SETUP.md index 6a4f44a..9ebe520 100644 --- a/SETUP.md +++ b/SETUP.md @@ -82,8 +82,8 @@ contains a `$comment`. "user@example", "127.0.0.1", "0.0.0.0" - // add any known-safe strings that the regex layer would otherwise hit — - // exact line substrings, e.g. a placeholder email in a README code block + // add any known-safe values that the regex layer would otherwise hit — + // matched against the DETECTED VALUE, not the whole line ] } } @@ -97,24 +97,19 @@ conservative and fail closed. Only `flag_examples`, `do_not_flag_examples`, and ## 3. Add the CI secret -The semantic layer in CI requires an Anthropic API key. Without it the workflow -logs a loud warning and skips the semantic check. +The semantic layer in CI requires an OpenRouter API key, and it is **mandatory**: +`.github/workflows/sanitization.yml` calls the canonical gate (a reusable workflow +in `cyborg-garden/swarm-map`), which runs `--require-semantic`. Until the secret +exists, every same-repo PR's `sanitization` check fails closed. Fork PRs never get +secrets, so they run the deterministic layer plus a loud partial-coverage warning. 1. Go to **Settings → Secrets and variables → Actions → New repository secret**. -2. Name: `ANTHROPIC_API_KEY`, value: your key. +2. Name: `OPENROUTER_API_KEY`, value: your key. 3. Use a **dedicated, capped key** (a separate key per package is easy to rotate - and lets you monitor usage per package in the Anthropic console). + and lets you monitor usage per package). -If you want CI to hard-fail when the key is missing (e.g. to catch misconfigured -forks), edit `.github/workflows/sanitization.yml` and add `--require-semantic` to -the gate step: - -```yaml -run: | - BASE="origin/${{ github.base_ref }}" - git diff --name-only "$BASE...HEAD" \ - | xargs -r python scripts/check_sanitization.py --require-semantic -``` +Do not copy the gate script into this repo. `scripts/check_sanitization.py` is a +launcher that fetches the canonical script at the tag pinned in the workflow. --- @@ -174,20 +169,21 @@ Read `docs/privacy-and-visibility.md` before flipping to public. The short versi ## 7. Verify the gate locally -Install the test dep and run the deterministic layer offline (no API key needed): +Run the deterministic layer locally (no API key needed; the launcher fetches the +pinned canonical gate once, or set `SANITIZE_GATE_PATH` to a local copy): ```bash -pip install anthropic # for the semantic layer pip install pytest # if not already installed -# Run the scanner self-tests (deterministic, no key needed): +# Check the gate pin and config (no key needed): python -m pytest tests/ -q +python scripts/check_sanitization.py --check-config # Full-tree audit, deterministic only (no key): python scripts/check_sanitization.py --full-tree # Full-tree audit with the semantic layer: -export ANTHROPIC_API_KEY=sk-ant-... +export OPENROUTER_API_KEY=... python scripts/check_sanitization.py --full-tree ``` @@ -197,7 +193,7 @@ Expected output when clean: ok hermes-skill/SKILL.md ok hermes-plugin/your_tool.py ... -sanitization: clean. +sanitization: clean (both layers ran). ``` A `SECRET/PII` line is a hard fail — remove the content before any commit. diff --git a/docs/promotion-and-upstream.md b/docs/promotion-and-upstream.md index 85b2aab..1593503 100644 --- a/docs/promotion-and-upstream.md +++ b/docs/promotion-and-upstream.md @@ -50,15 +50,16 @@ if the technique only worked once, it hasn't earned generalization yet. `main`. 3. **Let the gate run.** The sanitization gate - (`scripts/check_sanitization.py`, wired into - `.github/workflows/sanitization.yml`) runs on the diff: + (canonical in `cyborg-garden/swarm-map`, called from + `.github/workflows/sanitization.yml`; `scripts/check_sanitization.py` is a + local launcher for it) runs on the diff: - **Layer 1 — deterministic** (secrets/PII: API keys, private keys, emails, phones, IPs). No dependencies, fails closed, **hard fail on any hit.** A red deterministic check is a stop, not a discussion. - **Layer 2 — semantic** (use-case particulars, per your - `sanitize.config.json`). Advisory → routes to human review. Skipped with a - warning if no `ANTHROPIC_API_KEY` is present; CI can force it with - `--require-semantic`. + `sanitize.config.json`). Advisory → routes to human review. Needs + `OPENROUTER_API_KEY`; CI runs it with `--require-semantic`, so a missing + key fails closed (locally it skips with a warning). 4. **Review and merge.** A maintainer review is the discipline. If you're a solo operator, that's a self-review — and that's fine, because the deterministic diff --git a/sanitize.config.json b/sanitize.config.json index 010ee5d..3a66673 100644 --- a/sanitize.config.json +++ b/sanitize.config.json @@ -28,11 +28,11 @@ "tool, API, library, or product names", "well-known public reference material used illustratively" ], - "model": "claude-opus-4-8" + "model": "moonshotai/kimi-k3" }, "deterministic": { - "$comment": "The credential/PII layer runs with NO external dependencies and FAILS CLOSED. These patterns are conservative defaults; add your own. Allowlist false-positives by exact line substring in 'allow_substrings'.", + "$comment": "The credential/PII layer runs with NO external dependencies and FAILS CLOSED. These patterns are conservative defaults; add your own. Allowlist a false positive by putting a substring of the MATCHED VALUE in 'allow_substrings' — it is matched against the detected value, not the whole line.", "enabled": true, "allow_substrings": [ "noreply@anthropic.com", diff --git a/scripts/check_sanitization.py b/scripts/check_sanitization.py index ab33baa..a79780b 100644 --- a/scripts/check_sanitization.py +++ b/scripts/check_sanitization.py @@ -1,245 +1,52 @@ -"""Two-layer sanitization gate for a use-case agent package. +"""Local launcher for the canonical sanitization gate. NOT a copy of the gate. -Layer 1 (deterministic): scans changed text for credentials / PII — API keys, -private keys, tokens, emails, phones, IPs. NO external dependencies. FAILS CLOSED. -Always runs, even offline / without an API key. +The gate lives in ONE place: cyborg-garden/swarm-map, scripts/check_sanitization.py. +CI runs it through that repo's reusable workflow (.github/workflows/sanitization.yml +here pins it). This launcher fetches the script at the SAME tag and runs it against +this checkout, so `python scripts/check_sanitization.py ...` keeps working locally +and in docs. Every argument is passed through; see the canonical script for modes. -Layer 2 (semantic): sends content-bearing files (skills, souls, docs, prose) to -an LLM that flags operator/engagement *particulars* — names, case IDs, hostnames, -anything that would make a "generic" artifact actually specific to one operator. -Requires ANTHROPIC_API_KEY; skipped with a loud warning if unset (so local runs -work), REQUIRED in CI (pass --require-semantic to fail when the key is missing). +Do not paste the gate in here. Per-repo copies drifted apart, each one quietly +missing a protection another had. To change the gate, change it in swarm-map, +tag it, and bump GATE_TAG here and the `uses: ...@` pin in the workflow together +(tests/test_sanitization_pin.py fails if they disagree). -A flag routes to a human maintainer — it is advisory, not an automatic final -rejection. But the deterministic layer's hits (real secrets/PII) are hard fails. - -Modes: - check_sanitization.py a.md b.py # explicit file list (CI diff mode) - check_sanitization.py --full-tree # every git-tracked file (gardener/audit) - check_sanitization.py --all # alias for --full-tree - -Config: sanitize.config.json at repo root (see that file's comments). +Offline or testing an unreleased gate: SANITIZE_GATE_PATH=/path/to/check_sanitization.py """ -from __future__ import annotations - -import json import os -import re -import subprocess +import runpy import sys - -# ---------------------------------------------------------------- config - -def load_config(root="."): - path = os.path.join(root, "sanitize.config.json") - try: - with open(path, encoding="utf-8") as fh: - return json.load(fh) - except FileNotFoundError: - # Sensible defaults so the gate still runs on a fresh template. - return { - "package_kind": "SHARED, public agent package", - "sensitive_prefixes": ["hermes-skill/", "hermes-plugin/", "docker/SOUL", "docs/"], - "semantic": {"domain_noun": "operational engagement", "flag_examples": [], - "do_not_flag_examples": [], "model": "claude-opus-4-8"}, - "deterministic": {"enabled": True, "allow_substrings": []}, - } - -# ---------------------------------------------------------------- layer 1: deterministic - -# Conservative, low-false-positive credential/PII patterns. Each is (label, regex). -_DETERMINISTIC_PATTERNS = [ - ("anthropic-key", re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}")), - ("openai-key", re.compile(r"sk-(?:proj-)?[A-Za-z0-9]{20,}")), - ("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{30,}")), - ("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")), - ("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), - ("google-api-key", re.compile(r"\bAIza[0-9A-Za-z_\-]{35}\b")), - ("notion-token", re.compile(r"\bntn_[A-Za-z0-9]{20,}")), - ("private-key-block", re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----")), - ("email", re.compile(r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b")), - ("us-phone", re.compile(r"(?, "reasons": []}}.' - ) - - -def _extract_json(text): - start, end = text.find("{"), text.rfind("}") - if start == -1 or end == -1 or end < start: - raise ValueError(f"no JSON object in model reply: {text!r}") - try: - return json.loads(text[start: end + 1]) - except json.JSONDecodeError: - start2 = text.rfind("{") - if start2 != -1 and start2 < end: - return json.loads(text[start2: end + 1]) - raise - - -def assess(content, client, filename, system_prompt, model): - msg = client.messages.create( - model=model, max_tokens=1024, system=system_prompt, - messages=[{"role": "user", "content": f"File: {filename}\n\n---\n{content}\n---"}], - ) - verdict = _extract_json(msg.content[0].text) - if "flagged" not in verdict: - raise ValueError(f"model response missing 'flagged' key: {verdict!r}") - reasons = verdict.get("reasons", []) - if isinstance(reasons, str): - reasons = [reasons] - return {"flagged": bool(verdict["flagged"]), "reasons": list(reasons)} - - -def _make_client(): # pragma: no cover - thin wrapper, mocked in tests - import anthropic - return anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) - -# ---------------------------------------------------------------- file selection - -def select_sensitive_files(paths, prefixes): - """Content-bearing files for the SEMANTIC layer: anything under a sensitive - prefix, plus top-level prose .md (README, CONTRIBUTING).""" - out = [] - for p in paths: - if p.startswith(tuple(prefixes)): - out.append(p) - elif p.endswith(".md") and "/" not in p: - out.append(p) - return out - - -def git_tracked_files(root="."): - """All git-tracked files, for --full-tree mode.""" - res = subprocess.run(["git", "-C", root, "ls-files"], - capture_output=True, text=True, check=True) - return [ln for ln in res.stdout.splitlines() if ln.strip()] - -# ---------------------------------------------------------------- main - -def main(argv, root=".", client_factory=_make_client): - args = list(argv) - require_semantic = "--require-semantic" in args - args = [a for a in args if a != "--require-semantic"] - - cfg = load_config(root) - prefixes = cfg.get("sensitive_prefixes", []) - det_cfg = cfg.get("deterministic", {}) - allow = det_cfg.get("allow_substrings", []) - - if args and args[0] in ("--full-tree", "--all"): - candidates = git_tracked_files(root) - elif args: - candidates = args - else: - print("sanitization: no files given (use a file list or --full-tree).") - return 0 - - # ---- Layer 1: deterministic, over ALL candidate text files (fails closed) - det_hits = {} - skip_paths = tuple(det_cfg.get("skip_paths", [])) - if det_cfg.get("enabled", True): - for rel in candidates: - if skip_paths and rel.startswith(skip_paths): - continue # e.g. tests/ — legitimately holds credential-shaped fixtures - path = os.path.join(root, rel) - if not os.path.isfile(path): - continue - try: - with open(path, encoding="utf-8") as fh: - content = fh.read() - except (UnicodeDecodeError, OSError): - continue # binary / unreadable — skip - hits = scan_deterministic(content, allow) - if hits: - det_hits[rel] = hits - - # ---- Layer 2: semantic, over content-bearing files only - sensitive = select_sensitive_files(candidates, prefixes) - sem_flagged = {} - sem_ran = False - if sensitive: - if os.environ.get("ANTHROPIC_API_KEY"): - sem_ran = True - client = client_factory() - system_prompt = build_system_prompt(cfg) - model = cfg.get("semantic", {}).get("model", "claude-opus-4-8") - for rel in sensitive: - path = os.path.join(root, rel) - if not os.path.exists(path): - continue - with open(path, encoding="utf-8", errors="replace") as fh: - content = fh.read() - verdict = assess(content, client, rel, system_prompt, model) - if verdict["flagged"]: - sem_flagged[rel] = verdict["reasons"] - elif require_semantic: - print("ERROR: --require-semantic set but ANTHROPIC_API_KEY is missing.") - return 2 - else: - print("WARNING: ANTHROPIC_API_KEY unset — semantic layer SKIPPED " - "(deterministic layer still ran). Set the key for full coverage.") - - # ---- report - for rel, hits in det_hits.items(): - print(f"SECRET/PII {rel}:") - for label, text in hits: - shown = text if len(text) < 12 else text[:6] + "…" - print(f" - {label}: {shown}") - for rel, reasons in sem_flagged.items(): - print(f"FLAGGED {rel}:") - for r in reasons: - print(f" - {r}") - clean = [r for r in sensitive if r not in sem_flagged and r not in det_hits] - for rel in clean: - print(f"ok {rel}") - - if det_hits: - print("\nsanitization: credentials/PII detected — HARD FAIL (remove before merge).") - return 1 - if sem_flagged: - print("\nsanitization: possible particulars found — needs human review.") - return 1 - tail = "" if sem_ran else " (semantic layer skipped — deterministic only)" - print(f"\nsanitization: clean{tail}.") - return 0 - - -if __name__ == "__main__": # pragma: no cover - sys.exit(main(sys.argv[1:])) +import urllib.request + +GATE_TAG = "sanitization-gate-v1.0.0" +GATE_URL = ("https://raw.githubusercontent.com/cyborg-garden/swarm-map/" + f"{GATE_TAG}/scripts/check_sanitization.py") + + +def gate_path(): + override = os.environ.get("SANITIZE_GATE_PATH", "").strip() + if override: + return override + cache = os.path.join(os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), + "sanitization-gate", GATE_TAG, "check_sanitization.py") + if not os.path.isfile(cache): + os.makedirs(os.path.dirname(cache), exist_ok=True) + req = urllib.request.Request(GATE_URL, headers={"User-Agent": "sanitization-gate-launcher/1.0"}) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + body = resp.read() + except OSError as exc: + sys.exit(f"ERROR: could not fetch the sanitization gate {GATE_TAG} ({exc}). " + f"Failing closed. Offline? Set SANITIZE_GATE_PATH to a local copy of " + f"swarm-map's scripts/check_sanitization.py.") + tmp = cache + ".part" + with open(tmp, "wb") as fh: + fh.write(body) + os.replace(tmp, cache) + return cache + + +if __name__ == "__main__": + path = gate_path() + sys.argv = [path] + sys.argv[1:] + runpy.run_path(path, run_name="__main__") diff --git a/tests/test_check_sanitization.py b/tests/test_check_sanitization.py deleted file mode 100644 index 7562edb..0000000 --- a/tests/test_check_sanitization.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Tests for the two-layer sanitization gate.""" -import os -import subprocess -import sys - -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) -import check_sanitization as cs # noqa: E402 - - -# ---- Layer 1: deterministic -------------------------------------------------- - -def test_detects_anthropic_key(): - hits = cs.scan_deterministic("token = sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAA", []) - assert any(label == "anthropic-key" for label, _ in hits) - - -def test_detects_private_key_block(): - hits = cs.scan_deterministic("-----BEGIN OPENSSH PRIVATE KEY-----", []) - assert any(label == "private-key-block" for label, _ in hits) - - -def test_detects_real_email_but_allows_allowlisted(): - hits = cs.scan_deterministic("contact jane.doe@realcorp.io now", []) - assert any(label == "email" for label, _ in hits) - allowed = cs.scan_deterministic("from noreply@anthropic.com header", ["noreply@anthropic.com"]) - assert not any(label == "email" for label, _ in allowed) - - -def test_detects_ipv4_but_allows_loopback(): - assert any(l == "ipv4" for l, _ in cs.scan_deterministic("host 203.0.113.45", [])) - assert not any(l == "ipv4" for l, _ in cs.scan_deterministic("bind 127.0.0.1", ["127.0.0.1"])) - - -def test_detects_international_phone(): - # E.164 form with no separators — the us-phone pattern misses these entirely. - assert any(l == "intl-phone" for l, _ in cs.scan_deterministic("contact +442079460958 anytime", [])) - assert any(l == "intl-phone" for l, _ in cs.scan_deterministic("MATILDE_CONTACT=+4915123456789", [])) - # Not a phone: a short +N token (version, diff count) must not trip it. - assert not any(l == "intl-phone" for l, _ in cs.scan_deterministic("rebased +1234 ahead", [])) - - -def test_generic_methodology_is_clean(): - assert cs.scan_deterministic("Structure a handoff: decisions, threads, next step.", []) == [] - - -# ---- file selection ---------------------------------------------------------- - -def test_select_sensitive_files_prefixes_and_toplevel_md(): - paths = ["hermes-skill/SKILL.md", "engine/db.py", "README.md", "docs/guide.md", "src/x.py"] - got = cs.select_sensitive_files(paths, ["hermes-skill/", "docs/"]) - assert set(got) == {"hermes-skill/SKILL.md", "README.md", "docs/guide.md"} - - -# ---- semantic prompt build --------------------------------------------------- - -def test_build_system_prompt_uses_config_domain(): - cfg = {"package_kind": "X", "semantic": {"domain_noun": "patient case", - "flag_examples": ["patient names"], "do_not_flag_examples": ["clinical methodology"]}} - p = cs.build_system_prompt(cfg) - assert "patient case" in p and "patient names" in p and "clinical methodology" in p - - -# ---- main() integration with a fake client ----------------------------------- - -class _FakeMsg: - def __init__(self, text): - self.content = [type("C", (), {"text": text})()] - - -class _FakeClient: - def __init__(self, verdict_text): - self._t = verdict_text - self.messages = self - - def create(self, **kw): - return _FakeMsg(self._t) - - -def _write(root, rel, body): - path = os.path.join(root, rel) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as fh: - fh.write(body) - - -def test_main_hard_fails_on_secret(tmp_path, monkeypatch): - root = str(tmp_path) - _write(root, "sanitize.config.json", '{"deterministic":{"enabled":true,"allow_substrings":[]}}') - _write(root, "hermes-skill/SKILL.md", "key sk-ant-api03-BBBBBBBBBBBBBBBBBBBBBBBB") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - rc = cs.main(["hermes-skill/SKILL.md"], root=root) - assert rc == 1 # deterministic hard fail regardless of semantic - - -def test_main_semantic_flag(tmp_path): - root = str(tmp_path) - _write(root, "sanitize.config.json", - '{"sensitive_prefixes":["hermes-skill/"],"deterministic":{"enabled":true,"allow_substrings":[]},' - '"semantic":{"domain_noun":"case","flag_examples":[],"do_not_flag_examples":[],"model":"m"}}') - _write(root, "hermes-skill/SKILL.md", "Operation Bluebird targeted Acme Corp on 2024-01-02.") - os.environ["ANTHROPIC_API_KEY"] = "test" - try: - rc = cs.main(["hermes-skill/SKILL.md"], root=root, - client_factory=lambda: _FakeClient('{"flagged": true, "reasons": ["case codename"]}')) - finally: - del os.environ["ANTHROPIC_API_KEY"] - assert rc == 1 - - -def test_main_clean_passes(tmp_path): - root = str(tmp_path) - _write(root, "sanitize.config.json", - '{"sensitive_prefixes":["hermes-skill/"],"deterministic":{"enabled":true,"allow_substrings":[]},' - '"semantic":{"domain_noun":"case","flag_examples":[],"do_not_flag_examples":[],"model":"m"}}') - _write(root, "hermes-skill/SKILL.md", "Generic methodology: grade sources A-F.") - os.environ["ANTHROPIC_API_KEY"] = "test" - try: - rc = cs.main(["hermes-skill/SKILL.md"], root=root, - client_factory=lambda: _FakeClient('{"flagged": false, "reasons": []}')) - finally: - del os.environ["ANTHROPIC_API_KEY"] - assert rc == 0 - - -def test_main_require_semantic_fails_without_key(tmp_path, monkeypatch): - root = str(tmp_path) - _write(root, "sanitize.config.json", '{"sensitive_prefixes":["hermes-skill/"]}') - _write(root, "hermes-skill/SKILL.md", "generic text") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - rc = cs.main(["--require-semantic", "hermes-skill/SKILL.md"], root=root) - assert rc == 2 - - -def test_skip_paths_excludes_fixtures_from_deterministic(tmp_path, monkeypatch): - root = str(tmp_path) - _write(root, "sanitize.config.json", - '{"deterministic":{"enabled":true,"allow_substrings":[],"skip_paths":["tests/"]}}') - _write(root, "tests/fixtures.py", "fake AKIA1234567890ABCDEF in a test fixture") - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - rc = cs.main(["tests/fixtures.py"], root=root) - assert rc == 0 # skipped, not flagged - - -def test_full_tree_mode_scans_tracked_files(tmp_path, monkeypatch): - root = str(tmp_path) - subprocess.run(["git", "init", "-q", root], check=True) - subprocess.run(["git", "-C", root, "config", "user.email", "t@t"], check=True) - subprocess.run(["git", "-C", root, "config", "user.name", "t"], check=True) - _write(root, "sanitize.config.json", '{"deterministic":{"enabled":true,"allow_substrings":[]}}') - _write(root, "notes.md", "leaked AKIA1234567890ABCDEF here") - subprocess.run(["git", "-C", root, "add", "-A"], check=True) - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - rc = cs.main(["--full-tree"], root=root) - assert rc == 1 # AKIA key caught in full-tree sweep diff --git a/tests/test_sanitization_pin.py b/tests/test_sanitization_pin.py new file mode 100644 index 0000000..7f138e6 --- /dev/null +++ b/tests/test_sanitization_pin.py @@ -0,0 +1,50 @@ +"""The sanitization gate is canonical (cyborg-garden/swarm-map); this repo keeps +only its sanitize.config.json, a caller workflow and a local launcher. The one +thing that can drift here is the pin: the workflow's `uses: ...@` and the +launcher's GATE_TAG must name the same release, or local runs and CI would judge +the same diff with different gates.""" +import json +import os +import re + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +USES = re.compile(r"uses:\s*cyborg-garden/swarm-map/\.github/workflows/sanitization-gate\.yml@(\S+)") + + +def _workflow_pins(): + pins = {} + wf_dir = os.path.join(ROOT, ".github", "workflows") + for name in sorted(os.listdir(wf_dir)): + with open(os.path.join(wf_dir, name), encoding="utf-8") as fh: + for tag in USES.findall(fh.read()): + pins.setdefault(tag, []).append(name) + return pins + + +def _launcher_tag(): + with open(os.path.join(ROOT, "scripts", "check_sanitization.py"), encoding="utf-8") as fh: + return re.search(r'^GATE_TAG = "([^"]+)"', fh.read(), re.M).group(1) + + +def test_the_gate_is_called_and_pinned_to_a_release_tag(): + pins = _workflow_pins() + assert pins, "no workflow calls the canonical sanitization gate" + for tag in pins: + assert re.fullmatch(r"sanitization-gate-v\d+\.\d+\.\d+", tag), ( + f"pin the gate to a sanitization-gate-vX.Y.Z tag, not {tag!r}") + + +def test_launcher_and_workflows_pin_the_same_release(): + assert set(_workflow_pins()) == {_launcher_tag()} + + +def test_no_vendored_copy_of_the_gate(): + """The launcher must stay a launcher: no detector patterns pasted back in.""" + with open(os.path.join(ROOT, "scripts", "check_sanitization.py"), encoding="utf-8") as fh: + assert "_DETERMINISTIC_PATTERNS" not in fh.read() + + +def test_config_is_valid_json_with_sensitive_prefixes(): + with open(os.path.join(ROOT, "sanitize.config.json"), encoding="utf-8") as fh: + cfg = json.load(fh) + assert cfg.get("sensitive_prefixes"), "sanitize.config.json must declare sensitive_prefixes" From 2e1da956c05e9d1b119525ad634b55b9872bc55b Mon Sep 17 00:00:00 2001 From: Juniper Bevensee Date: Tue, 20 Oct 2026 16:28:19 +1300 Subject: [PATCH 2/4] docs(sanitization): note the gate tag must exist A pin to a missing sanitization-gate tag makes the caller job fail before it starts, with no check on the PR, so the PR can look green. Say so beside the pin and ask for the check to be required. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/sanitization.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/sanitization.yml b/.github/workflows/sanitization.yml index 6a96488..53acc19 100644 --- a/.github/workflows/sanitization.yml +++ b/.github/workflows/sanitization.yml @@ -29,6 +29,9 @@ permissions: jobs: sanitize: + # The pinned tag MUST exist in cyborg-garden/swarm-map. If it does not, this + # job fails at 0s with no check shown on the PR, which can look green. Keep + # this repo's sanitization check REQUIRED on main so that case blocks merges. uses: cyborg-garden/swarm-map/.github/workflows/sanitization-gate.yml@sanitization-gate-v1.0.0 with: mode: pr From 71ec7ab90821eaef66a00d51e4d580237c9cb7be Mon Sep 17 00:00:00 2001 From: Juniper Bevensee Date: Tue, 20 Oct 2026 16:39:25 +1300 Subject: [PATCH 3/4] docs(sanitization): describe exact-match allowlisting The canonical gate now matches allow entries exactly, refuses config that disables or narrows layer 1, and scans every PR with the base branch's config. Update the config comments and docs to match; in osint-engine, replace the '+1555' prefix entry with allow_fiction_phones, since prefixes no longer excuse anything. Co-Authored-By: Claude Opus 5.5 (1M context) --- sanitize.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanitize.config.json b/sanitize.config.json index 3a66673..6595b9c 100644 --- a/sanitize.config.json +++ b/sanitize.config.json @@ -32,7 +32,7 @@ }, "deterministic": { - "$comment": "The credential/PII layer runs with NO external dependencies and FAILS CLOSED. These patterns are conservative defaults; add your own. Allowlist a false positive by putting a substring of the MATCHED VALUE in 'allow_substrings' — it is matched against the detected value, not the whole line.", + "$comment": "The credential/PII layer runs with NO external dependencies and FAILS CLOSED. These patterns are conservative defaults; add your own. Allowlist a false positive by putting the EXACT detected value in 'allow_substrings' (despite the key's name, entries are never substrings). The only wider forms: an email domain such as 'example.com' excuses addresses at exactly that domain, and a three-octet prefix such as '198.51.100.' excuses that /24. Layer 1 cannot be switched off, and skip_paths may name only test-fixture paths.", "enabled": true, "allow_substrings": [ "noreply@anthropic.com", From 9ec9dc57a5ddca2249ba5a0894e7373053b5b890 Mon Sep 17 00:00:00 2001 From: Juniper Bevensee Date: Tue, 20 Oct 2026 16:53:48 +1300 Subject: [PATCH 4/4] chore(github): add CODEOWNERS for CI and the gate config Route changes to .github/, sanitize.config.json and the gate launcher to the owner for review. Advisory only: no branch protection requires code-owner review today, so this blocks nothing. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/CODEOWNERS | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..57396aa --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# CI (including the sanitization gate's caller), the gate's per-repo config, and +# the gate itself need the owner's review. Advisory today: no branch protection +# requires code-owner review, so this blocks nothing. +/.github/ @juniperbevensee +/sanitize.config.json @juniperbevensee +/scripts/check_sanitization.py @juniperbevensee