diff --git a/.claude/rules/capability-registry.md b/.claude/rules/capability-registry.md index 6088e3e5..0306d7b7 100644 --- a/.claude/rules/capability-registry.md +++ b/.claude/rules/capability-registry.md @@ -28,3 +28,13 @@ Before building any of these, check if it already exists. Use the existing imple | Pareto scoring | engine/scoring/pareto.py | multi-objective optimization | | Convergence loop | engine/feedback/convergence.py | `ConvergenceLoop` | | Label sanitization | engine/utils/security.py | `sanitize_label()` | +| CEG → Gate → EIE egress (the ONLY peer egress) | engine/gate_egress.py | `request_enrichment()`, `emit_graph_inference_result()` | +| Inference outputs → EIE wire shape | engine/gate_egress.py | `build_inference_outputs()` (0.55 floor, matches EIE) | +| Domain database provisioning | engine/graph/driver.py | `GraphDriver.ensure_database()` (flag: `auto_create_domain_database`) | +| Database-name validation for DDL | engine/utils/security.py | `sanitize_database_name()` — `sanitize_label()` forbids the dashes domain ids use | +| Gate run-time values as `$params` | engine/gates/types/all_gates.py | `BaseGate._bind_param()` / `BaseGate.query_params` | +| Cypher injection scanner (C-009) | tools/cypher_lint.py | `scan_tree()` — AST f-string role classifier | +| Inference rule execution | engine/inference_rule_registry.py | `execute_rule()`, `list_registered_rules()` | + +Do not add a second outbound client. `engine/gate_egress.py` is the only +CEG → peer egress; Gate resolves the destination from the action. diff --git a/.claude/rules/feature-flags.md b/.claude/rules/feature-flags.md index ce168d68..169ffa55 100644 --- a/.claude/rules/feature-flags.md +++ b/.claude/rules/feature-flags.md @@ -40,6 +40,15 @@ All in Settings class, controllable via env vars. Contract 21: every behavioral | strict_tenant_database | False | Require explicit `database=` on GraphDriver calls; no implicit 'neo4j' fallback (W7-01) | | require_sdk_chassis_in_prod | False | Fail startup if `L9_CHASSIS != sdk` when `l9_env == prod` (W7-02) | +## Constellation Seam (EIE ↔ Gate ↔ CEG) +| Flag | Default | Purpose | +|------|---------|---------| +| auto_enrich_via_gate | False | Dispatch `enrich` to EIE through Gate (spends EIE budget) | +| graph_inference_feedback_enabled | False | Emit `graph-inference-result` to EIE through Gate (EIE-008 / CEG-006) | +| auto_create_domain_database | False | Provision the tenant domain database on first use (CEG-008; Enterprise-only CREATE DATABASE) | +| health_api_enabled | False | Expose engine/health/api.py via admin health_* subactions (CEG-006) | +| unvalidated_domain_packs_enabled | False | Serve the 5 migrated packs whose gates do not compile to executable Cypher (CEG-009) | + ## Entity Resolution | Flag | Default | Purpose | |------|---------|---------| diff --git a/.claude/rules/subsystems.md b/.claude/rules/subsystems.md index ec6ed096..9a0491a9 100644 --- a/.claude/rules/subsystems.md +++ b/.claude/rules/subsystems.md @@ -28,6 +28,15 @@ All handlers: `async def handle_*(tenant: str, payload: dict) -> dict` | calibration_run | Score calibration vs expected ranges | | score_feedback | Compute weight adjustment proposal | | apply_weight_proposal | Apply proposed weight change | +| health_assess | AI-readiness assessment for one entity (`engine/health/api.py`) | +| health_batch_assess | Incremental batch readiness scan with cost ceilings | +| health_report | Health report for an entity (Seed tier) | +| emit_inference_feedback | Run inference rules and send `graph-inference-result` to EIE via Gate (flag: `graph_inference_feedback_enabled`) | + +CEG-006: the three `health_*` subactions are what makes `engine/health/api.py` +reachable. It was imported by nothing, so `trigger_reenrichment_v2` — and with +it the whole CEG → Gate → EIE enrichment request — had no trigger any inbound +packet could reach. ## Dependency Map ``` diff --git a/.env.template b/.env.template index 67f80c57..0d163756 100644 --- a/.env.template +++ b/.env.template @@ -21,9 +21,6 @@ NEO4J_USERNAME=neo4j NEO4J_PASSWORD=l9-dev-password NEO4J_POOL_SIZE=20 -── Redis Cache ───────────────────────────────────────────── -REDIS_URL=redis://redis:6379/0 - ── API Configuration ─────────────────────────────────────── API_KEY=dev-key-sha256-not-for-production LOG_LEVEL=debug @@ -47,7 +44,6 @@ LOCAL DEVELOPMENT ACCESS (from host machine, not Docker) API: http://localhost:8000/v1/health Neo4j Browser: http://localhost:7475 Neo4j Bolt: bolt://localhost:7688 -Redis: localhost:6379 Neo4j Credentials: User: neo4j Password: l9-dev-password @@ -68,11 +64,16 @@ KGE_EMBEDDING_DIM=300 GATE_URL=http://gate:9000 GATE_ADMIN_TOKEN=your-gate-admin-token-here L9_NODE_NAME=graph -L9_NODE_SPEC_PATH=engine/spec.yaml GATE_REGISTRATION_ENABLED=true GATE_REGISTER_OVERWRITE=true GATE_REGISTER_RETRIES=3 GATE_CLIENT_TIMEOUT_SECONDS=30.0 +# CEG-003: this is the name the SDK reads (gate/config.py: +# spec_path=os.getenv("GATE_NODE_SPEC_PATH", "engine/spec.yaml")). An +# L9_NODE_SPEC_PATH line sat above it with no reader at all, so an operator +# relocating the spec edited the variable that does nothing and registration +# kept using the default until that path stopped existing — at which point +# register_with_gate swallows FileNotFoundError and returns False. GATE_NODE_SPEC_PATH=engine/spec.yaml # ── SDK chassis (L9_CHASSIS=sdk) ───────────────────────────── @@ -104,7 +105,14 @@ L9_SIGNING_ALGORITHM=hmac-sha256 L9_SIGNING_KEY=change-me-shared-hmac-secret L9_SIGNING_KEY_ID=graph-engine-v1 # ed25519 alternative: L9_SIGNING_ALGORITHM=ed25519 + L9_SIGNING_PRIVATE_KEY -# L9_VERIFYING_KEYS_JSON={"gate-v1":""} +# +# CEG-002: REQUIRED whenever L9_REQUIRE_SIGNATURE=true, not optional. The same +# flag turns on verification of the responses Gate signs, and Gate signs with +# *its* key id, which CEG cannot resolve from its own signing key. Leave this +# unset and every signed Gate response is rejected with "no verifying key +# available for transport signature verification". Under hmac-sha256 the value +# is the shared secret; under ed25519 it is the peer's public key. +L9_VERIFYING_KEYS_JSON={"gate-v1":"change-me-shared-hmac-secret"} # Allowed actions — engine.handlers.ACTION_HANDLERS minus `enrich`: Gate owns # the `enrich` name for EIE, so CEG must never accept or advertise it. diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index e5a4ab6f..3651146a 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -182,15 +182,7 @@ jobs: --health-timeout 5s --health-retries 5 - redis: - image: redis:7-alpine - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + # CEG-007: redis service container removed with the dependency. steps: - name: Checkout code @@ -215,7 +207,6 @@ jobs: - name: Run tests with coverage env: DATABASE_URL: postgresql://${{ vars.TEST_DB_USER || 'test' }}:${{ vars.TEST_DB_PASSWORD || 'test' }}@localhost:5432/${{ vars.TEST_DB_NAME || 'test_db' }} - REDIS_URL: redis://localhost:6379 TESTING: "true" run: PYTHONPATH=. pytest tests/ -v --cov=engine --cov-report=xml --cov-report=term --ignore=tests/e2e diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa28a0f4..85037307 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,17 @@ jobs: python tools/check_deprecated_imports.py --check . echo "✅ No deprecated DomainSpecLoader usage found" + - name: Gate_SDK declares the v1 channel (offline) + run: | + python scripts/validate_sdk_pin.py + + - name: Gate_SDK lock agrees with the v1 channel (networked) + # The structural check above cannot see a stale lock: `v1` moves inside + # Gate_SDK and nothing here changes. This resolves the channel at the + # canonical remote and fails closed if poetry.lock no longer matches. + run: | + python scripts/validate_sdk_pin.py --verify-tag + # PacketEnvelope prohibition is governed by the baseline ratchet # (.github/workflows/baseline-ratchet-caller.yml → l9-ci-core # "Baseline Ratchet / Quarantined Debt", fail-closed). The l9-ci-sdk @@ -258,15 +269,9 @@ jobs: --health-retries 5 # Service only runs if POSTGRES_ENABLED=true (GitHub ignores services with falsy conditions) - redis: - image: ${{ vars.REDIS_IMAGE || 'redis:7-alpine' }} - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 + # CEG-007: the redis service container is gone with the dependency. No + # code under engine/ or chassis/ imports redis, so it started a container + # every run for nothing. steps: - name: Checkout Repository @@ -291,7 +296,6 @@ jobs: - name: Run Tests with Coverage env: DATABASE_URL: postgresql://${{ vars.POSTGRES_USER || 'test_user' }}:test_password@localhost:5432/${{ vars.POSTGRES_DB || 'test_db' }} - REDIS_URL: redis://localhost:6379/0 run: | echo "Running test suite..." PYTHONPATH=. pytest ${{ env.TEST_DIR }} \ diff --git a/.github/workflows/supply-chain.yml b/.github/workflows/supply-chain.yml index c2e2a227..e0f8236e 100644 --- a/.github/workflows/supply-chain.yml +++ b/.github/workflows/supply-chain.yml @@ -64,6 +64,8 @@ jobs: persist-credentials: false - name: Run OpenSSF Scorecard Analysis + # v2.4.4 action.yaml image is docker://ghcr.io/ossf/scorecard-action:v2.4.4 + # Do not pin v2.4.0 — that tag still docker-pulls gcr.io/openssf (billing denied). uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 with: results_file: scorecard.sarif diff --git a/.l9/ci.json b/.l9/ci.json new file mode 100644 index 00000000..ff5d59f1 --- /dev/null +++ b/.l9/ci.json @@ -0,0 +1,4 @@ +{ + "schema": "l9.ci-consumer/v1", + "repo_class": "python" +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a5acb3b5..edd2afd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,6 +78,12 @@ repos: fi; exit 0' language: system pass_filenames: false + - id: cypher-lint + name: make cypher-lint (C-009) + entry: python tools/cypher_lint.py + language: python + pass_filenames: false + types: [python] # L9 contract enforcement (24 invariants, 27 docs) - repo: local diff --git a/AGENTS.md b/AGENTS.md index 8152aa6e..305eda9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Cross-tool agent instructions for the CEG repository. Read by Claude Code, Codex ```bash make setup # Install deps, pre-commit hooks, verify Neo4j -make dev # docker-compose (app + Neo4j + Redis + Prometheus + Grafana) +make dev # docker-compose (app + Neo4j + Postgres) make test # Full pytest suite (unit + integration + compliance) make test-unit # Gate compilation, scoring math, parameter resolution make test-integration # testcontainers-neo4j full pipeline diff --git a/CHANGELOG.md b/CHANGELOG.md index 017fc46e..a6089f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,35 @@ All notable changes to L9 Engine will be documented in this file. Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Security (C-009) +- `tools/cypher_lint.py`: rewritten as an AST scanner that classifies every f-string + interpolation by syntactic role (quoted value, label, property, parameter name, + back-quoted identifier, `LIMIT`/`SKIP`, compiled fragment) instead of gating on a + keyword list; explicit reasoned waivers (`# cypher-lint: allow `) are + printed on every run. +- `engine/gates/compiler.py`, `engine/gates/types/all_gates.py`: query parameter + names are validated with `sanitize_label()`; operators and composite logic pass + through literal allow-lists; `EnumMapGate` mapping keys/values and GDS equipment + type names travel as `$parameters` (`BaseGate._bind_param()` / `query_params`). +- `engine/scoring/*`: dimension aliases validated, numeric spec values cast before + interpolation, `helpfulness`/`importance` builders validate property and + parameter names. +- `engine/utils/security.py`: `cypher_quoted_ident()` removed; `sanitize_database_name()` + added for `CREATE DATABASE`. + +### Fixed +- `GraphDriver.ensure_database()`: concurrent first-use callers await the single + in-flight `CREATE DATABASE` instead of racing past a pre-claimed name into a + database that does not exist yet (CEG-008 follow-up). +- `tests/unit/test_protocol_bodies.py`: enforces the docstring-only Protocol body + contract — any statement after the docstring fails with its location. + +### CI +- The consumer-local `l9-analysis.yml` workflow is not restored; organization + analysis stays owned by `l9-ci-core` (`Analyze (central Core)`). + ## [1.2.0] - 2026-03-10 — KGE Mathematical Core Hardening ### Patches Applied diff --git a/Makefile b/Makefile index 06f64a9c..b95132f1 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # ───────────────────────────────────────────────────────────── .PHONY: dev dev-build dev-down dev-logs dev-restart health -.PHONY: test test-unit test-integration seed shell neo4j-shell +.PHONY: test test-unit test-integration seed shell neo4j-shell cypher-lint # ── Governance ───────────────────────────────────────────── @@ -45,8 +45,6 @@ health:Check all service health @curl -sf http://localhost:8000/v1/health | python -m json.tool || echo "API: DOWN" @echo "── Neo4j ──" @docker exec l9-graph-neo4j cypher-shell -u neo4j -p l9-dev-password "RETURN 'ok'" 2>/dev/null || echo "Neo4j: DOWN" - @echo "── Redis ──" - @docker exec l9-graph-redis redis-cli ping || echo "Redis: DOWN" # ── Testing ──────────────────────────────────────────────── @@ -72,13 +70,10 @@ shell:Python shell inside API container neo4j-shell:Cypher shell into Neo4j docker exec -it l9-graph-neo4j cypher-shell -u neo4j -p l9-dev-password -redis-shell:Redis CLI - docker exec -it l9-graph-redis redis-cli - # ── Local Dev (API outside Docker, DBs in Docker) ───────── -local-dbs:Start only Neo4j + Redis - docker compose up -d neo4j redis +local-dbs:Start only Neo4j + Postgres + docker compose up -d neo4j postgres local-api:Run API locally against Dockerized DBs (SDK chassis; alias of local-api-sdk) $(MAKE) local-api-sdk @@ -86,7 +81,6 @@ local-api:Run API locally against Dockerized DBs (SDK chassis; alias of local-ap local-api-legacy:Run the legacy dict chassis locally (permitted in L9_ENV=dev|local|test) PLASTICOS_NEO4J_URI=bolt://localhost:7687 \ PLASTICOS_NEO4J_PASSWORD=l9-dev-password \ - PLASTICOS_REDIS_URL=redis://localhost:6379/0 \ PLASTICOS_LOG_LEVEL=debug \ L9_LIFECYCLE_HOOK=engine.boot:GraphLifecycle \ L9_CHASSIS=legacy \ @@ -95,7 +89,6 @@ local-api-legacy:Run the legacy dict chassis locally (permitted in L9_ENV=dev|lo local-api-sdk:Run API locally on the SDK chassis (L9_CHASSIS=sdk) PLASTICOS_NEO4J_URI=bolt://localhost:7687 \ PLASTICOS_NEO4J_PASSWORD=l9-dev-password \ - PLASTICOS_REDIS_URL=redis://localhost:6379/0 \ PLASTICOS_LOG_LEVEL=debug \ L9_LIFECYCLE_HOOK=engine.boot:GraphLifecycle \ L9_CHASSIS=sdk \ @@ -199,8 +192,6 @@ deploy-health: guard-vps-host ## Remote healthcheck over SSH (VPS ports may be f @ssh $(SSH_OPTS) $(SSH_TARGET) "curl -sf http://localhost:8000/v1/health && echo" || echo "API: DOWN" @echo "── Neo4j ──" @ssh $(SSH_OPTS) $(SSH_TARGET) "curl -sf http://localhost:7474 >/dev/null" && echo "Neo4j: UP" || echo "Neo4j: DOWN" - @echo "── Redis ──" - @ssh $(SSH_OPTS) $(SSH_TARGET) "docker exec l9-redis-prod redis-cli ping" || echo "Redis: DOWN" # ── Cleanup ──────────────────────────────────────────────── @@ -209,13 +200,16 @@ clean: ## Remove volumes + containers # ── Quality Gates (local, no Docker) ─────────────────────── -.PHONY: lint lint-fix typecheck check +.PHONY: lint lint-fix typecheck check cypher-lint lint: ## Ruff lint + format check (no mutation) + MyPy — matches CI's blocking gate ruff check . ruff format --check . mypy engine/ +cypher-lint: ## C-009: scan generated Cypher for injection vectors + python3 tools/cypher_lint.py + lint-fix: ## Autofix: ruff check --fix + ruff format . (run this when `make lint` fails) ruff check . --fix ruff format . diff --git a/README.md b/README.md index aa412e84..0083fa47 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ One engine. Any vertical. No custom code per domain. git clone && cd l9-engine ./scripts/setup.sh -# 2. Start local stack (Neo4j + Redis + API) +# 2. Start local stack (Neo4j + Postgres + API) ./scripts/dev.sh # 3. Seed sample data @@ -81,8 +81,8 @@ All intelligence features are disabled by default and activated per-domain via Y │ Domain Spec Loader │ │ YAML → Pydantic → Compiled Cypher │ ├──────────────────────────────────────────────────────┤ -│ Neo4j (multi-database) │ Redis (cache/scheduler) │ -└──────────────────────────┴───────────────────────────┘ +│ Neo4j (multi-database) │ +└──────────────────────────────────────────────────────┘ ``` ### Core Concepts @@ -121,8 +121,7 @@ l9-engine/ │ │ ├── pii.py # PII hash/encrypt/redact │ │ └── audit.py # Audit logging │ └── db/ # Database layer -│ ├── neo4j.py # Async Neo4j driver pool -│ └── redis.py # Redis connection +│ └── neo4j.py # Async Neo4j driver pool ├── domains/ # Domain specification packs │ ├── plasticos/ │ ├── mortgage-brokerage/ @@ -153,7 +152,6 @@ l9-engine/ | Component | Version | Required Plugins | |-----------|---------|------------------| | Neo4j | 5.15+ Enterprise | APOC, Graph Data Science (GDS) | -| Redis | 7.x | — | | Python | 3.12+ | — | **Neo4j Plugins:** @@ -174,7 +172,6 @@ Consistent across all L9 repos. Set in `.env` (local) or SSM Parameter Store (pr | `NEO4J_USERNAME` | `neo4j` | Neo4j username | | `NEO4J_PASSWORD` | — | Neo4j password (SSM in prod) | | `NEO4J_DATABASE` | `neo4j` | Default database | -| `REDIS_URL` | `redis://localhost:6379/0` | Redis connection | | `API_PORT` | `8000` | API listen port | | `API_WORKERS` | `4` | Uvicorn workers | | `DOMAINS_ROOT` | `./domains` | Path to domain specs | diff --git a/Readme-Requirements.md b/Readme-Requirements.md index e512527c..35207a32 100644 --- a/Readme-Requirements.md +++ b/Readme-Requirements.md @@ -31,7 +31,6 @@ status: active - `neo4j_uri`, `neo4j_username`, `neo4j_password` → consumed by `dependencies.py` to init `GraphDriver`[^1] - `domains_root` → consumed by `dependencies.py` to init `DomainPackLoader`[^2] -- `redis_url` → lazy-init Redis client - Scoring weights (`w_structural`, `w_geo`, etc.) → match the spec's config reference[^3] - Decay half-lives → match the temporal decay system in `engine-core-modules.py`[^1] @@ -39,7 +38,6 @@ status: active - `get_graph_driver()` → returns the shared async Neo4j driver - `get_domain_loader()` → returns the cached domain pack loader -- `get_redis()` → lazy Redis with graceful degradation - `startup()` / `shutdown()` → called from `create_app()` lifespan **`requirements.txt`** pins every dep from `pyproject.toml` with `>=X,"}. + L9_VERIFYING_KEYS_JSON: ${L9_VERIFYING_KEYS_JSON:?L9_VERIFYING_KEYS_JSON must be set when L9_REQUIRE_SIGNATURE is true} L9_ALLOWED_ACTIONS: match,sync,admin,outcomes,resolve,health,healthcheck L9_MAX_ATTACHMENTS: "0" L9_MAX_ATTACHMENT_SIZE_BYTES: "0" @@ -50,7 +59,6 @@ services: NEO4J_USERNAME: ${NEO4J_USERNAME:-neo4j} NEO4J_PASSWORD: ${NEO4J_PASSWORD:?NEO4J_PASSWORD must be set for production} NEO4J_POOL_SIZE: "50" - REDIS_URL: redis://redis:6379/0 API_SECRET_KEY: ${API_SECRET_KEY:?API_SECRET_KEY must be set for production} LOG_LEVEL: info CORS_ORIGINS: ${CORS_ORIGINS:-[]} @@ -59,11 +67,11 @@ services: KGE_CONFIDENCE_THRESHOLD: "0.3" KGE_EMBEDDING_DIM: "300" PARETO_ENABLED: "true" + # CEG-007: redis was a depends_on: service_healthy here for a dependency + # no code imports. depends_on: neo4j: condition: service_healthy - redis: - condition: service_healthy healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/v1/health')"] interval: 30s @@ -106,28 +114,9 @@ services: retries: 10 restart: always - # ── Redis 7.x ─────────────────────────────────────────── - redis: - image: redis:7-alpine - container_name: l9-redis-prod - ports: - - "6379:6379" - command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru - volumes: - - redis-data:/data - networks: - - l9-net - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - restart: always - volumes: neo4j-data: neo4j-logs: - redis-data: networks: l9-net: diff --git a/docker-compose.yml b/docker-compose.yml index 8f12b4b8..b6d1e80c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,6 @@ # Usage: docker compose up -d # API: http://localhost:8000/v1/health # Neo4j: http://localhost:7474 (browser), bolt://localhost:7687 -# Redis: localhost:6379 # Postgres: localhost:5432 (PacketStore + ComplianceEngine audit-flush pool) services: @@ -39,7 +38,13 @@ services: GATE_ADMIN_TOKEN: ${GATE_ADMIN_TOKEN:-dev-gate-admin-token-not-for-production} GATE_REGISTRATION_ENABLED: "true" L9_NODE_NAME: graph - L9_NODE_SPEC_PATH: engine/spec.yaml + # CEG-003: the SDK reads GATE_NODE_SPEC_PATH (gate/config.py). This was + # L9_NODE_SPEC_PATH, which nothing consumes — registration worked only + # because the SDK's default happens to be engine/spec.yaml and WORKDIR is + # /app. Relocating the spec would have silently had no effect until the + # default path stopped existing, at which point register_with_gate + # swallows FileNotFoundError and returns False. + GATE_NODE_SPEC_PATH: engine/spec.yaml L9_ENVIRONMENT: local L9_SERVICE_NAME: graph-engine L9_SERVICE_VERSION: 1.1.0 @@ -50,6 +55,16 @@ services: L9_SIGNING_ALGORITHM: hmac-sha256 L9_SIGNING_KEY: dev-shared-hmac-secret-not-for-production L9_SIGNING_KEY_ID: graph-engine-v1 + # CEG-002: L9_REQUIRE_SIGNATURE also turns on verification of the + # responses Gate signs, and Gate signs with *its* key id — not CEG's. With + # no verifying-key map this file shipped a node that rejected every signed + # Gate response with "no verifying key available for transport signature + # verification". The E2E only passed because its harness supplied the map. + # HMAC verification uses the same shared secret as signing; under ed25519 + # these are Gate's and CEG's public keys instead. + L9_VERIFYING_KEYS_JSON: >- + {"gate-v1": "dev-shared-hmac-secret-not-for-production", + "graph-engine-v1": "dev-shared-hmac-secret-not-for-production"} # `enrich` deliberately absent: Gate owns that action name for EIE. L9_ALLOWED_ACTIONS: match,sync,admin,outcomes,resolve,health,healthcheck L9_MAX_ATTACHMENTS: "0" @@ -61,8 +76,6 @@ services: NEO4J_USERNAME: neo4j NEO4J_PASSWORD: l9-dev-password NEO4J_POOL_SIZE: "20" - # Redis - REDIS_URL: redis://redis:6379/0 # API API_KEY: dev-key-sha256-not-for-production LOG_LEVEL: debug @@ -88,11 +101,11 @@ services: # Hot-reload: mount source code so changes reflect without rebuild - ./engine:/app/engine:ro - ./domains:/app/domains:ro + # CEG-007: redis was a depends_on: service_healthy here, so the api service + # could not start without a Redis nothing imports. depends_on: neo4j: condition: service_healthy - redis: - condition: service_healthy postgres: condition: service_healthy healthcheck: @@ -135,24 +148,6 @@ services: networks: - l9-graph - # ── Redis 7.x ─────────────────────────────────────────── - redis: - image: redis:7-alpine - container_name: l9-graph-redis - ports: - - "6379:6379" - command: redis-server --maxmemory 128mb --maxmemory-policy allkeys-lru - volumes: - - redis_data:/data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 5 - restart: unless-stopped - networks: - - l9-graph - # ── PostgreSQL 16 (PacketStore + ComplianceEngine audit) ── postgres: image: postgres:16-alpine @@ -183,7 +178,6 @@ volumes: neo4j_data: neo4j_logs: neo4j_plugins: - redis_data: postgres_data: networks: diff --git a/docs/CI_PIPELINE.md b/docs/CI_PIPELINE.md index c0f37ce1..02b75b39 100644 --- a/docs/CI_PIPELINE.md +++ b/docs/CI_PIPELINE.md @@ -46,7 +46,6 @@ status: active test instead of stopping at the first one, while still exiting non-zero (failing the job) if any test fails - PostgreSQL service (postgres:16) -- Redis service (redis:7-alpine) ### Phase 4: Security Scanning - Gitleaks (secret detection) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index c0f76dd2..f92cd909 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -39,7 +39,6 @@ The Graph Cognitive Engine is deployed on Hetzner Cloud. | **Health Check** | `http://178.104.43.11:8000/v1/health` | 8000 | | **Neo4j Browser** | `http://178.104.43.11:7474` | 7474 | | **Neo4j Bolt** | `bolt://178.104.43.11:7687` | 7687 | -| **Redis** | `redis://178.104.43.11:6379` | 6379 | ### SSH Access @@ -65,7 +64,6 @@ L9_LIFECYCLE_HOOK=engine.boot:GraphLifecycle NEO4J_URI=bolt://neo4j:7687 NEO4J_USERNAME=neo4j NEO4J_PASSWORD= -REDIS_URL=redis://redis:6379/0 API_PORT=8000 LOG_LEVEL=info CORS_ORIGINS=[] @@ -113,9 +111,9 @@ ssh root@178.104.43.11 "cd /opt/ceg && docker compose -f docker-compose.prod.yml │ l9-ceg (178.104.43.11) │ ├─────────────────────────────────────────────────────────────┤ │ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ API │ │ Neo4j │ │ Redis │ │ -│ │ :8000 │ │ :7474/7687 │ │ :6379 │ │ +│ ┌─────────────┐ ┌─────────────┐ │ +│ │ API │ │ Neo4j │ │ +│ │ :8000 │ │ :7474/7687 │ │ │ │ │ │ │ │ │ │ │ │ FastAPI + │ │ Graph DB + │ │ Cache + │ │ │ │ Uvicorn │ │ GDS Plugin │ │ Sessions │ │ diff --git a/docs/FEATURE_GATES.md b/docs/FEATURE_GATES.md index 75effa8d..d23aa75e 100644 --- a/docs/FEATURE_GATES.md +++ b/docs/FEATURE_GATES.md @@ -62,6 +62,10 @@ independently of the code default. | Capability Auth (domain-spec model) | `CAPABILITY_AUTH_ENABLED` | `True` | `True` | active | | PostgreSQL Audit Pool | `POSTGRES_DSN` | unset (`None`) | set | active (opt-in, soft dependency — see §7) | | Idea Portfolio Graph | `IDEA_PORTFOLIO_ENABLED` (`idea_portfolio_enabled`) | `False` | unset | dormant; opt-in IdeaOS portfolio reads/hydration | +| Graph Inference Feedback | `GRAPH_INFERENCE_FEEDBACK_ENABLED` (`graph_inference_feedback_enabled`) | `False` | unset | dormant; CEG → Gate → EIE `graph-inference-result` — see §13 | +| Domain Database Provisioning | `AUTO_CREATE_DOMAIN_DATABASE` (`auto_create_domain_database`) | `False` | unset | dormant; create the tenant domain database on first use — see §14 | +| Health API (admin health_* subactions) | `HEALTH_API_ENABLED` (`health_api_enabled`) | `False` | unset | dormant; AI-readiness assess/report surface — see §15 | +| Unvalidated Domain Packs | `UNVALIDATED_DOMAIN_PACKS_ENABLED` (`unvalidated_domain_packs_enabled`) | `False` | unset | dormant; seven packs whose gates do not compile to executable, faithful Cypher — see §16 | | Constellation Orchestration | — | — | — | accepted architectural gap — see §9 | --- @@ -325,6 +329,157 @@ intact. --- +## 13. Graph Inference Feedback (EIE-008 / CEG-006) + +**State**: Dormant +**Flag**: `GRAPH_INFERENCE_FEEDBACK_ENABLED=False` (default off) + +Emits `graph-inference-result` to Enrichment.Inference.Engine through Gate. +EIE advertises that action to Gate and implements the whole consumer side — +packet validation, per-tenant queues, target extraction, a 0.55 confidence +floor, injection into the convergence loop — and nothing in CEG ever produced +the packet, so the loop had a consumer and no producer. Both sides were even +built to the same confidence floor. + +### Prerequisites + +- `GATE_URL` configured and Gate reachable; `graph-inference-result` is owned by + `eie` in Gate's `CANONICAL_ACTION_OWNERS`, so Gate resolves the destination. +- EIE registered with Gate advertising `graph-inference-result`. + +### Activation Steps + +1. Set `GRAPH_INFERENCE_FEEDBACK_ENABLED=true`. +2. Drive the `admin` subaction `emit_inference_feedback` with an `entity`, + an `entity_id`, and optionally a `rules` list (defaults to every registered + inference rule). + +### Validation + +The dispatch result carries `sent_outputs`: how many findings cleared the 0.55 +floor and were actually sent. `status: "skipped"` with +`no_outputs_above_confidence_floor` means nothing qualified and no packet was +sent — an empty `inference_outputs` list is valid to EIE and would cost a Gate +round trip to queue nothing. + +### Rollback + +Set the flag back to `False`. Each emission queues re-enrichment targets in EIE +and therefore spends EIE budget, which is why it ships off — the same reason as +`AUTO_ENRICH_VIA_GATE`. + +--- + +## 14. Domain Database Provisioning (CEG-008) + +**State**: Dormant +**Flag**: `AUTO_CREATE_DOMAIN_DATABASE=False` (default off) + +`match` and `sync` route queries to a Neo4j database named after the domain id. +Neo4j does not create databases implicitly, so on a fresh instance every sync +and match failed with an `ExecutionError` until an operator ran +`CREATE DATABASE` by hand. With this flag on, `GraphDriver` provisions the +database on first use — once per database per process. + +### Prerequisites + +- **Neo4j Enterprise Edition.** `CREATE DATABASE` is an Enterprise + administrative command; Community Edition rejects it. +- Credentials with database administration privileges. + +### Activation Steps + +1. Set `AUTO_CREATE_DOMAIN_DATABASE=true`. +2. No restart of Neo4j is required; the next query against an unprovisioned + domain creates it. + +### Validation + +`Ensured Neo4j database '' exists` is logged on the provisioning call. +A refusal (Community Edition, or missing privilege) is logged as a warning and +does **not** raise: a deployment whose database already exists is never blocked +by a CREATE it is not allowed to run. + +### Rollback + +Set the flag back to `False`. Provisioning stops; a query against an absent +database then raises `DatabaseNotProvisionedError`, which names the missing +database and the exact `CREATE DATABASE` command. That message is present +whether or not the flag is on. + +--- + +## 15. Health API (CEG-006) + +**State**: Dormant +**Flag**: `HEALTH_API_ENABLED=False` (default off) + +Exposes `engine/health/api.py` through the `admin` subactions `health_assess`, +`health_batch_assess` and `health_report`. That module implemented three +handlers and was imported by nothing, so `request_enrichment` — the whole +CEG → Gate → EIE direction — had no trigger an inbound packet could reach. + +`AUTO_ENRICH_VIA_GATE` is not a substitute: it gates only the eventual outbound +Gate request, not assessment, reporting, or conversion-event tracking. The +surface needs a gate of its own, which is what this is. + +### Validation + +With the flag off the subactions return `{"status": "disabled"}` rather than +404-ing, so an operator can tell "switched off" from "not a subaction". + +### Rollback + +Set back to `False`. Note that assess/report append to the conversion-event +store in `engine/health/health_report.py`, which is bounded +(`CONVERSION_EVENT_MAX`) and discards oldest-first. + +--- + +## 16. Unvalidated Domain Packs (CEG-009) + +**State**: Dormant +**Flag**: `UNVALIDATED_DOMAIN_PACKS_ENABLED=False` (default off) + +CEG-009 moved nine domain packs out of a flat `_domain_spec.yaml` shape +the loader never reads into `domains//spec.yaml`. Making them readable +exposed that seven do not compile to executable, faithful Cypher: + +| Pack | Defect | +|---|---| +| `executive-assistant` | `RELATES_TO` fallback + `$85.0` | +| `repo-as-agent` | `RELATES_TO` fallback + `$5` | +| `roofing-company` | `RELATES_TO` fallback + `$1` | +| `aios-god-agent` | `RELATES_TO` fallback | +| `healthcare-referral` | `$1` | +| `legal-discovery` | `strictwhen` on a gate, consumed by no compiler | +| `research-agent` | `strictwhen` on two gates, consumed by no compiler | + +Three root causes, all in the spec-to-Cypher contract rather than in the files: + +1. `type: traversal` gates written with `pattern` and `condition`, which + `GateCompiler` does not consume. With no `edgetype` the compiler falls back + to `RELATES_TO`, an edge no ontology here declares, so the gate is a hard + filter that matches nothing. +2. A scalar `queryparam` (`85.0`, `5`, `1`). `GateSpec.coerce_queryparam_to_str` + turns it into a string; since C-009 validates parameter names like labels + the compiler refuses the gate outright instead of emitting `$85.0`. +3. `strictwhen` on a gate. `GateSpec` declares it and nothing under `engine/` + consumes it, so a gate the author wrote as conditional (`legal-discovery`, + `research-agent`) runs as an unconditional hard filter. + +**Do not turn this on to "see if they work".** They do not: the first serves +zero candidates, the second fails at execution. Fixing them means either +compiler support for `pattern`/`condition` and literal operands, or a declared +query-schema parameter per constant — a schema decision, not a file move. + +`tests/unit/test_domain_pack_shape.py` holds both ends: a discoverable pack +must compile to executable Cypher, and a gated pack that starts compiling +cleanly must be removed from `_DOMAIN_FEATURE_FLAGS`, so this list can only +shrink. + +--- + ## Querying Feature Status Use the `feature_status` admin subaction to get current state of all gates: diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 7d862bfe..ed9c2ff5 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -121,26 +121,62 @@ git commit -m "fix: gate implementation" ### Symptom ``` -❌ Unparameterized value interpolation detected +❌ quoted value interpolation — pass the value as a $parameter File: engine/sync/generator.py:89 -Pattern: f"SET n.status = '{status}'" +Expression: {status} +Pattern: cypher = f"SET n.status = '{status}'" ``` ### Diagnosis -This violates **Contract C-009** (Cypher Injection Prevention). Values must be parameterized. +This violates **Contract C-009** (Cypher Injection Prevention). `tools/cypher_lint.py` +parses every f-string that builds Cypher and classifies each `{expr}` by the text +just before it: + +| Text before `{expr}` | Role | Verdict | +|---|---|---| +| `'` or `"` | data value | always fails — pass it as a `$parameter` | +| `LIMIT` / `SKIP` | bound | always fails — pass it as a `$parameter` | +| `:` | label / relationship type | must be `sanitize_label(...)` or a name derived from it | +| `.` | property name | must be `sanitize_label(...)` or a name derived from it | +| `$` | parameter **name** | must be `sanitize_label(...)` or a name derived from it | +| `` ` `` | back-quoted identifier | must be `sanitize_database_name(...)` / `sanitize_label(...)` | +| anything else | compiled fragment | fails only when the expression reads a raw `spec` / `gate` / `dim` / `metadata` value | + +`int(...)` / `float(...)` casts and lookups in a literal allow-list dict count as +validated. Diagnostic strings (`raise`, `logger.*`, `msg = ...`, `reason=`) are +recognised by AST context and skipped. ### Resolution ```python -# ❌ WRONG +# ❌ WRONG — value quoted into the statement cypher = f"SET n.status = '{status}'" await driver.execute_query(cypher) -# ✅ CORRECT +# ✅ CORRECT — value travels as a parameter cypher = "SET n.status = $status" await driver.execute_query(cypher, {"status": status}) + +# ❌ WRONG — spec token interpolated raw (label, property, parameter name, operator) +cypher = f"candidate.{gate.candidateprop} {gate.operator} ${gate.queryparam}" + +# ✅ CORRECT — identifiers validated, operator from an allow-list +prop = sanitize_label(gate.candidateprop) +op = _OPERATORS[gate.operator] +param = sanitize_label(gate.queryparam) +cypher = f"candidate.{prop} {op} ${param}" ``` -**Agent Action**: Never interpolate values into Cypher. Only labels (after `sanitize_label()`). +Gate classes in `engine/gates/types/all_gates.py` bind run-time data with +`self._bind_param(suffix, value)` and expose it via `gate.query_params`; merge +that dict into the `execute_query` parameters. + +A finding may be waived only with `# cypher-lint: allow ` on the +interpolation's line. Waivers are printed on every run (never silent) and a +marker without a reason is ignored. + +**Agent Action**: Never interpolate values into Cypher. Only validated identifiers +(after `sanitize_label()` / `sanitize_database_name()`), and only `$parameters` +for data. --- diff --git a/docs/contracts/BANNED_PATTERNS.md b/docs/contracts/BANNED_PATTERNS.md index 42efa004..bcf886a3 100644 --- a/docs/contracts/BANNED_PATTERNS.md +++ b/docs/contracts/BANNED_PATTERNS.md @@ -126,3 +126,27 @@ def compile_traversal_gate(spec: GateSpec) -> str: **Scope note:** these rules apply to `engine/` only. Abstract base classes in `chassis/` (for example `AuditSink.write_batch`) raise `NotImplementedError` as their defining contract — that is the intended use, not a stub. + +### `typing.Protocol` method bodies (CEG#267) + +A Protocol method is a structural signature. It is never instantiated and never +executed. The only body that is valid Python *and* clean on every in-repo gate +is a docstring and nothing else: + +| Body | `STUB-001` | ruff `PIE790` | github-code-quality | +|---|---|---|---| +| `raise NotImplementedError` | CRITICAL (blocks merge) | clean | clean | +| `pass` | clean | PIE790 | clean | +| `...` | clean | clean | “Statement has no effect” | +| docstring only | clean | clean | clean | + +Do not “fix” a Protocol by raising. That is the opposite of a stub-free engine: +it trips the blocking scanner so a review bot can go quiet. Chassis ABCs may +still raise; `engine/` Protocols may not. + +```python +# ✅ CORRECT — Protocol signature, no executable statement +class GraphWriter(Protocol): + async def execute_write(self, *args: Any, **kwargs: Any) -> Any: + """Run one managed write transaction.""" +``` diff --git a/docs/contracts/SHARED_MODELS.md b/docs/contracts/SHARED_MODELS.md index 6d38c165..bbb1904d 100644 --- a/docs/contracts/SHARED_MODELS.md +++ b/docs/contracts/SHARED_MODELS.md @@ -38,9 +38,15 @@ from engine.packet_bridge import build_request_packet, build_response_packet Installation (pyproject.toml): ```toml -constellation-node-sdk = {git = "https://github.com/cryptoxdog/Gate_SDK.git"} +constellation-node-sdk = {git = "https://github.com/Quantum-L9/Gate_SDK.git", rev = "v1"} ``` +`Quantum-L9/Gate_SDK` is canonical — `cryptoxdog/Gate_SDK` is a forbidden fork +and `scripts/validate_sdk_pin.py` fails closed on it. `v1` is the moving major +compatibility channel owned by Gate_SDK's `contracts/RELEASE_IDENTITY_LEDGER.json`; +declare that channel, never a commit sha and never a branch. The concrete object +it resolves to belongs in `poetry.lock` as `resolved_reference`. + --- ## Package: l9-core (Internal Models) diff --git a/docs/contracts/config/env-contract.yaml b/docs/contracts/config/env-contract.yaml index 477d775c..20d2581a 100644 --- a/docs/contracts/config/env-contract.yaml +++ b/docs/contracts/config/env-contract.yaml @@ -94,17 +94,12 @@ variables: source: env sensitive: false - # ── Redis ───────────────────────────────────────────────── - - name: REDIS_URL - type: url - required: true - default: "redis://localhost:6379/0" - description: | - Redis connection URL. In Docker Compose: redis://redis:6379/0. - Used for scoring result caching and GDS output caching. - source: env - sensitive: false - example: "redis://redis:6379/0" + # CEG-007: REDIS_URL was declared here as required: true, described as backing + # "scoring result caching and GDS output caching". No such caching exists — + # nothing under engine/ or chassis/ imports redis at all. The variable, the + # dependency, the compose services and docs/contracts/dependencies/redis.yaml + # are removed together rather than leaving a contract that describes behaviour + # the code never had. # ── API Surface ─────────────────────────────────────────── - name: API_KEY diff --git a/docs/contracts/dependencies/redis.yaml b/docs/contracts/dependencies/redis.yaml deleted file mode 100644 index 31b25750..00000000 --- a/docs/contracts/dependencies/redis.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# ═══════════════════════════════════════════════════════════════ -# Contract: Redis 7 Dependency -# Source: docker-compose.yml (redis service) -# engine/config/settings.py:redis_url -# Version: 1.0.0 -# Updated: 2026-04-26 -# ═══════════════════════════════════════════════════════════════ - -service_name: redis -type: external -protocol: redis -version: "7-alpine" - -connection: - base_url_env: REDIS_URL - default_url: "redis://redis:6379/0" - auth_method: none - -usage: - - purpose: Scoring result caching - description: GDS and scoring dimension results are cached to avoid recomputation. - - purpose: Domain pack cache - description: Async domain spec cache with TTL (W4-03). TTL=DOMAIN_CACHE_TTL_SECONDS. - -docker_compose: - image: redis:7-alpine - port: "6379:6379" - command: "redis-server --maxmemory 128mb --maxmemory-policy allkeys-lru" - eviction_policy: allkeys-lru - max_memory: 128mb - -health_check: - command: "redis-cli ping" - -fallback: | - No explicit Redis fallback/circuit breaker implemented. - Engine will degrade if Redis is unavailable — scoring will proceed - without cache. diff --git a/domains/aios_god_agent_domain_spec.yaml b/domains/aios-god-agent/spec.yaml similarity index 100% rename from domains/aios_god_agent_domain_spec.yaml rename to domains/aios-god-agent/spec.yaml diff --git a/domains/executive_assistant_domain_spec.yaml b/domains/executive-assistant/spec.yaml similarity index 88% rename from domains/executive_assistant_domain_spec.yaml rename to domains/executive-assistant/spec.yaml index 3135e784..f68b5441 100644 --- a/domains/executive_assistant_domain_spec.yaml +++ b/domains/executive-assistant/spec.yaml @@ -53,6 +53,18 @@ ontology: - {name: category, type: enum} - {name: steps, type: string} + # CEG-009: HAS_SKILL below targets Skill, which was never declared, so this + # spec failed the ontology cross-reference validator and could not load at + # all. It went unnoticed because the file was also in the flat + # _domain_spec.yaml shape the loader never reads. + - label: Skill + managedby: static + auxiliary: true + properties: + - {name: skillid, type: string, required: true} + - {name: name, type: string} + - {name: category, type: enum} + edges: - {type: HAS_SKILL, from: Expert, to: Skill, direction: DIRECTED, category: capability, managedby: sync} - {type: COMPLETED_TASK, from: Expert, to: Task, direction: DIRECTED, category: transaction, managedby: api} diff --git a/domains/freight_matching_domain_spec.yaml b/domains/freight-matching/spec.yaml similarity index 100% rename from domains/freight_matching_domain_spec.yaml rename to domains/freight-matching/spec.yaml diff --git a/domains/healthcare_referral_domain_spec.yaml b/domains/healthcare-referral/spec.yaml similarity index 100% rename from domains/healthcare_referral_domain_spec.yaml rename to domains/healthcare-referral/spec.yaml diff --git a/domains/legal_discovery_domain_spec.yaml b/domains/legal-discovery/spec.yaml similarity index 100% rename from domains/legal_discovery_domain_spec.yaml rename to domains/legal-discovery/spec.yaml diff --git a/domains/mortgage_brokerage_domain_spec.yaml b/domains/mortgage-brokerage/spec.yaml similarity index 100% rename from domains/mortgage_brokerage_domain_spec.yaml rename to domains/mortgage-brokerage/spec.yaml diff --git a/domains/repo_as_agent_domain_spec.yaml b/domains/repo-as-agent/spec.yaml similarity index 100% rename from domains/repo_as_agent_domain_spec.yaml rename to domains/repo-as-agent/spec.yaml diff --git a/domains/research_agent_domain_spec.yaml b/domains/research-agent/spec.yaml similarity index 100% rename from domains/research_agent_domain_spec.yaml rename to domains/research-agent/spec.yaml diff --git a/domains/roofing_company_domain_spec.yaml b/domains/roofing-company/spec.yaml similarity index 87% rename from domains/roofing_company_domain_spec.yaml rename to domains/roofing-company/spec.yaml index 0f970460..35f75eb9 100644 --- a/domains/roofing_company_domain_spec.yaml +++ b/domains/roofing-company/spec.yaml @@ -57,6 +57,19 @@ ontology: - {name: priceperunit, type: float, unit: currency} - {name: warrantyears, type: int} + # CEG-009: SERVICES_ZIP below targets ZipCode, which was never declared, so + # this spec failed the ontology cross-reference validator and could not load + # at all. Hidden by the flat _domain_spec.yaml shape the loader never + # reads. Contractor.servicezips carries the same coverage as a raw string; + # the node is what the edge needs to resolve against. + - label: ZipCode + managedby: static + auxiliary: true + properties: + - {name: zipcode, type: string, required: true} + - {name: city, type: string} + - {name: state, type: string} + edges: - {type: SERVICES_ZIP, from: Contractor, to: ZipCode, direction: DIRECTED, category: capability, managedby: sync} - {type: USES_MATERIAL, from: Contractor, to: Material, direction: DIRECTED, category: capability, managedby: sync} diff --git a/engine/causal/attribution.py b/engine/causal/attribution.py index d9b20d4c..e3d57a84 100644 --- a/engine/causal/attribution.py +++ b/engine/causal/attribution.py @@ -71,7 +71,7 @@ async def compute_attribution( msg = f"Invalid attribution model: {model!r}. Must be one of {sorted(VALID_MODELS)}" raise ValueError(msg) - depth = max_depth or self._spec.causal.chain_depth_limit + depth = int(max_depth or self._spec.causal.chain_depth_limit) outcome_label = sanitize_label(self._spec.feedbackloop.outcome_node_label) # Build edge pattern from causal spec diff --git a/engine/causal/causal_compiler.py b/engine/causal/causal_compiler.py index 9b29ca0b..0f0babc7 100644 --- a/engine/causal/causal_compiler.py +++ b/engine/causal/causal_compiler.py @@ -86,7 +86,7 @@ def compile_causal_chain_query( up to the configured depth limit. """ safe_root = sanitize_label(root_label) - depth = max_depth or self._causal_spec.chain_depth_limit + depth = int(max_depth or self._causal_spec.chain_depth_limit) if edge_types: safe_types = [sanitize_label(t) for t in edge_types] diff --git a/engine/config/loader.py b/engine/config/loader.py index cec63d27..21c36534 100644 --- a/engine/config/loader.py +++ b/engine/config/loader.py @@ -32,7 +32,35 @@ MAX_SPEC_BYTES = 5 * 1024 * 1024 SPEC_FILENAME = "spec.yaml" -_DOMAIN_FEATURE_FLAGS = {"idea-portfolio": "idea_portfolio_enabled"} +_DOMAIN_FEATURE_FLAGS = { + "idea-portfolio": "idea_portfolio_enabled", + # CEG-009 moved nine packs into the shape this loader reads. Five of them + # carry gates that compile to Cypher that cannot execute, which was + # invisible while the loader never opened them: + # + # * `type: traversal` gates written with `pattern` + `condition`, which + # GateCompiler does not consume. With no `edgetype` they fall back to + # `RELATES_TO`, an edge no ontology here declares, so the gate rejects + # every candidate. + # * a scalar `queryparam` (85.0, 5, 1) — GateSpec coerces it to a string + # and the compiler refuses it as a parameter NAME (C-009). + # * `strictwhen` conditions — declared by the schema, consumed by no + # compiler, so a gate meant to be conditional runs as an unconditional + # hard filter and rejects candidates the pack author meant to keep. + # + # Making a pack readable is not the same as making it correct. Reaching + # these needs either compiler support for pattern/condition, literal + # operands and strictwhen, or a query-schema parameter per constant — a + # schema decision, not a file move. Dormant until then, guarded by + # tests/unit/test_domain_pack_shape.py so the set cannot grow silently. + "executive-assistant": "unvalidated_domain_packs_enabled", + "aios-god-agent": "unvalidated_domain_packs_enabled", + "repo-as-agent": "unvalidated_domain_packs_enabled", + "roofing-company": "unvalidated_domain_packs_enabled", + "healthcare-referral": "unvalidated_domain_packs_enabled", + "legal-discovery": "unvalidated_domain_packs_enabled", + "research-agent": "unvalidated_domain_packs_enabled", +} class DomainNotFoundError(Exception): diff --git a/engine/config/settings.py b/engine/config/settings.py index efb9cd16..46664855 100644 --- a/engine/config/settings.py +++ b/engine/config/settings.py @@ -56,8 +56,12 @@ class Settings(BaseSettings): neo4j_max_connection_lifetime: int = 3600 neo4j_connection_acquisition_timeout: int = 60 - # --- Redis --- - redis_url: str = "redis://localhost:6379/0" + # CEG-007: there was a `redis_url` here, and a redis service in both compose + # files, and a redis dependency in requirements — and no `import redis` + # anywhere under engine/ or chassis/. The declared contract claimed scoring + # and domain-pack caching that was never built. Operators could not start the + # api service without Redis running, for a dependency nothing used. All of it + # is removed; if shared state is wanted later, add it with the code that uses it. # --- API --- api_port: int = 8000 @@ -144,6 +148,29 @@ class Settings(BaseSettings): # Distinct from PACKET_STORE_DSN (engine/packet/packet_store.py), which manages # its own lazy pool — both may point at the same Postgres instance. + # CEG-008: `match` and `sync` route queries to a Neo4j database named after + # the domain id, and nothing ever created it — Neo4j does not create + # databases implicitly, so on a fresh instance every sync and match failed + # with an ExecutionError until an operator ran CREATE DATABASE by hand. + # With this on, GraphDriver provisions the database on first use (once per + # database per process). Off by default: CREATE DATABASE is an Enterprise + # Edition administrative command requiring privileges a read-only + # deployment is not expected to hold. Either way the failure now names the + # missing database and the command that provides it. + auto_create_domain_database: bool = False + # CEG-006: the `health_*` admin subactions make engine/health/api.py + # reachable. auto_enrich_via_gate gates only the eventual outbound Gate + # request, not assessment, reporting or conversion tracking, so the surface + # itself needs its own gate. Mechanism ships dormant; operator activates. + health_api_enabled: bool = False + # CEG-009: five migrated domain packs compile to Cypher that cannot execute + # (a `RELATES_TO` fallback for gates written with `pattern`/`condition`, and + # scalar `queryparam` values emitted as parameter names such as `$85.0`). + # They are readable now but not correct, so they stay undiscoverable until + # the compiler or the query schema grows the support they assume. Turning + # this on serves packs whose gates are known not to execute. + unvalidated_domain_packs_enabled: bool = False + # --- Wave 7: Explicit Tenant Database Binding --- strict_tenant_database: bool = ( False # W7-01: require explicit database= on GraphDriver calls; no implicit 'neo4j' fallback @@ -155,6 +182,12 @@ class Settings(BaseSettings): # Seam audit / PR remediation: paid-tier enrich_now Gate dispatch is opt-in. # Default off so deploy does not immediately spend EIE budget until enabled. auto_enrich_via_gate: bool = False + # EIE-008 / CEG-006: emit `graph-inference-result` to EIE through Gate. EIE + # advertises and implements the consumer side end to end; nothing here ever + # produced the packet, so the feedback loop had no producer. Off by default + # for the same reason as auto_enrich_via_gate — each emission queues + # re-enrichment targets and so spends EIE budget. + graph_inference_feedback_enabled: bool = False # IdeaOS portfolio graph is a new behavioral surface. Keep both corpus writes # and portfolio-context reads dormant until explicitly activated by an operator. idea_portfolio_enabled: bool = False diff --git a/engine/feedback/drift_detector.py b/engine/feedback/drift_detector.py index 2369dba8..536e21bb 100644 --- a/engine/feedback/drift_detector.py +++ b/engine/feedback/drift_detector.py @@ -72,6 +72,8 @@ async def _query_dimension_frequency( recent: bool, ) -> dict[str, float]: """Query dimension frequency distribution for recent or baseline outcomes.""" + # Validated again here so this helper is safe on its own, not only via its callers (C-009). + outcome_label = sanitize_label(outcome_label) comparator = ">" if recent else "<=" cypher = f""" MATCH (o:{outcome_label}) diff --git a/engine/feedback/signal_weights.py b/engine/feedback/signal_weights.py index 233b6a03..aa387a05 100644 --- a/engine/feedback/signal_weights.py +++ b/engine/feedback/signal_weights.py @@ -106,6 +106,8 @@ async def _compute_dimension_weight( Returns a dict with keys: weight, confidence, ci_width, lift, sample_size. """ + # Validated again here so this helper is safe on its own, not only via its callers (C-009). + outcome_label = sanitize_label(outcome_label) safe_dim = sanitize_label(dimension_name) # Query outcomes where this dimension scored above median diff --git a/engine/gate_egress.py b/engine/gate_egress.py index 7d7cbbc9..586b9d07 100644 --- a/engine/gate_egress.py +++ b/engine/gate_egress.py @@ -11,10 +11,23 @@ engine/gate_egress.py — the only CEG -> peer egress: CEG -> Gate -> EIE. -CEG never addresses the enrichment node. It asks Gate to run the `enrich` -action (owned by Enrichment.Inference.Engine in Gate's ownership map) and -receives Gate's response packet. The SDK owns packet construction, signing, -the single HTTP attempt, and the deadline derived from ``timeout_ms``. +CEG never addresses the enrichment node. It asks Gate to run an EIE-owned +action (``enrich``, ``graph-inference-result``; see Gate's +CANONICAL_ACTION_OWNERS) and receives Gate's response packet. The SDK owns +packet construction, signing, the single HTTP attempt, and the deadline derived +from ``timeout_ms``. + +Two directions live here: + +* ``request_enrichment`` — CEG asks EIE to research fields it is missing. +* ``emit_graph_inference_result`` — CEG hands EIE values it derived from the + graph, which EIE's ``GraphReturnChannel`` feeds into the convergence loop. + +The second closes a loop that was open in one direction only (EIE-008 / +CEG-006): EIE advertised ``graph-inference-result`` to Gate and implemented the +whole consumer side — validation, per-tenant queues, target extraction — while +no code in CEG ever constructed such a packet. The two halves were even built +to the same 0.55 confidence floor and never connected. Fail-closed rules (seam audit 2026-09-02): * no GATE_URL -> ``gate_not_configured``; there is no direct fallback; @@ -38,9 +51,18 @@ logger = logging.getLogger(__name__) ENRICH_ACTION = "enrich" +GRAPH_INFERENCE_ACTION = "graph-inference-result" DEFAULT_ENRICH_TIMEOUT_MS = 25_000 +DEFAULT_INFERENCE_TIMEOUT_MS = 10_000 _SEAM_TAGS: tuple[str, ...] = ("INTER_NODE",) +# EIE drops any inference output below this confidence +# (app/services/graph_return_channel.py CONFIDENCE_FLOOR). CEG's own rule +# registry suppresses below the same number (InferenceContext.confidence_floor). +# Filtering here too means a packet never carries outputs the receiver will +# silently discard, so "sent 6, queued 2" is visible at the sender. +INFERENCE_CONFIDENCE_FLOOR = 0.55 + def build_enrichment_request( *, @@ -112,6 +134,9 @@ async def request_enrichment( ) try: + # See emit_graph_inference_result: get_gate_client() raises ValueError on + # missing or invalid SDK environment material, which is a configuration + # failure this module reports as a typed result, never one it propagates. client = get_gate_client() response = await client.execute( action=ENRICH_ACTION, @@ -122,7 +147,7 @@ async def request_enrichment( correlation_id=correlation_id, compliance_tags=_SEAM_TAGS, ) - except GateClientError as exc: + except (GateClientError, ValueError) as exc: logger.warning("gate_egress: %s for entity=%s tenant=%s: %s", type(exc).__name__, entity_id, tenant, exc) return { "status": "failed", @@ -144,10 +169,158 @@ async def request_enrichment( } +def inference_idempotency_key(tenant: str, entity_id: str, outputs: Sequence[dict[str, Any]]) -> str: + """Stable over the same findings, so a retry cannot double-queue targets.""" + parts = sorted(f"{o.get('field')}={o.get('value')!r}@{o.get('confidence')}" for o in outputs) + digest = hashlib.sha256("|".join([tenant, entity_id, *parts]).encode("utf-8")).hexdigest() + return f"ceg:graph-inference:{tenant}:{entity_id}:{digest[:16]}" + + +def build_inference_outputs( + entity_id: str, + results: Sequence[Any], +) -> list[dict[str, Any]]: + """Shape CEG inference results into EIE's ``inference_outputs`` elements. + + Accepts ``InferenceResult`` (engine.inference_rule_registry) or a mapping + already in that shape. EIE requires ``entity_id``, ``field``, ``value``, + ``confidence`` and ``rule`` per element; ``InferenceResult.to_dict()`` + already produces all but ``entity_id``. + + Outputs below ``INFERENCE_CONFIDENCE_FLOOR`` are dropped here rather than + sent to be dropped there. + """ + outputs: list[dict[str, Any]] = [] + for result in results: + raw = result.to_dict() if hasattr(result, "to_dict") else dict(result) + field_name = raw.get("field") + if not field_name: + logger.warning("gate_egress: inference output without a field name — dropped") + continue + try: + confidence = float(raw.get("confidence", 0.0)) + except (TypeError, ValueError): + logger.warning("gate_egress: non-numeric confidence for field=%s — dropped", field_name) + continue + if confidence < INFERENCE_CONFIDENCE_FLOOR: + logger.debug( + "gate_egress: confidence %.3f below floor %.3f for field=%s — not sent", + confidence, + INFERENCE_CONFIDENCE_FLOOR, + field_name, + ) + continue + outputs.append( + { + "entity_id": entity_id, + "field": str(field_name), + "value": raw.get("value"), + "confidence": confidence, + "rule": str(raw.get("rule", "unknown")), + "provenance": str(raw.get("provenance", "inference")), + "rationale": str(raw.get("rationale", "")), + } + ) + return outputs + + +async def emit_graph_inference_result( + *, + tenant: str, + entity_id: str, + results: Sequence[Any], + timeout_ms: int = DEFAULT_INFERENCE_TIMEOUT_MS, + correlation_id: str | None = None, +) -> dict[str, Any]: + """Hand graph-derived field values to EIE via Gate. One attempt, fail closed. + + EIE's ``GraphReturnChannel`` converts each output into an ``EnrichmentTarget`` + and feeds the convergence loop, so this is the producer for the action EIE + has advertised and implemented all along (EIE-008). + + Returns a result dict shaped like ``request_enrichment``'s, with + ``sent_outputs`` so a caller can see how many findings survived the floor. + A call with nothing above the floor is reported as ``skipped`` and sends no + packet — an empty ``inference_outputs`` list is valid to EIE and would cost + a Gate round trip to queue nothing. + """ + outputs = build_inference_outputs(entity_id, results) + key = inference_idempotency_key(tenant, entity_id, outputs) + + if not outputs: + return { + "status": "skipped", + "error": "no_outputs_above_confidence_floor", + "action": GRAPH_INFERENCE_ACTION, + "idempotency_key": key, + "sent_outputs": 0, + } + + if not os.environ.get("GATE_URL", "").strip(): + logger.warning("gate_egress: GATE_URL unset — inference result for %s not sent", entity_id) + return { + "status": "failed", + "error": "gate_not_configured", + "action": GRAPH_INFERENCE_ACTION, + "idempotency_key": key, + "sent_outputs": 0, + } + + try: + # ValueError as well as GateClientError: get_gate_client() builds the SDK + # config from the environment and raises ValueError on missing or invalid + # material. Letting that escape would crash the admin subaction instead of + # returning the typed failure every other path in this module returns. + client = get_gate_client() + response = await client.execute( + action=GRAPH_INFERENCE_ACTION, + payload={"inference_outputs": outputs}, + tenant=tenant, + idempotency_key=key, + timeout_ms=timeout_ms, + correlation_id=correlation_id, + compliance_tags=_SEAM_TAGS, + ) + except (GateClientError, ValueError) as exc: + logger.warning( + "gate_egress: %s for inference entity=%s tenant=%s: %s", + type(exc).__name__, + entity_id, + tenant, + exc, + ) + return { + "status": "failed", + "error": type(exc).__name__, + "detail": str(exc), + "action": GRAPH_INFERENCE_ACTION, + "idempotency_key": key, + "sent_outputs": 0, + } + + failed = response.header.packet_type == "failure" + return { + "status": "failed" if failed else "ok", + "action": GRAPH_INFERENCE_ACTION, + "idempotency_key": key, + "sent_outputs": len(outputs), + "packet_id": str(response.header.packet_id), + "packet_type": response.header.packet_type, + "correlation_id": str(response.header.correlation_id) if response.header.correlation_id else None, + "payload": dict(response.payload), + } + + __all__ = [ "DEFAULT_ENRICH_TIMEOUT_MS", + "DEFAULT_INFERENCE_TIMEOUT_MS", "ENRICH_ACTION", + "GRAPH_INFERENCE_ACTION", + "INFERENCE_CONFIDENCE_FLOOR", "build_enrichment_request", + "build_inference_outputs", + "emit_graph_inference_result", "enrichment_idempotency_key", + "inference_idempotency_key", "request_enrichment", ] diff --git a/engine/gates/compiler.py b/engine/gates/compiler.py index 3e55cd0b..007272b3 100644 --- a/engine/gates/compiler.py +++ b/engine/gates/compiler.py @@ -36,6 +36,20 @@ # Cypher boolean conjunction used to join WHERE fragments. AND_JOINER = " AND " +# C-009: tokens read from the domain spec that are interpolated verbatim pass +# through a literal allow-list (a lookup raises on anything else) or through +# sanitize_label(); query parameter *names* are identifiers and must be +# validated exactly like labels, or `$name` becomes a statement injection. +_OPERATORS: dict[str, str] = { + op: op for op in (">=", "<=", ">", "<", "=", "!=", "<>", "IN", "CONTAINS", "STARTS WITH", "ENDS WITH") +} +_LOGIC: dict[str, str] = {"AND": "AND", "OR": "OR"} + + +def _param_name(name: str | None, fallback: str = "value") -> str: + """Validated Cypher parameter identifier for a gate's query parameter.""" + return sanitize_label(name or fallback) + class GateCompiler: """ @@ -226,7 +240,8 @@ def _compile_boolean(self, gate: GateSpec) -> str: """Boolean gate: candidate.prop = $param OR candidate.prop = true.""" prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "prop" if gate.queryparam: - return f"candidate.{prop} = ${gate.queryparam}" + param = _param_name(gate.queryparam) + return f"candidate.{prop} = ${param}" return f"candidate.{prop} = true" def _compile_threshold(self, gate: GateSpec) -> str: @@ -235,8 +250,9 @@ def _compile_threshold(self, gate: GateSpec) -> str: Supports operator override via gate.operator: >=, <=, >, <, = """ prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "prop" - op = gate.operator or ">=" - return f"candidate.{prop} {op} ${gate.queryparam}" + op = _OPERATORS[gate.operator or ">="] + param = _param_name(gate.queryparam) + return f"candidate.{prop} {op} ${param}" def _compile_range(self, gate: GateSpec) -> str: """ @@ -246,7 +262,7 @@ def _compile_range(self, gate: GateSpec) -> str: base_prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "prop" min_prop = sanitize_label(gate.candidateprop_min) if gate.candidateprop_min else f"min_{base_prop}" max_prop = sanitize_label(gate.candidateprop_max) if gate.candidateprop_max else f"max_{base_prop}" - param = gate.queryparam + param = _param_name(gate.queryparam) parts = [] parts.append(f"(candidate.{min_prop} IS NULL OR candidate.{min_prop} <= ${param})") @@ -259,9 +275,10 @@ def _compile_enum(self, gate: GateSpec) -> str: Or inverse: $param IN candidate.prop_list """ prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "prop" + param = _param_name(gate.queryparam) if gate.invertible: - return f"${gate.queryparam} IN candidate.{prop}" - return f"candidate.{prop} IN ${gate.queryparam}" + return f"${param} IN candidate.{prop}" + return f"candidate.{prop} IN ${param}" def _compile_exclusion(self, gate: GateSpec) -> str: """ @@ -287,12 +304,13 @@ def _compile_exclusion(self, gate: GateSpec) -> str: # values only ever flow through $ parameters. exclusion_edge = "(candidate)-[:" + edge_type + "]->" if gate.queryparam: + param = _param_name(gate.queryparam) return ( "NOT EXISTS { MATCH " + exclusion_edge + "(excluded) WHERE excluded." + target_prop - + f" = ${gate.queryparam} " + + f" = ${param} " + "}" ) return "NOT EXISTS { MATCH " + exclusion_edge + "() }" @@ -305,7 +323,7 @@ def _compile_composite(self, gate: GateSpec) -> str: if not gate.subgates: return "true" - combinator = f" {gate.logic or 'AND'} " + combinator = f" {_LOGIC[(gate.logic or 'AND').upper()]} " sub_fragments = [] for sub_gate_name in gate.subgates: sub_gate_spec = next((g for g in self._gates if g.name == sub_gate_name), None) @@ -330,7 +348,7 @@ def _compile_self_range(self, gate: GateSpec) -> str: base_prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "prop" min_prop = sanitize_label(gate.candidateprop_min) if gate.candidateprop_min else f"min_{base_prop}" max_prop = sanitize_label(gate.candidateprop_max) if gate.candidateprop_max else f"max_{base_prop}" - param = gate.queryparam + param = _param_name(gate.queryparam) return ( f"(candidate.{min_prop} IS NULL OR candidate.{min_prop} <= ${param}) AND " @@ -344,7 +362,7 @@ def _compile_freshness(self, gate: GateSpec) -> str: """ prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "updated_at" duration_field = "days" - duration_value = gate.maxagedays or 1 + duration_value = int(gate.maxagedays or 1) return f"candidate.{prop} >= datetime() - duration({{{duration_field}: {duration_value}}})" def _compile_temporal_range(self, gate: GateSpec) -> str: @@ -352,8 +370,8 @@ def _compile_temporal_range(self, gate: GateSpec) -> str: Temporal range gate: candidate.prop between two datetime parameters. """ prop = sanitize_label(gate.candidateprop) if gate.candidateprop else "timestamp" - start_param = gate.queryparam_start or f"{gate.queryparam}_start" - end_param = gate.queryparam_end or f"{gate.queryparam}_end" + start_param = sanitize_label(gate.queryparam_start or f"{gate.queryparam}_start") + end_param = sanitize_label(gate.queryparam_end or f"{gate.queryparam}_end") return f"candidate.{prop} >= ${start_param} AND candidate.{prop} <= ${end_param}" def _compile_traversal(self, gate: GateSpec) -> str: @@ -367,7 +385,8 @@ def _compile_traversal(self, gate: GateSpec) -> str: if gate.candidateprop and gate.queryparam: prop = sanitize_label(gate.candidateprop) - target_filter = f" {{{prop}: ${gate.queryparam}}}" + param = _param_name(gate.queryparam) + target_filter = f" {{{prop}: ${param}}}" label_clause = f":{target_label}" if target_label else "" return f"exists((candidate)-[:{edge_type}]->(t{label_clause}{target_filter}))" @@ -389,6 +408,6 @@ def _wrap_null_semantics(self, gate: GateSpec, predicate: str) -> str: gate_type=gate.type, null_behavior=null_behavior, gate_cypher=predicate, - candidate_prop=f"candidate.{gate.candidateprop}" if gate.candidateprop else None, - query_param=f"${gate.queryparam}" if gate.queryparam else None, + candidate_prop=f"candidate.{sanitize_label(gate.candidateprop)}" if gate.candidateprop else None, + query_param=f"${_param_name(gate.queryparam)}" if gate.queryparam else None, ) diff --git a/engine/gates/types/all_gates.py b/engine/gates/types/all_gates.py index e04e6505..d1d66bfb 100644 --- a/engine/gates/types/all_gates.py +++ b/engine/gates/types/all_gates.py @@ -13,13 +13,28 @@ Production-grade, enterprise-quality, frontier AI lab standard. """ +import hashlib import logging +import re from abc import ABC, abstractmethod +from typing import Any from engine.config.schema import DomainSpec, GateSpec +from engine.utils.security import sanitize_label logger = logging.getLogger(__name__) +_PARAM_KEY_UNSAFE_RE = re.compile(r"[^A-Za-z0-9_]") +_MAX_PARAM_KEY_LEN = 64 + +# C-009: spec tokens interpolated verbatim pass through a literal allow-list — +# a lookup raises on anything else — so an operator or combinator read from +# untrusted YAML can never carry Cypher. +_OPERATORS: dict[str, str] = { + op: op for op in (">=", "<=", ">", "<", "=", "!=", "<>", "IN", "CONTAINS", "STARTS WITH", "ENDS WITH") +} +_LOGIC: dict[str, str] = {"AND": "AND", "OR": "OR"} + # ============================================================================ # BASE GATE @@ -39,6 +54,11 @@ def __init__(self, spec: GateSpec, domain_spec: DomainSpec): """ self.spec = spec self.domain_spec = domain_spec + # C-009: data values a gate needs at run time never appear in the + # compiled fragment as literals. compile() registers them here under + # the parameter names the fragment references, and the caller merges + # `query_params` into the execute_query parameters. + self._query_params: dict[str, Any] = {} @abstractmethod def compile(self) -> str: @@ -49,13 +69,40 @@ def compile(self) -> str: Cypher clause (without NULL handling) """ + @property + def query_params(self) -> dict[str, Any]: + """Cypher parameters registered by the most recent compile() call.""" + return dict(self._query_params) + + def _bind_param(self, suffix: str, value: Any) -> str: + """Register ``value`` as a Cypher parameter and return its ``$name`` reference. + + The name is derived from the gate name so fragments from different gates + never collide; both parts are reduced to ``[A-Za-z0-9_]`` so the name is + always a valid Cypher parameter identifier. + """ + safe_gate = _PARAM_KEY_UNSAFE_RE.sub("_", self.spec.name) + safe_suffix = _PARAM_KEY_UNSAFE_RE.sub("_", suffix) + raw_key = f"gate_{safe_gate}_{safe_suffix}" + if len(raw_key) > _MAX_PARAM_KEY_LEN: + # Keep the key an identifier of bounded length; the digest keeps it unique per gate. + digest = hashlib.sha256(safe_gate.encode()).hexdigest()[:16] + raw_key = f"gate_{digest}_{safe_suffix}"[:_MAX_PARAM_KEY_LEN] + key = sanitize_label(raw_key) + self._query_params[key] = value + return f"${key}" + def _prop_ref(self, prop: str) -> str: - """Format property reference.""" - return f"candidate.{prop}" if not prop.startswith("candidate.") else prop + """Format property reference (property name validated as an identifier).""" + name = sanitize_label(prop.removeprefix("candidate.")) + return f"candidate.{name}" def _param_ref(self, param: str) -> str: - """Format query parameter reference.""" - return f"$query.{param}" if not param.startswith("$") else param + """Format query parameter reference (parameter path validated as identifiers).""" + parts = param.removeprefix("$").split(".") + if parts[0] != "query": + parts.insert(0, "query") + return "$" + ".".join(sanitize_label(part) for part in parts) # ============================================================================ @@ -104,7 +151,7 @@ def compile(self) -> str: prop = self._prop_ref(self.spec.candidateprop) param = self._param_ref(self.spec.queryparam) - operator = self.spec.operator + operator = _OPERATORS[self.spec.operator] return f"{prop} {operator} {param}" @@ -149,6 +196,7 @@ def compile(self) -> str: if not self.spec.logic: raise ValueError(f"Gate '{self.spec.name}': logic required (AND/OR)") + self._query_params = {} # Find subgate specs by name subgate_clauses = [] for subgate_name in self.spec.subgates: @@ -162,8 +210,9 @@ def compile(self) -> str: gate_class = GateRegistry.get_gate_class(subgate_spec.type) gate_instance = gate_class(subgate_spec, self.domain_spec) subgate_clauses.append(f"({gate_instance.compile()})") + self._query_params.update(gate_instance.query_params) - logic_op = f" {self.spec.logic.upper()} " + logic_op = f" {_LOGIC[self.spec.logic.upper()]} " return logic_op.join(subgate_clauses) @@ -186,14 +235,19 @@ def compile(self) -> str: prop = self._prop_ref(self.spec.candidateprop) param = self._param_ref(self.spec.queryparam) + self._query_params = {} # Check if mapping is provided (query value → candidate values) if self.spec.mapping: - # Build CASE WHEN for complex mapping + # Build CASE WHEN for complex mapping. Mapping keys and values are + # data compared against properties, not labels: they may contain + # spaces, dashes or anything else, so they travel as $parameters + # (C-009) rather than as quoted literals in the fragment. cases = [] - for query_val, candidate_vals in self.spec.mapping.items(): - val_list = ", ".join([f"'{v}'" for v in candidate_vals]) - cases.append(f"WHEN {param} = '{query_val}' THEN {prop} IN [{val_list}]") + for index, (query_val, candidate_vals) in enumerate(self.spec.mapping.items()): + key_ref = self._bind_param(f"key_{index}", query_val) + values_ref = self._bind_param(f"values_{index}", list(candidate_vals)) + cases.append(f"WHEN {param} = {key_ref} THEN {prop} IN {values_ref}") case_expr = " ".join(cases) return f"CASE {case_expr} ELSE false END" @@ -216,9 +270,12 @@ def compile(self) -> str: if not self.spec.edgetype: raise ValueError(f"Gate '{self.spec.name}': edgetype required") - edge = self.spec.edgetype - from_node = self.spec.fromnode or "query" - to_node = self.spec.tonode or "candidate" + # Relationship type and node variables are structural identifiers + # read straight from the domain spec, which is untrusted input: + # sanitize before interpolation (C-009). + edge = sanitize_label(self.spec.edgetype) + from_node = sanitize_label(self.spec.fromnode or "query") + to_node = sanitize_label(self.spec.tonode or "candidate") return f"NOT EXISTS(({from_node})-[:{edge}]->({to_node}))" @@ -265,7 +322,7 @@ def compile(self) -> str: raise ValueError(f"Gate '{self.spec.name}': maxagedays required") prop = self._prop_ref(self.spec.candidateprop) - max_age = self.spec.maxagedays + max_age = int(self.spec.maxagedays) return f"duration.between({prop}, datetime()).days <= {max_age}" @@ -314,10 +371,16 @@ def compile(self) -> str: if not self.spec.condition: raise ValueError(f"Gate '{self.spec.name}': condition required") + # `pattern` and `condition` are spec-authored Cypher by design — the + # traversal gate is the domain spec's escape hatch and has no + # value-level grammar to validate against. The live GateCompiler + # (engine/gates/compiler.py) does not honour these fields; packs that + # rely on them are withheld behind `unvalidated_domain_packs_enabled` + # (CEG-009). The waiver below keeps the scanner honest about that. pattern = self.spec.pattern condition = self.spec.condition - return f"EXISTS {{ MATCH {pattern} WHERE {condition} }}" + return f"EXISTS {{ MATCH {pattern} WHERE {condition} }}" # cypher-lint: allow spec-authored Cypher escape hatch, withheld by CEG-009 # ============================================================================ diff --git a/engine/gds/scheduler.py b/engine/gds/scheduler.py index efe5abde..c76ddbb4 100644 --- a/engine/gds/scheduler.py +++ b/engine/gds/scheduler.py @@ -305,14 +305,15 @@ async def _run_louvain(self, job_spec: GDSJobSpec) -> dict[str, Any]: graph_name = f"{safe_job_name}_graph" # Pre-cleanup: drop stale projection if it exists (fixes crash on re-run) - pre_drop = f""" - CALL gds.graph.exists('{graph_name}') YIELD exists + gds_params = {"graph_name": graph_name} + pre_drop = """ + CALL gds.graph.exists($graph_name) YIELD exists WITH exists WHERE exists - CALL gds.graph.drop('{graph_name}') YIELD graphName + CALL gds.graph.drop($graph_name) YIELD graphName RETURN graphName """ try: - await self.graph_driver.execute_query(pre_drop, database=db) + await self.graph_driver.execute_query(pre_drop, parameters=gds_params, database=db) except Exception as exc: exc_msg = str(exc).lower() if "not found" in exc_msg or "does not exist" in exc_msg: @@ -327,27 +328,28 @@ async def _run_louvain(self, job_spec: GDSJobSpec) -> dict[str, Any]: # Sanitize write property name write_prop = sanitize_label(job_spec.writeproperty or "communityId") + gds_params["write_prop"] = write_prop project_cypher = f""" - CALL gds.graph.project('{graph_name}', {node_labels}, {edge_types}) + CALL gds.graph.project($graph_name, {node_labels}, {edge_types}) YIELD graphName, nodeCount, relationshipCount RETURN graphName, nodeCount, relationshipCount """ try: - await self.graph_driver.execute_query(project_cypher, database=db) + await self.graph_driver.execute_query(project_cypher, parameters=gds_params, database=db) - louvain_cypher = f""" - CALL gds.louvain.write('{graph_name}', {{writeProperty: '{write_prop}'}}) + louvain_cypher = """ + CALL gds.louvain.write($graph_name, {writeProperty: $write_prop}) YIELD communityCount, modularity RETURN communityCount, modularity """ - result = await self.graph_driver.execute_query(louvain_cypher, database=db) + result = await self.graph_driver.execute_query(louvain_cypher, parameters=gds_params, database=db) data = result[0] if result else {} logger.info(f"Louvain: {data}") return {"communities": data.get("communityCount"), "modularity": data.get("modularity")} finally: - drop_cypher = f"CALL gds.graph.drop('{graph_name}') YIELD graphName RETURN graphName" + drop_cypher = "CALL gds.graph.drop($graph_name) YIELD graphName RETURN graphName" try: - await self.graph_driver.execute_query(drop_cypher, database=db) + await self.graph_driver.execute_query(drop_cypher, parameters=gds_params, database=db) except Exception: logger.exception(f"Failed to drop projected graph '{graph_name}'") @@ -587,8 +589,17 @@ async def _run_equipment_sync(self, job_spec: GDSJobSpec) -> dict[str, Any]: # Get equipment properties from ontology or use defaults equipment_props = self._get_equipment_properties(job_spec) - # Build dynamic CASE statements for equipment detection - case_statements = [f"CASE WHEN f.{prop} = true THEN '{name}' END" for prop, name in equipment_props] + # Build dynamic CASE statements for equipment detection. Property names + # are structural and pass sanitize_label; equipment type names are data + # (they may legitimately contain spaces or dashes) and travel as + # $parameters, never as quoted literals (C-009). + equipment_params: dict[str, Any] = {} + case_statements = [] + for index, (prop, name) in enumerate(equipment_props): + safe_prop = sanitize_label(prop) + param_key = sanitize_label(f"equipment_name_{index}") + equipment_params[param_key] = name + case_statements.append(f"CASE WHEN f.{safe_prop} = true THEN ${param_key} END") case_list = ",\n ".join(case_statements) cypher = f""" @@ -602,7 +613,7 @@ async def _run_equipment_sync(self, job_spec: GDSJobSpec) -> dict[str, Any]: MERGE (f)-[:HAS_EQUIPMENT]->(e) RETURN count(*) AS edges_created """ - result = await self.graph_driver.execute_query(cypher, database=db) + result = await self.graph_driver.execute_query(cypher, parameters=equipment_params, database=db) edges = result[0]["edges_created"] if result else 0 logger.info(f"Equipment sync: {edges} HAS_EQUIPMENT edges for {node_label}") return {"edges_created": edges} @@ -653,12 +664,13 @@ async def _run_causal_chain_scoring(self, job_spec: GDSJobSpec) -> dict[str, Any # Build edge pattern from causal spec causal_spec = self.domain_spec.causal + depth = int(causal_spec.chain_depth_limit) if causal_spec.causal_edges: safe_types = [sanitize_label(e.edge_type) for e in causal_spec.causal_edges] edge_pattern = "|".join(safe_types) - rel_pattern = f"[:{edge_pattern}*1..{causal_spec.chain_depth_limit}]" + rel_pattern = f"[:{edge_pattern}*1..{depth}]" else: - rel_pattern = f"[*1..{causal_spec.chain_depth_limit}]" + rel_pattern = f"[*1..{depth}]" # Calculate causal influence score per entity cypher = f""" diff --git a/engine/graph/driver.py b/engine/graph/driver.py index 973c5b78..8ba2936f 100644 --- a/engine/graph/driver.py +++ b/engine/graph/driver.py @@ -14,6 +14,7 @@ """ import asyncio +import functools import logging import os from typing import Any @@ -21,9 +22,37 @@ from neo4j import AsyncDriver, AsyncGraphDatabase from engine.graph.circuit_breaker import CircuitBreaker +from engine.utils.security import sanitize_database_name logger = logging.getLogger(__name__) +# Databases the DBMS always provides; never candidates for provisioning. +_BUILTIN_DATABASES = frozenset({"neo4j", "system"}) + +# Substrings Neo4j uses when the target database is absent. Matched case +# insensitively against the driver's message. +_DATABASE_ABSENT_MARKERS = ( + "database does not exist", + "databasenotfound", + "unable to get a routing table for database", +) + + +class DatabaseNotProvisionedError(RuntimeError): + """A query named a database the DBMS does not have. + + CEG-008: `match` and `sync` route to a database named after the domain id, + and nothing created it. Neo4j does not create databases implicitly, so on a + fresh instance every sync and match failed with the driver's own message + until an operator happened to run CREATE DATABASE by hand. This names the + missing database and the command that provides it. + """ + + +def _looks_like_absent_database(exc: Exception) -> bool: + message = str(exc).casefold() + return any(marker in message for marker in _DATABASE_ABSENT_MARKERS) + class GraphDriver: """Neo4j async driver manager.""" @@ -48,6 +77,13 @@ def __init__( self._driver: AsyncDriver | None = None self._lock = asyncio.Lock() + # Databases this process has already provisioned or confirmed, so the + # CREATE is attempted at most once per database per process ... + self._ensured_databases: set[str] = set() + # ... and the CREATE currently in flight per database, kept apart from + # the ensured set so a concurrent first use awaits that CREATE instead + # of either issuing its own or racing past it into the domain query. + self._provisioning: dict[str, asyncio.Task[bool]] = {} # W4-02: Circuit breaker — configured via settings, defaults provided from engine.config.settings import settings @@ -131,7 +167,96 @@ async def execute_query( ) raise ValueError(msg) database = "neo4j" - return await self._circuit_breaker.call(self._raw_execute_query, cypher, parameters, database) + + await self._provision_on_first_use(database) + try: + return await self._circuit_breaker.call(self._raw_execute_query, cypher, parameters, database) + except Exception as exc: + raise self._translate_absent_database(exc, database) from exc + + async def _provision_on_first_use(self, database: str) -> None: + """CEG-008: a tenant domain database has to exist before it can be used. + + Under ``auto_create_domain_database`` we provision it on first use — + for reads and writes alike, since a fresh deployment's first operation + is as likely to be a sync write as a match query. + """ + from engine.config.settings import settings as _db_settings + + if _db_settings.auto_create_domain_database: + await self.ensure_database(database) + + @staticmethod + def _translate_absent_database(exc: Exception, database: str) -> Exception: + """Name the missing database and the command that provides it, else pass the error through.""" + if _looks_like_absent_database(exc) and database not in _BUILTIN_DATABASES: + msg = ( + f"Neo4j database {database!r} does not exist. Domain queries route to a " + f"database named after the domain id, and Neo4j does not create one " + f"implicitly. Run this against the system database (Enterprise " + f"Edition): CREATE DATABASE `{database}` IF NOT EXISTS WAIT " + f"-- or set AUTO_CREATE_DOMAIN_DATABASE=true to have the engine " + f"create it on first use." + ) + return DatabaseNotProvisionedError(msg) + return exc + + async def ensure_database(self, name: str) -> bool: + """Create the domain database if it is absent. Idempotent per process. + + Returns True when the database is known to exist afterwards, False when + provisioning was not possible (Community Edition, or insufficient + privileges) — in which case the query that follows fails with the + message above rather than here, so a read-only deployment that has the + database already is not blocked by a CREATE it is not allowed to run. + + Concurrency: the first caller for a name starts the CREATE; every caller + that arrives while it is in flight awaits that same CREATE and receives + its result, so no domain query proceeds before provisioning completes + and the administrative command is issued once. A failed CREATE leaves + nothing ensured, so a later call retries it. + """ + if name in _BUILTIN_DATABASES or name in self._ensured_databases: + return True + sanitize_database_name(name) + + task = self._provisioning.get(name) + if task is None or task.done(): + # A finished task still in the map lost its race with the done + # callback below; its outcome is already reflected in + # `_ensured_databases` (success) or not (failure → retry now). + task = asyncio.get_running_loop().create_task(self._provision_database(name)) + self._provisioning[name] = task + task.add_done_callback(functools.partial(self._forget_provisioning, name)) + # shield: cancelling one waiting request must not cancel the CREATE the + # other waiters — and the ensured set — depend on. + return await asyncio.shield(task) + + def _forget_provisioning(self, name: str, done: asyncio.Task[bool]) -> None: + if self._provisioning.get(name) is done: + del self._provisioning[name] + + async def _provision_database(self, name: str) -> bool: + # Administrative commands must run against `system`, and the name is + # back-quoted because a dash is legal in a database name but not in a + # bare identifier. sanitize_database_name is what makes that quoting + # safe; it is re-applied here so the statement is safe by construction. + cypher = f"CREATE DATABASE `{sanitize_database_name(name)}` IF NOT EXISTS WAIT" + try: + await self._raw_execute_query(cypher, None, "system") + except Exception as exc: + logger.warning( + "Could not provision Neo4j database %r (%s: %s). " + "CREATE DATABASE is Enterprise Edition only and requires admin privileges.", + name, + type(exc).__name__, + exc, + ) + return False + + self._ensured_databases.add(name) + logger.info("Ensured Neo4j database %r exists", name) + return True async def _raw_execute_write( self, @@ -207,12 +332,16 @@ async def execute_write( ) raise ValueError(msg) database = "neo4j" - return await self._circuit_breaker.call( - self._raw_execute_write, - transaction_function, - *args, - cypher=cypher, - parameters=parameters, - database=database, - **kwargs, - ) + await self._provision_on_first_use(database) + try: + return await self._circuit_breaker.call( + self._raw_execute_write, + transaction_function, + *args, + cypher=cypher, + parameters=parameters, + database=database, + **kwargs, + ) + except Exception as exc: + raise self._translate_absent_database(exc, database) from exc diff --git a/engine/handlers.py b/engine/handlers.py index fb1b426d..d5033bfa 100644 --- a/engine/handlers.py +++ b/engine/handlers.py @@ -481,7 +481,7 @@ async def handle_match(tenant: str, payload: dict[str, Any]) -> dict[str, Any]: ) try: - parameters = {**resolved_query, "top_n": top_n} + parameters = {**resolved_query, "top_n": top_n, **scoring_assembler.last_query_params} results = await graph_driver.execute_query( cypher=cypher, parameters=parameters, @@ -1197,6 +1197,12 @@ async def handle_admin(tenant: str, payload: dict[str, Any]) -> dict[str, Any]: "gdpr_erasure_enabled": _fs.gdpr_erasure_enabled, "gdpr_dry_run": _fs.gdpr_dry_run, "gds_max_staleness_hours": _fs.gds_max_staleness_hours, + # --- Constellation seam (EIE <-> Gate <-> CEG) --- + "auto_enrich_via_gate": _fs.auto_enrich_via_gate, + "graph_inference_feedback_enabled": _fs.graph_inference_feedback_enabled, + "health_api_enabled": _fs.health_api_enabled, + "auto_create_domain_database": _fs.auto_create_domain_database, + "idea_portfolio_enabled": _fs.idea_portfolio_enabled, }, } @@ -1424,6 +1430,99 @@ async def handle_admin(tenant: str, payload: dict[str, Any]) -> dict[str, Any]: "outcome_history_size": len(outcome_history), } + # ── CEG-006: AI-readiness health, reachable from an inbound packet ─────── + # engine/health/api.py implemented three handlers and was imported by + # nothing, so `request_enrichment` — the whole CEG -> Gate -> EIE direction + # — had no trigger any packet could reach. It hangs off `admin` rather than + # a new advertised action for the same reason `trigger_gds` does: these are + # operator surfaces, not collaboration routes Gate load-balances. + if subaction in {"health_assess", "health_batch_assess", "health_report"}: + from engine.config.settings import settings as _health_settings + + # Mechanism ships dormant, operator activates — the same contract every + # other behavioural surface here follows. auto_enrich_via_gate gates only + # the eventual outbound request, not assessment, reporting or conversion + # tracking, so making these reachable needs a gate of their own. + if not _health_settings.health_api_enabled: + return { + "status": "disabled", + "subaction": subaction, + "message": "Health API is disabled. Set HEALTH_API_ENABLED=True.", + } + + from engine.health import api as health_api + + health_handlers = { + "health_assess": health_api.handle_health_assess, + "health_batch_assess": health_api.handle_health_batch_assess, + "health_report": health_api.handle_health_report, + } + return await health_handlers[subaction](tenant, payload) + + # ── EIE-008: hand graph-derived field values back to EIE through Gate ──── + # EIE advertises and implements `graph-inference-result` end to end and no + # code here ever produced one, so the feedback loop the architecture implies + # had a consumer and no producer. Flag-gated: it spends EIE budget. + if subaction == "emit_inference_feedback": + from engine.config.settings import settings as _inf_settings + + if not _inf_settings.graph_inference_feedback_enabled: + return { + "status": "disabled", + "message": ("Graph inference feedback is disabled. Set GRAPH_INFERENCE_FEEDBACK_ENABLED=True."), + } + + from engine.gate_egress import emit_graph_inference_result + from engine.inference_rule_registry import ( + InferenceContext, + execute_rule, + list_registered_rules, + ) + + entity = _require_key(payload, "entity", "admin", tenant) + entity_id = _require_key(payload, "entity_id", "admin", tenant) + domain_id = payload.get("domain_id", tenant) + + # `rules` absent means "every registered rule"; `rules: []` means "none" + # and must not silently become "all". A non-list is rejected rather than + # iterated — a bare string would otherwise be walked character by + # character and every char looked up as a rule name. + if "rules" in payload: + requested = payload["rules"] + if not isinstance(requested, list) or not all(isinstance(r, str) for r in requested): + raise ValidationError( + "admin.rules must be a list of rule-name strings", + action="admin", + tenant=tenant, + ) + else: + requested = list_registered_rules() + + context = InferenceContext( + tenant_id=tenant, + domain_id=domain_id, + pass_number=int(payload.get("pass_number", 1)), + known_fields=dict(entity), + ) + # Named `inferred`, not `result`: `result` is already bound to a + # dict[str, Any] earlier in this function, and reusing it here is a type + # error mypy catches (and did). + results = [ + inferred for rule_name in requested if (inferred := execute_rule(rule_name, entity, context)) is not None + ] + dispatch = await emit_graph_inference_result( + tenant=tenant, + entity_id=str(entity_id), + results=results, + ) + return { + "status": "inference_feedback_emitted", + "entity_id": entity_id, + "rules_run": list(requested), + "results_produced": len(results), + "dispatch": dispatch, + } + raise ValidationError(f"Unknown admin subaction: {subaction!r}", action="admin", tenant=tenant) diff --git a/engine/health/health_report.py b/engine/health/health_report.py index 94e4f3e0..cdcde7c3 100644 --- a/engine/health/health_report.py +++ b/engine/health/health_report.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +from collections import deque from datetime import UTC, datetime from typing import Any @@ -29,8 +30,17 @@ logger = logging.getLogger(__name__) -# In-memory conversion event store (replace with persistent store in production) -_conversion_events: list[ConversionEvent] = [] +# In-memory conversion event store (replace with persistent store in production). +# +# Bounded, deliberately. CEG-006 made the `health_*` admin subactions reachable, +# and every Seed-tier assess/report call appends a tenant- and entity-bearing +# record here. An unbounded list in a long-lived process is a memory leak that +# grows with traffic, and CLAUDE.md's "never create unbounded caches" covers +# exactly this. A deque discards oldest-first at the ceiling, so funnel analysis +# keeps a recent window rather than the whole history — the same trade the +# "replace with persistent store" note above already anticipates. +CONVERSION_EVENT_MAX = 10_000 +_conversion_events: deque[ConversionEvent] = deque(maxlen=CONVERSION_EVENT_MAX) def generate_health_report( @@ -166,7 +176,7 @@ def analyze_conversion_funnel( tenant: str | None = None, ) -> ConversionFunnelMetrics: """Analyze conversion funnel metrics, optionally filtered by tenant.""" - events = _conversion_events + events: list[ConversionEvent] = list(_conversion_events) if tenant: events = [e for e in events if e.tenant == tenant] diff --git a/engine/hoprag/indexer.py b/engine/hoprag/indexer.py index 7279c1ce..d422bbf4 100644 --- a/engine/hoprag/indexer.py +++ b/engine/hoprag/indexer.py @@ -67,7 +67,6 @@ async def fetch_passages( Returns: List of dicts with 'id' and 'text' keys. """ - ... async def write_edges( self, @@ -83,7 +82,6 @@ async def write_edges( Returns: Number of edges written. """ - ... async def get_vertex_count(self, label: str) -> int: """Count vertices with given label. @@ -94,7 +92,6 @@ async def get_vertex_count(self, label: str) -> int: Returns: Number of vertices. """ - ... @dataclass @@ -135,7 +132,7 @@ class GraphIndexBuilder: graph_store=neo4j_store, ) result = await builder.build(passage_label="Passage") - print(f"Created {result.edges_created} edges") + logger.info("Created %d edges", result.edges_created) """ def __init__( diff --git a/engine/resolution/similarity.py b/engine/resolution/similarity.py index 06698f36..ca7c55b5 100644 --- a/engine/resolution/similarity.py +++ b/engine/resolution/similarity.py @@ -123,6 +123,8 @@ async def _property_similarity( label: str, ) -> float: """Jaccard similarity over comparison properties.""" + # Validated again here so this helper is safe on its own, not only via its callers (C-009). + label = sanitize_label(label) comp_props = self._spec.comparison_properties if not comp_props: return 0.0 @@ -163,6 +165,8 @@ async def _structural_similarity( label: str, ) -> float: """Shared neighbor overlap (Jaccard on neighbor sets).""" + # Validated again here so this helper is safe on its own, not only via its callers (C-009). + label = sanitize_label(label) cypher = f""" MATCH (a:{label} {{entity_id: $a_id}})--(neighbor_a) WITH a, collect(DISTINCT id(neighbor_a)) AS neighbors_a @@ -220,6 +224,8 @@ async def _find_property_candidates( limit: int, ) -> list[str]: """Find candidate entity IDs that share property values.""" + # Validated again here so this helper is safe on its own, not only via its callers (C-009). + label = sanitize_label(label) comp_props = self._spec.comparison_properties if not comp_props: # Fallback: return all entities of the same label diff --git a/engine/scoring/assembler.py b/engine/scoring/assembler.py index b8a4c272..5109e2eb 100644 --- a/engine/scoring/assembler.py +++ b/engine/scoring/assembler.py @@ -27,7 +27,7 @@ ScoringDimensionSpec, ) from engine.graph.driver import GraphDriver -from engine.utils.security import sanitize_label +from engine.utils.security import cypher_number, sanitize_label logger = logging.getLogger(__name__) @@ -91,6 +91,7 @@ def __init__( self.domain_spec = domain_spec self.scoring_spec = domain_spec.scoring self._last_active_dims: list[str] = [] + self._query_params: dict[str, Any] = {} self._graph_driver = graph_driver self._learned_weights: dict[str, float] | None = None self._population_means: dict[str, float] = {} # S2-01: cached population means @@ -152,6 +153,7 @@ def assemble_scoring_clause( """ from engine.config.settings import settings + self._query_params = {} pareto_metadata: dict[str, Any] | None = None # Pareto pre-filter (lazy import to avoid circular deps) @@ -185,7 +187,8 @@ def assemble_scoring_clause( # W1-02: clamp each dimension expression to [0.0, 1.0] when enabled if settings.score_clamp_enabled: expr = self._clamp_expression(expr) - dimension_exprs.append(f"{expr} AS {dim.name}") + alias = sanitize_label(dim.name) + dimension_exprs.append(f"{expr} AS {alias}") weight = weights.get(dim.weightkey, dim.defaultweight) # Convergence loop: multiply spec weight by learned adjustment factor if dim.name in learned and sw_spec is not None: @@ -194,7 +197,7 @@ def assemble_scoring_clause( # apply negative penalty instead of just reduced weight. # Implements "primitive subtraction" from Lippl et al. weight = -abs(sw_spec.penalty_factor) if learned_w < sw_spec.penalty_threshold else weight * learned_w - weight_exprs.append(f"({weight} * {dim.name})") + weight_exprs.append(f"({cypher_number(weight)} * {alias})") active_dim_names.append(dim.name) self._last_active_dims = list(active_dim_names) @@ -214,6 +217,11 @@ def last_active_dimension_names(self) -> list[str]: """Return dimension names from the most recent assemble_scoring_clause call.""" return list(self._last_active_dims) + @property + def last_query_params(self) -> dict[str, Any]: + """Copy of Cypher parameters collected during the last assemble call.""" + return dict(self._query_params) + def _compile_dimension(self, dim: ScoringDimensionSpec) -> str: """Dispatch to computation-specific compiler. @@ -260,7 +268,7 @@ def _compile_dimension(self, dim: ScoringDimensionSpec) -> str: return base_expr def _compile_geodecay(self, dim: ScoringDimensionSpec) -> str: - k = dim.decayconstant or 50000.0 + k = cypher_number(dim.decayconstant or 50000.0) lat_prop = sanitize_label(dim.candidateprop or "lat") query_lat_param = sanitize_label(dim.queryprop or lat_prop) return ( @@ -271,7 +279,7 @@ def _compile_geodecay(self, dim: ScoringDimensionSpec) -> str: ) def _compile_lognormalized(self, dim: ScoringDimensionSpec) -> str: - max_val = dim.maxvalue or 1000.0 + max_val = cypher_number(dim.maxvalue or 1000.0) prop = sanitize_label(dim.candidateprop or "value") return f"log(1 + coalesce(candidate.{prop}, 0)) / log(1 + {max_val})" @@ -285,7 +293,7 @@ def _compile_communitymatch(self, dim: ScoringDimensionSpec) -> str: Returns bias score if communities match, graduated score otherwise. Does not require APOC - uses native Cypher only. """ - bias = dim.bias or 1.5 + bias = cypher_number(dim.bias or 1.5) cand_prop = sanitize_label(dim.candidateprop or "community_id") query_prop = sanitize_label(dim.queryprop or "community_id") @@ -310,8 +318,8 @@ def _compile_communitymatch(self, dim: ScoringDimensionSpec) -> str: ) def _compile_inverselinear(self, dim: ScoringDimensionSpec) -> str: - min_val = dim.minvalue or 0.0 - max_val = dim.maxvalue or 100.0 + min_val = cypher_number(dim.minvalue or 0.0) + max_val = cypher_number(dim.maxvalue or 100.0) prop = sanitize_label(dim.candidateprop or "value") return f"1.0 - (coalesce(candidate.{prop}, {max_val}) - {min_val}) / ({max_val} - {min_val})" @@ -335,7 +343,7 @@ def _compile_pricealignment(self, dim: ScoringDimensionSpec) -> str: """ cand_prop = sanitize_label(dim.candidateprop or "price_per_unit") query_prop = sanitize_label(dim.queryprop or "target_price") - tau = dim.maxvalue or 2.0 # tolerance: 2.0 = ~7.4x ratio scores 0 + tau = cypher_number(dim.maxvalue or 2.0) # tolerance: 2.0 = ~7.4x ratio scores 0 default = float(dim.defaultwhennull) # nosemgrep: float-requires-try-except return ( f"CASE " @@ -353,7 +361,7 @@ def _compile_temporalproximity(self, dim: ScoringDimensionSpec) -> str: Reads last_activity_date, touch_count_30d, and is_accelerating from node. """ date_prop = sanitize_label(dim.candidateprop or "last_activity_date") - decay_days = dim.maxvalue or 90.0 + decay_days = cypher_number(dim.maxvalue or 90.0) default = float(dim.defaultwhennull) # nosemgrep: float-requires-try-except # Weights for 3 signals w1, w2, w3 = 0.6, 0.25, 0.15 @@ -454,21 +462,31 @@ def _compile_preference_attention(self, dim: ScoringDimensionSpec) -> str: outcome_rel = sanitize_label(metadata.get("outcome_relation", "RESULTED_IN")) outcome_node = sanitize_label(metadata.get("outcome_node", "TransactionOutcome")) success_prop = sanitize_label(metadata.get("success_property", "outcome_type")) - success_value = sanitize_label(metadata.get("success_value", "closed_won")) + success_value = metadata.get("success_value", "closed_won") + if not isinstance(success_value, str): + msg = f"Dimension '{dim.name}': success_value must be a string" + raise ValueError(msg) cand_community_prop = sanitize_label(dim.candidateprop or "community_id") + safe_dim = sanitize_label(dim.name) + success_key = f"pref_success_{safe_dim}" + default_key = f"pref_default_{safe_dim}" + sample_key = f"pref_sample_k_{safe_dim}" + self._query_params[success_key] = success_value + self._query_params[default_key] = default + self._query_params[sample_key] = int(sample_k) return ( f"CASE " f" WHEN size([(qe)-[:{outcome_rel}]->(o:{outcome_node}) " - f" WHERE o.{success_prop} = '{success_value}' | o]) = 0 THEN {default} " + f" WHERE o.{success_prop} = ${success_key} | o]) = 0 THEN ${default_key} " f" ELSE toFloat(" f" size([(qe)-[:{outcome_rel}]->(o:{outcome_node}) " - f" WHERE o.{success_prop} = '{success_value}' " - f" AND o.community_id = candidate.{cand_community_prop} | o][0..{sample_k}])" + f" WHERE o.{success_prop} = ${success_key} " + f" AND o.community_id = candidate.{cand_community_prop} | o][0..${sample_key}])" f" ) / toFloat(" f" size([(qe)-[:{outcome_rel}]->(o:{outcome_node}) " - f" WHERE o.{success_prop} = '{success_value}' | o][0..{sample_k}])" + f" WHERE o.{success_prop} = ${success_key} | o][0..${sample_key}])" f" ) " f"END" ) @@ -537,11 +555,11 @@ def _apply_null_strategy(self, dim: ScoringDimensionSpec, base_expr: str) -> str return base_expr # No change — coalesce(..., 0.0) already handles this if strategy == NullStrategy.INHERIT_PRIOR: prop_name = sanitize_label(f"_prior_{dim.name}") - return f"coalesce(({base_expr}), candidate.{prop_name}, {dim.defaultwhennull})" + return f"coalesce(({base_expr}), candidate.{prop_name}, {cypher_number(dim.defaultwhennull)})" if strategy == NullStrategy.POPULATION_MEAN: # Population mean is injected as a parameter at query time param = sanitize_label(f"_popmean_{dim.name}") - return f"coalesce(({base_expr}), ${param}, {dim.defaultwhennull})" + return f"coalesce(({base_expr}), ${param}, {cypher_number(dim.defaultwhennull)})" return base_expr def _apply_cold_start_fallback(self, dim: ScoringDimensionSpec, base_expr: str) -> str: diff --git a/engine/scoring/helpfulness.py b/engine/scoring/helpfulness.py index 64feeb7e..5ba8c632 100644 --- a/engine/scoring/helpfulness.py +++ b/engine/scoring/helpfulness.py @@ -33,6 +33,8 @@ import logging from dataclasses import dataclass +from engine.utils.security import cypher_number, sanitize_label + logger = logging.getLogger(__name__) @@ -68,7 +70,7 @@ class HelpfulnessScorer: scorer = HelpfulnessScorer(alpha=0.5) result = scorer.compute(similarity=0.82, importance=0.45) - print(result.score) # 0.635 + logger.info("helpfulness=%s", result.score) # 0.635 """ def __init__(self, alpha: float = 0.5) -> None: @@ -199,8 +201,10 @@ def compile_helpfulness_cypher( Returns: Cypher expression string. """ + similarity_prop = sanitize_label(similarity_prop) + importance_prop = sanitize_label(importance_prop) return ( - f"CASE WHEN candidate.{similarity_prop} IS NULL THEN {default_when_null} " - f"ELSE ({alpha} * coalesce(candidate.{similarity_prop}, 0) + " - f"{1.0 - alpha} * coalesce(candidate.{importance_prop}, 0)) END" + f"CASE WHEN candidate.{similarity_prop} IS NULL THEN {cypher_number(default_when_null)} " + f"ELSE ({cypher_number(alpha)} * coalesce(candidate.{similarity_prop}, 0) + " + f"{1.0 - cypher_number(alpha)} * coalesce(candidate.{importance_prop}, 0)) END" ) diff --git a/engine/scoring/importance.py b/engine/scoring/importance.py index 9180e245..4c94c36e 100644 --- a/engine/scoring/importance.py +++ b/engine/scoring/importance.py @@ -34,6 +34,8 @@ import logging from dataclasses import dataclass +from engine.utils.security import cypher_number, sanitize_label + logger = logging.getLogger(__name__) @@ -64,7 +66,7 @@ class ImportanceScorer: scorer = ImportanceScorer() visit_counts = {"v1": 5, "v2": 3, "v3": 2} result = scorer.compute("v1", visit_counts) - print(result.score) # 0.5 + logger.info("importance=%s", result.score) # 0.5 """ def compute( @@ -185,7 +187,9 @@ def compile_importance_cypher( Returns: Cypher expression string. """ + visit_count_prop = sanitize_label(visit_count_prop) + total_visits_param = sanitize_label(total_visits_param) return ( - f"CASE WHEN candidate.{visit_count_prop} IS NULL THEN {default_when_null} " + f"CASE WHEN candidate.{visit_count_prop} IS NULL THEN {cypher_number(default_when_null)} " f"ELSE toFloat(candidate.{visit_count_prop}) / ${total_visits_param} END" ) diff --git a/engine/spec.yaml b/engine/spec.yaml index d58d5897..676eb1cf 100644 --- a/engine/spec.yaml +++ b/engine/spec.yaml @@ -15,6 +15,14 @@ node: id: graph version: "1.1.0" + # CEG-005: declare the semantic owner instead of letting Gate infer it. + # The SDK flattens node.owner into metadata.owner, and Gate's registration + # ownership assertion is fail-closed for canonical actions. With no owner + # here, registration succeeded only because Gate's _OWNER_ALIASES maps the + # node name "graph" to "ceg" — correct today, and a name-based alias rather + # than a declaration. Renaming the node would have broken registration for + # every canonical action at once. EIE declares owner=eie for the same reason. + owner: ceg type: engine internal_url: "http://graph:8000" health_endpoint: /v1/health diff --git a/engine/sync/idea_portfolio.py b/engine/sync/idea_portfolio.py index 32d8fdf4..dab4580e 100644 --- a/engine/sync/idea_portfolio.py +++ b/engine/sync/idea_portfolio.py @@ -222,7 +222,6 @@ async def execute_write( **kwargs: Any, ) -> dict[str, Any] | Any: """Run one managed write transaction, via a transaction function or `cypher`.""" - ... @dataclass(frozen=True) diff --git a/engine/traversal/multihop.py b/engine/traversal/multihop.py index 779ba673..9ef6c325 100644 --- a/engine/traversal/multihop.py +++ b/engine/traversal/multihop.py @@ -75,7 +75,6 @@ def evaluate_edges( Returns: Index of the selected edge in candidate_edges. """ - ... @dataclass(frozen=True) @@ -132,7 +131,6 @@ async def get_outgoing_edges(self, vertex_id: str) -> list[TraversalEdge]: Returns: List of TraversalEdge objects. """ - ... class MultiHopTraverser: @@ -158,7 +156,7 @@ class MultiHopTraverser: start_vertices=["v1", "v2", "v3"], query_embedding=query_emb, ) - print(result.visit_counts) # {"v1": 3, "v4": 2, ...} + logger.info("visit_counts=%s", result.visit_counts) # {"v1": 3, "v4": 2, ...} """ def __init__( diff --git a/engine/traversal/pseudo_query.py b/engine/traversal/pseudo_query.py index 0f71cbc2..a62ecd3b 100644 --- a/engine/traversal/pseudo_query.py +++ b/engine/traversal/pseudo_query.py @@ -126,7 +126,6 @@ def generate(self, prompt: str) -> str: Returns: Generated text response. """ - ... class KeywordExtractor(Protocol): @@ -141,7 +140,6 @@ def extract(self, text: str) -> frozenset[str]: Returns: Set of extracted keyword strings. """ - ... class EmbeddingEncoder(Protocol): @@ -156,7 +154,6 @@ def encode(self, text: str) -> tuple[float, ...]: Returns: Embedding vector as tuple of floats. """ - ... # ── Main Generator ─────────────────────────────────────────────────── @@ -182,8 +179,8 @@ class PseudoQueryGenerator: n_incoming=2, m_outgoing=4, ) - print(len(result.incoming)) # 2 - print(len(result.outgoing)) # 4 + logger.info("incoming=%d", len(result.incoming)) # 2 + logger.info("outgoing=%d", len(result.outgoing)) # 4 """ def __init__( diff --git a/engine/utils/security.py b/engine/utils/security.py index a1e813ef..46f66714 100644 --- a/engine/utils/security.py +++ b/engine/utils/security.py @@ -38,3 +38,50 @@ def sanitize_label(label: str) -> str: msg = f"Invalid label or type: {label!r}" raise ValueError(msg) return label + + +def cypher_number(value: object) -> float: + """ + Validate a domain-spec scalar before it is interpolated into Cypher as a numeric literal. + + SECURITY: a number cannot carry Cypher, so a value that survives ``float()`` + is safe to interpolate; anything else (a string payload, ``None`` where the + spec promised a number) is rejected here instead of reaching the query. + + Raises ValueError if the value is not numeric. + """ + try: + return float(value) # type: ignore[arg-type] + except (TypeError, ValueError) as exc: + msg = f"Expected a numeric Cypher literal, got {value!r}" + raise ValueError(msg) from exc + + +# Neo4j database naming rules: begins with an ASCII letter, then letters, +# digits, dots, dashes or underscores, 3-63 characters. Domain ids legitimately +# contain dashes ("healthcare-referral"), which is why sanitize_label does not +# apply here: its label grammar forbids them. +_DATABASE_NAME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9._-]{2,62}$") + + +def sanitize_database_name(name: str) -> str: + """ + Validate a Neo4j database name before it is quoted into an administrative command. + + SECURITY: ``CREATE DATABASE`` cannot take the name as a query parameter (it is + an administrative command, not a read/write query), so the name is back-quoted + into the statement — and therefore must be validated first. The grammar above + is what makes that quoting safe: no back-quote, whitespace or statement + separator can pass it. + + Raises ValueError if invalid. + """ + if not _DATABASE_NAME_RE.fullmatch(name): + msg = ( + f"refusing to provision Neo4j database {name!r}: a database name must begin " + f"with a letter and contain only letters, digits, dots, dashes or underscores " + f"(3-63 characters). CREATE DATABASE takes no query parameter, so an " + f"unvalidated name would be interpolated into an administrative command." + ) + raise ValueError(msg) + return name diff --git a/poetry.lock b/poetry.lock index d606da32..9c697805 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. [[package]] name = "annotated-doc" @@ -2036,26 +2036,6 @@ files = [ {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, ] -[[package]] -name = "redis" -version = "8.1.0" -description = "Python client for Redis database and key-value store" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [ - {file = "redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb"}, - {file = "redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25"}, -] - -[package.extras] -circuit-breaker = ["pybreaker (>=1.4.0)"] -hiredis = ["hiredis (>=3.2.0)"] -jwt = ["pyjwt (>=2.13.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"] -otel = ["opentelemetry-api (>=1.39.1)", "opentelemetry-exporter-otlp-proto-http (>=1.39.1)", "opentelemetry-sdk (>=1.39.1)"] -xxhash = ["xxhash (>=3.6.0,<3.7.0)"] - [[package]] name = "requests" version = "2.33.0" @@ -2715,4 +2695,4 @@ dev = ["pytest", "setuptools"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "1e726460d8433f929a08ddca9a953e4b2af22a87b594fc075e64a48c378147be" +content-hash = "8dd164d3ea9e0833dbca78643fefd7e161ef7bcdbea0625392c3a7bab26e51a0" diff --git a/pyproject.toml b/pyproject.toml index 58e285f1..42153b31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ pydantic = "^2.10.0" pydantic-settings = "^2.6.0" pyyaml = "^6.0" python-multipart = ">=0.0.9,<0.0.32" -redis = ">=7.4,<9.0" apscheduler = "^3.10.0" httpx = "^0.28.0" structlog = ">=25.5,<27.0" diff --git a/requirements.txt b/requirements.txt index 8deaa8c5..dec56400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ pydantic-settings>=2.6.0,<3.0.0 pyyaml>=6.0,<7.0 types-PyYAML>=6.0 python-multipart>=0.0.9,<1.0.0 -redis>=7.4.0,<8.0.0 apscheduler>=3.10.0,<4.0.0 httpx>=0.28.0,<0.29.0 structlog>=25.5.0,<27.0.0 diff --git a/scripts/validate_sdk_pin.py b/scripts/validate_sdk_pin.py index a7627888..e011244d 100644 --- a/scripts/validate_sdk_pin.py +++ b/scripts/validate_sdk_pin.py @@ -4,16 +4,46 @@ Policy (CEG#266 / PR #263): constellation-node-sdk tracks ``Quantum-L9/Gate_SDK@v1``. Manifests name the major tag. poetry.lock records that tag as ``reference`` and the resolved object as ``resolved_reference``. + +Gate_SDK owns what ``v1`` means (``contracts/RELEASE_IDENTITY_LEDGER.json``, +schema v2). + +CEG-001 asked for an immutable SHA in the manifests instead. The release set +kept the moving tag and closed the finding the other way: the **lock** is the +identity every deployed image installs, so the lock is what this script +verifies, and the resolved commit is printed on PASS so a build log records +which SDK object was actually taken. ``Enrichment.Inference.Engine/scripts/ +validate_sdk_pin.py`` enforces the same contract over that repo's +``requirements.lock``. + +That left one trade-off documented as accepted: a build resolving the manifest +live takes whatever ``v1`` points at that minute, while a lock-driven build +takes ``resolved_reference``, and moving the tag without refreshing the lock +separates them. ``--verify-tag`` turns that accepted risk into a detected +failure — it resolves the channel at the canonical remote and compares. The +divergence is still a deliberate act, but it is no longer one this repository +has to notice by hand. + +Modes +----- +default Offline structural checks over the active surfaces. +--verify-tag Resolves ``v1`` at the canonical remote and requires + ``resolved_reference`` to match. Fails closed when the remote + cannot be resolved — a lock that cannot be checked has not been + checked. """ from __future__ import annotations +import argparse import re +import subprocess from pathlib import Path ROOT = Path(__file__).resolve().parents[1] MAJOR_TAG = "v1" CANONICAL_REPO = "Quantum-L9/Gate_SDK" +CANONICAL_REMOTE = f"https://github.com/{CANONICAL_REPO}.git" FORBIDDEN_FORK = "cryptoxdog/Gate_SDK" SHA_RE = re.compile(r"\b[0-9a-f]{40}\b") LOCK_REFERENCE_RE = re.compile( @@ -57,13 +87,131 @@ def check_tree(root: Path) -> list[str]: return errors -def main() -> int: - errors = check_tree(ROOT) +def lock_resolution(text: str) -> str | None: + """The concrete object poetry recorded for the moving tag.""" + match = LOCK_REFERENCE_RE.search(text) + return match.group(2) if match else None + + +def safe_remote(remote: str) -> str: + """Return a remote from a fixed set — never the caller's string. + + Passing argv as a list and never invoking a shell stops *command* + injection, but not *argument* injection: ``git ls-remote + --upload-pack= `` runs ````, so a remote beginning with + ``-`` is an execution vector on its own (SonarCloud + pythonsecurity:S8705). + + The canonical remote is the contract, so it is matched by equality and the + module constant is returned: what reaches git is provably not built from + the argument. Anything else is a local fixture path used by the tests, + which must be an existing git repository directory. The CLI deliberately + exposes no --remote flag — an operator pointing this check at a + non-canonical repository is exactly what the release-identity contract + exists to prevent. + """ + if remote == CANONICAL_REMOTE: + return CANONICAL_REMOTE + if not remote or remote.startswith("-"): + msg = f"refusing Gate_SDK remote {remote!r}: a remote must not begin with '-'" + raise ValueError(msg) + candidate = Path(remote) + if not candidate.is_dir(): + msg = f"refusing Gate_SDK remote {remote!r}: not canonical and not a local repository" + raise ValueError(msg) + return str(candidate.resolve(strict=True)) + + +def resolve_remote_tag(remote: str, tag: str) -> str | None: + """Resolve ``refs/tags/`` at *remote*, preferring the peeled object.""" + completed = subprocess.run( + [ + "/usr/bin/git", + "ls-remote", + "--tags", + "--end-of-options", + safe_remote(remote), + f"refs/tags/{tag}", + ], + check=False, + text=True, + capture_output=True, + ) + if completed.returncode != 0: + return None + peeled: str | None = None + direct: str | None = None + for line in completed.stdout.splitlines(): + sha, _, name = line.partition("\t") + if name == f"refs/tags/{tag}^{{}}": + peeled = sha.strip() + elif name == f"refs/tags/{tag}": + direct = sha.strip() + return peeled or direct + + +def resolved_commit(root: Path) -> str | None: + """The commit poetry.lock records for the major tag, for the PASS line.""" + path = root / "poetry.lock" + if not path.exists(): + return None + match = LOCK_REFERENCE_RE.search(path.read_text(encoding="utf-8")) + return None if match is None else match.group(2) + + +def compare_lock_to_tag(resolved: str | None, tag_sha: str | None) -> list[str]: + """Stale-lock detection: the lock must hold what the channel points at now.""" + if tag_sha is None: + return [f"--verify-tag: could not resolve {CANONICAL_REPO}@{MAJOR_TAG}; an unverifiable lock does not pass"] + if resolved is None: + return ["--verify-tag: poetry.lock carries no resolved_reference to compare"] + if resolved != tag_sha: + return [ + ( + f"--verify-tag: poetry.lock resolved_reference is {resolved} but " + f"{MAJOR_TAG} now points at {tag_sha} — the lock is stale; re-run `poetry lock`" + ) + ] + return [] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Validate the Gate_SDK moving-major pin.") + parser.add_argument("--root", type=Path, default=ROOT) + parser.add_argument( + "--verify-tag", + action="store_true", + help="resolve the channel at the canonical remote and compare to poetry.lock", + ) + args = parser.parse_args(argv) + + errors = check_tree(args.root) + + if args.verify_tag: + try: + tag_sha = resolve_remote_tag(CANONICAL_REMOTE, MAJOR_TAG) + except ValueError as exc: + # Fails closed like any other unresolvable channel, but says which + # of the two reasons it was. + errors.append(f"--verify-tag: {exc}") + else: + errors.extend(compare_lock_to_tag(resolved_commit(args.root), tag_sha)) + if not errors: + print(f"NETWORK: {CANONICAL_REPO}@{MAJOR_TAG} == poetry.lock {tag_sha}") + if errors: print("FAIL") print("\n".join(errors)) return 1 - print(f"PASS CEG pin {CANONICAL_REPO}@{MAJOR_TAG}") + mode = "offline + networked" if args.verify_tag else "offline" + resolved = resolved_commit(args.root) + if resolved is None: + # check_tree() already accepts a tree with no poetry.lock, so this is + # reachable. Printing "PASS ... -> None" in a build log reads as a pin + # that resolved to nothing; say what is actually true instead. + print(f"PASS CEG pin {CANONICAL_REPO}@{MAJOR_TAG} ({mode}; no poetry.lock, no resolved commit recorded)") + return 0 + print(f"PASS CEG pin {CANONICAL_REPO}@{MAJOR_TAG} ({mode}) -> {resolved}") return 0 diff --git a/templates/.env.recommended.template b/templates/.env.recommended.template index 70f134e1..75e24555 100755 --- a/templates/.env.recommended.template +++ b/templates/.env.recommended.template @@ -2,5 +2,4 @@ POSTGRES_USER POSTGRES_DB POSTGRES_PASSWORD -REDIS_URL LOG_LEVEL diff --git a/tests/contracts/_constants.py b/tests/contracts/_constants.py index 722941ea..df298d78 100644 --- a/tests/contracts/_constants.py +++ b/tests/contracts/_constants.py @@ -15,7 +15,8 @@ OUTCOME_VALUES = {"success", "failure", "partial"} -REQUIRED_ENV_VARS = ["NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "REDIS_URL", "API_KEY"] +# CEG-007: REDIS_URL removed — no code under engine/ or chassis/ imports redis. +REQUIRED_ENV_VARS = ["NEO4J_URI", "NEO4J_USERNAME", "NEO4J_PASSWORD", "API_KEY"] FORBIDDEN_PROD_SECRETS: dict[str, list[str]] = { "NEO4J_PASSWORD": ["password", "change-me-in-production"], diff --git a/tests/contracts/conftest.py b/tests/contracts/conftest.py index 25f5e8a6..9f55c86d 100644 --- a/tests/contracts/conftest.py +++ b/tests/contracts/conftest.py @@ -69,14 +69,6 @@ def neo4j_dep() -> dict: return _load_yaml(path) -@pytest.fixture -def redis_dep() -> dict: - path = CONTRACTS_ROOT / "dependencies" / "redis.yaml" - if not path.exists(): - pytest.skip("redis.yaml not present") - return _load_yaml(path) - - @pytest.fixture def tool_index() -> dict: path = CONTRACTS_ROOT / "agents" / "tool-schemas" / "_index.yaml" diff --git a/tests/contracts/test_dependency_contracts.py b/tests/contracts/test_dependency_contracts.py index e098d817..2b11daa7 100644 --- a/tests/contracts/test_dependency_contracts.py +++ b/tests/contracts/test_dependency_contracts.py @@ -1,5 +1,5 @@ """ -External dependency contract tests for Neo4j and Redis. +External dependency contract tests for Neo4j. Sources: engine/graph/driver.py:GraphDriver, CircuitBreaker @@ -62,24 +62,9 @@ def test_neo4j_dep_direct_access_forbidden(neo4j_dep): assert direct == "forbidden" -# ── Redis dependency contract ──────────────────────────────────────────────── - - -def test_redis_dep_service_name(redis_dep): - assert redis_dep.get("service_name") == "redis" - - -def test_redis_dep_version_is_7(redis_dep): - version = str(redis_dep.get("version", "")) - assert "7" in version - - -def test_redis_dep_uses_env_var(redis_dep): - conn = redis_dep.get("connection", {}) - assert conn.get("base_url_env") == "REDIS_URL" - - -def test_redis_dep_usage_documents_scoring_cache(redis_dep): - usages = [u.get("purpose", "").lower() for u in redis_dep.get("usage", [])] - scoring_cached = any("scor" in u or "cache" in u or "gds" in u for u in usages) - assert scoring_cached +# CEG-007: the Redis dependency-contract tests are gone with the contract they +# asserted. redis.yaml declared "Scoring result caching" and "Domain pack cache" +# as its usages; neither was ever implemented, and nothing under engine/ or +# chassis/ imports redis. Four tests passed continuously against a description +# of behaviour that did not exist — which is what made the unused dependency +# look load-bearing. diff --git a/tests/unit/test_cypher_lint.py b/tests/unit/test_cypher_lint.py new file mode 100644 index 00000000..a131e7da --- /dev/null +++ b/tests/unit/test_cypher_lint.py @@ -0,0 +1,317 @@ +"""Unit tests — tools/cypher_lint.py, the C-009 scanner. + +The scanner is contract enforcement, so these tests pin both directions: +every unsafe shape it must catch, and every validated shape it must pass — +including the two cases the 2026-09-20 audit found the keyword-gated +predecessor wrong on (F280-2): a raw ``NOT EXISTS((query)-[:{edge}]->(c))`` +fragment with none of the old trigger keywords, and a module-constant label +in a multi-line ``MERGE``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools.cypher_lint import Finding, scan_source, scan_tree + +pytestmark = pytest.mark.unit + + +def _blocking(source: str) -> list[Finding]: + return [f for f in scan_source(source, rel_path="engine/x.py") if f.blocking] + + +def _kinds(source: str) -> list[str]: + return [f.kind.split(" —")[0] for f in _blocking(source)] + + +# ── must catch ────────────────────────────────────────────────────────────── + + +def test_quoted_value_interpolation_fails() -> None: + src = "def compile(status):\n return f\"SET n.status = '{status}'\"\n" + assert _kinds(src) == ["quoted value interpolation"] + + +def test_double_quoted_value_interpolation_fails() -> None: + src = "def compile(v):\n cypher = f'MATCH (n) WHERE n.x = \"{v}\" RETURN n'\n return cypher\n" + assert _kinds(src) == ["quoted value interpolation"] + + +def test_limit_and_skip_interpolation_fail() -> None: + # The keyword is interpolated so this fixture stays out of tools/contract_scanner.py SEC-004's line regex. + kw = "LIMIT" + src = f"def q(n, k):\n return f'MATCH (n) RETURN n SKIP {{k}} {kw} {{n}}'\n" + assert _kinds(src) == ["LIMIT / SKIP bound interpolated", "LIMIT / SKIP bound interpolated"] + + +def test_raw_spec_edge_type_in_exists_pattern_without_trigger_keywords_fails() -> None: + """F280-2 false negative: no MATCH/WHERE/… token, yet a live label interpolation.""" + src = ( + "class ExclusionGate:\n" + " def compile(self):\n" + " edge = self.spec.edgetype\n" + ' from_node = self.spec.fromnode or "query"\n' + ' to_node = self.spec.tonode or "candidate"\n' + ' return f"NOT EXISTS(({from_node})-[:{edge}]->({to_node}))"\n' + ) + findings = _blocking(src) + assert [f.expression for f in findings] == ["from_node", "edge", "to_node"] + assert findings[1].kind.startswith("label / relationship type is not validated") + assert findings[0].kind.startswith("raw domain-spec value") + + +def test_unvalidated_label_position_fails() -> None: + src = "def q(label):\n return f'MATCH (n:{label}) RETURN n'\n" + assert _kinds(src) == ["label / relationship type is not validated (sanitize_label)"] + + +def test_unvalidated_property_position_fails() -> None: + src = "def q(prop):\n return f'MATCH (n) WHERE n.{prop} IS NULL RETURN n'\n" + assert _kinds(src) == ["property name is not validated (sanitize_label)"] + + +def test_unvalidated_parameter_name_fails() -> None: + src = "def compile(gate):\n return f'candidate.x = ${gate.queryparam}'\n" + assert _kinds(src) == ["parameter name is not validated"] + + +def test_raw_spec_value_in_bare_position_inside_compile_fails() -> None: + """A fragment with no Cypher keyword at all is still scanned inside a compiler.""" + src = "def compile(self):\n op = self.spec.operator\n return f'{self._prop_ref()} {op} $x'\n" + assert _kinds(src) == ["raw domain-spec value interpolated into Cypher"] + + +def test_raw_metadata_get_fails() -> None: + src = "def _compile_x(dim):\n v = dim.metadata.get('success_value', 'won')\n return f'WHERE o.t = {v}'\n" + assert _kinds(src) == ["raw domain-spec value interpolated into Cypher"] + + +def test_multiline_fstring_is_analysed_as_one_unit() -> None: + src = 'def q(label):\n return f"""\n MATCH (n:{label})\n RETURN n\n """\n' + findings = _blocking(src) + assert len(findings) == 1 + assert findings[0].line_no == 3 + + +def test_unvalidated_backquoted_identifier_fails() -> None: + src = "async def _provision(self, name):\n cypher = f'CREATE DATABASE `{name}` IF NOT EXISTS WAIT'\n" + assert _kinds(src) == ["back-quoted identifier is not validated (sanitize_database_name / sanitize_label)"] + + +def test_fstring_flowing_to_execute_query_is_scanned_without_keywords() -> None: + src = "async def run(driver, x):\n await driver.execute_query(f\"'{x}'\", database='neo4j')\n" + assert _kinds(src) == ["quoted value interpolation"] + + +# ── must pass ─────────────────────────────────────────────────────────────── + + +def test_sanitize_label_call_passes() -> None: + src = ( + "from engine.utils.security import sanitize_label\n" + "def q(label):\n" + " return f'MATCH (n:{sanitize_label(label)}) RETURN n'\n" + ) + assert _blocking(src) == [] + + +def test_module_constant_label_in_multiline_merge_passes() -> None: + """F280-2 false positive: the shape that failed current main's idea_portfolio.py.""" + src = ( + 'STATE_LABEL = "IdeaPortfolioHydrationState"\n' + 'STATE_ID_PROPERTY = "state_id"\n' + '_LOCK_STATE_CYPHER = f"""MERGE (state:{STATE_LABEL} {{{STATE_ID_PROPERTY}: $state_id}})\n' + "ON CREATE SET state.tenant = $tenant\n" + 'RETURN state"""\n' + ) + assert _blocking(src) == [] + + +def test_name_assigned_from_sanitizer_passes() -> None: + src = "def q(self):\n prop = sanitize_label(self.spec.candidateprop)\n return f'candidate.{prop} >= $min'\n" + assert _blocking(src) == [] + + +def test_self_attribute_assigned_from_sanitizer_passes() -> None: + src = ( + "class X:\n" + " def __init__(self, spec):\n" + " self._label = sanitize_label(spec.label)\n" + " def q(self):\n" + " return f'MATCH (n:{self._label}) RETURN n'\n" + ) + assert _blocking(src) == [] + + +def test_same_module_helper_returning_validated_value_passes() -> None: + src = ( + "class S:\n" + " def _get_candidate_label(self, job):\n" + " return sanitize_label(job.label)\n" + " def q(self, job):\n" + " node_label = self._get_candidate_label(job)\n" + " return f'MATCH (f:{node_label}) RETURN f'\n" + ) + assert _blocking(src) == [] + + +def test_parameter_name_built_from_sanitized_parts_passes() -> None: + src = ( + "def _compile(dim):\n" + " safe_dim = sanitize_label(dim.name)\n" + " key = f'pref_success_{safe_dim}'\n" + " return f'WHERE o.t = ${key}'\n" + ) + assert _blocking(src) == [] + + +def test_join_over_sanitized_comprehension_passes() -> None: + src = ( + "def q(self):\n" + " safe_types = [sanitize_label(t) for t in self._spec.edge_types]\n" + " edge_pattern = '|'.join(safe_types)\n" + " depth = int(self._spec.chain_depth_limit)\n" + " return f'MATCH p = (a)-[:{edge_pattern}*1..{depth}]->(b) RETURN p'\n" + ) + assert _blocking(src) == [] + + +def test_numeric_cast_and_allow_list_lookup_pass() -> None: + src = ( + "_OPERATORS = {'>=': '>=', '<=': '<='}\n" + "def compile(gate):\n" + " op = _OPERATORS[gate.operator or '>=']\n" + " days = int(gate.maxagedays or 1)\n" + " return f'candidate.x {op} $y AND candidate.d >= datetime() - duration({{days: {days}}})'\n" + ) + assert _blocking(src) == [] + + +def test_parameterized_values_pass() -> None: + src = ( + "def compile(self):\n" + " ref = self._bind_param('key_0', self.spec.mapping)\n" + " return f'CASE WHEN $query.x = {ref} THEN candidate.y IN $vals ELSE false END'\n" + ) + assert _blocking(src) == [] + + +def test_diagnostic_fstrings_are_not_cypher() -> None: + src = ( + "def check(self, gate, name):\n" + " if not gate.queryparam:\n" + " raise ValueError(f\"Gate '{gate.name}': queryparam required\")\n" + " msg = f\"Gate '{gate.name}' MATCH failed\"\n" + ' logger.warning(f"Unknown gate type: {gate.type}, WHERE is it")\n' + " warnings.append(f\"Gate '{gate.name}': parameter '${gate.queryparam}' missing\")\n" + " return {'status': 'skipped', 'reason': f'unknown algorithm: {name}'}\n" + ) + assert _blocking(src) == [] + + +def test_fstring_passed_to_sanitizer_is_validated_by_the_call() -> None: + src = "def _x(dim):\n prop_name = sanitize_label(f'_prior_{dim.name}')\n return f'candidate.{prop_name}'\n" + assert _blocking(src) == [] + + +def test_non_cypher_fstrings_outside_compilers_are_ignored() -> None: + src = ( + "def cache_key(tenant, entity_id, digest):\n" + " return f'ceg:enrich:{tenant}:{entity_id}:{digest[:16]}'\n" + "def path(prefix, key):\n" + " return f'{prefix}.{key}'\n" + ) + assert _blocking(src) == [] + + +# ── facts are lexically scoped (Codex P1 / Copilot on PR #285) ────────────── + + +def test_sanitizer_binding_in_one_function_does_not_certify_another() -> None: + src = ( + "def safe(value):\n" + " label = sanitize_label(value)\n" + " return f'MATCH (n:{label}) RETURN n'\n" + "def unsafe(label):\n" + " return f'MATCH (n:{label}) RETURN n'\n" + ) + findings = _blocking(src) + assert [(f.line_no, f.expression) for f in findings] == [(5, "label")] + + +def test_binding_in_an_enclosing_scope_is_visible_to_nested_functions() -> None: + src = ( + "LABEL = sanitize_label(spec.label)\n" + "def outer(x):\n" + " prop = sanitize_label(x)\n" + " def inner():\n" + " return f'MATCH (n:{LABEL}) WHERE n.{prop} IS NULL RETURN n'\n" + " return inner()\n" + ) + assert _blocking(src) == [] + + +def test_helper_is_validated_only_by_its_own_returns() -> None: + """A validated return inside a nested def must not certify the outer helper.""" + src = ( + "class S:\n" + " def _label(self, job):\n" + " def _inner(v):\n" + " return sanitize_label(v)\n" + " return job.label\n" + " def q(self, job):\n" + " node_label = self._label(job)\n" + " return f'MATCH (f:{node_label}) RETURN f'\n" + ) + findings = _blocking(src) + assert [f.expression for f in findings] == ["node_label"] + + +# ── waivers are explicit and visible ──────────────────────────────────────── + + +def test_reasoned_waiver_is_reported_but_not_blocking() -> None: + src = ( + "def compile(self):\n" + " pattern = self.spec.pattern\n" + " return f'EXISTS {{ MATCH {pattern} }}' # cypher-lint: allow spec-authored escape hatch\n" + ) + findings = scan_source(src, rel_path="engine/x.py") + assert len(findings) == 1 + assert findings[0].waiver == "spec-authored escape hatch" + assert not findings[0].blocking + + +def test_waiver_without_a_reason_is_ignored() -> None: + src = "def compile(self):\n pattern = self.spec.pattern\n return f'EXISTS {{ MATCH {pattern} }}' # cypher-lint: allow\n" + assert len(_blocking(src)) == 1 + + +# ── tree scan ─────────────────────────────────────────────────────────────── + + +def test_scan_tree_reports_relative_path(tmp_path: Path) -> None: + target = tmp_path / "engine" / "sync" + target.mkdir(parents=True) + (target / "generator.py").write_text( + "def generate(status):\n return f\"SET n.status = '{status}'\"\n", + encoding="utf-8", + ) + findings = scan_tree(tmp_path) + assert [f.rel_path for f in findings] == ["engine/sync/generator.py"] + assert findings[0].line_no == 2 + assert "status" in findings[0].pattern + + +def test_scan_tree_without_engine_dir_is_empty(tmp_path: Path) -> None: + assert scan_tree(tmp_path) == [] + + +def test_live_engine_tree_has_no_blocking_findings() -> None: + """The scanner is the C-009 gate: the checked-in engine must pass it.""" + root = Path(__file__).resolve().parents[2] + blocking = [f for f in scan_tree(root) if f.blocking] + assert blocking == [], "\n".join(f"{f.rel_path}:{f.line_no} {f.kind} {{{f.expression}}}" for f in blocking) diff --git a/tests/unit/test_cypher_utils.py b/tests/unit/test_cypher_utils.py index 1a1b32e1..4f576d77 100644 --- a/tests/unit/test_cypher_utils.py +++ b/tests/unit/test_cypher_utils.py @@ -45,3 +45,30 @@ def test_sanitize_label_rejects_too_long(): with pytest.raises((ValueError, Exception)): sanitize_label("A" * 200) + + +@pytest.mark.unit +@pytest.mark.parametrize("name", ["plasticos", "healthcare-referral", "acme.tenant_01", "abc"]) +def test_sanitize_database_name_accepts_neo4j_database_names(name: str) -> None: + from engine.utils.security import sanitize_database_name + + assert sanitize_database_name(name) == name + + +@pytest.mark.unit +@pytest.mark.parametrize( + "name", + [ + "plasticos`; DROP DATABASE neo4j; --", # back-quote escape + "1leading-digit", + "ab", # under the 3-character minimum + "has space", + "", + "a" * 64, # over the 63-character maximum + ], +) +def test_sanitize_database_name_rejects_unsafe_names(name: str) -> None: + from engine.utils.security import sanitize_database_name + + with pytest.raises(ValueError, match="refusing to provision"): + sanitize_database_name(name) diff --git a/tests/unit/test_domain_database_provisioning.py b/tests/unit/test_domain_database_provisioning.py new file mode 100644 index 00000000..caf8b7bf --- /dev/null +++ b/tests/unit/test_domain_database_provisioning.py @@ -0,0 +1,303 @@ +"""CEG-008 — a domain database that does not exist must say so, or be created. + +`match` and `sync` route queries to a Neo4j database named after the domain id +(`engine/handlers.py`, `database=domain_spec.domain.id`). Neo4j does not create +databases implicitly, so on a fresh instance every sync and match failed with +the driver's own message until an operator ran CREATE DATABASE by hand — the +Constellation E2E hit exactly this and had to add the step to its boot sequence. + +Two halves, both tested here: with `auto_create_domain_database` on the engine +provisions on first use; with it off the failure names the missing database and +the exact command that provides it. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from engine.graph.driver import ( + DatabaseNotProvisionedError, + GraphDriver, + _looks_like_absent_database, +) + + +class _RecordingDriver(GraphDriver): + """GraphDriver with the Neo4j round trip replaced by a recorder.""" + + def __init__(self, *, fail_with: Exception | None = None) -> None: + super().__init__(uri="bolt://unused", username="u", password="p") + self.calls: list[tuple[str, str]] = [] + self._fail_with = fail_with + + async def _raw_execute_write( # type: ignore[override] + self, + transaction_function: Any = None, + *args: Any, + cypher: str | None = None, + parameters: dict[str, Any] | None = None, + database: str = "neo4j", + **kwargs: Any, + ) -> dict[str, Any]: + self.calls.append((f"WRITE {cypher}", database)) + if self._fail_with is not None and database != "system": + raise self._fail_with + return {"nodes_created": 0} + + async def _raw_execute_query( # type: ignore[override] + self, + cypher: str, + parameters: dict[str, Any] | None = None, + database: str = "neo4j", + ) -> list[Any]: + self.calls.append((cypher, database)) + if self._fail_with is not None and database != "system": + raise self._fail_with + return [] + + +@pytest.fixture +def auto_create(monkeypatch): + from engine.config import settings as settings_module + + monkeypatch.setattr(settings_module.settings, "auto_create_domain_database", True) + + +@pytest.fixture +def no_auto_create(monkeypatch): + from engine.config import settings as settings_module + + monkeypatch.setattr(settings_module.settings, "auto_create_domain_database", False) + + +# ── Off: the failure has to be actionable ─────────────────────────────────── + + +def test_absent_database_is_recognised() -> None: + assert _looks_like_absent_database(Exception("Database does not exist: plasticos")) + assert _looks_like_absent_database(Exception("Unable to get a routing table for database")) + assert not _looks_like_absent_database(Exception("SyntaxError: bad cypher")) + + +@pytest.mark.asyncio +async def test_missing_database_names_itself_and_the_fix(no_auto_create) -> None: + driver = _RecordingDriver(fail_with=Exception("Database does not exist: plasticos")) + with pytest.raises(DatabaseNotProvisionedError) as excinfo: + await driver.execute_query("MATCH (n) RETURN n", database="plasticos") + + message = str(excinfo.value) + assert "plasticos" in message + assert "CREATE DATABASE" in message + assert "AUTO_CREATE_DOMAIN_DATABASE" in message + + +@pytest.mark.asyncio +async def test_unrelated_errors_are_not_reinterpreted(no_auto_create) -> None: + boom = ValueError("SyntaxError: unexpected token") + driver = _RecordingDriver(fail_with=boom) + with pytest.raises(ValueError, match="unexpected token"): + await driver.execute_query("MATCH (n RETURN n", database="plasticos") + + +@pytest.mark.asyncio +async def test_nothing_is_provisioned_while_the_flag_is_off(no_auto_create) -> None: + driver = _RecordingDriver() + await driver.execute_query("MATCH (n) RETURN n", database="plasticos") + assert not any("CREATE DATABASE" in cypher for cypher, _ in driver.calls) + + +@pytest.mark.asyncio +async def test_missing_database_on_a_write_names_itself_and_the_fix(no_auto_create) -> None: + driver = _RecordingDriver(fail_with=Exception("Database does not exist: plasticos")) + with pytest.raises(DatabaseNotProvisionedError, match="CREATE DATABASE"): + await driver.execute_write(cypher="MERGE (n:X)", database="plasticos") + + +# ── On: provision once, on first use ──────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_first_use_write_is_provisioned_too(auto_create) -> None: + """A fresh deployment's first operation can be a managed write (idea-portfolio + corpus sync), not a match query; provisioning must not depend on which.""" + driver = _RecordingDriver() + await driver.execute_write(cypher="MERGE (n:X)", database="plasticos") + assert driver.calls == [ + ("CREATE DATABASE `plasticos` IF NOT EXISTS WAIT", "system"), + ("WRITE MERGE (n:X)", "plasticos"), + ] + # ...and the query that follows reuses the ensured state. + await driver.execute_query("MATCH (n) RETURN n", database="plasticos") + assert sum("CREATE DATABASE" in c for c, _ in driver.calls) == 1 + + +@pytest.mark.asyncio +async def test_database_is_created_on_first_use(auto_create) -> None: + driver = _RecordingDriver() + await driver.execute_query("MATCH (n) RETURN n", database="plasticos") + + creates = [(c, db) for c, db in driver.calls if "CREATE DATABASE" in c] + assert creates, "expected the domain database to be provisioned" + cypher, target = creates[0] + assert cypher == "CREATE DATABASE `plasticos` IF NOT EXISTS WAIT" + assert target == "system", "administrative commands must run against `system`" + + +@pytest.mark.asyncio +async def test_creation_is_attempted_only_once_per_process(auto_create) -> None: + driver = _RecordingDriver() + for _ in range(3): + await driver.execute_query("MATCH (n) RETURN n", database="plasticos") + assert sum("CREATE DATABASE" in c for c, _ in driver.calls) == 1 + + +@pytest.mark.asyncio +async def test_builtin_databases_are_never_provisioned(auto_create) -> None: + driver = _RecordingDriver() + await driver.execute_query("RETURN 1", database="neo4j") + await driver.execute_query("SHOW DATABASES", database="system") + assert not any("CREATE DATABASE" in c for c, _ in driver.calls) + + +@pytest.mark.asyncio +async def test_a_hyphenated_domain_id_is_accepted() -> None: + """Domain ids legitimately contain dashes, so sanitize_label cannot be used.""" + driver = _RecordingDriver() + assert await driver.ensure_database("healthcare-referral") is True + assert ("CREATE DATABASE `healthcare-referral` IF NOT EXISTS WAIT", "system") in driver.calls + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "name", + [ + "plasticos`; DROP DATABASE neo4j; --", # back-quote escape + "1leading-digit", + "ab", # under the 3-character minimum + "has space", + "", + ], +) +async def test_an_unsafe_database_name_is_refused(name: str) -> None: + """CREATE DATABASE takes no parameter, so the name is interpolated — and + therefore validated before it ever reaches the statement.""" + driver = _RecordingDriver() + with pytest.raises(ValueError, match="refusing to provision"): + await driver.ensure_database(name) + assert driver.calls == [] + + +class _RefusingDriver(_RecordingDriver): + """Community Edition: CREATE DATABASE is rejected, ordinary queries work.""" + + async def _raw_execute_query(self, cypher, parameters=None, database="neo4j"): + self.calls.append((cypher, database)) + if "CREATE DATABASE" in cypher: + msg = "Unsupported administration command in Community Edition" + raise RuntimeError(msg) + return [] + + +@pytest.mark.asyncio +async def test_provisioning_failure_is_reported_not_raised(auto_create) -> None: + """Community Edition rejects CREATE DATABASE. A deployment whose database + already exists must not be blocked by a CREATE it is not allowed to run.""" + driver = _RefusingDriver() + assert await driver.ensure_database("plasticos") is False + # The ordinary query still runs; it is the query that reports the truth. + assert await driver.execute_query("MATCH (n) RETURN n", database="plasticos") == [] + + +@pytest.mark.asyncio +async def test_failed_provisioning_is_retried_on_the_next_call() -> None: + driver = _RefusingDriver() + assert await driver.ensure_database("plasticos") is False + assert await driver.ensure_database("plasticos") is False + assert sum("CREATE DATABASE" in c for c, _ in driver.calls) == 2 + + +# ── On: concurrent first use ──────────────────────────────────────────────── + + +class _SlowCreateDriver(_RecordingDriver): + """The CREATE suspends until the test releases it, like a real `WAIT` would.""" + + def __init__(self) -> None: + super().__init__() + self.release = asyncio.Event() + self.create_started = asyncio.Event() + + async def _raw_execute_query(self, cypher, parameters=None, database="neo4j"): + self.calls.append((cypher, database)) + if "CREATE DATABASE" in cypher: + self.create_started.set() + await self.release.wait() + return [] + + +@pytest.mark.asyncio +async def test_concurrent_first_use_issues_one_create_and_no_query_runs_before_it_completes(auto_create) -> None: + """F283-1: a second request arriving while the CREATE is in flight must + wait for it — not see the name already claimed and race into the domain + query against a database that does not exist yet.""" + driver = _SlowCreateDriver() + requests = [asyncio.create_task(driver.execute_query("MATCH (n) RETURN n", database="plasticos")) for _ in range(5)] + + await driver.create_started.wait() + await asyncio.sleep(0) # let every request reach its await + assert driver.calls == [("CREATE DATABASE `plasticos` IF NOT EXISTS WAIT", "system")], ( + "no domain query may run while provisioning is in flight, and only one CREATE may be issued" + ) + assert all(not task.done() for task in requests) + + driver.release.set() + await asyncio.gather(*requests) + + creates = [c for c, _ in driver.calls if "CREATE DATABASE" in c] + queries = [(c, db) for c, db in driver.calls if "CREATE DATABASE" not in c] + assert len(creates) == 1 + assert queries == [("MATCH (n) RETURN n", "plasticos")] * 5 + # The CREATE precedes every domain query in the recorded order. + assert driver.calls[0][0].startswith("CREATE DATABASE") + + +@pytest.mark.asyncio +async def test_concurrent_callers_share_the_provisioning_outcome() -> None: + driver = _SlowCreateDriver() + waiters = [asyncio.create_task(driver.ensure_database("plasticos")) for _ in range(3)] + await driver.create_started.wait() + driver.release.set() + assert await asyncio.gather(*waiters) == [True, True, True] + assert sum("CREATE DATABASE" in c for c, _ in driver.calls) == 1 + + +@pytest.mark.asyncio +async def test_cancelling_one_waiter_does_not_cancel_the_create_for_the_others() -> None: + driver = _SlowCreateDriver() + first = asyncio.create_task(driver.ensure_database("plasticos")) + second = asyncio.create_task(driver.ensure_database("plasticos")) + await driver.create_started.wait() + await asyncio.sleep(0) + + first.cancel() + await asyncio.wait([first]) + assert first.cancelled() + + driver.release.set() + assert await second is True + assert "plasticos" in driver._ensured_databases + + +@pytest.mark.asyncio +async def test_in_flight_provisioning_is_forgotten_once_settled() -> None: + driver = _SlowCreateDriver() + task = asyncio.create_task(driver.ensure_database("plasticos")) + await driver.create_started.wait() + assert "plasticos" in driver._provisioning + driver.release.set() + assert await task is True + await asyncio.sleep(0) # done callbacks run on the next loop iteration + assert driver._provisioning == {} diff --git a/tests/unit/test_domain_pack_shape.py b/tests/unit/test_domain_pack_shape.py new file mode 100644 index 00000000..57c3e56a --- /dev/null +++ b/tests/unit/test_domain_pack_shape.py @@ -0,0 +1,165 @@ +"""CEG-009 — every domain under domains/ must be in the shape the loader reads. + +``DomainPackLoader`` resolves ``domains//spec.yaml`` +(``loader.SPEC_FILENAME``). Nine specs used a flat +``_domain_spec.yaml`` convention instead and were therefore unreachable: +they looked like available verticals in a directory listing, but a tenant id +matching one of them failed to resolve. Only ``plasticos`` loaded by default. + +Two of those nine were additionally invalid — ``executive-assistant`` declared +an edge to an undeclared ``Skill`` node, ``roofing-company`` one to ``ZipCode`` +— and nothing caught it, because a file the loader never opens is never +validated either. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +from engine.config.loader import SPEC_FILENAME, DomainPackLoader +from engine.config.schema import DomainSpec + +REPO_ROOT = Path(__file__).resolve().parents[2] +DOMAINS_ROOT = REPO_ROOT / "domains" + + +def _domain_dirs() -> list[Path]: + return sorted(p for p in DOMAINS_ROOT.iterdir() if p.is_dir()) + + +def test_no_flat_domain_spec_files_remain() -> None: + stragglers = sorted(p.name for p in DOMAINS_ROOT.glob("*_domain_spec.yaml")) + assert stragglers == [], ( + f"{stragglers} use the flat shape the loader never reads. Move each to " + f"domains//{SPEC_FILENAME}, or delete it if superseded." + ) + + +def test_every_domain_directory_holds_a_spec() -> None: + missing = [d.name for d in _domain_dirs() if not (d / SPEC_FILENAME).is_file()] + assert missing == [], f"domain directories without {SPEC_FILENAME}: {missing}" + + +@pytest.mark.parametrize("domain_dir", _domain_dirs(), ids=lambda p: p.name) +def test_domain_spec_validates(domain_dir: Path) -> None: + """A spec the loader can find is a spec that must survive validation.""" + raw = yaml.safe_load((domain_dir / SPEC_FILENAME).read_text(encoding="utf-8")) + DomainSpec.model_validate(raw) + + +@pytest.mark.parametrize("domain_dir", _domain_dirs(), ids=lambda p: p.name) +def test_directory_name_matches_declared_domain_id(domain_dir: Path) -> None: + """The loader resolves by domain id, so the directory name IS the lookup key.""" + raw = yaml.safe_load((domain_dir / SPEC_FILENAME).read_text(encoding="utf-8")) + assert raw["domain"]["id"] == domain_dir.name + + +def test_loader_discovers_every_unflagged_domain() -> None: + from engine.config.loader import _DOMAIN_FEATURE_FLAGS + + loader = DomainPackLoader(str(DOMAINS_ROOT)) + discovered = set(loader.list_domains()) + on_disk = {d.name for d in _domain_dirs()} + assert discovered <= on_disk + # Everything withheld is withheld deliberately, by a declared flag. + assert on_disk - discovered <= set(_DOMAIN_FEATURE_FLAGS) + + +def test_every_discovered_domain_actually_loads() -> None: + loader = DomainPackLoader(str(DOMAINS_ROOT)) + for domain_id in sorted(loader.list_domains()): + assert loader.load_domain(domain_id).domain.id == domain_id + + +# -------------------------------------------------------------------------- +# Readable is not the same as correct. +# -------------------------------------------------------------------------- + +_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +def _cypher_defects(spec) -> list[str]: + """Gate compilations that would fail or silently reject every candidate. + + Two shapes, both found only after CEG-009 made these packs readable: + + * ``RELATES_TO`` — the compiler's fallback when a ``type: traversal`` gate + declares no ``edgetype``. Packs written with ``pattern``/``condition`` + hit it, and no ontology here declares ``RELATES_TO``, so the gate is a + hard filter that matches nothing. + * ``strictwhen`` — declared by ``GateSpec`` and consumed by no compiler, so + a gate meant to be conditional runs as an unconditional hard filter. + * ``$`` — a scalar ``queryparam`` (``85.0``, ``5``, ``1``). + GateSpec coerces it to a string and the compiler used to emit it as a + parameter *name*, so Cypher received ``$85.0``. Since C-009 validates + parameter names like labels, the compiler now refuses the gate outright; + that refusal is the same defect, reported at compile time. + """ + from engine.gates.compiler import GateCompiler + + compiler = GateCompiler(spec) + defects: list[str] = [] + for gate in spec.gates: + if gate.strictwhen: + # Declared by GateSpec, consumed by nothing under engine/: the gate + # would run as an unconditional hard filter. + defects.append(f"{gate.name}: strictwhen {gate.strictwhen!r} is not consumed by any compiler") + try: + cypher = compiler.compile(gate) + except (ValueError, KeyError) as exc: + defects.append(f"{gate.name}: refused by the compiler — {exc}") + continue + if "RELATES_TO" in cypher and not gate.edgetype: + defects.append(f"{gate.name}: RELATES_TO fallback — {cypher}") + for param in re.findall(r"\$([^\s)]+)", cypher): + if not _IDENTIFIER.fullmatch(param): + defects.append(f"{gate.name}: '${param}' is not a parameter name — {cypher}") + return defects + + +@pytest.mark.parametrize("domain_dir", _domain_dirs(), ids=lambda p: p.name) +def test_discoverable_domain_gates_compile_to_executable_cypher(domain_dir: Path) -> None: + """A pack the loader will serve must produce Cypher that can run. + + The gap this closes: validating a spec against DomainSpec proves it parses, + not that its gates execute. Five packs passed Pydantic validation and still + compiled to `exists((candidate)-[:RELATES_TO]->(t))` or `$85.0`. + + A pack behind a feature flag is exempt — that is what the flag records. + """ + from engine.config.loader import _DOMAIN_FEATURE_FLAGS + + if domain_dir.name in _DOMAIN_FEATURE_FLAGS: + pytest.skip(f"{domain_dir.name} is flag-gated: {_DOMAIN_FEATURE_FLAGS[domain_dir.name]}") + + raw = yaml.safe_load((domain_dir / SPEC_FILENAME).read_text(encoding="utf-8")) + defects = _cypher_defects(DomainSpec.model_validate(raw)) + assert defects == [], f"{domain_dir.name} would serve unexecutable Cypher:\n " + "\n ".join(defects) + + +def test_flag_gated_packs_are_gated_for_a_reason_that_still_holds() -> None: + """Shrink-only: when a dormant pack starts compiling, un-gate it. + + Without this, a pack fixed later stays invisible and the flag becomes a + place defects go to be forgotten. + """ + from engine.config.loader import _DOMAIN_FEATURE_FLAGS + + still_broken, now_clean = [], [] + for name, flag in _DOMAIN_FEATURE_FLAGS.items(): + if flag != "unvalidated_domain_packs_enabled": + continue + path = DOMAINS_ROOT / name / SPEC_FILENAME + if not path.is_file(): + continue + spec = DomainSpec.model_validate(yaml.safe_load(path.read_text(encoding="utf-8"))) + (still_broken if _cypher_defects(spec) else now_clean).append(name) + + assert now_clean == [], ( + f"{now_clean} now compile cleanly — remove them from _DOMAIN_FEATURE_FLAGS so the loader serves them again." + ) + assert still_broken, "no pack is gated as unvalidated; drop the flag and this test" diff --git a/tests/unit/test_gates_all_types.py b/tests/unit/test_gates_all_types.py index 3d271cc0..8b706e00 100644 --- a/tests/unit/test_gates_all_types.py +++ b/tests/unit/test_gates_all_types.py @@ -22,6 +22,7 @@ from engine.gates.types.all_gates import ( BaseGate, BooleanGate, + EnumMapGate, ExclusionGate, FreshnessGate, ThresholdGate, @@ -431,3 +432,197 @@ def test_traversal_gate_compile(self) -> None: assert "EXISTS" in cypher assert "HAS" in cypher + + +# ============================================================================ +# C-009: GATE VALUES TRAVEL AS PARAMETERS, IDENTIFIERS ARE VALIDATED +# ============================================================================ + + +@pytest.mark.unit +class TestGateValueParameterization: + """F280-1: enum mapping keys/values are data, never quoted into the fragment.""" + + @staticmethod + def _enum_gate(mapping: dict) -> EnumMapGate: + spec = MagicMock() + spec.name = "polymer-map" + spec.candidateprop = "polymertype" + spec.queryparam = "polymertype" + spec.mapping = mapping + return EnumMapGate(spec, MagicMock()) + + def test_mapping_values_are_bound_as_parameters(self) -> None: + gate = self._enum_gate({"PET": ["PET", "rPET"], "HDPE": ["HDPE"]}) + + cypher = gate.compile() + params = gate.query_params + + assert cypher == ( + "CASE WHEN $query.polymertype = $gate_polymer_map_key_0 THEN candidate.polymertype IN $gate_polymer_map_values_0 " + "WHEN $query.polymertype = $gate_polymer_map_key_1 THEN candidate.polymertype IN $gate_polymer_map_values_1 " + "ELSE false END" + ) + assert params == { + "gate_polymer_map_key_0": "PET", + "gate_polymer_map_values_0": ["PET", "rPET"], + "gate_polymer_map_key_1": "HDPE", + "gate_polymer_map_values_1": ["HDPE"], + } + assert "'" not in cypher + + def test_legitimate_non_identifier_values_are_accepted(self) -> None: + """Spaces and hyphens are valid data — sanitize_label would have rejected them.""" + gate = self._enum_gate({"post-consumer": ["post consumer", "post-industrial", "mixed/bale"]}) + + cypher = gate.compile() + + assert "post-consumer" not in cypher + assert gate.query_params["gate_polymer_map_key_0"] == "post-consumer" + assert gate.query_params["gate_polymer_map_values_0"] == ["post consumer", "post-industrial", "mixed/bale"] + + def test_injection_shaped_values_never_reach_the_fragment(self) -> None: + payload = "x' OR 1=1 OR '" + gate = self._enum_gate({payload: ["a'] OR true OR ['b"]}) + + cypher = gate.compile() + + assert "OR 1=1" not in cypher + assert "OR true" not in cypher + assert gate.query_params["gate_polymer_map_key_0"] == payload + + def test_params_reset_between_compiles(self) -> None: + gate = self._enum_gate({"PET": ["PET"]}) + gate.compile() + gate.spec.mapping = {"HDPE": ["HDPE"]} + gate.compile() + assert gate.query_params == {"gate_polymer_map_key_0": "HDPE", "gate_polymer_map_values_0": ["HDPE"]} + + def test_no_mapping_binds_nothing(self) -> None: + gate = self._enum_gate({}) + assert gate.compile() == "$query.polymertype IN candidate.polymertype" + assert gate.query_params == {} + + def test_parameter_keys_stay_bounded_identifiers_for_long_gate_names(self) -> None: + gate = self._enum_gate({"PET": ["PET"]}) + gate.spec.name = "g" * 120 + gate.compile() + for key in gate.query_params: + assert len(key) <= 64 + assert key.replace("_", "a").isalnum() + + def test_composite_gate_collects_subgate_parameters(self) -> None: + from engine.config.schema import GateType + from engine.gates.types.all_gates import CompositeGate + + sub = MagicMock() + sub.name = "sub" + sub.type = GateType.ENUMMAP + sub.candidateprop = "p" + sub.queryparam = "q" + sub.mapping = {"a": ["b"]} + composite = MagicMock() + composite.name = "both" + composite.subgates = ["sub"] + composite.logic = "and" + domain_spec = MagicMock() + domain_spec.gates = [sub] + + gate = CompositeGate(composite, domain_spec) + cypher = gate.compile() + + assert " AND " in cypher or cypher.count("(") == 1 + assert gate.query_params == {"gate_sub_key_0": "a", "gate_sub_values_0": ["b"]} + + def test_composite_logic_outside_allow_list_is_rejected(self) -> None: + from engine.gates.types.all_gates import CompositeGate + + composite = MagicMock() + composite.name = "bad" + composite.subgates = ["x"] + composite.logic = "AND true OR" + domain_spec = MagicMock() + domain_spec.gates = [] + with pytest.raises((KeyError, ValueError)): + CompositeGate(composite, domain_spec).compile() + + +@pytest.mark.unit +class TestGateIdentifierValidation: + """Structural identifiers read from the spec are sanitized before interpolation.""" + + def test_exclusion_gate_sanitizes_edge_and_node_variables(self) -> None: + spec = MagicMock() + spec.name = "blocked" + spec.edgetype = "BLOCKED" + spec.fromnode = "q" + spec.tonode = "c" + assert ExclusionGate(spec, MagicMock()).compile() == "NOT EXISTS((q)-[:BLOCKED]->(c))" + + @pytest.mark.parametrize( + ("field", "value"), + [ + ("edgetype", "BLOCKED]->() RETURN 1 //"), + ("fromnode", "query) WHERE true OR (x"), + ("tonode", "candidate)) OR true OR (("), + ], + ) + def test_exclusion_gate_rejects_injection_shaped_identifiers(self, field: str, value: str) -> None: + spec = MagicMock() + spec.name = "blocked" + spec.edgetype = "BLOCKED" + spec.fromnode = None + spec.tonode = None + setattr(spec, field, value) + with pytest.raises(ValueError): + ExclusionGate(spec, MagicMock()).compile() + + def test_threshold_operator_outside_allow_list_is_rejected(self) -> None: + spec = MagicMock() + spec.name = "t" + spec.candidateprop = "score" + spec.queryparam = "score" + spec.operator = ">= 0 OR 1=1 OR candidate.score" + with pytest.raises(KeyError): + ThresholdGate(spec, MagicMock()).compile() + + def test_prop_and_param_refs_validate_identifiers(self) -> None: + spec = MagicMock() + spec.name = "b" + spec.candidateprop = "candidate.active" + spec.queryparam = "$query.active" + assert BooleanGate(spec, MagicMock()).compile() == "candidate.active = $query.active" + + spec.candidateprop = "active) OR true OR (x" + with pytest.raises(ValueError): + BooleanGate(spec, MagicMock()).compile() + spec.candidateprop = "active" + spec.queryparam = "active RETURN 1" + with pytest.raises(ValueError): + BooleanGate(spec, MagicMock()).compile() + + def test_gate_compiler_validates_query_parameter_names(self) -> None: + gate = make_mock_gate_spec( + name="credit_min", + gate_type=GateType.THRESHOLD, + candidate_prop="mincreditscore", + query_param="creditscore RETURN 1 //", + operator=">=", + ) + compiler = GateCompiler(make_mock_domain_spec(gates=[gate])) + with pytest.raises(ValueError): + compiler.compile(gate) + + def test_gate_compiler_validates_operator_and_logic(self) -> None: + threshold = make_mock_gate_spec( + name="t", gate_type=GateType.THRESHOLD, candidate_prop="score", query_param="score", operator="==" + ) + compiler = GateCompiler(make_mock_domain_spec(gates=[threshold])) + with pytest.raises(KeyError): + compiler.compile(threshold) + + composite = make_mock_gate_spec(name="c", gate_type=GateType.COMPOSITE, sub_gates=["t"], combinator="AND OR") + threshold.operator = ">=" + compiler = GateCompiler(make_mock_domain_spec(gates=[threshold, composite])) + with pytest.raises(KeyError): + compiler.compile(composite) diff --git a/tests/unit/test_graph_inference_egress.py b/tests/unit/test_graph_inference_egress.py new file mode 100644 index 00000000..2be3afe6 --- /dev/null +++ b/tests/unit/test_graph_inference_egress.py @@ -0,0 +1,145 @@ +"""EIE-008 / CEG-006 — the graph-inference feedback loop needs a producer. + +EIE advertises `graph-inference-result` to Gate and implements the entire +consumer side: packet validation, per-tenant queues, target extraction, a 0.55 +confidence floor, and injection into the convergence loop. No code in CEG ever +constructed such a packet, so the loop the architecture implies had a consumer +and nothing on the other end — and the two halves were built to the same +confidence floor without ever being connected. + +The mirror finding: CEG's own outbound `request_enrichment` was reachable only +through `engine/health/api.py`, which nothing imported, so no inbound packet +could cause CEG to call EIE either. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from engine.gate_egress import ( + GRAPH_INFERENCE_ACTION, + INFERENCE_CONFIDENCE_FLOOR, + build_inference_outputs, + emit_graph_inference_result, + inference_idempotency_key, +) +from engine.inference_rule_registry import InferenceResult + + +def _result(field: str, value: Any, confidence: float, rule: str = "r") -> InferenceResult: + return InferenceResult(field_name=field, value=value, confidence=confidence, rule_name=rule) + + +# ── Output shaping: EIE's contract is the spec ────────────────────────────── + + +def test_outputs_carry_every_key_eie_requires() -> None: + """EIE's extract_targets_from_packet reads entity_id, field, value, + confidence and rule off each element.""" + outputs = build_inference_outputs("e-1", [_result("tier", "mid_market", 0.85, "size_rule")]) + assert len(outputs) == 1 + assert outputs[0]["entity_id"] == "e-1" + assert outputs[0]["field"] == "tier" + assert outputs[0]["value"] == "mid_market" + assert outputs[0]["confidence"] == 0.85 + assert outputs[0]["rule"] == "size_rule" + + +def test_the_floor_matches_the_receiver() -> None: + """EIE drops below 0.55 (graph_return_channel.CONFIDENCE_FLOOR); so do we, + so a packet never carries outputs the receiver will silently discard.""" + assert INFERENCE_CONFIDENCE_FLOOR == 0.55 + outputs = build_inference_outputs( + "e-1", + [_result("kept", "a", 0.56), _result("dropped", "b", 0.54)], + ) + assert [o["field"] for o in outputs] == ["kept"] + + +def test_plain_mappings_are_accepted_too() -> None: + outputs = build_inference_outputs("e-1", [{"field": "f", "value": 1, "confidence": 0.9}]) + assert outputs[0]["rule"] == "unknown" + + +@pytest.mark.parametrize( + "bad", + [{"value": 1, "confidence": 0.9}, {"field": "f", "value": 1, "confidence": "high"}], +) +def test_malformed_outputs_are_dropped_not_sent(bad: dict) -> None: + assert build_inference_outputs("e-1", [bad]) == [] + + +def test_idempotency_key_is_stable_over_the_same_findings() -> None: + a = build_inference_outputs("e-1", [_result("f", "v", 0.9)]) + b = build_inference_outputs("e-1", [_result("f", "v", 0.9)]) + assert inference_idempotency_key("t", "e-1", a) == inference_idempotency_key("t", "e-1", b) + + +def test_idempotency_key_changes_when_the_finding_changes() -> None: + a = build_inference_outputs("e-1", [_result("f", "v1", 0.9)]) + b = build_inference_outputs("e-1", [_result("f", "v2", 0.9)]) + assert inference_idempotency_key("t", "e-1", a) != inference_idempotency_key("t", "e-1", b) + + +# ── Dispatch ──────────────────────────────────────────────────────────────── + + +class _FakeHeader: + packet_id = "pkt-1" + packet_type = "response" + correlation_id = None + + +class _FakeResponse: + header = _FakeHeader() + payload: dict[str, Any] = {"queued": 1} + + +@pytest.mark.asyncio +async def test_nothing_above_the_floor_sends_no_packet(monkeypatch) -> None: + """An empty inference_outputs list is valid to EIE and would cost a Gate + round trip to queue nothing.""" + + def _no_client(): + msg = "the Gate client must not be constructed" + raise AssertionError(msg) + + monkeypatch.setenv("GATE_URL", "http://gate.test") + monkeypatch.setattr("engine.gate_egress.get_gate_client", _no_client) + + result = await emit_graph_inference_result(tenant="t", entity_id="e-1", results=[_result("f", "v", 0.1)]) + assert result["status"] == "skipped" + assert result["sent_outputs"] == 0 + + +@pytest.mark.asyncio +async def test_unconfigured_gate_fails_closed(monkeypatch) -> None: + monkeypatch.delenv("GATE_URL", raising=False) + result = await emit_graph_inference_result(tenant="t", entity_id="e-1", results=[_result("f", "v", 0.9)]) + assert result["status"] == "failed" + assert result["error"] == "gate_not_configured" + + +@pytest.mark.asyncio +async def test_a_successful_emission_addresses_gate_with_eies_action(monkeypatch) -> None: + sent: dict[str, Any] = {} + + class _Client: + async def execute(self, **kwargs: Any) -> _FakeResponse: + sent.update(kwargs) + return _FakeResponse() + + monkeypatch.setenv("GATE_URL", "http://gate.test") + monkeypatch.setattr("engine.gate_egress.get_gate_client", _Client) + + result = await emit_graph_inference_result(tenant="acme", entity_id="e-1", results=[_result("tier", "small", 0.9)]) + + assert result["status"] == "ok" + assert result["sent_outputs"] == 1 + assert sent["action"] == GRAPH_INFERENCE_ACTION + assert sent["tenant"] == "acme" + assert sent["payload"]["inference_outputs"][0]["field"] == "tier" + # CEG never addresses EIE: the destination is Gate, resolved by action. + assert "destination_node" not in sent diff --git a/tests/unit/test_protocol_bodies.py b/tests/unit/test_protocol_bodies.py new file mode 100644 index 00000000..977d82b0 --- /dev/null +++ b/tests/unit/test_protocol_bodies.py @@ -0,0 +1,152 @@ +"""Protocol method bodies must be a docstring and nothing else (CEG#267). + +`docs/contracts/BANNED_PATTERNS.md` — "typing.Protocol method bodies": a +Protocol method is a structural signature, never executed, and the only body +that is valid Python *and* clean on every in-repo gate is a docstring alone. +The check therefore strips an optional leading docstring and requires the +remaining body to be empty; **any** statement fails, with its location. +""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +ENGINE = Path(__file__).resolve().parents[2] / "engine" + + +def _is_protocol_class(node: ast.ClassDef) -> bool: + return any( + (isinstance(base, ast.Name) and base.id == "Protocol") + or (isinstance(base, ast.Attribute) and base.attr == "Protocol") + for base in node.bases + ) + + +def _is_docstring(stmt: ast.stmt) -> bool: + return isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant) and isinstance(stmt.value.value, str) + + +def protocol_body_violations(tree: ast.AST, path: str) -> list[str]: + """Every statement after the optional docstring of every Protocol method, located.""" + violations: list[str] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef) or not _is_protocol_class(node): + continue + for item in node.body: + if not isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + body = item.body[1:] if item.body and _is_docstring(item.body[0]) else list(item.body) + violations.extend( + f"{path}:{stmt.lineno}: {node.name}.{item.name} body must be a docstring only, " + f"found {type(stmt).__name__} ({ast.unparse(stmt)})" + for stmt in body + ) + return violations + + +def _engine_protocol_methods() -> list[str]: + found: list[str] = [] + for path in sorted(ENGINE.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef) and _is_protocol_class(node): + found.extend( + f"{node.name}.{item.name}" + for item in node.body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + ) + return found + + +@pytest.mark.unit +def test_engine_has_protocol_methods() -> None: + assert _engine_protocol_methods(), "expected at least one engine Protocol method" + + +@pytest.mark.unit +def test_engine_protocol_methods_are_docstring_only() -> None: + failures: list[str] = [] + for path in sorted(ENGINE.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + failures.extend(protocol_body_violations(tree, path.as_posix())) + assert failures == [], "\n".join(failures) + + +# ── the checker itself must discriminate (F284-2) ─────────────────────────── + +_PROTOCOL = "from typing import Protocol\n\nclass P(Protocol):\n def m(self) -> int:\n" + + +@pytest.mark.unit +@pytest.mark.parametrize( + "body", + [ + ' """Doc."""\n', + ' """Doc.\n\n Multi-line.\n """\n', + ], + ids=["docstring", "multiline-docstring"], +) +def test_docstring_only_bodies_pass(body: str) -> None: + assert protocol_body_violations(ast.parse(_PROTOCOL + body), "p.py") == [] + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("body", "expected"), + [ + (' """Doc."""\n ...\n', "Expr (...)"), + (' """Doc."""\n pass\n', "Pass (pass)"), + (' """Doc."""\n raise NotImplementedError\n', "Raise (raise NotImplementedError)"), + (' """Doc."""\n raise NotImplementedError("x")\n', "Raise (raise NotImplementedError('x'))"), + (' """Doc."""\n raise RuntimeError("x")\n', "Raise (raise RuntimeError('x'))"), + (' """Doc."""\n return None\n', "Return (return None)"), + (' """Doc."""\n return 0\n', "Return (return 0)"), + (' """Doc."""\n x = 1\n', "Assign (x = 1)"), + (' """Doc."""\n self.x: int = 1\n', "AnnAssign (self.x: int = 1)"), + (' """Doc."""\n len("x")\n', "Expr (len('x'))"), + (' """Doc."""\n 1 + 1\n', "Expr (1 + 1)"), + (' """Doc."""\n if True:\n pass\n', "If (if True:"), + (" ...\n", "Expr (...)"), + (" pass\n", "Pass (pass)"), + (' """Doc."""\n """Second string is a statement."""\n', "Expr ('Second string is a statement.')"), + ], + ids=[ + "ellipsis", + "pass", + "raise-bare", + "raise-not-implemented", + "raise-other", + "return-none", + "return-value", + "assign", + "ann-assign", + "call", + "expression", + "compound", + "ellipsis-no-docstring", + "pass-no-docstring", + "second-string", + ], +) +def test_any_non_docstring_statement_fails_with_location(body: str, expected: str) -> None: + violations = protocol_body_violations(ast.parse(_PROTOCOL + body), "p.py") + assert len(violations) == 1 + assert violations[0].startswith("p.py:") + assert "P.m body must be a docstring only" in violations[0] + assert expected in violations[0] + + +@pytest.mark.unit +def test_every_offending_statement_is_reported() -> None: + body = ' """Doc."""\n x = 1\n return x\n' + violations = protocol_body_violations(ast.parse(_PROTOCOL + body), "p.py") + assert [v.split(": ", 1)[0] for v in violations] == ["p.py:6", "p.py:7"] + + +@pytest.mark.unit +def test_non_protocol_classes_are_not_checked() -> None: + src = "class Base:\n def m(self) -> int:\n raise NotImplementedError\n" + assert protocol_body_violations(ast.parse(src), "b.py") == [] diff --git a/tests/unit/test_validate_sdk_pin.py b/tests/unit/test_validate_sdk_pin.py index 4e2b4e4e..8f2b328f 100644 --- a/tests/unit/test_validate_sdk_pin.py +++ b/tests/unit/test_validate_sdk_pin.py @@ -21,6 +21,12 @@ MAJOR_TAG = _mod.MAJOR_TAG check_text = _mod.check_text check_tree = _mod.check_tree +lock_resolution = _mod.lock_resolution +compare_lock_to_tag = _mod.compare_lock_to_tag +safe_remote = _mod.safe_remote + +CHANNEL_OBJECT = "e9f829f982110be13752da8f18c7a9692e8ed908" +STALE_OBJECT = "69c6c67060b08440734a61473c03663423709964" @pytest.mark.unit @@ -62,3 +68,82 @@ def test_repo_tree_matches_v1_policy() -> None: errors = check_tree(_ROOT) assert errors == [], errors assert MAJOR_TAG == "v1" + + +# ── stale-lock agreement (--verify-tag logic, exercised without network) ────── +# +# The structural checks above cannot catch a stale lock: `v1` moves in Gate_SDK +# and nothing in this repository changes. These cover the comparison that does. + + +@pytest.mark.unit +def test_lock_resolution_reads_the_resolved_object() -> None: + lock = ( + 'name = "constellation-node-sdk"\n' + "[package.source]\n" + 'url = "https://github.com/Quantum-L9/Gate_SDK.git"\n' + 'reference = "v1"\n' + f'resolved_reference = "{CHANNEL_OBJECT}"\n' + ) + assert lock_resolution(lock) == CHANNEL_OBJECT + assert lock_resolution("no sdk source block here\n") is None + + +@pytest.mark.unit +def test_a_lock_current_with_the_channel_passes() -> None: + assert compare_lock_to_tag(CHANNEL_OBJECT, CHANNEL_OBJECT) == [] + + +@pytest.mark.unit +def test_a_stale_lock_fails() -> None: + errors = compare_lock_to_tag(STALE_OBJECT, CHANNEL_OBJECT) + assert any("stale" in item for item in errors), errors + + +@pytest.mark.unit +def test_an_unresolvable_channel_fails_closed() -> None: + """Required networked mode: inability to resolve is a failure, not a pass.""" + errors = compare_lock_to_tag(CHANNEL_OBJECT, None) + assert any("could not resolve" in item for item in errors), errors + + +@pytest.mark.unit +def test_a_lock_without_a_resolved_reference_fails_closed() -> None: + errors = compare_lock_to_tag(None, CHANNEL_OBJECT) + assert any("no resolved_reference" in item for item in errors), errors + + +# ── remote validation (SonarCloud pythonsecurity:S8705) ────────────────────── +# +# argv is a list and no shell is involved, which stops command injection but +# not argument injection: `git ls-remote --upload-pack= ` executes +# , so a --remote beginning with `-` is an execution vector by itself. + + +@pytest.mark.unit +def test_a_canonical_remote_is_accepted(tmp_path: Path) -> None: + url = "https://github.com/Quantum-L9/Gate_SDK.git" + assert safe_remote(url) == url + # An existing directory is accepted so tests can use a fixture repo. + assert safe_remote(str(tmp_path)) == str(tmp_path.resolve()) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "hostile", + [ + "--upload-pack=touch /tmp/pwned", + "-u", + "--exec=sh", + "", + ], +) +def test_a_remote_git_would_read_as_an_option_is_refused(hostile: str) -> None: + with pytest.raises(ValueError, match="must not begin with"): + safe_remote(hostile) + + +@pytest.mark.unit +def test_a_non_canonical_remote_that_is_not_a_repository_is_refused() -> None: + with pytest.raises(ValueError, match="not canonical and not a local repository"): + safe_remote("https://example.invalid/evil/Gate_SDK.git") diff --git a/tools/cypher_lint.py b/tools/cypher_lint.py new file mode 100644 index 00000000..2ae8e80d --- /dev/null +++ b/tools/cypher_lint.py @@ -0,0 +1,590 @@ +#!/usr/bin/env python3 +# --- L9_META --- +# l9_schema: 1 +# origin: engine-specific +# engine: graph +# layer: [tools, security] +# tags: [cypher, lint, C-009, injection] +# owner: platform +# status: active +# --- /L9_META --- +"""Scan engine/**/*.py for unparameterized Cypher interpolations (C-009). + +The scanner parses each module with :mod:`ast` and classifies **every** +interpolation of every f-string that is not a diagnostic message. There is no +keyword precondition: a fragment such as ``NOT EXISTS((query)-[:{edge}]->(c))`` +is analysed whether or not it happens to contain ``MATCH`` or ``WHERE``, and a +triple-quoted multi-line query is analysed as one unit. + +Each ``{expr}`` is classified by the literal text immediately before it: + +===================== ========================== ================================== +literal before role verdict +===================== ========================== ================================== +``'`` or ``"`` quoted data value FAIL — pass it as a ``$parameter`` +``LIMIT`` / ``SKIP`` pagination bound FAIL — pass it as a ``$parameter`` +``:`` label / relationship type OK only when the expression is *validated* +``.`` property name OK only when the expression is *validated* +``$`` parameter **name** OK only when the expression is *validated* +````` back-quoted identifier OK only when the expression is *validated* +anything else compiled fragment FAIL only when the expression is a *raw + domain-spec value* +===================== ========================== ================================== + +An expression is *validated* when it is a call to ``sanitize_label`` / +``sanitize_database_name`` / ``cypher_number``, a string literal, a name +bound from a validated expression in the same lexical scope (its function, +an enclosing function, or the module — never a sibling function), a ``self`` +attribute assigned from one anywhere in the module, a same-module function +whose own ``return`` statements (nested defs excluded) are all validated, an +f-string / ``str.join`` / comprehension built only from validated parts, or a +loop variable over a validated collection. + +An expression is a *raw domain-spec value* when it reads an attribute (or +``.get``) off ``spec``, ``gate``, ``job_spec``, ``dim``, ``metadata`` and +friends without passing through a validator — the shape that turns untrusted +YAML into Cypher. + +``int(...)`` / ``float(...)`` casts and lookups in a literal allow-list +(``_OPERATORS[gate.operator]``) are validated too: neither can carry Cypher. + +Which f-strings are Cypher is decided by context, not by a short keyword list: +the literal text (with each interpolation replaced by a placeholder) contains a +Cypher clause or structural shape (``-[:``, ``]->``, ``candidate.``, ``$x``), +**or** the f-string flows to a query sink (``execute_query``, ``cypher=``), +**or** it is built inside a compiler / assembler function, assigned to a +``cypher``/``clause``/``fragment``-named variable, or appended to a +``*_exprs`` / ``clauses`` / ``cases`` list. A bare ``f"{prop} {op} {param}"`` +fragment inside ``compile()`` is therefore scanned even though it contains no +keyword at all. + +Diagnostic f-strings (inside ``raise``, exception constructors, logger calls, +``warnings.append``, ``reason=`` / ``detail=`` keywords and assignments to +``msg``-like names) are not Cypher and are skipped by AST context, never by +line regex. + +A finding may be waived only with an explicit, reasoned trailing comment on +the interpolation's line — ``# cypher-lint: allow ``. Waivers are +printed on every run so they are never silent, and a marker without a reason +is ignored. +""" + +from __future__ import annotations + +import ast +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +SANITIZERS = frozenset({"sanitize_label", "sanitize_database_name", "cypher_number"}) +NUMERIC_CASTS = frozenset({"int", "float"}) + +# Names whose attributes are domain-spec / payload data, never Cypher-safe. +RAW_ROOTS = frozenset( + { + "spec", + "gate", + "gate_spec", + "job_spec", + "dim", + "dimension", + "domain_spec", + "metadata", + "payload", + "config", + "params", + "_spec", + "_domain_spec", + "_causal_spec", + "causal_spec", + "scoring_spec", + } +) + +LOG_METHODS = frozenset({"debug", "info", "warning", "warn", "error", "exception", "critical", "log"}) +MESSAGE_NAMES = frozenset( + {"msg", "message", "detail", "reason", "hint", "description", "warning", "note", "text", "summary", "error"} +) +MESSAGE_LISTS = frozenset( + {"warnings", "errors", "messages", "issues", "problems", "findings", "violations", "reasons", "notes"} +) +MESSAGE_KWARGS = MESSAGE_NAMES | frozenset({"resource", "title", "gate_impact", "crm_field_name"}) + +_LIMIT_SKIP_RE = re.compile(r"\b(?:LIMIT|SKIP)$") +_WAIVER_RE = re.compile(r"#\s*cypher-lint:\s*allow\b\s*(?P.*)$") + +# Cypher-likeness of the literal text, interpolations replaced by "X". +_CYPHER_TEXT_RE = re.compile( + r"\b(?:MATCH|MERGE|CREATE|DELETE|DETACH|SET|REMOVE|WHERE|WITH|RETURN|CALL|YIELD|UNWIND|LIMIT|SKIP" + r"|ORDER BY|UNION|FOREACH|EXISTS|OPTIONAL|CASE|WHEN|THEN|ELSE|END|AND|OR|NOT|XOR|IS NULL|IS NOT NULL" + r"|DISTINCT|coalesce|toFloat|toString|toInteger|datetime|duration|point)\b" + r"|-\[|\]->|<-\[|\bcandidate\.|\$query\.|\$X\b|\(\s*\w*\s*:\s*X" +) +_FRAGMENT_FUNC_RE = re.compile( + r"compile|assemble|cypher|query|clause|predicate|fragment|expr|where|_ref$|_pattern|render|generate", + re.IGNORECASE, +) +_FRAGMENT_NAME_RE = re.compile( + r"cypher|query|clause|fragment|predicate|expr|pattern|statement|stmt|where|match|filter|case", + re.IGNORECASE, +) +_FRAGMENT_LIST_RE = re.compile( + r"exprs|clauses|parts|cases|statements|fragments|predicates|conditions|filters|lines", re.IGNORECASE +) +SINK_CALLS = frozenset({"execute_query", "execute_write", "run", "_raw_execute_query", "_raw_execute_write"}) +SINK_KWARGS = frozenset({"cypher", "query", "statement"}) + + +@dataclass(frozen=True) +class Finding: + rel_path: str + line_no: int + pattern: str + kind: str + expression: str = "" + waiver: str | None = None + + @property + def blocking(self) -> bool: + return self.waiver is None + + +# ── module facts ──────────────────────────────────────────────────────────── + + +_SCOPE_NODES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda) +_NESTED_STOP = (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda, ast.ClassDef) + + +def _own_nodes(root: ast.AST): + """Yield the nodes of ``root``'s own body, not descending into nested scopes.""" + stack = list(ast.iter_child_nodes(root)) + while stack: + node = stack.pop() + yield node + if not isinstance(node, _NESTED_STOP): + stack.extend(ast.iter_child_nodes(node)) + + +class _Scope: + """Validated / raw name facts for one lexical scope (module or function).""" + + def __init__(self, node: ast.AST, parent: _Scope | None) -> None: + self.node = node + self.parent = parent + self.safe_names: set[str] = set() + self.raw_names: set[str] = set() + self.assignments: list[tuple[ast.expr, ast.expr]] = [] + self.loops: list[tuple[ast.expr, ast.expr]] = [] + + def chain(self): + scope: _Scope | None = self + while scope is not None: + yield scope + scope = scope.parent + + +class _ModuleFacts: + """Which names, ``self`` attributes and functions of a module are validated / raw. + + Name facts are lexically scoped: a binding made inside one function is + visible in that function and its nested functions, never in a sibling — + so ``label = sanitize_label(x)`` in one method cannot certify a bare + ``{label}`` in another. ``self`` attributes are object state and stay + module-wide. Assignment order inside one scope is not tracked. + """ + + def __init__(self, tree: ast.Module) -> None: + self.safe_attrs: set[str] = set() + self.raw_attrs: set[str] = set() + self.safe_funcs: set[str] = set() + self.parents: dict[ast.AST, ast.AST] = {} + for parent in ast.walk(tree): + for child in ast.iter_child_nodes(parent): + self.parents[child] = parent + self._scopes: dict[ast.AST, _Scope] = {tree: _Scope(tree, None)} + for node in ast.walk(tree): + if isinstance(node, _SCOPE_NODES): + self._scopes[node] = _Scope(node, None) + for node, scope in self._scopes.items(): + if node is not tree: + # A nested function's parent scope is the function that + # contains it (closures see enclosing bindings), else the + # scope that contains the parent node. + parent_node = self.parents[node] + scope.parent = self._scopes.get(parent_node) or self.scope_of(parent_node) + self._functions: list[ast.FunctionDef | ast.AsyncFunctionDef] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + self.scope_of(node).assignments.extend((target, node.value) for target in node.targets) + elif (isinstance(node, (ast.AnnAssign, ast.AugAssign)) and node.value is not None) or isinstance( + node, ast.NamedExpr + ): + self.scope_of(node).assignments.append((node.target, node.value)) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + self.scope_of(node).loops.append((node.target, node.iter)) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + self._functions.append(node) + self._fixpoint() + + def scope_of(self, node: ast.AST) -> _Scope: + """The lexical scope a node's bindings belong to (its nearest enclosing function, else module).""" + current: ast.AST | None = node + while current is not None: + if current in self._scopes and current is not node: + return self._scopes[current] + current = self.parents.get(current) + return next(iter(self._scopes.values())) + + def _fixpoint(self) -> None: + changed = True + while changed: + changed = False + for scope in self._scopes.values(): + for target, value in scope.assignments: + if self.is_safe(value, scope): + changed |= self._mark(target, scope.safe_names, self.safe_attrs) + if self.is_raw(value, scope): + changed |= self._mark(target, scope.raw_names, self.raw_attrs) + for target, iterable in scope.loops: + if self.is_safe(iterable, scope): + changed |= self._mark(target, scope.safe_names, self.safe_attrs) + if self.is_raw(iterable, scope): + changed |= self._mark(target, scope.raw_names, self.raw_attrs) + for fn in self._functions: + if fn.name in self.safe_funcs: + continue + # Only the function's own returns count — a validated return + # inside a nested helper says nothing about the outer function. + returns = [n.value for n in _own_nodes(fn) if isinstance(n, ast.Return) and n.value is not None] + fn_scope = self._scopes[fn] + if returns and all(self.is_safe(r, fn_scope) for r in returns): + self.safe_funcs.add(fn.name) + changed = True + + @staticmethod + def _mark(target: ast.expr, names: set[str], attrs: set[str]) -> bool: + if isinstance(target, ast.Name): + if target.id in names: + return False + names.add(target.id) + return True + if isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name) and target.value.id == "self": + if target.attr in attrs: + return False + attrs.add(target.attr) + return True + if isinstance(target, (ast.Tuple, ast.List)): + return any(_ModuleFacts._mark(elt, names, attrs) for elt in target.elts) + return False + + @staticmethod + def _name_in(name: str, scope: _Scope, attr: str) -> bool: + return any(name in getattr(s, attr) for s in scope.chain()) + + # -- validated? -- + + def is_safe(self, expr: ast.expr, scope: _Scope) -> bool: + if isinstance(expr, ast.Constant): + return isinstance(expr.value, str) + if isinstance(expr, ast.Name): + return self._name_in(expr.id, scope, "safe_names") + if isinstance(expr, ast.Attribute): + return isinstance(expr.value, ast.Name) and expr.value.id == "self" and expr.attr in self.safe_attrs + if isinstance(expr, ast.Call): + name = _call_name(expr) + if name in SANITIZERS or name in self.safe_funcs: + return True + if name in NUMERIC_CASTS and isinstance(expr.func, ast.Name): + return True # a number cannot carry Cypher + # ", ".join() + if ( + isinstance(expr.func, ast.Attribute) + and expr.func.attr == "join" + and isinstance(expr.func.value, ast.Constant) + and len(expr.args) == 1 + ): + return self.is_safe(expr.args[0], scope) + return False + if isinstance(expr, ast.JoinedStr): + return all(self.is_safe(v.value, scope) for v in expr.values if isinstance(v, ast.FormattedValue)) + if isinstance(expr, ast.FormattedValue): + return self.is_safe(expr.value, scope) + if isinstance(expr, ast.BoolOp): + return all(self.is_safe(v, scope) for v in expr.values) + if isinstance(expr, ast.IfExp): + return self.is_safe(expr.body, scope) and self.is_safe(expr.orelse, scope) + if isinstance(expr, (ast.ListComp, ast.SetComp, ast.GeneratorExp)): + return self.is_safe(expr.elt, scope) + if isinstance(expr, (ast.List, ast.Tuple, ast.Set)): + return bool(expr.elts) and all(self.is_safe(e, scope) for e in expr.elts) + if isinstance(expr, ast.Dict): + # a literal allow-list: `_OPERATORS[gate.operator]` raises on anything else + return bool(expr.values) and all(v is not None and self.is_safe(v, scope) for v in expr.values) + if isinstance(expr, ast.Subscript): + return self.is_safe(expr.value, scope) + return False + + # -- raw domain-spec value? -- + + def is_raw(self, expr: ast.expr, scope: _Scope) -> bool: + if isinstance(expr, ast.Name): + return self._name_in(expr.id, scope, "raw_names") + if isinstance(expr, ast.Attribute): + root = _attribute_root(expr) + if root in RAW_ROOTS: + return True + if isinstance(expr.value, ast.Name) and expr.value.id == "self": + return expr.attr in self.raw_attrs + return _attribute_chain_has_raw_segment(expr) + if isinstance(expr, ast.Call): + name = _call_name(expr) + if name in SANITIZERS or name in self.safe_funcs or name in NUMERIC_CASTS: + return False + if isinstance(expr.func, ast.Attribute) and expr.func.attr in {"get", "pop", "strip", "lower", "upper"}: + return self.is_raw(expr.func.value, scope) + if isinstance(expr.func, ast.Name) and expr.func.id == "str": + return bool(expr.args) and self.is_raw(expr.args[0], scope) + return False + if isinstance(expr, ast.BoolOp): + return any(self.is_raw(v, scope) for v in expr.values) + if isinstance(expr, ast.IfExp): + return self.is_raw(expr.body, scope) or self.is_raw(expr.orelse, scope) + if isinstance(expr, ast.Subscript): + return self.is_raw(expr.value, scope) + if isinstance(expr, ast.JoinedStr): + return any(self.is_raw(v.value, scope) for v in expr.values if isinstance(v, ast.FormattedValue)) + return False + + +def _call_name(call: ast.Call) -> str: + if isinstance(call.func, ast.Name): + return call.func.id + if isinstance(call.func, ast.Attribute): + return call.func.attr + return "" + + +def _attribute_root(expr: ast.Attribute) -> str: + node: ast.expr = expr + while isinstance(node, ast.Attribute): + node = node.value + return node.id if isinstance(node, ast.Name) else "" + + +def _attribute_chain_has_raw_segment(expr: ast.Attribute) -> bool: + """``self.spec.edgetype`` / ``self.domain_spec.domain.id`` → the chain passes through a raw segment.""" + node: ast.expr = expr + while isinstance(node, ast.Attribute): + if node.attr in RAW_ROOTS: + return True + node = node.value + return False + + +# ── message context ───────────────────────────────────────────────────────── + + +def _is_message_context(node: ast.AST, parents: dict[ast.AST, ast.AST]) -> bool: + """Diagnostic text, not Cypher: skip by AST context.""" + child: ast.AST = node + while child in parents: + parent = parents[child] + if isinstance(parent, ast.Raise): + return True + if isinstance(parent, ast.keyword) and parent.arg in MESSAGE_KWARGS: + return True + if isinstance(parent, ast.Dict): + for key, value in zip(parent.keys, parent.values, strict=True): + if value is child and isinstance(key, ast.Constant) and key.value in MESSAGE_NAMES: + return True + if isinstance(parent, ast.Call): + name = _call_name(parent) + if name in SANITIZERS and child in parent.args: + return True # sanitize_label(f"_prior_{dim.name}") — validated by the call around it + if name in LOG_METHODS or name in {"print", "warn"}: + return True + if name.endswith(("Error", "Exception", "Warning")): + return True + if ( + name in {"append", "extend"} + and isinstance(parent.func, ast.Attribute) + and isinstance(parent.func.value, ast.Name) + and parent.func.value.id in MESSAGE_LISTS + ): + return True + if isinstance(parent, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + targets = parent.targets if isinstance(parent, ast.Assign) else [parent.target] + if any(isinstance(t, ast.Name) and t.id in MESSAGE_NAMES for t in targets): + return True + if isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Module)): + return False + child = parent + return False + + +def _is_cypher_candidate(node: ast.JoinedStr, parents: dict[ast.AST, ast.AST]) -> bool: + """Does this f-string build Cypher? Decided by its text *or* by where it flows.""" + text = "".join(v.value if isinstance(v, ast.Constant) and isinstance(v.value, str) else "X" for v in node.values) + if _CYPHER_TEXT_RE.search(text): + return True + child: ast.AST = node + while child in parents: + parent = parents[child] + if isinstance(parent, ast.keyword) and parent.arg in SINK_KWARGS: + return True + if isinstance(parent, ast.Call): + name = _call_name(parent) + if name in SINK_CALLS: + return True + if ( + name in {"append", "extend"} + and isinstance(parent.func, ast.Attribute) + and isinstance(parent.func.value, ast.Name) + and _FRAGMENT_LIST_RE.search(parent.func.value.id) + ): + return True + if isinstance(parent, (ast.Assign, ast.AnnAssign, ast.AugAssign, ast.NamedExpr)): + targets = parent.targets if isinstance(parent, ast.Assign) else [parent.target] + if any( + (isinstance(t, ast.Name) and _FRAGMENT_NAME_RE.search(t.id)) + or (isinstance(t, ast.Attribute) and _FRAGMENT_NAME_RE.search(t.attr)) + for t in targets + ): + return True + if isinstance(parent, (ast.FunctionDef, ast.AsyncFunctionDef)): + return bool(_FRAGMENT_FUNC_RE.search(parent.name)) + if isinstance(parent, ast.Module): + return False + child = parent + return False + + +# ── classification ────────────────────────────────────────────────────────── + + +def _classify(before: str, expr: ast.expr, facts: _ModuleFacts, scope: _Scope) -> str | None: + """Return the failure kind for one interpolation, or None when it is acceptable.""" + tail = before[-1:] if before else "" + if tail in {"'", '"'}: + return "quoted value interpolation — pass the value as a $parameter" + if tail == "`": + if not facts.is_safe(expr, scope): + return "back-quoted identifier is not validated (sanitize_database_name / sanitize_label)" + return None + if tail == "$": + if not facts.is_safe(expr, scope): + return "parameter name is not validated — derive it from sanitize_label() or a literal" + return None + if tail == ":": + if not facts.is_safe(expr, scope): + return "label / relationship type is not validated (sanitize_label)" + return None + if tail == "." and not before.endswith(".."): + if not facts.is_safe(expr, scope): + return "property name is not validated (sanitize_label)" + return None + if _LIMIT_SKIP_RE.search(before.rstrip()): + return "LIMIT / SKIP bound interpolated — pass it as a $parameter" + if facts.is_safe(expr, scope): + return None + if facts.is_raw(expr, scope): + return "raw domain-spec value interpolated into Cypher — sanitize_label() it or pass it as a $parameter" + return None + + +def scan_source(source: str, *, rel_path: str) -> list[Finding]: + """Scan one module's source text.""" + tree = ast.parse(source, filename=rel_path) + facts = _ModuleFacts(tree) + parents = facts.parents + lines = source.splitlines() + + findings: list[Finding] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.JoinedStr): + continue + if isinstance(parents.get(node), ast.FormattedValue): + continue # nested format spec of another f-string + if _is_message_context(node, parents) or not _is_cypher_candidate(node, parents): + continue + scope = facts.scope_of(node) + before = "" + for value in node.values: + if isinstance(value, ast.Constant) and isinstance(value.value, str): + before = value.value + continue + if not isinstance(value, ast.FormattedValue): + continue + kind = _classify(before, value.value, facts, scope) + before = "" + if kind is None: + continue + line_no = getattr(value.value, "lineno", node.lineno) + line = lines[line_no - 1] if 0 < line_no <= len(lines) else "" + findings.append( + Finding( + rel_path=rel_path, + line_no=line_no, + pattern=line.strip(), + kind=kind, + expression=ast.unparse(value.value), + waiver=_waiver(line), + ) + ) + return findings + + +def _waiver(line: str) -> str | None: + """An explicit ``# cypher-lint: allow `` on the line; a bare marker is not a waiver.""" + match = _WAIVER_RE.search(line) + if match is None: + return None + reason = match.group("reason").strip(" -—:") + return reason or None + + +def scan_file(path: Path, *, root: Path) -> list[Finding]: + return scan_source(path.read_text(encoding="utf-8"), rel_path=path.relative_to(root).as_posix()) + + +def scan_tree(root: Path) -> list[Finding]: + engine = root / "engine" + if not engine.is_dir(): + return [] + findings: list[Finding] = [] + for path in sorted(engine.rglob("*.py")): + if path.is_file(): + findings.extend(scan_file(path, root=root)) + return findings + + +def render_findings(findings: list[Finding]) -> str: + blocks = [] + for item in findings: + head = f"❌ {item.kind}" if item.blocking else f"⚠️ waived ({item.waiver}) — {item.kind}" + blocks.append( + f"{head}\nFile: {item.rel_path}:{item.line_no}\nExpression: {{{item.expression}}}\nPattern: {item.pattern}" + ) + return "\n\n".join(blocks) + + +def main(argv: list[str] | None = None) -> int: + args = sys.argv[1:] if argv is None else argv + root = Path(args[0]).resolve() if args else Path.cwd() + findings = scan_tree(root) + blocking = [f for f in findings if f.blocking] + waived = [f for f in findings if not f.blocking] + if waived: + print(render_findings(waived), end="\n\n") + if blocking: + print(render_findings(blocking)) + print(f"\n{len(blocking)} C-009 finding(s), {len(waived)} waived") + return 1 + print(f"OK: cypher-lint — 0 injection vectors ({len(waived)} waived, listed above)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/domain_extractor.py b/tools/domain_extractor.py index 6c7a251c..d8f38358 100644 --- a/tools/domain_extractor.py +++ b/tools/domain_extractor.py @@ -22,8 +22,8 @@ OUTPUT: domains/ ├── plasticos/spec.yaml - ├── mortgage_brokerage_domain_spec.yaml - ├── healthcare_referral_domain_spec.yaml + ├── mortgage-brokerage/spec.yaml + ├── healthcare-referral/spec.yaml └── ... """ @@ -67,16 +67,17 @@ def extract_domains(input_file: Path) -> None: domain_id = domain_id_match.group(1) - # Normalize domain_id (replace hyphens with underscores for filename) - domain_id_normalized = domain_id.replace("-", "_") - # Clean up YAML (remove --- separator if at start) spec_clean = spec.strip() if spec_clean.startswith("---"): spec_clean = spec_clean[3:].lstrip() - # Write spec file with standard naming convention - spec_path = domains_dir / f"{domain_id_normalized}_domain_spec.yaml" + # Folder-shaped pack: domains//spec.yaml is the only layout + # DomainPackLoader discovers (tests/unit/test_domain_pack_shape.py + # rejects the former flat _domain_spec.yaml files). + pack_dir = domains_dir / domain_id + pack_dir.mkdir(exist_ok=True) + spec_path = pack_dir / "spec.yaml" spec_path.write_text(spec_clean) print(f"✅ Created: {spec_path} ({len(spec_clean)} bytes)")