From 569c75863175f7c06f54bfebe33aaf20e083b265 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 22:41:00 +0000 Subject: [PATCH 1/3] Close the six high findings and five follow-ups from the security review - targets: refuse a host that is not an IP literal or ASCII LDH, or that urllib would dial differently; refuse a name that does not resolve; credentialed openers ignore environment proxies - core: source URLs keep {key} in the path; action URLs take no query; manual attestations of keys the contract reads from another source are refused (attestation_denied) - server: observer-only roles cannot read cases, contracts, certificates or metrics; an empty bind host is refused; users.json is re-read on change - backup: export writes through temp files and refuses links - storage: migration 0006 widens amount_units to BIGINT on PostgreSQL - publish: timeouts end the whole process tree - compose: loopback publish, no default database password Regression tests in tests/test_security_review.py. Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 9 +- docs/ARCHITECTURE.md | 2 +- docs/DEPLOY.md | 11 +- scripts/compose_cell.sh | 6 + spec/GAPS.md | 2 +- syberlabs/outcome.py | 2 +- syberlabs/planner.py | 6 +- syberlabs/publish.py | 10 +- syberlabs/settle.py | 2 + syberlabs/targets.py | 57 ++++- syberwork/backup.py | 29 ++- syberwork/cli.py | 4 +- syberwork/core.py | 75 ++++++- syberwork/server.py | 59 ++++- syberwork/storage.py | 11 +- tests/console_browser.mjs | 32 +-- tests/test_cell.py | 4 +- tests/test_console.py | 20 +- tests/test_security_review.py | 402 ++++++++++++++++++++++++++++++++++ tests/test_witness.py | 5 +- 20 files changed, 688 insertions(+), 60 deletions(-) create mode 100644 tests/test_security_review.py diff --git a/docker-compose.yml b/docker-compose.yml index 1d87802f..41d90a2f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,14 @@ # One organization, one database. Run `syberwork init` before `serve`. # The home directory is /home/syber/cell, owned by uid 10001, the same path the image creates. +# SYBERWORK_DB_PASSWORD must be set (there is no default password); scripts/compose_cell.sh generates one. +# The service listens on 0.0.0.0 inside its container so Docker can forward to it, and Docker publishes that +# port on the host's loopback only. Widening the mapping is the operator's decision on the host (docs/DEPLOY.md). services: db: image: postgres:16 environment: POSTGRES_USER: syber - POSTGRES_PASSWORD: syber + POSTGRES_PASSWORD: ${SYBERWORK_DB_PASSWORD:?set SYBERWORK_DB_PASSWORD for the cell database} POSTGRES_DB: syberwork volumes: - syberwork-db:/var/lib/postgresql/data @@ -18,13 +21,13 @@ services: build: . environment: &cell SYBERWORK_HOME: /home/syber/cell - SYBERWORK_DATABASE: postgresql://syber:syber@db:5432/syberwork + SYBERWORK_DATABASE: postgresql://syber:${SYBERWORK_DB_PASSWORD:?set SYBERWORK_DB_PASSWORD for the cell database}@db:5432/syberwork SYBERWORK_ORGANIZATION: example SYBERWORK_BIND: 0.0.0.0 SYBERWORK_EFFECTS: worker SYBERWORK_TRACE: "1" ports: - - "8766:8766" + - "127.0.0.1:8766:8766" depends_on: db: condition: service_healthy diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 29a26f31..d6fe59d0 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,6 +34,6 @@ For each past proposal, replay evaluates a selected new contract and policy agai ## Integration -HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call. The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. +HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call and refuses a name it cannot resolve; a host with percent-escapes or other characters `urllib.request` would rewrite is refused before it is classified. The credentialed openers ignore `http_proxy`/`https_proxy` (`targets.credentialed_opener`), so a bearer token is never handed to an environment proxy. A manual attestation (`POST /api/cases/{id}/facts`) may not set a key the case's contract reads from another source or only verified, and an `observer`-only credential reads no case, contract, certificate or metric. The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. The [`economic_http` effect](ECONOMIC_ACTIONS.md) adds exact atomic units and a cross-case budget reservation to this same admission boundary. Its trusted settlement service is a separate authority; SyberWork records that service's matching receipt and status assertion, not independent blockchain finality. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 8b35b2a8..a33f9b09 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -10,17 +10,18 @@ The first deployment of the factory is the SyberWork service, its worker and its | Service | Image | Role | | --- | --- | --- | -| `db` | `postgres:16` | The cell's database `syberwork`, user `syber`, on volume `syberwork-db`. Health-checked with `pg_isready`. | -| `syberwork` | built from `Dockerfile` | The operator service, `syberwork serve --port 8766`, published on host port 8766. Binds `0.0.0.0` inside the container because `SYBERWORK_BIND` says so; the container port is reachable only through the published port. | +| `db` | `postgres:16` | The cell's database `syberwork`, user `syber`, password `SYBERWORK_DB_PASSWORD` (required; no default), on volume `syberwork-db`. Health-checked with `pg_isready`. | +| `syberwork` | built from `Dockerfile` | The operator service, `syberwork serve --port 8766`, published on the host's loopback, `127.0.0.1:8766`. Binds `0.0.0.0` inside the container because `SYBERWORK_BIND` says so; the container port is reachable only through that published port. | | `worker` | the same image | `syberwork worker`: leases queued effect obligations and settles them. Needed because `SYBERWORK_EFFECTS=worker` makes the service claim effects and return. | -Both processes share `SYBERWORK_HOME=/home/syber/cell` on volume `syberwork-home`, which holds `users.json` (credential hashes), `cell.json` and `traces.jsonl`. The environment (`SYBERWORK_DATABASE`, `SYBERWORK_ORGANIZATION=example`, `SYBERWORK_TRACE=1`) is the one in the file; this document changes nothing in it. +Both processes share `SYBERWORK_HOME=/home/syber/cell` on volume `syberwork-home`, which holds `users.json` (credential hashes), `cell.json` and `traces.jsonl`. The environment (`SYBERWORK_DATABASE`, `SYBERWORK_ORGANIZATION=example`, `SYBERWORK_TRACE=1`) is the one in the file; this document changes nothing in it. `users.json` is read again whenever it changes, so removing or rotating a credential there applies without restarting `serve`. ## 2. Running it on one host The host needs Docker with the Compose plugin and a checkout of this repository. The commands are the ones `scripts/compose_cell.sh` runs, in order, without the smoke steps. ```sh +export SYBERWORK_DB_PASSWORD="" # compose refuses to start without it docker compose build docker compose up -d db --wait docker compose run --rm -T syberwork syberwork init @@ -89,8 +90,8 @@ with `SYBERWORK_METRIC_TOKEN` set in the environment of the process that runs `s ## 4. What is not there yet - **No thread runner service.** Build Threads run from a checkout with `python -m syberlabs`; nothing in the stack starts, schedules or recovers them. That is a later packet. -- **No public exposure and no TLS.** Docker publishes port 8766 on the host's interfaces as `docker-compose.yml` is written; whether the host firewall closes it, or the mapping is narrowed to `127.0.0.1:8766:8766`, is the operator's decision on the host. The service speaks plain HTTP; the intervention effect will only read it over loopback. -- **No secrets broker.** The database password is the literal in `docker-compose.yml`, the credentials are what `init` printed, and the thread's metric token lives in an environment variable. Rotation is `down -v` and a new `init`. +- **No public exposure and no TLS.** Docker publishes port 8766 on the host's loopback only (`127.0.0.1:8766:8766`); widening that mapping, and the firewall that would then guard it, is the operator's decision on the host. The service speaks plain HTTP; the intervention effect will only read it over loopback. +- **No secrets broker.** The database password is whatever `SYBERWORK_DB_PASSWORD` held when the `syberwork-db` volume was first created (a volume created under the old literal `syber` keeps it until `down -v`), the credentials are what `init` printed, and the thread's metric token lives in an environment variable. Rotation is `down -v` and a new `init`. - **No principal management from the command line.** `Work.register_principal` exists for cell principals with a token hash and a delegation; the CLI exposes only the `init` credentials. - **Single host, single worker.** One Compose project, one database, one worker process. `syberwork backup` is the only redundancy. - **Counters are read on request.** `cases_open` reads every case's history on each call; fine for a cell this size, not a design for thousands of cases. diff --git a/scripts/compose_cell.sh b/scripts/compose_cell.sh index 21c0e9ff..11238130 100755 --- a/scripts/compose_cell.sh +++ b/scripts/compose_cell.sh @@ -4,6 +4,12 @@ set -euo pipefail cd "$(dirname "$0")/.." +# docker-compose.yml has no default database password. A throwaway cell gets a random one. +if [ -z "${SYBERWORK_DB_PASSWORD:-}" ]; then + SYBERWORK_DB_PASSWORD="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +fi +export SYBERWORK_DB_PASSWORD + docker compose build docker compose up -d db --wait docker compose run --rm -T syberwork syberwork init diff --git a/spec/GAPS.md b/spec/GAPS.md index fea8bb7f..4f868843 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -8,7 +8,7 @@ The schemas describe SyberWork as it behaves. They are not enforced at runtime e 3. **`install_contract` checks the fields admission indexes, including input kinds.** An acceptance `effect` without `action`, a `signoff` without `role`, a `fact` without `key`, or a `required_facts` item without `key` and `source` is rejected at publish. An input kind other than `string` or `integer` is `invalid_contract`. `create_case` rejects a value of the wrong kind with `input_schema`, including a boolean passed where an integer is required. -4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). DNS rebinding is checked at request time by `guard_request`, not at install. +4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). The host must be an IP literal or plain ASCII letter-digit-hyphen labels, so the name checked is the name `urllib.request` dials (a percent-escaped host is refused). DNS rebinding is checked at request time by `guard_request`, not at install; a name `guard_request` cannot resolve is refused, never allowed. Installed action URLs carry no query, and a source URL carries exactly one `{key}`, in its path. 5. **Proposals are appended before admission.** A non-dict `args` or a missing role list is stored, then denied with `invalid_proposal_shape`. The Proposal schema therefore does not require `args` to be an object. diff --git a/syberlabs/outcome.py b/syberlabs/outcome.py index fbc40c75..0777e669 100644 --- a/syberlabs/outcome.py +++ b/syberlabs/outcome.py @@ -139,7 +139,7 @@ def _read(self) -> Reading: headers["Authorization"] = "Bearer " + token try: guard_request(self.url) - except Rejected: + except (Rejected, OSError): # OSError: the guard could not resolve the host (URLError) raise MetricError("metric_unavailable") from None request = urllib.request.Request(self.url, headers=headers, method="GET") try: diff --git a/syberlabs/planner.py b/syberlabs/planner.py index 4cc64835..fb601f37 100644 --- a/syberlabs/planner.py +++ b/syberlabs/planner.py @@ -10,7 +10,7 @@ from syberlabs.canonical import canonical from syberlabs.errors import Rejected -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin class PlannerRefusal(Rejected): @@ -50,7 +50,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -_HTTP = urllib.request.build_opener(_NoRedirect) +_HTTP = credentialed_opener(_NoRedirect) class HttpPlanner: @@ -66,12 +66,12 @@ def ensure_configured(self) -> str: def propose(self, context: Mapping[str, Any]) -> dict[str, Any]: target = self.ensure_configured() - guard_request(target) headers = {"Content-Type": "application/json"} if os.getenv("SYBERWORK_PLANNER_TOKEN"): headers["Authorization"] = "Bearer " + os.environ["SYBERWORK_PLANNER_TOKEN"] request = urllib.request.Request(target, data=canonical(dict(context)).encode(), headers=headers, method="POST") try: + guard_request(target) # a refused address raises Rejected; an unresolved name, URLError with _HTTP.open(request, timeout=30) as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: diff --git a/syberlabs/publish.py b/syberlabs/publish.py index d4009ba9..bd0a1bb8 100644 --- a/syberlabs/publish.py +++ b/syberlabs/publish.py @@ -35,9 +35,10 @@ from urllib.parse import quote from syberlabs import critic, outcome +from syberlabs.checks import _stop from syberlabs.errors import Rejected from syberlabs.session import NoWrite -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin EFFECTS = ("git_push", "github_pull_request", "command", "intervention") _REMOTE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") @@ -52,7 +53,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -HTTP = urllib.request.build_opener(_NoRedirect) +HTTP = credentialed_opener(_NoRedirect) def _line(value) -> str: @@ -309,8 +310,9 @@ def _run(self, argv: list[str], case_id: str, args: dict, key: str) -> tuple[int deadline = time.monotonic() + self.timeout while proc.poll() is None: if time.monotonic() > deadline or os.fstat(out.fileno()).st_size > MAX_OUTPUT: - proc.kill() - proc.wait() + # End the command and everything it started, as a check's timeout does: ``proc.kill`` ends + # one process and leaves a deploy script's children running past the publish. + _stop(proc) return None, "" time.sleep(0.02) out.seek(0) diff --git a/syberlabs/settle.py b/syberlabs/settle.py index a8d2c1f0..44696ab8 100644 --- a/syberlabs/settle.py +++ b/syberlabs/settle.py @@ -148,6 +148,8 @@ def _get(target: Published, token: str, path: str, timeout: float): guard_request(url) except Rejected as exc: raise PollFailed(exc.code) from None + except urllib.error.URLError: # the guard could not resolve the host; no request was made + raise PollFailed("network failure (unresolved)") from None request = urllib.request.Request(url, method="GET", headers={ "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}) diff --git a/syberlabs/targets.py b/syberlabs/targets.py index e2a82af8..f8aad3b6 100644 --- a/syberlabs/targets.py +++ b/syberlabs/targets.py @@ -4,7 +4,10 @@ import ipaddress import os +import re import socket +import urllib.error +import urllib.request from urllib.parse import urlsplit from syberlabs.errors import Rejected @@ -16,6 +19,12 @@ "metadata.google.com", }) _BLOCKED_SUFFIXES = (".local", ".localhost", ".internal", ".localdomain") +# A host name the guard classifies must be the name ``urllib.request`` dials. ``urlsplit`` keeps +# percent-escapes in ``hostname`` while ``Request`` unquotes them, so ``%31%32%37.0.0.1`` was checked as a +# name and dialed as ``127.0.0.1``. Only plain ASCII letter-digit-hyphen labels (IDNA names in their +# ``xn--`` form) and IP literals are accepted; anything else is refused before it is classified. +_LABEL = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" +_HOSTNAME = re.compile(rf"(?:{_LABEL}\.)*{_LABEL}\.?", re.ASCII) def _public_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: @@ -39,6 +48,26 @@ def _https_host_blocked(host: str) -> bool: return False +def _plain_host(host: str) -> bool: + """A host ``urllib.request.Request`` dials exactly as written: an IP literal or plain LDH labels.""" + try: + ipaddress.ip_address(host) + return True + except ValueError: + pass + return len(host) <= 253 and bool(_HOSTNAME.fullmatch(host)) + + +def _dialed_host(url: str) -> str | None: + """The host ``urllib.request`` would connect to for ``url``, without its port and brackets.""" + try: + dialed = urllib.request.Request(url).host + except ValueError: + return None + found = urlsplit("//" + dialed).hostname + return found.lower() if found else None + + def _https_host_listed(host: str) -> bool: """``SYBERWORK_HTTPS_HOSTS`` narrows HTTPS. Unset or blank does not.""" raw = os.getenv("SYBERWORK_HTTPS_HOSTS", "") @@ -51,7 +80,9 @@ def _https_host_listed(host: str) -> bool: def trusted_origin(url: str) -> tuple[str, str, int]: """Reject userinfo, non-loopback HTTP, and non-public HTTPS hosts. - HTTPS names are checked as written. This function does not resolve DNS. + HTTPS names are checked as written, and the name checked must be the name ``urllib.request`` dials: + a host with percent-escapes, other non-LDH characters or non-ASCII is refused. This function does not + resolve DNS. """ try: parsed = urlsplit(url) @@ -59,6 +90,8 @@ def trusted_origin(url: str) -> tuple[str, str, int]: if (not isinstance(url, str) or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or not parsed.path.startswith("/")): raise ValueError("invalid URL") + if not _plain_host(parsed.hostname) or _dialed_host(url) != parsed.hostname: + raise ValueError("host is not dialed as written") if parsed.scheme == "https": if _https_host_blocked(parsed.hostname) or not _https_host_listed(parsed.hostname): raise ValueError("blocked host") @@ -73,16 +106,19 @@ def trusted_origin(url: str) -> tuple[str, str, int]: def guard_request(url: str) -> None: """Reject an HTTPS target that resolves to a non-public address. - Loopback HTTP is not resolved. A DNS failure is left to the caller, so an - unreachable planner stays ``planner_unavailable``. + Loopback HTTP is not resolved. A name the guard cannot resolve is refused, not allowed: it raises + ``urllib.error.URLError`` (no request is made), so each caller keeps the code it already gives an + unreachable destination (``planner_unavailable``, ``source_unavailable``, ``destination_unreachable``). """ scheme, host, port = trusted_origin(url) if scheme != "https": return try: answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) - except socket.gaierror: - return + except (socket.gaierror, UnicodeError, OSError): + raise urllib.error.URLError("target host did not resolve") from None + if not answers: + raise urllib.error.URLError("target host did not resolve") for info in answers: address = info[4][0].split("%", 1)[0] try: @@ -91,3 +127,14 @@ def guard_request(url: str) -> None: raise Rejected("invalid_target", "target host resolves to a non-public address") from None if not _public_ip(ip): raise Rejected("invalid_target", "target host resolves to a non-public address") + + +def credentialed_opener(*handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector: + """An opener for credential-bearing requests that never routes through an environment proxy. + + ``urllib.request.build_opener`` adds a ``ProxyHandler`` that reads ``http_proxy``/``https_proxy`` when the + opener is built. Through a proxy the guard's address check is moot (the proxy resolves and dials) and the + bearer token goes to the proxy, including on loopback HTTP, which the guard does not resolve. The empty + ``ProxyHandler`` disables that; a deployment that needs egress through a proxy must use a transparent one. + """ + return urllib.request.build_opener(urllib.request.ProxyHandler({}), *handlers) diff --git a/syberwork/backup.py b/syberwork/backup.py index b1c34a54..3d458246 100644 --- a/syberwork/backup.py +++ b/syberwork/backup.py @@ -9,6 +9,8 @@ from __future__ import annotations import json +import os +import tempfile from pathlib import Path from syberlabs.errors import Rejected @@ -73,6 +75,29 @@ def _columns(store, table: str) -> list[str]: return [item[0] for item in row.description] +def _write_file(directory: Path, name: str, text: str) -> None: + """Write ``directory/name`` as a new regular file; never through a link planted at that name. + + ``Path.write_text`` follows a symlink, so a ``manifest.json`` linked elsewhere had its target replaced with + the snapshot. A link (or any non-regular file) at the name is refused, and the bytes are written to a fresh + temporary file in the same directory (created exclusively) and moved over the name. + """ + target = directory / name + if target.is_symlink() or (target.exists() and not target.is_file()): + raise Rejected("backup_refused", f"{name} in the backup directory is a link or not a regular file") + handle, temporary = tempfile.mkstemp(prefix="." + name + ".", dir=directory) + try: + with os.fdopen(handle, "w", encoding="utf-8") as out: + out.write(text) + os.replace(temporary, target) + except BaseException: + try: + os.unlink(temporary) + except OSError: + pass + raise + + def export_cell(work, directory: str | Path) -> dict: directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -88,8 +113,8 @@ def export_cell(work, directory: str | Path) -> dict: rows = [] for record in db.execute(f"SELECT * FROM {table}").fetchall(): rows.append({column: record[column] for column in columns}) - (directory / f"{table}.json").write_text(json.dumps(rows, separators=(",", ":"), default=str), encoding="utf-8") - (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _write_file(directory, f"{table}.json", json.dumps(rows, separators=(",", ":"), default=str)) + _write_file(directory, "manifest.json", json.dumps(manifest, indent=2) + "\n") return manifest diff --git a/syberwork/cli.py b/syberwork/cli.py index e76d99bc..12a2e0c0 100644 --- a/syberwork/cli.py +++ b/syberwork/cli.py @@ -15,7 +15,7 @@ from .backup import export_cell, restore_cell from .certificates import CertificateShelf from .core import Work -from .server import serve +from .server import UserFile, serve from .trace import TraceLog from .worker import run_available, serve as serve_worker @@ -155,7 +155,7 @@ def main(): log = _witness_log(parser) client, role = _witness_request(parser, log) shelf = _certificates(parser) - serve(db, json.loads(userfile.read_text()), host=os.getenv("SYBERWORK_BIND", "127.0.0.1"), port=args.port, + serve(db, UserFile(userfile), host=os.getenv("SYBERWORK_BIND") or "127.0.0.1", port=args.port, witness_log=log, witness_client=client, witness_role=role, certificates=shelf) elif args.command == "backup": print(json.dumps(export_cell(db, args.directory))) diff --git a/syberwork/core.py b/syberwork/core.py index 49af68a3..ea00d7e3 100644 --- a/syberwork/core.py +++ b/syberwork/core.py @@ -31,7 +31,7 @@ from syberlabs.jcs import envelope_jcs from syberlabs import metrics as work_metrics from syberlabs.planner import HttpPlanner, planning_context -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin from syberlabs.values import at_path from syberwork.storage import ( ActionRegistry, @@ -61,7 +61,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -HTTP = urllib.request.build_opener(NoRedirect) +HTTP = credentialed_opener(NoRedirect) class StatusNotAuthoritative(Exception): @@ -203,6 +203,10 @@ def install_action(self, name: str, doc: dict) -> None: raise Rejected("invalid_action", "action kind must be local, http or economic_http") if doc["kind"] in ("http", "economic_http"): origin = trusted_origin(doc.get("url", "")) + # A definition names credential variables and never holds a secret (``registry``). A query string + # is where a secret would ride, so neither the write URL nor the status URL may carry one. + if urlsplit(doc["url"]).query or "?" in doc["url"] or any(c in doc["url"] for c in "{}"): + raise Rejected("invalid_action", "action URL takes no query string or template") allowed_methods = ("POST",) if doc["kind"] == "economic_http" else ("POST", "PUT", "PATCH") if doc.get("method", "POST") not in allowed_methods: raise Rejected("invalid_action", "HTTP method must write") @@ -219,6 +223,7 @@ def install_action(self, name: str, doc: dict) -> None: if (doc["kind"] not in ("http", "economic_http") or trusted_origin(status_url) != origin or status_url.count("{key}") != 1 or urlsplit(status_url).path.count("{key}") != 1 or + "?" in status_url or any(c in status_url.replace("{key}", "") for c in "{}")): raise Rejected("invalid_action", "status lookup needs one path key on the action origin") no_write = doc.get("no_write_statuses", []) @@ -238,8 +243,12 @@ def install_source(self, name: str, doc: dict) -> None: trusted = True except Rejected: trusted = False - if doc.get("kind") != "http" or not trusted or "{key}" not in url: - raise Rejected("invalid_source", "source needs a fixed HTTPS or local URL with {key}") + # The record key is substituted into the path only. A ``{key}`` in the host would let the case input + # choose which host receives the source's bearer token; a query would store a secret in the definition. + if (doc.get("kind") != "http" or not trusted or url.count("{key}") != 1 or + urlsplit(url).path.count("{key}") != 1 or "?" in url or + any(c in url.replace("{key}", "") for c in "{}")): + raise Rejected("invalid_source", "source needs a fixed HTTPS or local URL with one {key} in its path") with self.tx() as db: sources = SourceRegistry(db) prior = sources.get(name) @@ -447,6 +456,51 @@ def observe(self, case_id: str, key: str, value: Any, source: str, version: str, self._case(db, case_id) return self._append(db, case_id, "observed", {"key": key, "value": value, "source": source, "version": version, "actor": actor, "verified": verified}) + @staticmethod + def _attestable(contract: dict, key: str, source: str) -> bool: + """Whether a manual, unverified attestation of ``key`` from ``source`` may enter this contract's case. + + Admission (``facts.required``, ``resolution.current``) and argument binding read the newest observation + of a key. An unverified attestation of a key the contract relies on would shadow the verified fact, so + a case could be stalled, or its compiled arguments chosen, by anyone holding any attesting source. A key + the contract names (a required fact, an argument binding, a resolution's evidence) is attestable only + where a required fact names this very source and does not ask for verification. A key the contract never + reads stays attestable, as before: it is recorded unverified and decides nothing. + """ + named, allowed = set(), False + for action in (contract.get("actions") or {}).values(): + if not isinstance(action, dict): + continue + for requirement in action.get("required_facts") or []: + if isinstance(requirement, dict) and requirement.get("key") == key: + named.add(key) + if requirement.get("source") == source and not requirement.get("verified"): + allowed = True + bindings = action.get("arguments") or {} + if isinstance(bindings, dict): + for binding in bindings.values(): + if (isinstance(binding, str) and + binding.removeprefix("version:").removeprefix("fact:").split(".")[0] == key): + named.add(key) + for resolution in (contract.get("resolutions") or {}).values(): + if isinstance(resolution, dict) and any( + isinstance(part, dict) and part.get("key") == key for part in resolution.values()): + return False + return allowed or key not in named + + def attest(self, case_id: str, key: str, value: Any, source: str, version: str, actor: str) -> dict: + """Record a person's or service's unverified attestation (``POST /api/cases/{id}/facts``). + + Refused for a key the case's contract relies on unless that contract accepts this source's unverified + word for it (``_attestable``). ``refresh_fact`` is the route for a verified fact. + """ + with self.tx() as db: + row = self._case(db, case_id) + contract = self._contract(db, row["contract_id"], row["contract_version"]) + if not isinstance(key, str) or not self._attestable(contract, key, source): + raise Rejected("attestation_denied", "this contract reads that key only from its configured source") + return self.observe(case_id, key, value, source, version, actor) + def refresh_fact(self, case_id: str, source: str, key: str, record_key: str, actor: str, roles: list[str]) -> dict: with self.tx() as db: self._case(db, case_id) @@ -466,7 +520,10 @@ def refresh_fact(self, case_id: str, source: str, key: str, record_key: str, act raise Rejected("missing_credential", "source credential unavailable") headers["Authorization"] = "Bearer " + token target = config["url"].replace("{key}", quote(record_key, safe="")) - guard_request(target) + try: + guard_request(target) + except urllib.error.URLError: # the guard could not resolve the host; no request was made + raise Rejected("source_unavailable", "destination_unreachable") from None request = urllib.request.Request(target, headers=headers) try: with HTTP.open(request, timeout=min(30, config.get("timeout_seconds", 10))) as response: @@ -715,9 +772,15 @@ def reconcile(self, case_id: str, proposal_id: str, actor: str, roles: list[str] raise Rejected("missing_credential", "destination status credential unavailable") headers["Authorization"] = "Bearer " + token target = status_url.replace("{key}", quote(proposal_id, safe="")) - guard_request(target) + try: + guard_request(target) + unresolved = None + except urllib.error.URLError as exc: # the guard could not resolve the host: the lookup is unavailable + unresolved = exc request = urllib.request.Request(target, headers=headers) try: + if unresolved is not None: + raise unresolved with HTTP.open(request, timeout=min(30, config.get("timeout_seconds", 10))) as response: if getattr(response, "status", None) != 200: raise StatusNotAuthoritative(getattr(response, "status", None)) diff --git a/syberwork/server.py b/syberwork/server.py index 00121517..62509532 100644 --- a/syberwork/server.py +++ b/syberwork/server.py @@ -26,6 +26,12 @@ # reader is an administrator or the approver role, and not an automation credential (model, compiled, search). # There is no separate policy-read role; model.js mirrors this to suggest an identity that can read them. POLICY_READERS = frozenset({"admin", "manager"}) +# Roles that only attest facts (``syberwork init`` gives the inventory and supplier credentials ``observer``). +# A principal holding nothing else may post its own source's observations and read its own profile, but it +# reads no case, contract, certificate or metric: those name every case's inputs, history and amounts. +ATTEST_ONLY_ROLES = frozenset({"observer"}) +# The first path segments of the reads ``_case_reader`` guards. +CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates")) def _planner_configured() -> bool: @@ -41,6 +47,11 @@ def _policy_reader(roles) -> bool: return bool(POLICY_READERS.intersection(roles)) and not AUTOMATION_ROLES.intersection(roles) +def _case_reader(roles) -> bool: + """Whether a principal may read cases, contracts, certificates and metrics: any role beyond attesting.""" + return isinstance(roles, list) and bool({r for r in roles if isinstance(r, str)} - ATTEST_ONLY_ROLES) + + def _source_roles(work: Work) -> dict: """Each installed source's read roles, as refresh_fact checks them, and nothing else of its definition. @@ -106,7 +117,37 @@ def _preview(work: Work, case_id: str, data: dict, name: str, roles: list) -> di "written": False, "note": "Admission now for a proposal not yet made; nothing was recorded and nothing is reserved"} -def serve(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, +class UserFile: + """``users.json`` read again whenever it changes, so a removed or rotated credential stops working without a + restart. Read on each request by ``stat``; parsed only when its size or modification time moves. A file that + is missing, unreadable or not a JSON object of users answers with no users: every bearer is refused (cell + principals registered in the database still authenticate).""" + + def __init__(self, path: str | Path): + self.path = Path(path) + self._stamp = None + self._users: dict = {} + + def __call__(self) -> dict: + try: + status = self.path.stat() + stamp = (status.st_mtime_ns, status.st_size, status.st_ino) + except OSError: + self._stamp, self._users = None, {} + return {} + if stamp != self._stamp: + try: + loaded = json.loads(self.path.read_text(encoding="utf-8")) + valid = isinstance(loaded, dict) and all( + isinstance(info, dict) and isinstance(info.get("hash"), str) for info in loaded.values()) + except (OSError, ValueError): + loaded, valid = {}, False + self._users = loaded if valid else {} + self._stamp = stamp + return self._users + + +def serve(work: Work, users, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, witness_role=None, certificates=None): httpd = make_server(work, users, host, port, witness_log=witness_log, witness_client=witness_client, witness_role=witness_role, certificates=certificates) @@ -114,10 +155,13 @@ def serve(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, httpd.serve_forever() -def make_server(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, +def make_server(work: Work, users, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, witness_role=None, certificates=None) -> ThreadingHTTPServer: """Build the loopback server without starting it. Port 0 picks a free port. + ``users`` is a mapping of name to ``{hash, roles, sources}``, or a callable returning one on each request + (``UserFile``, which ``syberwork serve`` passes, so an edit to ``users.json`` applies without a restart). + ``witness_log`` is a ``TransparencyLog.open_readonly`` reader of the witness's log, or None. It holds no seed and cannot append; ``GET /api/cases/{id}/witness`` reads it. ``witness_client`` is a ``WitnessClient`` on the witness's loopback port and ``witness_role`` the one role whose holders ``POST /api/cases/{id}/witness`` lets ask it to sign: @@ -129,7 +173,9 @@ def make_server(work: Work, users: dict, host: str = "127.0.0.1", port: int = 87 SYBERWORK_CERTIFICATE_HOME and SYBERWORK_CERTIFICATE_PUBLIC. """ loopback = host in ("127.0.0.1", "::1", "localhost") - if not loopback and os.getenv("SYBERWORK_BIND", "") != host: + # An empty host binds every interface. SYBERWORK_BIND names a non-loopback address on purpose, so an empty or + # unset value never vouches for one, even when the host is empty too. + if not loopback and (not host or not isinstance(host, str) or os.getenv("SYBERWORK_BIND", "") != host): raise Rejected("unsafe_bind", "the operator service must bind to loopback") if (witness_client is None) != (witness_role is None) or (witness_client is not None and witness_log is None): raise Rejected("invalid_runtime", "a witness client needs a role and the witness log, and a role needs a client") @@ -144,7 +190,8 @@ class Handler(BaseHTTPRequestHandler): def _user(self): token = self.headers.get("Authorization", "").removeprefix("Bearer ") hashed = hashlib.sha256(token.encode()).hexdigest() - for name, info in users.items(): + current = users() if callable(users) else users + for name, info in current.items(): if token and hmac.compare_digest(info["hash"], hashed): return {"name": name, **info} principal = work.principal_for_token(token) @@ -186,6 +233,8 @@ def do_GET(self): self._static(parts[0], ASSETS[parts[0]]) return user = self._user() + if any(tuple(parts[:len(prefix)]) == prefix for prefix in CASE_READS) and not _case_reader(user.get("roles", [])): + raise Rejected("forbidden", "a role beyond observer is required to read cases, contracts, certificates and metrics") if len(parts) in (2, 3) and parts[:2] == ["metrics", "work_items"]: # Per-work-item metrics (docs/METRICS.md): every item, or one thread's, with each number's basis. try: @@ -325,7 +374,7 @@ def do_POST(self): if operation == "facts": if data["source"] not in user.get("sources", []): raise Rejected("source_denied", "actor cannot attest this source") - result = work.observe(case_id, data["key"], data["value"], data["source"], data["version"], actor) + result = work.attest(case_id, data["key"], data["value"], data["source"], data["version"], actor) elif operation == "refresh": result = work.refresh_fact(case_id, data["source"], data["key"], data["record_key"], actor, roles) elif operation == "resolution-request": diff --git a/syberwork/storage.py b/syberwork/storage.py index 253c572a..759a5499 100644 --- a/syberwork/storage.py +++ b/syberwork/storage.py @@ -27,7 +27,7 @@ def _statements(sql: str) -> list[str]: return [part.strip() for part in sql.split(";") if part.strip()] -_MIGRATIONS: list[tuple[str, str]] = [ +_MIGRATIONS: list[tuple[str, str | dict[str, str]]] = [ ("0001", f""" CREATE TABLE IF NOT EXISTS cell_lock ( id INTEGER PRIMARY KEY, @@ -273,6 +273,13 @@ def _statements(sql: str) -> list[str]: UNIQUE (evaluation_event_hash) ); """), + # ``units`` admits amounts up to 2**63-1, but 0001 created ``amount_units`` as INTEGER, which PostgreSQL + # stores as int4: a reservation above 2147483647 aborted the commit before the adapter ran. SQLite's + # INTEGER is already 64-bit, so only PostgreSQL changes. A dialect-keyed script runs only on its dialect. + ("0006", { + "sqlite": "", + "postgres": "ALTER TABLE economic_reservations ALTER COLUMN amount_units TYPE BIGINT", + }), ] @@ -409,6 +416,8 @@ def migrate(store) -> list[str]: for version, script in _MIGRATIONS: if version in applied: continue + if isinstance(script, dict): + script = script.get(store.dialect, "") for statement in _statements(script): store.execute(statement) store.execute( diff --git a/tests/console_browser.mjs b/tests/console_browser.mjs index 42f67909..9aeeee56 100644 --- a/tests/console_browser.mjs +++ b/tests/console_browser.mjs @@ -258,26 +258,29 @@ await page.waitForSelector('.replay table.diff'); report.replay = await page.locator('.replay table.diff').first().textContent(); await shot('14-replay'); -// A typed observation and an Attest receipt on a second case stay unverified. +// A typed observation and an Attest receipt on a second case stay unverified. The supplier holds only the +// observer role: it may attest keys the contract does not read from another source, is refused one it does +// (quote must be a verified supplier read), and cannot read the case, so the operator inspects the result. await switchTo('operator'); await page.keyboard.press('n'); await page.waitForSelector('#newcase-form input[name="in:part_number"]'); await page.click('#newcase-form button.primary'); await page.waitForSelector('.case-head'); -await switchTo('supplier'); +const secondCase = decodeURIComponent(page.url().split('#/case/')[1].split('/')[0]); +const attest = (key, version, value) => page.evaluate(async ([id, token, body]) => { + const response = await fetch('/api/cases/' + encodeURIComponent(id) + '/facts', {method: 'POST', cache: 'no-store', + headers: {Authorization: 'Bearer ' + token, 'Content-Type': 'application/json'}, body: JSON.stringify(body)}); + return {status: response.status, body: await response.json().catch(() => null)}; +}, [secondCase, tokens.supplier, {key, source: 'supplier', version, value}]); +report.quote_attest = await attest('quote', 'typed-1', {id: 'Q-7', price: 1, part_number: 'P-104', supplier: 'typed'}); +report.note_attest = await attest('supplier_note', 'typed-1', {text: 'typed by the supplier'}); +report.receipt_attest = await attest('attest_receipt', 'receipt-1', {schema: 'attest.receipt/v0', verified: false, evidence: {after: {status: 'proven'}}}); +report.supplier_case_read = await page.evaluate(async ([id, token]) => + (await fetch('/api/cases/' + encodeURIComponent(id), {headers: {Authorization: 'Bearer ' + token}, cache: 'no-store'})).status, + [secondCase, tokens.supplier]); +// The operator is still on the second case; r reads it again with the two attestations. await page.keyboard.press('2'); -await page.locator('.evidence details.warn-panel summary').click(); -const manual = page.locator('.evidence details.warn-panel'); -await manual.locator('input[name="key"]').fill('quote'); -await manual.locator('input[name="source"]').fill('supplier'); -await manual.locator('input[name="version"]').fill('typed-1'); -await manual.locator('textarea[name="value"]').fill('{"id":"Q-7","price":1,"part_number":"P-104","supplier":"typed"}'); -await manual.locator('button.btn').click(); -await page.waitForFunction(() => document.querySelectorAll('.obs tbody tr').length === 1); -await manual.locator('input[name="key"]').fill('attest_receipt'); -await manual.locator('input[name="version"]').fill('receipt-1'); -await manual.locator('textarea[name="value"]').fill('{"schema":"attest.receipt/v0","verified":false,"evidence":{"after":{"status":"proven"}}}'); -await manual.locator('button.btn').click(); +await page.keyboard.press('r'); await page.waitForFunction(() => document.querySelectorAll('.obs tbody tr').length === 2); report.manual = await page.locator('.obs tbody tr').allTextContents(); await shot('15-unverified'); @@ -286,6 +289,7 @@ await page.waitForSelector('.effects .notice.warn'); report.attest_notice = await page.textContent('.effects .notice.warn'); // The supplier holds no source read role; in the Studio it sees the read roles, not the definitions. +await switchTo('supplier'); await page.click('[data-view="studio"]'); await page.locator('.studio-item', { hasText: 'Sources' }).click(); await page.waitForSelector('.studio-body table.grid'); diff --git a/tests/test_cell.py b/tests/test_cell.py index f8cae019..e22c55ac 100644 --- a/tests/test_cell.py +++ b/tests/test_cell.py @@ -80,7 +80,7 @@ def test_inline_commit_still_settles_and_a_legacy_database_migrates(self): self.assertEqual(committed["status"], "succeeded") self.assertTrue(work.verify_chain(case_id)) versions = [row["version"] for row in work._db.execute("SELECT version FROM schema_migrations ORDER BY version")] - self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005"]) + self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005", "0006"]) work.close() copy = Path(folder) / "main.sqlite" shutil.copy(FIXTURE, copy) @@ -201,7 +201,7 @@ def open_db(): self.assertEqual(errors, []) check = Work(path) versions = [row["version"] for row in check._db.execute("SELECT version FROM schema_migrations ORDER BY version")] - self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005"]) + self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005", "0006"]) check.close() def test_failed_restore_does_not_keep_the_rows(self): diff --git a/tests/test_console.py b/tests/test_console.py index 274023cf..aaab07f9 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -413,7 +413,9 @@ def test_listed_read_roles_are_the_ones_refresh_enforces(self): def test_work_item_summary_carries_what_the_top_bar_reads(self): case = self.cell.json("/api/cases", {"contract_id": "purchase-order", "version": 1, "inputs": {"part_number": "P-104", "quantity": 2}})[1]["id"] - for who in ("operator", "manager", "supplier"): + # An observer-only credential (supplier) attests facts and reads no metric (security review, finding 4). + self.assertEqual(self.cell.json("/metrics/work_items", who="supplier")[0], 403) + for who in ("operator", "manager"): with self.subTest(who): status, body = self.cell.json("/metrics/work_items", who=who) self.assertEqual(status, 200) @@ -570,7 +572,9 @@ def test_side_records_are_work_side_channel_for_any_principal(self): case = self.case_with_decision() events = self.cell.work.inspect(case)["events"] expected = self.cell.work.side_channel(case) - for who in ("operator", "manager", "supplier", "planner"): + # Any principal beyond an observer-only credential (security review, finding 4). + self.assertEqual(self.cell.json(f"/api/cases/{case}/side", who="supplier")[0], 403) + for who in ("operator", "manager", "planner"): with self.subTest(who): self.assertEqual(self.cell.json(f"/api/cases/{case}/side", who=who), (200, expected)) self.assertEqual([row["seq"] for row in expected], [e["seq"] for e in events], "every new event has its row") @@ -662,7 +666,8 @@ def test_a_signed_head_is_witnessed_through_its_count_and_later_events_are_not(s self.attest(other_events) self.assertEqual(self.cell.json(f"/api/cases/{case}/witness")[1]["entries"], [], "another case's leaf is not this case's") self.attest(events) - for who in ("operator", "manager", "supplier", "planner"): + self.assertEqual(self.cell.json(f"/api/cases/{case}/witness", who="supplier")[0], 403) # observer-only + for who in ("operator", "manager", "planner"): with self.subTest(who): status, body = self.cell.json(f"/api/cases/{case}/witness", who=who) self.assertEqual(status, 200) @@ -997,7 +1002,9 @@ def journal(self) -> Path: def test_the_route_requires_a_token_and_is_read_by_any_principal(self): self.assertEqual(self.cell.json("/api/certificates", who=None)[0], 401) expected = self.listing() - for who in ("manager", "supplier", "planner", "admin"): + # Any principal beyond an observer-only credential (security review, finding 4). + self.assertEqual(self.cell.json("/api/certificates", who="supplier")[0], 403) + for who in ("manager", "planner", "admin"): with self.subTest(who): self.assertEqual(self.listing(who), expected) self.assertEqual(self.cell.json("/api/console", who="supplier")[1]["certificates"], {"configured": True}) @@ -1560,6 +1567,11 @@ def test_the_purchase_order_path_completes_in_a_browser(self): self.assertFalse(any("invented.rule" in row for row in report["mismatch_rules"]), "a rule that does not reproduce the decision is never shown") self.assertTrue(report["refused_rules"] and all("unavailable" in row and "unknown_case" in row for row in report["refused_rules"]), report["refused_rules"]) self.assertIn("amount_exceeds_limit", report["replay"]) + # The observer-only supplier is refused a key the contract reads from a verified source and cannot read the case. + self.assertEqual((report["quote_attest"]["status"], report["quote_attest"]["body"]["error"]), (409, "attestation_denied")) + self.assertEqual(report["note_attest"]["status"], 200, report["note_attest"]) + self.assertEqual(report["receipt_attest"]["status"], 200, report["receipt_attest"]) + self.assertEqual(report["supplier_case_read"], 403) self.assertEqual(len(report["manual"]), 2) self.assertTrue(all("unverified" in row for row in report["manual"])) self.assertIn("Attest claim", report["manual"][0]) diff --git a/tests/test_security_review.py b/tests/test_security_review.py new file mode 100644 index 00000000..822b10f4 --- /dev/null +++ b/tests/test_security_review.py @@ -0,0 +1,402 @@ +"""Regression tests for the trust-boundary findings of the 2026-11 SyberWork security review. + +Each class names the finding it pins. No test opens a connection beyond loopback, moves funds or reads a +real service: name resolution is patched, and the servers are SQLite cells on free loopback ports. +""" + +import hashlib +import json +import os +import socket +import sys +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +from syberlabs import planner as planner_module +from syberlabs import publish as publish_module +from syberlabs.errors import Rejected +from syberlabs.publish import Command +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin +from syberwork import core as core_module +from syberwork.backup import export_cell +from syberwork.core import Work +from syberwork.server import UserFile, make_server +from syberwork.storage import ReservationStore +from tests.test_cell import _drop_postgres, _install, _postgres_database + +EXAMPLES = Path(__file__).resolve().parents[1] / "syberwork" / "examples" +PUBLIC = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))] + + +def _token_hash(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +class HostCheckMatchesDialTarget(unittest.TestCase): + """Finding 1: the guard classified ``%31%32%37.0.0.1`` as a name while urllib dialed ``127.0.0.1``.""" + + ENCODED = ("https://%31%32%37.0.0.1/orders", "https://%6c%6fcalhost/x", "https://exa%6dple.com/x", + "https://169.254.169.%32%35%34/latest", "https://ex_ample.com/x", "https://éx.example/x", + "https://exa mple.com/x", "https://{key}.example.com/x") + + def test_a_host_urllib_would_rewrite_is_refused_before_classification(self): + for url in self.ENCODED: + with self.subTest(url=url), self.assertRaises(Rejected) as refused: + trusted_origin(url) + self.assertEqual(refused.exception.code, "invalid_target") + + def test_plain_names_and_ip_literals_still_pass(self): + self.assertEqual(trusted_origin("https://EXAMPLE.com/x"), ("https", "example.com", 443)) + self.assertEqual(trusted_origin("https://xn--bcher-kva.example/x"), ("https", "xn--bcher-kva.example", 443)) + self.assertEqual(trusted_origin("https://[2606:4700::1]:8443/x"), ("https", "2606:4700::1", 8443)) + self.assertEqual(trusted_origin("http://127.0.0.1:9/items/{key}"), ("http", "127.0.0.1", 9)) + + def test_the_guard_refuses_an_encoded_loopback_without_a_lookup(self): + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=AssertionError("no lookup")): + with self.assertRaises(Rejected): + guard_request("https://%31%32%37.0.0.1/orders") + + def test_a_name_the_guard_cannot_resolve_is_refused_not_allowed(self): + for failure in (socket.gaierror(), UnicodeError("label"), OSError("resolver")): + with self.subTest(failure=type(failure).__name__): + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=failure): + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path") + with patch("syberlabs.targets.socket.getaddrinfo", return_value=[]): + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path") + with patch("syberlabs.targets.socket.getaddrinfo", return_value=PUBLIC): + guard_request("https://example.com/path") + + def test_an_unresolvable_source_is_unavailable_and_makes_no_request(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + case = _install(work) + work.install_source("crm", {"kind": "http", "url": "https://crm.example/records/{key}"}) + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=socket.gaierror()), \ + patch.object(core_module.HTTP, "open", side_effect=AssertionError("no request")): + with self.assertRaises(Rejected) as refused: + work.refresh_fact(case, "crm", "c", "R-1", "operator", ["operator"]) + self.assertEqual((refused.exception.code, refused.exception.detail), ("source_unavailable", "destination_unreachable")) + work.close() + + +class SourceKeyStaysInThePath(unittest.TestCase): + """Finding 2: ``{key}`` in a source host let the case input pick the host that receives the bearer.""" + + def test_install_source_refuses_key_outside_the_path(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for url in ("https://{key}.example.com/records", "https://records.example/{key}/{key}", + "https://records.example/r?id={key}", "https://records.example/{key}?token=abc", + "https://records.example/{key}/{other}", "https://records.example/r#{key}"): + with self.subTest(url=url), self.assertRaises(Rejected) as refused: + work.install_source("crm", {"kind": "http", "url": url, "auth_env": "CRM_TOKEN"}) + self.assertEqual(refused.exception.code, "invalid_source") + work.install_source("crm", {"kind": "http", "url": "https://records.example/r/{key}", "auth_env": "CRM_TOKEN"}) + self.assertIn("crm", work.registry("sources")) + work.close() + + +class ActionUrlCarriesNoQuery(unittest.TestCase): + """Finding 3: a query string (where a secret rides) was stored in the action and served by ``registry``.""" + + def test_install_action_refuses_a_query_on_the_write_or_status_url(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for doc in ({"kind": "http", "url": "https://orders.example/po?api_key=s3cret"}, + {"kind": "http", "url": "https://orders.example/po?"}, + {"kind": "http", "url": "https://orders.example/{key}"}, + {"kind": "http", "url": "https://orders.example/po", + "status_url": "https://orders.example/po/{key}?api_key=s3cret"}): + with self.subTest(doc=doc), self.assertRaises(Rejected) as refused: + work.install_action("issue", doc) + self.assertEqual(refused.exception.code, "invalid_action") + self.assertEqual(work.registry("actions"), {}) + work.install_action("issue", {"kind": "http", "url": "https://orders.example/po", + "status_url": "https://orders.example/po/{key}", "auth_env": "ORDERS_TOKEN"}) + self.assertNotIn("?", json.dumps(work.registry("actions"))) + work.close() + + +class ObserverCannotStallOrRead(unittest.TestCase): + """Finding 4: an observer token could shadow a verified fact with an unverified one and read every case.""" + + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.work = Work(Path(folder.name) / "w.sqlite") + self.addCleanup(self.work.close) + self.work.install_policy(json.loads((EXAMPLES / "policy.json").read_text())) + self.work.install_contract(json.loads((EXAMPLES / "contract.json").read_text())) + self.work.install_action("record_review", {"kind": "local"}) + self.work.install_action("issue_order", {"kind": "local"}) + self.case = self.work.create_case("purchase-order", 1, {"part_number": "P-104", "quantity": 2}, "operator") + self.users = { + "inventory": {"hash": _token_hash("inv"), "roles": ["observer"], "sources": ["inventory"]}, + "supplier": {"hash": _token_hash("sup"), "roles": ["observer"], "sources": ["supplier"]}, + "operator": {"hash": _token_hash("op"), "roles": ["operator"], "sources": []}, + } + self.server = make_server(self.work, self.users, port=0) + self.server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = f"http://127.0.0.1:{self.server.server_port}" + + def call(self, path, token, data=None): + headers = {"Authorization": "Bearer " + token} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(self.base + path, data=None if data is None else json.dumps(data).encode(), + headers=headers, method="GET" if data is None else "POST") + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read()) + + def test_an_attestation_cannot_shadow_a_fact_the_contract_reads_verified(self): + self.work.observe(self.case, "part_number", "P-104", "inventory", "v1", "sync", verified=True) + before = len(self.work.inspect(self.case)["events"]) + # The supplier attests the inventory's key, and the inventory attests its own key unverified: the contract + # reads part_number only verified and only from inventory, so neither may land as the newest observation. + for token, source in (("sup", "supplier"), ("inv", "inventory")): + with self.subTest(source=source): + status, body = self.call(f"/api/cases/{self.case}/facts", token, + {"key": "part_number", "source": source, "version": "x", "value": "P-999"}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + # A key used only through an argument binding (``quote.id``) is protected too. + status, body = self.call(f"/api/cases/{self.case}/facts", "sup", + {"key": "quote", "source": "supplier", "version": "x", "value": {"id": "Q", "price": 1}}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + self.assertEqual(len(self.work.inspect(self.case)["events"]), before) + proposed = self.work.propose(self.case, "record_review", {"part_number": "P-104"}, "operator", ["operator"]) + self.assertEqual(proposed["decision"]["status"], "allowed") + + def test_a_key_the_contract_does_not_read_is_still_recorded_unverified(self): + status, body = self.call(f"/api/cases/{self.case}/facts", "inv", + {"key": "shelf_note", "source": "inventory", "version": "n1", "value": "bin 4"}) + self.assertEqual(status, 200) + self.assertIs(body["body"]["verified"], False) + + def test_a_contract_that_accepts_a_sources_word_still_takes_it(self): + contract = json.loads((EXAMPLES / "contract.json").read_text()) + contract["version"] = 2 + contract["actions"]["record_review"]["required_facts"][0]["verified"] = False + self.work.install_contract(contract) + case = self.work.create_case("purchase-order", 2, {"part_number": "P-104", "quantity": 2}, "operator") + status, _ = self.call(f"/api/cases/{case}/facts", "inv", + {"key": "part_number", "source": "inventory", "version": "n1", "value": "P-104"}) + self.assertEqual(status, 200) + status, body = self.call(f"/api/cases/{case}/facts", "sup", + {"key": "part_number", "source": "supplier", "version": "n1", "value": "P-1"}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + + def test_an_observer_reads_no_case_contract_certificate_or_metric(self): + for path in ("/api/cases", f"/api/cases/{self.case}", f"/api/cases/{self.case}/verify", + f"/api/cases/{self.case}/side", f"/api/cases/{self.case}/candidates", "/api/contracts", + "/api/contracts/purchase-order/1", "/api/contracts/purchase-order/1/artifacts", + "/api/certificates", "/metrics/cases_total", "/metrics/work_items"): + with self.subTest(path=path): + status, body = self.call(path, "inv") + self.assertEqual((status, body["error"]), (403, "forbidden")) + status, _ = self.call(path, "op") + self.assertEqual(status, 200) + status, body = self.call("/api/me", "inv") + self.assertEqual((status, body["name"]), (200, "inventory")) + + +class BackupDoesNotFollowLinks(unittest.TestCase): + """Finding 5: ``export_cell`` replaced the target of a ``manifest.json`` symlink with the snapshot.""" + + @unittest.skipIf(sys.platform == "win32", "creating symlinks needs a privilege on Windows") + def test_a_planted_link_is_refused_and_its_target_untouched(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + _install(work) + victim = Path(folder) / "victim.txt" + victim.write_text("keep me", encoding="utf-8") + for name in ("manifest.json", "cases.json"): + with self.subTest(name=name): + snapshot = Path(folder) / ("snap-" + name) + snapshot.mkdir() + (snapshot / name).symlink_to(victim) + with self.assertRaises(Rejected) as refused: + export_cell(work, snapshot) + self.assertEqual(refused.exception.code, "backup_refused") + self.assertEqual(victim.read_text(encoding="utf-8"), "keep me") + self.assertTrue((snapshot / name).is_symlink()) + clean = Path(folder) / "clean" + manifest = export_cell(work, clean) + self.assertEqual(json.loads((clean / "manifest.json").read_text(encoding="utf-8")), manifest) + self.assertEqual([p.name for p in clean.iterdir() if p.name.startswith(".")], []) + work.close() + + +class NoEnvironmentProxy(unittest.TestCase): + """Finding 6: ``build_opener`` installed a ProxyHandler from ``http_proxy`` at import, so the proxy got the bearer.""" + + def test_credentialed_openers_carry_no_proxy(self): + for opener in (core_module.HTTP, publish_module.HTTP, planner_module._HTTP): + with self.subTest(opener=opener): + self.assertFalse(any(isinstance(h, urllib.request.ProxyHandler) and h.proxies for h in opener.handlers)) + self.assertFalse(any(name.endswith("_open") and "proxy" in name for h in opener.handlers for name in dir(h) + if isinstance(h, urllib.request.ProxyHandler))) + + def test_a_proxy_set_before_the_opener_is_built_is_not_used(self): + hits = [] + + class Proxy(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(1) + self.sock.settimeout(5) + + def run(self): + try: + conn, _ = self.sock.accept() + hits.append(conn.recv(4096)) + conn.close() + except OSError: + pass + + proxy = Proxy() + proxy.start() + self.addCleanup(proxy.sock.close) + target = socket.socket() + target.bind(("127.0.0.1", 0)) + target.listen(1) + self.addCleanup(target.close) + address = f"http://127.0.0.1:{proxy.sock.getsockname()[1]}" + with patch.dict(os.environ, {"http_proxy": address, "HTTP_PROXY": address, "no_proxy": "", "NO_PROXY": ""}): + opener = credentialed_opener(core_module.NoRedirect) + request = urllib.request.Request(f"http://127.0.0.1:{target.getsockname()[1]}/x", + headers={"Authorization": "Bearer secret"}) + with self.assertRaises((urllib.error.URLError, OSError)): + opener.open(request, timeout=1) + proxy.sock.close() + proxy.join(2) + self.assertEqual(hits, []) + + +class ReservationAmountsAre64Bit(unittest.TestCase): + """Follow-up: PostgreSQL stored ``amount_units`` as int4 while ``units`` admits up to 2**63-1.""" + + LARGE = 2**62 + 7 + + def _reserve(self, work): + with work.tx() as db: + ReservationStore(db).reserve("p-large", "c", "b", "USD", self.LARGE) + with work.tx() as db: + self.assertEqual(ReservationStore(db).reserved_total("b", "USD"), self.LARGE) + + def test_sqlite_keeps_a_large_reservation(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + self._reserve(work) + work.close() + + def test_postgresql_keeps_a_large_reservation(self): + url, admin, name = _postgres_database() + self.addCleanup(_drop_postgres, admin, name) + work = Work(url) + try: + self._reserve(work) + kind = work._db.execute("SELECT data_type::text AS data_type FROM information_schema.columns " + "WHERE table_name = 'economic_reservations' AND column_name = 'amount_units'").fetchone() + self.assertEqual(kind["data_type"], "bigint") + finally: + work.close() + + +class PublishTimeoutEndsTheTree(unittest.TestCase): + """Follow-up: a publish command's timeout killed one process while a check's kills the whole tree.""" + + @unittest.skipUnless(hasattr(os, "killpg"), "the process-group path is POSIX; Windows uses taskkill /T") + def test_a_timed_out_command_leaves_no_child_running(self): + with tempfile.TemporaryDirectory() as folder: + beat = Path(folder) / "beat" + child = ("import time,pathlib\n" + f"p=pathlib.Path({str(beat)!r})\n" + "while True:\n p.open('a').write('.')\n time.sleep(0.05)\n") + parent = (f"import subprocess,sys,time\nsubprocess.Popen([sys.executable,'-c',{child!r}])\ntime.sleep(60)\n") + + class Repo: + @contextmanager + def worktree(self, commit): + yield Path(folder) + + class Kit: + repo = Repo() + + command = Command(Kit(), {"timeout_seconds": 1, "argv": [sys.executable, "-c", parent]}) + started = time.monotonic() + self.assertEqual(command._run(command.doc["argv"], "case", {"commit": "c"}, "k"), (None, "")) + self.assertLess(time.monotonic() - started, 30) + time.sleep(0.3) + size = beat.stat().st_size if beat.exists() else 0 + time.sleep(0.6) + self.assertEqual(beat.stat().st_size if beat.exists() else 0, size, "the grandchild kept running") + + +class BindAndUsers(unittest.TestCase): + """Follow-ups: an empty SYBERWORK_BIND passed the loopback guard, and users.json was read once.""" + + def test_an_empty_host_never_binds(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for env in ("", None): + with self.subTest(env=env): + environ = {"SYBERWORK_BIND": env} if env is not None else {} + with patch.dict(os.environ, environ, clear=False): + if env is None: + os.environ.pop("SYBERWORK_BIND", None) + with self.assertRaises(Rejected) as refused: + make_server(work, {}, "", 0) + self.assertEqual(refused.exception.code, "unsafe_bind") + work.close() + + def test_users_json_changes_apply_without_a_restart(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "users.json" + path.write_text(json.dumps({"op": {"hash": _token_hash("one"), "roles": ["operator"], "sources": []}})) + work = Work(Path(folder) / "w.sqlite") + server = make_server(work, UserFile(path), port=0) + server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(work.close) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + def me(token): + request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/api/me", + headers={"Authorization": "Bearer " + token}) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + self.assertEqual(me("one"), 200) + path.write_text(json.dumps({"op": {"hash": _token_hash("two-rotated"), "roles": ["operator"], "sources": []}})) + os.utime(path, ns=(time.time_ns(), time.time_ns() + 1_000_000)) + self.assertEqual((me("one"), me("two-rotated")), (401, 200)) + path.write_text("not json") + os.utime(path, ns=(time.time_ns(), time.time_ns() + 2_000_000)) + self.assertEqual(me("two-rotated"), 401) + path.unlink() + self.assertEqual(me("two-rotated"), 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_witness.py b/tests/test_witness.py index 1153f405..f5c404b8 100644 --- a/tests/test_witness.py +++ b/tests/test_witness.py @@ -10,6 +10,7 @@ import sqlite3 import tempfile import unittest +import urllib.error from pathlib import Path from unittest.mock import patch @@ -273,8 +274,10 @@ def test_request_guard_checks_resolved_addresses(self): guard_request("https://example.com/path") with patch("syberlabs.targets.socket.getaddrinfo", side_effect=AssertionError("dns")): guard_request("http://127.0.0.1:9/items/1") + # A name the guard cannot resolve is refused (URLError, so callers keep their unreachable codes). with patch("syberlabs.targets.socket.getaddrinfo", side_effect=socket.gaierror()): - guard_request("https://example.com/path") + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path") From 3de1e832db2e29786ceb73eae7731eeaf2232a11 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 22:55:56 +0000 Subject: [PATCH 2/3] Close the users.json test's server and database before removing its folder Windows refuses to delete an open SQLite file, and the cleanups ran after the temporary directory had already been removed. Co-Authored-By: Claude Opus 5.5 --- tests/test_security_review.py | 62 +++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/tests/test_security_review.py b/tests/test_security_review.py index 822b10f4..19906f07 100644 --- a/tests/test_security_review.py +++ b/tests/test_security_review.py @@ -367,35 +367,39 @@ def test_an_empty_host_never_binds(self): work.close() def test_users_json_changes_apply_without_a_restart(self): - with tempfile.TemporaryDirectory() as folder: - path = Path(folder) / "users.json" - path.write_text(json.dumps({"op": {"hash": _token_hash("one"), "roles": ["operator"], "sources": []}})) - work = Work(Path(folder) / "w.sqlite") - server = make_server(work, UserFile(path), port=0) - server.RequestHandlerClass.log_message = lambda *args: None - threading.Thread(target=server.serve_forever, daemon=True).start() - self.addCleanup(work.close) - self.addCleanup(server.server_close) - self.addCleanup(server.shutdown) - - def me(token): - request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/api/me", - headers={"Authorization": "Bearer " + token}) - try: - with urllib.request.urlopen(request, timeout=5) as response: - return response.status - except urllib.error.HTTPError as exc: - return exc.code - - self.assertEqual(me("one"), 200) - path.write_text(json.dumps({"op": {"hash": _token_hash("two-rotated"), "roles": ["operator"], "sources": []}})) - os.utime(path, ns=(time.time_ns(), time.time_ns() + 1_000_000)) - self.assertEqual((me("one"), me("two-rotated")), (401, 200)) - path.write_text("not json") - os.utime(path, ns=(time.time_ns(), time.time_ns() + 2_000_000)) - self.assertEqual(me("two-rotated"), 401) - path.unlink() - self.assertEqual(me("two-rotated"), 401) + # Cleanups run last in, first out: the folder is removed after the server and the database close, + # which Windows needs before it will delete an open SQLite file. + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + folder = scratch.name + path = Path(folder) / "users.json" + path.write_text(json.dumps({"op": {"hash": _token_hash("one"), "roles": ["operator"], "sources": []}})) + work = Work(Path(folder) / "w.sqlite") + server = make_server(work, UserFile(path), port=0) + server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(work.close) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + def me(token): + request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/api/me", + headers={"Authorization": "Bearer " + token}) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + self.assertEqual(me("one"), 200) + path.write_text(json.dumps({"op": {"hash": _token_hash("two-rotated"), "roles": ["operator"], "sources": []}})) + os.utime(path, ns=(time.time_ns(), time.time_ns() + 1_000_000)) + self.assertEqual((me("one"), me("two-rotated")), (401, 200)) + path.write_text("not json") + os.utime(path, ns=(time.time_ns(), time.time_ns() + 2_000_000)) + self.assertEqual(me("two-rotated"), 401) + path.unlink() + self.assertEqual(me("two-rotated"), 401) if __name__ == "__main__": From 5c287e5201d679341b66987e2eb6d155a2061420 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 3 Nov 2026 23:28:37 +0000 Subject: [PATCH 3/3] Close the review's remaining items: F6, F11, F14 to F18, and the F3/F8 edges - targets: refuse IANA special-purpose ranges explicitly (F6); pin HTTPS connections to the addresses the check accepted, name kept for SNI and Host (F11); header_value_ok for header values - core: refuse a version_arg that cannot be a header before the claim, and any unsafe header before a request (F14); a different result key is not a change unless the confirmation re-reads the trigger or the result's version moved (F15); compare-and-swap an expired lease to 'recovering' and keep a stored outcome when settling it (F16); acceptance fact keys are protected from manual attestation - evidence: a fact clause reads the newest observation and honors verified and source (F17) - server: frame-ancestors 'none' and X-Frame-Options on the console (F18); observer-only credentials read no builder view (F8) - planner: a planner URL takes no query string (F3) Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/DEPLOY.md | 2 +- spec/GAPS.md | 2 +- syberlabs/evidence.py | 7 +- syberlabs/planner.py | 2 + syberlabs/targets.py | 107 ++++++++++++++-- syberwork/core.py | 69 +++++++++-- syberwork/server.py | 8 +- tests/test_console.py | 4 +- tests/test_security_review.py | 224 +++++++++++++++++++++++++++++++++- 10 files changed, 398 insertions(+), 29 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d6fe59d0..e4efe801 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,6 +34,6 @@ For each past proposal, replay evaluates a selected new contract and policy agai ## Integration -HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call and refuses a name it cannot resolve; a host with percent-escapes or other characters `urllib.request` would rewrite is refused before it is classified. The credentialed openers ignore `http_proxy`/`https_proxy` (`targets.credentialed_opener`), so a bearer token is never handed to an environment proxy. A manual attestation (`POST /api/cases/{id}/facts`) may not set a key the case's contract reads from another source or only verified, and an `observer`-only credential reads no case, contract, certificate or metric. The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. +HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call and refuses a name it cannot resolve; a host with percent-escapes or other characters `urllib.request` would rewrite is refused before it is classified. The credentialed openers ignore `http_proxy`/`https_proxy` (`targets.credentialed_opener`), so a bearer token is never handed to an environment proxy, and their HTTPS connections are pinned: each resolves and checks its host again and dials only the addresses it accepted, keeping the name for SNI and Host. IANA special-purpose ranges are refused explicitly rather than by `is_global` alone. Header values (`If-Match` from `version_arg`) are refused if they hold a line break, control character or edge whitespace, before the effect is claimed. A manual attestation (`POST /api/cases/{id}/facts`) may not set a key the case's contract reads from another source or only verified, and an `observer`-only credential reads no case, contract, certificate, metric or builder view. The console document refuses framing (`frame-ancestors 'none'`). The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. The [`economic_http` effect](ECONOMIC_ACTIONS.md) adds exact atomic units and a cross-case budget reservation to this same admission boundary. Its trusted settlement service is a separate authority; SyberWork records that service's matching receipt and status assertion, not independent blockchain finality. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index a33f9b09..919bd51f 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -65,7 +65,7 @@ X-Record-Version: 6f1c...e2 | `effects_pending` | Claimed effects not yet settled: queued, running, or `effect_unknown` awaiting reconciliation. | | `effects_settled_applied_total` | Claimed effects with `effect_succeeded` or a verified `reconciled`. | | `effects_settled_not_applied_total` | Claimed effects with `effect_rejected` (a no-write refusal or `not_applied`). | -| `queue_depth` | Worker obligations queued or leased. Always 0 when effects settle inline. | +| `queue_depth` | Worker obligations queued, leased, or being recovered after an expired lease. Always 0 when effects settle inline. | | `uptime_seconds` | Seconds since this server process started. | Every claimed effect is in exactly one of the three effect counters. `observed_at` is the epoch time of the read, the form the effect's `max_age_seconds` rule understands; the version is a digest of key, value and `observed_at`, so a repeated read of an unchanged value is still a fresh, distinct reading. An unknown key is `404 {"error": "unknown_metric"}`. The route needs a bearer token like every other read on the service; there is no anonymous metrics endpoint. diff --git a/spec/GAPS.md b/spec/GAPS.md index 4f868843..c0a8ac42 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -8,7 +8,7 @@ The schemas describe SyberWork as it behaves. They are not enforced at runtime e 3. **`install_contract` checks the fields admission indexes, including input kinds.** An acceptance `effect` without `action`, a `signoff` without `role`, a `fact` without `key`, or a `required_facts` item without `key` and `source` is rejected at publish. An input kind other than `string` or `integer` is `invalid_contract`. `create_case` rejects a value of the wrong kind with `input_schema`, including a boolean passed where an integer is required. -4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). The host must be an IP literal or plain ASCII letter-digit-hyphen labels, so the name checked is the name `urllib.request` dials (a percent-escaped host is refused). DNS rebinding is checked at request time by `guard_request`, not at install; a name `guard_request` cannot resolve is refused, never allowed. Installed action URLs carry no query, and a source URL carries exactly one `{key}`, in its path. +4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). The host must be an IP literal or plain ASCII letter-digit-hyphen labels, so the name checked is the name `urllib.request` dials (a percent-escaped host is refused). DNS rebinding is checked at request time, not at install: `guard_request` refuses a name it cannot resolve, and the credentialed HTTPS connection resolves again and dials only the addresses it accepted. Installed action URLs and the planner URL carry no query, and a source URL carries exactly one `{key}`, in its path. 5. **Proposals are appended before admission.** A non-dict `args` or a missing role list is stored, then denied with `invalid_proposal_shape`. The Proposal schema therefore does not require `args` to be an object. diff --git a/syberlabs/evidence.py b/syberlabs/evidence.py index a34b0516..740d1e63 100644 --- a/syberlabs/evidence.py +++ b/syberlabs/evidence.py @@ -125,7 +125,12 @@ def acceptance_results(contract: dict, history: list) -> list[dict]: and e["body"]["action"] == clause.get("after_action")), 0) passed = any(e["kind"] == "signed" and e["body"]["role"] == clause["role"] and (not clause.get("after_action") or e["seq"] > completed_seq > 0) for e in history) elif clause["kind"] == "fact": - passed = any(e["kind"] == "observed" and e["body"]["key"] == clause["key"] and e["body"]["value"] == clause.get("equals") for e in history) + # The newest observation of the key decides, as admission reads facts; an older matching value does + # not. A clause that asks for ``verified`` (or names a ``source``) is held to it. + latest = next((e for e in reversed(history) if e["kind"] == "observed" and e["body"]["key"] == clause["key"]), None) + passed = (latest is not None and latest["body"]["value"] == clause.get("equals") + and (not clause.get("verified") or latest["body"].get("verified") is True) + and ("source" not in clause or latest["body"].get("source") == clause["source"])) else: passed = False results.append({"id": clause["id"], "passed": passed}) diff --git a/syberlabs/planner.py b/syberlabs/planner.py index fb601f37..a95032d2 100644 --- a/syberlabs/planner.py +++ b/syberlabs/planner.py @@ -59,6 +59,8 @@ class HttpPlanner: def ensure_configured(self) -> str: target = os.getenv("SYBERWORK_PLANNER_URL", "") try: + if "?" in target: # a planner URL names an endpoint; a credential in its query would be logged and sent + raise Rejected("invalid_target", "planner URL takes no query string") trusted_origin(target) except Rejected: raise PlannerRefusal("planner_unconfigured", "configure an HTTPS or local planner endpoint") from None diff --git a/syberlabs/targets.py b/syberlabs/targets.py index f8aad3b6..8c7deb90 100644 --- a/syberlabs/targets.py +++ b/syberlabs/targets.py @@ -2,6 +2,7 @@ from __future__ import annotations +import http.client import ipaddress import os import re @@ -27,10 +28,30 @@ _HOSTNAME = re.compile(rf"(?:{_LABEL}\.)*{_LABEL}\.?", re.ASCII) +# IANA special-purpose ranges (RFC 6890 registries) that are not a destination for a credentialed request. +# ``ipaddress`` flags most of them, but not all, and not the same set on every interpreter: some versions call +# 192.0.0.9, 192.0.0.10, 2001:1::1, 2001:1::2 and the deprecated site-local fec0::/10 global. They are listed here +# so the answer does not depend on the Python release. Anycast service addresses (PCP, TURN) are refused too: +# no installed source, action or planner is one of them. +_SPECIAL_NETWORKS = tuple(ipaddress.ip_network(net) for net in ( + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", + "192.0.0.0/24", "192.0.2.0/24", "192.31.196.0/24", "192.52.193.0/24", "192.88.99.0/24", "192.168.0.0/16", + "192.175.48.0/24", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", + "::/127", "::ffff:0:0/96", "64:ff9b:1::/48", "100::/64", "2001::/23", "2001:db8::/32", "2002::/16", + "2620:4f:8000::/48", "3fff::/20", "5f00::/16", "fc00::/7", "fe80::/10", "fec0::/10", "ff00::/8", +)) +# NAT64 (64:ff9b::/96) carries an IPv4 address in its last 32 bits; the gateway dials that address. +_NAT64 = ipaddress.ip_network("64:ff9b::/96") + + def _public_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: mapped = getattr(ip, "ipv4_mapped", None) if mapped is not None: ip = mapped + if ip.version == 6 and ip in _NAT64: + ip = ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF) + if any(ip.version == net.version and ip in net for net in _SPECIAL_NETWORKS): + return False if ip.is_multicast or ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified: return False return bool(ip.is_global) @@ -103,16 +124,12 @@ def trusted_origin(url: str) -> tuple[str, str, int]: raise Rejected("invalid_target", "target must be HTTPS or loopback HTTP without URL credentials") -def guard_request(url: str) -> None: - """Reject an HTTPS target that resolves to a non-public address. +def _accepted_addresses(host: str, port: int) -> list[tuple]: + """Resolve ``host`` once and return its addresses, refusing the lot if any of them is not public. - Loopback HTTP is not resolved. A name the guard cannot resolve is refused, not allowed: it raises - ``urllib.error.URLError`` (no request is made), so each caller keeps the code it already gives an - unreachable destination (``planner_unavailable``, ``source_unavailable``, ``destination_unreachable``). + A name that does not resolve raises ``urllib.error.URLError`` (callers report their unreachable code); a + name with any non-public address raises ``Rejected("invalid_target")``. """ - scheme, host, port = trusted_origin(url) - if scheme != "https": - return try: answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) except (socket.gaierror, UnicodeError, OSError): @@ -127,6 +144,66 @@ def guard_request(url: str) -> None: raise Rejected("invalid_target", "target host resolves to a non-public address") from None if not _public_ip(ip): raise Rejected("invalid_target", "target host resolves to a non-public address") + return answers + + +def guard_request(url: str) -> None: + """Reject an HTTPS target that resolves to a non-public address. + + Loopback HTTP is not resolved. A name the guard cannot resolve is refused, not allowed: it raises + ``urllib.error.URLError`` (no request is made), so each caller keeps the code it already gives an + unreachable destination (``planner_unavailable``, ``source_unavailable``, ``destination_unreachable``). + This early check gives callers their codes; the connection itself is pinned by ``credentialed_opener``, + which resolves and checks again and dials only the addresses it accepted. + """ + scheme, host, port = trusted_origin(url) + if scheme != "https": + return + _accepted_addresses(host, port) + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """An HTTPS connection that dials only addresses the guard accepts. + + ``urllib`` would otherwise resolve the name a second time when it connects, so a name whose answer changes + between the guard's lookup and the connection (DNS rebinding) could reach a private address. Here the + lookup and the check happen in ``connect`` and the socket goes to one of the addresses just accepted. The + original name is still used for SNI, certificate verification and the Host header. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._create_connection = self._pinned_connection + + def _pinned_connection(self, address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): + host, port = address + try: + answers = _accepted_addresses(host, port) + except Rejected as refused: + raise OSError(f"refused target address: {refused.detail}") from None + except urllib.error.URLError as unresolved: + raise OSError(str(unresolved.reason)) from None + last = None + for family, kind, proto, _name, sockaddr in answers: + sock = None + try: + sock = socket.socket(family, kind, proto) + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sockaddr) + return sock + except OSError as error: + last = error + if sock is not None: + sock.close() + raise last if last is not None else OSError("no address to connect to") + + +class _PinnedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open(_PinnedHTTPSConnection, req, context=self._context) def credentialed_opener(*handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector: @@ -136,5 +213,17 @@ def credentialed_opener(*handlers: urllib.request.BaseHandler) -> urllib.request opener is built. Through a proxy the guard's address check is moot (the proxy resolves and dials) and the bearer token goes to the proxy, including on loopback HTTP, which the guard does not resolve. The empty ``ProxyHandler`` disables that; a deployment that needs egress through a proxy must use a transparent one. + HTTPS connections are pinned: each resolves and checks its host and dials only the accepted addresses. + """ + return urllib.request.build_opener(urllib.request.ProxyHandler({}), _PinnedHTTPSHandler(), *handlers) + + +def header_value_ok(value: str) -> bool: + """A value safe to place in an HTTP header: printable, no CR or LF, no leading or trailing whitespace. + + ``http.client`` refuses a bare line break but accepts an obsolete fold (a break followed by a space or + tab), which a later hop may read as a second header. Anything outside visible ASCII and inner spaces + is refused rather than encoded. """ - return urllib.request.build_opener(urllib.request.ProxyHandler({}), *handlers) + return (isinstance(value, str) and value == value.strip(" \t") and value != "" + and all(" " <= ch <= "~" for ch in value)) diff --git a/syberwork/core.py b/syberwork/core.py index ea00d7e3..633588ec 100644 --- a/syberwork/core.py +++ b/syberwork/core.py @@ -31,7 +31,7 @@ from syberlabs.jcs import envelope_jcs from syberlabs import metrics as work_metrics from syberlabs.planner import HttpPlanner, planning_context -from syberlabs.targets import credentialed_opener, guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, header_value_ok, trusted_origin from syberlabs.values import at_path from syberwork.storage import ( ActionRegistry, @@ -351,8 +351,21 @@ def resolve_resolution(self, case_id: str, task_id: str, actor: str, roles: list identity = at_path(result["body"]["value"], spec["result"]["identity_path"]) confirmed = (not confirmation or at_path(confirmation["body"]["value"], spec["confirmation"]["value_path"]) == selected) - changed = (spec["trigger"]["key"] != spec["result"]["key"] or - result["body"]["version"] != task["trigger_version"]) + if spec["trigger"]["key"] == spec["result"]["key"]: + changed = result["body"]["version"] != task["trigger_version"] + else: + # A different result key is not itself a change. Either the confirmation re-reads the trigger and + # finds the decision at the path that was empty when the task was requested, or the result's own + # version moved from the one read before the request. Otherwise the task stays pending. + trigger, check = spec["trigger"], spec.get("confirmation") or {} + decided_at_trigger = (confirmation is not None and check.get("key") == trigger["key"] + and check.get("source") == trigger["source"] + and check.get("value_path") == trigger["missing_path"]) + requested_at = next(e["seq"] for e in history if e["kind"] == "resolution_requested" + and e["body"]["id"] == task_id) + baseline = self._result_baseline([e for e in history if e["seq"] < requested_at], spec) + changed = decided_at_trigger or (baseline is not None + and result["body"]["version"] != baseline["body"]["version"]) if (selected not in task["choices"] or identity != task["record_key"] or not changed or not confirmed): self._append(db, case_id, "resolution_checked", { @@ -368,6 +381,13 @@ def resolve_resolution(self, case_id: str, task_id: str, actor: str, roles: list }) return {"status": "completed", "choice": selected, "event": event} + @staticmethod + def _result_baseline(history: list, spec: dict) -> dict | None: + """The newest verified observation of a resolution's result key from its result source, if any.""" + result = spec["result"] + return next((e for e in reversed(history) if e["kind"] == "observed" and e["body"]["key"] == result["key"] + and e["body"]["source"] == result["source"] and e["body"].get("verified")), None) + def escalate_resolution(self, case_id: str, task_id: str, actor: str, roles: list[str]) -> dict: with self.tx() as db: history = self._events(db, case_id) @@ -463,8 +483,8 @@ def _attestable(contract: dict, key: str, source: str) -> bool: Admission (``facts.required``, ``resolution.current``) and argument binding read the newest observation of a key. An unverified attestation of a key the contract relies on would shadow the verified fact, so a case could be stalled, or its compiled arguments chosen, by anyone holding any attesting source. A key - the contract names (a required fact, an argument binding, a resolution's evidence) is attestable only - where a required fact names this very source and does not ask for verification. A key the contract never + the contract names (a required fact, an argument binding, a fact acceptance clause, a resolution's evidence) + is attestable only where a required fact names this very source and does not ask for verification. A key the contract never reads stays attestable, as before: it is recorded unverified and decides nothing. """ named, allowed = set(), False @@ -482,6 +502,9 @@ def _attestable(contract: dict, key: str, source: str) -> bool: if (isinstance(binding, str) and binding.removeprefix("version:").removeprefix("fact:").split(".")[0] == key): named.add(key) + for clause in contract.get("acceptance") or []: + if isinstance(clause, dict) and clause.get("kind") == "fact" and clause.get("key") == key: + named.add(key) # acceptance reads the newest observation of this key for resolution in (contract.get("resolutions") or {}).values(): if isinstance(resolution, dict) and any( isinstance(part, dict) and part.get("key") == key for part in resolution.values()): @@ -625,6 +648,11 @@ def commit(self, case_id: str, proposal_id: str, actor: str) -> dict: self._append(db, case_id, "decision", {"proposal_id": proposal_id, "policy_version": policy["version"], "phase": "commit", **decision}, rule=full["rule"]) return {"decision": decision} action = json.loads(ActionRegistry(db).get(proposal["action"])["body"]) + version_arg = action.get("version_arg") + if (action["kind"] != "local" and version_arg in proposal["args"] + and not header_value_ok(str(proposal["args"][version_arg]))): + # Sent as If-Match. Refused before the claim, so no effect is started for a value that cannot be sent. + raise Rejected("invalid_argument", f"{version_arg} is not a valid header value") extra = {} if action["kind"] == "economic_http": rule = policy["actions"][proposal["action"]]["economic"] @@ -698,13 +726,22 @@ def lease_obligation(self, *, now: float | None = None) -> dict | None: with self.tx() as db: row = db.execute( "SELECT proposal_id, case_id, state FROM effect_obligations " - "WHERE state='queued' OR (state='leased' AND lease_until IS NOT NULL AND lease_until <= ?) " + "WHERE state='queued' OR (state IN ('leased', 'recovering') AND lease_until IS NOT NULL AND lease_until <= ?) " "ORDER BY proposal_id LIMIT 1", (now,), ).fetchone() if not row: return None - if row["state"] == "leased": + if row["state"] in ("leased", "recovering"): + # Compare-and-swap the expired lease, so only one worker recovers it. The recovery holds a lease + # of its own, so a worker that dies while recovering leaves the obligation to the next one. + taken = db.execute( + "UPDATE effect_obligations SET state='recovering', lease_until=? " + "WHERE proposal_id=? AND state=? AND lease_until IS NOT NULL AND lease_until <= ?", + (now + 30, row["proposal_id"], row["state"], now), + ) + if getattr(taken, "rowcount", 1) != 1: + return None return {"state": "interrupted", "case_id": row["case_id"], "proposal_id": row["proposal_id"]} db.execute( "UPDATE effect_obligations SET state='leased', lease_until=?, attempts=attempts+1 " @@ -729,9 +766,17 @@ def settle_interrupted(self, job: dict) -> dict: case_id, proposal_id = job["case_id"], job["proposal_id"] with self.tx() as db: history = self._events(db, case_id) - if any(event["kind"] in ("effect_succeeded", "effect_rejected", "effect_unknown") and event["body"]["proposal_id"] == proposal_id for event in history): - self._close_obligation(db, proposal_id, "settled") - return {"status": "settled", "proposal_id": proposal_id} + terminal = next((event for event in reversed(history) + if event["kind"] in ("effect_succeeded", "effect_rejected", "effect_unknown") + and event["body"]["proposal_id"] == proposal_id), None) + reconciled = any(verified_reconciliation(event) and event["body"]["proposal_id"] == proposal_id + for event in history) + if terminal is not None or reconciled: + # Report the outcome already stored. An unknown effect stays unknown (still pending, budget held). + outcome = "succeeded" if reconciled else {"effect_succeeded": "succeeded", "effect_rejected": "rejected", + "effect_unknown": "unknown"}[terminal["kind"]] + self._close_obligation(db, proposal_id, {"succeeded": "settled"}.get(outcome, outcome)) + return {"status": outcome, "proposal_id": proposal_id} action = next(event["body"]["action"] for event in history if event["kind"] == "effect_started" and event["body"]["proposal_id"] == proposal_id) self._append(db, case_id, "effect_unknown", {"proposal_id": proposal_id, "action": action, "error": "worker_interrupted"}) self._close_obligation(db, proposal_id, "unknown") @@ -930,7 +975,7 @@ def metric(self, key: str) -> int: if key == "cases_total": return int(db.execute("SELECT COUNT(*) AS n FROM cases").fetchone()["n"]) if key == "queue_depth": - row = db.execute("SELECT COUNT(*) AS n FROM effect_obligations WHERE state IN ('queued', 'leased')").fetchone() + row = db.execute("SELECT COUNT(*) AS n FROM effect_obligations WHERE state IN ('queued', 'leased', 'recovering')").fetchone() return int(row["n"]) if key == "cases_open": count = 0 @@ -1139,6 +1184,8 @@ def _execute(action: dict, args: dict, key: str) -> dict: headers["Authorization"] = f"Bearer {token}" if "version_arg" in action and action["version_arg"] in args: headers["If-Match"] = str(args[action["version_arg"]]) + if not all(header_value_ok(value) for value in headers.values()): + raise Rejected("invalid_header", "a request header value has a line break, control character or edge whitespace") guard_request(action["url"]) req = urllib.request.Request(action["url"], data=canonical(args).encode(), headers=headers, method=action.get("method", "POST")) with HTTP.open(req, timeout=min(30, action.get("timeout_seconds", 10))) as response: diff --git a/syberwork/server.py b/syberwork/server.py index 62509532..0b5474b3 100644 --- a/syberwork/server.py +++ b/syberwork/server.py @@ -31,7 +31,7 @@ # reads no case, contract, certificate or metric: those name every case's inputs, history and amounts. ATTEST_ONLY_ROLES = frozenset({"observer"}) # The first path segments of the reads ``_case_reader`` guards. -CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates")) +CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates"), ("api", "builder")) def _planner_configured() -> bool: @@ -234,7 +234,7 @@ def do_GET(self): return user = self._user() if any(tuple(parts[:len(prefix)]) == prefix for prefix in CASE_READS) and not _case_reader(user.get("roles", [])): - raise Rejected("forbidden", "a role beyond observer is required to read cases, contracts, certificates and metrics") + raise Rejected("forbidden", "a role beyond observer is required to read cases, contracts, certificates, metrics and builder views") if len(parts) in (2, 3) and parts[:2] == ["metrics", "work_items"]: # Per-work-item metrics (docs/METRICS.md): every item, or one thread's, with each number's basis. try: @@ -335,7 +335,9 @@ def _static(self, name, mime): self.send_header("Content-Type", mime + "; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("X-Content-Type-Options", "nosniff") - self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'") + # default-src does not govern framing: frame-ancestors (and X-Frame-Options for older browsers) does. + self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'") + self.send_header("X-Frame-Options", "DENY") self.end_headers() self.wfile.write(body) diff --git a/tests/test_console.py b/tests/test_console.py index aaab07f9..06ec9bac 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -263,7 +263,7 @@ def test_every_new_route_requires_a_bearer_token(self): self.assertEqual((status, body["error"]), (401, "unauthorized")) def test_console_reports_planner_configuration_without_disclosing_it(self): - url = "http://127.0.0.1:9/plan?secret=abc" + url = "http://127.0.0.1:9/plan/secret-abc" previous = os.environ.pop("SYBERWORK_PLANNER_URL", None) self.addCleanup(lambda: os.environ.__setitem__("SYBERWORK_PLANNER_URL", previous) if previous is not None else os.environ.pop("SYBERWORK_PLANNER_URL", None)) status, body = self.cell.json("/api/console") @@ -274,6 +274,8 @@ def test_console_reports_planner_configuration_without_disclosing_it(self): self.assertEqual(json.loads(raw)["planner_configured"], True) self.assertNotIn(b"127.0.0.1:9", raw) self.assertNotIn(b"secret", raw) + os.environ["SYBERWORK_PLANNER_URL"] = "http://127.0.0.1:9/plan?secret=abc" + self.assertFalse(self.cell.json("/api/console")[1]["planner_configured"], "a planner URL takes no query string") os.environ["SYBERWORK_PLANNER_URL"] = "http://planner.example.com/plan" self.assertFalse(self.cell.json("/api/console")[1]["planner_configured"], "plain HTTP off loopback is not a usable planner") diff --git a/tests/test_security_review.py b/tests/test_security_review.py index 19906f07..ed940339 100644 --- a/tests/test_security_review.py +++ b/tests/test_security_review.py @@ -4,7 +4,9 @@ real service: name resolution is patched, and the servers are SQLite cells on free loopback ports. """ +import copy import hashlib +import ipaddress import json import os import socket @@ -29,7 +31,10 @@ from syberwork.core import Work from syberwork.server import UserFile, make_server from syberwork.storage import ReservationStore -from tests.test_cell import _drop_postgres, _install, _postgres_database +from syberlabs.evidence import acceptance_results +from syberlabs.targets import header_value_ok +from syberwork.worker import run_once +from tests.test_cell import NOTE, _drop_postgres, _install, _postgres_database EXAMPLES = Path(__file__).resolve().parents[1] / "syberwork" / "examples" PUBLIC = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))] @@ -212,6 +217,9 @@ def test_an_observer_reads_no_case_contract_certificate_or_metric(self): self.assertEqual(status, 200) status, body = self.call("/api/me", "inv") self.assertEqual((status, body["name"]), (200, "inventory")) + # Builder views keep the builder's own status for a refusal (409), as its writes already did. + status, body = self.call(f"/api/builder/work/{self.case}", "inv") + self.assertEqual((status, body["error"]), (409, "forbidden")) class BackupDoesNotFollowLinks(unittest.TestCase): @@ -402,5 +410,219 @@ def me(token): self.assertEqual(me("two-rotated"), 401) +class SpecialPurposeAddressesAreNotPublic(unittest.TestCase): + """Review F6: ``is_global`` alone let IANA special-purpose addresses through on some interpreters.""" + + def test_special_purpose_ranges_are_refused_whatever_ipaddress_says(self): + from syberlabs.targets import _public_ip + for address in ("192.0.0.9", "192.0.0.10", "2001:1::1", "2001:1::2", "fec0::1", "2001::1", "2002:7f00:1::1", + "64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "198.18.0.1", "100.64.0.1", "::ffff:127.0.0.1"): + with self.subTest(address=address): + self.assertFalse(_public_ip(ipaddress.ip_address(address))) + for address in ("93.184.216.34", "1.1.1.1", "2606:4700::1111", "2001:4860:4860::8888", "64:ff9b::808:808"): + with self.subTest(address=address): + self.assertTrue(_public_ip(ipaddress.ip_address(address))) + + def test_a_name_resolving_to_one_is_refused(self): + answer = [(socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2001:1::1", 443, 0, 0))] + with patch("syberlabs.targets.socket.getaddrinfo", return_value=answer): + with self.assertRaises(Rejected): + guard_request("https://example.com/x") + + +class ConnectionIsPinnedToTheCheckedAddress(unittest.TestCase): + """Review F11: the guard resolved once and urllib resolved again, so a rebinding name could reach loopback.""" + + def test_the_connection_resolves_checks_and_dials_only_the_accepted_address(self): + dialed = [] + + class Recorder: + def __init__(self, *args): + pass + + def settimeout(self, timeout): + pass + + def connect(self, sockaddr): + dialed.append(sockaddr) + raise OSError("stop before TLS") + + def close(self): + pass + + opener = credentialed_opener() + with patch("syberlabs.targets.socket.getaddrinfo", return_value=PUBLIC), \ + patch("syberlabs.targets.socket.socket", Recorder): + with self.assertRaises(urllib.error.URLError): + opener.open(urllib.request.Request("https://example.com/x"), timeout=2) + self.assertEqual(dialed, [("93.184.216.34", 443)]) + + def test_a_name_that_rebinds_to_loopback_after_the_guard_is_never_dialed(self): + loopback = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 443))] + answers = iter([PUBLIC, loopback]) + opener = credentialed_opener() + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=lambda *a, **k: next(answers)), \ + patch("syberlabs.targets.socket.socket", side_effect=AssertionError("no socket for a refused address")): + guard_request("https://example.com/x") + with self.assertRaises(urllib.error.URLError) as refused: + opener.open(urllib.request.Request("https://example.com/x"), timeout=2) + self.assertIn("refused target address", str(refused.exception.reason)) + + +class HeaderValuesAreChecked(unittest.TestCase): + """Review F14: ``version_arg`` was copied into If-Match with no character check.""" + + def test_folded_or_padded_values_are_refused(self): + for value in ("v1\r\n X-Injected: 1", "v1\n\tmore", " v1", "v1 ", "v\x00", "", "vé"): + with self.subTest(value=value): + self.assertFalse(header_value_ok(value)) + self.assertTrue(header_value_ok('"etag-1" W/x')) + + def test_commit_refuses_before_the_claim_and_execute_refuses_before_a_request(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + contract = dict(NOTE, id="versioned") + work.install_contract(contract) + work.install_policy({"version": 1, "actions": {"note": {"roles": ["operator"]}}}) + work.install_action("note", {"kind": "http", "url": "http://127.0.0.1:9/notes", "version_arg": "v"}) + case = work.create_case("versioned", 1, {"name": "a"}, "operator") + proposed = work.propose(case, "note", {"v": "q-1\r\n X-Injected: yes"}, "operator", ["operator"]) + self.assertEqual(proposed["decision"]["status"], "allowed", proposed) + with self.assertRaises(Rejected) as refused: + work.commit(case, proposed["proposal"]["id"], "operator") + self.assertEqual(refused.exception.code, "invalid_argument") + self.assertFalse(any(e["kind"] == "effect_started" for e in work.inspect(case)["events"])) + work.close() + with patch.object(core_module.HTTP, "open", side_effect=AssertionError("no request")): + with self.assertRaises(Rejected) as refused: + Work._execute({"kind": "http", "url": "http://127.0.0.1:9/x", "version_arg": "v"}, {"v": "a\n b"}, "k") + self.assertEqual(refused.exception.code, "invalid_header") + + +class ResolutionNeedsAChangedVersion(unittest.TestCase): + """Review F15: a result read under a key other than the trigger counted as a change by itself.""" + + def setUp(self): + from case_studies.enterprise_procurement import CONTRACT, SimulatedERP, configure + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.erp = SimulatedERP(Path(scratch.name) / "erp.sqlite3") + self.addCleanup(self.erp.close) + self.work = Work(Path(scratch.name) / "w.sqlite3") + self.addCleanup(self.work.close) + configure(self.work, self.erp) + self.contract = CONTRACT + + def _quote_task(self, version): + case = self.work.create_case(self.contract["id"], version, {"request_id": "REQ-4814"}, "analyst") + for source, key in (("requisitions", "request"), ("quote_options", "quote_options")): + self.work.refresh_fact(case, source, key, "REQ-4814", "analyst", ["operator"]) + return case, self.work.request_resolution(case, "quote", "analyst", ["operator"])["body"]["id"] + + def test_without_a_confirmation_or_captured_version_the_result_does_not_complete(self): + unconfirmed = copy.deepcopy(self.contract) + unconfirmed["version"] = 2 + del unconfirmed["resolutions"]["quote"]["confirmation"] + self.work.install_contract(unconfirmed) + case, task = self._quote_task(2) + self.erp.select_quote("procurement", "REQ-4814", "Q-882A") + result = self.work.resolve_resolution(case, task, "p.soto", ["procurement"]) + self.assertEqual((result["status"], result["reason"]), ("pending", "source_decision_unverified")) + + def test_a_confirmation_that_rereads_the_trigger_still_completes(self): + case, task = self._quote_task(1) + self.erp.select_quote("procurement", "REQ-4814", "Q-882A") + self.assertEqual(self.work.resolve_resolution(case, task, "p.soto", ["procurement"])["status"], "completed") + + +class InterruptedEffectsKeepTheirOutcome(unittest.TestCase): + """Review F16: a second worker could relabel a stored ``effect_unknown`` as settled.""" + + def test_one_worker_recovers_an_expired_lease_and_unknown_stays_unknown(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite", effects="worker") + case = _install(work) + proposal = work.propose(case, "note", {}, "operator", ["operator"])["proposal"]["id"] + work.commit(case, proposal, "operator") + with work.tx() as db: + db.execute("UPDATE effect_obligations SET state='leased', lease_until=? WHERE proposal_id=?", (0, proposal)) + first = work.lease_obligation(now=10) + self.assertEqual(first["state"], "interrupted") + self.assertIsNone(work.lease_obligation(now=10), "the expired lease is handed to one worker only") + self.assertEqual(work.settle_interrupted(first)["status"], "unknown") + # A worker that took the same job before the swap existed sees the stored outcome, not "settled". + self.assertEqual(work.settle_interrupted(dict(first))["status"], "unknown") + with work.tx() as db: + state = db.execute("SELECT state FROM effect_obligations WHERE proposal_id=?", (proposal,)).fetchone()["state"] + self.assertEqual(state, "unknown") + self.assertEqual(sum(e["kind"] == "effect_unknown" for e in work.inspect(case)["events"]), 1) + work.close() + + def test_a_recovery_that_died_is_picked_up_again(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite", effects="worker") + case = _install(work) + proposal = work.propose(case, "note", {}, "operator", ["operator"])["proposal"]["id"] + work.commit(case, proposal, "operator") + with work.tx() as db: + db.execute("UPDATE effect_obligations SET state='recovering', lease_until=? WHERE proposal_id=?", (0, proposal)) + self.assertEqual(work.metric("queue_depth"), 1) + self.assertEqual(run_once(work, now=10)["status"], "unknown") + work.close() + + +class FactAcceptanceReadsTheNewestObservation(unittest.TestCase): + """Review F17: a fact clause passed on any historical observation, verified or not.""" + + @staticmethod + def _observed(seq, value, verified, source="inventory"): + return {"seq": seq, "kind": "observed", "body": {"key": "status", "value": value, "verified": verified, "source": source}} + + def test_an_older_match_does_not_pass_and_verified_is_honored(self): + clause = {"id": "ready", "kind": "fact", "key": "status", "equals": "ready"} + contract = {"acceptance": [clause]} + superseded = [self._observed(1, "ready", True), self._observed(2, "held", True)] + self.assertFalse(acceptance_results(contract, superseded)[0]["passed"]) + current = [self._observed(1, "held", True), self._observed(2, "ready", False)] + self.assertTrue(acceptance_results(contract, current)[0]["passed"]) + self.assertFalse(acceptance_results({"acceptance": [dict(clause, verified=True)]}, current)[0]["passed"]) + self.assertFalse(acceptance_results({"acceptance": [dict(clause, source="erp")]}, current)[0]["passed"]) + + def test_an_acceptance_fact_key_is_not_open_to_manual_attestation(self): + contract = {"actions": {}, "acceptance": [{"id": "ready", "kind": "fact", "key": "status", "equals": "ready"}]} + self.assertFalse(Work._attestable(contract, "status", "supplier")) + self.assertTrue(Work._attestable(contract, "note", "supplier")) + + +class ConsoleCannotBeFramed(unittest.TestCase): + """Review F18: the console's policy set default-src, which does not govern framing.""" + + def test_the_console_document_refuses_framing(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + server = make_server(work, {}, port=0) + server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with urllib.request.urlopen(f"http://127.0.0.1:{server.server_port}/", timeout=5) as response: + headers = response.headers + finally: + server.shutdown() + server.server_close() + work.close() + self.assertIn("frame-ancestors 'none'", headers["Content-Security-Policy"]) + self.assertEqual(headers["X-Frame-Options"], "DENY") + + +class PlannerUrlTakesNoQuery(unittest.TestCase): + """Review F3, planner half: a credential could ride in the planner URL's query string.""" + + def test_a_planner_url_with_a_query_is_unconfigured(self): + with patch.dict(os.environ, {"SYBERWORK_PLANNER_URL": "https://planner.example/plan?key=s3cret"}): + with self.assertRaises(planner_module.PlannerRefusal) as refused: + planner_module.HttpPlanner().ensure_configured() + self.assertEqual(refused.exception.code, "planner_unconfigured") + + if __name__ == "__main__": unittest.main()