diff --git a/docker-compose.yml b/docker-compose.yml index 1d87802..41d90a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,14 @@ # One organization, one database. Run `syberwork init` before `serve`. # The home directory is /home/syber/cell, owned by uid 10001, the same path the image creates. +# SYBERWORK_DB_PASSWORD must be set (there is no default password); scripts/compose_cell.sh generates one. +# The service listens on 0.0.0.0 inside its container so Docker can forward to it, and Docker publishes that +# port on the host's loopback only. Widening the mapping is the operator's decision on the host (docs/DEPLOY.md). services: db: image: postgres:16 environment: POSTGRES_USER: syber - POSTGRES_PASSWORD: syber + POSTGRES_PASSWORD: ${SYBERWORK_DB_PASSWORD:?set SYBERWORK_DB_PASSWORD for the cell database} POSTGRES_DB: syberwork volumes: - syberwork-db:/var/lib/postgresql/data @@ -18,13 +21,13 @@ services: build: . environment: &cell SYBERWORK_HOME: /home/syber/cell - SYBERWORK_DATABASE: postgresql://syber:syber@db:5432/syberwork + SYBERWORK_DATABASE: postgresql://syber:${SYBERWORK_DB_PASSWORD:?set SYBERWORK_DB_PASSWORD for the cell database}@db:5432/syberwork SYBERWORK_ORGANIZATION: example SYBERWORK_BIND: 0.0.0.0 SYBERWORK_EFFECTS: worker SYBERWORK_TRACE: "1" ports: - - "8766:8766" + - "127.0.0.1:8766:8766" depends_on: db: condition: service_healthy diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 29a26f3..e4efe80 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -34,6 +34,6 @@ For each past proposal, replay evaluates a selected new contract and policy agai ## Integration -HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call. The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. +HTTP sources and effects are bound by administrator-installed definitions and do not accept an LLM-provided destination URL. Authorization headers use environment variable names in definitions; secrets never enter the contract. Cross-origin redirects are blocked. Source URLs and the planner URL go through `trusted_origin`. HTTPS must name a public host; loopback HTTP stays `127.0.0.1` with a port. `guard_request` checks the addresses HTTPS resolves to before the call and refuses a name it cannot resolve; a host with percent-escapes or other characters `urllib.request` would rewrite is refused before it is classified. The credentialed openers ignore `http_proxy`/`https_proxy` (`targets.credentialed_opener`), so a bearer token is never handed to an environment proxy, and their HTTPS connections are pinned: each resolves and checks its host again and dials only the addresses it accepted, keeping the name for SNI and Host. IANA special-purpose ranges are refused explicitly rather than by `is_global` alone. Header values (`If-Match` from `version_arg`) are refused if they hold a line break, control character or edge whitespace, before the effect is claimed. A manual attestation (`POST /api/cases/{id}/facts`) may not set a key the case's contract reads from another source or only verified, and an `observer`-only credential reads no case, contract, certificate, metric or builder view. The console document refuses framing (`frame-ancestors 'none'`). The signing key for a chain head is not stored in the case database. `python -m syberlabs.witness` holds that seed in a separate process, checks the chain, and appends to the transparency log. `WitnessClient` keeps the public key and opens the log read-only. The log refuses to open a file that already has a `cases` table. A process running as the same OS user can still open the file writable, and an operator who holds the seed can rebuild the log. It is not a public transparency service. Model suggestions are proposals accepted under separate model-bearing credentials, never direct network effects. The included `reference_system.py` is one fully running integration. `translate_stabilize` is a tested translator for one SyberRuntime outcome, not a live adapter. Barn, OmniOS, and customer tools still need adapters matching their authority and event semantics. The other rows in `spec/MAPPINGS.md` stay proposed. The [`economic_http` effect](ECONOMIC_ACTIONS.md) adds exact atomic units and a cross-case budget reservation to this same admission boundary. Its trusted settlement service is a separate authority; SyberWork records that service's matching receipt and status assertion, not independent blockchain finality. diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 8b35b2a..919bd51 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -10,17 +10,18 @@ The first deployment of the factory is the SyberWork service, its worker and its | Service | Image | Role | | --- | --- | --- | -| `db` | `postgres:16` | The cell's database `syberwork`, user `syber`, on volume `syberwork-db`. Health-checked with `pg_isready`. | -| `syberwork` | built from `Dockerfile` | The operator service, `syberwork serve --port 8766`, published on host port 8766. Binds `0.0.0.0` inside the container because `SYBERWORK_BIND` says so; the container port is reachable only through the published port. | +| `db` | `postgres:16` | The cell's database `syberwork`, user `syber`, password `SYBERWORK_DB_PASSWORD` (required; no default), on volume `syberwork-db`. Health-checked with `pg_isready`. | +| `syberwork` | built from `Dockerfile` | The operator service, `syberwork serve --port 8766`, published on the host's loopback, `127.0.0.1:8766`. Binds `0.0.0.0` inside the container because `SYBERWORK_BIND` says so; the container port is reachable only through that published port. | | `worker` | the same image | `syberwork worker`: leases queued effect obligations and settles them. Needed because `SYBERWORK_EFFECTS=worker` makes the service claim effects and return. | -Both processes share `SYBERWORK_HOME=/home/syber/cell` on volume `syberwork-home`, which holds `users.json` (credential hashes), `cell.json` and `traces.jsonl`. The environment (`SYBERWORK_DATABASE`, `SYBERWORK_ORGANIZATION=example`, `SYBERWORK_TRACE=1`) is the one in the file; this document changes nothing in it. +Both processes share `SYBERWORK_HOME=/home/syber/cell` on volume `syberwork-home`, which holds `users.json` (credential hashes), `cell.json` and `traces.jsonl`. The environment (`SYBERWORK_DATABASE`, `SYBERWORK_ORGANIZATION=example`, `SYBERWORK_TRACE=1`) is the one in the file; this document changes nothing in it. `users.json` is read again whenever it changes, so removing or rotating a credential there applies without restarting `serve`. ## 2. Running it on one host The host needs Docker with the Compose plugin and a checkout of this repository. The commands are the ones `scripts/compose_cell.sh` runs, in order, without the smoke steps. ```sh +export SYBERWORK_DB_PASSWORD="" # compose refuses to start without it docker compose build docker compose up -d db --wait docker compose run --rm -T syberwork syberwork init @@ -64,7 +65,7 @@ X-Record-Version: 6f1c...e2 | `effects_pending` | Claimed effects not yet settled: queued, running, or `effect_unknown` awaiting reconciliation. | | `effects_settled_applied_total` | Claimed effects with `effect_succeeded` or a verified `reconciled`. | | `effects_settled_not_applied_total` | Claimed effects with `effect_rejected` (a no-write refusal or `not_applied`). | -| `queue_depth` | Worker obligations queued or leased. Always 0 when effects settle inline. | +| `queue_depth` | Worker obligations queued, leased, or being recovered after an expired lease. Always 0 when effects settle inline. | | `uptime_seconds` | Seconds since this server process started. | Every claimed effect is in exactly one of the three effect counters. `observed_at` is the epoch time of the read, the form the effect's `max_age_seconds` rule understands; the version is a digest of key, value and `observed_at`, so a repeated read of an unchanged value is still a fresh, distinct reading. An unknown key is `404 {"error": "unknown_metric"}`. The route needs a bearer token like every other read on the service; there is no anonymous metrics endpoint. @@ -89,8 +90,8 @@ with `SYBERWORK_METRIC_TOKEN` set in the environment of the process that runs `s ## 4. What is not there yet - **No thread runner service.** Build Threads run from a checkout with `python -m syberlabs`; nothing in the stack starts, schedules or recovers them. That is a later packet. -- **No public exposure and no TLS.** Docker publishes port 8766 on the host's interfaces as `docker-compose.yml` is written; whether the host firewall closes it, or the mapping is narrowed to `127.0.0.1:8766:8766`, is the operator's decision on the host. The service speaks plain HTTP; the intervention effect will only read it over loopback. -- **No secrets broker.** The database password is the literal in `docker-compose.yml`, the credentials are what `init` printed, and the thread's metric token lives in an environment variable. Rotation is `down -v` and a new `init`. +- **No public exposure and no TLS.** Docker publishes port 8766 on the host's loopback only (`127.0.0.1:8766:8766`); widening that mapping, and the firewall that would then guard it, is the operator's decision on the host. The service speaks plain HTTP; the intervention effect will only read it over loopback. +- **No secrets broker.** The database password is whatever `SYBERWORK_DB_PASSWORD` held when the `syberwork-db` volume was first created (a volume created under the old literal `syber` keeps it until `down -v`), the credentials are what `init` printed, and the thread's metric token lives in an environment variable. Rotation is `down -v` and a new `init`. - **No principal management from the command line.** `Work.register_principal` exists for cell principals with a token hash and a delegation; the CLI exposes only the `init` credentials. - **Single host, single worker.** One Compose project, one database, one worker process. `syberwork backup` is the only redundancy. - **Counters are read on request.** `cases_open` reads every case's history on each call; fine for a cell this size, not a design for thousands of cases. diff --git a/scripts/compose_cell.sh b/scripts/compose_cell.sh index 21c0e9f..1123813 100755 --- a/scripts/compose_cell.sh +++ b/scripts/compose_cell.sh @@ -4,6 +4,12 @@ set -euo pipefail cd "$(dirname "$0")/.." +# docker-compose.yml has no default database password. A throwaway cell gets a random one. +if [ -z "${SYBERWORK_DB_PASSWORD:-}" ]; then + SYBERWORK_DB_PASSWORD="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" +fi +export SYBERWORK_DB_PASSWORD + docker compose build docker compose up -d db --wait docker compose run --rm -T syberwork syberwork init diff --git a/spec/GAPS.md b/spec/GAPS.md index fea8bb7..c0a8ac4 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -8,7 +8,7 @@ The schemas describe SyberWork as it behaves. They are not enforced at runtime e 3. **`install_contract` checks the fields admission indexes, including input kinds.** An acceptance `effect` without `action`, a `signoff` without `role`, a `fact` without `key`, or a `required_facts` item without `key` and `source` is rejected at publish. An input kind other than `string` or `integer` is `invalid_contract`. `create_case` rejects a value of the wrong kind with `input_schema`, including a boolean passed where an integer is required. -4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). DNS rebinding is checked at request time by `guard_request`, not at install. +4. **`trusted_origin` is procedural.** Source install and `SYBERWORK_PLANNER_URL` now call it. The public errors stay `invalid_source` and `planner_unconfigured`. A host that only begins with `http://127.0.0.1:` is rejected. HTTPS rejects non-public addresses, `localhost`, metadata hostnames, and the suffixes `.local`, `.localhost`, `.internal`, and `.localdomain`. `SYBERWORK_HTTPS_HOSTS`, when set, is an extra hostname allowlist. The schemas still do not encode the parser (no userinfo, no fragment, path required, HTTP only for `127.0.0.1` with a port). The host must be an IP literal or plain ASCII letter-digit-hyphen labels, so the name checked is the name `urllib.request` dials (a percent-escaped host is refused). DNS rebinding is checked at request time, not at install: `guard_request` refuses a name it cannot resolve, and the credentialed HTTPS connection resolves again and dials only the addresses it accepted. Installed action URLs and the planner URL carry no query, and a source URL carries exactly one `{key}`, in its path. 5. **Proposals are appended before admission.** A non-dict `args` or a missing role list is stored, then denied with `invalid_proposal_shape`. The Proposal schema therefore does not require `args` to be an object. diff --git a/syberlabs/evidence.py b/syberlabs/evidence.py index a34b051..740d1e6 100644 --- a/syberlabs/evidence.py +++ b/syberlabs/evidence.py @@ -125,7 +125,12 @@ def acceptance_results(contract: dict, history: list) -> list[dict]: and e["body"]["action"] == clause.get("after_action")), 0) passed = any(e["kind"] == "signed" and e["body"]["role"] == clause["role"] and (not clause.get("after_action") or e["seq"] > completed_seq > 0) for e in history) elif clause["kind"] == "fact": - passed = any(e["kind"] == "observed" and e["body"]["key"] == clause["key"] and e["body"]["value"] == clause.get("equals") for e in history) + # The newest observation of the key decides, as admission reads facts; an older matching value does + # not. A clause that asks for ``verified`` (or names a ``source``) is held to it. + latest = next((e for e in reversed(history) if e["kind"] == "observed" and e["body"]["key"] == clause["key"]), None) + passed = (latest is not None and latest["body"]["value"] == clause.get("equals") + and (not clause.get("verified") or latest["body"].get("verified") is True) + and ("source" not in clause or latest["body"].get("source") == clause["source"])) else: passed = False results.append({"id": clause["id"], "passed": passed}) diff --git a/syberlabs/outcome.py b/syberlabs/outcome.py index fbc40c7..0777e66 100644 --- a/syberlabs/outcome.py +++ b/syberlabs/outcome.py @@ -139,7 +139,7 @@ def _read(self) -> Reading: headers["Authorization"] = "Bearer " + token try: guard_request(self.url) - except Rejected: + except (Rejected, OSError): # OSError: the guard could not resolve the host (URLError) raise MetricError("metric_unavailable") from None request = urllib.request.Request(self.url, headers=headers, method="GET") try: diff --git a/syberlabs/planner.py b/syberlabs/planner.py index 4cc6483..a95032d 100644 --- a/syberlabs/planner.py +++ b/syberlabs/planner.py @@ -10,7 +10,7 @@ from syberlabs.canonical import canonical from syberlabs.errors import Rejected -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin class PlannerRefusal(Rejected): @@ -50,7 +50,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -_HTTP = urllib.request.build_opener(_NoRedirect) +_HTTP = credentialed_opener(_NoRedirect) class HttpPlanner: @@ -59,6 +59,8 @@ class HttpPlanner: def ensure_configured(self) -> str: target = os.getenv("SYBERWORK_PLANNER_URL", "") try: + if "?" in target: # a planner URL names an endpoint; a credential in its query would be logged and sent + raise Rejected("invalid_target", "planner URL takes no query string") trusted_origin(target) except Rejected: raise PlannerRefusal("planner_unconfigured", "configure an HTTPS or local planner endpoint") from None @@ -66,12 +68,12 @@ def ensure_configured(self) -> str: def propose(self, context: Mapping[str, Any]) -> dict[str, Any]: target = self.ensure_configured() - guard_request(target) headers = {"Content-Type": "application/json"} if os.getenv("SYBERWORK_PLANNER_TOKEN"): headers["Authorization"] = "Bearer " + os.environ["SYBERWORK_PLANNER_TOKEN"] request = urllib.request.Request(target, data=canonical(dict(context)).encode(), headers=headers, method="POST") try: + guard_request(target) # a refused address raises Rejected; an unresolved name, URLError with _HTTP.open(request, timeout=30) as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: diff --git a/syberlabs/publish.py b/syberlabs/publish.py index d4009ba..bd0a1bb 100644 --- a/syberlabs/publish.py +++ b/syberlabs/publish.py @@ -35,9 +35,10 @@ from urllib.parse import quote from syberlabs import critic, outcome +from syberlabs.checks import _stop from syberlabs.errors import Rejected from syberlabs.session import NoWrite -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin EFFECTS = ("git_push", "github_pull_request", "command", "intervention") _REMOTE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") @@ -52,7 +53,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -HTTP = urllib.request.build_opener(_NoRedirect) +HTTP = credentialed_opener(_NoRedirect) def _line(value) -> str: @@ -309,8 +310,9 @@ def _run(self, argv: list[str], case_id: str, args: dict, key: str) -> tuple[int deadline = time.monotonic() + self.timeout while proc.poll() is None: if time.monotonic() > deadline or os.fstat(out.fileno()).st_size > MAX_OUTPUT: - proc.kill() - proc.wait() + # End the command and everything it started, as a check's timeout does: ``proc.kill`` ends + # one process and leaves a deploy script's children running past the publish. + _stop(proc) return None, "" time.sleep(0.02) out.seek(0) diff --git a/syberlabs/settle.py b/syberlabs/settle.py index a8d2c1f..44696ab 100644 --- a/syberlabs/settle.py +++ b/syberlabs/settle.py @@ -148,6 +148,8 @@ def _get(target: Published, token: str, path: str, timeout: float): guard_request(url) except Rejected as exc: raise PollFailed(exc.code) from None + except urllib.error.URLError: # the guard could not resolve the host; no request was made + raise PollFailed("network failure (unresolved)") from None request = urllib.request.Request(url, method="GET", headers={ "Authorization": "Bearer " + token, "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}) diff --git a/syberlabs/targets.py b/syberlabs/targets.py index e2a82af..8c7deb9 100644 --- a/syberlabs/targets.py +++ b/syberlabs/targets.py @@ -2,9 +2,13 @@ from __future__ import annotations +import http.client import ipaddress import os +import re import socket +import urllib.error +import urllib.request from urllib.parse import urlsplit from syberlabs.errors import Rejected @@ -16,12 +20,38 @@ "metadata.google.com", }) _BLOCKED_SUFFIXES = (".local", ".localhost", ".internal", ".localdomain") +# A host name the guard classifies must be the name ``urllib.request`` dials. ``urlsplit`` keeps +# percent-escapes in ``hostname`` while ``Request`` unquotes them, so ``%31%32%37.0.0.1`` was checked as a +# name and dialed as ``127.0.0.1``. Only plain ASCII letter-digit-hyphen labels (IDNA names in their +# ``xn--`` form) and IP literals are accepted; anything else is refused before it is classified. +_LABEL = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" +_HOSTNAME = re.compile(rf"(?:{_LABEL}\.)*{_LABEL}\.?", re.ASCII) + + +# IANA special-purpose ranges (RFC 6890 registries) that are not a destination for a credentialed request. +# ``ipaddress`` flags most of them, but not all, and not the same set on every interpreter: some versions call +# 192.0.0.9, 192.0.0.10, 2001:1::1, 2001:1::2 and the deprecated site-local fec0::/10 global. They are listed here +# so the answer does not depend on the Python release. Anycast service addresses (PCP, TURN) are refused too: +# no installed source, action or planner is one of them. +_SPECIAL_NETWORKS = tuple(ipaddress.ip_network(net) for net in ( + "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", + "192.0.0.0/24", "192.0.2.0/24", "192.31.196.0/24", "192.52.193.0/24", "192.88.99.0/24", "192.168.0.0/16", + "192.175.48.0/24", "198.18.0.0/15", "198.51.100.0/24", "203.0.113.0/24", "224.0.0.0/4", "240.0.0.0/4", + "::/127", "::ffff:0:0/96", "64:ff9b:1::/48", "100::/64", "2001::/23", "2001:db8::/32", "2002::/16", + "2620:4f:8000::/48", "3fff::/20", "5f00::/16", "fc00::/7", "fe80::/10", "fec0::/10", "ff00::/8", +)) +# NAT64 (64:ff9b::/96) carries an IPv4 address in its last 32 bits; the gateway dials that address. +_NAT64 = ipaddress.ip_network("64:ff9b::/96") def _public_ip(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: mapped = getattr(ip, "ipv4_mapped", None) if mapped is not None: ip = mapped + if ip.version == 6 and ip in _NAT64: + ip = ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF) + if any(ip.version == net.version and ip in net for net in _SPECIAL_NETWORKS): + return False if ip.is_multicast or ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified: return False return bool(ip.is_global) @@ -39,6 +69,26 @@ def _https_host_blocked(host: str) -> bool: return False +def _plain_host(host: str) -> bool: + """A host ``urllib.request.Request`` dials exactly as written: an IP literal or plain LDH labels.""" + try: + ipaddress.ip_address(host) + return True + except ValueError: + pass + return len(host) <= 253 and bool(_HOSTNAME.fullmatch(host)) + + +def _dialed_host(url: str) -> str | None: + """The host ``urllib.request`` would connect to for ``url``, without its port and brackets.""" + try: + dialed = urllib.request.Request(url).host + except ValueError: + return None + found = urlsplit("//" + dialed).hostname + return found.lower() if found else None + + def _https_host_listed(host: str) -> bool: """``SYBERWORK_HTTPS_HOSTS`` narrows HTTPS. Unset or blank does not.""" raw = os.getenv("SYBERWORK_HTTPS_HOSTS", "") @@ -51,7 +101,9 @@ def _https_host_listed(host: str) -> bool: def trusted_origin(url: str) -> tuple[str, str, int]: """Reject userinfo, non-loopback HTTP, and non-public HTTPS hosts. - HTTPS names are checked as written. This function does not resolve DNS. + HTTPS names are checked as written, and the name checked must be the name ``urllib.request`` dials: + a host with percent-escapes, other non-LDH characters or non-ASCII is refused. This function does not + resolve DNS. """ try: parsed = urlsplit(url) @@ -59,6 +111,8 @@ def trusted_origin(url: str) -> tuple[str, str, int]: if (not isinstance(url, str) or not parsed.hostname or parsed.username or parsed.password or parsed.fragment or not parsed.path.startswith("/")): raise ValueError("invalid URL") + if not _plain_host(parsed.hostname) or _dialed_host(url) != parsed.hostname: + raise ValueError("host is not dialed as written") if parsed.scheme == "https": if _https_host_blocked(parsed.hostname) or not _https_host_listed(parsed.hostname): raise ValueError("blocked host") @@ -70,19 +124,18 @@ def trusted_origin(url: str) -> tuple[str, str, int]: raise Rejected("invalid_target", "target must be HTTPS or loopback HTTP without URL credentials") -def guard_request(url: str) -> None: - """Reject an HTTPS target that resolves to a non-public address. +def _accepted_addresses(host: str, port: int) -> list[tuple]: + """Resolve ``host`` once and return its addresses, refusing the lot if any of them is not public. - Loopback HTTP is not resolved. A DNS failure is left to the caller, so an - unreachable planner stays ``planner_unavailable``. + A name that does not resolve raises ``urllib.error.URLError`` (callers report their unreachable code); a + name with any non-public address raises ``Rejected("invalid_target")``. """ - scheme, host, port = trusted_origin(url) - if scheme != "https": - return try: answers = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) - except socket.gaierror: - return + except (socket.gaierror, UnicodeError, OSError): + raise urllib.error.URLError("target host did not resolve") from None + if not answers: + raise urllib.error.URLError("target host did not resolve") for info in answers: address = info[4][0].split("%", 1)[0] try: @@ -91,3 +144,86 @@ def guard_request(url: str) -> None: raise Rejected("invalid_target", "target host resolves to a non-public address") from None if not _public_ip(ip): raise Rejected("invalid_target", "target host resolves to a non-public address") + return answers + + +def guard_request(url: str) -> None: + """Reject an HTTPS target that resolves to a non-public address. + + Loopback HTTP is not resolved. A name the guard cannot resolve is refused, not allowed: it raises + ``urllib.error.URLError`` (no request is made), so each caller keeps the code it already gives an + unreachable destination (``planner_unavailable``, ``source_unavailable``, ``destination_unreachable``). + This early check gives callers their codes; the connection itself is pinned by ``credentialed_opener``, + which resolves and checks again and dials only the addresses it accepted. + """ + scheme, host, port = trusted_origin(url) + if scheme != "https": + return + _accepted_addresses(host, port) + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + """An HTTPS connection that dials only addresses the guard accepts. + + ``urllib`` would otherwise resolve the name a second time when it connects, so a name whose answer changes + between the guard's lookup and the connection (DNS rebinding) could reach a private address. Here the + lookup and the check happen in ``connect`` and the socket goes to one of the addresses just accepted. The + original name is still used for SNI, certificate verification and the Host header. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._create_connection = self._pinned_connection + + def _pinned_connection(self, address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None): + host, port = address + try: + answers = _accepted_addresses(host, port) + except Rejected as refused: + raise OSError(f"refused target address: {refused.detail}") from None + except urllib.error.URLError as unresolved: + raise OSError(str(unresolved.reason)) from None + last = None + for family, kind, proto, _name, sockaddr in answers: + sock = None + try: + sock = socket.socket(family, kind, proto) + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sockaddr) + return sock + except OSError as error: + last = error + if sock is not None: + sock.close() + raise last if last is not None else OSError("no address to connect to") + + +class _PinnedHTTPSHandler(urllib.request.HTTPSHandler): + def https_open(self, req): + return self.do_open(_PinnedHTTPSConnection, req, context=self._context) + + +def credentialed_opener(*handlers: urllib.request.BaseHandler) -> urllib.request.OpenerDirector: + """An opener for credential-bearing requests that never routes through an environment proxy. + + ``urllib.request.build_opener`` adds a ``ProxyHandler`` that reads ``http_proxy``/``https_proxy`` when the + opener is built. Through a proxy the guard's address check is moot (the proxy resolves and dials) and the + bearer token goes to the proxy, including on loopback HTTP, which the guard does not resolve. The empty + ``ProxyHandler`` disables that; a deployment that needs egress through a proxy must use a transparent one. + HTTPS connections are pinned: each resolves and checks its host and dials only the accepted addresses. + """ + return urllib.request.build_opener(urllib.request.ProxyHandler({}), _PinnedHTTPSHandler(), *handlers) + + +def header_value_ok(value: str) -> bool: + """A value safe to place in an HTTP header: printable, no CR or LF, no leading or trailing whitespace. + + ``http.client`` refuses a bare line break but accepts an obsolete fold (a break followed by a space or + tab), which a later hop may read as a second header. Anything outside visible ASCII and inner spaces + is refused rather than encoded. + """ + return (isinstance(value, str) and value == value.strip(" \t") and value != "" + and all(" " <= ch <= "~" for ch in value)) diff --git a/syberwork/backup.py b/syberwork/backup.py index b1c34a5..3d45824 100644 --- a/syberwork/backup.py +++ b/syberwork/backup.py @@ -9,6 +9,8 @@ from __future__ import annotations import json +import os +import tempfile from pathlib import Path from syberlabs.errors import Rejected @@ -73,6 +75,29 @@ def _columns(store, table: str) -> list[str]: return [item[0] for item in row.description] +def _write_file(directory: Path, name: str, text: str) -> None: + """Write ``directory/name`` as a new regular file; never through a link planted at that name. + + ``Path.write_text`` follows a symlink, so a ``manifest.json`` linked elsewhere had its target replaced with + the snapshot. A link (or any non-regular file) at the name is refused, and the bytes are written to a fresh + temporary file in the same directory (created exclusively) and moved over the name. + """ + target = directory / name + if target.is_symlink() or (target.exists() and not target.is_file()): + raise Rejected("backup_refused", f"{name} in the backup directory is a link or not a regular file") + handle, temporary = tempfile.mkstemp(prefix="." + name + ".", dir=directory) + try: + with os.fdopen(handle, "w", encoding="utf-8") as out: + out.write(text) + os.replace(temporary, target) + except BaseException: + try: + os.unlink(temporary) + except OSError: + pass + raise + + def export_cell(work, directory: str | Path) -> dict: directory = Path(directory) directory.mkdir(parents=True, exist_ok=True) @@ -88,8 +113,8 @@ def export_cell(work, directory: str | Path) -> dict: rows = [] for record in db.execute(f"SELECT * FROM {table}").fetchall(): rows.append({column: record[column] for column in columns}) - (directory / f"{table}.json").write_text(json.dumps(rows, separators=(",", ":"), default=str), encoding="utf-8") - (directory / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + _write_file(directory, f"{table}.json", json.dumps(rows, separators=(",", ":"), default=str)) + _write_file(directory, "manifest.json", json.dumps(manifest, indent=2) + "\n") return manifest diff --git a/syberwork/cli.py b/syberwork/cli.py index e76d99b..12a2e0c 100644 --- a/syberwork/cli.py +++ b/syberwork/cli.py @@ -15,7 +15,7 @@ from .backup import export_cell, restore_cell from .certificates import CertificateShelf from .core import Work -from .server import serve +from .server import UserFile, serve from .trace import TraceLog from .worker import run_available, serve as serve_worker @@ -155,7 +155,7 @@ def main(): log = _witness_log(parser) client, role = _witness_request(parser, log) shelf = _certificates(parser) - serve(db, json.loads(userfile.read_text()), host=os.getenv("SYBERWORK_BIND", "127.0.0.1"), port=args.port, + serve(db, UserFile(userfile), host=os.getenv("SYBERWORK_BIND") or "127.0.0.1", port=args.port, witness_log=log, witness_client=client, witness_role=role, certificates=shelf) elif args.command == "backup": print(json.dumps(export_cell(db, args.directory))) diff --git a/syberwork/core.py b/syberwork/core.py index 49af68a..633588e 100644 --- a/syberwork/core.py +++ b/syberwork/core.py @@ -31,7 +31,7 @@ from syberlabs.jcs import envelope_jcs from syberlabs import metrics as work_metrics from syberlabs.planner import HttpPlanner, planning_context -from syberlabs.targets import guard_request, trusted_origin +from syberlabs.targets import credentialed_opener, guard_request, header_value_ok, trusted_origin from syberlabs.values import at_path from syberwork.storage import ( ActionRegistry, @@ -61,7 +61,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return None -HTTP = urllib.request.build_opener(NoRedirect) +HTTP = credentialed_opener(NoRedirect) class StatusNotAuthoritative(Exception): @@ -203,6 +203,10 @@ def install_action(self, name: str, doc: dict) -> None: raise Rejected("invalid_action", "action kind must be local, http or economic_http") if doc["kind"] in ("http", "economic_http"): origin = trusted_origin(doc.get("url", "")) + # A definition names credential variables and never holds a secret (``registry``). A query string + # is where a secret would ride, so neither the write URL nor the status URL may carry one. + if urlsplit(doc["url"]).query or "?" in doc["url"] or any(c in doc["url"] for c in "{}"): + raise Rejected("invalid_action", "action URL takes no query string or template") allowed_methods = ("POST",) if doc["kind"] == "economic_http" else ("POST", "PUT", "PATCH") if doc.get("method", "POST") not in allowed_methods: raise Rejected("invalid_action", "HTTP method must write") @@ -219,6 +223,7 @@ def install_action(self, name: str, doc: dict) -> None: if (doc["kind"] not in ("http", "economic_http") or trusted_origin(status_url) != origin or status_url.count("{key}") != 1 or urlsplit(status_url).path.count("{key}") != 1 or + "?" in status_url or any(c in status_url.replace("{key}", "") for c in "{}")): raise Rejected("invalid_action", "status lookup needs one path key on the action origin") no_write = doc.get("no_write_statuses", []) @@ -238,8 +243,12 @@ def install_source(self, name: str, doc: dict) -> None: trusted = True except Rejected: trusted = False - if doc.get("kind") != "http" or not trusted or "{key}" not in url: - raise Rejected("invalid_source", "source needs a fixed HTTPS or local URL with {key}") + # The record key is substituted into the path only. A ``{key}`` in the host would let the case input + # choose which host receives the source's bearer token; a query would store a secret in the definition. + if (doc.get("kind") != "http" or not trusted or url.count("{key}") != 1 or + urlsplit(url).path.count("{key}") != 1 or "?" in url or + any(c in url.replace("{key}", "") for c in "{}")): + raise Rejected("invalid_source", "source needs a fixed HTTPS or local URL with one {key} in its path") with self.tx() as db: sources = SourceRegistry(db) prior = sources.get(name) @@ -342,8 +351,21 @@ def resolve_resolution(self, case_id: str, task_id: str, actor: str, roles: list identity = at_path(result["body"]["value"], spec["result"]["identity_path"]) confirmed = (not confirmation or at_path(confirmation["body"]["value"], spec["confirmation"]["value_path"]) == selected) - changed = (spec["trigger"]["key"] != spec["result"]["key"] or - result["body"]["version"] != task["trigger_version"]) + if spec["trigger"]["key"] == spec["result"]["key"]: + changed = result["body"]["version"] != task["trigger_version"] + else: + # A different result key is not itself a change. Either the confirmation re-reads the trigger and + # finds the decision at the path that was empty when the task was requested, or the result's own + # version moved from the one read before the request. Otherwise the task stays pending. + trigger, check = spec["trigger"], spec.get("confirmation") or {} + decided_at_trigger = (confirmation is not None and check.get("key") == trigger["key"] + and check.get("source") == trigger["source"] + and check.get("value_path") == trigger["missing_path"]) + requested_at = next(e["seq"] for e in history if e["kind"] == "resolution_requested" + and e["body"]["id"] == task_id) + baseline = self._result_baseline([e for e in history if e["seq"] < requested_at], spec) + changed = decided_at_trigger or (baseline is not None + and result["body"]["version"] != baseline["body"]["version"]) if (selected not in task["choices"] or identity != task["record_key"] or not changed or not confirmed): self._append(db, case_id, "resolution_checked", { @@ -359,6 +381,13 @@ def resolve_resolution(self, case_id: str, task_id: str, actor: str, roles: list }) return {"status": "completed", "choice": selected, "event": event} + @staticmethod + def _result_baseline(history: list, spec: dict) -> dict | None: + """The newest verified observation of a resolution's result key from its result source, if any.""" + result = spec["result"] + return next((e for e in reversed(history) if e["kind"] == "observed" and e["body"]["key"] == result["key"] + and e["body"]["source"] == result["source"] and e["body"].get("verified")), None) + def escalate_resolution(self, case_id: str, task_id: str, actor: str, roles: list[str]) -> dict: with self.tx() as db: history = self._events(db, case_id) @@ -447,6 +476,54 @@ def observe(self, case_id: str, key: str, value: Any, source: str, version: str, self._case(db, case_id) return self._append(db, case_id, "observed", {"key": key, "value": value, "source": source, "version": version, "actor": actor, "verified": verified}) + @staticmethod + def _attestable(contract: dict, key: str, source: str) -> bool: + """Whether a manual, unverified attestation of ``key`` from ``source`` may enter this contract's case. + + Admission (``facts.required``, ``resolution.current``) and argument binding read the newest observation + of a key. An unverified attestation of a key the contract relies on would shadow the verified fact, so + a case could be stalled, or its compiled arguments chosen, by anyone holding any attesting source. A key + the contract names (a required fact, an argument binding, a fact acceptance clause, a resolution's evidence) + is attestable only where a required fact names this very source and does not ask for verification. A key the contract never + reads stays attestable, as before: it is recorded unverified and decides nothing. + """ + named, allowed = set(), False + for action in (contract.get("actions") or {}).values(): + if not isinstance(action, dict): + continue + for requirement in action.get("required_facts") or []: + if isinstance(requirement, dict) and requirement.get("key") == key: + named.add(key) + if requirement.get("source") == source and not requirement.get("verified"): + allowed = True + bindings = action.get("arguments") or {} + if isinstance(bindings, dict): + for binding in bindings.values(): + if (isinstance(binding, str) and + binding.removeprefix("version:").removeprefix("fact:").split(".")[0] == key): + named.add(key) + for clause in contract.get("acceptance") or []: + if isinstance(clause, dict) and clause.get("kind") == "fact" and clause.get("key") == key: + named.add(key) # acceptance reads the newest observation of this key + for resolution in (contract.get("resolutions") or {}).values(): + if isinstance(resolution, dict) and any( + isinstance(part, dict) and part.get("key") == key for part in resolution.values()): + return False + return allowed or key not in named + + def attest(self, case_id: str, key: str, value: Any, source: str, version: str, actor: str) -> dict: + """Record a person's or service's unverified attestation (``POST /api/cases/{id}/facts``). + + Refused for a key the case's contract relies on unless that contract accepts this source's unverified + word for it (``_attestable``). ``refresh_fact`` is the route for a verified fact. + """ + with self.tx() as db: + row = self._case(db, case_id) + contract = self._contract(db, row["contract_id"], row["contract_version"]) + if not isinstance(key, str) or not self._attestable(contract, key, source): + raise Rejected("attestation_denied", "this contract reads that key only from its configured source") + return self.observe(case_id, key, value, source, version, actor) + def refresh_fact(self, case_id: str, source: str, key: str, record_key: str, actor: str, roles: list[str]) -> dict: with self.tx() as db: self._case(db, case_id) @@ -466,7 +543,10 @@ def refresh_fact(self, case_id: str, source: str, key: str, record_key: str, act raise Rejected("missing_credential", "source credential unavailable") headers["Authorization"] = "Bearer " + token target = config["url"].replace("{key}", quote(record_key, safe="")) - guard_request(target) + try: + guard_request(target) + except urllib.error.URLError: # the guard could not resolve the host; no request was made + raise Rejected("source_unavailable", "destination_unreachable") from None request = urllib.request.Request(target, headers=headers) try: with HTTP.open(request, timeout=min(30, config.get("timeout_seconds", 10))) as response: @@ -568,6 +648,11 @@ def commit(self, case_id: str, proposal_id: str, actor: str) -> dict: self._append(db, case_id, "decision", {"proposal_id": proposal_id, "policy_version": policy["version"], "phase": "commit", **decision}, rule=full["rule"]) return {"decision": decision} action = json.loads(ActionRegistry(db).get(proposal["action"])["body"]) + version_arg = action.get("version_arg") + if (action["kind"] != "local" and version_arg in proposal["args"] + and not header_value_ok(str(proposal["args"][version_arg]))): + # Sent as If-Match. Refused before the claim, so no effect is started for a value that cannot be sent. + raise Rejected("invalid_argument", f"{version_arg} is not a valid header value") extra = {} if action["kind"] == "economic_http": rule = policy["actions"][proposal["action"]]["economic"] @@ -641,13 +726,22 @@ def lease_obligation(self, *, now: float | None = None) -> dict | None: with self.tx() as db: row = db.execute( "SELECT proposal_id, case_id, state FROM effect_obligations " - "WHERE state='queued' OR (state='leased' AND lease_until IS NOT NULL AND lease_until <= ?) " + "WHERE state='queued' OR (state IN ('leased', 'recovering') AND lease_until IS NOT NULL AND lease_until <= ?) " "ORDER BY proposal_id LIMIT 1", (now,), ).fetchone() if not row: return None - if row["state"] == "leased": + if row["state"] in ("leased", "recovering"): + # Compare-and-swap the expired lease, so only one worker recovers it. The recovery holds a lease + # of its own, so a worker that dies while recovering leaves the obligation to the next one. + taken = db.execute( + "UPDATE effect_obligations SET state='recovering', lease_until=? " + "WHERE proposal_id=? AND state=? AND lease_until IS NOT NULL AND lease_until <= ?", + (now + 30, row["proposal_id"], row["state"], now), + ) + if getattr(taken, "rowcount", 1) != 1: + return None return {"state": "interrupted", "case_id": row["case_id"], "proposal_id": row["proposal_id"]} db.execute( "UPDATE effect_obligations SET state='leased', lease_until=?, attempts=attempts+1 " @@ -672,9 +766,17 @@ def settle_interrupted(self, job: dict) -> dict: case_id, proposal_id = job["case_id"], job["proposal_id"] with self.tx() as db: history = self._events(db, case_id) - if any(event["kind"] in ("effect_succeeded", "effect_rejected", "effect_unknown") and event["body"]["proposal_id"] == proposal_id for event in history): - self._close_obligation(db, proposal_id, "settled") - return {"status": "settled", "proposal_id": proposal_id} + terminal = next((event for event in reversed(history) + if event["kind"] in ("effect_succeeded", "effect_rejected", "effect_unknown") + and event["body"]["proposal_id"] == proposal_id), None) + reconciled = any(verified_reconciliation(event) and event["body"]["proposal_id"] == proposal_id + for event in history) + if terminal is not None or reconciled: + # Report the outcome already stored. An unknown effect stays unknown (still pending, budget held). + outcome = "succeeded" if reconciled else {"effect_succeeded": "succeeded", "effect_rejected": "rejected", + "effect_unknown": "unknown"}[terminal["kind"]] + self._close_obligation(db, proposal_id, {"succeeded": "settled"}.get(outcome, outcome)) + return {"status": outcome, "proposal_id": proposal_id} action = next(event["body"]["action"] for event in history if event["kind"] == "effect_started" and event["body"]["proposal_id"] == proposal_id) self._append(db, case_id, "effect_unknown", {"proposal_id": proposal_id, "action": action, "error": "worker_interrupted"}) self._close_obligation(db, proposal_id, "unknown") @@ -715,9 +817,15 @@ def reconcile(self, case_id: str, proposal_id: str, actor: str, roles: list[str] raise Rejected("missing_credential", "destination status credential unavailable") headers["Authorization"] = "Bearer " + token target = status_url.replace("{key}", quote(proposal_id, safe="")) - guard_request(target) + try: + guard_request(target) + unresolved = None + except urllib.error.URLError as exc: # the guard could not resolve the host: the lookup is unavailable + unresolved = exc request = urllib.request.Request(target, headers=headers) try: + if unresolved is not None: + raise unresolved with HTTP.open(request, timeout=min(30, config.get("timeout_seconds", 10))) as response: if getattr(response, "status", None) != 200: raise StatusNotAuthoritative(getattr(response, "status", None)) @@ -867,7 +975,7 @@ def metric(self, key: str) -> int: if key == "cases_total": return int(db.execute("SELECT COUNT(*) AS n FROM cases").fetchone()["n"]) if key == "queue_depth": - row = db.execute("SELECT COUNT(*) AS n FROM effect_obligations WHERE state IN ('queued', 'leased')").fetchone() + row = db.execute("SELECT COUNT(*) AS n FROM effect_obligations WHERE state IN ('queued', 'leased', 'recovering')").fetchone() return int(row["n"]) if key == "cases_open": count = 0 @@ -1076,6 +1184,8 @@ def _execute(action: dict, args: dict, key: str) -> dict: headers["Authorization"] = f"Bearer {token}" if "version_arg" in action and action["version_arg"] in args: headers["If-Match"] = str(args[action["version_arg"]]) + if not all(header_value_ok(value) for value in headers.values()): + raise Rejected("invalid_header", "a request header value has a line break, control character or edge whitespace") guard_request(action["url"]) req = urllib.request.Request(action["url"], data=canonical(args).encode(), headers=headers, method=action.get("method", "POST")) with HTTP.open(req, timeout=min(30, action.get("timeout_seconds", 10))) as response: diff --git a/syberwork/server.py b/syberwork/server.py index 0012151..0b5474b 100644 --- a/syberwork/server.py +++ b/syberwork/server.py @@ -26,6 +26,12 @@ # reader is an administrator or the approver role, and not an automation credential (model, compiled, search). # There is no separate policy-read role; model.js mirrors this to suggest an identity that can read them. POLICY_READERS = frozenset({"admin", "manager"}) +# Roles that only attest facts (``syberwork init`` gives the inventory and supplier credentials ``observer``). +# A principal holding nothing else may post its own source's observations and read its own profile, but it +# reads no case, contract, certificate or metric: those name every case's inputs, history and amounts. +ATTEST_ONLY_ROLES = frozenset({"observer"}) +# The first path segments of the reads ``_case_reader`` guards. +CASE_READS = (("metrics",), ("api", "cases"), ("api", "contracts"), ("api", "certificates"), ("api", "builder")) def _planner_configured() -> bool: @@ -41,6 +47,11 @@ def _policy_reader(roles) -> bool: return bool(POLICY_READERS.intersection(roles)) and not AUTOMATION_ROLES.intersection(roles) +def _case_reader(roles) -> bool: + """Whether a principal may read cases, contracts, certificates and metrics: any role beyond attesting.""" + return isinstance(roles, list) and bool({r for r in roles if isinstance(r, str)} - ATTEST_ONLY_ROLES) + + def _source_roles(work: Work) -> dict: """Each installed source's read roles, as refresh_fact checks them, and nothing else of its definition. @@ -106,7 +117,37 @@ def _preview(work: Work, case_id: str, data: dict, name: str, roles: list) -> di "written": False, "note": "Admission now for a proposal not yet made; nothing was recorded and nothing is reserved"} -def serve(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, +class UserFile: + """``users.json`` read again whenever it changes, so a removed or rotated credential stops working without a + restart. Read on each request by ``stat``; parsed only when its size or modification time moves. A file that + is missing, unreadable or not a JSON object of users answers with no users: every bearer is refused (cell + principals registered in the database still authenticate).""" + + def __init__(self, path: str | Path): + self.path = Path(path) + self._stamp = None + self._users: dict = {} + + def __call__(self) -> dict: + try: + status = self.path.stat() + stamp = (status.st_mtime_ns, status.st_size, status.st_ino) + except OSError: + self._stamp, self._users = None, {} + return {} + if stamp != self._stamp: + try: + loaded = json.loads(self.path.read_text(encoding="utf-8")) + valid = isinstance(loaded, dict) and all( + isinstance(info, dict) and isinstance(info.get("hash"), str) for info in loaded.values()) + except (OSError, ValueError): + loaded, valid = {}, False + self._users = loaded if valid else {} + self._stamp = stamp + return self._users + + +def serve(work: Work, users, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, witness_role=None, certificates=None): httpd = make_server(work, users, host, port, witness_log=witness_log, witness_client=witness_client, witness_role=witness_role, certificates=certificates) @@ -114,10 +155,13 @@ def serve(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, httpd.serve_forever() -def make_server(work: Work, users: dict, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, +def make_server(work: Work, users, host: str = "127.0.0.1", port: int = 8766, *, witness_log=None, witness_client=None, witness_role=None, certificates=None) -> ThreadingHTTPServer: """Build the loopback server without starting it. Port 0 picks a free port. + ``users`` is a mapping of name to ``{hash, roles, sources}``, or a callable returning one on each request + (``UserFile``, which ``syberwork serve`` passes, so an edit to ``users.json`` applies without a restart). + ``witness_log`` is a ``TransparencyLog.open_readonly`` reader of the witness's log, or None. It holds no seed and cannot append; ``GET /api/cases/{id}/witness`` reads it. ``witness_client`` is a ``WitnessClient`` on the witness's loopback port and ``witness_role`` the one role whose holders ``POST /api/cases/{id}/witness`` lets ask it to sign: @@ -129,7 +173,9 @@ def make_server(work: Work, users: dict, host: str = "127.0.0.1", port: int = 87 SYBERWORK_CERTIFICATE_HOME and SYBERWORK_CERTIFICATE_PUBLIC. """ loopback = host in ("127.0.0.1", "::1", "localhost") - if not loopback and os.getenv("SYBERWORK_BIND", "") != host: + # An empty host binds every interface. SYBERWORK_BIND names a non-loopback address on purpose, so an empty or + # unset value never vouches for one, even when the host is empty too. + if not loopback and (not host or not isinstance(host, str) or os.getenv("SYBERWORK_BIND", "") != host): raise Rejected("unsafe_bind", "the operator service must bind to loopback") if (witness_client is None) != (witness_role is None) or (witness_client is not None and witness_log is None): raise Rejected("invalid_runtime", "a witness client needs a role and the witness log, and a role needs a client") @@ -144,7 +190,8 @@ class Handler(BaseHTTPRequestHandler): def _user(self): token = self.headers.get("Authorization", "").removeprefix("Bearer ") hashed = hashlib.sha256(token.encode()).hexdigest() - for name, info in users.items(): + current = users() if callable(users) else users + for name, info in current.items(): if token and hmac.compare_digest(info["hash"], hashed): return {"name": name, **info} principal = work.principal_for_token(token) @@ -186,6 +233,8 @@ def do_GET(self): self._static(parts[0], ASSETS[parts[0]]) return user = self._user() + if any(tuple(parts[:len(prefix)]) == prefix for prefix in CASE_READS) and not _case_reader(user.get("roles", [])): + raise Rejected("forbidden", "a role beyond observer is required to read cases, contracts, certificates, metrics and builder views") if len(parts) in (2, 3) and parts[:2] == ["metrics", "work_items"]: # Per-work-item metrics (docs/METRICS.md): every item, or one thread's, with each number's basis. try: @@ -286,7 +335,9 @@ def _static(self, name, mime): self.send_header("Content-Type", mime + "; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.send_header("X-Content-Type-Options", "nosniff") - self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'") + # default-src does not govern framing: frame-ancestors (and X-Frame-Options for older browsers) does. + self.send_header("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'") + self.send_header("X-Frame-Options", "DENY") self.end_headers() self.wfile.write(body) @@ -325,7 +376,7 @@ def do_POST(self): if operation == "facts": if data["source"] not in user.get("sources", []): raise Rejected("source_denied", "actor cannot attest this source") - result = work.observe(case_id, data["key"], data["value"], data["source"], data["version"], actor) + result = work.attest(case_id, data["key"], data["value"], data["source"], data["version"], actor) elif operation == "refresh": result = work.refresh_fact(case_id, data["source"], data["key"], data["record_key"], actor, roles) elif operation == "resolution-request": diff --git a/syberwork/storage.py b/syberwork/storage.py index 253c572..759a549 100644 --- a/syberwork/storage.py +++ b/syberwork/storage.py @@ -27,7 +27,7 @@ def _statements(sql: str) -> list[str]: return [part.strip() for part in sql.split(";") if part.strip()] -_MIGRATIONS: list[tuple[str, str]] = [ +_MIGRATIONS: list[tuple[str, str | dict[str, str]]] = [ ("0001", f""" CREATE TABLE IF NOT EXISTS cell_lock ( id INTEGER PRIMARY KEY, @@ -273,6 +273,13 @@ def _statements(sql: str) -> list[str]: UNIQUE (evaluation_event_hash) ); """), + # ``units`` admits amounts up to 2**63-1, but 0001 created ``amount_units`` as INTEGER, which PostgreSQL + # stores as int4: a reservation above 2147483647 aborted the commit before the adapter ran. SQLite's + # INTEGER is already 64-bit, so only PostgreSQL changes. A dialect-keyed script runs only on its dialect. + ("0006", { + "sqlite": "", + "postgres": "ALTER TABLE economic_reservations ALTER COLUMN amount_units TYPE BIGINT", + }), ] @@ -409,6 +416,8 @@ def migrate(store) -> list[str]: for version, script in _MIGRATIONS: if version in applied: continue + if isinstance(script, dict): + script = script.get(store.dialect, "") for statement in _statements(script): store.execute(statement) store.execute( diff --git a/tests/console_browser.mjs b/tests/console_browser.mjs index 42f6790..9aeeee5 100644 --- a/tests/console_browser.mjs +++ b/tests/console_browser.mjs @@ -258,26 +258,29 @@ await page.waitForSelector('.replay table.diff'); report.replay = await page.locator('.replay table.diff').first().textContent(); await shot('14-replay'); -// A typed observation and an Attest receipt on a second case stay unverified. +// A typed observation and an Attest receipt on a second case stay unverified. The supplier holds only the +// observer role: it may attest keys the contract does not read from another source, is refused one it does +// (quote must be a verified supplier read), and cannot read the case, so the operator inspects the result. await switchTo('operator'); await page.keyboard.press('n'); await page.waitForSelector('#newcase-form input[name="in:part_number"]'); await page.click('#newcase-form button.primary'); await page.waitForSelector('.case-head'); -await switchTo('supplier'); +const secondCase = decodeURIComponent(page.url().split('#/case/')[1].split('/')[0]); +const attest = (key, version, value) => page.evaluate(async ([id, token, body]) => { + const response = await fetch('/api/cases/' + encodeURIComponent(id) + '/facts', {method: 'POST', cache: 'no-store', + headers: {Authorization: 'Bearer ' + token, 'Content-Type': 'application/json'}, body: JSON.stringify(body)}); + return {status: response.status, body: await response.json().catch(() => null)}; +}, [secondCase, tokens.supplier, {key, source: 'supplier', version, value}]); +report.quote_attest = await attest('quote', 'typed-1', {id: 'Q-7', price: 1, part_number: 'P-104', supplier: 'typed'}); +report.note_attest = await attest('supplier_note', 'typed-1', {text: 'typed by the supplier'}); +report.receipt_attest = await attest('attest_receipt', 'receipt-1', {schema: 'attest.receipt/v0', verified: false, evidence: {after: {status: 'proven'}}}); +report.supplier_case_read = await page.evaluate(async ([id, token]) => + (await fetch('/api/cases/' + encodeURIComponent(id), {headers: {Authorization: 'Bearer ' + token}, cache: 'no-store'})).status, + [secondCase, tokens.supplier]); +// The operator is still on the second case; r reads it again with the two attestations. await page.keyboard.press('2'); -await page.locator('.evidence details.warn-panel summary').click(); -const manual = page.locator('.evidence details.warn-panel'); -await manual.locator('input[name="key"]').fill('quote'); -await manual.locator('input[name="source"]').fill('supplier'); -await manual.locator('input[name="version"]').fill('typed-1'); -await manual.locator('textarea[name="value"]').fill('{"id":"Q-7","price":1,"part_number":"P-104","supplier":"typed"}'); -await manual.locator('button.btn').click(); -await page.waitForFunction(() => document.querySelectorAll('.obs tbody tr').length === 1); -await manual.locator('input[name="key"]').fill('attest_receipt'); -await manual.locator('input[name="version"]').fill('receipt-1'); -await manual.locator('textarea[name="value"]').fill('{"schema":"attest.receipt/v0","verified":false,"evidence":{"after":{"status":"proven"}}}'); -await manual.locator('button.btn').click(); +await page.keyboard.press('r'); await page.waitForFunction(() => document.querySelectorAll('.obs tbody tr').length === 2); report.manual = await page.locator('.obs tbody tr').allTextContents(); await shot('15-unverified'); @@ -286,6 +289,7 @@ await page.waitForSelector('.effects .notice.warn'); report.attest_notice = await page.textContent('.effects .notice.warn'); // The supplier holds no source read role; in the Studio it sees the read roles, not the definitions. +await switchTo('supplier'); await page.click('[data-view="studio"]'); await page.locator('.studio-item', { hasText: 'Sources' }).click(); await page.waitForSelector('.studio-body table.grid'); diff --git a/tests/test_cell.py b/tests/test_cell.py index f8cae01..e22c55a 100644 --- a/tests/test_cell.py +++ b/tests/test_cell.py @@ -80,7 +80,7 @@ def test_inline_commit_still_settles_and_a_legacy_database_migrates(self): self.assertEqual(committed["status"], "succeeded") self.assertTrue(work.verify_chain(case_id)) versions = [row["version"] for row in work._db.execute("SELECT version FROM schema_migrations ORDER BY version")] - self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005"]) + self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005", "0006"]) work.close() copy = Path(folder) / "main.sqlite" shutil.copy(FIXTURE, copy) @@ -201,7 +201,7 @@ def open_db(): self.assertEqual(errors, []) check = Work(path) versions = [row["version"] for row in check._db.execute("SELECT version FROM schema_migrations ORDER BY version")] - self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005"]) + self.assertEqual(versions, ["0001", "0002", "0003", "0004", "0005", "0006"]) check.close() def test_failed_restore_does_not_keep_the_rows(self): diff --git a/tests/test_console.py b/tests/test_console.py index 274023c..06ec9ba 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -263,7 +263,7 @@ def test_every_new_route_requires_a_bearer_token(self): self.assertEqual((status, body["error"]), (401, "unauthorized")) def test_console_reports_planner_configuration_without_disclosing_it(self): - url = "http://127.0.0.1:9/plan?secret=abc" + url = "http://127.0.0.1:9/plan/secret-abc" previous = os.environ.pop("SYBERWORK_PLANNER_URL", None) self.addCleanup(lambda: os.environ.__setitem__("SYBERWORK_PLANNER_URL", previous) if previous is not None else os.environ.pop("SYBERWORK_PLANNER_URL", None)) status, body = self.cell.json("/api/console") @@ -274,6 +274,8 @@ def test_console_reports_planner_configuration_without_disclosing_it(self): self.assertEqual(json.loads(raw)["planner_configured"], True) self.assertNotIn(b"127.0.0.1:9", raw) self.assertNotIn(b"secret", raw) + os.environ["SYBERWORK_PLANNER_URL"] = "http://127.0.0.1:9/plan?secret=abc" + self.assertFalse(self.cell.json("/api/console")[1]["planner_configured"], "a planner URL takes no query string") os.environ["SYBERWORK_PLANNER_URL"] = "http://planner.example.com/plan" self.assertFalse(self.cell.json("/api/console")[1]["planner_configured"], "plain HTTP off loopback is not a usable planner") @@ -413,7 +415,9 @@ def test_listed_read_roles_are_the_ones_refresh_enforces(self): def test_work_item_summary_carries_what_the_top_bar_reads(self): case = self.cell.json("/api/cases", {"contract_id": "purchase-order", "version": 1, "inputs": {"part_number": "P-104", "quantity": 2}})[1]["id"] - for who in ("operator", "manager", "supplier"): + # An observer-only credential (supplier) attests facts and reads no metric (security review, finding 4). + self.assertEqual(self.cell.json("/metrics/work_items", who="supplier")[0], 403) + for who in ("operator", "manager"): with self.subTest(who): status, body = self.cell.json("/metrics/work_items", who=who) self.assertEqual(status, 200) @@ -570,7 +574,9 @@ def test_side_records_are_work_side_channel_for_any_principal(self): case = self.case_with_decision() events = self.cell.work.inspect(case)["events"] expected = self.cell.work.side_channel(case) - for who in ("operator", "manager", "supplier", "planner"): + # Any principal beyond an observer-only credential (security review, finding 4). + self.assertEqual(self.cell.json(f"/api/cases/{case}/side", who="supplier")[0], 403) + for who in ("operator", "manager", "planner"): with self.subTest(who): self.assertEqual(self.cell.json(f"/api/cases/{case}/side", who=who), (200, expected)) self.assertEqual([row["seq"] for row in expected], [e["seq"] for e in events], "every new event has its row") @@ -662,7 +668,8 @@ def test_a_signed_head_is_witnessed_through_its_count_and_later_events_are_not(s self.attest(other_events) self.assertEqual(self.cell.json(f"/api/cases/{case}/witness")[1]["entries"], [], "another case's leaf is not this case's") self.attest(events) - for who in ("operator", "manager", "supplier", "planner"): + self.assertEqual(self.cell.json(f"/api/cases/{case}/witness", who="supplier")[0], 403) # observer-only + for who in ("operator", "manager", "planner"): with self.subTest(who): status, body = self.cell.json(f"/api/cases/{case}/witness", who=who) self.assertEqual(status, 200) @@ -997,7 +1004,9 @@ def journal(self) -> Path: def test_the_route_requires_a_token_and_is_read_by_any_principal(self): self.assertEqual(self.cell.json("/api/certificates", who=None)[0], 401) expected = self.listing() - for who in ("manager", "supplier", "planner", "admin"): + # Any principal beyond an observer-only credential (security review, finding 4). + self.assertEqual(self.cell.json("/api/certificates", who="supplier")[0], 403) + for who in ("manager", "planner", "admin"): with self.subTest(who): self.assertEqual(self.listing(who), expected) self.assertEqual(self.cell.json("/api/console", who="supplier")[1]["certificates"], {"configured": True}) @@ -1560,6 +1569,11 @@ def test_the_purchase_order_path_completes_in_a_browser(self): self.assertFalse(any("invented.rule" in row for row in report["mismatch_rules"]), "a rule that does not reproduce the decision is never shown") self.assertTrue(report["refused_rules"] and all("unavailable" in row and "unknown_case" in row for row in report["refused_rules"]), report["refused_rules"]) self.assertIn("amount_exceeds_limit", report["replay"]) + # The observer-only supplier is refused a key the contract reads from a verified source and cannot read the case. + self.assertEqual((report["quote_attest"]["status"], report["quote_attest"]["body"]["error"]), (409, "attestation_denied")) + self.assertEqual(report["note_attest"]["status"], 200, report["note_attest"]) + self.assertEqual(report["receipt_attest"]["status"], 200, report["receipt_attest"]) + self.assertEqual(report["supplier_case_read"], 403) self.assertEqual(len(report["manual"]), 2) self.assertTrue(all("unverified" in row for row in report["manual"])) self.assertIn("Attest claim", report["manual"][0]) diff --git a/tests/test_security_review.py b/tests/test_security_review.py new file mode 100644 index 0000000..ed94033 --- /dev/null +++ b/tests/test_security_review.py @@ -0,0 +1,628 @@ +"""Regression tests for the trust-boundary findings of the 2026-11 SyberWork security review. + +Each class names the finding it pins. No test opens a connection beyond loopback, moves funds or reads a +real service: name resolution is patched, and the servers are SQLite cells on free loopback ports. +""" + +import copy +import hashlib +import ipaddress +import json +import os +import socket +import sys +import tempfile +import threading +import time +import unittest +import urllib.error +import urllib.request +from contextlib import contextmanager +from pathlib import Path +from unittest.mock import patch + +from syberlabs import planner as planner_module +from syberlabs import publish as publish_module +from syberlabs.errors import Rejected +from syberlabs.publish import Command +from syberlabs.targets import credentialed_opener, guard_request, trusted_origin +from syberwork import core as core_module +from syberwork.backup import export_cell +from syberwork.core import Work +from syberwork.server import UserFile, make_server +from syberwork.storage import ReservationStore +from syberlabs.evidence import acceptance_results +from syberlabs.targets import header_value_ok +from syberwork.worker import run_once +from tests.test_cell import NOTE, _drop_postgres, _install, _postgres_database + +EXAMPLES = Path(__file__).resolve().parents[1] / "syberwork" / "examples" +PUBLIC = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 443))] + + +def _token_hash(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +class HostCheckMatchesDialTarget(unittest.TestCase): + """Finding 1: the guard classified ``%31%32%37.0.0.1`` as a name while urllib dialed ``127.0.0.1``.""" + + ENCODED = ("https://%31%32%37.0.0.1/orders", "https://%6c%6fcalhost/x", "https://exa%6dple.com/x", + "https://169.254.169.%32%35%34/latest", "https://ex_ample.com/x", "https://éx.example/x", + "https://exa mple.com/x", "https://{key}.example.com/x") + + def test_a_host_urllib_would_rewrite_is_refused_before_classification(self): + for url in self.ENCODED: + with self.subTest(url=url), self.assertRaises(Rejected) as refused: + trusted_origin(url) + self.assertEqual(refused.exception.code, "invalid_target") + + def test_plain_names_and_ip_literals_still_pass(self): + self.assertEqual(trusted_origin("https://EXAMPLE.com/x"), ("https", "example.com", 443)) + self.assertEqual(trusted_origin("https://xn--bcher-kva.example/x"), ("https", "xn--bcher-kva.example", 443)) + self.assertEqual(trusted_origin("https://[2606:4700::1]:8443/x"), ("https", "2606:4700::1", 8443)) + self.assertEqual(trusted_origin("http://127.0.0.1:9/items/{key}"), ("http", "127.0.0.1", 9)) + + def test_the_guard_refuses_an_encoded_loopback_without_a_lookup(self): + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=AssertionError("no lookup")): + with self.assertRaises(Rejected): + guard_request("https://%31%32%37.0.0.1/orders") + + def test_a_name_the_guard_cannot_resolve_is_refused_not_allowed(self): + for failure in (socket.gaierror(), UnicodeError("label"), OSError("resolver")): + with self.subTest(failure=type(failure).__name__): + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=failure): + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path") + with patch("syberlabs.targets.socket.getaddrinfo", return_value=[]): + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path") + with patch("syberlabs.targets.socket.getaddrinfo", return_value=PUBLIC): + guard_request("https://example.com/path") + + def test_an_unresolvable_source_is_unavailable_and_makes_no_request(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + case = _install(work) + work.install_source("crm", {"kind": "http", "url": "https://crm.example/records/{key}"}) + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=socket.gaierror()), \ + patch.object(core_module.HTTP, "open", side_effect=AssertionError("no request")): + with self.assertRaises(Rejected) as refused: + work.refresh_fact(case, "crm", "c", "R-1", "operator", ["operator"]) + self.assertEqual((refused.exception.code, refused.exception.detail), ("source_unavailable", "destination_unreachable")) + work.close() + + +class SourceKeyStaysInThePath(unittest.TestCase): + """Finding 2: ``{key}`` in a source host let the case input pick the host that receives the bearer.""" + + def test_install_source_refuses_key_outside_the_path(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for url in ("https://{key}.example.com/records", "https://records.example/{key}/{key}", + "https://records.example/r?id={key}", "https://records.example/{key}?token=abc", + "https://records.example/{key}/{other}", "https://records.example/r#{key}"): + with self.subTest(url=url), self.assertRaises(Rejected) as refused: + work.install_source("crm", {"kind": "http", "url": url, "auth_env": "CRM_TOKEN"}) + self.assertEqual(refused.exception.code, "invalid_source") + work.install_source("crm", {"kind": "http", "url": "https://records.example/r/{key}", "auth_env": "CRM_TOKEN"}) + self.assertIn("crm", work.registry("sources")) + work.close() + + +class ActionUrlCarriesNoQuery(unittest.TestCase): + """Finding 3: a query string (where a secret rides) was stored in the action and served by ``registry``.""" + + def test_install_action_refuses_a_query_on_the_write_or_status_url(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for doc in ({"kind": "http", "url": "https://orders.example/po?api_key=s3cret"}, + {"kind": "http", "url": "https://orders.example/po?"}, + {"kind": "http", "url": "https://orders.example/{key}"}, + {"kind": "http", "url": "https://orders.example/po", + "status_url": "https://orders.example/po/{key}?api_key=s3cret"}): + with self.subTest(doc=doc), self.assertRaises(Rejected) as refused: + work.install_action("issue", doc) + self.assertEqual(refused.exception.code, "invalid_action") + self.assertEqual(work.registry("actions"), {}) + work.install_action("issue", {"kind": "http", "url": "https://orders.example/po", + "status_url": "https://orders.example/po/{key}", "auth_env": "ORDERS_TOKEN"}) + self.assertNotIn("?", json.dumps(work.registry("actions"))) + work.close() + + +class ObserverCannotStallOrRead(unittest.TestCase): + """Finding 4: an observer token could shadow a verified fact with an unverified one and read every case.""" + + def setUp(self): + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.work = Work(Path(folder.name) / "w.sqlite") + self.addCleanup(self.work.close) + self.work.install_policy(json.loads((EXAMPLES / "policy.json").read_text())) + self.work.install_contract(json.loads((EXAMPLES / "contract.json").read_text())) + self.work.install_action("record_review", {"kind": "local"}) + self.work.install_action("issue_order", {"kind": "local"}) + self.case = self.work.create_case("purchase-order", 1, {"part_number": "P-104", "quantity": 2}, "operator") + self.users = { + "inventory": {"hash": _token_hash("inv"), "roles": ["observer"], "sources": ["inventory"]}, + "supplier": {"hash": _token_hash("sup"), "roles": ["observer"], "sources": ["supplier"]}, + "operator": {"hash": _token_hash("op"), "roles": ["operator"], "sources": []}, + } + self.server = make_server(self.work, self.users, port=0) + self.server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=self.server.serve_forever, daemon=True).start() + self.addCleanup(self.server.server_close) + self.addCleanup(self.server.shutdown) + self.base = f"http://127.0.0.1:{self.server.server_port}" + + def call(self, path, token, data=None): + headers = {"Authorization": "Bearer " + token} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(self.base + path, data=None if data is None else json.dumps(data).encode(), + headers=headers, method="GET" if data is None else "POST") + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read()) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read()) + + def test_an_attestation_cannot_shadow_a_fact_the_contract_reads_verified(self): + self.work.observe(self.case, "part_number", "P-104", "inventory", "v1", "sync", verified=True) + before = len(self.work.inspect(self.case)["events"]) + # The supplier attests the inventory's key, and the inventory attests its own key unverified: the contract + # reads part_number only verified and only from inventory, so neither may land as the newest observation. + for token, source in (("sup", "supplier"), ("inv", "inventory")): + with self.subTest(source=source): + status, body = self.call(f"/api/cases/{self.case}/facts", token, + {"key": "part_number", "source": source, "version": "x", "value": "P-999"}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + # A key used only through an argument binding (``quote.id``) is protected too. + status, body = self.call(f"/api/cases/{self.case}/facts", "sup", + {"key": "quote", "source": "supplier", "version": "x", "value": {"id": "Q", "price": 1}}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + self.assertEqual(len(self.work.inspect(self.case)["events"]), before) + proposed = self.work.propose(self.case, "record_review", {"part_number": "P-104"}, "operator", ["operator"]) + self.assertEqual(proposed["decision"]["status"], "allowed") + + def test_a_key_the_contract_does_not_read_is_still_recorded_unverified(self): + status, body = self.call(f"/api/cases/{self.case}/facts", "inv", + {"key": "shelf_note", "source": "inventory", "version": "n1", "value": "bin 4"}) + self.assertEqual(status, 200) + self.assertIs(body["body"]["verified"], False) + + def test_a_contract_that_accepts_a_sources_word_still_takes_it(self): + contract = json.loads((EXAMPLES / "contract.json").read_text()) + contract["version"] = 2 + contract["actions"]["record_review"]["required_facts"][0]["verified"] = False + self.work.install_contract(contract) + case = self.work.create_case("purchase-order", 2, {"part_number": "P-104", "quantity": 2}, "operator") + status, _ = self.call(f"/api/cases/{case}/facts", "inv", + {"key": "part_number", "source": "inventory", "version": "n1", "value": "P-104"}) + self.assertEqual(status, 200) + status, body = self.call(f"/api/cases/{case}/facts", "sup", + {"key": "part_number", "source": "supplier", "version": "n1", "value": "P-1"}) + self.assertEqual((status, body["error"]), (409, "attestation_denied")) + + def test_an_observer_reads_no_case_contract_certificate_or_metric(self): + for path in ("/api/cases", f"/api/cases/{self.case}", f"/api/cases/{self.case}/verify", + f"/api/cases/{self.case}/side", f"/api/cases/{self.case}/candidates", "/api/contracts", + "/api/contracts/purchase-order/1", "/api/contracts/purchase-order/1/artifacts", + "/api/certificates", "/metrics/cases_total", "/metrics/work_items"): + with self.subTest(path=path): + status, body = self.call(path, "inv") + self.assertEqual((status, body["error"]), (403, "forbidden")) + status, _ = self.call(path, "op") + self.assertEqual(status, 200) + status, body = self.call("/api/me", "inv") + self.assertEqual((status, body["name"]), (200, "inventory")) + # Builder views keep the builder's own status for a refusal (409), as its writes already did. + status, body = self.call(f"/api/builder/work/{self.case}", "inv") + self.assertEqual((status, body["error"]), (409, "forbidden")) + + +class BackupDoesNotFollowLinks(unittest.TestCase): + """Finding 5: ``export_cell`` replaced the target of a ``manifest.json`` symlink with the snapshot.""" + + @unittest.skipIf(sys.platform == "win32", "creating symlinks needs a privilege on Windows") + def test_a_planted_link_is_refused_and_its_target_untouched(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + _install(work) + victim = Path(folder) / "victim.txt" + victim.write_text("keep me", encoding="utf-8") + for name in ("manifest.json", "cases.json"): + with self.subTest(name=name): + snapshot = Path(folder) / ("snap-" + name) + snapshot.mkdir() + (snapshot / name).symlink_to(victim) + with self.assertRaises(Rejected) as refused: + export_cell(work, snapshot) + self.assertEqual(refused.exception.code, "backup_refused") + self.assertEqual(victim.read_text(encoding="utf-8"), "keep me") + self.assertTrue((snapshot / name).is_symlink()) + clean = Path(folder) / "clean" + manifest = export_cell(work, clean) + self.assertEqual(json.loads((clean / "manifest.json").read_text(encoding="utf-8")), manifest) + self.assertEqual([p.name for p in clean.iterdir() if p.name.startswith(".")], []) + work.close() + + +class NoEnvironmentProxy(unittest.TestCase): + """Finding 6: ``build_opener`` installed a ProxyHandler from ``http_proxy`` at import, so the proxy got the bearer.""" + + def test_credentialed_openers_carry_no_proxy(self): + for opener in (core_module.HTTP, publish_module.HTTP, planner_module._HTTP): + with self.subTest(opener=opener): + self.assertFalse(any(isinstance(h, urllib.request.ProxyHandler) and h.proxies for h in opener.handlers)) + self.assertFalse(any(name.endswith("_open") and "proxy" in name for h in opener.handlers for name in dir(h) + if isinstance(h, urllib.request.ProxyHandler))) + + def test_a_proxy_set_before_the_opener_is_built_is_not_used(self): + hits = [] + + class Proxy(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen(1) + self.sock.settimeout(5) + + def run(self): + try: + conn, _ = self.sock.accept() + hits.append(conn.recv(4096)) + conn.close() + except OSError: + pass + + proxy = Proxy() + proxy.start() + self.addCleanup(proxy.sock.close) + target = socket.socket() + target.bind(("127.0.0.1", 0)) + target.listen(1) + self.addCleanup(target.close) + address = f"http://127.0.0.1:{proxy.sock.getsockname()[1]}" + with patch.dict(os.environ, {"http_proxy": address, "HTTP_PROXY": address, "no_proxy": "", "NO_PROXY": ""}): + opener = credentialed_opener(core_module.NoRedirect) + request = urllib.request.Request(f"http://127.0.0.1:{target.getsockname()[1]}/x", + headers={"Authorization": "Bearer secret"}) + with self.assertRaises((urllib.error.URLError, OSError)): + opener.open(request, timeout=1) + proxy.sock.close() + proxy.join(2) + self.assertEqual(hits, []) + + +class ReservationAmountsAre64Bit(unittest.TestCase): + """Follow-up: PostgreSQL stored ``amount_units`` as int4 while ``units`` admits up to 2**63-1.""" + + LARGE = 2**62 + 7 + + def _reserve(self, work): + with work.tx() as db: + ReservationStore(db).reserve("p-large", "c", "b", "USD", self.LARGE) + with work.tx() as db: + self.assertEqual(ReservationStore(db).reserved_total("b", "USD"), self.LARGE) + + def test_sqlite_keeps_a_large_reservation(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + self._reserve(work) + work.close() + + def test_postgresql_keeps_a_large_reservation(self): + url, admin, name = _postgres_database() + self.addCleanup(_drop_postgres, admin, name) + work = Work(url) + try: + self._reserve(work) + kind = work._db.execute("SELECT data_type::text AS data_type FROM information_schema.columns " + "WHERE table_name = 'economic_reservations' AND column_name = 'amount_units'").fetchone() + self.assertEqual(kind["data_type"], "bigint") + finally: + work.close() + + +class PublishTimeoutEndsTheTree(unittest.TestCase): + """Follow-up: a publish command's timeout killed one process while a check's kills the whole tree.""" + + @unittest.skipUnless(hasattr(os, "killpg"), "the process-group path is POSIX; Windows uses taskkill /T") + def test_a_timed_out_command_leaves_no_child_running(self): + with tempfile.TemporaryDirectory() as folder: + beat = Path(folder) / "beat" + child = ("import time,pathlib\n" + f"p=pathlib.Path({str(beat)!r})\n" + "while True:\n p.open('a').write('.')\n time.sleep(0.05)\n") + parent = (f"import subprocess,sys,time\nsubprocess.Popen([sys.executable,'-c',{child!r}])\ntime.sleep(60)\n") + + class Repo: + @contextmanager + def worktree(self, commit): + yield Path(folder) + + class Kit: + repo = Repo() + + command = Command(Kit(), {"timeout_seconds": 1, "argv": [sys.executable, "-c", parent]}) + started = time.monotonic() + self.assertEqual(command._run(command.doc["argv"], "case", {"commit": "c"}, "k"), (None, "")) + self.assertLess(time.monotonic() - started, 30) + time.sleep(0.3) + size = beat.stat().st_size if beat.exists() else 0 + time.sleep(0.6) + self.assertEqual(beat.stat().st_size if beat.exists() else 0, size, "the grandchild kept running") + + +class BindAndUsers(unittest.TestCase): + """Follow-ups: an empty SYBERWORK_BIND passed the loopback guard, and users.json was read once.""" + + def test_an_empty_host_never_binds(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + for env in ("", None): + with self.subTest(env=env): + environ = {"SYBERWORK_BIND": env} if env is not None else {} + with patch.dict(os.environ, environ, clear=False): + if env is None: + os.environ.pop("SYBERWORK_BIND", None) + with self.assertRaises(Rejected) as refused: + make_server(work, {}, "", 0) + self.assertEqual(refused.exception.code, "unsafe_bind") + work.close() + + def test_users_json_changes_apply_without_a_restart(self): + # Cleanups run last in, first out: the folder is removed after the server and the database close, + # which Windows needs before it will delete an open SQLite file. + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + folder = scratch.name + path = Path(folder) / "users.json" + path.write_text(json.dumps({"op": {"hash": _token_hash("one"), "roles": ["operator"], "sources": []}})) + work = Work(Path(folder) / "w.sqlite") + server = make_server(work, UserFile(path), port=0) + server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(work.close) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + + def me(token): + request = urllib.request.Request(f"http://127.0.0.1:{server.server_port}/api/me", + headers={"Authorization": "Bearer " + token}) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + self.assertEqual(me("one"), 200) + path.write_text(json.dumps({"op": {"hash": _token_hash("two-rotated"), "roles": ["operator"], "sources": []}})) + os.utime(path, ns=(time.time_ns(), time.time_ns() + 1_000_000)) + self.assertEqual((me("one"), me("two-rotated")), (401, 200)) + path.write_text("not json") + os.utime(path, ns=(time.time_ns(), time.time_ns() + 2_000_000)) + self.assertEqual(me("two-rotated"), 401) + path.unlink() + self.assertEqual(me("two-rotated"), 401) + + +class SpecialPurposeAddressesAreNotPublic(unittest.TestCase): + """Review F6: ``is_global`` alone let IANA special-purpose addresses through on some interpreters.""" + + def test_special_purpose_ranges_are_refused_whatever_ipaddress_says(self): + from syberlabs.targets import _public_ip + for address in ("192.0.0.9", "192.0.0.10", "2001:1::1", "2001:1::2", "fec0::1", "2001::1", "2002:7f00:1::1", + "64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "198.18.0.1", "100.64.0.1", "::ffff:127.0.0.1"): + with self.subTest(address=address): + self.assertFalse(_public_ip(ipaddress.ip_address(address))) + for address in ("93.184.216.34", "1.1.1.1", "2606:4700::1111", "2001:4860:4860::8888", "64:ff9b::808:808"): + with self.subTest(address=address): + self.assertTrue(_public_ip(ipaddress.ip_address(address))) + + def test_a_name_resolving_to_one_is_refused(self): + answer = [(socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2001:1::1", 443, 0, 0))] + with patch("syberlabs.targets.socket.getaddrinfo", return_value=answer): + with self.assertRaises(Rejected): + guard_request("https://example.com/x") + + +class ConnectionIsPinnedToTheCheckedAddress(unittest.TestCase): + """Review F11: the guard resolved once and urllib resolved again, so a rebinding name could reach loopback.""" + + def test_the_connection_resolves_checks_and_dials_only_the_accepted_address(self): + dialed = [] + + class Recorder: + def __init__(self, *args): + pass + + def settimeout(self, timeout): + pass + + def connect(self, sockaddr): + dialed.append(sockaddr) + raise OSError("stop before TLS") + + def close(self): + pass + + opener = credentialed_opener() + with patch("syberlabs.targets.socket.getaddrinfo", return_value=PUBLIC), \ + patch("syberlabs.targets.socket.socket", Recorder): + with self.assertRaises(urllib.error.URLError): + opener.open(urllib.request.Request("https://example.com/x"), timeout=2) + self.assertEqual(dialed, [("93.184.216.34", 443)]) + + def test_a_name_that_rebinds_to_loopback_after_the_guard_is_never_dialed(self): + loopback = [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", 443))] + answers = iter([PUBLIC, loopback]) + opener = credentialed_opener() + with patch("syberlabs.targets.socket.getaddrinfo", side_effect=lambda *a, **k: next(answers)), \ + patch("syberlabs.targets.socket.socket", side_effect=AssertionError("no socket for a refused address")): + guard_request("https://example.com/x") + with self.assertRaises(urllib.error.URLError) as refused: + opener.open(urllib.request.Request("https://example.com/x"), timeout=2) + self.assertIn("refused target address", str(refused.exception.reason)) + + +class HeaderValuesAreChecked(unittest.TestCase): + """Review F14: ``version_arg`` was copied into If-Match with no character check.""" + + def test_folded_or_padded_values_are_refused(self): + for value in ("v1\r\n X-Injected: 1", "v1\n\tmore", " v1", "v1 ", "v\x00", "", "vé"): + with self.subTest(value=value): + self.assertFalse(header_value_ok(value)) + self.assertTrue(header_value_ok('"etag-1" W/x')) + + def test_commit_refuses_before_the_claim_and_execute_refuses_before_a_request(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + contract = dict(NOTE, id="versioned") + work.install_contract(contract) + work.install_policy({"version": 1, "actions": {"note": {"roles": ["operator"]}}}) + work.install_action("note", {"kind": "http", "url": "http://127.0.0.1:9/notes", "version_arg": "v"}) + case = work.create_case("versioned", 1, {"name": "a"}, "operator") + proposed = work.propose(case, "note", {"v": "q-1\r\n X-Injected: yes"}, "operator", ["operator"]) + self.assertEqual(proposed["decision"]["status"], "allowed", proposed) + with self.assertRaises(Rejected) as refused: + work.commit(case, proposed["proposal"]["id"], "operator") + self.assertEqual(refused.exception.code, "invalid_argument") + self.assertFalse(any(e["kind"] == "effect_started" for e in work.inspect(case)["events"])) + work.close() + with patch.object(core_module.HTTP, "open", side_effect=AssertionError("no request")): + with self.assertRaises(Rejected) as refused: + Work._execute({"kind": "http", "url": "http://127.0.0.1:9/x", "version_arg": "v"}, {"v": "a\n b"}, "k") + self.assertEqual(refused.exception.code, "invalid_header") + + +class ResolutionNeedsAChangedVersion(unittest.TestCase): + """Review F15: a result read under a key other than the trigger counted as a change by itself.""" + + def setUp(self): + from case_studies.enterprise_procurement import CONTRACT, SimulatedERP, configure + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.erp = SimulatedERP(Path(scratch.name) / "erp.sqlite3") + self.addCleanup(self.erp.close) + self.work = Work(Path(scratch.name) / "w.sqlite3") + self.addCleanup(self.work.close) + configure(self.work, self.erp) + self.contract = CONTRACT + + def _quote_task(self, version): + case = self.work.create_case(self.contract["id"], version, {"request_id": "REQ-4814"}, "analyst") + for source, key in (("requisitions", "request"), ("quote_options", "quote_options")): + self.work.refresh_fact(case, source, key, "REQ-4814", "analyst", ["operator"]) + return case, self.work.request_resolution(case, "quote", "analyst", ["operator"])["body"]["id"] + + def test_without_a_confirmation_or_captured_version_the_result_does_not_complete(self): + unconfirmed = copy.deepcopy(self.contract) + unconfirmed["version"] = 2 + del unconfirmed["resolutions"]["quote"]["confirmation"] + self.work.install_contract(unconfirmed) + case, task = self._quote_task(2) + self.erp.select_quote("procurement", "REQ-4814", "Q-882A") + result = self.work.resolve_resolution(case, task, "p.soto", ["procurement"]) + self.assertEqual((result["status"], result["reason"]), ("pending", "source_decision_unverified")) + + def test_a_confirmation_that_rereads_the_trigger_still_completes(self): + case, task = self._quote_task(1) + self.erp.select_quote("procurement", "REQ-4814", "Q-882A") + self.assertEqual(self.work.resolve_resolution(case, task, "p.soto", ["procurement"])["status"], "completed") + + +class InterruptedEffectsKeepTheirOutcome(unittest.TestCase): + """Review F16: a second worker could relabel a stored ``effect_unknown`` as settled.""" + + def test_one_worker_recovers_an_expired_lease_and_unknown_stays_unknown(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite", effects="worker") + case = _install(work) + proposal = work.propose(case, "note", {}, "operator", ["operator"])["proposal"]["id"] + work.commit(case, proposal, "operator") + with work.tx() as db: + db.execute("UPDATE effect_obligations SET state='leased', lease_until=? WHERE proposal_id=?", (0, proposal)) + first = work.lease_obligation(now=10) + self.assertEqual(first["state"], "interrupted") + self.assertIsNone(work.lease_obligation(now=10), "the expired lease is handed to one worker only") + self.assertEqual(work.settle_interrupted(first)["status"], "unknown") + # A worker that took the same job before the swap existed sees the stored outcome, not "settled". + self.assertEqual(work.settle_interrupted(dict(first))["status"], "unknown") + with work.tx() as db: + state = db.execute("SELECT state FROM effect_obligations WHERE proposal_id=?", (proposal,)).fetchone()["state"] + self.assertEqual(state, "unknown") + self.assertEqual(sum(e["kind"] == "effect_unknown" for e in work.inspect(case)["events"]), 1) + work.close() + + def test_a_recovery_that_died_is_picked_up_again(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite", effects="worker") + case = _install(work) + proposal = work.propose(case, "note", {}, "operator", ["operator"])["proposal"]["id"] + work.commit(case, proposal, "operator") + with work.tx() as db: + db.execute("UPDATE effect_obligations SET state='recovering', lease_until=? WHERE proposal_id=?", (0, proposal)) + self.assertEqual(work.metric("queue_depth"), 1) + self.assertEqual(run_once(work, now=10)["status"], "unknown") + work.close() + + +class FactAcceptanceReadsTheNewestObservation(unittest.TestCase): + """Review F17: a fact clause passed on any historical observation, verified or not.""" + + @staticmethod + def _observed(seq, value, verified, source="inventory"): + return {"seq": seq, "kind": "observed", "body": {"key": "status", "value": value, "verified": verified, "source": source}} + + def test_an_older_match_does_not_pass_and_verified_is_honored(self): + clause = {"id": "ready", "kind": "fact", "key": "status", "equals": "ready"} + contract = {"acceptance": [clause]} + superseded = [self._observed(1, "ready", True), self._observed(2, "held", True)] + self.assertFalse(acceptance_results(contract, superseded)[0]["passed"]) + current = [self._observed(1, "held", True), self._observed(2, "ready", False)] + self.assertTrue(acceptance_results(contract, current)[0]["passed"]) + self.assertFalse(acceptance_results({"acceptance": [dict(clause, verified=True)]}, current)[0]["passed"]) + self.assertFalse(acceptance_results({"acceptance": [dict(clause, source="erp")]}, current)[0]["passed"]) + + def test_an_acceptance_fact_key_is_not_open_to_manual_attestation(self): + contract = {"actions": {}, "acceptance": [{"id": "ready", "kind": "fact", "key": "status", "equals": "ready"}]} + self.assertFalse(Work._attestable(contract, "status", "supplier")) + self.assertTrue(Work._attestable(contract, "note", "supplier")) + + +class ConsoleCannotBeFramed(unittest.TestCase): + """Review F18: the console's policy set default-src, which does not govern framing.""" + + def test_the_console_document_refuses_framing(self): + with tempfile.TemporaryDirectory() as folder: + work = Work(Path(folder) / "w.sqlite") + server = make_server(work, {}, port=0) + server.RequestHandlerClass.log_message = lambda *args: None + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + with urllib.request.urlopen(f"http://127.0.0.1:{server.server_port}/", timeout=5) as response: + headers = response.headers + finally: + server.shutdown() + server.server_close() + work.close() + self.assertIn("frame-ancestors 'none'", headers["Content-Security-Policy"]) + self.assertEqual(headers["X-Frame-Options"], "DENY") + + +class PlannerUrlTakesNoQuery(unittest.TestCase): + """Review F3, planner half: a credential could ride in the planner URL's query string.""" + + def test_a_planner_url_with_a_query_is_unconfigured(self): + with patch.dict(os.environ, {"SYBERWORK_PLANNER_URL": "https://planner.example/plan?key=s3cret"}): + with self.assertRaises(planner_module.PlannerRefusal) as refused: + planner_module.HttpPlanner().ensure_configured() + self.assertEqual(refused.exception.code, "planner_unconfigured") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_witness.py b/tests/test_witness.py index 1153f40..f5c404b 100644 --- a/tests/test_witness.py +++ b/tests/test_witness.py @@ -10,6 +10,7 @@ import sqlite3 import tempfile import unittest +import urllib.error from pathlib import Path from unittest.mock import patch @@ -273,8 +274,10 @@ def test_request_guard_checks_resolved_addresses(self): guard_request("https://example.com/path") with patch("syberlabs.targets.socket.getaddrinfo", side_effect=AssertionError("dns")): guard_request("http://127.0.0.1:9/items/1") + # A name the guard cannot resolve is refused (URLError, so callers keep their unreachable codes). with patch("syberlabs.targets.socket.getaddrinfo", side_effect=socket.gaierror()): - guard_request("https://example.com/path") + with self.assertRaises(urllib.error.URLError): + guard_request("https://example.com/path")