diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6283f01..c929c96 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,20 @@ jobs: # the overlay tests hand keel's key and file to (tests/wgtools.py); # in CI they fail rather than skip without it. apt-packages: bats shellcheck gnupg gpgv wireguard-tools + # keel mesh end to end (decision 0048) in a Debian trixie system + # container, the release keel ships on: three network namespaces, + # the real wg-quick and TLS, as root in the container + # (tests/test_mesh_netns.py). The check is "mesh / trixie". + mesh: + uses: keel-linux/.github/.github/workflows/lxc-trixie.yml@main + with: + timeout: 20 + run: | + apt-get update -qq + apt-get install -y -qq --no-install-recommends python3-pytest \ + python3-yaml wireguard-tools iproute2 iputils-ping openssl + PYTHONPATH=. python3 -m pytest -q tests/test_mesh_netns.py \ + tests/test_mesh_channel.py # A change that ships must come with a changelog entry, or it cannot be # installed and never reaches an appliance. The rule and its tests are # bin/require-changelog of the .github repository. The check is diff --git a/README.md b/README.md index 78938b6..f07914f 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,11 @@ in [docs/apply.md](docs/apply.md). | `keel network revert` | Put back the interfaces file a pending network change replaced; what the revert timer runs. `--boot` restores the file only, for the boot unit | | `keel network wireguard key` | Print this node's WireGuard public key, making the key pair first when there is none (root, live system only); the private key is never printed (decision 0020, docs/spec.md) | | `keel network wireguard suggest-address` | Print a random unique local IPv6 address with its /64, for the first node of an overlay | -| `keel mesh invite` | Print the `keel mesh join keel1:` line a new node runs to join this node's WireGuard mesh, valid one hour and once, and reserve the new node's overlay address (decision 0048, [docs/mesh.md](docs/mesh.md)) | -| `keel mesh join` | With `--dry-run`, check a `keel1:` token and print the address and the peer it makes in this node's spec, applying nothing; the join itself comes later ([docs/mesh.md](docs/mesh.md)) | +| `keel mesh create` | On the first node: make the WireGuard mesh, a random unique local /64 with this node at `::1`, converged and confirmed by itself after the route check (decision 0048, [docs/mesh.md](docs/mesh.md)) | +| `keel mesh invite` | Print the `keel mesh join - <<< keel1:` line a new node runs to join this node's WireGuard mesh, valid one hour and once; reserve the new node's overlay address and start the invite's root helper, `keel-mesh-invite@`, and its unprivileged listener, `keel-mesh-listen@` ([docs/mesh.md](docs/mesh.md)) | +| `keel mesh join` | Join the mesh with that token: the request to the inviter's port, pinned and authenticated, this node's spec written and applied, and both windows confirmed over the overlay; prints `keel mesh accept keel1a:...` for the inviter when its port cannot be reached. With `--dry-run`, print the change and apply nothing ([docs/mesh.md](docs/mesh.md)) | +| `keel mesh accept` | On the inviter, the line a join printed when it could not reach the port ([docs/mesh.md](docs/mesh.md)) | +| `keel mesh status` | This node's peers, their last handshakes, and the pending invites; never a secret ([docs/mesh.md](docs/mesh.md)) | | `keel notify` | What monit runs when a check of the `monitor` section fails, lasts or recovers: sends one message, saying what happened and what to do, to every channel the section declares (email, Telegram, ntfy, a webhook). It reads the channels from `/etc/keel/monitor.json`, which `apply --system` writes as root, never from the spec, reads the tokens from their secret files and prints no URL (decision 0021, docs/apply.md) | | `keel inspect` | Write a spec from the running machine, or from an offline root, and report every field with its source or why it was not inferred; secrets are never read (brief sections 5.2 and 7) | | `keel diff` | Report drift between the spec and the running machine, or an offline root, field by field, through the same collector `inspect` uses; secrets are never compared and their files need not exist, nothing is written (brief section 5.2) | @@ -140,16 +143,16 @@ Defined in one place, `keel/exits.py`, and reproduced here. | 12 | `ASSEMBLE_FAILED` | The rootfs is not empty or cannot be created, or `tar` or `zstd` failed while extracting or packing | | 13 | `INSPECT_INCOMPLETE` | `inspect` wrote the spec, but a required field (hostname, fqdn, interfaces, alerts, updates) could not be inferred; or `diff` found no drift, but a declared field could not be observed. The report says which and why | | 14 | `DRIFT_FOUND` | `diff` found at least one declared field whose observed value differs. Drift wins over unobserved fields, so a report with both exits 14 | -| 15 | `APPLY_NEEDS_ROOT` | The system phase (`--system`, `--system-only`) on the live system was run by a user other than root; nothing was written, not even the conf. Also `keel network wireguard key` making a key, and `keel mesh invite`, on the live system | +| 15 | `APPLY_NEEDS_ROOT` | The system phase (`--system`, `--system-only`) on the live system was run by a user other than root; nothing was written, not even the conf. Also `keel network wireguard key` making a key, and `keel mesh create`, `invite`, `join`, `accept` and `serve`, on the live system | | 16 | `APPLY_FAILED` | The system phase could not make at least one change; the output names it. The conf was written, where the run writes one, and every other change was made, so the run can be repeated | | 17 | `CHANNEL_INVALID` | A channel pointer, or the record of the one this instance follows, does not parse or fails validation. Also a pointer signed in the future, one claiming more than 30 days of life, and a revision's archived pointer naming another revision | | 18 | `CHANNEL_UNVERIFIED` | A channel pointer is not signed by a key that may move a channel, or no keyring was given to check it against. A key that is revoked or expired is refused here: gpgv exits 0 for both and still prints `VALIDSIG`, so `GOODSIG` is what is required | | 19 | `CHANNEL_EXPIRED` | A channel pointer is past its expiry. A mirror that is stale, broken or hostile holds an appliance on an old release by not updating, so this is an error and never a warning | | 20 | `CHANNEL_ROLLBACK` | A channel pointer names an earlier revision than the one this instance is on; `--allow-rollback`, or naming the release and revision, is how going back is asked for | -| 21 | `NETWORK_NOT_CONFIRMED` | `keel network confirm` refused: no change is waiting, or it was not run from a new session over the new configuration, or a console (docs/apply.md) | +| 21 | `NETWORK_NOT_CONFIRMED` | `keel network confirm` refused: no change is waiting, or it was not run from a new session over the new configuration, or a console (docs/apply.md). Also `keel mesh create`, `join` and `accept` whose overlay change was not confirmed: it reverts by itself when its window ends | | 22 | `NOTIFY_FAILED` | `keel notify` reached no channel: `/etc/keel/monitor.json` is missing or refused, declares none, or every one failed; the message then went to syslog (user.crit) and root's mailbox. One channel that takes the message is success; the others' failures are printed | -| 23 | `MESH_TOKEN_INVALID` | `keel mesh join` could not use the token: not a `keel1:` token, mistyped or truncated (its checksum), of a later format, inconsistent, or expired. The message never holds the token or its secret | -| 24 | `MESH_REFUSED` | `keel mesh` refused: no overlay to invite into, no key yet, no endpoint, no free address in the prefix; or this node is in another mesh, at another address of it, or the change would make a spec validation refuses | +| 23 | `MESH_TOKEN_INVALID` | `keel mesh join` could not use the token: not a `keel1:` token, mistyped or truncated (its checksum), of a later format, inconsistent, or expired; or `keel mesh accept` could not read its `keel1a:` line. The message never holds the token or its secret | +| 24 | `MESH_REFUSED` | `keel mesh` refused: no overlay to invite into, no key yet, no endpoint, no free address in the prefix, another invite pending on the port; this node is in another mesh, at another address of it, or the change would make a spec validation refuses; a network change already waits in its window; the inviter refused the join (used, expired, bad HMAC), or the certificate is not the pinned one; no route to the inviter, or neither node can reach the other | Two rules that callers depend on: diff --git a/debian/changelog b/debian/changelog index 12514ea..c98b502 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,33 @@ +keel (0.17.0) unstable; urgency=medium + + * The second part of handbook decision 0048, joining the mesh with one + command. keel mesh create makes the mesh on its first node, a random + unique local /64 with the node at ::1, and confirms it itself once + the route check finds the gateways and the operator's SSH client + still on the uplink (0048 amending 0018). keel mesh invite starts + a root helper as the transient unit keel-mesh-invite@ and, from + it, the unprivileged listener keel-mesh-listen@ (DynamicUser, no + capability, sandboxed), HTTPS with the invite's own certificate on + TCP 51820 by default (--port), its TLS key handed over as a memfd on + a unix socket in its own runtime directory, the peer checked against + the unit's MainPID; the HMAC key never leaves the root side, and a + join's confirmation rests on a WireGuard handshake from the new + peer's key. The port is opened in keel's + firewall only where keel-firewall is enabled, through a new named + set of the derived ruleset. The printed line feeds the token on + standard input (keel mesh join - <<< keel1:...). keel mesh join + joins: the request pinned to the token's fingerprint and signed with + an HMAC of the invite's key, checked in constant time on the root + side, once, without replays; each node writes the other + into its own spec and applies it under the window, and the join's + own session over the overlay confirms both windows. Tested end to + end over 250 ms links with 2% loss. When the port cannot be reached, join + applies its side and prints keel mesh accept keel1a: for the + inviter. keel mesh status shows the peers, their handshakes and the + pending invites, never a secret. docs/mesh.md. + + -- Marcos Mendez Sat, 03 Oct 2026 23:00:00 +0000 + keel (0.16.0) unstable; urgency=medium * keel mesh invite and keel mesh join --dry-run, the first part of diff --git a/docs/apply.md b/docs/apply.md index 4cbd5f8..50679f8 100644 --- a/docs/apply.md +++ b/docs/apply.md @@ -1118,6 +1118,15 @@ drop, and these rules: - every `mesh` port of an enabled overlay, on the WireGuard interface only, and none (the step says which) while the spec declares no overlay; +- with an overlay, the TCP ports of the pending mesh invites: a named + set, `mesh_invites`, of ports with a timeout, and one rule that + accepts TCP to a port in it. `keel mesh invite` adds its port for the + time left to its expiry and the listener's end removes it, so the + port is open exactly while an invite is pending ([docs/mesh.md](mesh.md), + decision 0048). Adding an element is not a ruleset change, so it + neither waits on nor disturbs a change in its window. The set is + empty while no invite is pending, and a reload of the table empties + it; - on each of the machine's guest bridges, DNS on 53 and DHCP on 67 and 547: what lxc-net's or libvirt's dnsmasq serves the guests there, which a drop policy would otherwise cut, so every container lost its @@ -1373,6 +1382,8 @@ used survived, so: | SSH, over the overlay | the session started after the change, came from another machine, and arrived at an address the overlay declares (`address`, `ipv4_address`). confirm says the overlay was tested | | SSH, over the uplink | the session started after the change, came from another machine, and arrived at an address another interface of this machine holds now. confirm says the overlay itself was not tested, and at which address a peer would test it | | A console, a process attached from a container's host | always, as for the uplink | +| The mesh session of a join (decision 0048), for the overlay change that join made and no other | on the inviter, a WireGuard handshake from the new peer's key since the join (`wg show latest-handshakes`) and the join's confirmation request signed with the invite's HMAC key, both checked by the root side; on the new node, the inviter's signed answer over the tunnel ([docs/mesh.md](mesh.md)) | +| `keel mesh create`, for the change that created a mesh with no peer and no other | always, after the route check: an overlay with no peer routes its own private prefix only, and no peer exists to confirm it (decision 0048, second round, point 1) | | Anything else, a session older than the change, a session from this machine to itself, or one arriving at an address the overlay does not declare | refused (exit 21) | Requiring the overlay alone would make pairing impossible: the first @@ -1385,7 +1396,9 @@ first, with `ip route get`: to each gateway `network.interfaces` declares and each gateway of a default route in place now (`ip -6` and `ip -4 route show default`: what DHCP, SLAAC or a container's host configured, and what `--skip-uplink` left), IPv6 first, and, for an SSH -session that came over the uplink, back to its client. When one of them +session that came over the uplink, back to its client. `keel mesh +create` and `keel mesh join` also ask the route back to the client of +the SSH session they run in. When one of them leaves through the overlay's interface, the change routes the uplink's traffic into the overlay, and confirm refuses (exit 21) and leaves the change to revert when its window ends, or at once with `keel network diff --git a/docs/mesh.md b/docs/mesh.md index bf30bbc..ed01f0b 100644 --- a/docs/mesh.md +++ b/docs/mesh.md @@ -1,13 +1,16 @@ # keel mesh Joining a node to the WireGuard mesh takes one command (handbook decision -0048). On a node already in the mesh, `keel mesh invite` prints one line: +0048). The first node makes the mesh with `keel mesh create`. On a node +already in the mesh, `keel mesh invite` prints one line: ``` -keel mesh join keel1: +keel mesh join - <<< keel1: ``` -and that line, run on the new node, joins it. The token carries every +and that line, run on the new node, joins it. The token goes to `join` +on its standard input (a here-string of root's shell, bash), so it is +never an argument and never in the process list. The token carries every value the new node would otherwise copy by hand from [docs/spec.md](spec.md), "overlay": the inviter's public key, its endpoint and port, the overlay prefix and the address reserved for the @@ -15,17 +18,48 @@ new node. It is valid for one hour and can be used once. What a join writes is the same `network.overlay.wireguard` fields an operator writes by hand, `address` and a peer, converged by the same -`apply --system` under the same confirmation window. There is no `mesh` -section in the spec, and a mesh built by joins and one typed by hand are -the same spec. - -**What exists in this version:** the token, the address an invite -reserves, the pending invites, `keel mesh invite`, and `keel mesh join ---dry-run`, which prints the change a token makes and applies nothing. -The listener that answers a join, the join itself, the fallback command -`keel mesh accept`, `keel mesh create` for a node with no overlay, and -`keel mesh remove` come next; the token and the invite file already -carry what they need. +`apply --system` under the same confirmation window (decision 0018). +There is no `mesh` section in the spec, and a mesh built by joins and +one typed by hand are the same spec. + +**What exists in this version:** `create`, `invite` and its listener, +`join`, the fallback `accept`, and `status`. Not yet: etcd (it forms at +the third node in cloud advanced, Phase 5), the announcement of a new +node to the inviter's other peers, `keel mesh remove`, `keel mesh +invites` and `invite --cancel`, and the boot unit that removes invites +which expired while the machine was down. confconsole's "Invite a node" +and "Join a mesh" screens call these commands, in confconsole. + +## keel mesh create + +``` +keel mesh create [--network-window SECONDS] +``` + +Root, on the first node, whose spec declares no overlay address. It +writes into this node's spec `network.overlay.wireguard.address`, a +random unique local /64 with this node at `::1` (as `keel network +wireguard suggest-address` gives it), makes the mesh's identity, and +converges the overlay with `apply --system-only --skip-uplink` under the +window; apply makes the key pair on the machine (keel-core#8). WireGuard +listens on its default UDP port, 51820. + +No peer can confirm an overlay that has none, so `create` confirms it +itself (0048, second round, point 1, amending 0018): once apply.md's +route check finds every gateway, and the client of the SSH session +`create` runs in, still leaving through the uplink. An overlay with no +peer routes its own private prefix only; a route that does leave +through it refuses the confirmation and the change reverts when its +window ends. + +``` +# keel mesh create +created the mesh: this node is fd2a:9c41:7e03::1/64 on wg0, WireGuard on UDP 51820 +keel mesh invite prints the line that joins the next node +``` + +apply's own lines, and the confirmation's, go to standard error; +standard output holds the summary alone. ## keel mesh invite @@ -48,17 +82,22 @@ It: carries for the new node to pin; 4. makes the mesh's identity if this node has none (below); 5. under the mesh's lock, removes the expired invites, reserves a free - address (below) and writes the pending invite; -6. prints the `keel mesh join` line, alone, on standard output; what it - reserved and until when go to standard error, never the token. - -`--port` is the TCP port the join request will go to, 51820 by default, + address (below) and writes the pending invite; another invite pending + on the same TCP port is refused (each listener holds its port), and + `--port` gives this one another; +6. starts the invite's root helper as a transient unit, + `keel-mesh-invite@` (`systemd-run`), which starts the unprivileged + listener (below), so the command returns; an invite whose helper + cannot start is removed and no line is printed; +7. opens the port where keel's firewall exists (below); +8. prints the `keel mesh join` line, alone, on standard output; what it + reserved, until when, and whether the port was opened go to standard + error, never the token. + +`--port` is the TCP port the join request goes to, 51820 by default, the number of WireGuard's UDP port, so a provider's firewall needs one -number for both. - -No listener answers the invite yet in this version, so the line can be -checked on the new node with `keel mesh join --dry-run` and not used to -join. +number for both. Under `--root` no listener is started, and the line can +be checked with `keel mesh join --dry-run` only. ### The address @@ -73,13 +112,80 @@ the prefix is nearly full and is searched from the last draw, so its last free address is still found and a full prefix is refused. The prefix's own address (`::`, the subnet router anycast) is never given. -## keel mesh join --dry-run +## keel mesh join + +``` +keel mesh join - [--endpoint ADDRESS] [--network-window SECONDS] <<< TOKEN +keel mesh join TOKEN # accepted, with a warning: argv is public +keel mesh join - --dry-run <<< TOKEN +``` + +Root, on the new node. A token given as an argument still works, and +`join` warns that it shows in the process list and the shell's history. +It: + +1. reads the token whole (below), checks the spec can take it (a node + with no overlay, or one of this mesh at the reserved address), and + that no network change waits in its window here; +2. finds a route to the inviter's endpoint (`ip route get`); without + one it says so and names the rendezvous point of decision 0024, and + applies nothing; +3. keeps the mesh's identity the token carries (a node that keeps + another is in another mesh, and is refused), and makes its key pair + if it has none; +4. finds its own endpoint: `--endpoint`, else a static address + `network.interfaces` declares, else a global address of an interface + other than the overlay's; IPv6 first; none when it has none; +5. sends the join request to the inviter's HTTPS port, at its IPv6 + endpoint and then its IPv4 one, comparing the SHA-256 of the + certificate the inviter presents with the token's fingerprint before + a byte of the request is sent; +6. with the signed answer, which must carry its own nonce, the invite + id, the inviter's key and address, writes its own spec (its address + and the inviter as a peer) and applies it under the window, with + `--skip-uplink`; +7. opens the mesh session to the inviter's overlay address, retrying + every two seconds while the tunnel comes up: the inviter confirms its + window on receiving it, and this node confirms its own on the + inviter's signed answer. No `keel network confirm` is typed on + either node. + +At the end, on standard output: + +``` +joined the mesh: this node is fd2a:9c41:7e03::3/64 on wg0 +peer: nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E= at fd2a:9c41:7e03::1, through [2001:db8:1::10]:51820 +the mesh is up with 2 nodes; etcd starts when a third node joins, because 2 etcd members cannot lose one and keep a majority +``` + +apply's lines, the confirmation's, and every refusal go to standard +error. When the inviter knows other peers, the last line says how many +nodes the mesh has, and that this node reaches the inviter alone until +they learn of it, which this keel does not do yet (0048, "Until etcd +exists"). + +A join that fails after the request was accepted (the tunnel never +answers within the window, say) reverts on both sides by itself, and +the token is spent: the operator runs a new invite. + +### The fallback + +When the inviter's port gives no TCP answer within five seconds, `join` +does not give up, provided this node has an endpoint the inviter can +reach (behind NAT on both sides, neither can reach the other: `join` +says so, names the rendezvous point, and applies nothing). It applies +its own side under a window lengthened to the invite's remaining time, +up to 15 minutes, and prints on standard output, alone: ``` -keel mesh join TOKEN --dry-run -keel mesh join - --dry-run # the token on standard input +keel mesh accept keel1a: ``` +to run on the inviter, then waits for the inviter over the overlay as in +step 7. Too close to the expiry (under 30 seconds) it refuses instead. + +### --dry-run + Reads the token, checks it whole (below), and prints on standard output the fields of `network.overlay.wireguard` the join would set in this node's spec: @@ -101,8 +207,7 @@ and its overlay address as a `/128`. The spec is read and nothing is written or applied. It refuses a node whose overlay is on another prefix (another mesh) or at another address of this one (a node joins once), and a change `spec validate` would refuse once made, such as an overlay -prefix that overlaps this node's uplink, naming the error. Without -`--dry-run`, `join` applies nothing in this version and exits 9. +prefix that overlaps this node's uplink, naming the error. The overlay's optional `ipv4_address` is not carried: a join gives the new node an IPv6 overlay address and routes the inviter's IPv6 address @@ -110,7 +215,214 @@ only. A mesh that also uses IPv4 on the overlay adds those addresses by hand, as before. `-` reads the token from standard input, which keeps it out of the -process list and the shell's history; confconsole will use it. +process list and the shell's history; confconsole uses it. + +## keel mesh accept + +``` +keel mesh accept LINE +keel mesh accept - # the line on standard input +``` + +Root, on the inviter. It reads the `keel1a:` line (below), checks its +HMAC with the pending invite's key and the reserved address, refuses +while a network change waits, consumes the invite, stops the invite's +listener, adds the new node as a peer with its endpoint when it has one +and `persistent_keepalive` 25 (this node may be the one that has to +initiate, behind NAT), and applies. Then, in the foreground, it serves +the mesh session on its overlay address alone, on the invite's port +(opened in keel's firewall where there is one), until the new node +confirms or the window ends: + +``` +accepted: FHKH10gOWeK2bXHZPg8y+oPTprv556bwrmmRkbyEPgg= is a peer of this node at fd2a:9c41:7e03::3 +``` + +A line for another invite, or one changed in the copy, is refused +before anything is spent; a second use of the token finds no pending +invite. + +## keel mesh status + +``` +keel mesh status +``` + +This node's overlay, its public key, each peer of the spec with its +`allowed_ips`, its endpoint (as WireGuard last saw it) and its last +handshake, and the pending invites: id, reserved address, port, expiry, +and whether it was used and waits for its confirmation. `wg show` is +asked for `public-key`, `endpoints` and `latest-handshakes` only, never +`dump` or `private-key`, and no secret is printed. Off the live system +(`--root`) no handshake is read. + +## The protocol + +Two requests go to the inviter's HTTPS port, each a JSON body POSTed +with `X-Keel-Mesh-Signature`: the hex HMAC-SHA256, keyed with the +invite's HMAC key (derived from the token's secret, which never crosses +the wire), of `keel mesh 1\n`, the method, `\n`, the path, `\n`, and +the body. + +| Request | Fields | Over | +| --- | --- | --- | +| `POST /v1/join` | `invite_id`, `public_key`, `endpoint` (`[address]:port`, or null), `address` (the reserved one, with its length), `nonce` (16 random bytes, hex), `time` (seconds since the epoch) | the uplink | +| `POST /v1/confirm` | `invite_id`, `public_key`, `nonce`, `time` | the overlay | + +| Answer (200) | Fields | +| --- | --- | +| to a join | `invite_id`, `nonce` (the request's), `public_key` and `address` (the inviter's), `peers` (the other members the inviter knows: `public_key`, `endpoint` or null, `address`), `etcd` (`none` in this keel), `window` (the seconds the inviter's change waits) | +| to a confirmation | `invite_id`, `nonce`, `confirmed` (whether the inviter kept its change), `detail` | + +An answer is signed the same way, with the method `ANSWER` and the +request's path, so the new node knows it came from the node that made +the token as well as from the pinned certificate. A refusal is a JSON +`{"error": reason}`, unsigned: it gives nothing to act on. + +The fallback's `keel1a:` line carries the join request's fields in +binary, base64url without padding like the token, about 150 characters: +flags (1 byte: an IPv6 or an IPv4 endpoint follows, at most one), the +new node's key (32), the endpoint and its UDP port (16 or 4, and 2), +the reserved address and its length (16, 1), the invite id (8), the +time (4), an HMAC-SHA256 with the invite's key (32), and a checksum +(the first 4 bytes of the SHA-256 of `keel1a:` and the rest), so a +mistyped paste is told from a line for another invite. + +## How the window is confirmed + +0018 confirms a change only from a session that started after it, +arrived over the new configuration, and came from another machine. 0048 +adds two sources, for the overlay change keel mesh itself made and no +other ([docs/apply.md](apply.md)): + +- **the mesh session of a join**: on the inviter, what its root side + sees itself, never what the unprivileged listener reports: a + WireGuard handshake from the new peer's key since it was admitted + (`wg show latest-handshakes`), which only that key's holder, + reaching this node over the new tunnel, can complete, and the + confirmation request signed with the invite's key, which the listener + does not hold. On the new node, the inviter's signed answer, which + arrived over the tunnel to the inviter's overlay address; +- **`keel mesh create`**, for a mesh with no peer. + +Each knows its change by the marker apply left (the boot and the moment +the interface came up) and refuses any other: a change that is not its +own, or an uplink change, is never confirmed this way. The route check +of apply.md runs first, unchanged, with the client of the operator's SSH +session added for `create` and `join`. + +## The listener + +The port faces the internet, so what answers on it can change nothing +on the machine. An invite has two processes: + +- **the root helper**, `keel-mesh-invite@` running `keel mesh serve + `, which reads the invite, starts the listener, and is the only + one that holds the invite's HMAC key, verifies the HMAC, consumes the + invite, writes the spec and applies it (keel.mesh.admit, + keel.mesh.bridge); +- **the listener**, `keel-mesh-listen@` running `keel mesh listen`, + with `DynamicUser=yes`, an empty `CapabilityBoundingSet=` and + `AmbientCapabilities=`, `NoNewPrivileges`, `PrivateTmp`, + `PrivateDevices`, `ProtectSystem=strict`, `ProtectHome`, the kernel, + clock, hostname and cgroup protections, `RestrictAddressFamilies=AF_INET + AF_INET6 AF_UNIX`, `RestrictNamespaces`, `SystemCallFilter=@system-service`, + `MemoryMax=64M`, `TasksMax=32` and `LimitNOFILE=64`. It refuses to run + if it holds any capability. It does TLS 1.3 on the invite's TCP port, + on every address (IPv6 and IPv4; so a `--port` under 1024 cannot be + bound), and the checks below. + +The listener binds the bridge's unix socket, mode 0600, in its own +RuntimeDirectory, `/run/keel/mesh-listen-`, mode 0700 and its dynamic +user's, so no other user can reach it; it takes one connection, from +root, and removes the socket. The helper connects and reads the peer's +credentials (SO_PEERCRED) before it sends anything: not root, and the +unit's MainPID as systemd names it (`systemctl show -p MainPID`). A +socket held by anything else is skipped, and the helper tries again +until the listener shows (30 seconds). It then sends the listener's +parameters, which hold no key, and the invite's certificate and TLS +key as two memfds (SCM_RIGHTS): anonymous files that never have a name, +so the TLS key is written nowhere. **The invite's HMAC key never leaves +the root side.** The listener forwards each request that passed the +checks it can make without the key, as the bytes it received with the +signature header; the helper verifies the HMAC and checks the rest +again, and answers. The addresses the listener reports are logged and +decide nothing. The helper's `RuntimeMaxSec` is the time +left to the expiry plus the window a join made just before it needs to +be confirmed in (120 seconds, and 60 more); on `SIGTERM` (that limit, +or `systemctl stop`) it stops the listener and cleans up. + +A join request is checked in this order, and refused with its reason: + +| Check | Refused with | +| --- | --- | +| the path and the method | 404 | +| a body no longer than 16 KiB, of the protocol's shape | 413, 400 | +| the invite id: this listener's, no other | 403 | +| the request's time, within five minutes of this node's clock | 403 | +| its nonce, never seen | 403 `replayed` | +| (root side) all of the above again, and the HMAC with the invite's key, in constant time | 403 `bad HMAC` | +| (root side) the address, the one the invite reserved | 403 | +| (root side) a key this node already knows, its own or a peer's | 409: a join never replaces an entry | +| (root side) no network change waiting in its window here | 409, and the invite stays pending | +| (root side) the invite still pending: consumed under the mesh's lock, once | 410 `already used`, `expired` | + +Then the new node is written into this node's spec as a peer (its key, +its endpoint when it sent one, and the reserved address as a `/128` in +`allowed_ips`), applied under the window with `--skip-uplink`, and the +answer goes back signed. If the change does not come up the invite is +spent, the answer is a 500, and the operator runs a new invite. The +confirmation request is then taken, by the root side, only with a +WireGuard handshake from the new node's key since it was admitted +(waited for up to 10 seconds); it confirms this node's window, and the +listener stops. + +Against someone scanning or holding the port: + +- reading a request, TLS handshake included, has a deadline of 15 + seconds, and each read a timeout of 5; the apply that follows is not + under that deadline; +- at most 4 connections are read at once, and 2 more are kept for + connections to this node's overlay address, which only the mesh's + peers reach, so a join's confirmation is never starved; one source + address has one connection at a time, so it cannot fill the slots; +- the request line and headers are capped at 8 KiB, the body at 16 KiB; +- a source refused 5 times within a minute is not answered until the + minute has passed; +- only a request that names this invite and fails its HMAC counts + towards cancelling it, on the root side, and 5 of those do: someone + knows the invite and not its key. A scanner's requests cancel + nothing. The root side's answer then says the invite is done, and the + listener stops. + +When the helper ends, however it ends, the listener is stopped, the +invite's file is removed and its port closed. A used, cancelled or +expired invite's file is not kept. + +## Over a poor network + +Keel must work over links of 250 ms round trip, with jitter and 2% +loss. The join's TCP connection is given 10 seconds before the fallback +is taken (two lost SYNs still connect), an answer 180, the mesh session +is retried every 2 seconds for the whole window, the request's time may +be five minutes from the inviter's clock, and the listener's deadlines +are per request. The integration test runs on clean links and again on +250 ms ±25 ms with 2% loss on every link (`tc netem`). + +## The firewall port + +keel touches no firewall it does not own. Where keel-firewall is enabled +(cloud advanced, `firewall.enabled: true`), the derived ruleset in +`inet keel` has a named set, `mesh_invites`, of TCP ports with a +timeout, and one rule that accepts them ([docs/apply.md](apply.md)). +`invite` adds its port to the set for as long as the listener may run, +the time left to its expiry and 180 seconds more (`nft add element inet +keel mesh_invites { 51820 timeout 3780s }`), so the port is open exactly +while the invite is pending and closes by itself even if keel dies, and +the helper's end removes it. Adding an +element is not a ruleset change, so it neither waits on nor disturbs a +network change in its window. Everywhere else keel opens nothing and +`invite` says so: a closed port is what the fallback is for. ## The token @@ -179,42 +491,91 @@ backup set. The invite file holds an HMAC key derived from the secret, never the secret, so it cannot be turned back into the token. An invite is consumed once, under the lock: it is marked consumed, and a second -request is told it was already used. A consumed invite keeps its -address reserved until it expires, so no other invite takes the address -before the new node is a peer in the spec; an expired or damaged one is -removed by the next invite. - -**Pending:** an invite whose expiry passes while the machine is down is -removed by the next invite, not yet at boot; the boot unit 0048 -describes comes with the listener. +request is told it was already used. The listener's end, or accept's, +removes the file. The mesh identity is made once and kept: a file that does not hold one is an error, never replaced, since every other node of the mesh knows -the first. A mesh built by hand before `keel mesh` existed gets it from -its first invite. +the first. The node that joins keeps the token's, and one that keeps +another is refused. A mesh built by hand before `keel mesh` existed gets +it from its first invite. + +**The spec** is written by `create`, the root helper, `join` and +`accept` through keel's writer for its own state (keel.network.marker's +`write_private`: a temporary file, fsync, a rename), after the whole +document passes `spec validate`. Two things change with it, which an +operator who edits the spec by hand should know: **comments are +dropped** (it is written as YAML from the document keel read), and +**the file's mode becomes 0600**, root's alone. `keel spec set` +(keel#69) will make one writer of this and confconsole's. + +## Logs + +The listener's lines go to the journal of its unit: the invite id, the +reserved address, the new node's key and endpoint, each refusal with +its reason and the client's address, each line of the confirmation, and +how the invite ended. **Never the secret, the token, an HMAC or the +invite's TLS key.** The token is printed on the terminal of the root who +ran `invite`, and to nothing else. The line it prints feeds the token +to `join -` on standard input, so it is in no process list; the line +itself is in root's shell history on the new node. The unprivileged +listener's lines go to its own unit's journal, under the same rule. ## Exit codes | Code | When | | --- | --- | -| 0 | the line or the change was printed | +| 0 | the line, the change or the status was printed; the join or the accept confirmed | | 2, 3 | the spec cannot be read or is invalid | -| 9 | `join` without `--dry-run`: not implemented yet | -| 15 | `invite` on the live system, not as root | -| 16 | `invite`: `wg` cannot read the key file, or `openssl` cannot make the certificate, or the mesh identity is damaged | -| 23 | `join`: the token is mistyped, truncated, of a later format, inconsistent, or expired | -| 24 | refused: no overlay to invite into, no key yet, no endpoint, no free address; a node in another mesh or at another address of it, or a change the spec would refuse | +| 15 | `create`, `invite`, `join`, `accept`, `serve` on the live system, not as root | +| 16 | `wg` cannot read or make the key, `openssl` cannot make the certificate, the mesh identity is damaged, the invite's listener cannot start or bind, or this node's change did not come up | +| 21 | the change was applied and not confirmed (the other side did not answer within the window, or refused, or the route check did): it reverts by itself | +| 23 | the token, or `accept`'s line, is mistyped, truncated, of a later format, inconsistent, or expired | +| 24 | refused: no overlay to invite into, no key yet, no endpoint, no free address, another invite on the port; a node in another mesh or at another address of it, or a change the spec would refuse; a network change waiting; no route to the inviter, or neither node can reach the other; the inviter refused the join (used, expired, bad HMAC), its certificate is not the pinned one, or its answer is not the answer to the request | ## Tests -At four seams, each written down before its tests: the token's -`encode()` and `parse()` (`tests/test_mesh_token.py`, round trips and -every damaged, foreign, inconsistent and expired form, and the secret in -no message and no repr); the allocator's `free_address()` -(`tests/test_mesh_allocate.py`); the store's `reserve`, `find`, -`consume`, `pending` and `expire` against a scratch root -(`tests/test_mesh_invites.py`, modes, one use, expiry, no secret on -disk); and the commands through `keel.cli.main` -(`tests/test_mesh_cli.py`), with the real `wg` (tests/wgtools.py) and -the real `openssl`, so the key and the fingerprint in the token are -checked against what those tools say. +At these seams, each written down before its tests: + +- the token's `encode()` and `parse()` (`tests/test_mesh_token.py`), the + allocator (`tests/test_mesh_allocate.py`), the store + (`tests/test_mesh_invites.py`); +- the protocol's messages and their HMAC, pure + (`tests/test_mesh_protocol.py`), and the `keel1a:` line + (`tests/test_mesh_acceptline.py`); +- the HTTP handler as a function, twice: the root side's + `Admitter.forward(path, signature, body, local, peer)` against a + scratch root's invite and a node that records + (`tests/test_mesh_admit.py`: every refusal, one use, replays, known + keys, the confirmation over the overlay only), and the unprivileged + listener's `Listener.handle` (`tests/test_mesh_listener.py`: what + reaches the root side, what cancels the invite, the per source limit); +- the pinned channel and the listener's server, with real TLS on the + loopback (`tests/test_mesh_channel.py`: the deadline, the slots and + those kept for the overlay, the size cap, blocked sources); +- the bridge (`tests/test_mesh_bridge.py`): the whole path in one + process, TLS to the listener, the memfds over the unix socket, the + Admitter; a peer that is not the listener, one that never connects, + messages it should not send, a listener with capabilities, and the + sandbox of its unit; +- the spec writes and apply through `keel.mesh.node.Node`: the flows of + `join`, `accept`, `serve` and `create` against a real Node on scratch + roots, apply replaced by one that arms the window's marker, and the + real `keel.network.confirm` deciding (`tests/test_mesh_joining.py`, + `tests/test_mesh_inviting.py`, `tests/test_mesh_create_status.py`, + `tests/test_network_mesh_confirm.py`); +- the CLI through `keel.cli.main` (`tests/test_mesh_cli.py`, + `tests/test_mesh_cli_live.py`); +- end to end, `tests/test_mesh_netns.py` runs `tests/mesh_netns.py` as + root in a network namespace (tests/wgtools.py), with two more + namespaces joined to it by veths: real `wg-quick`, real TLS, the real + listener, join, accept and confirmation logic. One node joins through + the listener and one through the fallback, both ping the inviter over + the overlay with no `keel network confirm` typed, the spent invite is + refused, the inviter's journal holds no secret, and the listener, a + process of its own under `setpriv` with every capability dropped, + reports a CapEff of 0. apply there writes keel's rendered file and + runs `wg-quick up` in the node's namespace, since a live apply arms + timers on the host's systemd. It runs on clean links and on poor ones + (250 ms ±25 ms, 2% loss), and in a Debian trixie LXC container too + (`mesh / trixie`). diff --git a/keel/__init__.py b/keel/__init__.py index 14fcb39..1c3e915 100644 --- a/keel/__init__.py +++ b/keel/__init__.py @@ -6,4 +6,4 @@ clients of it. """ -__version__ = "0.16.0" +__version__ = "0.17.0" diff --git a/keel/cli.py b/keel/cli.py index fc47fdc..1612fa0 100644 --- a/keel/cli.py +++ b/keel/cli.py @@ -28,13 +28,12 @@ STATE_ENV, ) from keel.manifest import KINDS as MANIFEST_KINDS -from keel.mesh import commands as mesh_commands +from keel.mesh import parser as mesh_parser from keel.monitor.channelfile import PATH as NOTIFY_SETTINGS from keel.monitor.notify import CHECKS as NOTIFY_CHECKS from keel.monitor.notify import DIRECTIONS as NOTIFY_DIRECTIONS from keel.monitor.notify import LEVELS as NOTIFY_LEVELS from keel.spec import CONF_DEFAULT, CONF_ENV, SPEC_DEFAULT, SPEC_ENV -from keel.network.wireguard import DEFAULT_PORT as WIREGUARD_PORT from keel.system import DEFAULT_WINDOW @@ -188,7 +187,8 @@ def build_parser() -> argparse.ArgumentParser: add_root_option(revert_parser, "revert the change of") revert_parser.set_defaults(handler=commands.network_revert) add_wireguard_parser(network_actions) - add_mesh_parser(subparsers) + mesh_parser.add(subparsers, mesh_parser.Options( + add_common_options, add_root_option, port_number, window_seconds)) # no --spec: notify reads the settings apply wrote, never the spec notify_parser = subparsers.add_parser( @@ -323,57 +323,6 @@ def add_wireguard_parser(network_actions) -> None: suggest_parser.set_defaults(handler=commands.network_wireguard_suggest) -def add_mesh_parser(subparsers) -> None: - """keel mesh: joining the WireGuard mesh with one command (0048)""" - mesh_parser = subparsers.add_parser( - "mesh", - help="invite a node into this node's WireGuard mesh, or join one" - " with the line an invite printed (decision 0048, docs/mesh.md)", - ) - mesh_actions = mesh_parser.add_subparsers( - dest="action", metavar="ACTION" - ) - invite_parser = mesh_actions.add_parser( - "invite", - help="print the keel mesh join line for a new node, valid one hour" - " and once, and reserve its overlay address (root)", - ) - add_common_options(invite_parser) - invite_parser.add_argument( - "--endpoint", action="append", default=None, metavar="ADDRESS", - help="an address the new node reaches this one at, one per family;" - " may be repeated (default: the static addresses" - " network.interfaces declares)", - ) - invite_parser.add_argument( - "--port", type=port_number, default=WIREGUARD_PORT, metavar="PORT", - help="the TCP port the join request goes to (default: %(default)s," - " the number of WireGuard's UDP port)", - ) - add_root_option(invite_parser, "read the overlay of and keep the" - " invite in") - invite_parser.set_defaults(handler=mesh_commands.mesh_invite) - join_parser = mesh_actions.add_parser( - "join", - help="join the mesh of the node that printed TOKEN; with --dry-run," - " print the change of this node's spec it makes (the join itself" - " is not implemented yet)", - ) - add_common_options(join_parser) - join_parser.add_argument( - "token", metavar="TOKEN", - help="the keel1: token keel mesh invite printed, or - to read it" - " from standard input, which keeps it out of the process list", - ) - join_parser.add_argument( - "--dry-run", action="store_true", - help="print the address and the peer the join would write into the" - " spec, and change nothing", - ) - add_root_option(join_parser, "read the appliance manifests of") - join_parser.set_defaults(handler=mesh_commands.mesh_join) - - def port_number(text: str) -> int: if not text.isdigit() or not 1 <= int(text) <= 65535: raise argparse.ArgumentTypeError(f"not a port number: {text}") diff --git a/keel/exits.py b/keel/exits.py index 9fdff13..345ed50 100644 --- a/keel/exits.py +++ b/keel/exits.py @@ -67,11 +67,14 @@ " one this instance is on", NETWORK_NOT_CONFIRMED: "keel network confirm refused: nothing waiting" " (a change already confirmed is not refused), or not run over the new" - " configuration", + " configuration; or a keel mesh change that was not confirmed, which" + " reverts by itself", NOTIFY_FAILED: "keel notify reached no channel: none declared, or every" " one failed", - MESH_TOKEN_INVALID: "keel mesh join could not use the token: mistyped" - " or truncated, of a format this keel does not read, or expired", + MESH_TOKEN_INVALID: "keel mesh join could not use the token, or accept" + " its line: mistyped or truncated, of a format this keel does not" + " read, or expired", MESH_REFUSED: "keel mesh refused: no overlay to invite into, no" - " endpoint, no free address, or this node is in another mesh", + " endpoint, no free address, this node is in another mesh, a change" + " waits in its window, or the inviter refused the join", } diff --git a/keel/manifest/firewall.py b/keel/manifest/firewall.py index f54b2a4..c2422bb 100644 --- a/keel/manifest/firewall.py +++ b/keel/manifest/firewall.py @@ -5,7 +5,9 @@ `expose: public` of a process whose overlay is enabled (and every one of the appliance's own processes) is opened on every interface, every `expose: mesh` one on the WireGuard interface only, and the overlay's -own UDP port on the uplink. `loopback` opens nothing. This replaces +own UDP port on the uplink. `loopback` opens nothing. With an overlay, +a named set holds the TCP ports of the pending mesh invites (decision +0048), each with a timeout, and one rule accepts them. This replaces `WEBMIN_FW_TCP_INCOMING` of the recipes (0041, 6). nftables, one table of keel's own, `inet keel`, one input chain whose @@ -53,6 +55,12 @@ "udp dport 68 accept", ) ENABLED = "enabled" +# the TCP ports of the pending mesh invites (decision 0048): keel mesh +# invite adds its port with a timeout, so it closes by itself +INVITES_SET = "mesh_invites" +INVITES_SET_TEXT = ( + f"\tset {INVITES_SET} {{\n\t\ttype inet_service\n" + "\t\tflags timeout\n\t}\n") # What a host serves the guests on its bridges (lxcbr0, docker0, a # Proxmox vmbr): lxc-net's and libvirt's dnsmasq answer DNS, DHCPv4 and # DHCPv6 there, and a drop policy would make every guest lose its lease. @@ -101,13 +109,17 @@ def render(resolved: Resolved, states: dict, wireguard: dict | None, mesh = {} opened_mesh = sorted({port for ports in mesh.values() for port in ports}) rules = list(ALWAYS) + _rules("", sorted(public)) + sets = "" if iface: rules += _rules(f'iifname "{iface}" ', opened_mesh) + rules.append(f"tcp dport @{INVITES_SET} accept") + sets = INVITES_SET_TEXT if bridges: rules += _rules(_iifname(sorted(set(bridges))) + " ", list(BRIDGE_SERVICES)) body = "".join(f"\t\t{rule}\n" for rule in rules) - digest = hashlib.sha256(body.encode()).hexdigest()[:DIGEST_LENGTH] + digest = hashlib.sha256((sets + body).encode()).hexdigest()[ + :DIGEST_LENGTH] family, name = TABLE text = ( f"{HEADER}\n" @@ -115,6 +127,7 @@ def render(resolved: Resolved, states: dict, wireguard: dict | None, f"delete table {family} {name}\n" f"table {family} {name} {{\n" f'\tcomment "{COMMENT} {digest}"\n' + f"{sets}" "\tchain input {\n" "\t\ttype filter hook input priority filter; policy drop;\n" f"{body}" diff --git a/keel/mesh/__init__.py b/keel/mesh/__init__.py index 956f522..232cbc4 100644 --- a/keel/mesh/__init__.py +++ b/keel/mesh/__init__.py @@ -1,25 +1,31 @@ # Copyright (c) 2026 KeelLinux maintainers """Joining the WireGuard mesh with one command (handbook decision 0048) -`keel mesh invite`, on a node already in the mesh, prints one line, -`keel mesh join keel1:`, and that line run on the new node joins -it. What exists so far: +`keel mesh create` makes the mesh on its first node; `keel mesh invite`, +on a node already in it, prints one line, `keel mesh join - +<<< keel1:`, and that line run on the new node joins it. - keel.mesh.token, the `keel1:` token: what the new node needs from the inviter, in one URL safe line with a checksum; - keel.mesh.allocate, the address the invite reserves for the new node; - keel.mesh.invites, the pending invites under /var/lib/keel/mesh, which hold an HMAC key derived from the token's secret, never the secret; -- keel.mesh.identity, the mesh's random identity, which the token - carries and etcd will take as its cluster token; -- keel.mesh.certificate, the self signed certificate of one invite, - whose fingerprint the token carries for the new node to pin; +- keel.mesh.identity, the mesh's random identity; +- keel.mesh.certificate, the self signed certificate of one invite; +- keel.mesh.protocol, the join protocol's messages and their HMAC; +- keel.mesh.acceptline, the `keel1a:` line of the fallback; +- keel.mesh.channel, HTTPS with the invite's certificate, pinned; +- keel.mesh.listener, the unprivileged listener that faces the network; +- keel.mesh.admit, the root side that admits the new node, once; +- keel.mesh.bridge, between the two: a unix socket, the keys as memfds; +- keel.mesh.node, this machine's spec, apply and confirmation: the one + seam to the rest of keel; - keel.mesh.join, the change of the new node's spec a token makes; -- keel.mesh.commands, `keel mesh invite` and `keel mesh join --dry-run`. +- keel.mesh.joining, inviting, create, status: the commands' flows; +- keel.mesh.ports, the invite's port in keel's firewall; +- keel.mesh.commands and parser, the CLI. -The listener on the HTTPS port, the join itself, `accept`, `create` and -`remove` come later; the token and the invite file already carry what -they need. docs/mesh.md is the reference. +docs/mesh.md is the reference. """ # under the root: state, never configuration and never in the spec diff --git a/keel/mesh/acceptline.py b/keel/mesh/acceptline.py new file mode 100644 index 0000000..f242bfa --- /dev/null +++ b/keel/mesh/acceptline.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The keel1a: line the fallback prints, for the inviter (decision 0048) + +When the inviter's HTTPS port cannot be reached, `keel mesh join` applies +its own side and prints `keel mesh accept keel1a:` to run back on +the inviter. The answer carries what the join request would have: + + flags 1 bit 0: an IPv6 endpoint follows, bit 1: an IPv4 one + public key 32 the new node's WireGuard key + endpoint 16|4 when a flag is set, and its UDP port (2) + address 16 the address the invite reserved + prefix length 1 + invite id 8 + time 4 seconds since the epoch, UTC + HMAC 32 HMAC-SHA256 of the label, the prefix and the above, + keyed with the invite's HMAC key + checksum 4 the first four bytes of the SHA-256 of the prefix + and the above + +base64url without padding, as the token is. The checksum tells a +mistyped paste from a line for another invite, whose HMAC fails; so it +is accepted only by the inviter that made the token, once, before the +invite expires. The secret is in it no more than in the request. +""" + +import base64 +import binascii +import hashlib +import hmac +import ipaddress +import struct +from dataclasses import dataclass + +from keel.mesh.protocol import ProtocolError, endpoint +from keel.mesh.token import BODY_RE, CHECKSUM_BYTES, UNIQUE_LOCAL +from keel.network.wireguard import KEY_BYTES, key_bytes + +PREFIX = "keel1a:" +MAX_TEXT = 512 +IPV6_ENDPOINT = 0x01 +IPV4_ENDPOINT = 0x02 +LABEL = b"keel mesh accept\n" +MAC_BYTES = 32 +HEAD = struct.Struct(f">B{KEY_BYTES}s") +PORT = struct.Struct(">H") +BODY = struct.Struct(f">16sB8sI{MAC_BYTES}s") + + +class AcceptError(ValueError): + """A line accept cannot use, and why""" + + +@dataclass(frozen=True) +class Accept: + """`endpoint` is None for a new node that cannot be reached; `address` + has its prefix length""" + + public_key: str + endpoint: str | None + address: str + invite_id: str + time: int + + +@dataclass(frozen=True) +class Sealed: + """A parsed line: its fields, what the HMAC covers, and the HMAC""" + + accept: Accept + covered: bytes + mac: bytes + + def authentic(self, key: bytes) -> bool: + return hmac.compare_digest(mac(key, self.covered), self.mac) + + +def mac(key: bytes, covered: bytes) -> bytes: + return hmac.new(key, LABEL + PREFIX.encode() + covered, + hashlib.sha256).digest() + + +def checksum(payload: bytes) -> bytes: + return hashlib.sha256(PREFIX.encode() + payload).digest()[ + :CHECKSUM_BYTES] + + +def wrap(payload: bytes) -> str: + whole = payload + checksum(payload) + return PREFIX + base64.urlsafe_b64encode(whole).decode().rstrip("=") + + +def encode(accept: Accept, key: bytes) -> str: + """The line; raises AcceptError for one parse would refuse""" + check(accept) + flags, tail = 0, b"" + if accept.endpoint is not None: + host, _, port = accept.endpoint.rpartition(":") + address = ipaddress.ip_address(host.strip("[]")) + flags = IPV6_ENDPOINT if address.version == 6 else IPV4_ENDPOINT + tail = address.packed + PORT.pack(int(port)) + own = ipaddress.IPv6Interface(accept.address) + covered = HEAD.pack(flags, key_bytes(accept.public_key)) + tail + covered += BODY.pack(own.ip.packed, own.network.prefixlen, + bytes.fromhex(accept.invite_id), accept.time, + b"")[:-MAC_BYTES] + return wrap(covered + mac(key, covered)) + + +def parse(text: str) -> Sealed: + """The line in `text`, checked; raises AcceptError with the reason""" + if len(text) > MAX_TEXT: + raise AcceptError(f"not a keel mesh accept line: more than" + f" {MAX_TEXT} characters") + text = text.strip() + if not text.startswith(PREFIX): + raise AcceptError(f"not the answer of a keel mesh join: it starts" + f" with {PREFIX} (paste it alone)") + body = text[len(PREFIX):] + if not BODY_RE.match(body): + raise AcceptError("the line holds a character base64url does not" + " use: it was changed in the copy") + try: + whole = base64.urlsafe_b64decode(body + "=" * (-len(body) % 4)) + except (binascii.Error, ValueError): + whole = b"" + payload, found = whole[:-CHECKSUM_BYTES], whole[-CHECKSUM_BYTES:] + if len(whole) <= CHECKSUM_BYTES or not hmac.compare_digest( + found, checksum(payload)): + raise AcceptError("the line is mistyped or truncated: its checksum" + " does not match; copy it again") + try: + sealed = unpack(payload) + except (struct.error, ValueError) as e: + raise AcceptError(f"the line is malformed: {e}") from None + check(sealed.accept) + return sealed + + +def unpack(payload: bytes) -> Sealed: + flags, public = HEAD.unpack_from(payload) + if flags not in (0, IPV6_ENDPOINT, IPV4_ENDPOINT): + raise ValueError(f"flags {flags:#04x}: one endpoint at most") + offset, shown = HEAD.size, None + if flags: + size = 16 if flags == IPV6_ENDPOINT else 4 + host = ipaddress.ip_address(payload[offset:offset + size]) + (port,) = PORT.unpack_from(payload, offset + size) + offset += size + PORT.size + shown = f"[{host}]:{port}" if host.version == 6 else f"{host}:{port}" + address, length, invite, time, sealed = BODY.unpack_from(payload, offset) + if offset + BODY.size != len(payload): + raise ValueError(f"{len(payload)} bytes where its flags say" + f" {offset + BODY.size}") + return Sealed( + accept=Accept(public_key=base64.b64encode(public).decode(), + endpoint=shown, + address=f"{ipaddress.IPv6Address(address)}/{length}", + invite_id=invite.hex(), time=time), + covered=payload[:-MAC_BYTES], mac=sealed) + + +def check(accept: Accept) -> None: + if key_bytes(accept.public_key) is None: + raise AcceptError("the new node's key is not a WireGuard key") + try: + endpoint(accept.endpoint) + except ProtocolError as e: + raise AcceptError(str(e)) from None + own = ipaddress.IPv6Interface(accept.address) + if not own.network.subnet_of(UNIQUE_LOCAL): + raise AcceptError(f"{own} is not an overlay address (fc00::/7)") diff --git a/keel/mesh/admit.py b/keel/mesh/admit.py new file mode 100644 index 0000000..009b03b --- /dev/null +++ b/keel/mesh/admit.py @@ -0,0 +1,321 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The root side of an invite: admit the new node, once (decision 0048) + +The network faces an unprivileged listener (keel.mesh.listener), which +hands each request that passed its checks of shape, invite id, time and +nonce, as the bytes it received, to the Admitter in the root helper +(keel.mesh.bridge). The listener never holds the invite's HMAC key: the +HMAC is verified here alone, in constant time, with the shape, the id, +the time and the nonce checked again. Five requests that name the +invite and fail its HMAC cancel it, here too; the response then says +so (`final`), and the listener stops. + +A join is then checked against the address the invite reserved, a key +this node already knows (its own, or a peer's: a join never replaces an +entry), and a network change waiting in its window. The invite is +consumed under the mesh's lock, so a second request finds it used, the +new node is written into the spec as a peer and applied under decision +0018's window, and the answer goes back signed. The confirmation +rests on what this side sees itself, never on the addresses the +listener reports: the confirmation request signed with the invite's +key, and a WireGuard handshake from the new node's key since it was +admitted (`wg show latest-handshakes`), which only the holder of +that key, reaching this node over the new tunnel, can make. It confirms +this node's window. + +`handle` is a function of the path, the signature header, the body and +the connection's two addresses, so every decision is tested without a +socket. Nothing logged holds the secret, a token, an HMAC or a key. +""" + +import ipaddress +import json +import sys +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta + +from keel.mesh import invites, protocol +from keel.mesh.node import Node, NodeError +from keel.mesh.protocol import ProtocolError +from keel.network import marker, session +from keel.network.wireguard import same_key + +# how a refusal is counted by the listener (keel.mesh.listener) +SHAPE = "shape" +HMAC = "hmac" +MAX_FORGED = 5 +# how long a confirmation waits for the new peer's handshake to show +HANDSHAKE_WAIT = 10 +HANDSHAKE_POLL = 0.5 + + +@dataclass(frozen=True) +class Response: + """`final`: the invite is done with, and the listener stops""" + + status: int + body: bytes + signature: str | None = None + final: bool = False + + +@dataclass(frozen=True) +class Joined: + """The join that was accepted: who, where, the change, and until when + its confirmation is waited for""" + + public_key: str + address: str + made: marker.Pending | None + until: datetime + # when it was admitted, in seconds since the epoch: a handshake from + # its key at or after this was made over the new tunnel + since: int = 0 + + +class Refusal(Exception): + def __init__(self, status: int, reason: str, kind: str = SHAPE): + super().__init__(reason) + self.status = status + self.reason = reason + self.kind = kind + + +def refusal(status: int, reason: str) -> Response: + return Response(status, json.dumps({"error": reason}).encode()) + + +def parsed(invite_id: str, path: str, body: bytes | None, now: datetime, + parse: Callable[[bytes], object]): + """The request, if it is this invite's and fresh: what the listener + can check without the HMAC key. Raises Refusal""" + if body is None: + raise Refusal(413, "the request is longer than any join") + try: + request = parse(body) + except ProtocolError as e: + raise Refusal(400, f"malformed request: {e}") from None + if request.invite_id != invite_id: + raise Refusal(403, f"invite {request.invite_id} is not the one" + " this listener serves") + if not protocol.fresh(request.time, now): + raise Refusal(403, "the request's time is more than" + f" {protocol.SKEW} from this node's clock") + return request + + +def verify(invite_id: str, key: bytes, path: str, signature: str | None, + body: bytes | None, now: datetime, + parse: Callable[[bytes], object]): + """The request, if it is this invite's, fresh, and signed with its + key. Raises Refusal; one of kind HMAC is a request that names this + invite and is not signed with its key""" + request = parsed(invite_id, path, body, now, parse) + if not protocol.signed(key, protocol.METHOD, path, body, signature): + raise Refusal(403, "bad HMAC: the request is not signed with this" + " invite's key", HMAC) + return request + + +def parser(path: str) -> Callable[[bytes], object]: + return (protocol.join_request if path == protocol.JOIN + else protocol.confirm_request) + + +def default_log(line: str) -> None: + print(line, file=sys.stderr, flush=True) + + +class Admitter: + """One invite, on the root side; `clock` gives an aware UTC datetime""" + + def __init__(self, root: str, invite: invites.Pending, public_key: str, + address: str, node: Node, clock: Callable[[], datetime], + log: Callable[[str], None] | None = None, + joined: Joined | None = None, + sleep: Callable[[float], None] = time.sleep): + self.root = root + self.invite = invite + self.public_key = public_key + self.address = address + self.node = node + self.clock = clock + self.log = log or default_log + self.joined = joined + self.sleep = sleep + self.nonces: set[str] = set() + self.forged = 0 + self.confirmed: bool | None = None + self.cancelled = False + + @property + def name(self) -> str: + return f"invite {self.invite.invite_id}" + + def forward(self, path: str, signature: str | None, body: bytes | None, + local: str, peer: str) -> Response: + """The answer to one request; never raises + + `local` and `peer` are what the listener reports: for the log, + never for a decision. + """ + try: + if self.cancelled: + raise Refusal(410, f"{self.name} was cancelled") + if path == protocol.JOIN: + return self.join(signature, body, peer) + if path == protocol.CONFIRM: + return self.confirm(signature, body) + raise Refusal(404, f"{path[:64]!r}: no such request") + except Refusal as e: + self.log(f"{self.name}: refused a request from {peer}:" + f" {e.reason}") + if e.kind == HMAC: + self.forged += 1 + if self.forged >= MAX_FORGED and self.joined is None: + self.cancel(f"{MAX_FORGED} requests named it and" + " failed its HMAC") + found = refusal(e.status, e.reason) + return Response(found.status, found.body, + final=self.cancelled or self.confirmed is False) + + def cancel(self, reason: str) -> None: + """The invite is given up on: it goes""" + invites.remove(self.root, self.invite.invite_id) + self.cancelled = True + self.log(f"{self.name}: cancelled, {reason}; run keel mesh invite" + " again") + + def checked(self, path: str, signature: str | None, body: bytes | None): + request = verify(self.invite.invite_id, self.invite.hmac_key, path, + signature, body, self.clock(), parser(path)) + if request.nonce in self.nonces: + raise Refusal(403, "replayed request: its nonce was seen") + self.nonces.add(request.nonce) + return request + + def join(self, signature: str | None, body: bytes | None, + peer: str) -> Response: + if self.joined is not None: + raise Refusal(410, f"{self.name} was already used") + request = self.checked(protocol.JOIN, signature, body) + if request.address != self.invite.address: + raise Refusal(403, f"{request.address} is not the address this" + " invite reserved") + if self.known(request.public_key): + raise Refusal(409, "that key is already this node's or one of" + " its peers': a new node joins with a key of its" + " own") + if self.node.waiting(): + self.log(f"{self.name}: a network change waits for its" + " confirmation here; the invite stays") + return refusal(409, "another network change waits for its" + " confirmation on the inviter; try again once" + " it is confirmed or reverted") + try: + invites.consume(self.root, self.invite.invite_id, self.clock()) + except invites.InviteError as e: + raise Refusal(410, str(e)) from None + return self.admitted(request, peer) + + def known(self, key: str) -> bool: + """This node's own key, or a peer's of its spec""" + if same_key(key, self.public_key): + return True + return any(same_key(str(one.get("public_key")), key) + for one in self.node.overlay().get("peers") or []) + + def admitted(self, request: protocol.JoinRequest, peer: str) -> Response: + """The invite is spent: the new node becomes a peer, applied""" + address = str(ipaddress.IPv6Interface(request.address).ip) + self.log(f"{self.name}: join from {peer}, key {request.public_key}," + f" endpoint {request.endpoint or 'none'}, at {address}") + entry = {"public_key": request.public_key, + "allowed_ips": [f"{address}/128"]} + since = int(self.clock().timestamp()) + if request.endpoint: + entry["endpoint"] = request.endpoint + try: + change = self.node.admit(entry) + problem = None if change.made else ( + f"this node's change did not come up (apply exited" + f" {change.code})") + except NodeError as e: + problem = str(e) + if problem: + self.confirmed = False + self.log(f"{self.name}: {problem}; the invite is spent") + found = refusal(500, "the inviter could not apply the new peer;" + " ask it for a new invite") + return Response(found.status, found.body, final=True) + window = change.made.window + self.joined = Joined(request.public_key, address, change.made, + self.clock() + timedelta(seconds=window), since) + answer = protocol.JoinAnswer( + invite_id=self.invite.invite_id, nonce=request.nonce, + public_key=self.public_key, address=self.address, + peers=self.node.peers(request.public_key), etcd="none", + window=window) + self.log(f"{self.name}: peer applied; waiting {window} s for the" + " new node over the overlay") + return self.signed(protocol.JOIN, protocol.dumps(answer)) + + def confirm(self, signature: str | None, + body: bytes | None) -> Response: + if self.joined is None: + raise Refusal(409, "no join to confirm yet") + request = self.checked(protocol.CONFIRM, signature, body) + if request.public_key != self.joined.public_key: + raise Refusal(403, "the confirmation is not the new node's") + seen = self.handshake() + if seen is None: + raise Refusal(409, f"no WireGuard handshake from" + f" {self.joined.public_key} since it was admitted:" + " the tunnel is not up yet") + own = str(ipaddress.IPv6Interface(self.address).ip) + origin = session.Origin( + session.MESH, f"a WireGuard handshake from the new peer's key" + f" ({seen}) and the confirmation of {self.name}", None, own, + self.joined.address) + confirmed, lines = self.node.confirm(origin, self.joined.made) + for line in lines: + self.log(f"{self.name}: {line}") + self.confirmed = confirmed + answer = protocol.ConfirmAnswer( + invite_id=self.invite.invite_id, nonce=request.nonce, + confirmed=confirmed, detail="; ".join(lines)) + found = self.signed(protocol.CONFIRM, protocol.dumps(answer)) + return Response(found.status, found.body, found.signature, True) + + def handshake(self) -> str | None: + """When the new peer's key last completed a handshake with this + node, if it did since it was admitted; waited for a little, as the + request that comes over the tunnel can come right after it""" + waited = 0.0 + while True: + seen = self.node.handshake(self.joined.public_key) + if seen is not None and seen >= self.joined.since: + return datetime.fromtimestamp( + seen, self.clock().tzinfo).strftime("%H:%M:%S UTC") + if waited >= HANDSHAKE_WAIT: + return None + self.sleep(HANDSHAKE_POLL) + waited += HANDSHAKE_POLL + + def signed(self, path: str, body: bytes) -> Response: + return Response(200, body, protocol.sign( + self.invite.hmac_key, protocol.ANSWER, path, body)) + + +def stopped(admitter: Admitter) -> str: + """How the invite ended, in one line for the log""" + if admitter.cancelled: + return f"{admitter.name}: cancelled" + if admitter.confirmed: + return f"{admitter.name}: joined and confirmed" + if admitter.joined is not None or admitter.confirmed is False: + return (f"{admitter.name}: joined, not confirmed; its change" + " reverts when its window ends") + return f"{admitter.name}: expired unused" diff --git a/keel/mesh/bridge.py b/keel/mesh/bridge.py new file mode 100644 index 0000000..c589def --- /dev/null +++ b/keel/mesh/bridge.py @@ -0,0 +1,361 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""Between the unprivileged listener and the root side (decision 0048) + +The root helper (`keel mesh serve` in keel-mesh-invite@, or `keel +mesh accept` in the foreground) starts the listener as its own unit, +keel-mesh-listen@, with a dynamic user, no capability and the +sandbox of LISTENER_PROPERTIES. The listener binds the bridge's unix +socket in its RuntimeDirectory, /run/keel/mesh-listen-, mode 0700 +and its dynamic user's, so no other user can reach it; the socket is +0600. The helper connects, and checks with SO_PEERCRED that the other +end is the unit's MainPID, as systemd names it, and not root; a socket +held by anything else is skipped, and the helper tries again until the +listener shows. It then sends the listener's parameters, which hold no +key, and the invite's certificate and TLS key as two memfds over +SCM_RIGHTS, so the TLS key is never written to a file. The invite's +HMAC key never leaves the root side. The listener accepts the helper +only as root (or as itself, in a test). + +From then on the listener forwards each request that passed its checks, +as the bytes it received, and the helper answers with the Admitter's +response (keel.mesh.admit), which verifies the HMAC and decides. +Messages are JSON, one per line, at most MAX_MESSAGE bytes; bodies in +base64. The helper treats everything it reads as untrusted: a message +it cannot read ends the exchange. +""" + +import base64 +import binascii +import ipaddress +import json +import os +import socket +import struct +import sys +import time +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from datetime import datetime, timezone + +from keel import exits +from keel.mesh.admit import Admitter, Response +from keel.mesh.channel import context_from_paths, pem_fds +from keel.mesh.listener import Listener, Params, bound, run + +UNIT = "keel-mesh-listen@{id}" +# the listener's RuntimeDirectory, under /run +RUNTIME = "keel/mesh-listen-{id}" +SOCKET = "bridge.sock" +MAX_MESSAGE = 65536 +CONNECT_WAIT = 30 +RETRY = 0.05 +PEERCRED = struct.Struct("3i") +STATUS = "/proc/self/status" +# the listener's unit: a dynamic user, no capability, nothing writable, +# IP and unix sockets only, the system calls of a service +LISTENER_PROPERTIES = ( + "DynamicUser=yes", + "NoNewPrivileges=yes", + "PrivateTmp=yes", + "PrivateDevices=yes", + "ProtectSystem=strict", + "ProtectHome=yes", + "ProtectKernelTunables=yes", + "ProtectKernelModules=yes", + "ProtectKernelLogs=yes", + "ProtectControlGroups=yes", + "ProtectClock=yes", + "ProtectHostname=yes", + "RestrictNamespaces=yes", + "RestrictRealtime=yes", + "RestrictSUIDSGID=yes", + "LockPersonality=yes", + "RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX", + "SystemCallFilter=@system-service", + "SystemCallArchitectures=native", + "CapabilityBoundingSet=", + "AmbientCapabilities=", + "MemoryMax=64M", + "TasksMax=32", + "LimitNOFILE=64", +) + + +class BridgeError(Exception): + """The listener could not be started, trusted, or understood""" + + +def unit(invite_id: str) -> str: + return UNIT.format(id=invite_id) + + +def socket_path(root: str, invite_id: str) -> str: + return os.path.join(root, "run", RUNTIME.format(id=invite_id), SOCKET) + + +def listener_command(path: str) -> tuple[str, ...]: + return (sys.executable, "-m", "keel", "mesh", "listen", path) + + +@dataclass +class Unit: + """A listener started as its unit""" + + name: str + run: Callable[[tuple[str, ...]], str | None] + output: Callable[[tuple[str, ...]], str | None] + + def trusted(self, pid: int, uid: int) -> bool: + """Not root, and the unit's main process as systemd knows it""" + found = (self.output(("systemctl", "show", "--property=MainPID", + "--value", self.name)) or "").strip() + return uid != 0 and found.isdigit() and int(found) == pid + + def stop(self) -> None: + self.run(("systemctl", "stop", self.name)) + + +def start_unit(invite_id: str, path: str, seconds: int, + run: Callable[[tuple[str, ...]], str | None], + output: Callable[[tuple[str, ...]], str | None]) -> Unit: + """Start keel-mesh-listen@; raises BridgeError""" + name = unit(invite_id) + problem = run(("systemd-run", f"--unit={name}", "--collect", "--quiet", + f"--property=RuntimeMaxSec={seconds}", + f"--property=RuntimeDirectory={RUNTIME.format(id=invite_id)}", + "--property=RuntimeDirectoryMode=0700", + *(f"--property={one}" for one in LISTENER_PROPERTIES), + f"--description=keel: the unprivileged listener of mesh" + f" invite {invite_id}", *listener_command(path))) + if problem: + raise BridgeError(f"the listener could not start: {problem}") + return Unit(name, run, output) + + +def params_json(params: Params) -> bytes: + data = {"invite_id": params.invite_id, + "expires": int(params.expires.timestamp()), + "host": params.host, "port": params.port, + "overlay": params.overlay, + "joined": None if params.joined is None else [ + params.joined[0], params.joined[1], + int(params.joined[2].timestamp())]} + return json.dumps(data).encode() + + +def params_of(data: bytes) -> Params: + found = json.loads(data.decode()) + joined = found["joined"] + return Params( + invite_id=str(found["invite_id"]), + expires=datetime.fromtimestamp(found["expires"], timezone.utc), + host=str(found["host"]), port=int(found["port"]), + overlay=str(found["overlay"]), + joined=None if joined is None else ( + str(joined[0]), str(joined[1]), + datetime.fromtimestamp(joined[2], timezone.utc))) + + +def line(sock: socket.socket) -> bytes | None: + """One message, None at the end; raises BridgeError past the cap""" + found = bytearray() + while not found.endswith(b"\n"): + chunk = sock.recv(1) + if not chunk: + return None + found += chunk + if len(found) > MAX_MESSAGE: + raise BridgeError("a message longer than any the bridge has") + return bytes(found) + + +def send(sock: socket.socket, data: dict) -> None: + sock.sendall(json.dumps(data).encode() + b"\n") + + +def address(value: object) -> str: + return str(ipaddress.ip_address(str(value))) + + +def answered(admitter: Admitter, data: bytes) -> dict: + """The helper's answer to one message; raises BridgeError""" + try: + found = json.loads(data.decode()) + if found.get("op") != "forward": + raise ValueError(f"unknown operation {found.get('op')!r}") + body = found.get("body") + signature = found.get("signature") + response = admitter.forward( + str(found["path"]), + signature if isinstance(signature, str) else None, + None if body is None else base64.b64decode(body, validate=True), + address(found["local"]), address(found["peer"])) + except (ValueError, KeyError, TypeError, AttributeError, + binascii.Error) as e: + raise BridgeError(f"the listener sent what it should not: {e}") \ + from None + return {"status": response.status, + "body": base64.b64encode(response.body).decode(), + "signature": response.signature, "final": response.final} + + +@dataclass +class Bridge: + """The root helper of one invite + + `start(path)` starts the listener, which binds `path`, and returns + what can tell its process (trusted(pid, uid)) and stop it (stop()). + """ + + admitter: Admitter + params: Params + certificate: str + tls_key: str + path: str + start: Callable[[str], object] + log: Callable[[str], None] + sleep: Callable[[float], None] = time.sleep + + def run(self) -> None: + """Until the listener is done; raises BridgeError""" + started = self.start(self.path) + try: + with self.connected(started) as conn: + self.exchange(conn) + finally: + started.stop() + + def connected(self, started) -> socket.socket: + """The listener's socket, its peer checked; untrusted ones are + skipped until the listener shows or CONNECT_WAIT passes""" + deadline = time.monotonic() + CONNECT_WAIT + while time.monotonic() < deadline: + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.connect(self.path) + except OSError: + conn.close() + self.sleep(RETRY) + continue + pid, uid, _ = PEERCRED.unpack(conn.getsockopt( + socket.SOL_SOCKET, socket.SO_PEERCRED, PEERCRED.size)) + if started.trusted(pid, uid): + self.log(f"invite {self.params.invite_id}: the listener" + f" (pid {pid}, uid {uid}) serves TCP port" + f" {self.params.port}") + return conn + conn.close() + self.log(f"invite {self.params.invite_id}: skipped process" + f" {pid} (uid {uid}) at {self.path}: it is not the" + " listener") + self.sleep(RETRY) + raise BridgeError(f"the listener did not connect within" + f" {CONNECT_WAIT} s") + + def exchange(self, conn: socket.socket) -> None: + with pem_fds(self.certificate, self.tls_key) as fds: + socket.send_fds(conn, [params_json(self.params) + b"\n"], fds) + while True: + data = line(conn) + if data is None: + return + send(conn, answered(self.admitter, data)) + + +class Remote: + """The listener's backend: the root side, over the bridge""" + + def __init__(self, sock: socket.socket): + self.sock = sock + + def ask(self, data: dict) -> dict: + try: + send(self.sock, data) + found = line(self.sock) + except OSError as e: + raise BridgeError(f"the root side is gone: {e}") from None + if found is None: + raise BridgeError("the root side is gone") + return json.loads(found.decode()) + + def forward(self, path: str, signature: str | None, body: bytes | None, + local: str, peer: str) -> Response: + try: + found = self.ask({ + "op": "forward", "path": path, "signature": signature, + "body": None if body is None else base64.b64encode( + body).decode(), "local": local, "peer": peer}) + except BridgeError as e: + return Response(503, json.dumps({"error": str(e)}).encode()) + return Response(int(found["status"]), + base64.b64decode(found["body"]), + found.get("signature"), bool(found.get("final"))) + + +def capabilities(text: str) -> int: + """CapEff of /proc/PID/status, as an integer""" + for one in text.splitlines(): + if one.startswith("CapEff:"): + return int(one.split()[1], 16) + return -1 + + +def helper(path: str, own_uid: int) -> socket.socket | None: + """The root helper's connection to the socket the listener binds at + `path`, 0600 in its 0700 directory; None when none came. A peer that + is neither root nor this user is dropped""" + os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True) + with suppress(FileNotFoundError): + os.remove(path) + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: + server.bind(path) + os.chmod(path, 0o600) + server.listen(1) + server.settimeout(CONNECT_WAIT) + while True: + try: + conn, _ = server.accept() + except TimeoutError: + return None + _, uid, _ = PEERCRED.unpack(conn.getsockopt( + socket.SOL_SOCKET, socket.SO_PEERCRED, PEERCRED.size)) + if uid in (0, own_uid): + # one helper: nothing else will connect + os.remove(path) + return conn + conn.close() + + +def listen(path: str, clock: Callable[[], datetime], + log: Callable[[str], None], status: str = STATUS, + poll: float = 1.0) -> int: + """`keel mesh listen`: the unprivileged listener's whole life""" + with open(status) as fob: + found = capabilities(fob.read()) + if found != 0: + log(f"the mesh listener runs without capabilities, and this" + f" process has {found:#x}: refused") + return exits.APPLY_FAILED + sock = helper(path, os.getuid()) + if sock is None: + log(f"no root helper came to {path} within {CONNECT_WAIT} s") + return exits.APPLY_FAILED + with sock: + message, fds, _, _ = socket.recv_fds(sock, MAX_MESSAGE, 2) + try: + params = params_of(message) + context = context_from_paths(*(f"/proc/self/fd/{one}" + for one in fds)) + except (ValueError, KeyError, TypeError, OSError) as e: + log(f"the root side sent no parameters the listener can use" + f" ({e or 'nothing'}): refused") + return exits.APPLY_FAILED + finally: + for one in fds: + os.close(one) + listener = Listener(params, Remote(sock), clock, log) + log(f"invite {params.invite_id}: listening on TCP port" + f" {params.port}, without capabilities") + run(bound(params.host, params.port), listener, context, poll) + return exits.OK diff --git a/keel/mesh/channel.py b/keel/mesh/channel.py new file mode 100644 index 0000000..b64f533 --- /dev/null +++ b/keel/mesh/channel.py @@ -0,0 +1,180 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""HTTPS between the new node and the inviter, pinned (decision 0048) + +The inviter serves with the invite's own key and self signed +certificate; the new node trusts no authority and compares the SHA-256 +of the certificate the inviter presents with the fingerprint in the +token, before it sends a byte of the request. Both ends speak TLS 1.3. + +The invite's TLS key lives in the invite's file, not on disk as a PEM: +ssl loads a key from a path only, so it is handed over through a +memfd, an anonymous file that never has a name in any filesystem and +goes when it is closed. +""" + +import hashlib +import hmac +import http.client +import ipaddress +import json +import os +import socket +import ssl +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass + +from keel.mesh import protocol + +# a TCP answer within this many seconds, or the fallback (0048): two +# lost SYNs on a poor link (1 s, then 2 s more, then 4 s) still connect +CONNECT_TIMEOUT = 10 +# the inviter applies its change before it answers a join +ANSWER_TIMEOUT = 180 + + +class ChannelError(Exception): + """The exchange did not complete, and why""" + + +class Unreachable(ChannelError): + """No TCP answer: the port is closed, filtered, or nothing listens""" + + +class Refused(ChannelError): + """The inviter answered with a refusal; the reason it gave""" + + +class Forged(ChannelError): + """The certificate is not the pinned one, or the answer is not + signed with the invite's key""" + + +@dataclass(frozen=True) +class Reply: + """A signed answer, and the addresses the connection used""" + + body: bytes + local: str + peer: str + + +def plain(address: str) -> str: + """A socket's address as keel compares it: an IPv4-mapped one as IPv4""" + value = ipaddress.ip_address(address.split("%", 1)[0]) + if isinstance(value, ipaddress.IPv6Address) and value.ipv4_mapped: + return str(value.ipv4_mapped) + return str(value) + + +def fingerprint(der: bytes) -> bytes: + return hashlib.sha256(der).digest() + + +@contextmanager +def pem_fds(*texts: str) -> Iterator[list[int]]: + """memfds that hold `texts`, closed afterwards""" + fds = [] + try: + for text in texts: + fd = os.memfd_create("keel-mesh-invite", os.MFD_CLOEXEC) + fds.append(fd) + os.write(fd, text.encode()) + yield fds + finally: + for fd in fds: + os.close(fd) + + +def context_from_paths(cert_path: str, key_path: str) -> ssl.SSLContext: + """The listener's context, from paths that read as the PEMs""" + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_3 + context.load_cert_chain(cert_path, key_path) + return context + + +def server_context(certificate: str, key: str) -> ssl.SSLContext: + """The listener's context, with the invite's certificate and key""" + with pem_fds(certificate, key) as fds: + return context_from_paths(*(f"/proc/self/fd/{fd}" for fd in fds)) + + +def client_context() -> ssl.SSLContext: + """No authority is trusted: the certificate is pinned instead""" + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.minimum_version = ssl.TLSVersion.TLSv1_3 + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + +def shown(host: str, port: int) -> str: + return f"[{host}]:{port}" if ":" in host else f"{host}:{port}" + + +def post(host: str, port: int, path: str, body: bytes, key: bytes, + pinned: bytes, connect_timeout: float = CONNECT_TIMEOUT, + answer_timeout: float = ANSWER_TIMEOUT) -> Reply: + """POST a signed request; the answer, its signature checked + + Raises Unreachable without a TCP answer, Forged for another + certificate or an answer not signed with `key`, Refused for a + refusal, and ChannelError for an exchange cut short. + """ + where = shown(host, port) + try: + raw = socket.create_connection((host, port), timeout=connect_timeout) + except OSError as e: + raise Unreachable(f"{where}: {e.strerror or e}") from None + raw.settimeout(answer_timeout) + try: + tls = client_context().wrap_socket(raw) + except OSError as e: + raw.close() + raise Forged(f"{where}: the TLS handshake failed ({e}): it is not" + " the inviter's listener") from None + with tls: + der = tls.getpeercert(binary_form=True) + if not hmac.compare_digest(fingerprint(der), pinned): + raise Forged(f"{where} presented a certificate that is not the" + " one the token pins: nothing was sent") + local, peer = plain(tls.getsockname()[0]), plain(tls.getpeername()[0]) + status, signature, data = exchange(tls, host, port, path, body, key) + if status != 200: + raise Refused(reason(data, status)) + if not protocol.signed(key, protocol.ANSWER, path, data, signature): + raise Forged(f"the answer from {where} is not signed with this" + " invite's key") + return Reply(data, local, peer) + + +def exchange(tls: ssl.SSLSocket, host: str, port: int, path: str, + body: bytes, key: bytes) -> tuple[int, str | None, bytes]: + connection = http.client.HTTPConnection(host, port) + connection.sock = tls + headers = {"Content-Type": protocol.CONTENT_TYPE, "Connection": "close", + protocol.SIGNATURE: protocol.sign(key, protocol.METHOD, path, + body)} + try: + connection.request(protocol.METHOD, path, body, headers) + response = connection.getresponse() + data = response.read(protocol.MAX_BODY + 1) + except (OSError, http.client.HTTPException) as e: + raise ChannelError(f"{shown(host, port)} did not answer: {e}") \ + from None + if len(data) > protocol.MAX_BODY: + raise ChannelError(f"the answer from {shown(host, port)} is longer" + " than any the protocol has") + return response.status, response.getheader(protocol.SIGNATURE), data + + +def reason(data: bytes, status: int) -> str: + """What a refusal says, or its status when it says nothing readable""" + try: + found = json.loads(data.decode()).get("error") + except (UnicodeDecodeError, ValueError, AttributeError): + found = None + if isinstance(found, str) and found.isprintable(): + return found[:300] + return f"HTTP {status}" diff --git a/keel/mesh/commands.py b/keel/mesh/commands.py index a41441c..f4c935a 100644 --- a/keel/mesh/commands.py +++ b/keel/mesh/commands.py @@ -1,27 +1,50 @@ # Copyright (c) 2026 KeelLinux maintainers -"""keel mesh invite and keel mesh join --dry-run (decision 0048) - -`invite` prints the one line a new node runs, `keel mesh join -keel1:`, alone on standard output, and reserves the new node's -address in a pending invite. No listener answers it yet: the token can -be read with `join --dry-run`, which prints the change of the new node's -spec it makes and applies nothing. Like every command, nothing here -prompts, so confconsole calls the same functions. +"""keel mesh: create, invite, join, accept, status (decision 0048) + +`invite` reserves the new node's address in a pending invite, starts +its root helper as a transient unit, and prints the one line a new node +runs, `keel mesh join - <<< keel1:`, alone on standard output. +`join` runs the join (keel.mesh.joining), or with `--dry-run` prints the +change of the spec it would make. `accept` is the fallback's half on +the inviter (keel.mesh.inviting), `create` makes the mesh on the first +node (keel.mesh.create), `serve` is what the root helper's unit runs, +`listen` what the unprivileged listener's runs (keel.mesh.bridge), and +`status` shows the peers and the pending invites. Like every command, +nothing here prompts, so confconsole calls the same commands. """ import ipaddress import os import secrets +import signal import sys from dataclasses import replace from datetime import datetime, timezone import yaml -from keel import exits, spec +from keel import exits, spec, system from keel.commands import error, manifest_facts, read_spec from keel.inspect import ROOT_DEFAULT -from keel.mesh import allocate, certificate, identity, invites, join +from keel.mesh import ( + allocate, + bridge, + certificate, + create, + identity, + invites, + inviting, + join, + joining, + ports, + status, +) +from keel.mesh.node import ( + Node, + NodeError, + global_addresses, + static_addresses, +) from keel.mesh.token import ( LIFETIME, MAX_TEXT, @@ -34,7 +57,7 @@ parse, shown, ) -from keel.network import wgkeys, wireguard +from keel.network import live, session, wgkeys, wireguard from keel.system.ovstate import overlay_of STDIN = "-" @@ -78,11 +101,37 @@ def mesh_invite(args) -> int: except (allocate.AllocationError, TokenError, invites.InviteError) as e: error(str(e)) return exits.MESH_REFUSED - print(f"keel mesh join {line}") - report(made) + opened, code = listening(made, args.spec, root, now) + if code != exits.OK: + return code + print(f"keel mesh join - <<< {line}") + report(made, opened) return exits.OK +def listening(made: invites.Pending, path: str, root: str, + now: datetime) -> tuple[str, int]: + """Start the invite's listener and open its port; what was done + + Off the live system nothing is started. An invite whose listener + cannot start is removed: a line nothing answers is not printed. + """ + if root != ROOT_DEFAULT: + return ("not the live system: no listener started, so the line" + " can be checked with keel mesh join --dry-run only"), \ + exits.OK + problem = inviting.start(made, os.path.abspath(path), now, live.run) + if problem: + invites.remove(root, made.invite_id) + error(f"the invite's listener could not start ({problem}); nothing" + " is reserved") + return "", exits.APPLY_FAILED + # as long as the listener may wait for a join's confirmation + seconds = int((made.expires - now).total_seconds()) + inviting.LINGER + return ports.open_port(made.https_port, seconds, live.run, + live.output), exits.OK + + def invite(root: str, now: datetime, draft: Token, overlay: dict, tls: tuple[str, str]) -> tuple[invites.Pending, str]: """The pending invite, reserved, and its token @@ -93,6 +142,12 @@ def invite(root: str, now: datetime, draft: Token, overlay: dict, lines: list[str] = [] def make(others: tuple[invites.Pending, ...]) -> invites.Pending: + for one in others: + if one.https_port == draft.https_port: + raise invites.InviteError( + f"invite {one.invite_id} is pending on TCP port" + f" {one.https_port} until {shown(one.expires)}: give" + " this one another with --port, or wait") assigned = allocate.free_address( draft.address, allocate.taken(overlay) + [ @@ -109,15 +164,16 @@ def make(others: tuple[invites.Pending, ...]) -> invites.Pending: return made, lines[0] -def report(made: invites.Pending) -> None: +def report(made: invites.Pending, opened: str) -> None: """What the invite did, on standard error: never the token""" print("on the new node, keel mesh join - reads the token from standard" " input, out of the process list and the shell's history", file=sys.stderr) print(f"invite {made.invite_id}: {made.address} reserved for the new" - f" node until {shown(made.expires)}", file=sys.stderr) - print("no listener answers it in this keel yet: the line can be checked" - " with keel mesh join --dry-run on the new node", file=sys.stderr) + f" node until {shown(made.expires)}, the listener on TCP port" + f" {made.https_port} ({inviting.unit(made.invite_id)})", + file=sys.stderr) + print(opened, file=sys.stderr) def inviter_spec(path: str, root: str) -> tuple[dict | None, int]: @@ -132,8 +188,7 @@ def inviter_spec(path: str, root: str) -> tuple[dict | None, int]: overlay = overlay_of(doc) if overlay is None or not overlay.get("address"): error(f"{path} declares no network.overlay.wireguard: this node" - " is in no mesh to invite into (keel mesh create comes in a" - " later keel; docs/mesh.md)") + " is in no mesh to invite into; keel mesh create makes one") return None, exits.MESH_REFUSED return doc, exits.OK @@ -182,29 +237,59 @@ def endpoints(declared: list[str] | None, doc: dict) -> tuple[str, ...]: if version in found) -def static_addresses(doc: dict): - interfaces = (doc.get("network") or {}).get("interfaces") or {} - for iface in interfaces.values(): - for family in ("ipv6", "ipv4"): - block = (iface or {}).get(family) or {} - if block.get("method") == "static" and block.get("address"): - yield ipaddress.ip_interface(str(block["address"])).ip +ARGV_WARNING = ( + "the token is an argument, so it shows in the process list and the" + " shell's history: keel mesh join - <<< TOKEN, the line keel mesh" + " invite prints, reads it from standard input") -def mesh_join(args) -> int: - """Print the change a token makes to this node's spec; apply nothing""" - if not args.dry_run: - error("keel mesh join applies nothing in this keel yet: with" - " --dry-run it prints the change it will make") - return exits.NOT_IMPLEMENTED +def read_token(args) -> tuple[Token | None, int]: # never more than a token can be: parse refuses the excess unread - text = sys.stdin.read(MAX_TEXT + 1) if args.token == STDIN \ - else args.token + if args.token == STDIN: + text = sys.stdin.read(MAX_TEXT + 1) + else: + text = args.token + err(f"Warning: {ARGV_WARNING}") try: - token = parse(text, datetime.now(timezone.utc)) + return parse(text, datetime.now(timezone.utc)), exits.OK except TokenError as e: error(str(e)) - return exits.MESH_TOKEN_INVALID + return None, exits.MESH_TOKEN_INVALID + + +def mesh_join(args) -> int: + """Join the mesh the token names; with --dry-run, print the change""" + if args.dry_run: + return join_dry_run(args) + refusal = system.needs_root(os.path.abspath(args.root), "keel mesh join") + if refusal: + error(refusal) + return exits.APPLY_NEEDS_ROOT + token, code = read_token(args) + if token is None: + return code + node = live_node(args) + try: + doc = node.document() + endpoint = joining.own_endpoint( + args.endpoint, doc, + global_addresses(live.output(GLOBAL_ADDRESSES) or ""), + wireguard.interface(overlay_of(doc) or {})) + except (NodeError, ValueError) as e: + error(str(e)) + return exits.MESH_REFUSED + return joining.run(joining.Joiner(node, token, utcnow, out, err), + endpoint) + + +GLOBAL_ADDRESSES = ("ip", "-o", "address", "show", "scope", "global") + + +def join_dry_run(args) -> int: + """Print the change a token makes to this node's spec; apply nothing""" + token, code = read_token(args) + if token is None: + return code doc: dict = {"version": 1} if os.path.exists(args.spec): doc, code = read_spec(args.spec, False, args.root) @@ -229,3 +314,88 @@ def mesh_join(args) -> int: f" (invite {token.invite_id}, valid until {shown(token.expires)}," f" etcd: {token.etcd})", file=sys.stderr) return exits.OK + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def out(line: str) -> None: + print(line, flush=True) + + +def err(line: str) -> None: + print(line, file=sys.stderr, flush=True) + + +def operator_clients() -> tuple[str, ...]: + """The client of the SSH session this command runs in, if any: the + route check asks that it still leaves through the uplink""" + origin = session.origin("/proc", os.getpid(), live.sockets) + if origin.kind == session.SSH and origin.peer: + return (origin.peer,) + return () + + +def live_node(args) -> Node: + return Node(os.path.abspath(args.root), args.spec, + window=getattr(args, "network_window", system.DEFAULT_WINDOW), + clients=operator_clients()) + + +def as_root(args, verb: str) -> int: + refusal = system.needs_root(os.path.abspath(args.root), verb) + if refusal: + error(refusal) + return exits.APPLY_NEEDS_ROOT + return exits.OK + + +def mesh_create(args) -> int: + """Make the mesh on its first node""" + code = as_root(args, "keel mesh create") + if code != exits.OK: + return code + return create.create(live_node(args), out, err) + + +def inviter(args) -> inviting.Inviter: + return inviting.Inviter(live_node(args), utcnow, out, err) + + +def mesh_serve(args) -> int: + """What keel-mesh-invite@ runs: the invite's listener""" + code = as_root(args, "keel mesh serve") + if code != exits.OK: + return code + signal.signal(signal.SIGTERM, inviting.terminated) + return inviting.serve_invite(inviter(args), args.invite) + + +def mesh_accept(args) -> int: + """The fallback's line, run on the inviter""" + code = as_root(args, "keel mesh accept") + if code != exits.OK: + return code + text = sys.stdin.read(512 + 1) if args.line == STDIN else args.line + return inviting.accept(inviter(args), text) + + +def mesh_listen(args) -> int: + """What keel-mesh-listen@ runs: the unprivileged listener""" + return bridge.listen(args.socket, utcnow, err) + + +def mesh_status(args) -> int: + """The peers, their handshakes and the pending invites; no secret""" + root = os.path.abspath(args.root) + overlay = None + if os.path.exists(args.spec): + doc, code = read_spec(args.spec, False, root) + if doc is None: + return code + overlay = overlay_of(doc) + for line in status.lines(overlay, root, utcnow(), + live.output if root == ROOT_DEFAULT else None): + print(line) + return exits.OK diff --git a/keel/mesh/create.py b/keel/mesh/create.py new file mode 100644 index 0000000..a3dfc33 --- /dev/null +++ b/keel/mesh/create.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh create: the first node makes the mesh (decision 0048) + +A random unique local /64 with this node at ::1 (as `keel network +wireguard suggest-address` gives it), WireGuard's default port, and +the mesh's identity. The overlay is converged by apply under 0018's +window like any first overlay; apply makes the key pair on the machine +(keel-core#8). No peer can confirm an overlay that has none, so create +confirms it itself, once apply.md's route check finds every gateway, +and the operator's SSH client, still leaving through the uplink: an +overlay with no peer routes its own private prefix only (0048, second +round, point 1). +""" + +import secrets +from collections.abc import Callable + +from keel import exits +from keel.mesh import identity +from keel.mesh.node import Node, NodeError, with_overlay +from keel.network import session, wireguard +from keel.network.wireguard import GLOBAL_ID_BYTES +from keel.system.ovstate import overlay_of + + +def create(node: Node, out: Callable[[str], None], + err: Callable[[str], None]) -> int: + try: + doc = node.document() + except NodeError as e: + err(str(e)) + return exits.MESH_REFUSED + current = overlay_of(doc) or {} + if current.get("address"): + err(f"this node is already in a mesh, at {current['address']}:" + " keel mesh invite brings a new node into it") + return exits.MESH_REFUSED + address = wireguard.suggest_address( + secrets.token_bytes(GLOBAL_ID_BYTES)) + after = with_overlay(doc, {"address": address}) + try: + identity.ensure(node.root) + change = node.change(after) + except (NodeError, ValueError) as e: + err(str(e)) + return exits.MESH_REFUSED + if change.made is None: + err(f"the overlay did not come up under the window (apply exited" + f" {change.code}); the spec declares it, and the next apply" + " --system brings it up") + return exits.APPLY_FAILED + kept, lines = node.confirm( + session.Origin(session.SELF, "keel mesh create"), change.made) + for line in lines: + err(line) + if not kept: + return exits.NETWORK_NOT_CONFIRMED + overlay = overlay_of(after) + out(f"created the mesh: this node is {address} on" + f" {wireguard.interface(overlay)}, WireGuard on UDP" + f" {wireguard.port(overlay)}") + out("keel mesh invite prints the line that joins the next node") + return exits.OK diff --git a/keel/mesh/identity.py b/keel/mesh/identity.py index 6f0709b..77a9d55 100644 --- a/keel/mesh/identity.py +++ b/keel/mesh/identity.py @@ -33,6 +33,21 @@ def ensure(root: str) -> bytes: return found +def adopt(root: str, found: bytes) -> None: + """Keep the identity a token carries, on the node that joins + + Raises ValueError when this node already keeps another one: it is + in another mesh, and a node is in one. + """ + with locked(root): + current = read(root) + if current is None: + write_private(root, IDENTITY, found.hex() + "\n") + elif current != found: + raise ValueError(f"this node is in another mesh: /{IDENTITY}" + " names another one than the token's") + + def read(root: str) -> bytes | None: """The identity, None when not made yet; ValueError when damaged""" try: diff --git a/keel/mesh/inviting.py b/keel/mesh/inviting.py new file mode 100644 index 0000000..59700c8 --- /dev/null +++ b/keel/mesh/inviting.py @@ -0,0 +1,249 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The inviter's side: the listener's unit, and accept (decision 0048) + +`keel mesh invite` starts the invite's root helper as a transient unit, +`keel-mesh-invite@`, so the command returns and the invite closes +by itself even if nothing else runs again: the unit's RuntimeMaxSec is +the time left to the expiry, plus the window a join made just before +it needs to be confirmed in. The unit runs `keel mesh serve ` +(`serve` here), which starts the unprivileged listener on the invite's +TCP port, on every address, and answers what it forwards through +keel.mesh.bridge; when it ends, however it ends, it stops the listener +and removes the invite's file and its port from keel's firewall. + +`keel mesh accept keel1a:...` is the fallback's other half: the new node +could not reach the port, applied its own side, and printed the line. +accept checks the line's HMAC with the invite's key, consumes the +invite, adds the new node as a peer (with a keepalive, since this node +may be the one that has to initiate), applies, and then, in the +foreground, does what serve does, with the listener on its overlay +address alone, until the new node confirms or the window ends. +""" + +import ipaddress +import ssl +import sys +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timedelta + +from keel import exits +from keel.mesh import acceptline, bridge, invites, ports +from keel.mesh.admit import Admitter, Joined, stopped +from keel.mesh.bridge import Bridge, BridgeError, socket_path +from keel.mesh.listener import Params +from keel.mesh.node import Node, NodeError +from keel.network import live, wgkeys, wireguard +from keel.system import DEFAULT_WINDOW + +UNIT = "keel-mesh-invite@{id}" +# a join accepted just before the expiry is confirmed within its window +LINGER = DEFAULT_WINDOW + 60 +KEEPALIVE = 25 +EVERY_ADDRESS = "::" + + +def unit(invite_id: str) -> str: + return UNIT.format(id=invite_id) + + +def serve_command(invite_id: str, spec: str) -> tuple[str, ...]: + """What the unit runs: this interpreter, so no PATH is involved""" + return (sys.executable, "-m", "keel", "mesh", "serve", invite_id, + "--spec", spec) + + +def start(made: invites.Pending, spec: str, now: datetime, + run: Callable[[tuple[str, ...]], str | None]) -> str | None: + """Start the invite's listener; None, or why it could not be""" + seconds = int((made.expires - now).total_seconds()) + LINGER + return run(( + "systemd-run", f"--unit={unit(made.invite_id)}", "--collect", + "--quiet", f"--property=RuntimeMaxSec={seconds}", + f"--description=keel: the listener of mesh invite {made.invite_id}", + *serve_command(made.invite_id, spec))) + + +@dataclass +class Inviter: + """This node as the inviter; the outside world as fields""" + + node: Node + clock: Callable[[], datetime] + out: Callable[[str], None] + err: Callable[[str], None] + run: Callable[[tuple[str, ...]], str | None] = field(default=live.run) + # starts the listener: (invite id, socket path, seconds) -> what can + # tell its process and stop it (keel.mesh.bridge.Unit) + start: Callable[[str, str, int], object] | None = None + bridged: Callable[[Bridge], None] = Bridge.run + output: Callable[[tuple[str, ...]], str | None] = field( + default=live.output) + + @property + def root(self) -> str: + return self.node.root + + def own(self) -> tuple[str | None, str]: + """(public key, overlay address with its length); key None and + the reason when it cannot be read""" + overlay = self.node.overlay() + address = str(overlay.get("address") or "") + key = wireguard.key_path(overlay) + public, problem = wgkeys.public(f"{self.root.rstrip('/')}/" + f"{key.lstrip('/')}") + return public, (str(problem) if public is None else address) + + def starter(self, invite_id: str, seconds: int) -> Callable[[str], object]: + if self.start is not None: + return lambda path: self.start(invite_id, path, seconds) + return lambda path: bridge.start_unit(invite_id, path, seconds, + self.run, self.output) + + def bridge(self, admitter: Admitter, invite: invites.Pending, + params: Params) -> None: + """The listener with `params`, until it is done; raises + BridgeError""" + seconds = int((params.joined[2] if params.joined else + params.expires).timestamp() + - self.clock().timestamp()) + 1 + self.bridged(Bridge( + admitter, params, invite.certificate, invite.tls_key, + socket_path(self.root, invite.invite_id), + self.starter(invite.invite_id, max(seconds, 1)), self.err)) + + def close(self, invite: invites.Pending) -> None: + """The invite is done: its file and its port go""" + invites.remove(self.root, invite.invite_id) + others = [one for one in invites.pending(self.root, self.clock()) + if one.https_port == invite.https_port] + if not others: + ports.close_port(invite.https_port, self.run) + + +def terminated(signum, frame): + """systemctl stop and RuntimeMaxSec send SIGTERM: end through the + `finally` that removes the invite (keel.mesh.commands.mesh_serve + installs it)""" + raise SystemExit(0) + + +def serve_invite(inviter: Inviter, invite_id: str) -> int: + """`keel mesh serve`: what the unit runs, until the invite is done""" + found = invites.find(inviter.root, invite_id, inviter.clock()) + if found is None: + inviter.err(f"invite {invite_id}: not pending (used, cancelled or" + " expired); nothing to serve") + return exits.MESH_REFUSED + public, address = inviter.own() + if public is None: + inviter.err(address) + inviter.close(found) + return exits.APPLY_FAILED + admitter = Admitter(inviter.root, found, public, address, inviter.node, + inviter.clock, inviter.err) + own = str(ipaddress.IPv6Interface(address).ip) + inviter.err(f"invite {invite_id}: serving TCP port {found.https_port}" + f" until {found.expires:%H:%M:%S} UTC") + code = exits.OK + try: + inviter.bridge(admitter, found, Params( + found.invite_id, found.expires, EVERY_ADDRESS, + found.https_port, own)) + except (BridgeError, OSError, ssl.SSLError) as e: + inviter.err(f"invite {invite_id}: {e}") + code = exits.APPLY_FAILED + finally: + inviter.close(found) + inviter.err(stopped(admitter)) + return code + + +def accept(inviter: Inviter, text: str) -> int: + """`keel mesh accept keel1a:...`: the fallback, on the inviter""" + try: + sealed = acceptline.parse(text) + except acceptline.AcceptError as e: + inviter.err(str(e)) + return exits.MESH_TOKEN_INVALID + line = sealed.accept + now = inviter.clock() + found = invites.find(inviter.root, line.invite_id, now) + if found is None: + inviter.err(f"no pending invite {line.invite_id} on this node: it" + " expired, was used, or was made by another node") + return exits.MESH_REFUSED + if not sealed.authentic(found.hmac_key) or line.address != found.address: + inviter.err(f"the line is not the answer to invite {line.invite_id}:" + " its HMAC does not match the invite's key") + return exits.MESH_REFUSED + if inviter.node.waiting(): + inviter.err("a network change waits for its confirmation on this" + " node: confirm or revert it, then accept again") + return exits.MESH_REFUSED + public, address = inviter.own() + if public is None: + inviter.err(address) + return exits.APPLY_FAILED + try: + spent = invites.consume(inviter.root, line.invite_id, now) + except invites.InviteError as e: + inviter.err(str(e)) + return exits.MESH_REFUSED + # the invite's listener holds the port on every address: it goes + inviter.run(("systemctl", "stop", unit(line.invite_id))) + try: + return accepted(inviter, spent, line, public, address) + finally: + inviter.close(spent) + + +def accepted(inviter: Inviter, spent: invites.Pending, + line: acceptline.Accept, public: str, address: str) -> int: + joiner = str(ipaddress.IPv6Interface(line.address).ip) + peer = {"public_key": line.public_key, + "allowed_ips": [f"{joiner}/128"], + "persistent_keepalive": KEEPALIVE} + if line.endpoint: + peer["endpoint"] = line.endpoint + inviter.err(f"invite {spent.invite_id}: accepting key {line.public_key}," + f" endpoint {line.endpoint or 'none'}, at {joiner}") + if wireguard.same_key(line.public_key, public): + inviter.err("that key is this node's own: a new node joins with a" + " key of its own; the invite is spent") + return exits.MESH_REFUSED + since = int(inviter.clock().timestamp()) + try: + change = inviter.node.admit(peer) + except NodeError as e: + inviter.err(f"{e}; the invite is spent") + return exits.MESH_REFUSED + if change.made is None: + inviter.err(f"this node's change did not come up (apply exited" + f" {change.code}); the invite is spent") + return exits.APPLY_FAILED + window = change.made.window + until = inviter.clock() + timedelta(seconds=window) + admitter = Admitter( + inviter.root, spent, public, address, inviter.node, inviter.clock, + inviter.err, joined=Joined(line.public_key, joiner, change.made, + until, since)) + own = str(ipaddress.IPv6Interface(address).ip) + inviter.err(ports.open_port(spent.https_port, window, inviter.run, + inviter.output)) + inviter.err(f"waiting up to {window} s for the new node over the" + f" overlay, at [{own}]:{spent.https_port}") + try: + inviter.bridge(admitter, spent, Params( + spent.invite_id, spent.expires, own, + spent.https_port, own, (line.public_key, joiner, until))) + except (BridgeError, OSError, ssl.SSLError) as e: + inviter.err(f"cannot serve [{own}]:{spent.https_port}: {e}; this" + " node's change reverts by itself") + return exits.NETWORK_NOT_CONFIRMED + if not admitter.confirmed: + inviter.err(stopped(admitter)) + return exits.NETWORK_NOT_CONFIRMED + inviter.out(f"accepted: {line.public_key} is a peer of this node at" + f" {joiner}") + return exits.OK diff --git a/keel/mesh/joining.py b/keel/mesh/joining.py new file mode 100644 index 0000000..0cddf73 --- /dev/null +++ b/keel/mesh/joining.py @@ -0,0 +1,303 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh join TOKEN: the new node's side (decision 0048) + +1. The token is read whole (keel.mesh.token), the spec checked to take + it (keel.mesh.join), and nothing waits in 0018's window here. +2. The node keeps the mesh's identity, makes its key pair if it has + none, and finds a route to the inviter's endpoint; without one it + says so and names the rendezvous point (0024), applying nothing. +3. It sends the join request to the inviter's HTTPS port, pinning the + certificate the token names: its key, its endpoint, the reserved + address, a nonce and the time, under an HMAC with the invite's key. +4. With the signed answer, it writes its own spec (its address, the + inviter as a peer) and applies it under the window. +5. It opens the mesh session over the overlay, to the inviter's overlay + address: the inviter confirms its own window on receiving it, and + this node confirms its own on the inviter's signed answer. + +When the HTTPS port gives no TCP answer, this node applies its own side +under a window lengthened to the invite's remaining time (up to 15 +minutes), prints `keel mesh accept keel1a:...` for the inviter, and +waits for the inviter over the overlay, as in step 5. +""" + +import ipaddress +import os +import time +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime + +from keel import exits +from keel.mesh import acceptline, channel, identity, join, protocol +from keel.mesh.acceptline import Accept +from keel.mesh.channel import ChannelError, Forged, Refused, Unreachable +from keel.mesh.node import Node, NodeError, static_addresses +from keel.mesh.token import Token, hmac_key, shown +from keel.network import live, marker, session, wgkeys, wireguard +from keel.system.ovstate import overlay_of + +# how often the mesh session is tried while the tunnel comes up +RETRY = 2.0 +FALLBACK_LONGEST = 900 +FALLBACK_SHORTEST = 30 +TWO_NODES = ("the mesh is up with 2 nodes; etcd starts when a third node" + " joins, because 2 etcd members cannot lose one and keep a" + " majority") +RENDEZVOUS = ("the way in is the rendezvous point of decision 0024, which" + " the operator runs") + + +@dataclass +class Joiner: + """The new node's side; every function the outside world answers is + a field, so the flow is tested without a network""" + + node: Node + token: Token + clock: Callable[[], datetime] + out: Callable[[str], None] + err: Callable[[str], None] + post: Callable[..., channel.Reply] = channel.post + sleep: Callable[[float], None] = time.sleep + route_dev: Callable[[str], str | None] = field(default=live.route_dev) + + @property + def key(self) -> bytes: + return hmac_key(self.token.secret) + + def refused(self, message: str, code: int = exits.MESH_REFUSED) -> int: + self.err(message) + return code + + +def run(joiner: Joiner, endpoint: str | None) -> int: + """Join; `endpoint` is where the inviter reaches this node, or None""" + token = joiner.token + try: + doc = joiner.node.document() + after = join.merged(doc, join.change(overlay_of(doc), token)) + except (NodeError, join.JoinError) as e: + return joiner.refused(str(e)) + problems = joiner.node.problems(after) + if problems: + return joiner.refused("; ".join(f"{joiner.node.path}, once joined:" + f" {one}" for one in problems)) + if joiner.node.waiting(): + return joiner.refused( + "a network change waits for its confirmation on this node:" + " confirm or revert it (keel network confirm, keel network" + " revert), then join") + reach = [host for host in token.endpoints + if joiner.route_dev(host) is not None] + if not reach: + return joiner.refused( + f"this node has no route to the inviter at" + f" {', '.join(token.endpoints)}: {RENDEZVOUS}; nothing was" + " applied") + try: + kept = identity.read(joiner.node.root) + except ValueError as e: + return joiner.refused(str(e)) + if kept not in (None, token.mesh_id): + return joiner.refused(f"this node is in another mesh:" + f" /{identity.IDENTITY} names another one" + " than the token's") + overlay = overlay_of(after) + public, problem = own_key(joiner.node.root, overlay) + if public is None: + return joiner.refused(problem, exits.APPLY_FAILED) + own = f"{endpoint_text(endpoint)}:{wireguard.port(overlay)}" \ + if endpoint else None + request = protocol.JoinRequest( + invite_id=token.invite_id, public_key=public, endpoint=own, + address=token.assigned, nonce=protocol.new_nonce(), + time=protocol.seconds(joiner.clock())) + return requested(joiner, after, request, reach) + + +def requested(joiner: Joiner, after: dict, request: protocol.JoinRequest, + reach: list[str]) -> int: + token = joiner.token + body = protocol.dumps(request) + for host in reach: + try: + reply = joiner.post(host, token.https_port, protocol.JOIN, body, + joiner.key, token.fingerprint) + break + except Unreachable as e: + joiner.err(f"no answer from the inviter's port: {e}") + except ChannelError as e: + return joiner.refused(f"the inviter refused the join: {e}" + if isinstance(e, Refused) else str(e)) + else: + return fallback(joiner, after, request) + try: + answer = protocol.join_answer(reply.body) + except protocol.ProtocolError as e: + return joiner.refused(f"the inviter's answer is malformed: {e}") + if (answer.invite_id, answer.nonce, answer.address) != ( + token.invite_id, request.nonce, token.address) or \ + not wireguard.same_key(answer.public_key, token.public_key): + return joiner.refused("the inviter's answer is not the answer to" + " this request") + # kept only once the inviter has admitted this node + identity.adopt(joiner.node.root, token.mesh_id) + change = applied(joiner, after) + if change is None: + return exits.APPLY_FAILED + return confirmed(joiner, change, request.public_key, answer.peers) + + +def fallback(joiner: Joiner, after: dict, + request: protocol.JoinRequest) -> int: + """No TCP answer: apply this side, print the line for the inviter""" + token = joiner.token + if request.endpoint is None: + return joiner.refused( + "neither node can reach the other: the inviter's port gave no" + " answer and this node has no endpoint the inviter could" + f" reach; {RENDEZVOUS}; nothing was applied") + left = int((token.expires - joiner.clock()).total_seconds()) + window = min(left, FALLBACK_LONGEST) + if window < FALLBACK_SHORTEST: + return joiner.refused(f"the token expires at {shown(token.expires)}," + " too soon to wait for the inviter; ask it" + " for a new invite") + identity.adopt(joiner.node.root, token.mesh_id) + change = applied(joiner, after, window) + if change is None: + return exits.APPLY_FAILED + line = acceptline.encode(Accept( + public_key=request.public_key, endpoint=request.endpoint, + address=token.assigned, invite_id=token.invite_id, + time=protocol.seconds(joiner.clock())), joiner.key) + joiner.out(f"keel mesh accept {line}") + joiner.err(f"the inviter's TCP port {token.https_port} gave no answer:" + " run the line above on the inviter within" + f" {window // 60} minutes. This node waits for it over the" + " overlay, and its change reverts by itself if it never" + " comes") + return confirmed(joiner, change, request.public_key, ()) + + +def applied(joiner: Joiner, after: dict, + window: int | None = None) -> marker.Pending | None: + try: + change = joiner.node.change(after, window) + except NodeError as e: + joiner.err(str(e)) + return None + if change.made is None: + joiner.err(f"this node's overlay did not come up under the window" + f" (apply exited {change.code}); the token is spent and" + " the inviter's change reverts by itself: ask it for a" + " new invite") + return change.made + + +def confirmed(joiner: Joiner, made: marker.Pending, public: str, + peers: tuple[protocol.Peer, ...]) -> int: + """The mesh session over the overlay, then this node's confirmation""" + token = joiner.token + inviter = str(ipaddress.IPv6Interface(token.address).ip) + deadline = joiner.clock().timestamp() + made.window + while True: + request = protocol.ConfirmRequest( + invite_id=token.invite_id, public_key=public, + nonce=protocol.new_nonce(), + time=protocol.seconds(joiner.clock())) + body = protocol.dumps(request) + try: + reply = joiner.post(inviter, token.https_port, protocol.CONFIRM, + body, joiner.key, token.fingerprint, + connect_timeout=RETRY) + break + except (Refused, Forged) as e: + return joiner.refused(f"the mesh session was refused: {e}; this" + " node's change reverts by itself", + exits.NETWORK_NOT_CONFIRMED) + except ChannelError: + if joiner.clock().timestamp() + RETRY >= deadline: + return joiner.refused( + f"the inviter did not answer over the overlay at" + f" [{inviter}]:{token.https_port} within the window;" + " this node's change reverts by itself", + exits.NETWORK_NOT_CONFIRMED) + joiner.sleep(RETRY) + try: + answer = protocol.confirm_answer(reply.body) + except protocol.ProtocolError as e: + return joiner.refused(f"the inviter's answer is malformed: {e}", + exits.NETWORK_NOT_CONFIRMED) + if (answer.invite_id, answer.nonce) != (token.invite_id, request.nonce): + return joiner.refused("the inviter's answer is not the answer to" + " this mesh session; this node's change" + " reverts by itself", + exits.NETWORK_NOT_CONFIRMED) + if not answer.confirmed: + return joiner.refused( + f"the inviter did not keep its change ({answer.detail}); this" + " node's change reverts by itself", exits.NETWORK_NOT_CONFIRMED) + origin = session.Origin( + session.MESH, f"the inviter's answer over the overlay (invite" + f" {token.invite_id})", None, reply.local, reply.peer) + kept, lines = joiner.node.confirm(origin, made) + for line in lines: + joiner.err(line) + if not kept: + return exits.NETWORK_NOT_CONFIRMED + summary(joiner, peers) + return exits.OK + + +def summary(joiner: Joiner, peers: tuple[protocol.Peer, ...]) -> None: + """What `join` prints at the end, and confconsole shows""" + token = joiner.token + overlay = joiner.node.overlay() + inviter = ipaddress.IPv6Interface(token.address).ip + joiner.out(f"joined the mesh: this node is {token.assigned} on" + f" {wireguard.interface(overlay)}") + joiner.out(f"peer: {token.public_key} at {inviter}, through" + f" {token.endpoint()}") + if not peers: + joiner.out(TWO_NODES) + return + joiner.out(f"the mesh has {len(peers) + 2} nodes; this node reaches the" + f" inviter, and the {len(peers)} other(s) once they learn of" + " it, which this keel does not do yet; etcd is not part of" + " this keel") + + +def own_key(root: str, overlay: dict) -> tuple[str | None, str]: + """This node's public key, the pair made first when there is none + (keel-core#8: on the machine itself)""" + path = os.path.join(root, wireguard.key_path(overlay).lstrip("/")) + if not os.path.exists(path): + problem = wgkeys.generate(path) + if problem: + return None, problem + public, problem = wgkeys.public(path) + return public, str(problem or "") + + +def endpoint_text(address: str) -> str: + found = ipaddress.ip_address(address) + return f"[{found}]" if found.version == 6 else str(found) + + +def own_endpoint(declared: str | None, doc: dict, + held: list[tuple[str, str]], iface: str) -> str | None: + """Where the inviter reaches this node: `--endpoint`, else a static + address network.interfaces declares, else a global address `held` + by an interface other than the overlay's; IPv6 first, None without + one another node could reach""" + if declared: + return str(ipaddress.ip_address(declared)) + found = list(static_addresses(doc)) + [ + ipaddress.ip_address(address) for name, address in held + if name != iface] + usable = sorted((one for one in found if not ( + one.is_loopback or one.is_link_local or one.is_multicast + or one.is_unspecified)), key=lambda one: one.version, reverse=True) + return str(usable[0]) if usable else None diff --git a/keel/mesh/listener.py b/keel/mesh/listener.py new file mode 100644 index 0000000..d40fc38 --- /dev/null +++ b/keel/mesh/listener.py @@ -0,0 +1,352 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The invite's listener, the only part that faces the network (0048) + +It runs without privileges: as a DynamicUser unit with no capability +(keel.mesh.bridge starts it), it holds the invite's TLS key in memory +only, never the invite's HMAC key, and it can change nothing on the +machine. It does TLS, reads one HTTP request per connection, and checks +what it can without the key (keel.mesh.admit.parsed: the shape, the +invite id, the time; and the nonce); it hands what passed, as the bytes +it received, to its backend, the Admitter behind the bridge, which +verifies the HMAC, checks the rest again and decides. A compromised +listener can therefore forge nothing the root side accepts. + +Against someone scanning or holding the port: + +- a request's whole reading, TLS handshake included, has a deadline + (REQUEST_DEADLINE), and every read a timeout; +- at most SLOTS connections are read at once, and OVERLAY_SLOTS more + are kept for connections to this node's overlay address, which only + the mesh's peers reach, so a join's confirmation is never starved; + and one source has one connection at a time (PER_SOURCE), so no one + address fills the slots; +- the request line and headers are capped (MAX_HEADERS), the body by + the protocol (MAX_BODY); +- a source refused MAX_REFUSED times within PERIOD is not answered + until the period has passed; +- only a request that names this invite and fails its HMAC counts + towards cancelling it, on the root side: that is someone who knows + the invite and not its key, while a scanner learns nothing and costs + the operator nothing. The root side's `final` answer stops the + listener. +""" + +import ipaddress +import socket +import threading +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta +from http.server import BaseHTTPRequestHandler + +from keel.mesh import protocol +from keel.mesh.admit import ( + Refusal, + Response, + default_log, + parsed, + parser, + refusal, +) +from keel.mesh.channel import plain + +MAX_REFUSED = 5 +PERIOD = timedelta(seconds=60) +REQUEST_DEADLINE = 15 +READ_TIMEOUT = 5 +SLOTS = 4 +OVERLAY_SLOTS = 2 +PER_SOURCE = 1 +MAX_HEADERS = 8192 +POLL = 1.0 +# how long the threads of the last connections are waited for +DRAIN = 5 + + +@dataclass(frozen=True) +class Params: + """What the root side hands the listener: no path, no file + + `overlay` is this node's overlay address, whose connections have + slots of their own; `joined` is set for accept, which serves the + confirmation alone: (public key, address, until). + """ + + invite_id: str + expires: datetime + host: str + port: int + overlay: str + joined: tuple[str, str, datetime] | None = None + + +class Limiter: + """The sources refused too often lately, and until when""" + + def __init__(self, clock: Callable[[], datetime]): + self.clock = clock + self.seen: dict[str, list[datetime]] = {} + + def recent(self, source: str) -> list[datetime]: + since = self.clock() - PERIOD + found = [one for one in self.seen.get(source, []) if one > since] + self.seen[source] = found + return found + + def refused(self, source: str) -> None: + self.seen[source] = self.recent(source) + [self.clock()] + + def blocked(self, source: str) -> bool: + return len(self.recent(source)) >= MAX_REFUSED + + +class Listener: + """The front of one invite; `clock` gives an aware UTC datetime + + `backend` is the root side: keel.mesh.admit.Admitter in one process, + keel.mesh.bridge.Remote across the bridge; both have forward(path, + signature, body, local, peer) -> Response. + """ + + def __init__(self, params: Params, backend, + clock: Callable[[], datetime], + log: Callable[[str], None] | None = None): + self.params = params + self.backend = backend + self.clock = clock + self.log = log or default_log + self.lock = threading.Lock() + self.limiter = Limiter(clock) + self.nonces: set[str] = set() + self.joined = params.joined + self.confirmed: bool | None = None + self.ended = False + + @property + def name(self) -> str: + return f"invite {self.params.invite_id}" + + @property + def finished(self) -> bool: + if self.ended or self.confirmed is not None: + return True + until = self.joined[2] if self.joined else self.params.expires + return self.clock() >= until + + def blocked(self, source: str) -> bool: + with self.lock: + return self.limiter.blocked(source) + + def handle(self, method: str, path: str, signature: str | None, + body: bytes | None, local: str, peer: str) -> Response: + """The answer to one request; never raises""" + with self.lock: + try: + return self.checked(method, path, signature, body, local, + peer) + except Refusal as e: + return self.refuse(e, peer) + + def refuse(self, refused: Refusal, peer: str) -> Response: + self.limiter.refused(peer) + self.log(f"{self.name}: refused a request from {peer}:" + f" {refused.reason}") + return refusal(refused.status, refused.reason) + + def checked(self, method: str, path: str, signature: str | None, + body: bytes | None, local: str, peer: str) -> Response: + if method != protocol.METHOD or path not in (protocol.JOIN, + protocol.CONFIRM): + raise Refusal(404, f"{method} {path[:64]!r}: no such request") + if path == protocol.JOIN and self.joined is not None: + raise Refusal(410, f"{self.name} was already used") + if path == protocol.CONFIRM and self.joined is None: + raise Refusal(409, "no join to confirm yet") + request = parsed(self.params.invite_id, path, body, self.clock(), + parser(path)) + if request.nonce in self.nonces: + raise Refusal(403, "replayed request: its nonce was seen") + self.nonces.add(request.nonce) + if path == protocol.CONFIRM and (local, peer) != ( + self.params.overlay, self.joined[1]): + raise Refusal(403, f"the confirmation must come over the" + f" overlay, from {self.joined[1]} to" + f" {self.params.overlay}") + found = self.backend.forward(path, signature, body, local, peer) + self.learn(path, request, found, peer) + return found + + def learn(self, path: str, request, found: Response, peer: str) -> None: + """What the root side's answer says about this invite's state""" + if found.final or found.status == 503: + self.ended = True + if found.status not in (200, 409): + self.limiter.refused(peer) + if found.status != 200: + return + try: + if path == protocol.JOIN: + answer = protocol.join_answer(found.body) + address = str(ipaddress.IPv6Interface(request.address).ip) + self.joined = (request.public_key, address, self.clock() + + timedelta(seconds=answer.window)) + else: + self.confirmed = protocol.confirm_answer( + found.body).confirmed + except protocol.ProtocolError as e: + self.ended = True + self.log(f"{self.name}: the root side's answer cannot be read" + f" ({e}); the listener stops") + + +class Capped: + """A request's reader that gives out `budget` bytes: MAX_HEADERS for + the request line and the headers, then MAX_BODY for the body""" + + def __init__(self, inner, budget: int): + self.inner = inner + self.budget = budget + + def spend(self, count: int) -> None: + self.budget -= count + if self.budget < 0: + raise ValueError("the request is longer than any join") + + def readline(self, limit: int = -1) -> bytes: + found = self.inner.readline(limit) + self.spend(len(found)) + return found + + def read(self, count: int = -1) -> bytes: + found = self.inner.read(count) + self.spend(len(found)) + return found + + def close(self) -> None: + self.inner.close() + + +@dataclass +class Connection: + """What a handler needs of the connection it serves""" + + listener: Listener + local: str + read: threading.Timer + + +class Handler(BaseHTTPRequestHandler): + server_version = "keel-mesh" + sys_version = "" + + def setup(self): + super().setup() + self.rfile = Capped(self.rfile, MAX_HEADERS) + + def answer(self): + try: + length = int(self.headers.get("Content-Length") or 0) + except ValueError: + length = -1 + self.rfile.budget = protocol.MAX_BODY + body = (self.rfile.read(length) + if 0 <= length <= protocol.MAX_BODY else None) + # read whole: the deadline is for reading, not for the apply + self.server.read.cancel() + found = self.server.listener.handle( + self.command, self.path, self.headers.get(protocol.SIGNATURE), + body, self.server.local, self.client_address[0]) + self.send_response(found.status) + self.send_header("Content-Type", protocol.CONTENT_TYPE) + self.send_header("Content-Length", str(len(found.body))) + if found.signature: + self.send_header(protocol.SIGNATURE, found.signature) + self.end_headers() + self.wfile.write(found.body) + + do_POST = do_GET = do_PUT = answer + + def log_message(self, format, *args): # noqa: A002 + """The listener logs each request itself, and no header""" + + +def shut(held: list[socket.socket]) -> None: + """The reading's deadline: whatever was being read fails. `held` is + the connection's socket now: the raw one, then its TLS wrapper, + which takes the raw one's descriptor""" + try: + held[-1].shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def bound(host: str, port: int) -> socket.socket: + """IPv6 and IPv4 on one socket; raises OSError when the port is + taken or the address is not this machine's""" + server = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + try: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + server.bind((host, port)) + server.listen(16) + except OSError: + server.close() + raise + return server + + +def serve_one(listener: Listener, context, raw: socket.socket, source: str, + local: str, release: Callable[[], None]) -> None: + held = [raw] + read = threading.Timer(REQUEST_DEADLINE, shut, (held,)) + read.start() + try: + raw.settimeout(READ_TIMEOUT) + held.append(context.wrap_socket(raw, server_side=True)) + Handler(held[-1], (source, 0), Connection(listener, local, read)) + except (OSError, ValueError): + pass + finally: + read.cancel() + held[-1].close() + release() + + +def run(server: socket.socket, listener: Listener, context, + poll: float = POLL) -> None: + """Answer until the listener is finished, then close the port""" + slots = threading.BoundedSemaphore(SLOTS) + overlay_slots = threading.BoundedSemaphore(OVERLAY_SLOTS) + active: dict[str, int] = {} + guard = threading.Lock() + threads: list[threading.Thread] = [] + server.settimeout(poll) + with server: + while not listener.finished: + try: + raw, address = server.accept() + except TimeoutError: + continue + source, local = plain(address[0]), plain(raw.getsockname()[0]) + slot = overlay_slots if local == listener.params.overlay \ + else slots + with guard: + held = active.get(source, 0) + if held >= PER_SOURCE or listener.blocked(source) or \ + not slot.acquire(blocking=False): + raw.close() + continue + with guard: + active[source] = held + 1 + + def release(slot=slot, source=source): + slot.release() + with guard: + active[source] -= 1 + thread = threading.Thread(target=serve_one, args=( + listener, context, raw, source, local, release), + daemon=True) + thread.start() + threads = [one for one in threads if one.is_alive()] + [thread] + for thread in threads: + thread.join(DRAIN) diff --git a/keel/mesh/node.py b/keel/mesh/node.py new file mode 100644 index 0000000..777cbe2 --- /dev/null +++ b/keel/mesh/node.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""This machine as keel mesh changes it: its spec, apply, confirm + +The one seam between the mesh commands and the rest of keel. A join +writes the same `network.overlay.wireguard` fields an operator writes +by hand, into this node's own spec (0013), and converges them with the +same `apply --system` under the same window (0018), with the uplink +left alone (`--skip-uplink`, as confconsole's overlay screen asks) and +no certificate requested. The spec is validated whole before it is +written, and written as keel writes its state, through a temporary +file and a rename (keel.network.marker.write_private), so a crash +leaves the old spec or the new one. It is written as YAML from the +document keel read, so comments in it are not kept. + +The change apply made is read back from its marker: that is the change +keel mesh may confirm, and no other (keel.network.confirm). +""" + +import contextlib +import ipaddress +import os +import sys +from collections.abc import Callable +from dataclasses import dataclass, field + +import yaml + +from keel import spec +from keel.commands import apply_system, manifest_facts +from keel.mesh.protocol import Peer +from keel.network import confirm as netconfirm +from keel.network import live, marker, session, wireguard +from keel.network.marker import write_private +from keel.network.wireguard import allowed, same_key +from keel.system import DEFAULT_WINDOW +from keel.system.ovstate import overlay_of + + +class NodeError(Exception): + """The spec cannot take the change, and why""" + + +@dataclass(frozen=True) +class Change: + """apply's exit code, and the overlay change waiting, if it made one""" + + code: int + made: marker.Pending | None + + +def live_apply(doc: dict, root: str, window: int) -> int: + """apply --system-only --skip-uplink, its lines on standard error + + Standard output stays the command's own: the line to paste, or the + summary confconsole shows. + """ + with contextlib.redirect_stdout(sys.stderr): + return apply_system(doc, root, False, "keel mesh", + defer_certificate=True, network_window=window, + skip_uplink=True) + + +@dataclass +class Node: + root: str + path: str + window: int = DEFAULT_WINDOW + clients: tuple[str, ...] = () + apply: Callable[[dict, str, int], int] = live_apply + probes: Callable[[], netconfirm.Probes] = field(default=live.probes) + run: Callable[[tuple[str, ...]], str | None] = field(default=live.run) + output: Callable[[tuple[str, ...]], str | None] = field( + default=live.output) + + def document(self) -> dict: + """The spec as keel reads it; an absent one is an empty spec""" + if not os.path.exists(self.path): + return {"version": 1} + try: + return spec.canonical(spec.load(self.path)) + except spec.SpecError as e: + raise NodeError(str(e)) from None + + def overlay(self) -> dict: + return overlay_of(self.document()) or {} + + def problems(self, doc: dict) -> list[str]: + return spec.validate(doc, check_secret_files=False, + facts=manifest_facts(doc, self.root)) + + def write(self, doc: dict) -> None: + """Validate the whole spec, then replace the file with it""" + problems = self.problems(doc) + if problems: + raise NodeError("; ".join(f"{self.path}: {one}" + for one in problems)) + text = yaml.safe_dump(doc, sort_keys=False, default_flow_style=False, + allow_unicode=True) + directory, name = os.path.split(os.path.abspath(self.path)) + try: + write_private(directory, name, text) + except OSError as e: + raise NodeError(f"{self.path} cannot be written:" + f" {e.strerror or e}") from None + + def change(self, doc: dict, window: int | None = None) -> Change: + """Write `doc` and apply it; the change waiting, if any""" + self.write(doc) + code = self.apply(doc, self.root, window or self.window) + made = marker.read(self.root) if marker.exists(self.root) else None + if made is not None and (made.kind != marker.OVERLAY + or made.changed_at is None): + made = None + return Change(code, made) + + def waiting(self) -> bool: + return marker.exists(self.root) + + def admit(self, peer: dict, window: int | None = None) -> Change: + """This node's spec with `peer` added, written and applied; a + peer of the same key is refused (NodeError), never replaced""" + return self.change(with_peer(self.document(), peer), window) + + def peers(self, but: str) -> tuple[Peer, ...]: + """The peers the spec declares, but the one of key `but`""" + found = [] + for peer in self.overlay().get("peers") or []: + key = str(peer.get("public_key")) + if same_key(key, but): + continue + hosts = [one for one in allowed(peer) if one.endswith("/128")] + if hosts: + found.append(Peer(key, peer.get("endpoint"), + hosts[0].split("/")[0])) + return tuple(found) + + def handshake(self, key: str) -> int | None: + """When `key` last completed a WireGuard handshake with this node, + in seconds since the epoch, as `wg show IFACE latest-handshakes` + says; None for no handshake, or no answer""" + iface = wireguard.interface(self.overlay()) + text = self.output(("wg", "show", iface, "latest-handshakes")) or "" + for line in text.splitlines(): + fields = line.split("\t") + if len(fields) == 2 and same_key(fields[0], key) and \ + fields[1].isdigit() and fields[1] != "0": + return int(fields[1]) + return None + + def confirm(self, origin: session.Origin, + made: marker.Pending | None) -> tuple[bool, list[str]]: + return netconfirm.confirm(self.root, origin, self.probes(), self.run, + expected=made, clients=self.clients) + + +def static_addresses(doc: dict): + """The static addresses network.interfaces declares, in its order""" + interfaces = (doc.get("network") or {}).get("interfaces") or {} + for iface in interfaces.values(): + for family in ("ipv6", "ipv4"): + block = (iface or {}).get(family) or {} + if block.get("method") == "static" and block.get("address"): + yield ipaddress.ip_interface(str(block["address"])).ip + + +def global_addresses(text: str) -> list[tuple[str, str]]: + """`ip -o address show scope global` output: (interface, address)""" + found = [] + for line in text.splitlines(): + fields = line.split() + if len(fields) > 3 and fields[2] in ("inet", "inet6"): + found.append((fields[1].split("@")[0], + fields[3].split("/")[0])) + return found + + +def with_peer(doc: dict, peer: dict) -> dict: + """`doc` with the peer added to its overlay, as a new document + + A peer of the same key is never replaced: raises NodeError. + """ + network = dict(doc.get("network") or {}) + overlay = dict(network.get("overlay") or {}) + wireguard = dict(overlay.get("wireguard") or {}) + kept = list(wireguard.get("peers") or []) + if any(same_key(str(one.get("public_key")), peer["public_key"]) + for one in kept): + raise NodeError(f"{peer['public_key']} is already a peer of this" + " node; a join never replaces one") + wireguard["peers"] = kept + [peer] + return {**doc, "network": {**network, "overlay": { + **overlay, "wireguard": wireguard}}} + + +def with_overlay(doc: dict, fields: dict) -> dict: + """`doc` with fields of its overlay set, as a new document""" + network = dict(doc.get("network") or {}) + overlay = dict(network.get("overlay") or {}) + wireguard = {**(overlay.get("wireguard") or {}), **fields} + return {**doc, "network": {**network, "overlay": { + **overlay, "wireguard": wireguard}}} diff --git a/keel/mesh/parser.py b/keel/mesh/parser.py new file mode 100644 index 0000000..03d4b6d --- /dev/null +++ b/keel/mesh/parser.py @@ -0,0 +1,144 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The keel mesh subcommands' arguments (decision 0048, docs/mesh.md) + +Apart from keel.cli for length; the options every subcommand shares +are keel.cli's own, handed in, so they are defined once. +""" + +import argparse +from collections.abc import Callable +from dataclasses import dataclass + +from keel.mesh import commands +from keel.network.wireguard import DEFAULT_PORT as WIREGUARD_PORT +from keel.system import DEFAULT_WINDOW + + +@dataclass(frozen=True) +class Options: + """keel.cli's shared helpers: --spec and friends, --root, and the + types of a port and of a window""" + + common: Callable[[argparse.ArgumentParser], None] + root: Callable[[argparse.ArgumentParser, str], None] + port: Callable[[str], int] + window: Callable[[str], int] + + +def add(subparsers, options: Options) -> None: + """keel mesh: joining the WireGuard mesh with one command (0048)""" + mesh_parser = subparsers.add_parser( + "mesh", + help="invite a node into this node's WireGuard mesh, or join one" + " with the line an invite printed (decision 0048, docs/mesh.md)", + ) + mesh_actions = mesh_parser.add_subparsers( + dest="action", metavar="ACTION" + ) + invite_parser = mesh_actions.add_parser( + "invite", + help="print the keel mesh join line for a new node, valid one hour" + " and once, and reserve its overlay address (root)", + ) + options.common(invite_parser) + invite_parser.add_argument( + "--endpoint", action="append", default=None, metavar="ADDRESS", + help="an address the new node reaches this one at, one per family;" + " may be repeated (default: the static addresses" + " network.interfaces declares)", + ) + invite_parser.add_argument( + "--port", type=options.port, default=WIREGUARD_PORT, + metavar="PORT", + help="the TCP port the join request goes to (default: %(default)s," + " the number of WireGuard's UDP port)", + ) + options.root(invite_parser, "read the overlay of and keep the invite in") + invite_parser.set_defaults(handler=commands.mesh_invite) + join_parser = mesh_actions.add_parser( + "join", + help="join the mesh of the node that printed TOKEN (root); with" + " --dry-run, print the change of this node's spec it makes", + ) + options.common(join_parser) + join_parser.add_argument( + "token", metavar="TOKEN", + help="the keel1: token keel mesh invite printed, or - to read it" + " from standard input, which keeps it out of the process list", + ) + join_parser.add_argument( + "--dry-run", action="store_true", + help="print the address and the peer the join would write into the" + " spec, and change nothing", + ) + join_parser.add_argument( + "--endpoint", default=None, metavar="ADDRESS", + help="the address the inviter reaches this node at (default: a" + " static address network.interfaces declares, else a global one" + " this node holds; none behind NAT)", + ) + window(options, join_parser) + options.root(join_parser, "read the appliance manifests of") + join_parser.set_defaults(handler=commands.mesh_join) + actions(mesh_actions, options) + + +def window(options: Options, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--network-window", type=options.window, + default=DEFAULT_WINDOW, metavar="SECONDS", + help="how long this node's overlay change waits for its" + " confirmation before it reverts (default: %(default)s)", + ) + + +def actions(mesh_actions, options: Options) -> None: + """keel mesh create, accept, status, and the listener's serve""" + create_parser = mesh_actions.add_parser( + "create", + help="make the mesh on its first node: a random overlay prefix," + " this node at ::1, confirmed by itself (root)", + ) + options.common(create_parser) + window(options, create_parser) + options.root(create_parser, "create the mesh in") + create_parser.set_defaults(handler=commands.mesh_create) + accept_parser = mesh_actions.add_parser( + "accept", + help="on the inviter, the keel mesh accept line a join printed when" + " it could not reach this node's port (root)", + ) + options.common(accept_parser) + accept_parser.add_argument( + "line", metavar="LINE", + help="the keel1a: line, or - to read it from standard input", + ) + window(options, accept_parser) + options.root(accept_parser, "accept the new node in") + accept_parser.set_defaults(handler=commands.mesh_accept) + status_parser = mesh_actions.add_parser( + "status", + help="this node's peers, their last handshakes, and the pending" + " invites; never a secret", + ) + options.common(status_parser) + options.root(status_parser, "read the mesh of") + status_parser.set_defaults(handler=commands.mesh_status) + serve_parser = mesh_actions.add_parser( + "serve", + help="the invite's root helper, which keel mesh invite starts as" + " the unit keel-mesh-invite@ID; not run by hand", + ) + options.common(serve_parser) + serve_parser.add_argument("invite", metavar="ID", + help="the invite to answer") + options.root(serve_parser, "serve the invite of") + serve_parser.set_defaults(handler=commands.mesh_serve) + listen_parser = mesh_actions.add_parser( + "listen", + help="the unprivileged listener keel mesh serve starts as the unit" + " keel-mesh-listen@ID; not run by hand", + ) + listen_parser.add_argument("socket", metavar="SOCKET", + help="the root helper's unix socket") + listen_parser.set_defaults(handler=commands.mesh_listen) diff --git a/keel/mesh/ports.py b/keel/mesh/ports.py new file mode 100644 index 0000000..59fcb03 --- /dev/null +++ b/keel/mesh/ports.py @@ -0,0 +1,46 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The invite's TCP port in keel's firewall, where it has one (0048) + +keel touches no firewall it does not own. Where keel-firewall is +enabled (cloud advanced, `firewall.enabled: true`), its derived ruleset +holds a named set of the pending invites' ports, each with a timeout, +and one rule that accepts them (keel.manifest.firewall): an invite adds +its port for the time left to its expiry, so it closes by itself even +if keel dies, and the listener's end removes it. Adding an element is +not a ruleset change, so it neither waits on nor disturbs a network +change in its window. Everywhere else nothing is opened, and the +command says so: a closed port is what the fallback is for. +""" + +from collections.abc import Callable + +from keel.manifest.firewall import INVITES_SET, TABLE + +Runner = Callable[[tuple[str, ...]], str | None] +Reader = Callable[[tuple[str, ...]], str | None] + + +def element(port: int, seconds: int | None = None) -> str: + timeout = f" timeout {seconds}s" if seconds else "" + return f"{{ {port}{timeout} }}" + + +def open_port(port: int, seconds: int, run: Runner, output: Reader) -> str: + """Open the port for `seconds` where keel's firewall has the set; + one line that says what was done""" + if output(("nft", "list", "set", *TABLE, INVITES_SET)) is None: + return (f"keel's firewall is not enabled here, so keel opened" + f" nothing: TCP port {port} must reach this node, or the" + " new node's join prints a line to run here instead") + problem = run(("nft", "add", "element", *TABLE, INVITES_SET, + element(port, seconds))) + if problem: + return (f"TCP port {port} could not be opened in keel's firewall" + f" ({problem}); the new node's join prints a line to run" + " here instead") + return f"TCP port {port} opened in keel's firewall while the invite waits" + + +def close_port(port: int, run: Runner) -> None: + """Remove the element; one that timed out or never was is fine""" + run(("nft", "delete", "element", *TABLE, INVITES_SET, element(port))) diff --git a/keel/mesh/protocol.py b/keel/mesh/protocol.py new file mode 100644 index 0000000..3598b0f --- /dev/null +++ b/keel/mesh/protocol.py @@ -0,0 +1,254 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The join protocol's messages and their HMAC (decision 0048) + +Two requests go to the inviter's HTTPS port, each a JSON body POSTed +with an HMAC-SHA256 of the method, the path and the body in the header +X-Keel-Mesh-Signature, keyed with the invite's HMAC key +(keel.mesh.token.hmac_key): the token's secret never crosses the wire. + + POST /v1/join JoinRequest -> JoinAnswer, over the uplink + POST /v1/confirm ConfirmRequest -> ConfirmAnswer, over the overlay + +An answer of 200 is signed the same way under the method ANSWER and the +request's path, and carries the request's nonce, so the new node knows +it came from the node that made the token as well as from the pinned +certificate. A refusal is a JSON {"error": reason}, unsigned: it gives +nothing to act on. Everything here is pure. +""" + +import hashlib +import hmac +import ipaddress +import json +import re +import secrets +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta + +from keel.mesh.token import ETCD_STATES +from keel.network.wireguard import is_key, split_endpoint + +JOIN = "/v1/join" +CONFIRM = "/v1/confirm" +METHOD = "POST" +ANSWER = "ANSWER" +SIGNATURE = "X-Keel-Mesh-Signature" +CONTENT_TYPE = "application/json" +# a join request is about 300 bytes, an answer grows with the peers +MAX_BODY = 16384 +# how far a request's time may be from the inviter's clock +SKEW = timedelta(minutes=5) +NONCE_BYTES = 16 +LABEL = b"keel mesh 1\n" +ID_RE = re.compile(r"^[0-9a-f]{16}$") +NONCE_RE = re.compile(rf"^[0-9a-f]{{{NONCE_BYTES * 2}}}$") +SIGNATURE_RE = re.compile(r"^[0-9a-f]{64}$") + + +class ProtocolError(ValueError): + """A message that is not one of the protocol's, and why""" + + +@dataclass(frozen=True) +class Peer: + """A member as the answer names it: key, endpoint or None, address""" + + public_key: str + endpoint: str | None + address: str + + +@dataclass(frozen=True) +class JoinRequest: + """The new node: its key, its endpoint or None, the address the + invite reserved, with its prefix length""" + + invite_id: str + public_key: str + endpoint: str | None + address: str + nonce: str + time: int + + +@dataclass(frozen=True) +class JoinAnswer: + """The inviter: its key and overlay address, the peers it knows + besides the new node, the etcd state, and the seconds its change + waits for the confirmation""" + + invite_id: str + nonce: str + public_key: str + address: str + peers: tuple[Peer, ...] + etcd: str + window: int + + +@dataclass(frozen=True) +class ConfirmRequest: + invite_id: str + public_key: str + nonce: str + time: int + + +@dataclass(frozen=True) +class ConfirmAnswer: + """Whether the inviter kept its change, and what it said""" + + invite_id: str + nonce: str + confirmed: bool + detail: str + + +def sign(key: bytes, method: str, path: str, body: bytes) -> str: + message = LABEL + method.encode() + b"\n" + path.encode() + b"\n" + body + return hmac.new(key, message, hashlib.sha256).hexdigest() + + +def signed(key: bytes, method: str, path: str, body: bytes, + signature: str | None) -> bool: + """Whether `signature` is the HMAC of the message, in constant time""" + if signature is None or not SIGNATURE_RE.match(signature): + return False + return hmac.compare_digest(sign(key, method, path, body), signature) + + +def new_nonce() -> str: + return secrets.token_hex(NONCE_BYTES) + + +def seconds(now: datetime) -> int: + return int(now.timestamp()) + + +def fresh(time: int, now: datetime) -> bool: + return abs(time - now.timestamp()) <= SKEW.total_seconds() + + +def dumps(message) -> bytes: + return json.dumps(asdict(message), sort_keys=True).encode() + + +def loaded(body: bytes) -> dict: + try: + data = json.loads(body.decode()) + except (UnicodeDecodeError, ValueError, RecursionError): + raise ProtocolError("the body is not JSON") from None + if not isinstance(data, dict): + raise ProtocolError("the body is not a JSON object") + return data + + +def field(data: dict, name: str, kind: type): + value = data.get(name) + if type(value) is not kind: + raise ProtocolError(f"{name} is missing or not a {kind.__name__}") + return value + + +def matching(data: dict, name: str, pattern: re.Pattern) -> str: + value = field(data, name, str) + if not pattern.match(value): + raise ProtocolError(f"{name} is malformed") + return value + + +def key(data: dict, name: str) -> str: + value = data.get(name) + if not isinstance(value, str) or not is_key(value): + raise ProtocolError(f"{name} is not a WireGuard key") + return value + + +def endpoint(value: object) -> str | None: + """None, or an address and port another node can reach""" + if value is None: + return None + try: + host, number = split_endpoint(str(value)) + address = ipaddress.ip_address(host) + except ValueError as e: + raise ProtocolError(f"endpoint {value!r}: {e}") from None + if (address.is_unspecified or address.is_loopback + or address.is_multicast or address.is_link_local): + raise ProtocolError(f"endpoint {address} cannot be reached from" + " another node") + shown = f"[{address}]" if address.version == 6 else str(address) + return f"{shown}:{number}" + + +def overlay(data: dict, name: str, prefixed: bool) -> str: + """An overlay address: with its prefix length when `prefixed`""" + value = field(data, name, str) + try: + found = ipaddress.IPv6Interface(value) + except ValueError: + raise ProtocolError(f"{name} is not an IPv6 address") from None + if prefixed != ("/" in value): + raise ProtocolError(f"{name} is not an IPv6 address" + f"{' with its prefix length' if prefixed else ''}") + return str(found) if prefixed else str(found.ip) + + +def number(data: dict, name: str) -> int: + value = field(data, name, int) + if value < 0: + raise ProtocolError(f"{name} is negative") + return value + + +def join_request(body: bytes) -> JoinRequest: + data = loaded(body) + return JoinRequest( + invite_id=matching(data, "invite_id", ID_RE), + public_key=key(data, "public_key"), + endpoint=endpoint(data.get("endpoint")), + address=overlay(data, "address", True), + nonce=matching(data, "nonce", NONCE_RE), + time=number(data, "time")) + + +def join_answer(body: bytes) -> JoinAnswer: + data = loaded(body) + etcd = field(data, "etcd", str) + if etcd not in ETCD_STATES: + raise ProtocolError(f"etcd state {etcd!r} is not one this keel" + " knows") + peers = field(data, "peers", list) + return JoinAnswer( + invite_id=matching(data, "invite_id", ID_RE), + nonce=matching(data, "nonce", NONCE_RE), + public_key=key(data, "public_key"), + address=overlay(data, "address", True), + peers=tuple(peer(one) for one in peers), + etcd=etcd, window=number(data, "window")) + + +def peer(data: object) -> Peer: + if not isinstance(data, dict): + raise ProtocolError("a peer is not a JSON object") + return Peer(public_key=key(data, "public_key"), + endpoint=endpoint(data.get("endpoint")), + address=overlay(data, "address", False)) + + +def confirm_request(body: bytes) -> ConfirmRequest: + data = loaded(body) + return ConfirmRequest( + invite_id=matching(data, "invite_id", ID_RE), + public_key=key(data, "public_key"), + nonce=matching(data, "nonce", NONCE_RE), + time=number(data, "time")) + + +def confirm_answer(body: bytes) -> ConfirmAnswer: + data = loaded(body) + return ConfirmAnswer( + invite_id=matching(data, "invite_id", ID_RE), + nonce=matching(data, "nonce", NONCE_RE), + confirmed=field(data, "confirmed", bool), + detail=field(data, "detail", str)) diff --git a/keel/mesh/status.py b/keel/mesh/status.py new file mode 100644 index 0000000..bc4848c --- /dev/null +++ b/keel/mesh/status.py @@ -0,0 +1,82 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh status: the peers, their handshakes, the pending invites + +Read from the spec, from `wg show` (its public-key, endpoints and +latest-handshakes views, never `dump` or `private-key`, which hold the +private key) and from the invite files, of which only the id, the +reserved address, the port and the expiry are shown. Nothing secret is +printed. +""" + +import ipaddress +from collections.abc import Callable +from datetime import datetime + +from keel.mesh import invites +from keel.mesh.token import shown +from keel.network import wireguard +from keel.network.wireguard import allowed, same_key + +Reader = Callable[[tuple[str, ...]], str | None] + + +def table(text: str | None) -> dict[str, str]: + """`wg show IFACE endpoints|latest-handshakes`: key to value""" + found = {} + for line in (text or "").splitlines(): + fields = line.split("\t") + if len(fields) == 2: + found[fields[0]] = fields[1] + return found + + +def handshake(value: str | None, now: datetime) -> str: + if value is None: + return "no handshake known" + if not value.isdigit() or value == "0": + return "no handshake yet" + ago = int(now.timestamp()) - int(value) + return f"handshake {max(ago, 0)} s ago" + + +def lines(overlay: dict | None, root: str, now: datetime, + output: Reader | None) -> list[str]: + """What status prints; `output` None off the live system""" + if not overlay or not overlay.get("address"): + return ["this node is in no mesh: keel mesh create makes one, keel" + " mesh join joins one"] + iface = wireguard.interface(overlay) + found = [f"this node: {overlay['address']} on {iface}, WireGuard on" + f" UDP {wireguard.port(overlay)}"] + if output is None: + found.append("not the live system: no handshake read") + endpoints = handshakes = {} + else: + public = (output(("wg", "show", iface, "public-key")) or "").strip() + found.append("public key: " + f"{public or 'unknown (is the interface up?)'}") + endpoints = table(output(("wg", "show", iface, "endpoints"))) + handshakes = table(output(("wg", "show", iface, + "latest-handshakes"))) + peers = overlay.get("peers") or [] + found.append(f"peers: {len(peers)}") + for peer in peers: + key = str(peer.get("public_key")) + live_key = next((one for one in endpoints | handshakes + if same_key(one, key)), None) + endpoint = endpoints.get(live_key) or peer.get("endpoint") + if endpoint in (None, "(none)"): + endpoint = "no endpoint" + state = (handshake(handshakes.get(live_key), now) + if output is not None else "") + found.append(f" {key} {', '.join(allowed(peer))} {endpoint}" + f"{' ' + state if state else ''}") + waiting = invites.pending(root, now) + found.append(f"pending invites: {len(waiting)}") + for one in waiting: + used = "used, waiting for its confirmation" if one.consumed \ + else "pending" + address = ipaddress.IPv6Interface(one.address) + found.append(f" {one.invite_id} {address} TCP {one.https_port}" + f" until {shown(one.expires)} {used}") + return found diff --git a/keel/network/confirm.py b/keel/network/confirm.py index 01a771c..1802e66 100644 --- a/keel/network/confirm.py +++ b/keel/network/confirm.py @@ -9,6 +9,12 @@ process attached from the container's host has seen the machine and needs no such proof. +Decision 0048 adds two sources, for an overlay change keel mesh made and +no other: a join's authenticated session over the overlay (session.MESH), +and `keel mesh create` for a mesh with no peer yet (session.SELF). Each +names the change it made, the marker it read back after its apply, and +confirms only that one, after the same route check. + Two limits are said rather than hidden: a session tests one family, so a static address of the other family is reported as not tested; and a client on the same link reaches the machine without its gateway, so when @@ -42,18 +48,28 @@ class Probes: gateways: Callable[[], list[str] | None] = lambda: [] +MESH_ORIGINS = (session.MESH, session.SELF) + + def confirm(root: str, origin: session.Origin, probes: Probes, - run: switch.Runner) -> tuple[bool, list[str]]: - """(confirmed, what to tell the operator)""" + run: switch.Runner, expected: marker.Pending | None = None, + clients: tuple[str, ...] = ()) -> tuple[bool, list[str]]: + """(confirmed, what to tell the operator) + + `expected` is the change keel mesh made, which a MESH or SELF origin + may confirm and no other; `clients` are the operator's SSH clients + the route check asks too, for an origin that is not their session. + """ with marker.locked(root): if not marker.exists(root): return nothing_waiting(marker.last(root)) pending = marker.read(root) - refusal = not_ready(pending, probes.boot_id()) + refusal = not_ready(pending, probes.boot_id()) or not_made_here( + pending, origin, expected) if refusal: return False, [refusal] overlay = pending.kind == marker.OVERLAY - refusal = captured(pending, origin, probes) or ( + refusal = captured(pending, origin, probes, clients) or ( overlay_not_proof if overlay else not_proof)( pending, origin, probes) if refusal: @@ -113,6 +129,25 @@ def not_ready(pending: marker.Pending | None, boot_id: str | None) -> ( return None +def not_made_here(pending: marker.Pending, origin: session.Origin, + expected: marker.Pending | None) -> str | None: + """A mesh origin confirms the overlay change keel mesh made, alone + + The change is known by the boot and the moment it came up, which + the marker records once the interface is up (decision 0048, "It + confirms only its own change"). + """ + if origin.kind not in MESH_ORIGINS: + return None + if expected is None or pending.kind != marker.OVERLAY or ( + expected.boot_id, expected.changed_at) != ( + pending.boot_id, pending.changed_at): + return ("refused: the network change waiting is not the one keel" + f" mesh made, so {origin.detail} cannot confirm it;" + f" {LEFT_TO_REVERT}") + return None + + def not_proof(pending: marker.Pending, origin: session.Origin, probes: Probes) -> str | None: if origin.kind in (session.CONSOLE_KIND, session.HOST): @@ -164,9 +199,20 @@ def overlay_not_proof(pending: marker.Pending, origin: session.Origin, make pairing impossible, since the first node's overlay carries nothing until the second node has declared it too. confirm then says which path was tested and which was not. + + keel mesh's own sources (decision 0048): `create` for a mesh with no + peer, and a join's session, which must arrive at an address the + overlay declares; WireGuard binds its source to the key of the peer + the change added, which the listener checks. """ - if origin.kind in (session.CONSOLE_KIND, session.HOST): + if origin.kind in (session.CONSOLE_KIND, session.HOST, session.SELF): return None + if origin.kind == session.MESH: + if origin.local and same_address(origin.local, pending.addresses): + return None + return (f"refused: {origin.detail} arrived at {origin.local}, not" + " at an address the overlay declares" + f" ({', '.join(pending.addresses)})") if origin.kind != session.SSH: return f"refused: {origin.detail}" if origin.started is None or origin.started <= pending.changed_at: @@ -199,7 +245,7 @@ def overlay_not_proof(pending: marker.Pending, origin: session.Origin, def captured(pending: marker.Pending, origin: session.Origin, - probes: Probes) -> str | None: + probes: Probes, clients: tuple[str, ...] = ()) -> str | None: """An overlay change that routes the uplink into itself Whoever confirms, a console included, would keep a change that cuts @@ -209,7 +255,8 @@ def captured(pending: marker.Pending, origin: session.Origin, to each gateway the spec declared, to each gateway of a default route in place (what DHCP, SLAAC or a container's host configured, which the spec does not show, and what --skip-uplink left), IPv6 first, and - back to the client of an SSH session that came over the uplink. One + back to the client of an SSH session that came over the uplink, and + to the operator's SSH `clients` keel mesh names. One through the overlay refuses; so does one `ip` gives no answer for, since an unknown route is not a clean one. """ @@ -220,7 +267,8 @@ def captured(pending: marker.Pending, origin: session.Origin, return ("refused: `ip route show default` gave no answer, so it" " cannot be told whether the change routes the uplink into" f" the overlay; {LEFT_TO_REVERT}") - for address, what in route_targets(pending, origin, live_gateways): + for address, what in route_targets(pending, origin, live_gateways, + clients): dev = probes.route_dev(address) if dev is None: return (f"refused: `ip route get {address}` ({what}) gave no" @@ -235,7 +283,8 @@ def captured(pending: marker.Pending, origin: session.Origin, def route_targets(pending: marker.Pending, origin: session.Origin, - live_gateways: list[str]) -> list[tuple[str, str]]: + live_gateways: list[str], + clients: tuple[str, ...] = ()) -> list[tuple[str, str]]: """(address, what it is) to ask the route of, gateways IPv6 first A session that came over the overlay is answered through it, as it @@ -252,12 +301,21 @@ def route_targets(pending: marker.Pending, origin: session.Origin, if origin.kind == session.SSH and origin.peer and origin.local and \ not same_address(origin.local, pending.addresses): found.append((origin.peer, "this session's client")) + found += [(client, "the operator's SSH client") for client in clients] return found def overlay_lines(pending: marker.Pending, origin: session.Origin, probes: Probes) -> list[str]: """Which of the two paths this confirmation tested""" + if origin.kind == session.SELF: + return ["the overlay has no peer yet, so nothing can cross it:" + " keel mesh create confirmed it once the routes to the" + " gateways and to the operator's session were found to" + " still leave through the uplink"] + if origin.kind == session.MESH: + return [f"the overlay was tested: {origin.detail} arrived at" + f" {origin.local} on {pending.iface}, from {origin.peer}"] if origin.kind != session.SSH or origin.local is None: return [] if same_address(origin.local, pending.addresses): diff --git a/keel/network/session.py b/keel/network/session.py index dcdb4a3..3bb9a48 100644 --- a/keel/network/session.py +++ b/keel/network/session.py @@ -39,6 +39,12 @@ CONSOLE_KIND = "console" HOST = "host" UNKNOWN = "unknown" +# decision 0048: sources keel mesh hands confirm itself, never read from +# the process tree. MESH is a join's authenticated session over the +# overlay, from the peer the change added; SELF is `keel mesh create` +# confirming an overlay with no peer, which no peer can. +MESH = "mesh" +SELF = "self" @dataclass(frozen=True) diff --git a/pyproject.toml b/pyproject.toml index 9f126e1..c81bd1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "keel" -version = "0.16.0" +version = "0.17.0" authors = [{name = "KeelLinux maintainers"}] description = "Declarative management of one appliance instance" readme = "README.md" diff --git a/tests/README.md b/tests/README.md index 612d5ed..639380e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -55,6 +55,14 @@ needs the `wireguard` kernel module. Without the tools those tests skip, except in CI, where the workflow installs `wireguard-tools` (tests/wgtools.py). +`keel mesh` (decision 0048) is tested end to end by +`tests/test_mesh_netns.py`, which runs `tests/mesh_netns.py` as root in +a network namespace the same way, with two more namespaces joined to it +by veths, the real `wg-quick`, real TLS on the namespaces' addresses and +`ping` over the overlay; it needs the `wireguard` kernel module, and +skips without it outside CI. `tests/test_mesh_channel.py` serves the +real TLS listener on `[::1]`. + The one shell file the package ships, the firstboot hook `firstboot.d/10keel-system`, is tested with bats in `tests/hook.bats`, with `keel` replaced by a script on `PATH` that records its arguments and diff --git a/tests/mesh_helpers.py b/tests/mesh_helpers.py new file mode 100644 index 0000000..c97fa45 --- /dev/null +++ b/tests/mesh_helpers.py @@ -0,0 +1,113 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""What the mesh tests share: a pending invite, a node that records + +`FakeNode` stands in for keel.mesh.node.Node at the listener's and the +commands' seam: it records the peers it is asked to admit and the +origins it is asked to confirm, and answers as told. `reserved` writes +a pending invite with a real certificate from openssl (a dependency of +keel), so the TLS tests serve and pin a real one. +""" + +import json +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from keel.mesh import certificate, invites, protocol +from keel.mesh.node import Change +from keel.mesh.token import hmac_key, invite_id +from keel.network import marker + +SECRET = bytes(range(32)) +KEY = hmac_key(SECRET) +INVITE = invite_id(SECRET) +JOINER = "FHKH10gOWeK2bXHZPg8y+oPTprv556bwrmmRkbyEPgg=" +INVITER = "nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E=" +OTHER = "9vm/AzCVQQsWt1cU2eyjom4cABkNuodiHL8nDjKohEE=" +NOW = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) +ASSIGNED = "fd00:6b65:1::3/64" +OWN = "fd00:6b65:1::1/64" +MADE = marker.Pending(iface="wg0", path="etc/wireguard/wg0.conf", + window=120, addresses=("fd00:6b65:1::1",), + kind=marker.OVERLAY).up("b1", 50.0) + +_TLS: list[tuple[str, str]] = [] + + +def tls() -> tuple[str, str]: + """One key and certificate for the whole run: openssl takes a while""" + if not _TLS: + _TLS.append(certificate.make()) + return _TLS[0] + + +def reserved(root: str, port: int = 51820, expires: datetime | None = None, + secret: bytes = SECRET) -> invites.Pending: + key, cert = tls() + made = invites.Pending( + invite_id=invite_id(secret), address=ASSIGNED, + expires=expires or NOW + timedelta(hours=1), https_port=port, + certificate=cert, hmac_key=hmac_key(secret), tls_key=key) + return invites.reserve(root, NOW, lambda others: made) + + +@dataclass +class FakeNode: + is_waiting: bool = False + code: int = 0 + made: marker.Pending | None = MADE + confirmed: bool = True + known: tuple[protocol.Peer, ...] = () + admitted: list[dict] = field(default_factory=list) + origins: list = field(default_factory=list) + overlay_value: dict = field(default_factory=dict) + # what `wg show IFACE latest-handshakes` gives for the new node's key + handshake_at: int | None = int(NOW.timestamp()) + asked: list[str] = field(default_factory=list) + + def handshake(self, key: str) -> int | None: + self.asked.append(key) + return self.handshake_at + + def overlay(self) -> dict: + return self.overlay_value + + def waiting(self) -> bool: + return self.is_waiting + + def admit(self, peer: dict, window: int | None = None) -> Change: + self.admitted.append(peer) + return Change(self.code, self.made) + + def peers(self, but: str) -> tuple[protocol.Peer, ...]: + return self.known + + def confirm(self, origin, made): + self.origins.append((origin, made)) + return self.confirmed, [f"confirmed from {origin.detail}"] + + +class Clock: + def __init__(self, now: datetime = NOW): + self.now = now + + def __call__(self) -> datetime: + return self.now + + +def join_body(**changed) -> bytes: + data = {"invite_id": INVITE, "public_key": JOINER, + "endpoint": "[2001:db8:2::20]:51820", "address": ASSIGNED, + "nonce": protocol.new_nonce(), "time": protocol.seconds(NOW)} + data.update(changed) + return json.dumps(data).encode() + + +def confirm_body(**changed) -> bytes: + data = {"invite_id": INVITE, "public_key": JOINER, + "nonce": protocol.new_nonce(), "time": protocol.seconds(NOW)} + data.update(changed) + return json.dumps(data).encode() + + +def signature(path: str, body: bytes, key: bytes = KEY) -> str: + return protocol.sign(key, protocol.METHOD, path, body) diff --git a/tests/mesh_netns.py b/tests/mesh_netns.py new file mode 100644 index 0000000..c21eea0 --- /dev/null +++ b/tests/mesh_netns.py @@ -0,0 +1,343 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""Three nodes in three network namespaces, joined with keel mesh + +Run as root in a network namespace of its own by tests/test_mesh_netns.py +(tests/wgtools.py says how); never run by hand on a machine. This +namespace is the inviter A. Two children, B and C, are network +namespaces of their own (`unshare -n`), each joined to A by a veth on +its own /64. Then, with the real wg-quick, the real TLS listener and +client, the real flows and the real confirmation logic of 0018: + +1. A is in the mesh: its overlay up, and `keel mesh invite` through + keel.cli prints the line; +2. A's listener serves the invite (keel.mesh.inviting.serve_invite) and + B joins with the token (keel.mesh.joining), both windows confirmed + by the mesh session over the overlay; B pings A over the overlay; +3. a second invite is made with nothing listening: C's join gets no + TCP answer, applies its side and prints `keel mesh accept`, which A + runs (keel.mesh.inviting.accept); both windows are confirmed over + the overlay, and C pings A; +4. the second token used again, from a fresh scratch root on C: the + port gives no answer, so the join falls back again, and A refuses + its line: the invite is spent. + +Two things are not keel's own here. apply: a live apply arms systemd +timers on the host, which no namespace isolates, so apply writes keel's +rendered wg-quick file under the node's scratch root, runs `wg-quick up` +on it in the node's namespace, and records the change as apply does +(keel.network.marker); everything that confirms a window is the real +keel.network.confirm with the real `ip` probes. And the listener's +unit: the root side starts `keel mesh listen` as a process of its own +under `setpriv` with every capability dropped and no new privileges, +where keel-mesh-listen@ adds a dynamic user and systemd's sandbox; the +bridge checks it is that process (SO_PEERCRED), and the listener, which +refuses to run with any capability, reports its CapEff. + +With MESH_NETEM set (`250ms 25ms 2%`, say) every veth, both ends, +delays by the first, with the second as jitter, and loses the third: +a poor network. + +The result is one JSON line on standard output, `RESULT {...}`. +""" + +import contextlib +import io +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +from datetime import datetime, timezone + +from keel import cli +from keel.mesh import inviting, joining +from keel.mesh.node import Node +from keel.mesh.token import parse +from keel.network import marker, wireguard + +PREFIX = "fd00:6b65:e2e" +OVERLAY_A = f"{PREFIX}::1" +LINKS = {"B": "2001:db8:e2e:1", "C": "2001:db8:e2e:2"} + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +class Recorder: + """systemctl, systemd-run and nft: nothing of the host is touched""" + + def __init__(self): + self.calls = [] + + def __call__(self, argv): + self.calls.append(argv) + + +def sh(*argv: str, pid: int | None = None) -> str: + prefix = ["nsenter", "-t", str(pid), "-n"] if pid else [] + done = subprocess.run(prefix + list(argv), capture_output=True, + text=True, check=False) + if done.returncode: + raise RuntimeError(f"{' '.join(argv)}: {done.stderr.strip()}") + return done.stdout + + +def netns_apply(doc: dict, root: str, window: int) -> int: + """apply's overlay step, in this namespace, under the node's root""" + overlay = doc["network"]["overlay"]["wireguard"] + iface = wireguard.interface(overlay) + relative = wireguard.conf_path(iface) + conf = os.path.join(root, relative) + os.makedirs(os.path.dirname(conf), exist_ok=True) + # the key is under the scratch root, where keel made it + key = os.path.join(root, wireguard.key_path(overlay).lstrip("/")) + with open(os.open(conf, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), + "w") as fob: + fob.write(wireguard.render({**overlay, "private_key": {"file": key}})) + subprocess.run(["wg-quick", "down", conf], capture_output=True, + check=False) + up = subprocess.run(["wg-quick", "up", conf], capture_output=True, + text=True, check=False) + if up.returncode: + print(up.stderr, file=sys.stderr) + return 16 + marker.save(root, "", marker.Target(relative, marker.OVERLAY, True)) + marker.write(root, marker.Pending( + iface=iface, path=relative, window=window, + addresses=tuple(one.split("/")[0] + for one in wireguard.addresses(overlay)), + kind=marker.OVERLAY, absent=True).up(marker.boot_id(), + marker.process_clock())) + return 0 + + +def scratch(spec: str) -> tuple[str, str]: + root = tempfile.mkdtemp(prefix="keel-mesh-") + path = os.path.join(root, "instance.yaml") + with open(path, "w") as fob: + fob.write(spec.format(root=root)) + return root, path + + +def node(root: str, path: str) -> Node: + return Node(root, path, apply=netns_apply, run=Recorder()) + + +def ping(address: str) -> bool: + """One answer of five is enough: the link may lose packets""" + return subprocess.run(["ping", "-6", "-c", "5", "-i", "0.3", "-W", "3", + address], capture_output=True, + check=False).returncode == 0 + + +def joiner(token_text: str, endpoint: str) -> None: + """In B's or C's namespace: join, then ping A over the overlay""" + root, path = scratch("version: 1\n") + out, err = [], [] + + def said(line): + out.append(line) + print(line, flush=True) + found = joining.Joiner(node(root, path), parse(token_text, utcnow()), + utcnow, said, err.append) + code = joining.run(found, endpoint) + last = marker.last(root) + print("RESULT " + json.dumps({ + "code": code, "out": out, "err": err, "ping": ping(OVERLAY_A), + "outcome": last.outcome if last else None}), flush=True) + + +CHILDREN: list[subprocess.Popen] = [] +CAPABILITIES: list[str] = [] +NETEM = os.environ.get("MESH_NETEM", "").split() + + +# The listener holds no capability, so as root without +# CAP_DAC_READ_SEARCH it cannot traverse a home directory the checkout +# may be in (a CI runner's): it imports a copy of keel kept beside the +# nodes' scratch roots, and logs where the driver can read it. +LISTENER_HOME = tempfile.mkdtemp(prefix="keel-mesh-listener-") +LISTENER_LOG = os.path.join(LISTENER_HOME, "listener.log") +shutil.copytree(os.path.dirname(os.path.dirname(os.path.abspath( + inviting.__file__))), os.path.join(LISTENER_HOME, "keel")) + + +class Spawned: + """`keel mesh listen` as its own process, with no capability""" + + def __init__(self, invite_id: str, path: str, seconds: int): + with open(LISTENER_LOG, "a") as log: + self.process = subprocess.Popen( + ["setpriv", "--no-new-privs", "--bounding-set", "-all", + "--inh-caps", "-all", "--ambient-caps", "-all", + sys.executable, "-m", "keel", "mesh", "listen", path], + stdout=log, stderr=log, cwd=LISTENER_HOME, + env={**os.environ, "PYTHONPATH": LISTENER_HOME}) + + def trusted(self, pid: int, uid: int) -> bool: + with open(f"/proc/{pid}/status") as fob: + CAPABILITIES.extend(one.split()[1] for one in fob + if one.startswith("CapEff:")) + return pid == self.process.pid + + def stop(self) -> None: + self.process.terminate() + self.process.wait(10) + + +def shaped(device: str, pid: int | None = None) -> None: + if NETEM: + delay, jitter, loss = NETEM + sh("tc", "qdisc", "add", "dev", device, "root", "netem", "delay", + delay, jitter, "loss", loss, pid=pid) + + +def listening(host: str, port: int) -> None: + """Wait until the listener's port takes a connection""" + for _ in range(300): + try: + socket.create_connection((host, port), 1).close() + return + except OSError: + time.sleep(0.1) + with open(LISTENER_LOG) as fob: + said = fob.read()[-2000:] + raise RuntimeError(f"nothing listens on [{host}]:{port}; the" + f" listener said: {said}") + + +def link(name: str) -> int: + """A child namespace, and a veth from here to it on its own /64""" + child = subprocess.Popen(["unshare", "-n", "sleep", "600"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + CHILDREN.append(child) + net = LINKS[name] + sh("ip", "link", "add", f"to{name}", "type", "veth", "peer", "name", + "uplink") + sh("ip", "link", "set", "uplink", "netns", str(child.pid)) + sh("ip", "-6", "address", "add", f"{net}::1/64", "dev", f"to{name}", + "nodad") + sh("ip", "link", "set", f"to{name}", "up") + for argv in (("ip", "link", "set", "lo", "up"), + ("ip", "-6", "address", "add", f"{net}::2/64", "dev", + "uplink", "nodad"), + ("ip", "link", "set", "uplink", "up")): + sh(*argv, pid=child.pid) + shaped(f"to{name}") + shaped("uplink", child.pid) + return child.pid + + +def invite(root: str, path: str, name: str, port: int) -> str: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = cli.main(["mesh", "invite", "--root", root, "--spec", path, + "--endpoint", f"{LINKS[name]}::1", "--port", + str(port)]) + if code: + raise RuntimeError(err.getvalue()) + return out.getvalue().split()[-1] + + +def run_joiner(pid: int, token_text: str, name: str) -> subprocess.Popen: + return subprocess.Popen( + ["nsenter", "-t", str(pid), "-n", sys.executable, __file__, "join", + token_text, f"{LINKS[name]}::2"], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) + + +def result(process: subprocess.Popen) -> dict: + out, err = process.communicate(timeout=300) + for line in out.splitlines(): + if line.startswith("RESULT "): + return json.loads(line[len("RESULT "):]) + raise RuntimeError(f"no result: {out}\n{err}") + + +def accept_line(process: subprocess.Popen) -> str: + """The line a joiner that fell back prints, as an operator reads it""" + for text in process.stdout: + if text.startswith("keel mesh accept "): + return text.split()[-1] + return "" + + +def inviter() -> None: + sh("ip", "link", "set", "lo", "up") + pids = {name: link(name) for name in LINKS} + root, path = scratch("version: 1\nnetwork:\n overlay:\n wireguard:\n" + f" address: {OVERLAY_A}/64\n") + key = os.path.join(root, "etc/wireguard/wg0.key") + os.makedirs(os.path.dirname(key)) + with open(os.open(key, os.O_WRONLY | os.O_CREAT, 0o600), "w") as fob: + subprocess.run(["wg", "genkey"], stdout=fob, check=True) + a = node(root, path) + netns_apply(a.document(), root, 120) + marker.clear(root) + journal = [] + report = {} + + # 1-2. the root helper and its unprivileged listener, and B joins + first = invite(root, path, "B", 51900) + a_side = inviting.Inviter(a, utcnow, journal.append, journal.append, + run=Recorder(), output=lambda argv: None, + start=Spawned) + codes = [] + thread = threading.Thread(target=lambda: codes.append( + inviting.serve_invite(a_side, parse(first, utcnow()).invite_id))) + thread.start() + listening(f"{LINKS['B']}::1", 51900) + report["b"] = result(run_joiner(pids["B"], first, "B")) + thread.join(120) + report["serve"] = codes + report["a_after_b"] = marker.last(root).outcome + + # 3. nothing listens on the second invite's port: the fallback + second = invite(root, path, "C", 51901) + accepting = inviting.Inviter(a, utcnow, journal.append, journal.append, + run=Recorder(), output=lambda argv: None, + start=Spawned) + process = run_joiner(pids["C"], second, "C") + report["accept"] = inviting.accept(accepting, accept_line(process)) + rest, err = process.communicate(timeout=300) + report["c"] = next( + (json.loads(one[len("RESULT "):]) for one in rest.splitlines() + if one.startswith("RESULT ")), {"err": err}) + report["a_after_c"] = marker.last(root).outcome + + # 4. the second token again: A refuses the line, the join is stopped + process = run_joiner(pids["C"], second, "C") + refused = [] + again = inviting.Inviter(a, utcnow, refused.append, refused.append, + run=Recorder(), output=lambda argv: None, + start=Spawned) + report["again"] = [inviting.accept(again, accept_line(process)), + refused] + process.kill() + process.wait() + report["peers"] = [peer.get("persistent_keepalive") + for peer in a.overlay()["peers"]] + report["invites_left"] = os.listdir(os.path.join( + root, "var/lib/keel/mesh/invites")) + report["journal"] = journal + report["tokens"] = [first, second] + report["capabilities"] = CAPABILITIES + report["netem"] = NETEM + print("RESULT " + json.dumps(report), flush=True) + + +if __name__ == "__main__": + if sys.argv[1:2] == ["join"]: + joiner(sys.argv[2], sys.argv[3]) + else: + try: + inviter() + finally: + for one in CHILDREN: + one.kill() diff --git a/tests/mesh_wire.py b/tests/mesh_wire.py new file mode 100644 index 0000000..3c7f2b7 --- /dev/null +++ b/tests/mesh_wire.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""Two nodes in one process: a real Node each, a wire between them + +`node()` is keel.mesh.node.Node on a scratch root with apply replaced +by `Armed`, which arms the overlay change a live apply would (the marker +of decision 0018, dated as up) and records the document; the real +keel.network.confirm decides each confirmation, with probes that say +every route leaves through eth0. `Wire` stands for channel.post: it +hands the joiner's request to the inviter's Listener.handle with the +addresses a connection would carry, over the uplink for a join and +over the overlay for the mesh session, and checks the answer's HMAC as +the client does. +""" + +import json +import os +from datetime import timedelta + +from mesh_helpers import ASSIGNED, INVITER, JOINER, MADE, NOW, OWN, SECRET +from test_network_window import Recorder + +from keel.mesh import certificate, protocol +from keel.mesh.channel import Forged, Refused, Reply, Unreachable +from keel.mesh.node import Node +from keel.mesh.token import Token, invite_id +from keel.network import marker, wireguard +from keel.network.confirm import Probes + +INVITER_UPLINK = "2001:db8:1::10" +JOINER_UPLINK = "2001:db8:2::20" +INVITER_OVERLAY = "fd00:6b65:1::1" +JOINER_OVERLAY = "fd00:6b65:1::3" +INVITER_SPEC = f"""\ +version: 1 +network: + overlay: + wireguard: + address: {OWN} + listen_port: 51821 +""" + + +class Armed: + """apply, as far as keel mesh sees it: the overlay change armed""" + + def __init__(self, code: int = 0, arms: bool = True): + self.code = code + self.arms = arms + self.documents = [] + + def __call__(self, doc: dict, root: str, window: int) -> int: + self.documents.append((doc, window)) + if self.arms: + overlay = doc["network"]["overlay"]["wireguard"] + iface = wireguard.interface(overlay) + marker.save(root, "", marker.Target( + wireguard.conf_path(iface), marker.OVERLAY, True)) + marker.write(root, marker.Pending( + iface=iface, path=wireguard.conf_path(iface), window=window, + addresses=tuple(one.split("/")[0] for one in + wireguard.addresses(overlay)), + kind=marker.OVERLAY, absent=True).up("b1", 50.0)) + return self.code + + +def probes(route: str = "eth0") -> Probes: + """Every route, the default one's gateway's included, through `route`""" + return Probes(boot_id=lambda: "b1", addresses=lambda iface: [], + route_via=lambda peer: None, holder=lambda address: None, + route_dev=lambda address: route, + gateways=lambda: ["2001:db8:1::1"]) + + +def node(root: str, text: str | None = None, apply=None, + route: str = "eth0", clients=()) -> Node: + path = os.path.join(root, "instance.yaml") + if text is not None: + with open(path, "w") as fob: + fob.write(text) + found = Node(root, path, clients=clients, apply=apply or Armed(), + probes=lambda: probes(route), run=Recorder(), + output=handshakes) + return found + + +def handshakes(argv: tuple[str, ...]) -> str | None: + """`wg show wg0 latest-handshakes`: the new node's key, just now""" + if argv[-1] == "latest-handshakes": + return f"{JOINER}\t{int(NOW.timestamp())}\n" + return None + + +def token(invite, **changed) -> Token: + values = dict( + public_key=INVITER, endpoints=(INVITER_UPLINK,), port=51821, + https_port=51820, + fingerprint=certificate.fingerprint(invite.certificate), + address=OWN, assigned=ASSIGNED, mesh_id=bytes(range(16)), + invite_id=invite_id(SECRET), expires=NOW + timedelta(hours=1), + secret=SECRET) + values.update(changed) + return Token(**values) + + +class Wire: + """channel.post, through a Listener in this process""" + + def __init__(self, listener, pinned: bytes): + self.listener = listener + self.pinned = pinned + self.unreachable: set[str] = set() + self.calls: list[tuple[str, str]] = [] + + def __call__(self, host, port, path, body, key, pinned, **timeouts): + self.calls.append((host, path)) + if path in self.unreachable: + raise Unreachable(f"[{host}]:{port}: Connection timed out") + if pinned != self.pinned: + raise Forged("another certificate") + if path == protocol.JOIN: + local, peer = INVITER_UPLINK, JOINER_UPLINK + else: + local, peer = host, JOINER_OVERLAY + found = self.listener.handle( + protocol.METHOD, path, + protocol.sign(key, protocol.METHOD, path, body), body, local, + peer) + if found.status != 200: + raise Refused(json.loads(found.body)["error"]) + if not protocol.signed(key, protocol.ANSWER, path, found.body, + found.signature): + raise Forged("not signed") + return Reply(found.body, peer, local) + + +__all__ = ["MADE", "Wire", "node", "token"] diff --git a/tests/test_manifest_firewall.py b/tests/test_manifest_firewall.py index 1a52c29..d13a4e3 100644 --- a/tests/test_manifest_firewall.py +++ b/tests/test_manifest_firewall.py @@ -95,6 +95,19 @@ def test_cloud_advanced_opens_the_mesh_on_the_overlay_only(self): self.assertNotIn("8080", found.text) self.assertNftAccepts(found.text) + def test_a_named_set_for_the_pending_invites_ports(self): + """keel mesh invite adds its TCP port to the set with a timeout, + so the port is open while the invite is pending and closes by + itself (decision 0048); empty while none is""" + found = self.render(ADVANCED, WG) + self.assertIn("\tset mesh_invites {\n\t\ttype inet_service\n" + "\t\tflags timeout\n\t}\n\tchain input {\n", + found.text) + self.assertIn("\t\ttcp dport @mesh_invites accept\n", found.text) + self.assertNftAccepts(found.text) + without = self.render(CORE_OFF) + self.assertNotIn("mesh_invites", without.text) + def test_the_overlay_s_defaults(self): found = self.render(ADVANCED, {"address": "fd00:1::1/64"}) self.assertIn('iifname "wg0"', found.text) diff --git a/tests/test_mesh_acceptline.py b/tests/test_mesh_acceptline.py new file mode 100644 index 0000000..11bf56f --- /dev/null +++ b/tests/test_mesh_acceptline.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The keel1a: line the fallback prints for the inviter (decision 0048) + +The new node's key, its endpoint, the reserved address, the invite id +and the time, with an HMAC keyed with the invite's HMAC key and a +checksum, so a mistyped paste and a line for another invite are told +apart and neither is accepted. +""" + +import unittest + +from keel.mesh import acceptline +from keel.mesh.acceptline import Accept, AcceptError + +KEY = bytes(range(32)) +JOINER = "FHKH10gOWeK2bXHZPg8y+oPTprv556bwrmmRkbyEPgg=" +LINE = Accept(public_key=JOINER, endpoint="[2001:db8:2::20]:51820", + address="fd00:6b65:1::3/64", invite_id="0123456789abcdef", + time=1_790_000_000) + + +class TestRoundTrip(unittest.TestCase): + def test_each_endpoint(self): + for endpoint in ("[2001:db8:2::20]:51820", "198.51.100.7:51821", + None): + with self.subTest(endpoint=endpoint): + made = Accept(**{**LINE.__dict__, "endpoint": endpoint}) + text = acceptline.encode(made, KEY) + self.assertTrue(text.startswith("keel1a:")) + self.assertLess(len(text), 200) + sealed = acceptline.parse(text) + self.assertEqual(sealed.accept, made) + self.assertTrue(sealed.authentic(KEY)) + self.assertFalse(sealed.authentic(bytes(32))) + + def test_the_whole_command_line_pasted_is_read(self): + text = acceptline.encode(LINE, KEY) + self.assertEqual(acceptline.parse(f" {text}\n").accept, LINE) + + +class TestRefused(unittest.TestCase): + def refused(self, text, words): + with self.assertRaises(AcceptError) as caught: + acceptline.parse(text) + self.assertIn(words, str(caught.exception)) + + def test_damaged(self): + text = acceptline.encode(LINE, KEY) + changed = text[:20] + ("A" if text[20] != "A" else "B") + text[21:] + for found, words in ( + ("x" * 2000, "more than"), + ("keel1:abc", "starts with keel1a:"), + ("keel1a:ab*c", "base64url"), + ("keel1a:", "mistyped or truncated"), + ("keel1a:AAAAA", "mistyped or truncated"), + (changed, "mistyped or truncated"), + (text[:-3], "mistyped or truncated")): + with self.subTest(found=found[:12]): + self.refused(found, words) + + def test_a_layout_its_flags_do_not_match(self): + for payload, words in ( + (bytes([4]) + bytes(100), "flags"), + (bytes([3]) + bytes(100), "flags"), + (bytes([1]) + bytes(10), "malformed"), + (bytes([0]) + bytes(32) + bytes(61) + b"x", "bytes where")): + with self.subTest(payload=payload[:2]): + self.refused(acceptline.wrap(payload), words) + + def test_values_no_inviter_could_use(self): + for changed, words in ( + ({"public_key": "A" * 43 + "B"}, "not a WireGuard key"), + ({"endpoint": "[::1]:51820"}, "cannot be reached"), + ({"address": "2001:db8::3/64"}, "not an overlay address")): + with self.subTest(changed=changed): + with self.assertRaises(AcceptError) as caught: + acceptline.encode(Accept(**{**LINE.__dict__, **changed}), + KEY) + self.assertIn(words, str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_admit.py b/tests/test_mesh_admit.py new file mode 100644 index 0000000..80093b3 --- /dev/null +++ b/tests/test_mesh_admit.py @@ -0,0 +1,337 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The root side of an invite, its handler a function (decision 0048) + +Admitter.forward against a scratch root's pending invite and a node that +records (tests/mesh_helpers.py): every refusal with its status and +reason, checked again whatever the listener checked, one use under the +lock, replays, a key the inviter already knows, the join that admits +the new node and the confirmation that must arrive over the overlay. +""" + +import json +import os +import shutil +import tempfile +import unittest +from os.path import join +from unittest import mock + +from mesh_helpers import ( + INVITE, + INVITER, + JOINER, + KEY, + MADE, + NOW, + OTHER, + OWN, + Clock, + FakeNode, + confirm_body, + join_body, + reserved, + signature, +) + +from keel.mesh import admit, invites, protocol +from keel.mesh.admit import Admitter, stopped +from keel.mesh.node import NodeError +from keel.mesh.protocol import Peer +from keel.network import session + +UPLINK = "2001:db8:2::20" +OVERLAY_OWN = "fd00:6b65:1::1" +OVERLAY_NEW = "fd00:6b65:1::3" + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.invite = reserved(self.root) + self.node = FakeNode() + self.clock = Clock() + self.logged = [] + self.slept = [] + self.listener = Admitter(self.root, self.invite, INVITER, OWN, + self.node, self.clock, self.logged.append, + sleep=self.slept.append) + + def join(self, body=None, sign=None, path=protocol.JOIN, peer=UPLINK): + body = join_body() if body is None else body + found = sign if sign is not None else signature(path, body) + return self.listener.forward(path, found, body, "::", peer) + + def confirm(self, body=None, local=OVERLAY_OWN, peer=OVERLAY_NEW, + sign=None): + body = confirm_body() if body is None else body + found = sign if sign is not None else signature(protocol.CONFIRM, + body) + return self.listener.forward(protocol.CONFIRM, found, body, + local, peer) + + def error(self, response): + return json.loads(response.body)["error"] + + def file(self): + return join(self.root, invites.INVITES, f"{INVITE}.json") + + +class TestJoin(Case): + def test_a_join_admits_the_new_node_and_answers_signed(self): + self.node.known = (Peer(OTHER, "[2001:db8:3::30]:51820", + "fd00:6b65:1::7"),) + body = join_body() + found = self.join(body) + self.assertEqual(found.status, 200, found.body) + self.assertTrue(protocol.signed(KEY, protocol.ANSWER, protocol.JOIN, + found.body, found.signature)) + answer = protocol.join_answer(found.body) + self.assertEqual(answer.nonce, json.loads(body)["nonce"]) + self.assertEqual((answer.invite_id, answer.public_key, + answer.address, answer.etcd, answer.window), + (INVITE, INVITER, OWN, "none", 120)) + self.assertEqual(answer.peers, self.node.known) + self.assertEqual(self.node.admitted, [{ + "public_key": JOINER, "allowed_ips": [f"{OVERLAY_NEW}/128"], + "endpoint": "[2001:db8:2::20]:51820"}]) + self.assertTrue(invites.read(self.root, INVITE).consumed) + self.assertIsNone(self.listener.confirmed) + self.assertIn(f"invite {INVITE}: join from {UPLINK}, key {JOINER}," + f" endpoint [2001:db8:2::20]:51820, at {OVERLAY_NEW}", + self.logged) + + def test_a_new_node_without_an_endpoint(self): + self.assertEqual(self.join(join_body(endpoint=None)).status, 200) + self.assertNotIn("endpoint", self.node.admitted[0]) + self.assertIn("endpoint none", self.logged[0]) + + def test_a_second_join_finds_the_invite_used(self): + self.join() + found = self.join() + self.assertEqual(found.status, 410) + self.assertIn("already used", self.error(found)) + self.assertEqual(len(self.node.admitted), 1) + + def test_an_invite_consumed_elsewhere_is_refused(self): + invites.consume(self.root, INVITE, NOW) + found = self.join() + self.assertEqual(found.status, 410) + self.assertIn("already used", self.error(found)) + self.assertEqual(self.node.admitted, []) + + def test_a_change_that_did_not_come_up_spends_the_invite(self): + self.node.made, self.node.code = None, 16 + found = self.join() + self.assertEqual(found.status, 500) + self.assertIn("could not apply", self.error(found)) + self.assertIs(self.listener.confirmed, False) + self.assertIn("apply exited 16", self.logged[-1]) + + def test_a_spec_that_cannot_take_the_peer_spends_the_invite(self): + def refuse(peer, window=None): + raise NodeError("instance.yaml cannot be written: disk full") + self.node.admit = refuse + found = self.join() + self.assertEqual(found.status, 500) + self.assertIs(self.listener.confirmed, False) + self.assertIn("cannot be written: disk full; the invite is spent", + self.logged[-1]) + + def test_a_change_waiting_leaves_the_invite_pending(self): + self.node.is_waiting = True + found = self.join() + self.assertEqual(found.status, 409) + self.assertIn("another network change waits", self.error(found)) + self.assertFalse(invites.read(self.root, INVITE).consumed) + self.assertIsNone(self.listener.joined) + + +class TestRefusals(Case): + def test_each_refusal_with_its_status_and_reason(self): + good = join_body() + for kwargs, status, words in ( + (dict(path="/"), 404, "no such request"), + (dict(body=b"{}"), 400, "malformed request"), + (dict(body=join_body(invite_id="00" * 8)), 403, + "is not the one this listener serves"), + (dict(body=good, sign="00" * 32), 403, "bad HMAC"), + (dict(body=good, sign=""), 403, "bad HMAC"), + (dict(body=join_body(time=protocol.seconds(NOW) - 600)), + 403, "time is more than"), + (dict(body=join_body(address="fd00:6b65:1::9/64")), 403, + "not the address this invite reserved")): + with self.subTest(kwargs=list(kwargs)): + self.setUp() + found = self.join(**kwargs) + self.assertEqual(found.status, status) + self.assertIn(words, self.error(found)) + self.assertIsNone(found.signature) + self.assertEqual(self.node.admitted, []) + self.assertFalse(invites.read(self.root, INVITE).consumed) + self.assertIn("refused a request from 2001:db8:2::20", + self.logged[0]) + + def test_a_body_longer_than_any_join(self): + found = self.listener.forward(protocol.JOIN, None, None, "::", + UPLINK) + self.assertEqual(found.status, 413) + + def test_a_replayed_request_is_refused(self): + self.node.is_waiting = True + body = join_body() + self.assertEqual(self.join(body).status, 409) + self.node.is_waiting = False + found = self.join(body) + self.assertEqual(found.status, 403) + self.assertIn("replayed", self.error(found)) + + def test_a_cancel_removes_the_invite(self): + self.listener.cancel("5 requests named it and failed its HMAC") + self.assertFalse(os.path.exists(self.file())) + self.assertIn("cancelled, 5 requests named it", self.logged[-1]) + self.assertEqual(stopped(self.listener), f"invite {INVITE}: cancelled") + + def test_no_secret_in_the_log(self): + body = join_body() + sign = signature(protocol.JOIN, body) + self.join(body, sign) + self.join(sign="00" * 32) + logged = "\n".join(self.logged) + for secret in (KEY.hex(), sign, self.invite.tls_key): + self.assertNotIn(secret, logged) + + +class TestConfirm(Case): + def test_before_any_join(self): + found = self.confirm() + self.assertEqual(found.status, 409) + self.assertEqual(self.node.origins, []) + + def test_the_handshake_and_the_signed_confirmation(self): + self.join() + body = confirm_body() + found = self.confirm(body) + self.assertEqual(found.status, 200, found.body) + self.assertTrue(found.final) + self.assertTrue(protocol.signed(KEY, protocol.ANSWER, + protocol.CONFIRM, found.body, + found.signature)) + answer = protocol.confirm_answer(found.body) + self.assertTrue(answer.confirmed) + self.assertEqual(answer.nonce, json.loads(body)["nonce"]) + [(origin, made)] = self.node.origins + # the root side's own evidence: the addresses are its own facts + self.assertEqual((origin.kind, origin.local, origin.peer), + (session.MESH, OVERLAY_OWN, OVERLAY_NEW)) + self.assertIn("a WireGuard handshake from the new peer's key", + origin.detail) + self.assertEqual(self.node.asked, [JOINER]) + self.assertEqual(made, MADE) + self.assertIs(self.listener.confirmed, True) + self.assertEqual(stopped(self.listener), + f"invite {INVITE}: joined and confirmed") + + def test_a_refused_confirmation_is_answered_and_ends_it(self): + self.join() + self.node.confirmed = False + answer = protocol.confirm_answer(self.confirm().body) + self.assertFalse(answer.confirmed) + self.assertIs(self.listener.confirmed, False) + self.assertIn("not confirmed", stopped(self.listener)) + + def test_another_key_s_confirmation(self): + self.join() + found = self.confirm(body=confirm_body(public_key=OTHER)) + self.assertEqual(found.status, 403) + self.assertEqual(self.node.origins, []) + + def test_the_handshake_is_waited_for_a_little(self): + self.join() + seen = iter([None, None, int(NOW.timestamp())]) + self.node.handshake = lambda key: next(seen) + self.assertEqual(self.confirm().status, 200) + self.assertEqual(self.slept, [admit.HANDSHAKE_POLL] * 2) + + +class TestALyingListener(Case): + """What a compromised listener can forward, and what it gets: it + holds no HMAC key, and its addresses decide nothing""" + + def test_it_cannot_choose_the_joiner_s_key(self): + forged = join_body(public_key=OTHER) + found = self.listener.forward(protocol.JOIN, "00" * 32, forged, + "::", UPLINK) + self.assertEqual(found.status, 403) + self.assertIn("bad HMAC", self.error(found)) + self.assertEqual(self.node.admitted, []) + self.assertFalse(invites.read(self.root, INVITE).consumed) + + def test_it_cannot_confirm_without_the_tunnel(self): + """A join was admitted; the new node never came over the tunnel. + The listener reports the right addresses for a request it forged + or replayed: no handshake from the key, no confirmation""" + self.join() + self.node.handshake_at = None + found = self.confirm(local=OVERLAY_OWN, peer=OVERLAY_NEW) + self.assertEqual(found.status, 409) + self.assertIn("no WireGuard handshake", self.error(found)) + self.assertEqual(self.node.origins, []) + self.assertIsNone(self.listener.confirmed) + self.assertEqual(len(self.slept), int( + admit.HANDSHAKE_WAIT / admit.HANDSHAKE_POLL)) + + def test_a_handshake_older_than_the_join_is_not_the_tunnel(self): + self.join() + self.node.handshake_at = int(NOW.timestamp()) - 1 + self.assertEqual(self.confirm().status, 409) + + def test_its_addresses_decide_nothing(self): + self.join() + found = self.confirm(local="2001:db8:1::10", peer="2001:db8:9::9") + self.assertEqual(found.status, 200) + [(origin, _)] = self.node.origins + self.assertEqual((origin.local, origin.peer), + (OVERLAY_OWN, OVERLAY_NEW)) + + def test_five_forged_requests_naming_the_invite_cancel_it(self): + for number in range(5): + found = self.join(sign="00" * 32) + self.assertEqual(found.final, number == 4) + self.assertFalse(os.path.exists(self.file())) + self.assertTrue(self.listener.cancelled) + found = self.join() + self.assertEqual((found.status, found.final), (410, True)) + + def test_a_scanner_s_requests_cancel_nothing(self): + for _ in range(6): + self.join(body=join_body(invite_id="00" * 8), sign="00" * 32) + self.join(body=b"{}") + self.assertFalse(self.listener.cancelled) + self.assertTrue(os.path.exists(self.file())) + + +class TestLog(Case): + def test_the_journal_by_default(self): + listener = Admitter(self.root, self.invite, INVITER, OWN, self.node, + self.clock) + with mock.patch("builtins.print") as printed: + listener.forward("/", None, b"", "::", UPLINK) + self.assertIn("refused a request", printed.call_args.args[0]) + + +class TestKnownKeys(Case): + def test_a_key_the_inviter_knows_is_refused_before_it_is_spent(self): + for key in (INVITER, OTHER): + with self.subTest(key=key): + self.setUp() + self.node.overlay_value = {"peers": [{"public_key": OTHER}]} + found = self.join(join_body(public_key=key)) + self.assertEqual(found.status, 409) + self.assertIn("a key of its own", self.error(found)) + self.assertFalse(invites.read(self.root, INVITE).consumed) + self.assertEqual(self.node.admitted, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_bridge.py b/tests/test_mesh_bridge.py new file mode 100644 index 0000000..c6e3530 --- /dev/null +++ b/tests/test_mesh_bridge.py @@ -0,0 +1,365 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The bridge between the unprivileged listener and the root side + +The whole path in one process: Bridge.run binds its unix socket and +"starts" the listener as a thread running keel.mesh.bridge.listen, which +receives its parameters and the invite's certificate and TLS key as +memfds over SCM_RIGHTS, serves real TLS on [::1], and forwards to the +Admitter; then each refusal of the bridge: a peer that is not the +listener, one that never connects, messages it should not send, a +listener with capabilities. And the unit the listener runs as. +""" + +import base64 +import json +import os +import shutil +import socket +import tempfile +import threading +import unittest +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from unittest import mock + +from mesh_helpers import ( + INVITER, + KEY, + OWN, + FakeNode, + confirm_body, + join_body, + reserved, + signature, +) +from test_mesh_channel import free_port +from test_network_window import Recorder + +from keel import exits +from keel.mesh import bridge, certificate, channel, protocol +from keel.mesh.admit import Admitter +from keel.mesh.bridge import Bridge, BridgeError, Remote, Unit +from keel.mesh.channel import Refused +from keel.mesh.listener import Params + +NO_CAPABILITIES = "Name:\tpython3\nCapEff:\t0000000000000000\n" + + +class Started: + """The listener as a thread of this process""" + + def __init__(self, status, trust=True): + self.status = status + self.trust = trust + self.stopped = False + self.codes = [] + self.logged = [] + + def __call__(self, path): + self.thread = threading.Thread(target=lambda: self.codes.append( + bridge.listen(path, lambda: datetime.now(timezone.utc), + self.logged.append, self.status, poll=0.05)), + daemon=True) + self.thread.start() + return self + + def trusted(self, pid, uid): + return self.trust and pid == os.getpid() + + def stop(self): + self.stopped = True + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.status = os.path.join(self.root, "status") + with open(self.status, "w") as fob: + fob.write(NO_CAPABILITIES) + self.port = free_port() + now = datetime.now(timezone.utc) + self.invite = reserved(self.root, port=self.port, + expires=now + timedelta(hours=1)) + self.node = FakeNode() + self.logged = [] + self.admitter = Admitter(self.root, self.invite, INVITER, OWN, + self.node, lambda: now, self.logged.append) + self.params = Params(self.invite.invite_id, + self.invite.expires, "::", self.port, + "fd00:6b65:1::1") + + def bridge(self, started): + return Bridge(self.admitter, self.params, self.invite.certificate, + self.invite.tls_key, + bridge.socket_path(self.root, self.invite.invite_id), + started, self.logged.append) + + def post(self, body=None, key=KEY): + now = protocol.seconds(datetime.now(timezone.utc)) + return channel.post( + "::1", self.port, protocol.JOIN, + join_body(time=now) if body is None else body, key, + certificate.fingerprint(self.invite.certificate), + connect_timeout=2, answer_timeout=10) + + def wait_for_port(self): + """From a source of its own: one connection per source""" + for _ in range(200): + try: + socket.create_connection( + ("127.0.0.1", self.port), 0.1, + source_address=("127.0.0.9", 0)).close() + return + except OSError: + threading.Event().wait(0.02) + self.fail("the listener never bound its port") + + +class TestTheWholePath(Case): + def run_bridge(self): + started = Started(self.status) + running = threading.Thread(target=self.bridge(started).run, + daemon=True) + running.start() + self.wait_for_port() + return started, running + + def ended(self, started, running): + running.join(10) + self.assertFalse(running.is_alive()) + self.assertEqual(started.codes, [exits.OK]) + self.assertTrue(started.stopped) + self.assertFalse(os.path.exists(bridge.socket_path( + self.root, self.invite.invite_id))) + self.assertIn("without capabilities", started.logged[0]) + self.assertIn("the listener (pid", self.logged[0]) + + def test_a_join_through_the_unprivileged_side(self): + # a window of a second: the listener stops waiting soon after + self.node.made = replace(self.node.made, window=1) + started, running = self.run_bridge() + answer = protocol.join_answer(self.post().body) + self.assertEqual((answer.public_key, answer.window), (INVITER, 1)) + self.assertEqual(len(self.node.admitted), 1) + self.ended(started, running) + + def test_forged_requests_cancel_the_invite_through_it(self): + started, running = self.run_bridge() + for _ in range(5): + with self.assertRaises(Refused): + self.post(key=bytes(32)) + self.ended(started, running) + self.assertTrue(self.admitter.cancelled) + self.assertEqual(self.node.admitted, []) + + +class TestRefused(Case): + def test_a_peer_that_is_not_the_listener_is_skipped(self): + started = Started(self.status, trust=False) + with mock.patch.object(bridge, "CONNECT_WAIT", 0.5), \ + self.assertRaises(BridgeError) as caught: + self.bridge(started).run() + self.assertIn("did not connect", str(caught.exception)) + self.assertIn("skipped process", self.logged[0]) + self.assertTrue(started.stopped) + + def test_a_listener_that_never_binds(self): + idle = mock.Mock() + with mock.patch.object(bridge, "CONNECT_WAIT", 0.1), \ + self.assertRaises(BridgeError) as caught: + self.bridge(lambda path: idle).run() + self.assertIn("did not connect", str(caught.exception)) + idle.stop.assert_called_once_with() + + def test_a_listener_no_helper_comes_to(self): + logged = [] + path = bridge.socket_path(self.root, self.invite.invite_id) + with mock.patch.object(bridge, "CONNECT_WAIT", 0.1): + self.assertEqual(bridge.listen(path, None, logged.append, + self.status), exits.APPLY_FAILED) + self.assertIn("no root helper came", logged[0]) + + def test_the_listener_s_socket_is_its_own(self): + path = bridge.socket_path(self.root, self.invite.invite_id) + modes = [] + + def connect(): + while not os.path.exists(path): + threading.Event().wait(0.01) + modes.append(os.stat(path).st_mode & 0o777) + modes.append(os.stat(os.path.dirname(path)).st_mode & 0o777) + with socket.socket(socket.AF_UNIX) as client: + client.connect(path) + client.recv(1) + threading.Thread(target=connect, daemon=True).start() + with bridge.helper(path, os.getuid()) as conn: + self.assertIsNotNone(conn) + self.assertEqual(modes, [0o600, 0o700]) + self.assertFalse(os.path.exists(path)) + + def test_a_helper_neither_root_nor_itself_is_dropped(self): + if os.getuid() == 0: + self.skipTest("as root, every peer is root") + path = bridge.socket_path(self.root, self.invite.invite_id) + + def connect(): + while not os.path.exists(path): + threading.Event().wait(0.01) + with socket.socket(socket.AF_UNIX) as client: + client.connect(path) + client.recv(1) + threading.Thread(target=connect, daemon=True).start() + with mock.patch.object(bridge, "CONNECT_WAIT", 0.3): + self.assertIsNone(bridge.helper(path, os.getuid() + 1)) + + def test_a_listener_with_capabilities_refuses_to_run(self): + with open(self.status, "w") as fob: + fob.write("CapEff:\t000001ffffffffff\n") + logged = [] + self.assertEqual(bridge.listen("/nonexistent", None, logged.append, + self.status), exits.APPLY_FAILED) + self.assertIn("without capabilities", logged[0]) + self.assertIn("0x1ffffffffff", logged[0]) + + def test_messages_the_listener_should_not_send(self): + for data in (b"not json\n", b'{"op": "rm"}\n', + b'{"op": "cancel", "reason": "x"}\n', + b'{"op": "forward", "path": "/v1/join"}\n', + b'{"op": "forward", "path": "/v1/join", "body": "@",' + b' "local": "::1", "peer": "::1"}\n', + b'{"op": "forward", "path": "/v1/join", "body": null,' + b' "local": "x", "peer": "::1"}\n'): + with self.subTest(data=data): + with self.assertRaises(BridgeError): + bridge.answered(self.admitter, data) + self.assertFalse(self.admitter.cancelled) + + def test_a_forward(self): + found = bridge.answered(self.admitter, json.dumps({ + "op": "forward", "path": "/v1/join", "signature": None, + "body": None, "local": "::1", "peer": "::1"}).encode()) + self.assertEqual((found["status"], found["final"]), (413, False)) + + def test_a_lying_listener_gets_nothing_through_the_bridge(self): + """It forges a join with a key of its choosing, and a confirmation + with the addresses the tunnel would have: neither is taken""" + forged = join_body(public_key="9vm/AzCVQQsWt1cU2eyjom4cABkNuodiHL8" + "nDjKohEE=") + found = bridge.answered(self.admitter, json.dumps({ + "op": "forward", "path": "/v1/join", "signature": "00" * 32, + "body": base64.b64encode(forged).decode(), "local": "::1", + "peer": "::1"}).encode()) + self.assertEqual(found["status"], 403) + self.assertEqual(self.node.admitted, []) + self.admitter.forward(protocol.JOIN, signature( + protocol.JOIN, join_body()), join_body(), "::", "::1") + self.node.handshake_at = None + self.admitter.sleep = lambda seconds: None + body = confirm_body() + found = bridge.answered(self.admitter, json.dumps({ + "op": "forward", "path": "/v1/confirm", + "signature": signature(protocol.CONFIRM, body), + "body": base64.b64encode(body).decode(), + "local": "fd00:6b65:1::1", "peer": "fd00:6b65:1::3"}).encode()) + self.assertEqual(found["status"], 409) + self.assertEqual(self.node.origins, []) + + def test_a_message_longer_than_the_cap(self): + left, right = socket.socketpair() + with left, right: + right.sendall(b"x" * 64) + with mock.patch.object(bridge, "MAX_MESSAGE", 16), \ + self.assertRaises(BridgeError): + bridge.line(left) + + +class TestRemote(unittest.TestCase): + def test_the_root_side_gone(self): + left, right = socket.socketpair() + right.close() + with left: + found = Remote(left).forward("/v1/join", None, b"{}", "::1", + "::1") + self.assertEqual(found.status, 503) + + def test_the_root_side_closed_its_end(self): + left, right = socket.socketpair() + with left, right: + right.shutdown(socket.SHUT_WR) + self.assertEqual(Remote(left).forward( + "/v1/join", None, b"{}", "::1", "::1").status, 503) + + def test_a_closed_socket(self): + left, right = socket.socketpair() + right.close() + left.close() + self.assertEqual(Remote(left).forward("/", None, None, "::1", + "::1").status, 503) + + +class TestUnit(unittest.TestCase): + def test_the_listener_s_unit_is_unprivileged_and_sandboxed(self): + run = Recorder() + found = bridge.start_unit("0123456789abcdef", "/run/x.sock", 60, run, + lambda argv: None) + [argv] = run.calls + self.assertEqual(argv[:5], ( + "systemd-run", "--unit=keel-mesh-listen@0123456789abcdef", + "--collect", "--quiet", "--property=RuntimeMaxSec=60")) + for one in ("DynamicUser=yes", "NoNewPrivileges=yes", + "PrivateTmp=yes", "ProtectSystem=strict", + "ProtectHome=yes", "CapabilityBoundingSet=", + "AmbientCapabilities=", "SystemCallFilter=@system-service", + "RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX", + "MemoryMax=64M", "LimitNOFILE=64", + "RuntimeDirectory=keel/mesh-listen-0123456789abcdef", + "RuntimeDirectoryMode=0700"): + self.assertIn(f"--property={one}", argv) + self.assertEqual(argv[-3:], ("mesh", "listen", "/run/x.sock")) + found.stop() + self.assertEqual(run.calls[-1], ( + "systemctl", "stop", "keel-mesh-listen@0123456789abcdef")) + + def test_the_socket_is_in_the_listener_s_runtime_directory(self): + self.assertEqual(bridge.socket_path("/", "0123456789abcdef"), + "/run/keel/mesh-listen-0123456789abcdef/bridge.sock") + + def test_a_unit_that_cannot_start(self): + with self.assertRaises(BridgeError): + bridge.start_unit("0123456789abcdef", "/run/x.sock", 60, + Recorder(fail={"systemd-run": 1}), + lambda argv: None) + + def test_trusted_only_as_the_unit_s_main_process_and_never_root(self): + asked = [] + + def main_pid(argv): + asked.append(argv) + return "4242\n" + found = Unit("keel-mesh-listen@0123456789abcdef", Recorder(), + main_pid) + self.assertTrue(found.trusted(4242, 61234)) + self.assertEqual(asked[0], ( + "systemctl", "show", "--property=MainPID", "--value", + "keel-mesh-listen@0123456789abcdef")) + self.assertFalse(found.trusted(4242, 0)) + self.assertFalse(found.trusted(4243, 61234)) + self.assertFalse(Unit("u", Recorder(), lambda argv: None).trusted( + 4242, 61234)) + + def test_the_parameters_round_trip(self): + until = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) + for joined in (None, ("k", "fd00::3", until)): + params = Params("0123456789abcdef", until, "::", 51820, + "fd00::1", joined) + self.assertEqual(bridge.params_of(bridge.params_json(params)), + params) + + def test_capabilities(self): + self.assertEqual(bridge.capabilities(NO_CAPABILITIES), 0) + self.assertEqual(bridge.capabilities("Name: x\n"), -1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_channel.py b/tests/test_mesh_channel.py new file mode 100644 index 0000000..f62b21a --- /dev/null +++ b/tests/test_mesh_channel.py @@ -0,0 +1,308 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The pinned HTTPS channel, end to end on the loopback + +The real unprivileged listener's server (keel.mesh.listener.run) with +the invite's real certificate from openssl, in front of the real +Admitter, served from a thread on [::1], and the real client +(keel.mesh.channel.post): the pin checked before anything is sent, the +answer's HMAC, each failure told apart; and against someone holding +the port, the reading's deadline, the slots, the slots kept for the +overlay, the size cap and the sources refused too often. +""" + +import contextlib +import json +import shutil +import socket +import ssl +import tempfile +import threading +import unittest +from datetime import datetime, timedelta, timezone +from unittest import mock + +from mesh_helpers import ( + INVITER, + KEY, + OWN, + FakeNode, + join_body, + reserved, + tls, +) + +from keel.mesh import certificate, channel, listener, protocol +from keel.mesh.admit import Admitter, Response +from keel.mesh.channel import ChannelError, Forged, Refused, Unreachable +from keel.mesh.listener import Listener, Params, bound, run + + +def free_port() -> int: + with socket.socket(socket.AF_INET6) as probe: + probe.bind(("::1", 0)) + return probe.getsockname()[1] + + +class Served(unittest.TestCase): + """A listener on [::1] for the length of one test""" + + listener_class = Listener + host = "::1" + overlay = "fd00:6b65:1::1" + constants: dict = {} + + def setUp(self): + for name, value in self.constants.items(): + patcher = mock.patch.object(listener, name, value) + patcher.start() + self.addCleanup(patcher.stop) + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.port = free_port() + now = datetime.now(timezone.utc) + self.invite = reserved(self.root, port=self.port, + expires=now + timedelta(hours=1)) + self.node = FakeNode() + self.logged = [] + admitter = Admitter(self.root, self.invite, INVITER, OWN, self.node, + lambda: now, self.logged.append) + self.listener = self.listener_class( + Params(self.invite.invite_id, + self.invite.expires, self.host, self.port, self.overlay), + admitter, lambda: now, self.logged.append) + context = channel.server_context(self.invite.certificate, + self.invite.tls_key) + server = bound(self.host, self.port) + self.thread = threading.Thread( + target=run, args=(server, self.listener, context), + kwargs={"poll": 0.05}, daemon=True) + self.thread.start() + self.addCleanup(self.stop) + self.pin = certificate.fingerprint(self.invite.certificate) + + def stop(self): + self.listener.ended = True + self.thread.join(10) + + def post(self, body=None, path=protocol.JOIN, key=KEY, pin=None, + host="::1"): + now = protocol.seconds(datetime.now(timezone.utc)) + body = join_body(time=now) if body is None else body + return channel.post(host, self.port, path, body, key, + pin or self.pin, connect_timeout=2, + answer_timeout=5) + + def held(self, source=None, host="127.0.0.1"): + """A connection that never starts its TLS handshake""" + raw = socket.create_connection( + (host, self.port), timeout=5, + source_address=(source, 0) if source else None) + self.addCleanup(raw.close) + return raw + + def dropped(self, source): + """Whether a connection from `source` is closed unanswered""" + raw = self.held(source) + try: + channel.client_context().wrap_socket(raw) + except OSError: + return True + return False + + +class TestPost(Served): + def test_a_join_round_trip_through_the_admitter(self): + reply = self.post() + self.assertEqual((reply.local, reply.peer), ("::1", "::1")) + answer = protocol.join_answer(reply.body) + self.assertEqual(answer.public_key, INVITER) + self.assertEqual(len(self.node.admitted), 1) + + def test_another_certificate_is_refused_before_anything_is_sent(self): + other = certificate.fingerprint(certificate.make()[1]) + with self.assertRaises(Forged) as caught: + self.post(pin=other) + self.assertIn("not the one the token pins", str(caught.exception)) + self.assertEqual(self.logged, []) + + def test_a_refusal_says_why(self): + with self.assertRaises(Refused) as caught: + self.post(key=bytes(32)) + self.assertIn("bad HMAC", str(caught.exception)) + + def test_a_body_too_long_is_refused_unread(self): + """Refused without reading it; the client may see the refusal or, + as the unread body is discarded, a reset first""" + with self.assertRaises(ChannelError): + self.post(body=b"x" * (protocol.MAX_BODY + 1)) + self.stop() + self.assertIn("longer than any join", self.logged[-1]) + + def test_a_bad_content_length(self): + context = channel.client_context() + with socket.create_connection(("::1", self.port), timeout=5) as raw, \ + context.wrap_socket(raw) as conn: + conn.sendall(b"POST /v1/join HTTP/1.1\r\nHost: x\r\n" + b"Content-Length: many\r\n\r\n") + self.assertIn(b" 413 ", conn.recv(4096)) + + def test_a_client_without_tls_does_not_stop_the_listener(self): + with socket.create_connection(("::1", self.port), timeout=5) as raw: + raw.sendall(b"GET / HTTP/1.0\r\n\r\n") + with contextlib.suppress(OSError): + raw.recv(100) + self.assertEqual(protocol.join_answer(self.post().body).public_key, + INVITER) + + +class TestHeld(Served): + constants = {"REQUEST_DEADLINE": 0.5} + + def test_a_slow_client_is_cut_at_the_deadline(self): + context = channel.client_context() + with socket.create_connection(("::1", self.port), timeout=5) as raw, \ + context.wrap_socket(raw) as conn: + conn.sendall(b"POST /v1/join HTTP/1.1\r\n") + self.assertEqual(conn.recv(100), b"") + + def test_headers_longer_than_the_cap_are_not_read(self): + context = channel.client_context() + with socket.create_connection(("::1", self.port), timeout=5) as raw, \ + context.wrap_socket(raw) as conn: + with contextlib.suppress(OSError): + conn.sendall(b"POST /v1/join HTTP/1.1\r\n" + b"X-A: " + + b"a" * 9000 + b"\r\n\r\n") + with contextlib.suppress(OSError): + self.assertNotIn(b"HTTP/1", conn.recv(100)) + + +class TestSlots(Served): + """Sources on the IPv4 loopback, 127.0.0.2 and up, take the slots; + [::1] is this test's overlay address""" + + constants = {"SLOTS": 1} + host = "::" + overlay = "::1" + + def test_the_slots_and_those_kept_for_the_overlay(self): + self.held("127.0.0.2") + self.assertTrue(self.dropped("127.0.0.3")) + # the overlay's connections have slots of their own + answer = protocol.join_answer(self.post().body) + self.assertEqual(answer.public_key, INVITER) + + +class TestPerSource(Served): + host = "::" + + def test_one_connection_at_a_time_from_one_source(self): + self.held("127.0.0.2") + self.assertTrue(self.dropped("127.0.0.2")) + self.assertFalse(self.dropped("127.0.0.3")) + + +class TestBlocked(Served): + def test_a_source_refused_too_often_is_not_answered(self): + for _ in range(listener.MAX_REFUSED): + with self.assertRaises(Refused): + self.post(body=b"{}") + with self.assertRaises(ChannelError) as caught: + self.post() + self.assertNotIsInstance(caught.exception, Refused) + + +class TestUnreachable(unittest.TestCase): + def test_nothing_listening(self): + with self.assertRaises(Unreachable) as caught: + channel.post("::1", free_port(), protocol.JOIN, b"{}", KEY, + bytes(32), connect_timeout=2) + self.assertIn("[::1]:", str(caught.exception)) + + def test_not_tls(self): + with socket.socket(socket.AF_INET6) as server: + server.bind(("::1", 0)) + server.listen() + port = server.getsockname()[1] + + def close_at_once(): + conn, _ = server.accept() + conn.close() + threading.Thread(target=close_at_once, daemon=True).start() + with self.assertRaises(Forged) as caught: + channel.post("::1", port, protocol.JOIN, b"{}", KEY, + bytes(32), connect_timeout=2) + self.assertIn("TLS handshake failed", str(caught.exception)) + + +class Unsigned(Listener): + def handle(self, *args): + return Response(200, b"{}", "00" * 32) + + +class TestForgedAnswer(Served): + listener_class = Unsigned + + def test_an_answer_not_signed_with_the_key(self): + with self.assertRaises(Forged) as caught: + self.post() + self.assertIn("not signed with this invite's key", + str(caught.exception)) + + +class TooLong(Listener): + def handle(self, *args): + return Response(200, b"x" * (protocol.MAX_BODY + 1), None) + + +class TestTooLongAnswer(Served): + listener_class = TooLong + + def test_an_answer_longer_than_the_protocol_allows(self): + with self.assertRaises(ChannelError) as caught: + self.post() + self.assertIn("longer than any", str(caught.exception)) + + +class Silent(Listener): + def handle(self, *args): + raise ConnectionResetError("gone") + + +class TestCutShort(Served): + listener_class = Silent + + def test_an_exchange_cut_short(self): + with self.assertRaises(ChannelError) as caught: + self.post() + self.assertIn("did not answer", str(caught.exception)) + + +class TestHelpers(unittest.TestCase): + def test_reason(self): + for data, found in ((b'{"error": "expired"}', "expired"), + (b"", "HTTP 502"), + (b'{"error": 5}', "HTTP 502"), + (b'{"error": "a\\u0007"}', "HTTP 502"), + (b"[]", "HTTP 502")): + with self.subTest(data=data): + self.assertEqual(channel.reason(data, 502), found) + + def test_plain(self): + self.assertEqual(channel.plain("::ffff:192.0.2.5"), "192.0.2.5") + self.assertEqual(channel.plain("fe80::1%eth0"), "fe80::1") + self.assertEqual(channel.plain("fd00::1"), "fd00::1") + + def test_the_server_context_loads_the_invite_pair(self): + key, cert = tls() + context = channel.server_context(cert, key) + self.assertIsInstance(context, ssl.SSLContext) + with self.assertRaises(ssl.SSLError): + channel.server_context(cert, certificate.make()[0]) + + def test_shown(self): + self.assertEqual(channel.shown("192.0.2.1", 5), "192.0.2.1:5") + self.assertEqual(json.loads('"x"'), "x") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_cli.py b/tests/test_mesh_cli.py index 1378572..e4ebd33 100644 --- a/tests/test_mesh_cli.py +++ b/tests/test_mesh_cli.py @@ -65,9 +65,11 @@ def run_cli(*argv, stdin=""): def token_of(out: str) -> str: + """The token of the line invite prints, which feeds it on standard + input, never as an argument""" line = out.strip() - assert line.startswith("keel mesh join keel1:"), line - return line[len("keel mesh join "):] + assert line.startswith("keel mesh join - <<< keel1:"), line + return line[len("keel mesh join - <<< "):] def secret_forms(secret: bytes) -> tuple[str, ...]: @@ -164,7 +166,7 @@ def test_the_line_and_the_invite_it_reserves(self): def test_the_next_invite_takes_the_next_address(self): first = mesh_token.parse(token_of(self.invite()[1]), datetime.now(timezone.utc)) - second = mesh_token.parse(token_of(self.invite()[1]), + second = mesh_token.parse(token_of(self.invite("--port", "8443")[1]), datetime.now(timezone.utc)) self.assertEqual((first.assigned, second.assigned), ("fd00:6b65:1::3/64", "fd00:6b65:1::4/64")) @@ -172,6 +174,15 @@ def test_the_next_invite_takes_the_next_address(self): self.assertNotEqual(first.fingerprint, second.fingerprint) self.assertEqual(first.mesh_id, second.mesh_id) + def test_one_pending_invite_per_port(self): + """the listener of each holds its port on every address""" + self.invite() + code, out, err = self.invite() + self.assertEqual((code, out), (exits.MESH_REFUSED, "")) + self.assertIn("is pending on TCP port 51820", err) + self.assertIn("--port", err) + self.assertEqual(len(self.invite_files()), 1) + def test_endpoint_and_port_given(self): code, out, err = self.invite("--endpoint", "198.51.100.7", "--endpoint", "2001:db8:9::7", @@ -218,6 +229,15 @@ def test_no_key_yet(self): self.assertEqual(code, exits.MESH_REFUSED) self.assertIn("converge the overlay first", err) + def test_a_listener_that_cannot_start_prints_no_line(self): + self.write_spec(OVERLAY) + self.key_file() + with self.public(), mock.patch( + "keel.mesh.commands.listening", + return_value=("", exits.APPLY_FAILED)): + code, out, _ = self.invite() + self.assertEqual((code, out), (exits.APPLY_FAILED, "")) + def test_a_key_wg_cannot_read(self): self.write_spec(OVERLAY) self.key_file() @@ -417,10 +437,13 @@ def test_standard_input_is_read_no_further_than_a_token_can_be(self): self.assertEqual((code, out), (exits.MESH_TOKEN_INVALID, "")) self.assertIn("more than 1024 characters", err) - def test_without_dry_run_nothing_is_done_yet(self): - code, out, err = self.join(self.token()) - self.assertEqual((code, out), (exits.NOT_IMPLEMENTED, "")) - self.assertIn("--dry-run", err) + def test_without_dry_run_on_the_live_system_needs_root(self): + with mock.patch("os.geteuid", return_value=1000): + code, out, err = run_cli("mesh", "join", self.token(), "--spec", + self.spec) + self.assertEqual((code, out), (exits.APPLY_NEEDS_ROOT, "")) + self.assertIn("keel mesh join on the live system must run as root", + err) def test_no_action(self): code, _, _ = run_cli("mesh") diff --git a/tests/test_mesh_cli_live.py b/tests/test_mesh_cli_live.py new file mode 100644 index 0000000..544090c --- /dev/null +++ b/tests/test_mesh_cli_live.py @@ -0,0 +1,244 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh create, join, accept, serve, status through keel.cli.main + +The handlers are thin: root on the live system, the token or the line +read (from standard input with -), the live Node built, and the flow +called. Each flow is replaced here at its seam (keel.mesh.joining.run, +inviting.accept, ...) and tested on its own in test_mesh_joining.py, +test_mesh_inviting.py and test_mesh_create_status.py. +""" + +import shutil +import signal +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from os.path import join +from unittest import mock + +from mesh_helpers import NOW, reserved +from mesh_wire import token +from test_mesh_cli import run_cli + +from keel import exits +from keel.mesh import commands, invites, inviting +from keel.mesh.token import encode +from keel.network import session + +SPEC = """\ +version: 1 +network: + interfaces: + eth0: + ipv6: {method: static, address: '2001:db8:2::20/64'} +""" + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.spec = join(self.root, "instance.yaml") + with open(self.spec, "w") as fob: + fob.write(SPEC) + clients = mock.patch("keel.mesh.commands.operator_clients", + return_value=("2001:db8:9::5",)) + clients.start() + self.addCleanup(clients.stop) + + def mesh(self, *argv, stdin=""): + return run_cli("mesh", *argv, "--spec", self.spec, "--root", + self.root, stdin=stdin) + + def line(self): + invite = reserved(self.root) + return encode(token(invite, expires=datetime.now(timezone.utc) + + timedelta(hours=1))) + + +class TestNeedsRoot(Case): + def test_each_command_that_changes_the_machine(self): + for argv in (("create",), ("accept", "keel1a:x"), + ("serve", "0123456789abcdef"), + ("join", "keel1:x")): + with self.subTest(argv=argv), \ + mock.patch("os.geteuid", return_value=1000): + code, out, err = run_cli("mesh", *argv) + self.assertEqual((code, out), (exits.APPLY_NEEDS_ROOT, "")) + self.assertIn("must run as root", err) + + +class TestHandlers(Case): + def test_create(self): + with mock.patch("keel.mesh.commands.create.create", + return_value=exits.OK) as made: + code, _, _ = self.mesh("create", "--network-window", "60") + self.assertEqual(code, exits.OK) + node = made.call_args.args[0] + self.assertEqual((node.root, node.path, node.window, node.clients), + (self.root, self.spec, 60, ("2001:db8:9::5",))) + + def test_join(self): + line = self.line() + with mock.patch("keel.mesh.commands.joining.run", + return_value=exits.OK) as joined, \ + mock.patch("keel.mesh.commands.live.output", + return_value=""): + code, _, err = self.mesh("join", line) + self.assertEqual(code, exits.OK, err) + joiner, endpoint = joined.call_args.args + self.assertEqual(endpoint, "2001:db8:2::20") + self.assertEqual(joiner.token.invite_id, + invites.ids(self.root)[0]) + self.assertIsInstance(joiner.clock(), datetime) + + def test_join_reads_the_token_from_standard_input(self): + line = self.line() + with mock.patch("keel.mesh.commands.joining.run", + return_value=exits.OK) as joined, \ + mock.patch("keel.mesh.commands.live.output", + return_value=None): + code, _, _ = self.mesh("join", "-", "--endpoint", "2001:db8::7", + stdin=line) + self.assertEqual(code, exits.OK) + self.assertEqual(joined.call_args.args[1], "2001:db8::7") + + def test_a_token_given_as_an_argument_is_warned_about(self): + line = self.line() + with mock.patch("keel.mesh.commands.joining.run", + return_value=exits.OK), \ + mock.patch("keel.mesh.commands.live.output", + return_value=""): + _, _, err = self.mesh("join", line) + self.assertIn("Warning: the token is an argument, so it shows in" + " the process list", err) + with mock.patch("keel.mesh.commands.joining.run", + return_value=exits.OK), \ + mock.patch("keel.mesh.commands.live.output", + return_value=""): + _, _, err = self.mesh("join", "-", stdin=line) + self.assertNotIn("Warning", err) + + def test_listen_needs_no_root(self): + with mock.patch("os.geteuid", return_value=61234), \ + mock.patch("keel.mesh.commands.bridge.listen", + return_value=exits.OK) as listened: + code, _, _ = run_cli("mesh", "listen", "/run/keel/mesh/x.sock") + self.assertEqual(code, exits.OK) + self.assertEqual(listened.call_args.args[0], "/run/keel/mesh/x.sock") + + def test_join_with_a_bad_token_or_endpoint(self): + code, _, err = self.mesh("join", "keel1:abc") + self.assertEqual(code, exits.MESH_TOKEN_INVALID) + code, _, err = self.mesh("join", self.line(), "--endpoint", "x") + self.assertEqual(code, exits.MESH_REFUSED) + self.assertIn("does not appear to be an IPv4 or IPv6", err) + + def test_accept(self): + with mock.patch("keel.mesh.commands.inviting.accept", + return_value=exits.OK) as accepted: + self.assertEqual(self.mesh("accept", "-", + stdin="keel1a:x")[0], exits.OK) + inviter, text = accepted.call_args.args + self.assertEqual(text, "keel1a:x") + self.assertEqual(inviter.node.root, self.root) + self.assertEqual(inviter.clock().tzinfo, timezone.utc) + + def test_serve(self): + """SIGTERM, from systemctl stop or RuntimeMaxSec, ends it through + the cleanup that removes the invite""" + before = signal.getsignal(signal.SIGTERM) + self.addCleanup(signal.signal, signal.SIGTERM, before) + with mock.patch("keel.mesh.commands.inviting.serve_invite", + return_value=exits.OK) as served: + code, _, _ = self.mesh("serve", "0123456789abcdef") + self.assertEqual(code, exits.OK) + self.assertEqual(served.call_args.args[1], "0123456789abcdef") + self.assertEqual(signal.getsignal(signal.SIGTERM), + inviting.terminated) + + def test_status(self): + with open(self.spec, "a") as fob: + fob.write(" overlay:\n wireguard:\n" + " address: fd00:6b65:1::1/64\n") + code, out, _ = self.mesh("status") + self.assertEqual(code, exits.OK) + self.assertIn("this node: fd00:6b65:1::1/64 on wg0", out) + self.assertIn("not the live system", out) + + def test_status_without_a_spec_or_with_a_broken_one(self): + code, out, _ = run_cli("mesh", "status", "--spec", + join(self.root, "none.yaml"), "--root", + self.root) + self.assertEqual(code, exits.OK) + self.assertIn("in no mesh", out) + with open(self.spec, "w") as fob: + fob.write("version: [") + self.assertEqual(self.mesh("status")[0], exits.SPEC_UNREADABLE) + + def test_status_on_the_live_system_asks_wg(self): + with mock.patch("keel.mesh.commands.ROOT_DEFAULT", self.root), \ + mock.patch("keel.mesh.commands.live.output", + return_value=None) as asked: + self.mesh("status") + self.assertEqual(asked.call_count, 0) + with open(self.spec, "a") as fob: + fob.write(" overlay:\n wireguard:\n" + " address: fd00:6b65:1::1/64\n") + with mock.patch("keel.mesh.commands.ROOT_DEFAULT", self.root), \ + mock.patch("keel.mesh.commands.live.output", + return_value=None) as asked: + self.mesh("status") + self.assertGreater(asked.call_count, 0) + + +class TestListening(Case): + def test_the_unit_started_and_the_port_opened(self): + made = reserved(self.root) + with mock.patch("keel.mesh.commands.inviting.start", + return_value=None) as started, \ + mock.patch("keel.mesh.commands.ports.open_port", + return_value="opened") as opened: + found = commands.listening(made, "spec.yaml", "/", NOW) + self.assertEqual(found, ("opened", exits.OK)) + self.assertTrue(started.call_args.args[1].endswith("/spec.yaml")) + # as long as the listener may wait for a late join's confirmation + self.assertEqual(opened.call_args.args[:2], + (51820, 3600 + inviting.LINGER)) + + def test_a_unit_that_cannot_start_removes_the_invite(self): + made = reserved(self.root) + with mock.patch("keel.mesh.commands.inviting.start", + return_value="systemd-run exited 1"), \ + mock.patch("keel.mesh.commands.invites.remove") as removed: + found = commands.listening(made, "spec.yaml", "/", NOW) + self.assertEqual(found, ("", exits.APPLY_FAILED)) + removed.assert_called_once_with("/", made.invite_id) + + def test_off_the_live_system_nothing_starts(self): + made = reserved(self.root) + text, code = commands.listening(made, "spec.yaml", self.root, NOW) + self.assertEqual(code, exits.OK) + self.assertIn("no listener started", text) + + +class TestOperatorClients(unittest.TestCase): + def test_the_client_of_this_ssh_session(self): + for origin, found in ( + (session.Origin(session.SSH, "ssh", 1.0, "2001:db8::1", + "2001:db8::9"), ("2001:db8::9",)), + (session.Origin(session.CONSOLE_KIND, "tty1"), ())): + with self.subTest(kind=origin.kind), \ + mock.patch("keel.mesh.commands.session.origin", + return_value=origin): + self.assertEqual(commands.operator_clients(), found) + + def test_out_and_err(self): + with mock.patch("builtins.print") as printed: + commands.out("a") + commands.err("b") + self.assertEqual(printed.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_create_status.py b/tests/test_mesh_create_status.py new file mode 100644 index 0000000..5c9c13f --- /dev/null +++ b/tests/test_mesh_create_status.py @@ -0,0 +1,292 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh create, keel mesh status, the invite's port, the node seam + +create against a real Node on a scratch root (tests/mesh_wire.py), +confirmed by the real keel.network.confirm; status from fixture `wg +show` output; the firewall element through a recording runner. +""" + +import ipaddress +import os +import shutil +import tempfile +import unittest +from datetime import timedelta +from unittest import mock + +import yaml +from mesh_helpers import INVITE, JOINER, NOW, OTHER, reserved +from mesh_helpers import INVITER as INVITER_KEY +from mesh_wire import INVITER_SPEC, Armed, node, probes +from test_network_window import Recorder + +from keel import exits +from keel.mesh import create, identity, invites, ports, status +from keel.mesh.node import ( + Node, + NodeError, + global_addresses, + live_apply, + with_overlay, +) +from keel.network import marker + + +class TestCreate(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.node = node(self.root, clients=("2001:db8:9::5",)) + self.out, self.err = [], [] + + def create(self): + return create.create(self.node, self.out.append, self.err.append) + + def overlay(self): + with open(self.node.path) as fob: + return yaml.safe_load(fob)["network"]["overlay"]["wireguard"] + + def test_a_random_prefix_this_node_at_one_confirmed_by_itself(self): + self.assertEqual(self.create(), exits.OK, self.err) + address = ipaddress.IPv6Interface(self.overlay()["address"]) + self.assertEqual(address.network.prefixlen, 64) + self.assertEqual(int(address.ip) & 0xFFFF, 1) + self.assertTrue(str(address).startswith("fd")) + self.assertIsNotNone(identity.read(self.root)) + self.assertFalse(marker.exists(self.root)) + self.assertEqual(marker.last(self.root).outcome, marker.CONFIRMED) + self.assertIn("confirmed from keel mesh create", self.err) + self.assertEqual(self.out, [ + f"created the mesh: this node is {address} on wg0, WireGuard" + " on UDP 51820", + "keel mesh invite prints the line that joins the next node"]) + + def test_the_operator_session_routed_into_it_reverts_it(self): + self.node.probes = lambda: probes("wg0") + self.assertEqual(self.create(), exits.NETWORK_NOT_CONFIRMED) + self.assertTrue(marker.exists(self.root)) + self.assertIn("leaves through wg0", self.err[0]) + + def test_a_node_already_in_a_mesh(self): + with open(self.node.path, "w") as fob: + fob.write(INVITER_SPEC) + self.assertEqual(self.create(), exits.MESH_REFUSED) + self.assertIn("already in a mesh, at fd00:6b65:1::1/64", self.err[0]) + + def test_a_spec_that_cannot_be_read(self): + with open(self.node.path, "w") as fob: + fob.write("version: [") + self.assertEqual(self.create(), exits.MESH_REFUSED) + + def test_a_damaged_identity(self): + os.makedirs(os.path.join(self.root, "var/lib/keel/mesh")) + with open(os.path.join(self.root, identity.IDENTITY), "w") as fob: + fob.write("nope\n") + self.assertEqual(self.create(), exits.MESH_REFUSED) + self.assertIn("does not hold a mesh identity", self.err[0]) + + def test_the_overlay_does_not_come_up(self): + self.node.apply = Armed(code=16, arms=False) + self.assertEqual(self.create(), exits.APPLY_FAILED) + self.assertIn("apply exited 16", self.err[0]) + + +class TestStatus(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.overlay = {"address": "fd00:6b65:1::1/64", "peers": [ + {"public_key": JOINER, "endpoint": "[2001:db8:2::20]:51820", + "allowed_ips": ["fd00:6b65:1::3/128"]}, + {"public_key": OTHER, "allowed_ips": ["fd00:6b65:1::7/128"]}]} + + def wg(self, argv): + return { + "public-key": "nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E=\n", + "endpoints": f"{JOINER}\t[2001:db8:2::21]:51820\n" + f"{OTHER}\t(none)\nnot a line\n", + "latest-handshakes": f"{JOINER}\t" + f"{int(NOW.timestamp()) - 12}\n{OTHER}\t0\n", + }[argv[-1]] + + def test_the_peers_their_handshakes_and_the_invites(self): + reserved(self.root) + found = status.lines(self.overlay, self.root, NOW, self.wg) + self.assertEqual(found[:4], [ + "this node: fd00:6b65:1::1/64 on wg0, WireGuard on UDP 51820", + "public key: nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E=", + "peers: 2", + f" {JOINER} fd00:6b65:1::3/128 [2001:db8:2::21]:51820" + " handshake 12 s ago"]) + self.assertEqual(found[4], f" {OTHER} fd00:6b65:1::7/128 no" + " endpoint no handshake yet") + self.assertEqual(found[5], "pending invites: 1") + self.assertIn(f" {INVITE} fd00:6b65:1::3/64 TCP 51820 until" + " 2026-10-03 13:00:00 UTC pending", found[6]) + invites.consume(self.root, INVITE, NOW) + self.assertIn("used, waiting for its confirmation", + status.lines(self.overlay, self.root, NOW, + self.wg)[-1]) + + def test_no_secret_is_asked_for(self): + asked = [] + status.lines(self.overlay, self.root, NOW, + lambda argv: asked.append(argv) or self.wg(argv)) + for argv in asked: + self.assertNotIn("dump", argv) + self.assertNotIn("private-key", argv) + + def test_an_interface_that_is_down(self): + found = status.lines(self.overlay, self.root, NOW, lambda argv: None) + self.assertIn("public key: unknown (is the interface up?)", found) + self.assertIn("no handshake known", found[3]) + + def test_off_the_live_system(self): + found = status.lines(self.overlay, self.root, NOW, None) + self.assertEqual(found[1], "not the live system: no handshake read") + self.assertTrue(found[3].endswith("[2001:db8:2::20]:51820")) + + def test_no_mesh(self): + for overlay in (None, {}): + self.assertIn("in no mesh", status.lines(overlay, self.root, + NOW, None)[0]) + + def test_a_handshake_in_the_future_reads_as_now(self): + self.assertEqual(status.handshake( + str(int((NOW + timedelta(seconds=5)).timestamp())), NOW), + "handshake 0 s ago") + + +class TestPorts(unittest.TestCase): + def test_opened_where_keel_s_firewall_has_the_set(self): + run = Recorder() + line = ports.open_port(51820, 3600, run, lambda argv: "set") + self.assertEqual(run.calls, [("nft", "add", "element", "inet", + "keel", "mesh_invites", + "{ 51820 timeout 3600s }")]) + self.assertIn("opened in keel's firewall", line) + + def test_nothing_opened_elsewhere(self): + run = Recorder() + line = ports.open_port(51820, 3600, run, lambda argv: None) + self.assertEqual(run.calls, []) + self.assertIn("keel opened nothing", line) + + def test_an_element_that_cannot_be_added(self): + line = ports.open_port(51820, 3600, Recorder(fail={"nft": 1}), + lambda argv: "set") + self.assertIn("could not be opened", line) + + +class TestNode(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + + def test_an_absent_spec_is_empty(self): + self.assertEqual(node(self.root).document(), {"version": 1}) + + def test_an_invalid_document_is_not_written(self): + found = node(self.root) + with self.assertRaises(NodeError) as caught: + found.write({"version": 1, "network": {"overlay": { + "wireguard": {"address": "not an address"}}}}) + self.assertIn("instance.yaml", str(caught.exception)) + self.assertFalse(os.path.exists(found.path)) + + def test_the_spec_is_written_root_s_alone(self): + found = node(self.root) + found.write(with_overlay({"version": 1}, + {"address": "fd00:1::1/64"})) + self.assertEqual(os.stat(found.path).st_mode & 0o777, 0o600) + self.assertEqual(found.overlay(), {"address": "fd00:1::1/64"}) + + def test_a_change_that_is_not_the_overlay_is_not_made(self): + found = node(self.root, apply=lambda doc, root, window: ( + marker.save(root, ""), marker.write(root, marker.Pending( + "eth0", "x", 120).up("b1", 1.0)), 0)[-1]) + change = found.change(with_overlay({"version": 1}, + {"address": "fd00:1::1/64"})) + self.assertIsNone(change.made) + + def test_a_peer_is_never_replaced(self): + found = node(self.root, INVITER_SPEC) + found.admit({"public_key": JOINER, + "allowed_ips": ["fd00:6b65:1::3/128"]}) + with self.assertRaises(NodeError) as caught: + found.admit({"public_key": JOINER, + "allowed_ips": ["fd00:6b65:1::9/128"]}) + self.assertIn("never replaces", str(caught.exception)) + self.assertEqual(len(found.overlay()["peers"]), 1) + + def test_the_peers_the_answer_carries(self): + found = node(self.root, INVITER_SPEC) + found.admit({"public_key": JOINER, + "allowed_ips": ["fd00:6b65:1::3/128"], + "endpoint": "[2001:db8:2::20]:51820"}) + found.admit({"public_key": OTHER, "allowed_ips": ["fd00:6b65:2::/64"]}) + self.assertEqual(found.peers(OTHER)[0].address, "fd00:6b65:1::3") + self.assertEqual(found.peers(JOINER), ()) + + def test_the_live_apply_writes_its_lines_to_standard_error(self): + with mock.patch("keel.mesh.node.apply_system", + side_effect=lambda *args, **kwargs: print("line") + or 0) as applied, \ + mock.patch("sys.stderr") as err, \ + mock.patch("sys.stdout") as out: + self.assertEqual(live_apply({"version": 1}, "/", 150), 0) + applied.assert_called_once_with( + {"version": 1}, "/", False, "keel mesh", defer_certificate=True, + network_window=150, skip_uplink=True) + self.assertTrue(err.write.called) + self.assertFalse(out.write.called) + + def test_a_spec_that_cannot_be_written(self): + found = node(self.root) + with mock.patch("keel.mesh.node.write_private", + side_effect=OSError(28, "No space left on device")): + with self.assertRaises(NodeError) as caught: + found.write({"version": 1}) + self.assertIn("cannot be written: No space left on device", + str(caught.exception)) + + def test_the_identity_a_node_keeps_is_adopted_again(self): + identity.adopt(self.root, bytes(16)) + identity.adopt(self.root, bytes(16)) + self.assertEqual(identity.read(self.root), bytes(16)) + with self.assertRaises(ValueError) as caught: + identity.adopt(self.root, bytes(range(16))) + self.assertIn("in another mesh", str(caught.exception)) + self.assertEqual(identity.read(self.root), bytes(16)) + + def test_the_latest_handshake_of_a_key(self): + found = node(self.root, INVITER_SPEC) + asked = [] + + def wg(argv): + asked.append(argv) + return (f"{OTHER}\t0\n{JOINER}\t1790000000\nnot a line\n" + f"{INVITER_KEY}\tsoon\n") + found.output = wg + self.assertEqual(found.handshake(JOINER), 1790000000) + self.assertEqual(asked[0], ("wg", "show", "wg0", "latest-handshakes")) + for key in (OTHER, INVITER_KEY, + "nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82F="): + self.assertIsNone(found.handshake(key)) + found.output = lambda argv: None + self.assertIsNone(found.handshake(JOINER)) + + def test_global_addresses(self): + text = ("2: eth0 inet6 2001:db8::5/64 scope global \\\n" + "3: wg0@NONE inet6 fd00::1/64 scope global\n" + "garbage\n") + self.assertEqual(global_addresses(text), [ + ("eth0", "2001:db8::5"), ("wg0", "fd00::1")]) + + def test_defaults_are_live(self): + found = Node("/", "/etc/keel/instance.yaml") + self.assertIs(found.apply, live_apply) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_inviting.py b/tests/test_mesh_inviting.py new file mode 100644 index 0000000..add4aba --- /dev/null +++ b/tests/test_mesh_inviting.py @@ -0,0 +1,313 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The inviter's side: the listener's unit, serve, and accept (0048) + +serve and accept against a scratch root's pending invite and a real +Node (tests/mesh_wire.py), with the socket loop replaced by a function +that plays the new node's requests into the Listener; the unit's argv +through a recording runner. +""" + +import os +import shutil +import signal +import sys +import tempfile +import unittest +from unittest import mock + +from mesh_helpers import ( + ASSIGNED, + INVITE, + INVITER, + JOINER, + KEY, + NOW, + Clock, + confirm_body, + join_body, + reserved, + signature, +) +from mesh_wire import ( + INVITER_SPEC, + JOINER_OVERLAY, + Armed, + node, + probes, +) +from test_network_window import Recorder + +from keel import exits +from keel.mesh import acceptline, invites, inviting, protocol +from keel.mesh.acceptline import Accept +from keel.mesh.node import NodeError +from keel.network import marker + +UPLINK = "2001:db8:2::20" + + +class Played: + """Bridge.run, without a listener: the requests it would forward""" + + def __init__(self, *requests, fail=None): + self.requests = requests + self.fail = fail + self.calls = [] + self.answers = [] + + def __call__(self, bridge): + self.calls.append((bridge.params.host, bridge.params.port)) + self.bridge = bridge + if self.fail: + raise self.fail + for path, body, local, peer in self.requests: + self.answers.append(bridge.admitter.forward( + path, signature(path, body), body, local, peer)) + + +def join_request(): + return (protocol.JOIN, join_body(), "2001:db8:1::10", UPLINK) + + +def confirm_request(): + return (protocol.CONFIRM, confirm_body(), "fd00:6b65:1::1", + JOINER_OVERLAY) + + +class Case(unittest.TestCase): + def setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.root) + self.invite = reserved(self.root) + self.node = node(self.root, INVITER_SPEC) + self.out, self.err = [], [] + self.run = Recorder() + self.inviter = inviting.Inviter( + self.node, Clock(), self.out.append, self.err.append, + run=self.run, output=lambda argv: "table inet keel {}") + keys = mock.patch("keel.mesh.inviting.wgkeys.public", + return_value=(INVITER, None)) + self.public = keys.start() + self.addCleanup(keys.stop) + + def file(self): + return os.path.join(self.root, invites.INVITES, f"{INVITE}.json") + + def peers(self): + return self.node.overlay().get("peers") + + +class TestUnit(unittest.TestCase): + def test_the_listener_runs_as_a_transient_unit(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root) + made = reserved(root) + run = Recorder() + self.assertIsNone(inviting.start(made, "/etc/keel/instance.yaml", + NOW, run)) + self.assertEqual(run.calls, [( + "systemd-run", f"--unit=keel-mesh-invite@{INVITE}", "--collect", + "--quiet", f"--property=RuntimeMaxSec={3600 + inviting.LINGER}", + f"--description=keel: the listener of mesh invite {INVITE}", + sys.executable, "-m", "keel", "mesh", "serve", INVITE, "--spec", + "/etc/keel/instance.yaml")]) + + +class TestServe(Case): + def test_a_join_and_its_confirmation(self): + played = Played(join_request(), confirm_request()) + self.inviter.bridged = played + self.assertEqual(inviting.serve_invite(self.inviter, INVITE), + exits.OK) + self.assertEqual(played.calls, [("::", 51820)]) + self.assertEqual([one.status for one in played.answers], [200, 200]) + self.assertEqual(self.peers()[0]["public_key"], JOINER) + self.assertFalse(marker.exists(self.root)) + self.assertFalse(os.path.exists(self.file())) + self.assertIn(("nft", "delete", "element", "inet", "keel", + "mesh_invites", "{ 51820 }"), self.run.calls) + self.assertEqual(self.err[-1], f"invite {INVITE}: joined and" + " confirmed") + + def test_sigterm_ends_through_the_cleanup(self): + with self.assertRaises(SystemExit): + inviting.terminated(signal.SIGTERM, None) + + def test_an_invite_that_is_not_pending(self): + invites.remove(self.root, INVITE) + self.assertEqual(inviting.serve_invite(self.inviter, INVITE), + exits.MESH_REFUSED) + self.assertIn("not pending", self.err[0]) + + def test_no_key_to_answer_with(self): + self.public.return_value = (None, "wg cannot read the key") + self.assertEqual(inviting.serve_invite(self.inviter, INVITE), + exits.APPLY_FAILED) + self.assertEqual(self.err, ["wg cannot read the key"]) + self.assertFalse(os.path.exists(self.file())) + + def test_a_port_already_taken(self): + self.inviter.bridged = Played(fail=OSError(98, "Address in use")) + self.assertEqual(inviting.serve_invite(self.inviter, INVITE), + exits.APPLY_FAILED) + self.assertIn("Address in use", self.err[1]) + self.assertFalse(os.path.exists(self.file())) + self.assertIn("expired unused", self.err[-1]) + + def test_another_invite_on_the_port_keeps_it_open(self): + reserved(self.root, secret=bytes(32)) + self.inviter.bridged = Played() + inviting.serve_invite(self.inviter, INVITE) + self.assertNotIn("delete", [one[1] for one in self.run.calls]) + + +class TestAccept(Case): + def line(self, key=KEY, **changed): + values = dict(public_key=JOINER, endpoint="[2001:db8:2::20]:51820", + address=ASSIGNED, invite_id=INVITE, + time=protocol.seconds(NOW)) + values.update(changed) + return acceptline.encode(Accept(**values), key) + + def accept(self, text=None, played=None): + self.inviter.bridged = played or Played(confirm_request()) + return inviting.accept(self.inviter, text or self.line()) + + def test_the_peer_applied_and_the_mesh_session_on_the_overlay(self): + played = Played(confirm_request()) + self.assertEqual(self.accept(played=played), exits.OK, self.err) + self.assertEqual(played.calls, [("fd00:6b65:1::1", 51820)]) + self.assertEqual(self.peers(), [{ + "public_key": JOINER, "allowed_ips": [f"{JOINER_OVERLAY}/128"], + "persistent_keepalive": 25, + "endpoint": "[2001:db8:2::20]:51820"}]) + self.assertIn(("systemctl", "stop", f"keel-mesh-invite@{INVITE}"), + self.run.calls) + self.assertIn(("nft", "add", "element", "inet", "keel", + "mesh_invites", "{ 51820 timeout 120s }"), + self.run.calls) + self.assertEqual(self.out, [f"accepted: {JOINER} is a peer of this" + f" node at {JOINER_OVERLAY}"]) + self.assertFalse(os.path.exists(self.file())) + self.assertFalse(marker.exists(self.root)) + + def test_a_new_node_without_an_endpoint(self): + self.assertEqual(self.accept(self.line(endpoint=None)), exits.OK) + self.assertNotIn("endpoint", self.peers()[0]) + + def test_refused_before_anything_is_spent(self): + for text, code, words in ( + ("keel1:x", exits.MESH_TOKEN_INVALID, "keel1a:"), + (self.line(key=bytes(32)), exits.MESH_REFUSED, + "HMAC does not match"), + (self.line(address="fd00:6b65:1::9/64"), exits.MESH_REFUSED, + "HMAC does not match"), + (self.line(invite_id="00" * 8), exits.MESH_REFUSED, + "no pending invite")): + with self.subTest(words=words): + self.err.clear() + self.assertEqual(self.accept(text), code) + self.assertIn(words, self.err[0]) + self.assertFalse(invites.read(self.root, INVITE).consumed) + self.assertEqual(self.node.apply.documents, []) + + def test_the_inviter_s_own_key_is_refused(self): + self.assertEqual(self.accept(self.line(public_key=INVITER)), + exits.MESH_REFUSED) + self.assertIn("this node's own", self.err[-1]) + self.assertIsNone(self.node.overlay().get("peers")) + + def test_a_change_waiting(self): + marker.save(self.root, "") + marker.write(self.root, marker.Pending("eth0", "x", 120)) + self.assertEqual(self.accept(), exits.MESH_REFUSED) + self.assertIn("a network change waits", self.err[0]) + + def test_no_key(self): + self.public.return_value = (None, "no key") + self.assertEqual(self.accept(), exits.APPLY_FAILED) + self.assertFalse(invites.read(self.root, INVITE).consumed) + + def test_an_invite_used_already(self): + invites.consume(self.root, INVITE, NOW) + self.assertEqual(self.accept(), exits.MESH_REFUSED) + self.assertIn("already used", self.err[0]) + + def test_the_change_does_not_come_up(self): + self.node.apply = Armed(code=16, arms=False) + self.assertEqual(self.accept(), exits.APPLY_FAILED) + self.assertIn("apply exited 16", self.err[-1]) + self.assertFalse(os.path.exists(self.file())) + + def test_a_spec_that_cannot_take_the_peer(self): + with mock.patch.object(self.node, "write", + side_effect=NodeError("full")): + self.assertEqual(self.accept(), exits.MESH_REFUSED) + self.assertEqual(self.err[-1], "full; the invite is spent") + + def test_the_new_node_never_comes(self): + self.assertEqual(self.accept(played=Played()), + exits.NETWORK_NOT_CONFIRMED) + self.assertIn("not confirmed", self.err[-1]) + self.assertTrue(marker.exists(self.root)) + + def test_the_overlay_address_cannot_be_bound(self): + found = self.accept(played=Played(fail=OSError(99, "not here"))) + self.assertEqual(found, exits.NETWORK_NOT_CONFIRMED) + self.assertIn("cannot serve [fd00:6b65:1::1]:51820", + self.err[-1]) + + def test_a_confirmation_the_route_check_refuses(self): + self.node.probes = lambda: probes("wg0") + played = Played(confirm_request()) + self.assertEqual(self.accept(played=played), + exits.NETWORK_NOT_CONFIRMED) + answer = protocol.confirm_answer(played.answers[0].body) + self.assertFalse(answer.confirmed) + + +class TestClose(Case): + def test_the_port_closes_with_the_last_invite_on_it(self): + self.inviter.close(self.invite) + self.assertFalse(os.path.exists(self.file())) + self.assertEqual(self.run.calls, [( + "nft", "delete", "element", "inet", "keel", "mesh_invites", + "{ 51820 }")]) + + def test_the_listener_of_each_invite_and_its_socket(self): + played = Played() + self.inviter.bridged = played + inviting.serve_invite(self.inviter, INVITE) + found = played.bridge + self.assertEqual(found.path, os.path.join( + self.root, "run/keel", f"mesh-listen-{INVITE}", "bridge.sock")) + self.assertEqual((found.params.host, found.params.overlay, + found.params.joined), ("::", "fd00:6b65:1::1", + None)) + # the listener is never given the invite's HMAC key + self.assertNotIn("hmac", repr(found.params)) + self.assertEqual((found.certificate, found.tls_key), + (self.invite.certificate, self.invite.tls_key)) + + def test_the_listener_starts_as_its_unit_unless_told(self): + started = [] + self.inviter.start = lambda *args: started.append(args) or "unit" + self.assertEqual(self.inviter.starter(INVITE, 30)("path"), "unit") + self.assertEqual(started, [(INVITE, "path", 30)]) + self.inviter.start = None + with mock.patch("keel.mesh.inviting.bridge.start_unit", + return_value="unit") as unit: + self.assertEqual(self.inviter.starter(INVITE, 30)("path"), + "unit") + unit.assert_called_once_with(INVITE, "path", 30, self.run, + self.inviter.output) + + def test_own(self): + self.assertEqual(self.inviter.own(), (INVITER, "fd00:6b65:1::1/64")) + self.public.return_value = (None, "unreadable") + self.assertEqual(self.inviter.own(), (None, "unreadable")) + self.assertEqual(self.inviter.root, self.root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_joining.py b/tests/test_mesh_joining.py new file mode 100644 index 0000000..efb7e82 --- /dev/null +++ b/tests/test_mesh_joining.py @@ -0,0 +1,412 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh join, the new node's side, against a real listener + +Two scratch roots, a real Node each and the inviter's real Listener, +joined by a wire in this process (tests/mesh_wire.py): what each spec +holds afterwards, that each window is confirmed by the mesh session +and by nothing else, the fallback's line, and every refusal before +anything is applied. +""" + +import os +import shutil +import tempfile +import unittest +from datetime import timedelta +from unittest import mock + +import yaml +from mesh_helpers import ( + ASSIGNED, + INVITER, + JOINER, + NOW, + OTHER, + Clock, + reserved, +) +from mesh_wire import ( + INVITER_SPEC, + INVITER_UPLINK, + JOINER_OVERLAY, + Armed, + Wire, + node, + probes, + token, +) + +from keel import exits +from keel.mesh import acceptline, identity, invites, joining, protocol +from keel.mesh.admit import Admitter, Joined +from keel.mesh.listener import Listener, Params +from keel.mesh.node import NodeError +from keel.mesh.token import hmac_key +from keel.network import marker + +JOINER_ENDPOINT = "2001:db8:2::20" + + +def spec_of(found): + with open(found.path) as fob: + return yaml.safe_load(fob) + + +class Case(unittest.TestCase): + def setUp(self): + self.inviter_root = tempfile.mkdtemp() + self.joiner_root = tempfile.mkdtemp() + for root in (self.inviter_root, self.joiner_root): + self.addCleanup(shutil.rmtree, root) + self.invite = reserved(self.inviter_root) + self.inviter = node(self.inviter_root, INVITER_SPEC) + self.joiner = node(self.joiner_root) + self.clock = Clock() + self.logged = [] + self.admitter = Admitter(self.inviter_root, self.invite, INVITER, + "fd00:6b65:1::1/64", self.inviter, + self.clock, self.logged.append) + self.listener = Listener( + Params(self.invite.invite_id, + self.invite.expires, "::", 51820, "fd00:6b65:1::1"), + self.admitter, self.clock, self.logged.append) + self.wire = Wire(self.listener, token(self.invite).fingerprint) + self.out, self.err, self.slept = [], [], [] + keys = mock.patch.multiple( + "keel.mesh.joining.wgkeys", generate=mock.DEFAULT, + public=mock.DEFAULT) + found = keys.start() + self.addCleanup(keys.stop) + found["generate"].return_value = None + found["public"].return_value = (JOINER, None) + self.keys = found + + def joining(self, endpoint=JOINER_ENDPOINT, route="eth0", **changed): + self.found = joining.Joiner( + self.joiner, token(self.invite, **changed), self.clock, + self.out.append, self.err.append, post=self.wire, + sleep=self.slept.append, + route_dev=lambda host: route) + return joining.run(self.found, endpoint) + + +class TestJoin(Case): + def test_both_specs_and_both_windows(self): + code = self.joining() + self.assertEqual(code, exits.OK, self.err) + mine = spec_of(self.joiner)["network"]["overlay"]["wireguard"] + self.assertEqual(mine, {"address": ASSIGNED, "peers": [{ + "public_key": INVITER, "endpoint": f"[{INVITER_UPLINK}]:51821", + "allowed_ips": ["fd00:6b65:1::1/128"]}]}) + theirs = spec_of(self.inviter)["network"]["overlay"]["wireguard"] + self.assertEqual(theirs["peers"], [{ + "public_key": JOINER, "allowed_ips": [f"{JOINER_OVERLAY}/128"], + "endpoint": f"[{JOINER_ENDPOINT}]:51820"}]) + for root in (self.inviter_root, self.joiner_root): + self.assertFalse(marker.exists(root)) + self.assertEqual(marker.last(root).outcome, marker.CONFIRMED) + self.assertEqual(self.out, [ + f"joined the mesh: this node is {ASSIGNED} on wg0", + f"peer: {INVITER} at fd00:6b65:1::1, through" + f" [{INVITER_UPLINK}]:51821", + joining.TWO_NODES]) + self.assertIn("the overlay was tested: the inviter's answer over the" + " overlay", "\n".join(self.err)) + self.assertTrue(self.listener.confirmed) + self.assertEqual(identity.read(self.joiner_root), bytes(range(16))) + self.assertEqual([path for _, path in self.wire.calls], + [protocol.JOIN, protocol.CONFIRM]) + self.assertEqual(self.wire.calls[1][0], "fd00:6b65:1::1") + + def test_a_node_behind_nat_sends_no_endpoint(self): + self.assertEqual(self.joining(endpoint=None), exits.OK, self.err) + theirs = spec_of(self.inviter)["network"]["overlay"]["wireguard"] + self.assertNotIn("endpoint", theirs["peers"][0]) + + def test_a_mesh_of_three_says_what_this_keel_does_not_do(self): + with open(self.inviter.path, "a") as fob: + fob.write(f" peers:\n - public_key: {OTHER}\n" + " allowed_ips: [fd00:6b65:1::7/128]\n") + self.assertEqual(self.joining(), exits.OK, self.err) + self.assertIn("the mesh has 3 nodes", self.out[-1]) + + def test_the_mesh_session_is_retried_while_the_tunnel_comes_up(self): + calls = [] + real = self.wire + + def flaky(host, port, path, *args, **kwargs): + calls.append(path) + if path == protocol.CONFIRM and calls.count(path) < 3: + raise joining.Unreachable("not yet") + return real(host, port, path, *args, **kwargs) + self.wire = flaky + self.assertEqual(self.joining(), exits.OK, self.err) + self.assertEqual(self.slept, [joining.RETRY, joining.RETRY]) + + +class TestRefusedBeforeApplying(Case): + def refused(self, code, words, **kwargs): + found = self.joining(**kwargs) + self.assertEqual(found, code, self.err) + self.assertIn(words, "\n".join(self.err)) + self.assertEqual(self.joiner.apply.documents, []) + self.assertFalse(os.path.exists(self.joiner.path)) + return found + + def test_no_route_to_the_inviter(self): + self.refused(exits.MESH_REFUSED, "no route to the inviter", + route=None) + self.assertIn("rendezvous point", self.err[0]) + + def test_a_change_waiting(self): + marker.save(self.joiner_root, "") + marker.write(self.joiner_root, marker.Pending("eth0", "x", 120)) + self.refused(exits.MESH_REFUSED, "a network change waits") + + def test_another_mesh(self): + identity.adopt(self.joiner_root, bytes(16)) + self.refused(exits.MESH_REFUSED, "in another mesh") + + def test_a_damaged_identity(self): + os.makedirs(os.path.join(self.joiner_root, "var/lib/keel/mesh")) + with open(os.path.join(self.joiner_root, identity.IDENTITY), + "w") as fob: + fob.write("x\n") + self.refused(exits.MESH_REFUSED, "does not hold a mesh identity") + + def test_a_refused_join_keeps_no_identity(self): + """a spent token must not tie this node to its mesh""" + invites.consume(self.inviter_root, self.invite.invite_id, NOW) + self.joining() + self.assertIsNone(identity.read(self.joiner_root)) + + def test_a_spec_that_cannot_be_read(self): + with open(self.joiner.path, "w") as fob: + fob.write("version: [") + self.assertEqual(self.joining(), exits.MESH_REFUSED) + + def test_a_spec_the_join_would_make_invalid(self): + with open(self.joiner.path, "w") as fob: + fob.write("version: 1\nnetwork:\n interfaces:\n eth0:\n" + " ipv6: {method: static, address:" + " 'fd00:6b65:1::9/48'}\n") + found = self.joining() + self.assertEqual(found, exits.MESH_REFUSED) + self.assertIn("once joined", self.err[0]) + + def test_a_node_at_another_address(self): + with open(self.joiner.path, "w") as fob: + fob.write("version: 1\nnetwork:\n overlay:\n wireguard:\n" + " address: fd00:6b65:1::9/64\n") + self.assertEqual(self.joining(), exits.MESH_REFUSED) + self.assertIn("a node joins a mesh once", self.err[0]) + + def test_no_key(self): + self.keys["generate"].return_value = "wg is not installed" + self.refused(exits.APPLY_FAILED, "wg is not installed") + + def test_the_inviter_refuses(self): + invites.consume(self.inviter_root, self.invite.invite_id, NOW) + self.refused(exits.MESH_REFUSED, "the inviter refused the join:" + " invite") + + def test_another_certificate(self): + self.wire.pinned = bytes(32) + self.refused(exits.MESH_REFUSED, "another certificate") + + def test_an_answer_to_another_request(self): + real = self.wire + + def other(host, port, path, body, *args, **kwargs): + reply = real(host, port, path, body, *args, **kwargs) + data = protocol.join_answer(reply.body) + forged = protocol.dumps(protocol.JoinAnswer( + **{**data.__dict__, "nonce": "ff" * 16})) + return reply.__class__(forged, reply.local, reply.peer) + self.wire = other + self.refused(exits.MESH_REFUSED, "not the answer to this request") + + def test_a_malformed_answer(self): + real = self.wire + + def broken(*args, **kwargs): + reply = real(*args, **kwargs) + return reply.__class__(b"{}", reply.local, reply.peer) + self.wire = broken + self.refused(exits.MESH_REFUSED, "answer is malformed") + + +class TestAfterTheAnswer(Case): + def test_this_side_does_not_come_up(self): + self.joiner.apply = Armed(code=16, arms=False) + self.assertEqual(self.joining(), exits.APPLY_FAILED) + self.assertIn("apply exited 16", self.err[-1]) + self.assertIn("the token is spent", self.err[-1]) + + def test_a_spec_that_cannot_be_written(self): + with mock.patch.object(self.joiner, "write", + side_effect=NodeError("cannot be written:" + " No space left")): + self.assertEqual(self.joining(), exits.APPLY_FAILED) + self.assertIn("cannot be written: No space left", self.err[-1]) + self.assertEqual(self.joiner.apply.documents, []) + + def test_the_inviter_never_answers_over_the_overlay(self): + self.wire.unreachable.add(protocol.CONFIRM) + clock = self.clock + + def later(seconds): + clock.now += timedelta(seconds=seconds) + self.slept = [] + found = joining.Joiner( + self.joiner, token(self.invite), clock, self.out.append, + self.err.append, post=self.wire, sleep=later, + route_dev=lambda host: "eth0") + self.assertEqual(joining.run(found, JOINER_ENDPOINT), + exits.NETWORK_NOT_CONFIRMED) + self.assertIn("did not answer over the overlay", self.err[-1]) + self.assertTrue(marker.exists(self.joiner_root)) + + def test_the_inviter_does_not_keep_its_change(self): + self.inviter.probes = lambda: probes("wg0") + self.assertEqual(self.joining(), exits.NETWORK_NOT_CONFIRMED) + self.assertIn("the inviter did not keep its change", self.err[-1]) + self.assertTrue(marker.exists(self.joiner_root)) + + def test_this_node_does_not_keep_its_own(self): + self.joiner.probes = lambda: probes("wg0") + self.assertEqual(self.joining(), exits.NETWORK_NOT_CONFIRMED) + self.assertIn("leaves through wg0", self.err[-1]) + + def test_a_refused_mesh_session(self): + real = self.wire + + def refuse(host, port, path, *args, **kwargs): + if path == protocol.CONFIRM: + raise joining.Refused("no") + return real(host, port, path, *args, **kwargs) + self.wire = refuse + self.assertEqual(self.joining(), exits.NETWORK_NOT_CONFIRMED) + self.assertIn("the mesh session was refused: no", self.err[-1]) + + def test_a_confirmation_of_another_session(self): + real = self.wire + + def other(host, port, path, *args, **kwargs): + reply = real(host, port, path, *args, **kwargs) + if path != protocol.CONFIRM: + return reply + data = protocol.confirm_answer(reply.body) + forged = protocol.dumps(protocol.ConfirmAnswer( + **{**data.__dict__, "nonce": "ff" * 16})) + return reply.__class__(forged, reply.local, reply.peer) + self.wire = other + self.assertEqual(self.joining(), exits.NETWORK_NOT_CONFIRMED) + self.assertIn("not the answer to this mesh session", self.err[-1]) + self.assertTrue(marker.exists(self.joiner_root)) + + def test_a_malformed_confirmation(self): + real = self.wire + + def broken(host, port, path, *args, **kwargs): + reply = real(host, port, path, *args, **kwargs) + if path == protocol.CONFIRM: + return reply.__class__(b"[]", reply.local, reply.peer) + return reply + self.wire = broken + self.assertEqual(self.joining(), exits.NETWORK_NOT_CONFIRMED) + self.assertIn("malformed", self.err[-1]) + + +class TestFallback(Case): + def setUp(self): + super().setUp() + self.wire.unreachable.add(protocol.JOIN) + + def accepted(self): + """What accept leaves the inviter in: the peer applied, the + listener waiting for the mesh session""" + self.inviter.admit({"public_key": JOINER, + "allowed_ips": [f"{JOINER_OVERLAY}/128"]}) + until = NOW + timedelta(seconds=120) + self.admitter.joined = Joined(JOINER, JOINER_OVERLAY, + marker.read(self.inviter_root), until) + self.listener.joined = (JOINER, JOINER_OVERLAY, until) + + def test_the_line_for_the_inviter_and_the_wait(self): + self.accepted() + code = self.joining() + self.assertEqual(code, exits.OK, self.err) + line = self.out[0] + self.assertTrue(line.startswith("keel mesh accept keel1a:")) + sealed = acceptline.parse(line.split()[-1]) + self.assertTrue(sealed.authentic(hmac_key(token( + self.invite).secret))) + self.assertEqual(sealed.accept.endpoint, f"[{JOINER_ENDPOINT}]:51820") + self.assertEqual(sealed.accept.address, ASSIGNED) + self.assertIn("run the line above on the inviter within 15 minutes", + self.err[1]) + [(doc, window)] = self.joiner.apply.documents + self.assertEqual(window, 900) + self.assertEqual(self.out[-1], joining.TWO_NODES) + + def test_the_window_is_the_time_left_to_the_expiry(self): + self.accepted() + self.clock.now = NOW + timedelta(minutes=55) + self.joining() + self.assertEqual(self.joiner.apply.documents[0][1], 300) + + def test_too_close_to_the_expiry(self): + self.clock.now = NOW + timedelta(minutes=59, seconds=40) + self.assertEqual(self.joining(), exits.MESH_REFUSED) + self.assertIn("too soon", self.err[-1]) + self.assertEqual(self.joiner.apply.documents, []) + + def test_both_behind_nat(self): + self.assertEqual(self.joining(endpoint=None), exits.MESH_REFUSED) + self.assertIn("neither node can reach the other", self.err[-1]) + self.assertEqual(self.joiner.apply.documents, []) + + def test_this_side_does_not_come_up(self): + self.joiner.apply = Armed(code=16, arms=False) + self.assertEqual(self.joining(), exits.APPLY_FAILED) + self.assertEqual(self.out, []) + + +class TestOwnEndpoint(unittest.TestCase): + def test_declared_first(self): + self.assertEqual(joining.own_endpoint("2001:db8::5", {}, [], "wg0"), + "2001:db8::5") + + def test_static_then_global_ipv6_first(self): + doc = {"network": {"interfaces": {"eth0": { + "ipv4": {"method": "static", "address": "192.0.2.5/24"}}}}} + held = [("eth0", "198.51.100.1"), ("wg0", "fd00::1"), + ("eth1", "2001:db8::9"), ("lo", "::1")] + self.assertEqual(joining.own_endpoint(None, doc, held, "wg0"), + "2001:db8::9") + self.assertEqual(joining.own_endpoint(None, doc, [], "wg0"), + "192.0.2.5") + self.assertIsNone(joining.own_endpoint(None, {}, [("lo", "::1")], + "wg0")) + + def test_endpoint_text(self): + self.assertEqual(joining.endpoint_text("192.0.2.1"), "192.0.2.1") + self.assertEqual(joining.endpoint_text("2001:db8::1"), + "[2001:db8::1]") + + +class TestOwnKey(unittest.TestCase): + def test_an_existing_key_is_read_not_made(self): + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root) + os.makedirs(os.path.join(root, "etc/wireguard")) + open(os.path.join(root, "etc/wireguard/wg0.key"), "w").close() + with mock.patch("keel.mesh.joining.wgkeys") as keys: + keys.public.return_value = (None, "unreadable") + self.assertEqual(joining.own_key(root, {}), (None, "unreadable")) + keys.generate.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_listener.py b/tests/test_mesh_listener.py new file mode 100644 index 0000000..780953b --- /dev/null +++ b/tests/test_mesh_listener.py @@ -0,0 +1,247 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The unprivileged listener's checks, before anything reaches root + +Listener.handle with a backend that records: what is forwarded (the +signature unchecked: the listener has no HMAC key) and what is refused, +what a refusal counts towards (the source's limit), what the root +side's answers teach the listener (its `final` ends it), and when it +is finished. +""" + +import io +import json +import socket +import unittest +from datetime import timedelta +from unittest import mock + +from mesh_helpers import ( + INVITE, + JOINER, + NOW, + Clock, + confirm_body, + join_body, + signature, +) + +from keel.mesh import listener, protocol +from keel.mesh.admit import Response +from keel.mesh.listener import Capped, Limiter, Listener, Params + +UPLINK = "2001:db8:2::20" +OVERLAY = "fd00:6b65:1::1" +JOINER_OVERLAY = "fd00:6b65:1::3" +ANSWER = protocol.dumps(protocol.JoinAnswer( + invite_id=INVITE, nonce="00" * 16, + public_key="nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E=", + address="fd00:6b65:1::1/64", peers=(), etcd="none", window=120)) + + +def params(**changed) -> Params: + values = dict(invite_id=INVITE, + expires=NOW + timedelta(hours=1), host="::", port=51820, + overlay=OVERLAY) + values.update(changed) + return Params(**values) + + +class Backend: + def __init__(self, status=200, body=ANSWER, final=False): + self.status = status + self.body = body + self.final = final + self.forwarded = [] + + def forward(self, path, signature, body, local, peer): + self.forwarded.append((path, signature, local, peer)) + return Response(self.status, self.body, "sig", self.final) + + +class Case(unittest.TestCase): + def setUp(self): + self.clock = Clock() + self.backend = Backend() + self.logged = [] + self.listener = Listener(params(), self.backend, self.clock, + self.logged.append) + + def join(self, body=None, sign=None, method="POST", path=protocol.JOIN, + peer=UPLINK): + body = join_body() if body is None else body + found = sign if sign is not None else signature(path, body) + return self.listener.handle(method, path, found, body, "::", peer) + + def confirm(self, local=OVERLAY, peer=JOINER_OVERLAY, body=None): + body = confirm_body() if body is None else body + return self.listener.handle( + "POST", protocol.CONFIRM, signature(protocol.CONFIRM, body), + body, local, peer) + + +class TestForwarded(Case): + def test_a_good_join_is_forwarded_and_its_answer_learnt(self): + body = join_body() + found = self.join(body) + self.assertEqual(found.status, 200) + self.assertEqual(self.backend.forwarded, + [(protocol.JOIN, signature(protocol.JOIN, body), + "::", UPLINK)]) + self.assertEqual(self.listener.joined, + (JOINER, JOINER_OVERLAY, NOW + timedelta( + seconds=120))) + self.assertEqual(self.join().status, 410) + + def test_the_confirmation_over_the_overlay_only(self): + self.join() + self.backend.body = protocol.dumps(protocol.ConfirmAnswer( + invite_id=INVITE, nonce="00" * 16, confirmed=True, detail="")) + for local, peer in ((UPLINK, JOINER_OVERLAY), (OVERLAY, UPLINK)): + self.assertEqual(self.confirm(local, peer).status, 403) + self.assertEqual(self.confirm().status, 200) + self.assertTrue(self.listener.confirmed) + self.assertTrue(self.listener.finished) + + def test_no_confirmation_before_a_join(self): + self.assertEqual(self.confirm().status, 409) + self.assertEqual(self.backend.forwarded, []) + + def test_a_refusal_from_the_root_side_is_passed_on(self): + self.backend.status, self.backend.body = 409, b'{"error": "x"}' + self.assertEqual(self.join().status, 409) + self.assertIsNone(self.listener.joined) + self.assertFalse(self.listener.finished) + + def test_an_answer_it_cannot_read_ends_it(self): + self.backend.body = b"{}" + self.assertEqual(self.join().status, 200) + self.assertTrue(self.listener.finished) + self.assertIn("cannot be read", self.logged[-1]) + + def test_a_final_answer_or_a_root_side_gone_ends_it(self): + for status, final in ((500, True), (410, True), (503, False)): + with self.subTest(status=status): + self.setUp() + self.backend.status, self.backend.final = status, final + self.join() + self.assertTrue(self.listener.finished) + + def test_a_signature_is_the_root_side_s_to_check(self): + """Without the key the listener forwards a forged request, and + the root side refuses it""" + self.backend.status, self.backend.body = 403, b'{"error": "x"}' + self.assertEqual(self.join(sign="00" * 32).status, 403) + self.assertEqual(len(self.backend.forwarded), 1) + self.assertEqual(self.backend.forwarded[0][1], "00" * 32) + + +class TestRefused(Case): + def test_nothing_unchecked_reaches_the_root_side(self): + for kwargs in (dict(path="/"), dict(method="GET"), + dict(body=b"{}"), + dict(body=join_body(invite_id="00" * 8)), + dict(body=join_body( + time=protocol.seconds(NOW) - 600))): + with self.subTest(kwargs=list(kwargs)): + self.assertGreaterEqual(self.join(**kwargs).status, 400) + self.assertEqual(self.backend.forwarded, []) + + def test_a_replay_is_refused_here(self): + body = join_body() + self.backend.status = 409 + self.join(body) + found = self.join(body) + self.assertEqual(found.status, 403) + self.assertIn("replayed", json.loads(found.body)["error"]) + + def test_a_scanner_does_not_cancel_the_invite(self): + for kwargs in (dict(path="/"), dict(body=b"{}"), + dict(body=join_body(invite_id="00" * 8), + sign="00" * 32)) * 3: + self.join(peer="2001:db8:9::1", **kwargs) + self.assertEqual(self.backend.forwarded, []) + self.assertFalse(self.listener.finished) + + def test_the_root_side_s_refusals_count_for_the_source(self): + self.backend.status, self.backend.body = 403, b'{"error": "x"}' + for _ in range(5): + self.join() + self.assertTrue(self.listener.blocked(UPLINK)) + + def test_a_source_refused_five_times_is_not_answered(self): + for _ in range(5): + self.assertFalse(self.listener.blocked(UPLINK)) + self.join(path="/") + self.assertTrue(self.listener.blocked(UPLINK)) + self.assertFalse(self.listener.blocked("2001:db8:9::1")) + self.clock.now += timedelta(seconds=61) + self.assertFalse(self.listener.blocked(UPLINK)) + + def test_each_refusal_is_logged_with_its_source(self): + self.join(path="/") + self.assertIn(f"invite {INVITE}: refused a request from {UPLINK}", + self.logged[0]) + + +class TestFinished(Case): + def test_at_the_expiry_unused(self): + self.assertFalse(self.listener.finished) + self.clock.now = NOW + timedelta(hours=1) + self.assertTrue(self.listener.finished) + + def test_a_join_waits_one_window_for_its_confirmation(self): + self.clock.now = NOW + timedelta(minutes=59, seconds=50) + self.join(join_body(time=protocol.seconds(self.clock.now))) + self.clock.now += timedelta(seconds=119) + self.assertFalse(self.listener.finished) + self.clock.now += timedelta(seconds=1) + self.assertTrue(self.listener.finished) + + def test_accept_s_listener_serves_the_confirmation_alone(self): + found = Listener(params(joined=(JOINER, JOINER_OVERLAY, + NOW + timedelta(seconds=120))), + self.backend, self.clock) + self.assertEqual(found.name, f"invite {INVITE}") + with mock.patch("builtins.print") as printed: + self.assertEqual(found.handle( + "POST", protocol.JOIN, None, b"", "::", UPLINK).status, 410) + self.assertIn("already used", printed.call_args.args[0]) + + +class TestCapped(unittest.TestCase): + def test_the_budget_covers_lines_and_bodies(self): + reader = Capped(io.BytesIO(b"line\n" + b"x" * 20), 15) + self.assertEqual(reader.readline(), b"line\n") + self.assertEqual(reader.read(10), b"x" * 10) + with self.assertRaises(ValueError): + reader.read(10) + reader.close() + self.assertTrue(reader.inner.closed) + + +class TestSockets(unittest.TestCase): + def test_a_port_already_taken(self): + first = listener.bound("::1", 0) + with first, self.assertRaises(OSError): + listener.bound("::1", first.getsockname()[1]) + + def test_the_deadline_on_a_socket_already_gone(self): + gone = socket.socket() + gone.close() + listener.shut([gone]) + + +class TestLimiter(unittest.TestCase): + def test_old_refusals_are_forgotten(self): + clock = Clock() + found = Limiter(clock) + for _ in range(4): + found.refused("a") + clock.now += listener.PERIOD + timedelta(seconds=1) + found.refused("a") + self.assertFalse(found.blocked("a")) + self.assertEqual(len(found.seen["a"]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_netns.py b/tests/test_mesh_netns.py new file mode 100644 index 0000000..d6709d4 --- /dev/null +++ b/tests/test_mesh_netns.py @@ -0,0 +1,125 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""keel mesh across three network namespaces, with WireGuard and TLS + +tests/mesh_netns.py, as root in a network namespace of its own +(tests/wgtools.py: `unshare -n` as root, `unshare -rn`, or `sudo -n +unshare -n`, which is what Ubuntu's CI runners allow), with the real +wg and wg-quick, which need the wireguard kernel module. Three nodes +joined by veths: B joins A through A's listener, C through the fallback +and `keel mesh accept`, both ping A over the overlay with no `keel +network confirm` typed, the spent invite is refused, no secret reaches +A's journal, and the listener that faced the network held no +capability. In CI a missing tool or namespace fails instead of +skipping. + +Twice: on clean links, and on poor ones (POOR: 250 ms of delay with +25 ms of jitter, and 2% loss, on every veth, both ends; `tc` and the +sch_netem module), which Keel must work over. +""" + +import base64 +import json +import os +import shutil +import subprocess +import sys +import unittest +from datetime import datetime, timezone + +import wgtools + +from keel import exits +from keel.mesh.token import parse + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DRIVER = os.path.join(REPO, "tests", "mesh_netns.py") + + +def secret_forms(token: str) -> list[str]: + secret = parse(token, datetime(2000, 1, 1, tzinfo=timezone.utc)).secret + return [token, secret.hex(), base64.b64encode(secret).decode()[:40], + base64.urlsafe_b64encode(secret).decode()[:40]] + + +POOR = "250ms 25ms 2%" + + +class TestThreeNamespaces(unittest.TestCase): + netem = "" + reports: dict = {} + + def setUp(self): + tools = wgtools.require(self) + if not os.path.exists("/sys/module/wireguard") and \ + not os.environ.get("CI"): + self.skipTest("the wireguard kernel module is not loaded") + if self.netem and not shutil.which("tc") and \ + not os.environ.get("CI"): + self.skipTest("tc not found (iproute2)") + if self.netem not in self.reports: + self.reports[self.netem] = self.run_driver(tools) + self.found = self.reports[self.netem] + + def run_driver(self, tools: str) -> dict: + prefix = wgtools.namespace_prefix() + if prefix is None: + if os.environ.get("CI"): + self.fail("no way to run wg-quick as root in a namespace") + self.skipTest("neither root, user namespaces nor sudo -n") + done = subprocess.run( + prefix + ["env", f"PATH={tools}:{os.environ.get('PATH', '')}", + f"PYTHONPATH={REPO}", f"MESH_NETEM={self.netem}", + sys.executable, DRIVER], + capture_output=True, text=True, check=False, timeout=600) + for line in done.stdout.splitlines(): + if line.startswith("RESULT "): + return json.loads(line[len("RESULT "):]) + self.fail(f"no result from {DRIVER}:\n{done.stdout[-3000:]}\n" + f"{done.stderr[-3000:]}") + + def test_b_joins_through_the_listener(self): + b = self.found["b"] + self.assertEqual(b["code"], exits.OK, b) + self.assertTrue(b["ping"], b) + self.assertEqual(b["outcome"], "confirmed") + self.assertEqual(self.found["a_after_b"], "confirmed") + self.assertEqual(self.found["serve"], [exits.OK]) + self.assertTrue(b["out"][-1].startswith("the mesh is up with 2")) + self.assertIn("the overlay was tested", "\n".join(b["err"])) + + def test_c_joins_through_the_fallback(self): + c = self.found["c"] + self.assertEqual(self.found["accept"], exits.OK) + self.assertEqual(c["code"], exits.OK, c) + self.assertTrue(c["out"][0].startswith("keel mesh accept keel1a:")) + self.assertTrue(c["ping"], c) + self.assertEqual(c["outcome"], "confirmed") + self.assertEqual(self.found["a_after_c"], "confirmed") + # B through the listener, C through accept, with a keepalive + self.assertEqual(self.found["peers"], [None, 25]) + self.assertEqual(self.found["invites_left"], []) + + def test_a_spent_invite_is_refused(self): + code, said = self.found["again"] + self.assertEqual(code, exits.MESH_REFUSED) + self.assertIn("no pending invite", said[0]) + + def test_the_listener_held_no_capability(self): + self.assertEqual(self.found["capabilities"], + ["0000000000000000"] * 2) + self.assertEqual(self.found["netem"], self.netem.split()) + + def test_no_secret_in_the_inviter_s_journal(self): + journal = "\n".join(self.found["journal"]) + self.assertIn("joined and confirmed", journal) + for token in self.found["tokens"]: + for form in secret_forms(token): + self.assertNotIn(form, journal) + + +class TestThreeNamespacesOnAPoorNetwork(TestThreeNamespaces): + netem = POOR + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_mesh_protocol.py b/tests/test_mesh_protocol.py new file mode 100644 index 0000000..3aa0141 --- /dev/null +++ b/tests/test_mesh_protocol.py @@ -0,0 +1,155 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The join protocol's messages and their HMAC (decision 0048) + +Pure: each message to JSON and back, every malformed field refused with +its reason, the HMAC over the method, the path and the body checked in +constant time, and the request's time against the clock. +""" + +import json +import unittest +from datetime import datetime, timedelta, timezone + +from keel.mesh import protocol +from keel.mesh.protocol import ( + ConfirmAnswer, + ConfirmRequest, + JoinAnswer, + JoinRequest, + Peer, + ProtocolError, +) + +KEY = bytes(range(32)) +JOINER = "FHKH10gOWeK2bXHZPg8y+oPTprv556bwrmmRkbyEPgg=" +INVITER = "nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E=" +NOW = datetime(2026, 10, 3, 12, 0, tzinfo=timezone.utc) +NONCE = "00112233445566778899aabbccddeeff" +REQUEST = JoinRequest( + invite_id="0123456789abcdef", public_key=JOINER, + endpoint="[2001:db8:2::20]:51820", address="fd00:6b65:1::3/64", + nonce=NONCE, time=int(NOW.timestamp())) +ANSWER = JoinAnswer( + invite_id="0123456789abcdef", nonce=NONCE, public_key=INVITER, + address="fd00:6b65:1::1/64", + peers=(Peer(public_key=JOINER, endpoint=None, + address="fd00:6b65:1::7"),), + etcd="none", window=120) + + +def body(message, **changed): + data = json.loads(protocol.dumps(message)) + data.update(changed) + return json.dumps(data).encode() + + +class TestSignature(unittest.TestCase): + def test_the_hmac_covers_the_method_the_path_and_the_body(self): + found = protocol.sign(KEY, "POST", protocol.JOIN, b"{}") + self.assertEqual(len(found), 64) + self.assertTrue(protocol.signed(KEY, "POST", protocol.JOIN, b"{}", + found)) + for method, path, data, key in ( + ("PUT", protocol.JOIN, b"{}", KEY), + ("POST", protocol.CONFIRM, b"{}", KEY), + ("POST", protocol.JOIN, b"{ }", KEY), + ("POST", protocol.JOIN, b"{}", bytes(32))): + with self.subTest(method=method, path=path, data=data): + self.assertFalse(protocol.signed(key, method, path, data, + found)) + + def test_a_missing_or_malformed_signature_is_false(self): + for found in (None, "", "zz" * 32, "00" * 31, "é" * 64): + with self.subTest(found=found): + self.assertFalse(protocol.signed(KEY, "POST", protocol.JOIN, + b"{}", found)) + + def test_the_answer_is_signed_apart_from_the_request(self): + request = protocol.sign(KEY, "POST", protocol.JOIN, b"{}") + answer = protocol.sign(KEY, protocol.ANSWER, protocol.JOIN, b"{}") + self.assertNotEqual(request, answer) + + +class TestMessages(unittest.TestCase): + def test_round_trips(self): + confirm = ConfirmRequest(invite_id="0123456789abcdef", + public_key=JOINER, nonce=NONCE, + time=int(NOW.timestamp())) + confirmed = ConfirmAnswer(invite_id="0123456789abcdef", nonce=NONCE, + confirmed=True, detail="confirmed") + for message, load in ( + (REQUEST, protocol.join_request), + (REQUEST.__class__(**{**REQUEST.__dict__, "endpoint": None}), + protocol.join_request), + (ANSWER, protocol.join_answer), + (confirm, protocol.confirm_request), + (confirmed, protocol.confirm_answer)): + with self.subTest(message=type(message).__name__): + self.assertEqual(load(protocol.dumps(message)), message) + + def test_an_ipv4_endpoint(self): + found = protocol.join_request(body(REQUEST, + endpoint="198.51.100.7:51820")) + self.assertEqual(found.endpoint, "198.51.100.7:51820") + + def test_a_malformed_join_request_is_refused_with_its_reason(self): + for changed, words in ( + ({"invite_id": "0123"}, "invite_id"), + ({"public_key": "not a key"}, "public_key"), + ({"public_key": 7}, "public_key"), + ({"endpoint": "[::1]:51820"}, "cannot be reached"), + ({"endpoint": "[fe80::1]:51820"}, "cannot be reached"), + ({"endpoint": "host.example:51820"}, "endpoint"), + ({"endpoint": "[2001:db8::1]"}, "endpoint"), + ({"address": "192.0.2.1/24"}, "address"), + ({"address": "fd00::3"}, "address"), + ({"nonce": "00"}, "nonce"), + ({"time": "now"}, "time"), + ({"time": True}, "time")): + with self.subTest(changed=changed): + with self.assertRaises(ProtocolError) as caught: + protocol.join_request(body(REQUEST, **changed)) + self.assertIn(words, str(caught.exception)) + + def test_not_a_message(self): + for data in (b"", b"[]", b"\xff", b'{"invite_id": "x"}', + b"{" * 2000): + with self.subTest(data=data[:10]): + with self.assertRaises(ProtocolError): + protocol.join_request(data) + + def test_a_malformed_answer_is_refused(self): + for changed in ({"peers": "none"}, {"peers": [1]}, + {"peers": [{"public_key": 1}]}, + {"peers": [{"public_key": JOINER, "address": "x", + "endpoint": None}]}, + {"etcd": "maybe"}, {"window": -1}, + {"window": "120"}, {"address": "fd00::1"}): + with self.subTest(changed=changed): + with self.assertRaises(ProtocolError): + protocol.join_answer(body(ANSWER, **changed)) + with self.assertRaises(ProtocolError): + protocol.confirm_answer(b'{"invite_id": "0123456789abcdef",' + b' "nonce": "' + NONCE.encode() + + b'", "confirmed": 1, "detail": ""}') + + +class TestFreshness(unittest.TestCase): + def test_within_five_minutes_either_way(self): + for offset, fresh in ((0, True), (299, True), (-299, True), + (301, False), (-301, False)): + with self.subTest(offset=offset): + self.assertEqual(protocol.fresh( + int(NOW.timestamp()) + offset, NOW), fresh) + + def test_a_nonce_is_new_each_time(self): + self.assertNotEqual(protocol.new_nonce(), protocol.new_nonce()) + self.assertEqual(len(protocol.new_nonce()), 32) + + def test_now_in_seconds(self): + self.assertEqual(protocol.seconds(NOW + timedelta(seconds=0.7)), + int(NOW.timestamp())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_network_mesh_confirm.py b/tests/test_network_mesh_confirm.py new file mode 100644 index 0000000..5c98d29 --- /dev/null +++ b/tests/test_network_mesh_confirm.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The two sources decision 0048 adds to 0018's confirmation + +A join's own authenticated session over the overlay, from the peer the +change added, and `keel mesh create` confirming a mesh with no peer +itself. Both confirm only the overlay change keel mesh made (the marker +it read back after its apply), only after the route check, and never an +uplink change. +""" + +import unittest + +from helpers import spec # noqa: F401 + +# keel.inspect before keel.network.confirm, the order keel.cli imports +# them in: alone, this file would meet their import cycle the other way +import keel.inspect # noqa: F401, I001 +from test_network_overlay_window import ( + ENABLE, + STOP, + RootCase, + pending, + probes, +) +from test_network_window import Recorder + +from keel.network import confirm as netconfirm +from keel.network import marker, session + +MADE = pending(absent=True).up("b1", 50.0) + + +def mesh(local="fd00:1::9", peer="fd00:1::2"): + return session.Origin(session.MESH, "the mesh session of invite" + " 0123456789abcdef", 60.0, local, peer) + + +def own(): + return session.Origin(session.SELF, "keel mesh create") + + +class TestMeshOrigins(RootCase): + def setUp(self): + super().setUp() + marker.save(self.root, "") + marker.write(self.root, MADE) + + def confirm(self, origin, expected=MADE, routes=None, clients=()): + run = Recorder() + found = netconfirm.confirm(self.root, origin, probes(routes=routes), + run, expected=expected, clients=clients) + return found, run + + def test_the_join_session_over_the_overlay_confirms(self): + (confirmed, lines), run = self.confirm(mesh()) + self.assertTrue(confirmed, lines) + self.assertEqual(lines[0], "confirmed from the mesh session of" + " invite 0123456789abcdef") + self.assertEqual(lines[1], "the overlay was tested: the mesh session" + " of invite 0123456789abcdef arrived at fd00:1::9" + " on wg0, from fd00:1::2") + self.assertEqual(run.calls, [STOP, ENABLE]) + self.assertFalse(marker.exists(self.root)) + + def test_create_confirms_a_mesh_with_no_peer_itself(self): + (confirmed, lines), _ = self.confirm(own(), clients=("2001:db8:9::5",)) + self.assertTrue(confirmed, lines) + self.assertIn("no peer yet", lines[1]) + self.assertIn("still leave through the uplink", lines[1]) + + def test_only_the_change_keel_mesh_made(self): + for expected in (None, pending(absent=True).up("b1", 49.0), + pending(absent=True).up("b0", 50.0)): + for origin in (mesh(), own()): + with self.subTest(expected=expected, origin=origin.kind): + (confirmed, lines), run = self.confirm(origin, expected) + self.assertFalse(confirmed) + self.assertIn("not the one keel mesh made", lines[0]) + self.assertEqual(run.calls, []) + self.assertTrue(marker.exists(self.root)) + + def test_a_mesh_session_at_another_address_is_refused(self): + (confirmed, lines), _ = self.confirm(mesh(local="2001:db8:1::20")) + self.assertFalse(confirmed) + self.assertIn("arrived at 2001:db8:1::20, not at an address the" + " overlay declares", lines[0]) + + def test_the_route_check_runs_first(self): + routes = {"2001:db8:9::5": "wg0"} + (confirmed, lines), _ = self.confirm(own(), routes=routes, + clients=("2001:db8:9::5",)) + self.assertFalse(confirmed) + self.assertIn("the route to the operator's SSH client 2001:db8:9::5" + " leaves through wg0", lines[0]) + + +class TestNeverAnUplinkChange(RootCase): + def test_a_mesh_origin_cannot_confirm_the_uplink(self): + uplink = marker.Pending(iface="eth0", path="etc/network/interfaces", + window=120, addresses=("2001:db8:1::20",)) + marker.save(self.root, "") + marker.write(self.root, uplink.up("b1", 50.0)) + made = marker.read(self.root) + for origin in (mesh(local="2001:db8:1::20"), own()): + with self.subTest(origin=origin.kind): + confirmed, lines = netconfirm.confirm( + self.root, origin, probes(), Recorder(), expected=made) + self.assertFalse(confirmed) + self.assertIn("not the one keel mesh made", lines[0]) + + def test_without_the_check_the_uplink_refuses_them_too(self): + uplink = marker.Pending(iface="eth0", path="etc/network/interfaces", + window=120).up("b1", 50.0) + for origin in (mesh(), own()): + with self.subTest(origin=origin.kind): + self.assertIn("refused", netconfirm.not_proof( + uplink, origin, probes())) + + +class TestLines(unittest.TestCase): + def test_overlay_lines_of_each_origin(self): + self.assertEqual(netconfirm.overlay_lines( + pending(), own(), probes()), [ + "the overlay has no peer yet, so nothing can cross it:" + " keel mesh create confirmed it once the routes to the" + " gateways and to the operator's session were found to" + " still leave through the uplink"]) + + +if __name__ == "__main__": + unittest.main()