From a9da8dda986edadf0afa08fb51182914cb5370dc Mon Sep 17 00:00:00 2001 From: navigator Date: Sat, 3 Oct 2026 13:30:23 +0000 Subject: [PATCH 1/2] feat: compare WireGuard keys by their 32 bytes, refuse spellings wg refuses Spec validation compared peer keys as text, for this node's own key and for a key declared twice. Both now go through one helper, wireguard.key_bytes and same_key. A key whose last base64 character sets the two spare bits decodes to the same bytes in Python but is refused by wg pubkey; key_bytes refuses that spelling too, so is_key no longer takes a key wg-quick would reject. --- keel/network/wireguard.py | 27 ++++++++++++++++++++++++--- keel/spec/validate_overlay.py | 25 +++++++++++++++++++------ tests/test_spec_overlay.py | 8 ++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/keel/network/wireguard.py b/keel/network/wireguard.py index 67a7a76..8b45059 100644 --- a/keel/network/wireguard.py +++ b/keel/network/wireguard.py @@ -83,12 +83,33 @@ def addresses(overlay: dict) -> tuple[str, ...]: def is_key(text: str) -> bool: """A WireGuard key as wg prints it: 32 bytes in base64, 44 characters""" + return key_bytes(text) is not None + + +def key_bytes(text: str) -> bytes | None: + """The 32 bytes of a key written as wg prints it, or None + + The last character of the base64 carries two bits no byte uses. + Python's decoder ignores them; wg refuses a key that sets them ("Key + is not the correct length or format"), so such a spelling is no key + here either. Keys are compared by these bytes (same_key), never as + text. + """ if len(text) != KEY_LENGTH: - return False + return None try: - return len(base64.b64decode(text, validate=True)) == KEY_BYTES + found = base64.b64decode(text, validate=True) except (binascii.Error, ValueError): - return False + return None + if len(found) != KEY_BYTES or base64.b64encode(found).decode() != text: + return None + return found + + +def same_key(one: str, other: str) -> bool: + """Whether two spellings are the same WireGuard key""" + found = key_bytes(one) + return found is not None and found == key_bytes(other) def split_endpoint(text: str) -> tuple[str, int]: diff --git a/keel/spec/validate_overlay.py b/keel/spec/validate_overlay.py index ea01a39..6a63c7f 100644 --- a/keel/spec/validate_overlay.py +++ b/keel/spec/validate_overlay.py @@ -234,11 +234,23 @@ def peers_errors(key: str, peers: Any, reserved: Reserved) -> list[str]: errors: list[str] = [] for index, peer in enumerate(peers or []): errors += peer_errors(f"{key}[{index}]", peer, reserved) - keys = [str(peer.get("public_key")) for peer in peers or [] - if isinstance(peer, dict) and peer.get("public_key")] - errors += [f"{key}: public key {one} is declared twice" - for one in sorted(set(keys)) if keys.count(one) > 1] - return errors + shared_ips(key, peers or []) + return errors + twice(key, peers or []) + shared_ips(key, peers or []) + + +def twice(key: str, peers: list) -> list[str]: + """A key two peers declare, compared by its bytes (same_key)""" + seen: list[bytes] = [] + errors = [] + for peer in peers: + found = wireguard.key_bytes(str(peer.get("public_key"))) \ + if isinstance(peer, dict) else None + if found is None: + continue + if seen.count(found) == 1: + errors.append(f"{key}: public key {peer['public_key']} is" + " declared twice") + seen.append(found) + return errors def own_key_errors(key: str, wg: dict) -> list[str]: @@ -265,7 +277,8 @@ def own_key_errors(key: str, wg: dict) -> list[str]: return [f"{key}[{index}].public_key: this is this node's own public key" f" (of {path}); a node is not its own peer, list the other" " nodes' keys" for index, peer in enumerate(peers) - if isinstance(peer, dict) and peer.get("public_key") == own] + if isinstance(peer, dict) + and wireguard.same_key(str(peer.get("public_key")), own)] def peer_errors(key: str, peer: Any, reserved: Reserved) -> list[str]: diff --git a/tests/test_spec_overlay.py b/tests/test_spec_overlay.py index 250fd1b..8b232fd 100644 --- a/tests/test_spec_overlay.py +++ b/tests/test_spec_overlay.py @@ -152,6 +152,14 @@ def test_peers(self): one(self, document({"peers": [peer(public_key="abc")]}), "not a WireGuard key") + def test_a_key_spelled_as_wg_never_writes_it(self): + # the last character's two spare bits set: Python's base64 decodes + # it to PEER_KEY's bytes, wg pubkey refuses it ("Key is not the + # correct length or format"), so it is no key here either + spelled = PEER_KEY[:-2] + "F=" + one(self, document({"peers": [peer(public_key=spelled)]}), + "not a WireGuard key") + def test_endpoint(self): for endpoint, words in ( ("2001:db8::20:51820", "brackets"), From 16a26f8dbf38ce77e659b1e95237dd61c5ffe4f6 Mon Sep 17 00:00:00 2001 From: navigator Date: Sat, 3 Oct 2026 13:30:23 +0000 Subject: [PATCH 2/2] feat: keel mesh invite and join --dry-run, the keel1: token (0048) First part of handbook decision 0048, joining the mesh with one command. keel mesh invite prints keel mesh join keel1:, valid one hour and once, and reserves the new node's overlay address, a random free one of the prefix (secrets), so two members inviting at once before etcd exists almost never collide, in a pending invite under /var/lib/keel/mesh (0700 directories, 0600 files, atomic writes, one lock). The invite keeps an HMAC key derived from the token's secret, never the secret, and the invite's own certificate, whose SHA-256 the token carries for pinning. Consuming an invite marks it used and keeps its address reserved until it expires. keel mesh join TOKEN|- --dry-run checks the token (length, prefix, checksum, layout, values, expiry) and prints the address and peer entry it makes in the spec, validated as the spec would be after the join; it applies nothing. The token already carries the HTTPS port, the mesh identity and the etcd state, so the listener, the join, accept, create and remove need no format change. Removing expired invites at boot comes with the listener. New exit codes 23 and 24. docs/mesh.md. keel 0.16.0. --- README.md | 6 +- debian/changelog | 21 ++ docs/mesh.md | 220 ++++++++++++++++++ docs/spec.md | 9 + keel/__init__.py | 2 +- keel/cli.py | 62 +++++- keel/exits.py | 6 + keel/mesh/__init__.py | 29 +++ keel/mesh/allocate.py | 88 ++++++++ keel/mesh/certificate.py | 59 +++++ keel/mesh/commands.py | 231 +++++++++++++++++++ keel/mesh/identity.py | 51 +++++ keel/mesh/invites.py | 222 +++++++++++++++++++ keel/mesh/join.py | 67 ++++++ keel/mesh/token.py | 329 +++++++++++++++++++++++++++ pyproject.toml | 5 +- tests/test_mesh_allocate.py | 96 ++++++++ tests/test_mesh_cli.py | 431 ++++++++++++++++++++++++++++++++++++ tests/test_mesh_invites.py | 292 ++++++++++++++++++++++++ tests/test_mesh_token.py | 277 +++++++++++++++++++++++ 20 files changed, 2498 insertions(+), 5 deletions(-) create mode 100644 docs/mesh.md create mode 100644 keel/mesh/__init__.py create mode 100644 keel/mesh/allocate.py create mode 100644 keel/mesh/certificate.py create mode 100644 keel/mesh/commands.py create mode 100644 keel/mesh/identity.py create mode 100644 keel/mesh/invites.py create mode 100644 keel/mesh/join.py create mode 100644 keel/mesh/token.py create mode 100644 tests/test_mesh_allocate.py create mode 100644 tests/test_mesh_cli.py create mode 100644 tests/test_mesh_invites.py create mode 100644 tests/test_mesh_token.py diff --git a/README.md b/README.md index 2705a96..78938b6 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ in [docs/apply.md](docs/apply.md). | `keel network revert` | Put back the interfaces file a pending network change replaced; what the revert timer runs. `--boot` restores the file only, for the boot unit | | `keel network wireguard key` | Print this node's WireGuard public key, making the key pair first when there is none (root, live system only); the private key is never printed (decision 0020, docs/spec.md) | | `keel network wireguard suggest-address` | Print a random unique local IPv6 address with its /64, for the first node of an overlay | +| `keel mesh invite` | Print the `keel mesh join keel1:` line a new node runs to join this node's WireGuard mesh, valid one hour and once, and reserve the new node's overlay address (decision 0048, [docs/mesh.md](docs/mesh.md)) | +| `keel mesh join` | With `--dry-run`, check a `keel1:` token and print the address and the peer it makes in this node's spec, applying nothing; the join itself comes later ([docs/mesh.md](docs/mesh.md)) | | `keel notify` | What monit runs when a check of the `monitor` section fails, lasts or recovers: sends one message, saying what happened and what to do, to every channel the section declares (email, Telegram, ntfy, a webhook). It reads the channels from `/etc/keel/monitor.json`, which `apply --system` writes as root, never from the spec, reads the tokens from their secret files and prints no URL (decision 0021, docs/apply.md) | | `keel inspect` | Write a spec from the running machine, or from an offline root, and report every field with its source or why it was not inferred; secrets are never read (brief sections 5.2 and 7) | | `keel diff` | Report drift between the spec and the running machine, or an offline root, field by field, through the same collector `inspect` uses; secrets are never compared and their files need not exist, nothing is written (brief section 5.2) | @@ -138,7 +140,7 @@ Defined in one place, `keel/exits.py`, and reproduced here. | 12 | `ASSEMBLE_FAILED` | The rootfs is not empty or cannot be created, or `tar` or `zstd` failed while extracting or packing | | 13 | `INSPECT_INCOMPLETE` | `inspect` wrote the spec, but a required field (hostname, fqdn, interfaces, alerts, updates) could not be inferred; or `diff` found no drift, but a declared field could not be observed. The report says which and why | | 14 | `DRIFT_FOUND` | `diff` found at least one declared field whose observed value differs. Drift wins over unobserved fields, so a report with both exits 14 | -| 15 | `APPLY_NEEDS_ROOT` | The system phase (`--system`, `--system-only`) on the live system was run by a user other than root; nothing was written, not even the conf | +| 15 | `APPLY_NEEDS_ROOT` | The system phase (`--system`, `--system-only`) on the live system was run by a user other than root; nothing was written, not even the conf. Also `keel network wireguard key` making a key, and `keel mesh invite`, on the live system | | 16 | `APPLY_FAILED` | The system phase could not make at least one change; the output names it. The conf was written, where the run writes one, and every other change was made, so the run can be repeated | | 17 | `CHANNEL_INVALID` | A channel pointer, or the record of the one this instance follows, does not parse or fails validation. Also a pointer signed in the future, one claiming more than 30 days of life, and a revision's archived pointer naming another revision | | 18 | `CHANNEL_UNVERIFIED` | A channel pointer is not signed by a key that may move a channel, or no keyring was given to check it against. A key that is revoked or expired is refused here: gpgv exits 0 for both and still prints `VALIDSIG`, so `GOODSIG` is what is required | @@ -146,6 +148,8 @@ Defined in one place, `keel/exits.py`, and reproduced here. | 20 | `CHANNEL_ROLLBACK` | A channel pointer names an earlier revision than the one this instance is on; `--allow-rollback`, or naming the release and revision, is how going back is asked for | | 21 | `NETWORK_NOT_CONFIRMED` | `keel network confirm` refused: no change is waiting, or it was not run from a new session over the new configuration, or a console (docs/apply.md) | | 22 | `NOTIFY_FAILED` | `keel notify` reached no channel: `/etc/keel/monitor.json` is missing or refused, declares none, or every one failed; the message then went to syslog (user.crit) and root's mailbox. One channel that takes the message is success; the others' failures are printed | +| 23 | `MESH_TOKEN_INVALID` | `keel mesh join` could not use the token: not a `keel1:` token, mistyped or truncated (its checksum), of a later format, inconsistent, or expired. The message never holds the token or its secret | +| 24 | `MESH_REFUSED` | `keel mesh` refused: no overlay to invite into, no key yet, no endpoint, no free address in the prefix; or this node is in another mesh, at another address of it, or the change would make a spec validation refuses | Two rules that callers depend on: diff --git a/debian/changelog b/debian/changelog index 98e9ca7..12514ea 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,24 @@ +keel (0.16.0) unstable; urgency=medium + + * keel mesh invite and keel mesh join --dry-run, the first part of + handbook decision 0048 (joining the mesh with one command). invite + prints the keel mesh join keel1: line, valid one hour and + once, and reserves the new node's overlay address, a random free + one of the prefix, in a pending invite under /var/lib/keel/mesh + (0700, 0600) that keeps an HMAC key derived from the token's + secret, never the secret; a consumed invite keeps its address + until it expires. join + --dry-run checks the token (prefix, checksum, layout, expiry) and + prints the address and the peer it makes in the spec; it applies + nothing yet. The listener, the join itself, accept, create and + remove come next. docs/mesh.md. New exit codes 23 and 24. + * WireGuard keys are compared by their 32 bytes, through one helper, + in the peer checks of spec validation; and a key spelled with the + two spare bits of its last character set, which wg refuses, is no + longer taken for a key. + + -- Marcos Mendez Sat, 03 Oct 2026 20:00:00 +0000 + keel (0.15.4) unstable; urgency=medium * spec validation refuses a peer of network.overlay.wireguard that holds diff --git a/docs/mesh.md b/docs/mesh.md new file mode 100644 index 0000000..bf30bbc --- /dev/null +++ b/docs/mesh.md @@ -0,0 +1,220 @@ +# keel mesh + +Joining a node to the WireGuard mesh takes one command (handbook decision +0048). On a node already in the mesh, `keel mesh invite` prints one line: + +``` +keel mesh join keel1: +``` + +and that line, run on the new node, joins it. The token carries every +value the new node would otherwise copy by hand from +[docs/spec.md](spec.md), "overlay": the inviter's public key, its +endpoint and port, the overlay prefix and the address reserved for the +new node. It is valid for one hour and can be used once. + +What a join writes is the same `network.overlay.wireguard` fields an +operator writes by hand, `address` and a peer, converged by the same +`apply --system` under the same confirmation window. There is no `mesh` +section in the spec, and a mesh built by joins and one typed by hand are +the same spec. + +**What exists in this version:** the token, the address an invite +reserves, the pending invites, `keel mesh invite`, and `keel mesh join +--dry-run`, which prints the change a token makes and applies nothing. +The listener that answers a join, the join itself, the fallback command +`keel mesh accept`, `keel mesh create` for a node with no overlay, and +`keel mesh remove` come next; the token and the invite file already +carry what they need. + +## keel mesh invite + +``` +keel mesh invite [--endpoint ADDRESS]... [--port PORT] +``` + +Root, on a node whose spec declares `network.overlay.wireguard` and whose +key file exists (apply makes it the first time it converges the overlay). +It: + +1. reads this node's public key from its private key file with `wg + pubkey`, as `keel network wireguard key` does; the key is not made + here; +2. takes the endpoints the new node will reach this one at: the + `--endpoint` addresses (one per family, for a port forward say), or + else the static addresses `network.interfaces` declares, IPv6 first; +3. makes a key pair and a self signed certificate for this invite only + (`openssl req -x509`, P-256), whose SHA-256 fingerprint the token + carries for the new node to pin; +4. makes the mesh's identity if this node has none (below); +5. under the mesh's lock, removes the expired invites, reserves a free + address (below) and writes the pending invite; +6. prints the `keel mesh join` line, alone, on standard output; what it + reserved and until when go to standard error, never the token. + +`--port` is the TCP port the join request will go to, 51820 by default, +the number of WireGuard's UDP port, so a provider's firewall needs one +number for both. + +No listener answers the invite yet in this version, so the line can be +checked on the new node with `keel mesh join --dry-run` and not used to +join. + +### The address + +A random address of the inviter's overlay prefix, drawn with Python's +`secrets`, that is not its own, not inside any peer's `allowed_ips`, and +not reserved by another pending invite. Random rather than the next +free one: until etcd exists, a member does not know the invites another +member has pending, and two members inviting at the same time would both +hand out `::2`; two draws in a `/64` practically never meet. A draw that +lands on a taken address is drawn again; after 16 taken draws in a row +the prefix is nearly full and is searched from the last draw, so its +last free address is still found and a full prefix is refused. The +prefix's own address (`::`, the subnet router anycast) is never given. + +## keel mesh join --dry-run + +``` +keel mesh join TOKEN --dry-run +keel mesh join - --dry-run # the token on standard input +``` + +Reads the token, checks it whole (below), and prints on standard output +the fields of `network.overlay.wireguard` the join would set in this +node's spec: + +```yaml +network: + overlay: + wireguard: + address: fd00:6b65:1::3/64 + peers: + - public_key: nb9/izIukqWXM7gnBpe7hki4jKZZChWOW1wEfONn82E= + endpoint: '[2001:db8:1::10]:51820' + allowed_ips: + - fd00:6b65:1::1/128 +``` + +The inviter is the peer: its key, its endpoint (IPv6 when it has one), +and its overlay address as a `/128`. The spec is read and nothing is +written or applied. It refuses a node whose overlay is on another prefix +(another mesh) or at another address of this one (a node joins once), +and a change `spec validate` would refuse once made, such as an overlay +prefix that overlaps this node's uplink, naming the error. Without +`--dry-run`, `join` applies nothing in this version and exits 9. + +The overlay's optional `ipv4_address` is not carried: a join gives the +new node an IPv6 overlay address and routes the inviter's IPv6 address +only. A mesh that also uses IPv4 on the overlay adds those addresses by +hand, as before. + +`-` reads the token from standard input, which keeps it out of the +process list and the shell's history; confconsole will use it. + +## The token + +`keel1:` followed by the base64url encoding, without padding, of the +fields below in this order, big endian, and a checksum: the first four +bytes of the SHA-256 of the prefix and the fields. A typical invite, +with one IPv6 endpoint, is 250 characters; with an IPv4 endpoint as +well, 256. + +| Field | Bytes | Why the new node needs it | +| --- | --- | --- | +| flags | 1 | which endpoints follow: bit 0 IPv6, bit 1 IPv4; no other bit is set | +| inviter's public key | 32 | its peer entry for the inviter | +| IPv6 endpoint | 16 | where to send the handshake, when the inviter has one | +| IPv4 endpoint | 4 | the same, for a new node without IPv6 | +| WireGuard port | 2 | the UDP port at the endpoint | +| HTTPS port | 2 | the TCP port of the join request, at the same address | +| certificate fingerprint | 32 | SHA-256 of the invite's certificate, to pin it | +| inviter's overlay address | 16 | its peer's `allowed_ips`, as a `/128` | +| prefix length | 1 | of the overlay prefix, which the inviter's address is on | +| assigned address | 16 | its own `network.overlay.wireguard.address` | +| mesh identity | 16 | names the mesh; etcd's cluster token | +| etcd state | 1 | 0 none, 1 forms at this join, 2 running | +| etcd port | 2 | only when running: the inviter's member port | +| invite id | 8 | which pending invite this is; derived from the secret | +| secret | 32 | 256 random bits, used once; the HMAC key is derived from it | +| expiry | 4 | seconds since the epoch, UTC | +| checksum | 4 | a truncated or mistyped paste is refused before anything is sent | + +`join` refuses, each with its reason and exit 23: + +- more than 1024 characters, before anything is decoded (a token is + about 256); +- text that does not start with `keel1:` (the whole `keel mesh join` + line pasted, say), and a `keel2:` or later token, naming the keel it + needs; +- a character outside base64url, and any token whose checksum does not + match: one character changed, or the end cut off; +- a layout that does not match its flags, unknown flags or an unknown + etcd state; +- a value no join could use: a key that is not 32 bytes, an endpoint + that cannot be reached from another machine (loopback, link local, + multicast), a port 0, an overlay prefix outside `fc00::/7`, an + assigned address outside the prefix or equal to the inviter's, an + invite id that is not the one its secret gives; +- an expired token, with the time it expired. + +The secret is the only value in the token that is not public, and it is +in no message: no error names the token or its secret, and the `repr` +of a parsed token leaves it out. `invite` writes the token to its +standard output and nowhere else. + +## Where the state lives + +Under `/var/lib/keel/mesh`, root's, directories mode 0700 and files 0600, +written through a temporary file and a rename as `/var/lib/keel/network` +is, every change under the lock `/var/lib/keel/mesh/lock`. State, not +configuration: never in the spec, never emitted by `inspect`, never in a +backup set. + +| Path | What it holds | +| --- | --- | +| `invites/.json` | one pending invite: the reserved address, the expiry, the HTTPS port, the invite's certificate and TLS key, and the HMAC key | +| `identity` | the mesh's identity, 16 bytes in hex | + +The invite file holds an HMAC key derived from the secret, never the +secret, so it cannot be turned back into the token. An invite is +consumed once, under the lock: it is marked consumed, and a second +request is told it was already used. A consumed invite keeps its +address reserved until it expires, so no other invite takes the address +before the new node is a peer in the spec; an expired or damaged one is +removed by the next invite. + +**Pending:** an invite whose expiry passes while the machine is down is +removed by the next invite, not yet at boot; the boot unit 0048 +describes comes with the listener. + +The mesh identity is made once and kept: a file that does not hold one +is an error, never replaced, since every other node of the mesh knows +the first. A mesh built by hand before `keel mesh` existed gets it from +its first invite. + +## Exit codes + +| Code | When | +| --- | --- | +| 0 | the line or the change was printed | +| 2, 3 | the spec cannot be read or is invalid | +| 9 | `join` without `--dry-run`: not implemented yet | +| 15 | `invite` on the live system, not as root | +| 16 | `invite`: `wg` cannot read the key file, or `openssl` cannot make the certificate, or the mesh identity is damaged | +| 23 | `join`: the token is mistyped, truncated, of a later format, inconsistent, or expired | +| 24 | refused: no overlay to invite into, no key yet, no endpoint, no free address; a node in another mesh or at another address of it, or a change the spec would refuse | + +## Tests + +At four seams, each written down before its tests: the token's +`encode()` and `parse()` (`tests/test_mesh_token.py`, round trips and +every damaged, foreign, inconsistent and expired form, and the secret in +no message and no repr); the allocator's `free_address()` +(`tests/test_mesh_allocate.py`); the store's `reserve`, `find`, +`consume`, `pending` and `expire` against a scratch root +(`tests/test_mesh_invites.py`, modes, one use, expiry, no secret on +disk); and the commands through `keel.cli.main` +(`tests/test_mesh_cli.py`), with the real `wg` (tests/wgtools.py) and +the real `openssl`, so the key and the fingerprint in the token are +checked against what those tools say. diff --git a/docs/spec.md b/docs/spec.md index 5e8f8cf..9f5a9dc 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -300,6 +300,15 @@ network: | `...peers[].allowed_ips` | system | Required, at least one prefix: what is routed to this peer and accepted from it. For a node, its overlay address as a `/128` (and `/32`). A prefix with host bits set is an error, and so is a prefix given to two peers, since wg would silently keep it for the last one only. Each prefix must be inside the private ranges the overlay addresses use (`fc00::/7`; RFC 1918 or `100.64.0.0/10`): a `/0`, a public prefix (a `::/1` and `8000::/1` pair takes as much as `::/0`) and a prefix that overlaps the uplink or a peer's endpoint are refused (see "Routes" below) | | `...peers[].persistent_keepalive` | system | Optional, seconds between 1 and 65535: keeps a path through a NAT or a stateful firewall open. Leave it out for none | +**Joining with one command.** The peers and the address need not be +typed: `keel mesh invite`, on a node already in the mesh, prints a `keel +mesh join keel1:` line that carries the inviter's key, endpoint +and port and an address reserved for the new node, and `keel mesh join +--dry-run` on the new node prints the `address` and the peer entry it +makes of them, exactly the fields above (handbook decision 0048, +[docs/mesh.md](mesh.md)). Nothing in the spec records how a peer was +added. + **Routes.** `wg-quick` adds a route for the overlay's own prefixes and for each peer's `allowed_ips`. A route that covers the uplink would send the uplink's replies into the overlay, and the machine would be cut off diff --git a/keel/__init__.py b/keel/__init__.py index 46486c4..14fcb39 100644 --- a/keel/__init__.py +++ b/keel/__init__.py @@ -6,4 +6,4 @@ clients of it. """ -__version__ = "0.15.4" +__version__ = "0.16.0" diff --git a/keel/cli.py b/keel/cli.py index b8cc099..fc47fdc 100644 --- a/keel/cli.py +++ b/keel/cli.py @@ -28,11 +28,13 @@ STATE_ENV, ) from keel.manifest import KINDS as MANIFEST_KINDS +from keel.mesh import commands as mesh_commands from keel.monitor.channelfile import PATH as NOTIFY_SETTINGS from keel.monitor.notify import CHECKS as NOTIFY_CHECKS from keel.monitor.notify import DIRECTIONS as NOTIFY_DIRECTIONS from keel.monitor.notify import LEVELS as NOTIFY_LEVELS from keel.spec import CONF_DEFAULT, CONF_ENV, SPEC_DEFAULT, SPEC_ENV +from keel.network.wireguard import DEFAULT_PORT as WIREGUARD_PORT from keel.system import DEFAULT_WINDOW @@ -186,6 +188,7 @@ def build_parser() -> argparse.ArgumentParser: add_root_option(revert_parser, "revert the change of") revert_parser.set_defaults(handler=commands.network_revert) add_wireguard_parser(network_actions) + add_mesh_parser(subparsers) # no --spec: notify reads the settings apply wrote, never the spec notify_parser = subparsers.add_parser( @@ -320,6 +323,63 @@ def add_wireguard_parser(network_actions) -> None: suggest_parser.set_defaults(handler=commands.network_wireguard_suggest) +def add_mesh_parser(subparsers) -> None: + """keel mesh: joining the WireGuard mesh with one command (0048)""" + mesh_parser = subparsers.add_parser( + "mesh", + help="invite a node into this node's WireGuard mesh, or join one" + " with the line an invite printed (decision 0048, docs/mesh.md)", + ) + mesh_actions = mesh_parser.add_subparsers( + dest="action", metavar="ACTION" + ) + invite_parser = mesh_actions.add_parser( + "invite", + help="print the keel mesh join line for a new node, valid one hour" + " and once, and reserve its overlay address (root)", + ) + add_common_options(invite_parser) + invite_parser.add_argument( + "--endpoint", action="append", default=None, metavar="ADDRESS", + help="an address the new node reaches this one at, one per family;" + " may be repeated (default: the static addresses" + " network.interfaces declares)", + ) + invite_parser.add_argument( + "--port", type=port_number, default=WIREGUARD_PORT, metavar="PORT", + help="the TCP port the join request goes to (default: %(default)s," + " the number of WireGuard's UDP port)", + ) + add_root_option(invite_parser, "read the overlay of and keep the" + " invite in") + invite_parser.set_defaults(handler=mesh_commands.mesh_invite) + join_parser = mesh_actions.add_parser( + "join", + help="join the mesh of the node that printed TOKEN; with --dry-run," + " print the change of this node's spec it makes (the join itself" + " is not implemented yet)", + ) + add_common_options(join_parser) + join_parser.add_argument( + "token", metavar="TOKEN", + help="the keel1: token keel mesh invite printed, or - to read it" + " from standard input, which keeps it out of the process list", + ) + join_parser.add_argument( + "--dry-run", action="store_true", + help="print the address and the peer the join would write into the" + " spec, and change nothing", + ) + add_root_option(join_parser, "read the appliance manifests of") + join_parser.set_defaults(handler=mesh_commands.mesh_join) + + +def port_number(text: str) -> int: + if not text.isdigit() or not 1 <= int(text) <= 65535: + raise argparse.ArgumentTypeError(f"not a port number: {text}") + return int(text) + + def add_layer_options(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--layers-dir", @@ -667,7 +727,7 @@ def main(argv: list[str] | None = None) -> int: return exits.USAGE system = getattr(args, "system", False) or getattr( args, "system_only", False - ) or args.command == "database" + ) or args.command in ("database", "mesh") if getattr(args, "dry_run", False) and not system: parser.error("--dry-run requires --system or --system-only") if getattr(args, "defer_certificate", False) and not system: diff --git a/keel/exits.py b/keel/exits.py index 1520d47..9fdff13 100644 --- a/keel/exits.py +++ b/keel/exits.py @@ -28,6 +28,8 @@ CHANNEL_ROLLBACK = 20 NETWORK_NOT_CONFIRMED = 21 NOTIFY_FAILED = 22 +MESH_TOKEN_INVALID = 23 +MESH_REFUSED = 24 DESCRIPTIONS = { OK: "success, including the no-op cases", @@ -68,4 +70,8 @@ " configuration", NOTIFY_FAILED: "keel notify reached no channel: none declared, or every" " one failed", + MESH_TOKEN_INVALID: "keel mesh join could not use the token: mistyped" + " or truncated, of a format this keel does not read, or expired", + MESH_REFUSED: "keel mesh refused: no overlay to invite into, no" + " endpoint, no free address, or this node is in another mesh", } diff --git a/keel/mesh/__init__.py b/keel/mesh/__init__.py new file mode 100644 index 0000000..956f522 --- /dev/null +++ b/keel/mesh/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""Joining the WireGuard mesh with one command (handbook decision 0048) + +`keel mesh invite`, on a node already in the mesh, prints one line, +`keel mesh join keel1:`, and that line run on the new node joins +it. What exists so far: + +- keel.mesh.token, the `keel1:` token: what the new node needs from the + inviter, in one URL safe line with a checksum; +- keel.mesh.allocate, the address the invite reserves for the new node; +- keel.mesh.invites, the pending invites under /var/lib/keel/mesh, which + hold an HMAC key derived from the token's secret, never the secret; +- keel.mesh.identity, the mesh's random identity, which the token + carries and etcd will take as its cluster token; +- keel.mesh.certificate, the self signed certificate of one invite, + whose fingerprint the token carries for the new node to pin; +- keel.mesh.join, the change of the new node's spec a token makes; +- keel.mesh.commands, `keel mesh invite` and `keel mesh join --dry-run`. + +The listener on the HTTPS port, the join itself, `accept`, `create` and +`remove` come later; the token and the invite file already carry what +they need. docs/mesh.md is the reference. +""" + +# under the root: state, never configuration and never in the spec +DIR = "var/lib/keel/mesh" +LOCK = f"{DIR}/lock" +DIR_MODE = 0o700 +FILE_MODE = 0o600 diff --git a/keel/mesh/allocate.py b/keel/mesh/allocate.py new file mode 100644 index 0000000..63399fb --- /dev/null +++ b/keel/mesh/allocate.py @@ -0,0 +1,88 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The address an invite reserves for the new node (decision 0048) + +A random address of the inviter's overlay prefix that is not its own, +not inside a peer's allowed_ips and not reserved by another pending +invite. Random, from `secrets`, rather than the lowest free one: until +etcd exists no member knows what another member's pending invites hold, +so two members inviting at the same time would both pick ::2, while two +random draws in a /64 practically never meet. The prefix's own address, +the subnet router anycast of RFC 4291, is never given. + +A draw that lands on a taken address is drawn again. When ATTEMPTS +draws in a row are taken the prefix is nearly full, and it is searched +from the last draw, wrapping around once, so a prefix with one address +left still gives it and a full one is refused. +""" + +import ipaddress +import secrets +from collections.abc import Callable, Iterable + +from keel.network.wireguard import allowed + +ATTEMPTS = 16 +Network = ipaddress.IPv4Network | ipaddress.IPv6Network + + +class AllocationError(ValueError): + """No address left in the prefix""" + + +def taken(overlay: dict) -> list[str]: + """What the declared peers already route: every allowed_ips prefix""" + return [one for peer in overlay.get("peers") or [] + for one in allowed(peer)] + + +def free_address(own: str, used: Iterable[str], + randbelow: Callable[[int], int] | None = None) -> str: + """A free address of `own`'s prefix, with its prefix length + + `own` is the inviter's overlay address with its length; `used` holds + prefixes and bare addresses of either family (an address written + with a length is that whole prefix), and what lies outside the + prefix is ignored. `randbelow(n)` gives an integer in [0, n); tests + pass their own (default: secrets.randbelow). Raises AllocationError + when none is left. + """ + randbelow = randbelow or secrets.randbelow + mine = ipaddress.IPv6Interface(own) + prefix = mine.network + blocked = [network for network in ( + ipaddress.ip_network(str(one), strict=False) for one in used) + if network.version == 6 and network.overlaps(prefix)] + blocked.append(ipaddress.IPv6Network(mine.ip)) + first = int(prefix.network_address) + 1 + last = int(prefix.broadcast_address) + found = None + if last >= first: + for _ in range(ATTEMPTS): + drawn = first + randbelow(last - first + 1) + if first_free(drawn, drawn, blocked) is not None: + found = drawn + break + else: + found = first_free(drawn, last, blocked) + if found is None: + found = first_free(first, drawn, blocked) + if found is None: + raise AllocationError( + f"no free address in {prefix}: every one is this node's, a" + " peer's or a pending invite's; wait for an invite to" + " expire, or remove a peer") + return f"{ipaddress.IPv6Address(found)}/{prefix.prefixlen}" + + +def first_free(start: int, end: int, blocked: list[Network]) -> int | None: + """The lowest address in [start, end] no blocked prefix covers""" + candidate = start + moved = True + while moved and candidate <= end: + moved = False + for network in blocked: + if int(network.network_address) <= candidate <= int( + network.broadcast_address): + candidate = int(network.broadcast_address) + 1 + moved = True + return candidate if candidate <= end else None diff --git a/keel/mesh/certificate.py b/keel/mesh/certificate.py new file mode 100644 index 0000000..b056c2c --- /dev/null +++ b/keel/mesh/certificate.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 KeelLinux maintainers +"""The self signed certificate of one invite (decision 0048) + +Each invite gets a key pair and a certificate of its own, made by +`openssl req -x509` (openssl is a dependency of keel), so nothing about +it outlives the invite. The token carries the SHA-256 of the +certificate's DER form; the new node pins that fingerprint instead of +trusting any authority. The key stays in the invite's file, mode 0600, +and is never printed. +""" + +import base64 +import hashlib +import re +import subprocess + +OPENSSL = "openssl" +TIMEOUT = 30 +SUBJECT = "/CN=keel-mesh-invite" +# a day: the invite itself expires within the hour; keel checks that +DAYS = "1" +PEM_RE = re.compile( + r"-----BEGIN (?P