diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bc7cac..a573eab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,18 +20,11 @@ jobs: with: go-version-file: go.mod - - uses: actions/setup-python@v7 - with: - python-version: "3.13" - - uses: oven-sh/setup-bun@v2 - name: Build the web dashboard run: make web - - name: Install Python dependencies - run: python -m pip install cryptography - - name: gofmt run: | unformatted=$(gofmt -l .) @@ -49,14 +42,27 @@ jobs: - name: Go tests (race detector) run: go test -race ./... - - name: Python peer tests - run: make test-python + - name: Cross-compile the release artifacts + run: scripts/dist.sh 0.0.0-ci - - name: Go/Python interop - run: make interop + # The carriers that go through the internet. They depend on tailcat's + # public DERP relays and on Cloudflare's quick tunnels, so they report + # without blocking a merge. + carriers: + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + steps: + - uses: actions/checkout@v7 - - name: Conformance suite against the Python peer - run: make conformance + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod - - name: Cross-compile the release artifacts - run: scripts/dist.sh 0.0.0-ci + - name: Install cloudflared + run: | + curl -fsSL -o /usr/local/bin/cloudflared https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 + chmod +x /usr/local/bin/cloudflared + + - name: tailcat and Cloudflare carriers + run: make carriers diff --git a/CHANGELOG.md b/CHANGELOG.md index de5f9f3..218c689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,31 @@ Release versions of this implementation. The protocol version (`v` in `hello`) i ## [Unreleased] +Draft 2 of the protocol (protocol v1), implemented. Not compatible with 0.6.0 or earlier: peers of the two drafts cannot connect, and addresses shared before do not work. `awp up` prints a new address. + ### Changed -- SPEC.md draft 2 (protocol v1). A connection is a WireGuard tunnel between the two peers' keys: the Ed25519 identity key, converted, is the WireGuard key, so the tunnel's authentication is the protocol's and `auth` is gone. Carriers move WireGuard datagrams and are interchangeable and untrusted: `udp`, `tailcat` (the default), `ws` (any HTTP tunnel, including Cloudflare quick tunnels), `unix`. Addresses are `awp1…` blobs of key, pre-shared key and endpoints; `hello` carries `ep` for reconnecting to the sender. Draft 1's notes are folded in: `aud` on grants, cumulative acks, chunks before their msg, what `seen` counts, strictly increasing ids, one connection per pair, the `refused` code, encodings and canonical JSON. Not compatible with draft 1. The implementation still speaks draft 1; the spec example test skips until the wire package follows. +- SPEC.md draft 2. A connection is a WireGuard tunnel between the two peers' keys: the Ed25519 identity key, converted to X25519, is the WireGuard key, so the tunnel's authentication is the protocol's. `auth` and the hello nonce are gone; `hello.key` must be the key on the other end of the tunnel, and `grants` travel in hello. Carriers move WireGuard datagrams and are untrusted: tailcat (the default), UDP, WebSockets (any HTTP tunnel, including Cloudflare quick tunnels), Unix sockets. There is no plaintext binding. Draft 1's notes are folded in: `aud` on grants, cumulative acks, chunks before their msg, what `seen` counts, strictly increasing ids, one connection per pair, the `refused` code, encodings and canonical JSON. +- Addresses are `awp1` strings: the key, the listener's pre-shared key and every endpoint it listens on, in one. `hello.addr` and `introduce.peer.address` are addresses too. Pre-shared keys are per pair once two peers have met, so `awp address --rotate` shuts out agents not met yet without cutting off the ones you know. +- The engine runs on a new `tunnel` package: Tailscale's wireguard-go and a gVisor TCP/IP stack in userspace, with a `conn.Bind` over the carriers that follows a peer to wherever its last authenticated packet came from. The `transport` package is gone. `awp daemon --listen` takes carriers: `tailcat`, `udp:HOST:PORT`, `unix:/path`, `ws:HOST:PORT[=URL]` and `cloudflare`. +- The JSON Schema is `schema/v1/awp.schema.json`, `$id` https://agentwireprotocol.com/schema/v1/awp.schema.json. +- `awp conform` reaches the peer through a tunnel of its own and checks the tunnel binding: two new scenarios, `hello-key` and `hello-own-key`, replace `auth-bad-sig`. `--listen` takes a carrier. +- `awp status` and the control API report the address, its endpoints, and carriers still coming up, in place of the tailcat fields. + +### Added + +- `awp tunnel`: the tunnel alone, as a helper process for SDKs in languages without WireGuard. It holds the SDK's key, prints the address, opens streams on request over a control socket, forwards the streams peers open, and refuses a hello that names another key before the SDK sees it. +- `awp address --rotate`. +- A CI job for the carriers that need the internet (tailcat, Cloudflare), and `make carriers`. + +### Fixed + +- `awp connect` right after a bye dials afresh instead of returning the connection that is closing. + +### Removed + +- The single-file Python peer (`python/`) and its interop and conformance scripts. The Python SDK (agentwireprotocol/python-sdk) runs the conformance suite in its own CI. +- `AWP_ALLOW_PLAINTEXT` and the `tcp:` binding. ## [0.6.0] - 2026-09-29 diff --git a/Makefile b/Makefile index 90e894e..4af4b15 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PLATFORMS := linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 LDFLAGS := -s -w PLUGIN := plugin/awp -.PHONY: build web test test-go test-python interop conformance schema dist plugin plugin-local clean +.PHONY: build web test test-go carriers schema dist plugin plugin-local clean build: go build -trimpath -ldflags "$(LDFLAGS)" -o bin/awp ./cmd/awp @@ -16,24 +16,20 @@ web: find internal/web/dist -mindepth 1 ! -name README -exec rm -rf {} + cp -R web/dist/. internal/web/dist/ -test: test-go test-python interop conformance +test: test-go +# The Go tests include the conformance suite against the Go node, both +# ways. The SDKs run it against themselves in their own repositories. test-go: go vet ./... go test -race ./... -test-python: - cd python && python3 -m unittest test_awp_peer +# The carriers that need the internet: tailcat through its DERP relays, and +# a WebSocket through a Cloudflare quick tunnel (needs cloudflared). +carriers: + AWP_TEST_TAILCAT=1 AWP_TEST_CLOUDFLARE=1 go test -count=1 -v -run 'TestTailcat|TestCloudflare' ./tunnel -interop: build - cd python && AWP_BIN=$(CURDIR)/bin/awp python3 -m unittest -v interop_test - -# The conformance suite (awp conform) against the Python peer, both ways. -# The Go node runs it in its own tests (conformance). -conformance: build - AWP_BIN=$(CURDIR)/bin/awp scripts/conformance.sh - -# The JSON Schema and its reference page, schema/v0/, from the wire +# The JSON Schema and its reference page, schema/v1/, from the wire # package. A test fails when they are stale. schema: go generate ./wire diff --git a/NOTES.md b/NOTES.md index 0bb02dd..7aef7e6 100644 --- a/NOTES.md +++ b/NOTES.md @@ -1,5 +1,7 @@ # Implementation notes for SPEC.md draft 1 +Draft 2 (SPEC.md) folds in the proposals below and replaces the transport and the handshake they discuss: `auth` is gone, and so are the plain bindings. These notes stay as the record of why draft 2 says what it does. + These notes come from building AWP twice, independently. The Go reference daemon is in this repository. The single-file Python peer (`python/awp_peer.py`) was written from SPEC.md alone, without looking at the Go code. The two interoperate (`python/interop_test.py`). The places where either implementation had to guess are listed below, with the choice made and a proposed spec change. ## Decisions on the open questions diff --git a/README.md b/README.md index 0c3b855..f988f17 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # awp -awp is the reference implementation of the Agent Wire Protocol (AWP): peer-to-peer messaging between coding agents, over [tailcat](https://github.com/tailscale/tailcat). AWP is a wire protocol between two peers, not an API on a server. Nothing sits between them: no server, no provider, no account. The protocol is in [SPEC.md](SPEC.md). The spec is at draft 2, which makes WireGuard the connection itself and tailcat one of several pluggable carriers; this implementation still speaks draft 1 (protocol v0) until the engine follows. +awp is the reference implementation of the Agent Wire Protocol (AWP): peer-to-peer messaging between coding agents, inside a WireGuard tunnel between the two agents' own keys. AWP is a wire protocol between two peers, not an API on a server. Nothing sits between them: no server, no provider, no account. The protocol is in [SPEC.md](SPEC.md) (draft 2, protocol v1). + +The tunnel rides on whatever can carry a packet: [tailcat](https://github.com/tailscale/tailcat) by default, which gets through any NAT, or plain UDP, a WebSocket through a Cloudflare quick tunnel or any HTTP proxy, or a local socket. The carriers see only WireGuard, so none of them has to be trusted. One agent runs `awp up` and gets an address. The other runs `awp connect
`. After that, both sides are equal. Either one can: @@ -9,14 +11,14 @@ One agent runs `awp up` and gets an address. The other runs `awp connect - libexec/ binaries, built by `make plugin` .claude-plugin/plugin.json Claude Code manifest @@ -130,6 +132,7 @@ In a test, a Claude Code session with only this plugin loaded was told in plain | command | what it does | |---------|--------------| | `awp up` | start the daemon (if needed); print identity and the address to share | +| `awp address [--rotate]` | print the address; `--rotate` replaces its admission secret, so copies shared before stop admitting agents not met yet | | `awp listen` | as the spec describes: print the address, then stream inbound messages as NDJSON until killed | | `awp connect
` | connect; the daemon keeps the connection and resumes after drops | | `awp send [] ` | send; `--thread`, `--subject`, `--re`, `--code FILE`, `--data JSON`, `--file PATH` (blob), `--wait-ack 30s` | @@ -148,9 +151,25 @@ In a test, a Claude Code session with only this plugin loaded was told in plain | `awp share []`, `awp private ` | share this agent's conversations with a dashboard host; keep a thread out (see NOTES.md) | | `awp model []` | show or set the model this agent runs on (Claude Code, Cursor and opencode report it automatically) | | `awp mcp`, `awp hook ` | MCP server; harness hook helper | +| `awp tunnel` | the WireGuard tunnel alone, as a helper process for SDKs in other languages | +| `awp conform`, `awp schema` | the conformance runner; the JSON Schema | A peer can be named by the name it announced, a local alias (`awp alias`), a unique prefix of its key, or an address. Every command takes `--home` (default `$AWP_HOME` or `~/.awp`), and most take `--json`. +### Carriers + +The daemon listens on tailcat unless told otherwise. `awp daemon --listen`, repeatable, or `AWP_LISTEN` (comma-separated) choose: + +| carrier | listen as | for | +|---------|-----------|-----| +| tailcat | `tailcat` | anything behind NAT: laptops, sandboxes. The default. | +| UDP | `udp:HOST:PORT` (`udp::41641` advertises every local address) | machines that can reach each other: a LAN, Fly's 6PN, public addresses | +| WebSocket | `ws:HOST:PORT`, or `ws:HOST:PORT=wss://public/url` behind a proxy | networks that block UDP; browsers; any HTTP tunnel | +| Cloudflare | `cloudflare` | a `ws` endpoint through a Cloudflare quick tunnel; needs `cloudflared`, no account | +| Unix socket | `unix:/path` | agents on one machine | + +Every carrier the daemon listens on goes into its one address, and a dialer tries them all. `awp status` lists them. + ## Watching the network ```sh @@ -165,15 +184,16 @@ Agents on other hosts appear only if they share presence (the `presence` extensi ## How it works ``` - CLI ─┐ ┌─ tailcat (embedded library; tunnel port 1) - MCP ─┼─ ~/.awp/awp.sock ─ daemon ─ node engine ─────┼─ tcp:host:port (loopback / private only) -hook ─┘ (local control API) │ └─ unix:/path + CLI ─┐ ┌─ tailcat + MCP ─┼─ ~/.awp/awp.sock ─ daemon ─ node engine ─ WireGuard tunnel ─┼─ udp +hook ─┘ (local control API) │ (userspace, TCP ├─ ws / cloudflare + │ inside on port 1) └─ unix ~/.awp/awp.db (SQLite: outbox, seen ids, log, threads, blobs, grants) ``` - **Daemon** (`internal/daemon`). One per home. Any command starts it on demand. It serves a newline-delimited JSON API on a `0600` Unix socket. - **Engine** (`node`). It implements the spec: - - the hello/auth handshake over the exact hello bytes + - the hello handshake, with `hello.key` checked against the tunnel's key - resume, the outbox and acks - dedup by id - blobs, grants and introductions @@ -181,67 +201,66 @@ hook ─┘ (local control API) │ └─ unix:/p - ping/pong liveness (two missed pongs mean a dead connection) - bye, and the error codes with their close rules - reconnection with exponential backoff capped at 60s, no give-up, for as long as there are unacked messages or open threads +- **Tunnel** (`tunnel`). WireGuard (Tailscale's wireguard-go) with a userspace TCP/IP stack (gVisor), keyed by the identity. Its `conn.Bind` spreads datagrams over the carriers and follows a peer to whichever path its last authenticated packet came from. Pre-shared keys (the listener's own, and one per pair of peers) are in `~/.awp/tunnel.json`, the tailcat carrier's key in `~/.awp/tailcat.json`, so an address survives restarts: a sandbox that wakes from sleep is back at the address its peers already have. - **Store** (`store`). All state lives in SQLite (WAL, `synchronous=FULL`), so a `kill -9` at any moment loses nothing. - Received messages are acked only after they are committed. - Ids are allocated inside the enqueue transaction, so id order, outbox order and send order always agree. Resume depends on that. -- **Tailcat** (`transport`). The listener's WireGuard key, pre-shared key and DERP region are saved in `~/.awp/tailcat.json`. The address therefore survives restarts: a sandbox that wakes from sleep is back at the address its peers already have. - **Wire** (`wire/`). The message types, NDJSON framing (1 MiB lines), ULIDs, key encoding, canonical JSON and grants. It is importable by other Go peers, and it is the source the JSON Schema is generated from (below). -### Extensions beyond draft 1 +### Extensions -Unknown fields are ignored (section 5), so all of these are compatible with peers that do not know them. [NOTES.md](NOTES.md) explains why each is needed. +Unknown fields and types are ignored (section 8), so these are compatible with peers that do not know them. Draft 1's extensions (`hello.addr`, `grant.aud`, `chunk.th`, `ref` on `blob_refused`) are part of draft 2; [NOTES.md](NOTES.md) explains why each was needed. -- `hello.addr`: the sender's own reachable address. It lets a listener with queued results reconnect to a dialer that went away. -- `grant.aud`: binds an introduction grant to the peer it is meant for. Without it, the grant would also give the recipient powers over the introducer. -- `chunk.th`: chunks carry their thread, so resume can replay them. Chunks are sent before the msg that references them. -- `err ref`: `blob_refused` names the refused blob. - `presence`: an opt-in, signed summary of what an agent is doing: its peers, and its threads' subjects and states. It is gossiped across the network, so `awp web` on any connected host can show every agent. It is sent only to peers that list `presence` in their hello `caps`. ## Schema and conformance The protocol has no OpenAPI description, since it is not HTTP. It has the equivalent for a line protocol: -- **A JSON Schema** (draft 2020-12) for every message, at [`schema/v0/awp.schema.json`](schema/v0/awp.schema.json) and served at https://agentwireprotocol.com/schema/v0/awp.schema.json. It is generated from the `wire` package by `make schema` (`go generate ./wire`), together with the [schema reference page](schema/v0/schema.mdx) of the docs site, so the reference implementation's types are the source of truth and a test fails when the files are stale. Every JSON example in [SPEC.md](SPEC.md) validates against it, and so does one of every message the Go peer encodes. `awp schema` prints it. -- **A conformance runner**, `awp conform`. It is a peer of its own with a key of its own: it connects to the peer under test once per scenario, drives each exchange, checks every line it receives against the schema, and reports. +- **A JSON Schema** (draft 2020-12) for every message, at [`schema/v1/awp.schema.json`](schema/v1/awp.schema.json) and served at https://agentwireprotocol.com/schema/v1/awp.schema.json. It is generated from the `wire` package by `make schema` (`go generate ./wire`), together with the [schema reference page](schema/v1/schema.mdx) of the docs site, so the reference implementation's types are the source of truth and a test fails when the files are stale. Every JSON example in [SPEC.md](SPEC.md) validates against it, and so does one of every message the Go peer encodes. `awp schema` prints it. +- **A conformance runner**, `awp conform`. It is a peer of its own with a key of its own: it reaches the peer under test through a tunnel like any peer, once per scenario, drives each exchange, checks every line it receives against the schema, and reports. ```sh -awp conform tcp:127.0.0.1:7000 # the peer listens; every scenario, each on its own connection -awp conform --listen tcp:127.0.0.1:0 \ +awp conform awp1... # the peer listens; every scenario, each on its own connection +awp conform --listen udp:127.0.0.1:0 \ --run 'my-peer connect {addr}' # the peer connects to the runner; the scenarios that share a connection awp conform --list # the scenarios, with the spec section each one checks -awp conform --scenario handshake --trace tcp:127.0.0.1:7000 +awp conform --scenario handshake --trace awp1... ``` -The scenarios cover the handshake (hello without waiting, auth signatures, resume), the closing errors (`version`, `auth`, `bad_frame`, `too_large`) and that the connection closes after them, ping/pong, acks for `msg` and `state` with the right `re` and `th`, dedup, unknown types and fields, non-closing errors, blobs in chunks, grants and bye. The report is text, or JSON with `--json`; the exit status is 1 when a scenario fails. The reference implementation runs the suite against itself in `go test`, and `make conformance` runs it against the Python peer both ways. +The scenarios cover the handshake (hello without waiting, `hello.key` bound to the tunnel, resume), the closing errors (`version`, `auth` for a hello naming another key or the peer's own, `bad_frame`, `too_large`) and that the connection closes after them, ping/pong, acks for `msg` and `state` with the right `re` and `th`, dedup, unknown types and fields, non-closing errors, blobs in chunks, grants and bye. The report is text, or JSON with `--json`; the exit status is 1 when a scenario fails. The reference implementation runs the suite against itself in `go test`, and each SDK runs it against itself in its own CI. -Writing a peer in another language: generate your types from the schema (or check hand-written ones against it, as the MCP SDKs do), keep the examples in SPEC.md as test vectors (the grant in section 10.2 has a real signature), and run `awp conform` against your peer while you go. +Writing a peer in another language: generate your types from the schema (or check hand-written ones against it, as the MCP SDKs do), keep the examples in SPEC.md as test vectors (the key conversion in section 4.2 and the grant in section 13.2 are real), get the tunnel from `awp tunnel` (below), and run `awp conform` against your peer while you go. ## SDKs Your own program can be a peer. Each SDK has the whole protocol inside (resume with an outbox on disk, acks, dedup, blobs, grants, reconnection) and runs the conformance suite in its own tests: -- [Go](https://github.com/agentwireprotocol/go-sdk): `github.com/agentwireprotocol/go-sdk/awp`, a `Peer` on this repository's engine (the `node`, `store`, `transport` and `conformance` packages). -- [Python](https://github.com/agentwireprotocol/python-sdk): `pip install awp`, an asyncio `Peer` with no dependencies, grown out of `python/awp_peer.py`. +- [Go](https://github.com/agentwireprotocol/go-sdk): `github.com/agentwireprotocol/go-sdk/awp`, a `Peer` on this repository's engine (the `node`, `store`, `tunnel` and `conformance` packages). +- [Python](https://github.com/agentwireprotocol/python-sdk): `pip install awp`, an asyncio `Peer` with no dependencies. - [TypeScript](https://github.com/agentwireprotocol/typescript-sdk): `npm install @agentwireprotocol/sdk`, a `Peer` for Node and Bun with types generated from the schema. +Python and TypeScript have no WireGuard of their own, so their peers run `awp tunnel`: the tunnel alone, one process per peer, holding the SDK's key. It prints the address, opens streams on request over a control socket, forwards the streams other peers open, and checks each `hello.key` against the tunnel before the SDK sees it (SPEC.md section 18.2). + The docs have a section for each: https://docs.agentwireprotocol.com/go, /python, /typescript. ## Security -- The address is a bearer secret for reaching `hello`, and nothing more. Share it like a password. -- The default admission policy accepts any key and logs it, as section 7.3 recommends. `--accept allowlist` admits only allowed keys, trusted keys, keys you dialed, and keys that present a grant you honor. +- The address is a bearer secret for reaching `hello`, and nothing more. Share it like a password; `awp address --rotate` revokes every copy for agents not met yet. +- Every byte between two agents is inside WireGuard between their keys. Carriers, relays and tunnel providers see ciphertext only; there is no plaintext mode. +- The default admission policy accepts any key and logs it, as section 10.2 recommends. `--accept allowlist` admits only allowed keys, trusted keys, keys you dialed, and keys that present a grant you honor. - `exec` and `fs:write` are remote code execution. - Requests that need a capability the sender lacks are refused with `err forbidden`. - The daemon *serves* requests itself only when started with `--serve` (see [PROFILE.md](PROFILE.md)). Otherwise the request goes to the agent, marked as allowed or denied. - File access is confined to `--root` through `os.Root`. -- Plain TCP to public addresses is refused. Use tailcat, or a private network such as Fly's 6PN. - The skill and the hooks frame inbound messages as untrusted input from another agent, not instructions from the user. ## Development ```sh -make test # go vet, go test -race, the Python peer's own tests, Go↔Python interop, conformance -make schema # regenerate schema/v0/ from the wire package +make test # go vet, go test -race (the conformance suite against the Go node included) +make carriers # the tailcat and Cloudflare carriers, live (needs the internet and cloudflared) +make schema # regenerate schema/v1/ from the wire package make build # ./bin/awp make plugin # plugin/awp/libexec/awp-{linux,darwin}-{amd64,arm64} make dist # release artifacts in dist/ @@ -262,16 +281,15 @@ The release workflow runs CI, checks that the version markers match the tag, bui The test suite covers: - the wire format (including canonical JSON checked against Python's `json.dumps`) -- two-node integration: resume across repeated `kill -9`, byte-identical multi-chunk blobs, refused blobs, bad auth, version, bad frame, oversized lines, unknown fields, dead-peer detection, served `exec` and `fs:read` behind grants, introductions with attenuation and audience binding, bye and parking, reconnecting via `hello.addr` -- the MCP server, including channel push -- interop against the independent Python peer (`python/`), over TCP and Unix sockets, with `kill -9` on each side -- the schema: the generated files are current, SPEC.md's examples and the Go peer's messages validate, and the conformance suite passes against the Go node in both modes and against the Python peer +- the tunnel: the key conversion vectors from SPEC.md, addresses, streams over the udp, unix and ws carriers, wrong and rotated pre-shared keys, restarts, and (with `make carriers`) tailcat and Cloudflare live +- two-node integration: resume across repeated `kill -9`, byte-identical multi-chunk blobs, refused blobs, a hello naming another key or our own, version, bad frame, oversized lines, unknown fields, dead-peer detection, served `exec` and `fs:read` behind grants, introductions with attenuation and audience binding, bye and parking, reconnecting via `hello.addr` +- the MCP server, including channel push; the `awp tunnel` helper in both directions +- the schema: the generated files are current, SPEC.md's examples and the Go peer's messages validate, and the conformance suite passes against the Go node in both modes; the SDKs run it against themselves Not done yet: - multi-party rooms -- key rotation -- a WebSocket binding (for wake-on-connect through a sandbox's HTTP URL) +- identity key rotation - Windows ## License diff --git a/cmd/awp/commands.go b/cmd/awp/commands.go index 6c9cb67..7b2b018 100644 --- a/cmd/awp/commands.go +++ b/cmd/awp/commands.go @@ -21,7 +21,7 @@ import ( "github.com/agentwireprotocol/awp/internal/mcp" "github.com/agentwireprotocol/awp/internal/version" "github.com/agentwireprotocol/awp/store" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) @@ -32,19 +32,19 @@ func cmdVersion(ctx context.Context, args []string) error { func cmdDaemon(ctx context.Context, args []string) error { f := newFlags("daemon", "", "Run the awp daemon in the foreground. Other commands start it on demand;\nrun it yourself under a service manager, or to watch its log.") - listen := f.StringArray("listen", nil, "address to listen on: tailcat, tcp:HOST:PORT or unix:/path (repeatable; default from config, else tailcat)") + listen := f.StringArray("listen", nil, "carrier to listen on: tailcat, udp:HOST:PORT, unix:/path, ws:HOST:PORT[=URL] or cloudflare\n(repeatable; default from config, else tailcat)") noTailcat := f.Bool("no-tailcat", false, "do not listen on tailcat") name := f.String("name", "", "name sent in hello (default awp@HOSTNAME)") about := f.String("about", "", "free-text description sent in hello") harnessFlag := f.String("harness", "", "agent harness this agent runs in: "+strings.Join(harness.IDs(), ", ")+" (default: detected; remembered)") - advertise := f.String("advertise", "", "address sent in hello for the peer to dial back (none to disable)") + advertise := f.String("advertise", "", "none: do not send our address in hello for the peer to dial back") serve := f.StringSlice("serve", nil, "capabilities to fulfil automatically for granted peers: exec, fs:read, fs:write") root := f.String("root", "", "directory that fs:read/fs:write are confined to and exec runs in") accept := f.String("accept", "", "admission policy: any (default) or allowlist") allow := f.StringArray("allow", nil, "key to admit under the allowlist policy (repeatable)") trust := f.StringArray("trust", nil, "issuer key whose grants to honor (repeatable)") trace := f.Bool("trace", false, "log every protocol line") - verbose := f.Bool("verbose", false, "include tailcat's own logs") + verbose := f.Bool("verbose", false, "include tailcat's and WireGuard's own logs") presence := f.Bool("presence", false, "publish signed presence (threads, states, peers) so awp web on any connected host can see this agent") if err := f.Parse(args); err != nil { return err @@ -92,7 +92,7 @@ func cmdDaemon(ctx context.Context, args []string) error { } func cmdUp(ctx context.Context, args []string) error { - f := newFlags("up", "", "Start the daemon in the background if it is not running, wait for the tailcat\naddress, and print the identity and the address to share.") + f := newFlags("up", "", "Start the daemon in the background if it is not running, wait for its carriers\nto come up, and print the identity and the address to share.") name := f.String("name", "", "name to present to peers, e.g. claude-code@myhost (remembered)") about := f.String("about", "", "what you are working on, sent in hello (remembered)") harnessFlag := f.String("harness", "", "agent harness this agent runs in: "+strings.Join(harness.IDs(), ", ")+" (default: detected; remembered)") @@ -139,8 +139,8 @@ func cmdUp(ctx context.Context, args []string) error { fmt.Println("share the address with the other agent; they run: awp connect
") } else { fmt.Printf(" address none yet") - if st.TailcatErr != "" { - fmt.Printf(" (tailcat: %s)", st.TailcatErr) + if e := st.PendingError(); e != "" { + fmt.Printf(" (%s)", e) } fmt.Println() } @@ -212,19 +212,17 @@ func cmdStatus(ctx context.Context, args []string) error { } fmt.Printf("%s %s\n", st.Name, st.Key) fmt.Printf(" pid %d, up %s, home %s\n", st.PID, time.Since(st.Started).Round(time.Second), st.Home) - if st.Tailcat != "" { - fmt.Printf(" address %s\n", st.Tailcat) - } else if st.TailcatWant { - msg := "starting" - if st.TailcatErr != "" { - msg = st.TailcatErr - } - fmt.Printf(" tailcat %s\n", msg) + if st.Address != "" { + fmt.Printf(" address %s\n", st.Address) + } + for _, e := range st.Endpoints { + fmt.Printf(" via %s\n", shortEndpoint(e)) } - for _, a := range st.Addresses { - if !strings.HasPrefix(a, "tailcat:") { - fmt.Printf(" listen %s\n", a) + for spec, e := range st.Pending { + if e == "" { + e = "starting" } + fmt.Printf(" %-8s %s\n", spec, e) } if len(st.Serve) > 0 { fmt.Printf(" serves %s (to peers holding a grant)\n", strings.Join(st.Serve, ", ")) @@ -239,7 +237,8 @@ func cmdStatus(ctx context.Context, args []string) error { } func cmdAddress(ctx context.Context, args []string) error { - f := newFlags("address", "", "Print the address to share (the tailcat address when available).") + f := newFlags("address", "", "Print the address to share: this agent's key, admission secret and every\nendpoint it listens on. --rotate replaces the admission secret, so every\naddress shared before stops admitting agents not met yet.") + rotate := f.Bool("rotate", false, "replace the pre-shared key first") if err := f.Parse(args); err != nil { return err } @@ -247,13 +246,18 @@ func cmdAddress(ctx context.Context, args []string) error { if err != nil { return err } + if *rotate { + if err := c.Call(ctx, "rotate_psk", nil, nil); err != nil { + return err + } + } st, err := waitAddress(ctx, c, 45*time.Second) if err != nil { return err } addr := st.ShareAddress() if addr == "" { - return fmt.Errorf("no address yet (tailcat: %s)", st.TailcatErr) + return fmt.Errorf("no address yet (%s)", st.PendingError()) } fmt.Println(addr) return nil @@ -279,7 +283,7 @@ func cmdListen(ctx context.Context, args []string) error { if *text { fmt.Printf("listening as %s (%s)\naddress %s\n", st.Name, st.Key, st.ShareAddress()) } else { - printLine(map[string]any{"event": "listening", "address": st.ShareAddress(), "addresses": st.Addresses, "key": st.Key, "name": st.Name}) + printLine(map[string]any{"event": "listening", "address": st.ShareAddress(), "endpoints": st.Endpoints, "key": st.Key, "name": st.Name}) } p := &printer{w: os.Stdout} err = c.Stream(ctx, "subscribe", api.SubscribeParams{Inbox: true, Mark: *mark}, func(raw json.RawMessage) error { @@ -299,7 +303,7 @@ func cmdListen(ctx context.Context, args []string) error { } func cmdConnect(ctx context.Context, args []string) error { - f := newFlags("connect", "
", "Connect to a peer. The address is what the other side's `awp up` or\n`awp listen` printed (tc..., tcp:HOST:PORT or unix:/path), or the name of a\npeer you have met before. The daemon keeps the connection and reconnects\nwhenever there is unfinished business.") + f := newFlags("connect", "
", "Connect to a peer. The address is what the other side's `awp up` or\n`awp listen` printed (awp1...), or the name of a\npeer you have met before. The daemon keeps the connection and reconnects\nwhenever there is unfinished business.") timeout := f.Duration("timeout", 60*time.Second, "how long to wait for the connection") if err := f.Parse(args); err != nil { return err @@ -438,7 +442,7 @@ func sendStatus(res api.SendResult, waited bool) string { func looksLikePeer(ctx context.Context, c interface { Call(context.Context, string, any, any) error }, s string) bool { - if _, err := transport.Parse(s); err == nil { + if tunnel.IsAddress(s) { return true } var peers []api.PeerView @@ -783,7 +787,7 @@ func printPeers(peers []api.PeerView) { } func cmdGrant(ctx context.Context, args []string) error { - f := newFlags("grant", " ...", "Grant capabilities to a peer (section 10 of the spec): exec, fs:read, fs:write,\nintroduce, admin. exec and fs:write are remote code execution; grant them only\nwhen your user has explicitly agreed, and keep the ttl short.") + f := newFlags("grant", " ...", "Grant capabilities to a peer (section 13 of the spec): exec, fs:read, fs:write,\nintroduce, admin. exec and fs:write are remote code execution; grant them only\nwhen your user has explicitly agreed, and keep the ttl short.") ttl := f.Duration("ttl", time.Hour, "how long the grant is valid") if err := f.Parse(args); err != nil { return err @@ -1092,3 +1096,13 @@ func cmdPrivate(ctx context.Context, args []string) error { fmt.Printf("thread %s is private: it is not shared, and hosts that had a copy are asked to forget it\n", th) return nil } + +// shortEndpoint abbreviates long endpoint values (tailcat addresses) for +// display. +func shortEndpoint(e string) string { + kind, v, _ := strings.Cut(e, ":") + if len(v) > 40 { + v = v[:16] + "…" + v[len(v)-8:] + } + return kind + " " + v +} diff --git a/cmd/awp/conform.go b/cmd/awp/conform.go index 3a24cd2..693a3ea 100644 --- a/cmd/awp/conform.go +++ b/cmd/awp/conform.go @@ -19,8 +19,8 @@ func cmdConform(ctx context.Context, args []string) error { "to the address once per scenario, drives each exchange, checks every line against the\n"+ "JSON Schema, and reports. With --listen the peer connects to the runner instead, and the\n"+ "scenarios that fit one connection run in turn. Exit status 1 if any scenario fails.") - listen := f.String("listen", "", "listen here (tcp:HOST:PORT or unix:/path) and let the peer connect") - run := f.String("run", "", "with --listen: a shell command that starts the peer, {addr} replaced by the listening address") + listen := f.String("listen", "", "listen on this carrier (udp:127.0.0.1:0, unix:/path, tailcat, ...) and let the peer connect") + run := f.String("run", "", "with --listen: a shell command that starts the peer, {addr} replaced by the runner's address") only := f.StringArray("scenario", nil, "run only this scenario (repeatable; see --list)") timeout := f.Duration("timeout", 15*time.Second, "how long to wait for the peer at each step") list := f.Bool("list", false, "list the scenarios and exit") diff --git a/cmd/awp/main.go b/cmd/awp/main.go index 77a529a..514c00f 100644 --- a/cmd/awp/main.go +++ b/cmd/awp/main.go @@ -1,5 +1,5 @@ // Command awp is the reference implementation of the Agent Wire Protocol -// (AWP): a daemon that holds the identity, the tailcat listener and every +// (AWP): a daemon that holds the identity, the WireGuard tunnel and every // connection, plus a small CLI, harness hooks and an MCP server on top. package main @@ -65,6 +65,7 @@ func init() { {"model", "[]", "show or set the model this agent runs on (shared with the network)", cmdModel}, {"share", "[...]", "show or set the hosts this agent shares its conversations with", cmdShare}, {"private", "[] ", "keep a thread out of conversation sharing", cmdPrivate}, + {"tunnel", "", "the WireGuard tunnel as a helper process, for SDKs in other languages", cmdTunnel}, {"conform", "[
]", "check a peer against the protocol: run the conformance scenarios and report", cmdConform}, {"schema", "", "print the protocol's JSON Schema", cmdSchema}, {"version", "", "print the version", cmdVersion}, @@ -112,7 +113,7 @@ func findCommand(name string) *command { } func usage(w io.Writer) { - fmt.Fprintln(w, "awp: peer-to-peer messaging between agents, over the Agent Wire Protocol and tailcat.") + fmt.Fprintln(w, "awp: peer-to-peer messaging between agents, over the Agent Wire Protocol: WireGuard\nbetween the agents' keys, carried by tailcat, UDP, WebSockets or a local socket.") fmt.Fprintln(w) fmt.Fprintln(w, "usage: awp [flags] [args]") fmt.Fprintln(w) @@ -300,8 +301,8 @@ func printLine(v any) error { return enc.Encode(v) } -// waitAddress polls status until tailcat is up (if enabled), for commands -// that need an address to hand out. +// waitAddress polls status until the carriers are up, for commands that +// need an address to hand out. func waitAddress(ctx context.Context, c *control.Client, timeout time.Duration) (*api.Status, error) { deadline := time.Now().Add(timeout) for { @@ -309,7 +310,7 @@ func waitAddress(ctx context.Context, c *control.Client, timeout time.Duration) if err := c.Call(ctx, "status", nil, &st); err != nil { return nil, err } - if st.Tailcat != "" || !st.TailcatWant || time.Now().After(deadline) { + if st.Settled() || time.Now().After(deadline) { return &st, nil } select { diff --git a/cmd/awp/tunnel.go b/cmd/awp/tunnel.go new file mode 100644 index 0000000..e04be0d --- /dev/null +++ b/cmd/awp/tunnel.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/signal" + "syscall" + + "github.com/agentwireprotocol/awp/internal/helper" +) + +func cmdTunnel(ctx context.Context, args []string) error { + f := newFlags("tunnel", "", + "The WireGuard tunnel as a helper process, for SDKs that speak the protocol themselves\n"+ + "(SPEC.md section 18.2). It prints JSON lines on stdout (a ready event, then an address\n"+ + "event whenever the address changes), dials on request over --socket, forwards the\n"+ + "streams peers open to --forward, and exits when stdin closes. The identity comes from\n"+ + "--identity (a JSON file with \"seed\") or $AWP_IDENTITY_SEED.") + identity := f.String("identity", "", "identity file: JSON with \"seed\", 32 bytes of base64url") + state := f.String("state", "", "directory for pre-shared keys and the tailcat key (default: in memory)") + listen := f.StringArray("listen", nil, "carrier to listen on: tailcat, udp:HOST:PORT, unix:/path, ws:HOST:PORT[=URL] or cloudflare (repeatable)") + socket := f.String("socket", "", "control socket to create, for dial requests") + forward := f.String("forward", "", "where streams peers open go: a unix socket path, or tcp:HOST:PORT") + verbose := f.Bool("verbose", false, "log the tunnel's activity on stderr") + if err := f.Parse(args); err != nil { + return err + } + var cfg helper.Config + var err error + switch { + case *identity != "": + cfg.Identity, err = helper.LoadIdentity(*identity) + case os.Getenv("AWP_IDENTITY_SEED") != "": + cfg.Identity, err = helper.SeedKey(os.Getenv("AWP_IDENTITY_SEED")) + default: + err = errors.New("give --identity or $AWP_IDENTITY_SEED") + } + if err != nil { + return err + } + if *socket == "" { + return errors.New("--socket is required") + } + if len(*listen) > 0 && *forward == "" { + return errors.New("--listen needs --forward, where accepted streams go") + } + cfg.StateDir, cfg.Listen, cfg.Socket, cfg.Forward = *state, *listen, *socket, *forward + if *verbose { + lg := log.New(os.Stderr, "awp tunnel: ", log.LstdFlags) + cfg.Logf = func(format string, args ...any) { lg.Output(2, fmt.Sprintf(format, args...)) } + } + ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) + defer stop() + return helper.Run(ctx, cfg, os.Stdin, os.Stdout) +} diff --git a/conformance/conformance.go b/conformance/conformance.go index 2525c70..ea73f22 100644 --- a/conformance/conformance.go +++ b/conformance/conformance.go @@ -3,8 +3,12 @@ // (or listens for it), drives each scenario over the wire, checks every // line it receives against the JSON Schema, and reports what the peer did. // +// The runner reaches the peer the way any peer does, through a WireGuard +// tunnel from a key of its own, so the tunnel's binding of hello.key is +// checked too. +// // It is what `awp conform` runs. The reference implementation passes it in -// its own tests; scripts/conformance.sh runs it against the Python peer. +// its own tests, and so do the SDKs. package conformance import ( @@ -23,20 +27,22 @@ import ( "time" "github.com/agentwireprotocol/awp/schema" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) // Options say which peer to check and how. type Options struct { - // Addr is the peer's address. The runner dials it once per scenario. + // Addr is the peer's address (awp1...). The runner dials it once per + // scenario. Addr string - // Listen is an address (tcp:host:port or unix:/path) to listen on - // instead: the peer dials the runner, and the scenarios that fit one - // connection run in turn on it. Solo scenarios are skipped. + // Listen is a carrier to listen on instead (tunnel.Config.Listen: + // "udp:127.0.0.1:0", "unix:/path", "tailcat", ...): the peer dials the + // runner, and the scenarios that fit one connection run in turn on it. + // Solo scenarios are skipped. Listen string // Run is a shell command to start once the runner listens, with - // {addr} replaced by the listening address; it is killed at the end. + // {addr} replaced by the runner's address; it is killed at the end. Run string // Only names the scenarios to run; nil runs all of them. Only []string @@ -139,6 +145,15 @@ func Run(ctx context.Context, o Options) (*Report, error) { return nil, err } r := &runner{o: o, v: v, priv: priv, key: wire.FormatKey(pub), ids: wire.NewIDGen()} + var listen []string + if o.Listen != "" { + listen = []string{o.Listen} + } + r.tun, err = tunnel.New(tunnel.Config{Identity: priv, Listen: listen}) + if err != nil { + return nil, err + } + defer r.tun.Close() started := time.Now() rep := &Report{Addr: o.Addr, Mode: "connect"} if o.Listen != "" { @@ -167,6 +182,7 @@ func Run(ctx context.Context, o Options) (*Report, error) { type runner struct { o Options + tun *tunnel.Tunnel v *schema.Validator priv ed25519.PrivateKey key string @@ -176,16 +192,14 @@ type runner struct { // connect runs every scenario on a connection of its own. func (r *runner) connect(ctx context.Context, scenarios []Scenario, rep *Report) error { - addr, err := transport.Parse(r.o.Addr) + addr, err := tunnel.ParseAddress(r.o.Addr) if err != nil { return err } - t := &transport.Transport{AllowPublicTCP: true} - defer t.Close() for _, sc := range scenarios { r.o.Logf("%s: connecting", sc.Name) - dctx, cancel := context.WithTimeout(ctx, r.o.Timeout) - conn, err := t.Dial(dctx, addr) + dctx, cancel := context.WithTimeout(ctx, 3*r.o.Timeout) + conn, err := r.tun.Dial(dctx, addr) cancel() if err != nil { if len(rep.Results) == 0 { @@ -204,17 +218,22 @@ func (r *runner) connect(ctx context.Context, scenarios []Scenario, rep *Report) // listen accepts one connection from the peer and runs the scenarios that // share a connection on it, in order. func (r *runner) listen(ctx context.Context, scenarios []Scenario, rep *Report) error { - addr, err := transport.Parse(r.o.Listen) - if err != nil { + if err := r.tun.Start(); err != nil { return err } - ln, err := transport.Listen(addr) - if err != nil { - return err + deadline := time.Now().Add(90 * time.Second) + for len(r.tun.Address().Endpoints) == 0 { + if time.Now().After(deadline) { + return fmt.Errorf("listen %s: %v", r.o.Listen, r.tun.Pending()) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(100 * time.Millisecond): + } } - defer ln.Close() - rep.Addr = addr.Kind + ":" + ln.Addr().String() - r.o.Logf("listening on %s", rep.Addr) + rep.Addr = r.tun.Address().String() + r.o.Logf("listening at %s", rep.Addr) if r.o.Run != "" { cmd := exec.CommandContext(ctx, "sh", "-c", strings.ReplaceAll(r.o.Run, "{addr}", rep.Addr)) cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr @@ -230,15 +249,15 @@ func (r *runner) listen(ctx context.Context, scenarios []Scenario, rep *Report) }() } type accepted struct { - conn net.Conn + conn *tunnel.Stream err error } ch := make(chan accepted, 1) go func() { - c, err := ln.Accept() + c, err := r.tun.Accept() ch <- accepted{c, err} }() - var conn net.Conn + var conn *tunnel.Stream select { case a := <-ch: if a.err != nil { @@ -268,7 +287,7 @@ func (r *runner) listen(ctx context.Context, scenarios []Scenario, rep *Report) return nil } -func (r *runner) session(conn net.Conn) *session { +func (r *runner) session(conn *tunnel.Stream) *session { return &session{r: r, conn: conn, rd: wire.NewReader(conn), wr: wire.NewWriter(conn), timeout: r.o.Timeout, trace: r.o.Trace} } @@ -311,7 +330,7 @@ func (r *runner) runOne(s *session, sc Scenario) Result { // session is one connection to the peer, as the scenarios see it. type session struct { r *runner - conn net.Conn + conn *tunnel.Stream rd *wire.Reader wr *wire.Writer timeout time.Duration @@ -471,11 +490,11 @@ func (s *session) expectClosed(what string) error { var helloCaps = []string{"chat", "blob", "grant", "introduce"} func (s *session) hello(v int) wire.Hello { - return wire.Hello{Envelope: s.env(wire.THello), V: v, Key: s.r.key, Name: s.r.o.Name, Nonce: wire.Nonce(32), + return wire.Hello{Envelope: s.env(wire.THello), V: v, Key: s.r.key, Name: s.r.o.Name, Caps: helloCaps, About: "conformance runner"} } -// handshake does section 7. With waitFirst the runner holds its own hello +// handshake does section 10. With waitFirst the runner holds its own hello // until the peer's arrives and checks that the peer did not wait. func (s *session) handshake(waitFirst bool) error { if !waitFirst { @@ -516,31 +535,20 @@ func (s *session) handshake(waitFirst bool) error { if !s.check(err == nil, "hello.key is an ed25519 key (%v)", err) { return errors.New("bad hello key") } - s.check(h.Nonce != "", "hello carries a nonce") - if s.r.peer == nil { - s.r.peer = &Peer{Key: h.Key, Name: h.Name, About: h.About, V: h.V, Caps: h.Caps} - } - - if err := s.send(wire.Auth{Envelope: s.env(wire.TAuth), Sig: wire.SignAuth(s.r.priv, s.myHello, s.peerHello)}); err != nil { - return err - } - _, line, err = s.expect(wire.TAuth) - if err != nil { - return fmt.Errorf("waiting for auth: %w", err) - } - var a wire.Auth - if err := wire.Decode(line, &a); err != nil { - return fmt.Errorf("malformed auth: %v", err) - } - err = wire.VerifyAuth(pub, a.Sig, s.peerHello, s.myHello) - if !s.check(err == nil, "auth.sig verifies with hello.key over both hello lines (%v)", err) { - return errors.New("auth signature does not verify") - } - for i, g := range a.Grants { + s.check(s.conn.Is(pub), "hello.key is the key on the other end of the tunnel") + for i, g := range h.Grants { if _, err := wire.ParseGrant(g, time.Time{}); err != nil { - s.check(false, "auth.grants[%d] verifies (%v)", i, err) + s.check(false, "hello.grants[%d] verifies (%v)", i, err) } } + if h.Addr != "" { + a, err := tunnel.ParseAddress(h.Addr) + s.check(err == nil && a.KeyString() == h.Key, "hello.addr is an address of hello.key (%v)", err) + s.check(err != nil || a.PSK == nil, "hello.addr carries no pre-shared key") + } + if s.r.peer == nil { + s.r.peer = &Peer{Key: h.Key, Name: h.Name, About: h.About, V: h.V, Caps: h.Caps} + } env, line, err = s.expect(wire.TResume) if err != nil { @@ -550,7 +558,7 @@ func (s *session) handshake(waitFirst bool) error { if err := wire.Decode(line, &res); err != nil { return fmt.Errorf("malformed resume: %v", err) } - s.check(true, "peer sent resume after auth (%d threads seen)", len(res.Seen)) + s.check(true, "peer sent resume after hello (%d threads seen)", len(res.Seen)) if err := s.send(wire.Resume{Envelope: s.env(wire.TResume), Seen: map[string]string{}}); err != nil { return err } diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go index ad4f422..321300b 100644 --- a/conformance/conformance_test.go +++ b/conformance/conformance_test.go @@ -2,6 +2,7 @@ package conformance_test import ( "context" + "os" "path/filepath" "strings" "testing" @@ -13,7 +14,11 @@ import ( func startNode(t *testing.T) *node.Node { t.Helper() - home := t.TempDir() + home, err := os.MkdirTemp("", "awpc") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(home) }) n, err := node.Open(node.Config{ Home: home, Name: "go@test", @@ -53,7 +58,7 @@ func TestReferenceImplementation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() rep, err := conformance.Run(ctx, conformance.Options{ - Addr: n.Addresses()[0], + Addr: n.Address().String(), Timeout: 10 * time.Second, Logf: t.Logf, }) @@ -76,17 +81,22 @@ func TestReferenceImplementation(t *testing.T) { // connection run, the solo ones are skipped. func TestListenMode(t *testing.T) { n := startNode(t) - sock := "unix:" + filepath.Join(t.TempDir(), "runner.sock") ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() + addr := make(chan string, 1) go func() { - // Give the runner a moment to listen, then connect. - time.Sleep(300 * time.Millisecond) - if _, err := n.Connect(ctx, sock); err != nil { + // Connect once the runner says where it listens. + a := <-addr + if _, err := n.Connect(ctx, a); err != nil { t.Logf("connect: %v", err) } }() - rep, err := conformance.Run(ctx, conformance.Options{Listen: sock, Timeout: 10 * time.Second, Logf: t.Logf}) + rep, err := conformance.Run(ctx, conformance.Options{Listen: "udp:127.0.0.1:0", Timeout: 10 * time.Second, Logf: func(f string, a ...any) { + t.Logf(f, a...) + if strings.HasPrefix(f, "listening at") { + addr <- a[0].(string) + } + }}) if err != nil { t.Fatal(err) } @@ -103,7 +113,7 @@ func TestListenMode(t *testing.T) { if rep.Skipped != solo { t.Fatalf("%d skipped, want the %d solo scenarios", rep.Skipped, solo) } - if !strings.HasPrefix(rep.Addr, "unix:") { + if !strings.HasPrefix(rep.Addr, "awp1") { t.Fatalf("report address %q", rep.Addr) } } @@ -112,7 +122,7 @@ func TestListenMode(t *testing.T) { func TestOnly(t *testing.T) { n := startNode(t) ctx := context.Background() - rep, err := conformance.Run(ctx, conformance.Options{Addr: n.Addresses()[0], Only: []string{"ping", "bye"}, Timeout: 10 * time.Second}) + rep, err := conformance.Run(ctx, conformance.Options{Addr: n.Address().String(), Only: []string{"ping", "bye"}, Timeout: 10 * time.Second}) if err != nil { t.Fatal(err) } @@ -120,7 +130,7 @@ func TestOnly(t *testing.T) { report(t, rep) t.Fatalf("results %+v", rep.Results) } - if _, err := conformance.Run(ctx, conformance.Options{Addr: n.Addresses()[0], Only: []string{"nope"}}); err == nil { + if _, err := conformance.Run(ctx, conformance.Options{Addr: n.Address().String(), Only: []string{"nope"}}); err == nil { t.Fatal("unknown scenario accepted") } } diff --git a/conformance/scenarios.go b/conformance/scenarios.go index 91e7b15..5802b1e 100644 --- a/conformance/scenarios.go +++ b/conformance/scenarios.go @@ -1,6 +1,7 @@ package conformance import ( + "crypto/ed25519" "encoding/base64" "errors" "fmt" @@ -13,10 +14,10 @@ import ( // Scenarios lists every scenario, in the order they run. func Scenarios() []Scenario { return []Scenario{ - {Name: "handshake", Section: "7", Handshake: true, - Doc: "hello without waiting, then auth and resume in order; the auth signature verifies", + {Name: "handshake", Section: "10", Handshake: true, + Doc: "hello without waiting, naming the tunnel's key, then resume", run: func(s *session) error { return s.handshake(true) }}, - {Name: "version", Section: "7.1", Solo: true, + {Name: "version", Section: "10.1", Solo: true, Doc: "a hello with an unknown v gets err version and the connection closes", run: func(s *session) error { if err := s.send(s.hello(99)); err != nil { @@ -24,25 +25,39 @@ func Scenarios() []Scenario { } return expectErr(s, wire.ErrVersion, "after a hello with v 99") }}, - {Name: "auth-bad-sig", Section: "7.2", Solo: true, - Doc: "an auth whose signature does not verify gets err auth and the connection closes", + {Name: "hello-key", Section: "10.1", Solo: true, + Doc: "a hello naming a key other than the tunnel's gets err auth and the connection closes", run: func(s *session) error { - my, err := wire.Encode(s.hello(wire.Version)) + other, _, err := ed25519.GenerateKey(nil) if err != nil { return err } - if err := s.sendRaw(my); err != nil { + h := s.hello(wire.Version) + h.Key = wire.FormatKey(other) + if err := s.send(h); err != nil { return err } - if _, _, err := s.expect(wire.THello); err != nil { + return expectErr(s, wire.ErrAuth, "after a hello whose key is not the tunnel's") + }}, + {Name: "hello-own-key", Section: "10.1", Solo: true, + Doc: "a hello naming the receiver's own key gets err auth and the connection closes", + run: func(s *session) error { + _, line, err := s.expect(wire.THello) + if err != nil { return fmt.Errorf("waiting for hello: %w", err) } - if err := s.send(wire.Auth{Envelope: s.env(wire.TAuth), Sig: wire.SignAuth(s.r.priv, []byte("wrong"), []byte("transcript"))}); err != nil { + var theirs wire.Hello + if err := wire.Decode(line, &theirs); err != nil { + return err + } + h := s.hello(wire.Version) + h.Key = theirs.Key + if err := s.send(h); err != nil { return err } - return expectErr(s, wire.ErrAuth, "after an auth with a bad signature") + return expectErr(s, wire.ErrAuth, "after a hello carrying the peer's own key") }}, - {Name: "bad-frame", Section: "5", Solo: true, + {Name: "bad-frame", Section: "8", Solo: true, Doc: "a line that is not a JSON object gets err bad_frame and the connection closes", run: func(s *session) error { if err := s.handshake(false); err != nil { @@ -53,7 +68,7 @@ func Scenarios() []Scenario { } return expectErr(s, wire.ErrBadFrame, "after a line that is not JSON") }}, - {Name: "too-large", Section: "5", Solo: true, + {Name: "too-large", Section: "8", Solo: true, Doc: "a line over 1 MiB gets err too_large and the connection closes", run: func(s *session) error { if err := s.handshake(false); err != nil { @@ -68,7 +83,7 @@ func Scenarios() []Scenario { } return expectErr(s, wire.ErrTooLarge, "after a line over 1 MiB") }}, - {Name: "ping", Section: "9.4", + {Name: "ping", Section: "12.4", Doc: "a ping is answered by a pong whose re is the ping's id", run: func(s *session) error { ping := wire.Ping{Envelope: s.env(wire.TPing)} @@ -82,7 +97,7 @@ func Scenarios() []Scenario { s.check(env.Re == ping.ID, "pong.re is the ping's id (got %q)", env.Re) return nil }}, - {Name: "msg-ack", Section: "9.1, 9.2", + {Name: "msg-ack", Section: "12.1, 12.2", Doc: "a msg with text, code and data parts is acked, with the msg's id and thread", run: func(s *session) error { m := wire.Msg{Envelope: s.env(wire.TMsg), Subject: "conformance", Parts: []wire.Part{ @@ -93,7 +108,7 @@ func Scenarios() []Scenario { m.Th = "conform-" + m.ID return expectAck(s, m.Envelope, m) }}, - {Name: "state-ack", Section: "8.1, 9.2", + {Name: "state-ack", Section: "11.1, 12.2", Doc: "a state is acked like a msg", run: func(s *session) error { m := wire.Msg{Envelope: s.env(wire.TMsg), Subject: "conformance state", Parts: text("state follows")} @@ -105,7 +120,7 @@ func Scenarios() []Scenario { st.Th = m.Th return expectAck(s, st.Envelope, st) }}, - {Name: "dedup", Section: "9.5", + {Name: "dedup", Section: "12.5", Doc: "a msg delivered twice with the same id is acked both times", run: func(s *session) error { m := wire.Msg{Envelope: s.env(wire.TMsg), Subject: "conformance dedup", Parts: text("once")} @@ -124,7 +139,7 @@ func Scenarios() []Scenario { s.check(env.Re == m.ID, "the replayed msg is acked again (ack.re %q)", env.Re) return nil }}, - {Name: "unknown-type", Section: "5", + {Name: "unknown-type", Section: "8", Doc: "a message of an unknown type is ignored, or answered with a non-closing err unsupported", run: func(s *session) error { line := []byte(`{"t":"conformance-future","id":"` + s.r.ids.New() + `","ts":"` + wire.Now() + `","payload":[1,2,3]}`) @@ -154,7 +169,7 @@ func Scenarios() []Scenario { s.check(env.Re == ping.ID, "connection survives an unknown message type: pong answers the ping") return nil }}, - {Name: "unknown-field", Section: "5", + {Name: "unknown-field", Section: "8", Doc: "unknown fields in a msg and its parts are ignored; the msg is acked", run: func(s *session) error { id, th := s.r.ids.New(), "conform-fields" @@ -165,7 +180,7 @@ func Scenarios() []Scenario { } return expectAck(s, wire.Envelope{T: wire.TMsg, ID: id, Th: th}) }}, - {Name: "err-nonclosing", Section: "9.7", + {Name: "err-nonclosing", Section: "12.7", Doc: "an err with a non-closing code (unsupported) leaves the connection open", run: func(s *session) error { if err := s.send(wire.Err{Envelope: s.env(wire.TErr), Code: wire.ErrUnsupported, Detail: "conformance: a non-closing err"}); err != nil { @@ -173,7 +188,7 @@ func Scenarios() []Scenario { } return s.alive("err unsupported") }}, - {Name: "chunk", Section: "9.3", Needs: "blob", + {Name: "chunk", Section: "12.3", Needs: "blob", Doc: "a msg with a blob part, followed by the blob in two chunks, is acked and refused only with blob_refused", run: func(s *session) error { data := []byte("hello, conformance runner\n") @@ -205,7 +220,7 @@ func Scenarios() []Scenario { } return s.alive("the chunks") }}, - {Name: "grant", Section: "10.3", Needs: "grant", + {Name: "grant", Section: "13.3", Needs: "grant", Doc: "a grant message carrying a grant to the peer is accepted without an err", run: func(s *session) error { g, err := wire.MintGrant(s.r.priv, s.r.peer.Key, []string{"fs:read"}, time.Now().Add(time.Hour), "") @@ -221,7 +236,7 @@ func Scenarios() []Scenario { } return s.alive("the grant") }}, - {Name: "bye", Section: "9.6", + {Name: "bye", Section: "12.6", Doc: "a bye is answered with a bye and the connection closes", run: func(s *session) error { if err := s.send(wire.Bye{Envelope: s.env(wire.TBye), Reason: "conformance done"}); err != nil { diff --git a/go.mod b/go.mod index fa427ed..fab48b8 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,19 @@ go 1.27.1 require ( charm.land/huh/v2 v2.0.3 charm.land/lipgloss/v2 v2.0.6 + filippo.io/edwards25519 v1.2.0 + github.com/coder/websocket v1.8.14 + github.com/fxamacker/cbor/v2 v2.9.0 github.com/invopop/jsonschema v0.14.0 github.com/pb33f/ordered-map/v2 v2.3.1 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 github.com/spf13/pflag v1.0.10 github.com/tailscale/tailcat v0.7.0 + github.com/tailscale/wireguard-go v0.0.0-20260911194433-e3222a3340cd + golang.org/x/crypto v0.57.0 golang.org/x/term v0.46.0 golang.org/x/text v0.42.0 + gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 modernc.org/sqlite v1.59.0 tailscale.com v1.103.0-pre.0.20260916030321-a2263542f260 ) @@ -19,7 +25,6 @@ require ( require ( charm.land/bubbles/v2 v2.2.1 // indirect charm.land/bubbletea/v2 v2.0.10 // indirect - filippo.io/edwards25519 v1.2.0 // indirect github.com/akutz/memconn v0.1.0 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/atotto/clipboard v0.1.4 // indirect @@ -36,12 +41,10 @@ require ( github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/coder/websocket v1.8.14 // indirect github.com/creachadair/msync v0.8.1 // indirect github.com/creack/pty v1.1.24 // indirect github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gaissmai/bart v0.26.1 // indirect github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect @@ -73,14 +76,12 @@ require ( github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd // indirect github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc // indirect github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 // indirect - github.com/tailscale/wireguard-go v0.0.0-20260911194433-e3222a3340cd // indirect github.com/u-root/u-root v0.14.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20260823151212-3075585bcbeb // indirect - golang.org/x/crypto v0.57.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/net v0.59.0 // indirect golang.org/x/sync v0.23.0 // indirect @@ -88,7 +89,6 @@ require ( golang.org/x/time v0.15.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard/windows v0.5.3 // indirect - gvisor.dev/gvisor v0.0.0-20260915211658-a6f909f08a72 // indirect modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect diff --git a/internal/api/api.go b/internal/api/api.go index 1ed8964..e1924c8 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -4,6 +4,7 @@ package api import ( "encoding/json" + "strings" "time" "github.com/agentwireprotocol/awp/store" @@ -12,44 +13,63 @@ import ( // Status describes the running daemon. type Status struct { - Key string `json:"key"` - Short string `json:"short"` - Name string `json:"name"` - About string `json:"about"` - Harness string `json:"harness,omitempty"` // the agent harness this daemon runs in - ShareWith []PeerRef `json:"share_with,omitempty"` // hosts this agent mirrors its conversations to - Model string `json:"model,omitempty"` // the model the agent runs on, as last reported - Active *time.Time `json:"active,omitempty"` // when the agent last acted through awp - Waiting bool `json:"waiting,omitempty"` // blocked in awp wait now - Host string `json:"host,omitempty"` // this machine's hostname - Fingerprint string `json:"fingerprint"` - Version string `json:"version"` - Home string `json:"home"` - PID int `json:"pid"` - Started time.Time `json:"started"` - Addresses []string `json:"addresses"` - Tailcat string `json:"tailcat,omitempty"` // bare tc... address to share - TailcatErr string `json:"tailcat_error,omitempty"` - TailcatWant bool `json:"tailcat_enabled"` - Presence bool `json:"presence,omitempty"` // publishing presence (see NOTES.md) - Serve []string `json:"serve,omitempty"` - Accept string `json:"accept"` - Peers []PeerView `json:"peers"` - Unread int `json:"unread"` // messages and states from peers not yet read - Notices int `json:"notices"` // unread local notices (connection, sharing and file events) - Outbox int `json:"outbox"` + Key string `json:"key"` + Short string `json:"short"` + Name string `json:"name"` + About string `json:"about"` + Harness string `json:"harness,omitempty"` // the agent harness this daemon runs in + ShareWith []PeerRef `json:"share_with,omitempty"` // hosts this agent mirrors its conversations to + Model string `json:"model,omitempty"` // the model the agent runs on, as last reported + Active *time.Time `json:"active,omitempty"` // when the agent last acted through awp + Waiting bool `json:"waiting,omitempty"` // blocked in awp wait now + Host string `json:"host,omitempty"` // this machine's hostname + Fingerprint string `json:"fingerprint"` + Version string `json:"version"` + Home string `json:"home"` + PID int `json:"pid"` + Started time.Time `json:"started"` + Address string `json:"address,omitempty"` // the awp1 address to share, once a carrier is up + Endpoints []string `json:"endpoints"` // kind:value, one per carrier endpoint in the address + Listen []string `json:"listen"` // the carriers configured + Pending map[string]string `json:"pending,omitempty"` // carriers still coming up, and their last error + Presence bool `json:"presence,omitempty"` // publishing presence (see NOTES.md) + Serve []string `json:"serve,omitempty"` + Accept string `json:"accept"` + Peers []PeerView `json:"peers"` + Unread int `json:"unread"` // messages and states from peers not yet read + Notices int `json:"notices"` // unread local notices (connection, sharing and file events) + Outbox int `json:"outbox"` } -// ShareAddress is the address to hand to another agent: the tailcat -// address when there is one, else the first listen address. -func (s *Status) ShareAddress() string { - if s.Tailcat != "" { - return s.Tailcat +// ShareAddress is the address to hand to another agent. +func (s *Status) ShareAddress() string { return s.Address } + +// Settled reports whether every configured carrier is up, or has failed +// while others are up: a caller waiting for an address can stop. +func (s *Status) Settled() bool { + if len(s.Pending) == 0 { + return true + } + if s.Address == "" { + return false + } + for _, e := range s.Pending { + if e == "" { + return false + } } - if len(s.Addresses) > 0 { - return s.Addresses[0] + return true +} + +// PendingError summarizes why carriers are not up yet. +func (s *Status) PendingError() string { + var out []string + for k, e := range s.Pending { + if e != "" { + out = append(out, k+": "+e) + } } - return "" + return strings.Join(out, "; ") } // PeerView is a peer as the daemon sees it right now. diff --git a/internal/daemon/config.go b/internal/daemon/config.go index c63a2d2..522b63f 100644 --- a/internal/daemon/config.go +++ b/internal/daemon/config.go @@ -3,7 +3,7 @@ // // One daemon per awp home (~/.awp by default) owns the identity key, // the tailcat listener and every connection, so switching harness mid-task -// keeps the identity and the conversations (spec section 15, question 1). +// keeps the identity and the conversations (spec section 18, question 1). package daemon import ( @@ -37,9 +37,8 @@ type Config struct { OutboxTTL string `json:"outbox_ttl,omitempty"` Policy node.Policy `json:"policy,omitzero"` Trace bool `json:"trace,omitempty"` - Plaintext bool `json:"allow_plaintext,omitempty"` // plain TCP to public addresses - Verbose bool `json:"verbose,omitempty"` // include tailcat's own logs - Presence bool `json:"presence,omitempty"` // publish signed presence (NOTES.md) + Verbose bool `json:"verbose,omitempty"` // include tailcat's own logs + Presence bool `json:"presence,omitempty"` // publish signed presence (NOTES.md) } // DefaultHome is $AWP_HOME or ~/.awp. @@ -134,9 +133,6 @@ func LoadConfig(home string) (Config, error) { if v, ok := env("AWP_PRESENCE"); ok { cfg.Presence = v != "0" && v != "false" } - if v, ok := env("AWP_ALLOW_PLAINTEXT"); ok && v != "0" { - cfg.Plaintext = true - } if len(cfg.Listen) == 0 { cfg.Listen = []string{"tailcat"} } @@ -159,8 +155,7 @@ func (c Config) nodeConfig() (node.Config, error) { Policy: c.Policy, Trace: c.Trace, - AllowPlaintext: c.Plaintext, - Presence: c.Presence, + Presence: c.Presence, } if c.PingInterval != "" { d, err := time.ParseDuration(c.PingInterval) diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index cd9308f..3fafc7f 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -20,7 +20,7 @@ import ( "github.com/agentwireprotocol/awp/internal/version" "github.com/agentwireprotocol/awp/node" "github.com/agentwireprotocol/awp/store" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) @@ -132,6 +132,11 @@ func (d *Daemon) Call(ctx context.Context, method string, params json.RawMessage return map[string]bool{"ok": true}, nil case "status": return d.status() + case "rotate_psk": + if err := d.n.RotatePSK(); err != nil { + return nil, err + } + return map[string]bool{"ok": true}, nil case "mirrored": return d.n.Store().MirroredThreads() case "set_share": @@ -299,7 +304,6 @@ func (d *Daemon) Stream(ctx context.Context, method string, params json.RawMessa func (d *Daemon) status() (*api.Status, error) { n := d.n st := n.Store() - tc, tcErr := n.TailcatAddress() peers, err := d.peerViews() if err != nil { return nil, err @@ -336,10 +340,10 @@ func (d *Daemon) status() (*api.Status, error) { Home: d.cfg.Home, PID: os.Getpid(), Started: d.started, - Addresses: n.Addresses(), - Tailcat: tc, - TailcatErr: tcErr, - TailcatWant: slices.Contains(d.cfg.Listen, "tailcat"), + Address: addrString(n.Address()), + Endpoints: endpoints(n.Address()), + Listen: d.cfg.Listen, + Pending: n.Pending(), Presence: n.SharesPresence(), Serve: d.cfg.Policy.Serve, Accept: d.cfg.Policy.Accept, @@ -430,10 +434,10 @@ func (d *Daemon) connect(ctx context.Context, p api.ConnectParams) (*api.Connect defer cancel() addr := p.Address // A known peer's name or key reconnects to its best known address. - if _, err := transport.Parse(addr); err != nil { + if !tunnel.IsAddress(addr) { key, rerr := d.n.ResolvePeer(addr) if rerr != nil { - return nil, fmt.Errorf("%v; and %v", err, rerr) + return nil, fmt.Errorf("%q is not an address (awp1...) and %v", addr, rerr) } pv := d.view(key) if pv.Connected { @@ -442,15 +446,10 @@ func (d *Daemon) connect(ctx context.Context, p api.ConnectParams) (*api.Connect if len(pv.Addrs) == 0 { return nil, fmt.Errorf("no known address for %s", pv.Label()) } - var lastErr error - for _, a := range pv.Addrs { - if k, err := d.n.Connect(ctx, a.Addr); err == nil && k == key { - return d.connected(k, start), nil - } else if err != nil { - lastErr = err - } + if err := d.n.ConnectKey(ctx, key); err != nil { + return nil, err } - return nil, lastErr + return d.connected(key, start), nil } key, err := d.n.Connect(ctx, addr) if err != nil { @@ -478,7 +477,7 @@ func (d *Daemon) resolve(ctx context.Context, peer, th string) (string, error) { } return d.n.ResolveThread(th) } - if _, err := transport.Parse(peer); err == nil { + if tunnel.IsAddress(peer) { res, err := d.connect(ctx, api.ConnectParams{Address: peer}) if err != nil { return "", err @@ -819,3 +818,19 @@ func (d *Daemon) shareRefs() []api.PeerRef { } return refs } + +// addrString is the address to share, or "" while no carrier is up. +func addrString(a tunnel.Address) string { + if len(a.Endpoints) == 0 { + return "" + } + return a.String() +} + +func endpoints(a tunnel.Address) []string { + out := []string{} + for _, e := range a.Endpoints { + out = append(out, e.String()) + } + return out +} diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index b57e8e2..27aafba 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "os" "path/filepath" "testing" "time" @@ -39,7 +40,11 @@ func TestTouches(t *testing.T) { // testDaemon serves a node listening on a Unix socket in its own home. func testDaemon(t *testing.T, name string) *Daemon { t.Helper() - home := t.TempDir() + home, err := os.MkdirTemp("", "awpd") // short: unix socket paths are limited + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(home) }) n, err := node.Open(node.Config{ Home: home, Name: name, @@ -67,7 +72,7 @@ func testDaemon(t *testing.T, name string) *Daemon { func TestConnectExisting(t *testing.T) { a, b := testDaemon(t, "a"), testDaemon(t, "b") ctx := context.Background() - addr := b.n.Addresses()[0] + addr := b.n.Address().String() res, err := a.connect(ctx, api.ConnectParams{Address: addr}) if err != nil { t.Fatal(err) @@ -94,7 +99,7 @@ func TestConnectExisting(t *testing.T) { func TestReadNoMark(t *testing.T) { a, b := testDaemon(t, "a"), testDaemon(t, "b") ctx := context.Background() - if _, err := a.connect(ctx, api.ConnectParams{Address: b.n.Addresses()[0]}); err != nil { + if _, err := a.connect(ctx, api.ConnectParams{Address: b.n.Address().String()}); err != nil { t.Fatal(err) } res, err := a.n.Send(node.SendRequest{Peer: b.n.Key(), Subject: "Read", Parts: []wire.Part{{K: wire.PartText, Text: "needle"}}}) @@ -131,7 +136,7 @@ func link(t *testing.T, from, to *Daemon) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if _, err := from.connect(ctx, api.ConnectParams{Address: to.n.Addresses()[0]}); err != nil { + if _, err := from.connect(ctx, api.ConnectParams{Address: to.n.Address().String()}); err != nil { t.Fatalf("connect %s -> %s: %v", from.n.Name(), to.n.Name(), err) } waitFor(t, to.n, "inbound connection", func() bool { return to.n.Connected(from.n.Key()) }) diff --git a/internal/helper/helper.go b/internal/helper/helper.go new file mode 100644 index 0000000..13e370a --- /dev/null +++ b/internal/helper/helper.go @@ -0,0 +1,413 @@ +// Package helper is `awp tunnel`: the WireGuard tunnel of SPEC.md sections +// 4 to 7 as a separate process, for SDKs in languages without a WireGuard +// implementation (section 18.2). The SDK keeps its identity and speaks the +// protocol from section 8 on; the helper moves its streams. +// +// One helper serves one peer. It prints JSON lines on stdout: a "ready" +// event, then an "address" event each time the address changes. It exits +// when stdin closes, so it never outlives the SDK that started it. +// +// Requests: the SDK connects to the control socket and writes one line. +// +// {"dial": ["awp1...", ...], "key": "ed25519:..."} +// +// dials a peer (addresses of one peer, and/or a key the tunnel already +// knows); the helper answers {"ok":true,"key":...,"via":...} or +// {"ok":false,"error":...}, and the connection becomes the stream. +// +// {"listen": "tailcat"} +// +// adds a carrier and answers, once it is up, {"ok":true,"address":..., +// "public":...}. +// +// {"rotate": true} +// +// replaces the listener's pre-shared key and answers with the new address. +// +// Accepting: each stream a peer opens is connected to the SDK's forward +// socket. The helper first writes one line of its own, +// +// {"tunnel": {"remote": ""}} +// +// so the SDK knows which grants to present in its hello before the peer's +// hello arrives; then it splices the stream unchanged. A dial reply carries +// the same "remote". +// +// In both directions the helper reads the peer's first line and, if it is +// a hello whose key is not the tunnel's, sends err auth and closes, so the +// SDK can trust hello.key as section 10.1 requires. +package helper + +import ( + "bufio" + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "strings" + "sync" + "time" + + "github.com/agentwireprotocol/awp/tunnel" + "github.com/agentwireprotocol/awp/wire" +) + +// Config configures a helper. +type Config struct { + Identity ed25519.PrivateKey + StateDir string // pre-shared keys and the tailcat key; empty keeps them in memory + Listen []string // carriers, as tunnel.Config.Listen + Socket string // the control socket to create + Forward string // where inbound streams go: a unix socket path, or tcp:HOST:PORT + Logf func(string, ...any) +} + +// Event is one line on stdout. +type Event struct { + Event string `json:"event"` // ready or address + Key string `json:"key"` + Address string `json:"address,omitempty"` // with the pre-shared key: the one to share + Public string `json:"public,omitempty"` // without: for hello.addr + Socket string `json:"socket,omitempty"` +} + +// Run serves until ctx ends or stdin closes. +func Run(ctx context.Context, cfg Config, stdin io.Reader, stdout io.Writer) error { + if cfg.Logf == nil { + cfg.Logf = func(string, ...any) {} + } + if cfg.Socket == "" { + return errors.New("no control socket") + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + key := wire.FormatKey(cfg.Identity.Public().(ed25519.PublicKey)) + + var outMu sync.Mutex + emit := func(ev Event) { + outMu.Lock() + defer outMu.Unlock() + b, _ := json.Marshal(ev) + stdout.Write(append(b, '\n')) + } + var tu *tunnel.Tunnel + changed := make(chan struct{}, 1) + tu, err := tunnel.New(tunnel.Config{ + Identity: cfg.Identity, + StateDir: cfg.StateDir, + Listen: cfg.Listen, + Logf: cfg.Logf, + Changed: func() { + select { + case changed <- struct{}{}: + default: + } + }, + }) + if err != nil { + return err + } + defer tu.Close() + if err := tu.Start(); err != nil { + return err + } + os.Remove(cfg.Socket) + ln, err := net.Listen("unix", cfg.Socket) + if err != nil { + return err + } + defer ln.Close() + defer os.Remove(cfg.Socket) + + addrEvent := func(name string) Event { + ev := Event{Event: name, Key: key, Socket: cfg.Socket} + if a := tu.Address(); len(a.Endpoints) > 0 { + ev.Address, ev.Public = a.String(), a.Public().String() + } + return ev + } + emit(addrEvent("ready")) + go func() { + for { + select { + case <-ctx.Done(): + return + case <-changed: + emit(addrEvent("address")) + } + } + }() + + // Stdin closing means the SDK is gone. + if stdin != nil { + go func() { + io.Copy(io.Discard, stdin) + cancel() + }() + } + go func() { + <-ctx.Done() + ln.Close() + tu.Close() + }() + + go func() { + for { + s, err := tu.Accept() + if err != nil { + return + } + go forward(cfg, s) + } + }() + for { + c, err := ln.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return err + } + go dial(ctx, cfg, tu, c) + } +} + +type request struct { + Dial []string `json:"dial,omitempty"` + Key string `json:"key,omitempty"` + Listen string `json:"listen,omitempty"` + Rotate bool `json:"rotate,omitempty"` +} + +type dialReply struct { + OK bool `json:"ok"` + Key string `json:"key,omitempty"` + Remote string `json:"remote,omitempty"` + Via string `json:"via,omitempty"` + Address string `json:"address,omitempty"` + Public string `json:"public,omitempty"` + Error string `json:"error,omitempty"` +} + +func dial(ctx context.Context, cfg Config, tu *tunnel.Tunnel, c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + c.SetReadDeadline(time.Now().Add(30 * time.Second)) + line, err := br.ReadBytes('\n') + if err != nil { + return + } + c.SetReadDeadline(time.Time{}) + reply := func(r dialReply) error { + b, _ := json.Marshal(r) + _, err := c.Write(append(b, '\n')) + return err + } + var req request + if err := json.Unmarshal(line, &req); err != nil { + reply(dialReply{Error: "bad request: " + err.Error()}) + return + } + switch { + case req.Listen != "": + reply(listen(ctx, tu, req.Listen)) + return + case req.Rotate: + if err := tu.RotatePSK(); err != nil { + reply(dialReply{Error: err.Error()}) + return + } + reply(addressReply(tu)) + return + } + var addrs []tunnel.Address + for _, s := range req.Dial { + a, err := tunnel.ParseAddress(s) + if err != nil { + reply(dialReply{Error: err.Error()}) + return + } + addrs = append(addrs, a) + } + if req.Key != "" { + pub, err := wire.ParseKey(req.Key) + if err != nil { + reply(dialReply{Error: err.Error()}) + return + } + addrs = append(addrs, tunnel.Address{Key: pub}) + } + if len(addrs) == 0 { + reply(dialReply{Error: "nothing to dial"}) + return + } + dctx, cancel := context.WithTimeout(ctx, 45*time.Second) + s, err := tu.Dial(dctx, addrs...) + cancel() + if err != nil { + reply(dialReply{Error: err.Error()}) + return + } + defer s.Close() + if err := reply(dialReply{OK: true, Key: addrs[0].KeyString(), Remote: s.Remote.String(), Via: s.Via()}); err != nil { + return + } + splice(cfg, &bufConn{Conn: c, r: br}, s) +} + +func addressReply(tu *tunnel.Tunnel) dialReply { + r := dialReply{OK: true} + if a := tu.Address(); len(a.Endpoints) > 0 { + r.Address, r.Public = a.String(), a.Public().String() + } + return r +} + +// listen adds a carrier and waits, up to 90 seconds, for it to come up. +func listen(ctx context.Context, tu *tunnel.Tunnel, spec string) dialReply { + if err := tu.Listen(spec); err != nil { + return dialReply{Error: err.Error()} + } + deadline := time.Now().Add(90 * time.Second) + for { + e, pending := tu.Pending()[spec] + switch { + case !pending: + return addressReply(tu) + case e != "": + return dialReply{Error: spec + ": " + e} + case time.Now().After(deadline): + return dialReply{Error: spec + ": not up after 90s"} + } + select { + case <-ctx.Done(): + return dialReply{Error: "shutting down"} + case <-time.After(100 * time.Millisecond): + } + } +} + +func forward(cfg Config, s *tunnel.Stream) { + defer s.Close() + network, target := "unix", cfg.Forward + if t, ok := strings.CutPrefix(cfg.Forward, "tcp:"); ok { + network, target = "tcp", t + } + if target == "" { + return + } + c, err := net.DialTimeout(network, target, 10*time.Second) + if err != nil { + cfg.Logf("forward to %s: %v", cfg.Forward, err) + return + } + defer c.Close() + pre, _ := json.Marshal(map[string]any{"tunnel": map[string]string{"remote": s.Remote.String()}}) + if _, err := c.Write(append(pre, '\n')); err != nil { + return + } + splice(cfg, c, s) +} + +// splice copies both ways between the SDK's end and the tunnel stream, +// checking that the peer's first line, if it is a hello, names the +// tunnel's key. +func splice(cfg Config, local net.Conn, s *tunnel.Stream) { + done := make(chan struct{}, 2) + go func() { + io.Copy(s, local) + s.CloseWrite() + done <- struct{}{} + }() + go func() { + defer func() { done <- struct{}{} }() + defer closeWrite(local) + r := bufio.NewReaderSize(s, 64<<10) + first, err := r.ReadSlice('\n') + if len(first) > 0 { + if msg := badHello(first, s); msg != "" { + cfg.Logf("closing a stream: %s", msg) + e, _ := wire.Encode(wire.Err{Envelope: wire.Envelope{T: wire.TErr, ID: wire.NewIDGen().New(), TS: wire.Now()}, Code: wire.ErrAuth, Detail: msg}) + s.Write(append(e, '\n')) + s.Close() + local.Close() + return + } + if _, err := local.Write(first); err != nil { + return + } + } + if err != nil && !errors.Is(err, bufio.ErrBufferFull) { + return + } + io.Copy(local, r) + }() + <-done + <-done +} + +// badHello says what is wrong with a first line that is a hello naming a +// key other than the tunnel's, or "". +func badHello(line []byte, s *tunnel.Stream) string { + var h struct { + T string `json:"t"` + Key string `json:"key"` + } + if json.Unmarshal(line, &h) != nil || h.T != wire.THello { + return "" + } + pub, err := wire.ParseKey(h.Key) + if err != nil { + return "" // the SDK answers a malformed key itself + } + if !s.Is(pub) { + return fmt.Sprintf("hello key %s is not the key on the other end of the tunnel", wire.ShortKey(h.Key)) + } + return "" +} + +type bufConn struct { + net.Conn + r *bufio.Reader +} + +func (b *bufConn) Read(p []byte) (int, error) { return b.r.Read(p) } + +func closeWrite(c net.Conn) { + if cw, ok := c.(interface{ CloseWrite() error }); ok { + cw.CloseWrite() + return + } + if b, ok := c.(*bufConn); ok { + closeWrite(b.Conn) + } +} + +// LoadIdentity reads an identity file as the SDKs and the daemon write it: +// JSON with "seed", 32 bytes of base64url. +func LoadIdentity(path string) (ed25519.PrivateKey, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f struct { + Seed string `json:"seed"` + } + if err := json.Unmarshal(b, &f); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return SeedKey(f.Seed) +} + +// SeedKey decodes a base64 seed into a key. +func SeedKey(seed string) (ed25519.PrivateKey, error) { + raw, err := wire.DecodeB64(strings.TrimSpace(seed)) + if err != nil || len(raw) != ed25519.SeedSize { + return nil, errors.New("identity seed must be 32 bytes of base64") + } + return ed25519.NewKeyFromSeed(raw), nil +} diff --git a/internal/helper/helper_test.go b/internal/helper/helper_test.go new file mode 100644 index 0000000..079d577 --- /dev/null +++ b/internal/helper/helper_test.go @@ -0,0 +1,303 @@ +package helper + +import ( + "bufio" + "context" + "crypto/ed25519" + "encoding/json" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/agentwireprotocol/awp/node" + "github.com/agentwireprotocol/awp/tunnel" + "github.com/agentwireprotocol/awp/wire" +) + +func tempDir(t *testing.T) string { + d, err := os.MkdirTemp("", "awph") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(d) }) + return d +} + +func startNode(t *testing.T) *node.Node { + home := tempDir(t) + n, err := node.Open(node.Config{Home: home, Name: "go@test", Listen: []string{"unix:" + filepath.Join(home, "s")}, + HandshakeTimeout: 5 * time.Second, MaxBackoff: time.Second, Logf: func(f string, a ...any) { t.Logf("[node] "+f, a...) }}) + if err != nil { + t.Fatal(err) + } + if err := n.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { n.Close() }) + return n +} + +type sdk struct { + t *testing.T + key string + events *bufio.Scanner + ready Event + socket string + fwd net.Listener +} + +// startHelper runs a helper in-process, as the SDK would run `awp tunnel`. +func startHelper(t *testing.T, listen ...string) *sdk { + dir := tempDir(t) + _, priv, _ := ed25519.GenerateKey(nil) + fwd, err := net.Listen("unix", filepath.Join(dir, "fwd.sock")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { fwd.Close() }) + outR, outW := io.Pipe() + inR, inW := io.Pipe() + cfg := Config{Identity: priv, StateDir: dir, Listen: listen, Socket: filepath.Join(dir, "ctl.sock"), Forward: filepath.Join(dir, "fwd.sock"), Logf: t.Logf} + done := make(chan error, 1) + go func() { done <- Run(context.Background(), cfg, inR, outW) }() + t.Cleanup(func() { + inW.Close() // stdin closing stops the helper + select { + case err := <-done: + if err != nil { + t.Errorf("helper: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("helper did not stop when stdin closed") + } + }) + s := &sdk{t: t, key: wire.FormatKey(priv.Public().(ed25519.PublicKey)), events: bufio.NewScanner(outR), socket: cfg.Socket, fwd: fwd} + s.ready = s.event() + if s.ready.Event != "ready" || s.ready.Key != s.key { + t.Fatalf("first event %+v", s.ready) + } + return s +} + +func (s *sdk) event() Event { + s.t.Helper() + if !s.events.Scan() { + s.t.Fatal("helper stdout ended") + } + var ev Event + if err := json.Unmarshal(s.events.Bytes(), &ev); err != nil { + s.t.Fatal(err) + } + return ev +} + +func (s *sdk) hello() string { + return fmt.Sprintf(`{"t":"hello","id":"%s","ts":"%s","v":%d,"key":"%s","name":"sdk@test","caps":["chat"]}`, wire.NewIDGen().New(), wire.Now(), wire.Version, s.key) +} + +func readType(t *testing.T, r *bufio.Reader, want string) []byte { + t.Helper() + line, err := r.ReadBytes('\n') + if err != nil { + t.Fatalf("waiting for %s: %v", want, err) + } + env, err := wire.ParseEnvelope(line) + if err != nil || env.T != want { + t.Fatalf("want %s, got %s", want, line) + } + return line +} + +func TestDialThroughHelper(t *testing.T) { + n := startNode(t) + s := startHelper(t) + c, err := net.Dial("unix", s.socket) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(30 * time.Second)) + fmt.Fprintf(c, `{"dial":["%s"]}`+"\n", n.Address()) + r := bufio.NewReader(c) + line, _ := r.ReadBytes('\n') + var rep dialReply + if json.Unmarshal(line, &rep); !rep.OK || rep.Key != n.Key() { + t.Fatalf("dial reply %s", line) + } + fmt.Fprintln(c, s.hello()) + readType(t, r, wire.THello) + readType(t, r, wire.TResume) + fmt.Fprintf(c, `{"t":"resume","id":"r1","ts":"%s","seen":{}}`+"\n", wire.Now()) + fmt.Fprintf(c, `{"t":"msg","id":"%s","ts":"%s","th":"t1","parts":[{"k":"text","text":"through the helper"}]}`+"\n", wire.NewIDGen().New(), wire.Now()) + readType(t, r, wire.TAck) + if !n.Connected(s.key) { + t.Fatal("node does not see the SDK's key") + } + + // A bad request is answered, not hung up on. + c2, _ := net.Dial("unix", s.socket) + defer c2.Close() + fmt.Fprintln(c2, `{"dial":["tcp:127.0.0.1:1"]}`) + line, _ = bufio.NewReader(c2).ReadBytes('\n') + if json.Unmarshal(line, &rep); rep.OK || rep.Error == "" { + t.Fatalf("bad dial reply %s", line) + } +} + +func TestAcceptThroughHelper(t *testing.T) { + s := startHelper(t, "unix:"+filepath.Join(tempDir(t), "h.sock")) + addr := s.ready.Address + if addr == "" { + addr = s.event().Address + } + a, err := tunnel.ParseAddress(addr) + if err != nil || a.PSK == nil || a.KeyString() != s.key { + t.Fatalf("helper address %q: %v", addr, err) + } + if p, _ := tunnel.ParseAddress(s.ready.Public); s.ready.Public != "" && p.PSK != nil { + t.Fatal("public address carries the psk") + } + + n := startNode(t) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err := n.Connect(ctx, addr); err != nil { + t.Logf("connect: %v", err) + } + }() + c, err := s.fwd.Accept() + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(30 * time.Second)) + r := bufio.NewReader(c) + pre, _ := r.ReadBytes('\n') + var p struct { + Tunnel struct{ Remote string } `json:"tunnel"` + } + nk, _ := wire.ParseKey(n.Key()) + want, _ := tunnel.X25519Public(nk) + if json.Unmarshal(pre, &p); p.Tunnel.Remote != want.String() { + t.Fatalf("preamble %s, want remote %s", pre, want) + } + fmt.Fprintln(c, s.hello()) + h := readType(t, r, wire.THello) + if !strings.Contains(string(h), n.Key()) { + t.Fatalf("hello %s", h) + } + readType(t, r, wire.TResume) + fmt.Fprintf(c, `{"t":"resume","id":"r1","ts":"%s","seen":{}}`+"\n", wire.Now()) + deadline := time.Now().Add(10 * time.Second) + for !n.Connected(s.key) { + if time.Now().After(deadline) { + t.Fatal("node never connected") + } + time.Sleep(50 * time.Millisecond) + } + + // A peer whose hello names another key is refused by the helper; the + // SDK sees the stream end without the lie. + _, priv, _ := ed25519.GenerateKey(nil) + tu, err := tunnel.New(tunnel.Config{Identity: priv}) + if err != nil { + t.Fatal(err) + } + defer tu.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + st, err := tu.Dial(ctx, a) + if err != nil { + t.Fatal(err) + } + defer st.Close() + other, _, _ := ed25519.GenerateKey(nil) + fmt.Fprintf(st, `{"t":"hello","id":"x","ts":"%s","v":%d,"key":"%s"}`+"\n", wire.Now(), wire.Version, wire.FormatKey(other)) + fc, err := s.fwd.Accept() + if err != nil { + t.Fatal(err) + } + defer fc.Close() + fc.SetDeadline(time.Now().Add(20 * time.Second)) + if b, _ := io.ReadAll(fc); strings.Contains(string(b), `"t":"hello"`) { + t.Fatalf("the SDK got the lying hello: %s", b) + } + st.SetDeadline(time.Now().Add(20 * time.Second)) + sr := bufio.NewReader(st) + for { + line, err := sr.ReadBytes('\n') + if err != nil { + t.Fatal("no err auth before close") + } + var e wire.Err + if json.Unmarshal(line, &e); e.T == wire.TErr { + if e.Code != wire.ErrAuth { + t.Fatalf("err %+v", e) + } + break + } + } +} + +func control(t *testing.T, s *sdk, req string) dialReply { + t.Helper() + c, err := net.Dial("unix", s.socket) + if err != nil { + t.Fatal(err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(30 * time.Second)) + fmt.Fprintln(c, req) + line, err := bufio.NewReader(c).ReadBytes('\n') + if err != nil { + t.Fatal(err) + } + var r dialReply + if err := json.Unmarshal(line, &r); err != nil { + t.Fatal(err) + } + return r +} + +// TestListenAndRotate: carriers added at run time, and a rotated +// pre-shared key. +func TestListenAndRotate(t *testing.T) { + s := startHelper(t) + if s.ready.Address != "" { + t.Fatalf("an address before listening: %q", s.ready.Address) + } + r := control(t, s, `{"listen":"udp:127.0.0.1:0"}`) + if !r.OK || r.Address == "" || r.Public == "" { + t.Fatalf("listen: %+v", r) + } + if ev := s.event(); ev.Event != "address" || ev.Address != r.Address { + t.Fatalf("address event %+v", ev) + } + r2 := control(t, s, fmt.Sprintf(`{"listen":"unix:%s"}`, filepath.Join(tempDir(t), "l.sock"))) + a, err := tunnel.ParseAddress(r2.Address) + if err != nil || len(a.Endpoints) != 2 { + t.Fatalf("two carriers: %+v %v", a, err) + } + if r := control(t, s, `{"listen":"nope:1"}`); r.OK || r.Error == "" { + t.Fatalf("bad carrier accepted: %+v", r) + } + rot := control(t, s, `{"rotate":true}`) + if !rot.OK || rot.Address == r2.Address || rot.Public != r2.Public { + t.Fatalf("rotate: %+v", rot) + } + n := startNode(t) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + go n.Connect(ctx, rot.Address) + c, err := s.fwd.Accept() + if err != nil { + t.Fatal(err) + } + c.Close() +} diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 3f02bb1..3f173cc 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -1,5 +1,5 @@ // Package mcp serves awp as a Model Context Protocol server over stdio, -// for harnesses that prefer tools to shell commands (spec section 15). The +// for harnesses that prefer tools to shell commands (spec section 18). The // tools call the same daemon the CLI does, so behavior is identical. // // When the client registers for Claude Code channel notifications (the @@ -213,7 +213,7 @@ func str(desc string) map[string]any { return map[string]any{"type": "string", " var tools = []tool{ {"awp_listen", "Start this machine's awp peer if needed and return its identity and the address to share with another agent (via your user or a task prompt). The address is a secret bearer credential for reaching you.", obj(map[string]any{})}, - {"awp_connect", "Connect to another agent's awp address (tc..., tcp:host:port or unix:/path), or reconnect to a known peer by name. The connection is kept alive and resumed after drops.", obj(map[string]any{"address": str("address the other agent shared, or a known peer's name")}, "address")}, + {"awp_connect", "Connect to another agent's awp address (awp1...), or reconnect to a known peer by name. The connection is kept alive and resumed after drops.", obj(map[string]any{"address": str("address the other agent shared, or a known peer's name")}, "address")}, {"awp_send", "Send a message to a peer. Without `thread`, opens a new thread; give it a `subject` that reads like a task title. To reply, pass the thread id. Queued (never fails) if the peer is away.", obj(map[string]any{ "peer": str("peer name, alias, key prefix or address; optional when thread is given"), @@ -289,8 +289,8 @@ func (s *Server) dispatch(ctx context.Context, c *control.Client, name string, r fmt.Fprintf(&b, "You are %s, key %s.\n", st.Name, st.Key) if addr := st.ShareAddress(); addr != "" { fmt.Fprintf(&b, "Address to share: %s\n", addr) - } else if st.TailcatErr != "" { - fmt.Fprintf(&b, "No address yet (tailcat: %s)\n", st.TailcatErr) + } else if e := st.PendingError(); e != "" { + fmt.Fprintf(&b, "No address yet (%s)\n", e) } if len(st.Peers) == 0 { b.WriteString("No peers yet.\n") @@ -450,7 +450,7 @@ func waitStatus(ctx context.Context, c *control.Client, forAddress bool) (*api.S if err := c.Call(ctx, "status", nil, &st); err != nil { return nil, err } - if !forAddress || st.Tailcat != "" || !st.TailcatWant || time.Now().After(deadline) { + if !forAddress || st.Settled() || time.Now().After(deadline) { return &st, nil } select { diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go index b576576..2bfa756 100644 --- a/internal/mcp/mcp_test.go +++ b/internal/mcp/mcp_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "io" + "os" "path/filepath" "strings" "testing" @@ -19,7 +20,11 @@ import ( // startDaemon runs a daemon with its own home, listening on a Unix socket. func startDaemon(t *testing.T, name string) (*control.Client, string) { t.Helper() - home := t.TempDir() + home, err := os.MkdirTemp("", "awpm") // short: unix socket paths are limited + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(home) }) sock := "unix:" + filepath.Join(home, "p.sock") cfg := daemon.Config{Home: home, Name: name, Listen: []string{sock}} errc := make(chan error, 1) @@ -36,7 +41,11 @@ func startDaemon(t *testing.T, name string) (*control.Client, string) { c.Call(context.Background(), "shutdown", nil, nil) <-errc }) - return c, sock + var st api.Status + if err := c.Call(context.Background(), "status", nil, &st); err != nil || st.Address == "" { + t.Fatalf("status: %v %+v", err, st) + } + return c, st.Address } // session is an MCP client talking to a Server over pipes. diff --git a/internal/schemagen/gen.go b/internal/schemagen/gen.go index af2a4b8..324a1b4 100644 --- a/internal/schemagen/gen.go +++ b/internal/schemagen/gen.go @@ -16,7 +16,7 @@ import ( // Module is the Go module the wire package lives in; ID is the schema's $id. const ( Module = "github.com/agentwireprotocol/awp" - ID = "https://agentwireprotocol.com/schema/v0/awp.schema.json" + ID = "https://agentwireprotocol.com/schema/v1/awp.schema.json" ) // required adds, per message type, the envelope fields the spec requires @@ -34,7 +34,7 @@ var required = map[string][]string{ // root, so that the wire package's comments can be read. func generate() (*Output, error) { r := &jsonschema.Reflector{ - // Unknown fields must be ignored (section 5), so no object is closed. + // Unknown fields must be ignored (section 8), so no object is closed. AllowAdditionalProperties: true, ExpandedStruct: true, Anonymous: true, @@ -82,7 +82,7 @@ func generate() (*Output, error) { variants = append(variants, ref(m.Type.Name())) } defs["Message"] = &jsonschema.Schema{ - Description: "One line of a connection: a message of SPEC.md sections 7 to 10, or one of the extensions marked x-extension. " + + Description: "One line of a connection: a message of SPEC.md sections 10 to 13, or one of the extensions marked x-extension. " + "A receiver ignores lines of a type it does not know, after checking the envelope.", OneOf: variants, } @@ -121,7 +121,7 @@ func generate() (*Output, error) { presence.Required = append(presence.Required, "sig") // omitempty in Go only for signing defs["Presence"] = presence - if err := point(defs, "Auth", "grants", "Grant", true); err != nil { + if err := point(defs, "Hello", "grants", "Grant", true); err != nil { return nil, err } if err := point(defs, "GrantMsg", "grant", "Grant", false); err != nil { @@ -159,7 +159,7 @@ func generate() (*Output, error) { root := &jsonschema.Schema{ Version: jsonschema.Version, ID: ID, - Title: "Agent Wire Protocol v0", + Title: "Agent Wire Protocol v1", Description: "The messages of the Agent Wire Protocol, one JSON object per line (SPEC.md). Every message is an envelope (t, id, ts, and th or re where they apply) with the fields of its type beside it.", Comments: "Generated from the wire package of " + Module + " by go generate ./wire. Do not edit by hand.", Ref: "#/$defs/Message", diff --git a/internal/schemagen/gen_test.go b/internal/schemagen/gen_test.go index 4594182..d925008 100644 --- a/internal/schemagen/gen_test.go +++ b/internal/schemagen/gen_test.go @@ -18,7 +18,7 @@ func TestGenerated(t *testing.T) { if err != nil { t.Fatal(err) } - for path, want := range map[string][]byte{"schema/v0/awp.schema.json": out.JSON, "schema/v0/schema.mdx": out.MDX} { + for path, want := range map[string][]byte{"schema/v1/awp.schema.json": out.JSON, "schema/v1/schema.mdx": out.MDX} { have, err := os.ReadFile(path) if err != nil { t.Fatal(err) diff --git a/internal/schemagen/main.go b/internal/schemagen/main.go index b68761b..38b87e1 100644 --- a/internal/schemagen/main.go +++ b/internal/schemagen/main.go @@ -2,7 +2,7 @@ // the wire package, which is the source of truth: every message type, its // fields, their constraints and the doc comments come from there. // -// go run ./internal/schemagen # writes schema/v0/awp.schema.json and schema.mdx +// go run ./internal/schemagen # writes schema/v1/awp.schema.json and schema.mdx // go run ./internal/schemagen -check # fails if either file is stale // // It must run from the repository root (go generate ./wire and make schema @@ -19,8 +19,8 @@ import ( ) func main() { - jsonOut := flag.String("json", "schema/v0/awp.schema.json", "schema file to write") - mdxOut := flag.String("mdx", "schema/v0/schema.mdx", "reference page to write") + jsonOut := flag.String("json", "schema/v1/awp.schema.json", "schema file to write") + mdxOut := flag.String("mdx", "schema/v1/schema.mdx", "reference page to write") check := flag.Bool("check", false, "write nothing; exit 1 if a file is not what would be generated") flag.Parse() diff --git a/internal/web/state.go b/internal/web/state.go index 09fa236..0f39bb9 100644 --- a/internal/web/state.go +++ b/internal/web/state.go @@ -110,8 +110,8 @@ type State struct { HostName string `json:"host_name"` Presence bool `json:"presence"` Address string `json:"address,omitempty"` - // Listeners are the addresses this host accepts connections on, and - // TailcatError why its tailcat listener is not up, if it is not. + // Listeners are the endpoints this host accepts connections on, and + // TailcatError why its tailcat carrier is not up, if it is not. Listeners []string `json:"listeners"` TailcatError string `json:"tailcat_error,omitempty"` Agents []Agent `json:"agents"` @@ -163,7 +163,7 @@ func realAbout(s string) string { // agent's presence. func buildState(st *api.Status, local []*store.Thread, net *api.Network, mirrored []store.MirroredThread, now time.Time) *State { s := &State{At: now, Version: st.Version, Self: st.Key, HostName: st.Name, Presence: st.Presence, Address: st.ShareAddress(), - Listeners: append([]string{}, st.Addresses...), TailcatError: st.TailcatErr, + Listeners: append([]string{}, st.Endpoints...), TailcatError: st.Pending["tailcat"], Agents: []Agent{}, Links: []Link{}, Threads: []Thread{}} if net == nil { net = &api.Network{} diff --git a/node/conn.go b/node/conn.go index b35bcd7..8d4ec76 100644 --- a/node/conn.go +++ b/node/conn.go @@ -13,6 +13,7 @@ import ( "github.com/agentwireprotocol/awp/internal/version" "github.com/agentwireprotocol/awp/store" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) @@ -23,6 +24,7 @@ var helloCaps = []string{"chat", "blob", "grant", "introduce", wire.TPresence, w // Conn is one connection to a peer, from hello to close. type Conn struct { n *Node + stream *tunnel.Stream raw net.Conn r *wire.Reader w *wire.Writer @@ -64,14 +66,15 @@ type ctrlItem struct { after func() // runs after the line is written } -func newConn(n *Node, raw net.Conn, outbound bool, via string) *Conn { +func newConn(n *Node, s *tunnel.Stream, outbound bool) *Conn { return &Conn{ n: n, - raw: raw, - r: wire.NewReader(raw), - w: wire.NewWriter(raw), + stream: s, + raw: s, + r: wire.NewReader(s), + w: wire.NewWriter(s), outbound: outbound, - via: via, + via: s.Via(), ctrl: make(chan ctrlItem, 4096), pumpCh: make(chan struct{}, 1), accepted: make(chan struct{}), @@ -150,25 +153,24 @@ func (e errPeerSaid) Error() string { return fmt.Sprintf("peer sent err %s: %s", e.e.Code, e.e.Detail) } -// handshake runs section 7: both sides send hello at once, then auth over -// the transcript of both hellos. +// handshake runs section 10: both sides send hello at once. The tunnel +// already authenticated the other side's key; hello must name that key. func (c *Conn) handshake() error { n := c.n c.raw.SetDeadline(time.Now().Add(n.cfg.HandshakeTimeout)) defer c.raw.SetDeadline(time.Time{}) - myHello, err := c.writeNow(wire.Hello{ + if _, err := c.writeNow(wire.Hello{ Envelope: wire.Envelope{T: wire.THello, ID: n.ids.New(), TS: wire.Now()}, V: wire.Version, Key: n.key, Name: n.cfg.Name, - Nonce: wire.Nonce(32), Caps: helloCaps, About: n.about(), Addr: n.advertised(), + Grants: n.grantsToPresent(c.stream), Shares: n.ShareWith(), - }) - if err != nil { + }); err != nil { return fmt.Errorf("send hello: %w", err) } @@ -191,33 +193,15 @@ func (c *Conn) handshake() error { return fmt.Errorf("bad hello key: %v", err) } if h.Key == n.key { - c.sendErrNow(wire.ErrAuth, h.ID, "connected to self") - return errors.New("connected to self") + c.sendErrNow(wire.ErrAuth, h.ID, "hello carries this peer's own key") + return errors.New("hello carries our own key") } - c.peerKey, c.peerPub, c.hello = h.Key, pub, h - - if _, err := c.writeNow(wire.Auth{ - Envelope: wire.Envelope{T: wire.TAuth, ID: n.ids.New(), TS: wire.Now()}, - Sig: wire.SignAuth(n.priv, myHello, peerHello), - Grants: n.grantsToPresent(h.Key), - }); err != nil { - return fmt.Errorf("send auth: %w", err) + if !c.stream.Is(pub) { + c.sendErrNow(wire.ErrAuth, h.ID, "hello key is not the key on the other end of the tunnel") + return fmt.Errorf("hello key %s does not match the tunnel", wire.ShortKey(h.Key)) } - - authLine, env, err := c.readHandshakeLine(wire.TAuth) - if err != nil { - return err - } - var a wire.Auth - if err := wire.Decode(authLine, &a); err != nil { - c.sendErrNow(wire.ErrBadFrame, env.ID, "malformed auth") - return fmt.Errorf("malformed auth: %v", err) - } - if err := wire.VerifyAuth(pub, a.Sig, peerHello, myHello); err != nil { - c.sendErrNow(wire.ErrAuth, a.ID, "auth signature does not verify") - return fmt.Errorf("auth from %s: %v", wire.ShortKey(h.Key), err) - } - c.peerGrants = a.Grants + c.peerKey, c.peerPub, c.hello, c.peerGrants = h.Key, pub, h, h.Grants + c.via = c.stream.Via() return nil } @@ -243,11 +227,7 @@ func (c *Conn) readHandshakeLine(want string) ([]byte, wire.Envelope, error) { return nil, env, errPeerSaid{e} } if env.T != want { - code := wire.ErrBadFrame - if want == wire.TAuth { - code = wire.ErrAuth - } - c.sendErrNow(code, env.ID, "expected "+want+", got "+env.T) + c.sendErrNow(wire.ErrBadFrame, env.ID, "expected "+want+", got "+env.T) return nil, env, fmt.Errorf("expected %s, got %q", want, env.T) } return line, env, nil @@ -435,7 +415,7 @@ func (c *Conn) writer() { } } -// pinger implements section 9.4: ping when idle, and two missed pongs mean +// pinger implements section 12.4: ping when idle, and two missed pongs mean // the connection is dead. Any received line counts as proof of life. func (c *Conn) pinger() { interval := c.n.cfg.PingInterval @@ -481,7 +461,7 @@ func (c *Conn) onPong(re string) { // RTT is the last measured round trip time, or zero. func (c *Conn) RTT() time.Duration { return time.Duration(c.rtt.Load()) } -// sendBye starts a graceful close (section 9.6): nothing else is sent, and +// sendBye starts a graceful close (section 12.6): nothing else is sent, and // the connection closes on the peer's bye or after five seconds. func (c *Conn) sendBye(reason string) { if c.byeSent.Swap(true) { @@ -494,8 +474,8 @@ func (c *Conn) sendBye(reason string) { // serveConn runs a connection from handshake to close. For connections we // dialed, done receives the handshake outcome. -func (n *Node) serveConn(raw net.Conn, outbound bool, via string, done chan<- dialResult) { - c := newConn(n, raw, outbound, via) +func (n *Node) serveConn(s *tunnel.Stream, outbound bool, done chan<- dialResult) { + c := newConn(n, s, outbound) err := c.handshake() if err == nil { err = n.afterHandshake(c) @@ -505,7 +485,7 @@ func (n *Node) serveConn(raw net.Conn, outbound bool, via string, done chan<- di } if err == nil && done != nil { // A listener that refuses us (admission policy) says so with an err - // right after auth, so the dial only counts once the peer's resume, + // right after hello, so the dial only counts once the peer's resume, // or anything other than an err, has arrived. go func() { select { @@ -520,14 +500,14 @@ func (n *Node) serveConn(raw net.Conn, outbound bool, via string, done chan<- di } if err != nil { if c.peerKey != "" { - n.logf("connection with %s via %s failed: %v", wire.ShortKey(c.peerKey), via, err) + n.logf("connection with %s via %s failed: %v", wire.ShortKey(c.peerKey), c.via, err) } else { - n.logf("connection via %s failed: %v", via, err) + n.logf("connection via %s failed: %v", c.via, err) } c.Close(err.Error()) return } - n.logf("connected to %s (%s) via %s", c.hello.Name, wire.ShortKey(c.peerKey), via) + n.logf("connected to %s (%s) via %s", c.hello.Name, wire.ShortKey(c.peerKey), c.via) c.run() n.logf("disconnected from %s (%s): %s", c.hello.Name, wire.ShortKey(c.peerKey), c.closeReason()) n.deactivate(c) @@ -542,11 +522,8 @@ func (n *Node) afterHandshake(c *Conn) error { if err := store.TouchPeer(q, c.peerKey, h.Name, h.About, h.Caps, now); err != nil { return err } - if c.outbound { - return store.AddAddr(q, c.peerKey, c.via, "connect", now) - } - if h.Addr != "" { - return store.AddAddr(q, c.peerKey, h.Addr, "hello", now) + if a, err := tunnel.ParseAddress(h.Addr); err == nil && a.KeyString() == c.peerKey { + return store.AddAddr(q, c.peerKey, a.Public().String(), "hello", now) } return nil }) @@ -555,19 +532,16 @@ func (n *Node) afterHandshake(c *Conn) error { return err } for _, raw := range c.peerGrants { - n.acceptPresentedGrant(c.peerKey, raw, "auth") + n.acceptPresentedGrant(c.peerKey, raw, "hello") } if err := n.admit(c.peerKey, c.outbound); err != nil { - c.sendErrNow(wire.ErrAuth, "", err.Error()) + c.sendErrNow(wire.ErrRefused, "", err.Error()) n.sysEvent(c.peerKey, "refused", map[string]any{"reason": err.Error(), "name": h.Name, "via": c.via}) return err } if err := n.activate(c); err != nil { return err } - if c.outbound { - n.st.AddrResult(c.peerKey, c.via, nil, now) - } n.sysEvent(c.peerKey, "connected", map[string]any{"name": h.Name, "about": h.About, "via": c.via, "outbound": c.outbound, "caps": h.Caps}) n.noteShares(c.peerKey, h.Shares) return nil diff --git a/node/dial.go b/node/dial.go index fcc880b..f0d0507 100644 --- a/node/dial.go +++ b/node/dial.go @@ -9,7 +9,7 @@ import ( "time" "github.com/agentwireprotocol/awp/store" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) @@ -19,14 +19,11 @@ type dialResult struct { err error } -// dial connects to addr and runs the handshake. On success the connection -// keeps running in its own goroutine and the peer's key is returned. -func (n *Node) dial(ctx context.Context, addr string) (string, error) { - a, err := transport.Parse(addr) - if err != nil { - return "", err - } - raw, err := n.tr.Dial(ctx, a) +// dial opens a tunnel stream to the peer the addresses describe and runs +// the handshake. On success the connection keeps running in its own +// goroutine and the peer's key is returned. +func (n *Node) dial(ctx context.Context, addrs ...tunnel.Address) (string, error) { + s, err := n.tun.Dial(ctx, addrs...) if err != nil { return "", err } @@ -34,13 +31,13 @@ func (n *Node) dial(ctx context.Context, addr string) (string, error) { n.wg.Add(1) go func() { defer n.wg.Done() - n.serveConn(raw, true, a.String(), res) + n.serveConn(s, true, res) }() select { case r := <-res: return r.key, r.err case <-ctx.Done(): - raw.Close() + s.Close() r := <-res if r.err == nil { return r.key, nil @@ -49,29 +46,122 @@ func (n *Node) dial(ctx context.Context, addr string) (string, error) { } } -// Connect dials an address given out of band (section 4.1) and returns the -// key of the peer that answered. If a peer known at this address is -// already connected, that connection is reused. +// Connect dials an address given out of band (section 6) and returns the +// key of the peer. If that peer is already connected, the connection is +// reused. func (n *Node) Connect(ctx context.Context, addr string) (string, error) { - a, err := transport.Parse(addr) + a, err := tunnel.ParseAddress(addr) if err != nil { return "", err } - var existing string - n.st.DB().QueryRow(`SELECT peer FROM addrs WHERE addr = ? ORDER BY last_ok DESC LIMIT 1`, a.String()).Scan(&existing) - if existing != "" && n.Connected(existing) { - n.st.SetParked(existing, false) - return existing, nil + key := a.KeyString() + if key == n.key { + return "", errors.New("that is this peer's own address") + } + if n.settle(ctx, key) { + n.st.SetParked(key, false) + return key, nil } - key, err := n.dial(ctx, a.String()) + known := n.knownAddresses(key) + got, err := n.dial(ctx, append([]tunnel.Address{a}, known...)...) if err != nil { return "", err } + if err := n.st.Tx(func(q store.Q) error { return store.AddAddr(q, key, a.String(), "connect", time.Now()) }); err != nil { + n.logf("remember address: %v", err) + } + n.st.SetParked(got, false) + return got, nil +} + +// settle reports whether key has a live connection to reuse. A connection +// that is saying bye is not one: settle waits for it to end, so that a +// connect right after a bye dials afresh instead of returning it. +func (n *Node) settle(ctx context.Context, key string) bool { + n.mu.Lock() + var c *Conn + if p := n.peers[key]; p != nil { + c = p.conn + } + n.mu.Unlock() + if c == nil { + return false + } + if !c.byeSent.Load() { + return true + } + // Wait for the node to let go of it, not just for its reader to stop. + for { + ch := n.Changed() + n.mu.Lock() + p := n.peers[key] + gone := p == nil || p.conn != c + live := p != nil && p.conn != nil && p.conn != c + n.mu.Unlock() + if gone { + return live + } + select { + case <-ch: + case <-time.After(100 * time.Millisecond): + case <-ctx.Done(): + return false + } + } +} + +// ConnectKey dials a known peer at everything known about reaching it. +func (n *Node) ConnectKey(ctx context.Context, key string) error { + if n.settle(ctx, key) { + n.st.SetParked(key, false) + return nil + } + a, ok := n.addressOf(key) + if !ok { + return fmt.Errorf("no known address for %s", wire.ShortKey(key)) + } + got, err := n.dial(ctx, a) + if err != nil { + return err + } + if got != key { + return fmt.Errorf("address now belongs to %s", wire.ShortKey(got)) + } n.st.SetParked(key, false) - return key, nil + return nil } -// dialer is a peer's reconnect loop (section 4.3): exponential backoff +// knownAddresses parses the stored addresses of key, those given out of +// band (which carry the pre-shared key) first. +func (n *Node) knownAddresses(key string) []tunnel.Address { + rows, _ := store.Addrs(n.st.DB(), key) + var first, rest []tunnel.Address + for _, r := range rows { + a, err := tunnel.ParseAddress(r.Addr) + if err != nil || a.KeyString() != key { + continue + } + if r.Source == "connect" { + first = append(first, a) + } else { + rest = append(rest, a) + } + } + return append(first, rest...) +} + +// addressOf is everything known about reaching key, merged: what the +// reconnect loop dials and what an introduction hands on. +func (n *Node) addressOf(key string) (tunnel.Address, bool) { + as := n.knownAddresses(key) + if len(as) == 0 { + return tunnel.Address{}, false + } + m, err := tunnel.Merge(as...) + return m, err == nil +} + +// dialer is a peer's reconnect loop (section 10.3): exponential backoff // capped at MaxBackoff, no give-up, for as long as wantConnection holds. type dialer struct { n *Node @@ -123,7 +213,7 @@ func (n *Node) ensureDialer(key string, immediate bool) { if !n.wantConnection(key) { return } - if addrs, _ := store.Addrs(n.st.DB(), key); len(addrs) == 0 { + if _, ok := n.addressOf(key); !ok { return } @@ -150,7 +240,6 @@ func (d *dialer) run(immediate bool) { }() backoff := time.Second - failures := 0 first := true for { if n.ctx.Err() != nil || n.Connected(d.key) || !n.wantConnection(d.key) { @@ -171,31 +260,29 @@ func (d *dialer) run(immediate bool) { continue } first = false - for _, a := range addrs { - if n.Connected(d.key) { - return - } - ctx, cancel := context.WithTimeout(n.ctx, 30*time.Second) - key, err := n.dial(ctx, a.Addr) - cancel() - if err == nil && key != d.key { - err = fmt.Errorf("address now belongs to %s", wire.ShortKey(key)) - } - if err == nil { - return - } - var peerErr errPeerSaid - if errors.As(err, &peerErr) && (peerErr.e.Code == wire.ErrAuth || peerErr.e.Code == wire.ErrVersion) { - n.logf("%s refused us: %v", wire.ShortKey(d.key), err) + addr, ok := n.addressOf(d.key) + if !ok { + return + } + ctx, cancel := context.WithTimeout(n.ctx, 45*time.Second) + key, err := n.dial(ctx, addr) + cancel() + if err == nil && key != d.key { + err = fmt.Errorf("address now belongs to %s", wire.ShortKey(key)) + } + if err == nil { + for _, a := range addrs { + n.st.AddrResult(d.key, a.Addr, nil, time.Now()) } - d.setErr(err) + return + } + var peerErr errPeerSaid + if errors.As(err, &peerErr) && (peerErr.e.Code == wire.ErrRefused || peerErr.e.Code == wire.ErrAuth || peerErr.e.Code == wire.ErrVersion) { + n.logf("%s refused us: %v", wire.ShortKey(d.key), err) + } + d.setErr(err) + for _, a := range addrs { n.st.AddrResult(d.key, a.Addr, err, time.Now()) - failures++ - if failures%3 == 0 { - if pa, err := transport.Parse(a.Addr); err == nil { - n.tr.Reset(pa) // start the next attempt from a fresh tailcat node - } - } } wait := backoff/2 + rand.N(backoff) // jitter in [b/2, 3b/2) n.logf("reconnect to %s failed (%s); retrying in %v", wire.ShortKey(d.key), d.lastErr(), wait.Round(100*time.Millisecond)) diff --git a/node/identity.go b/node/identity.go index 1d0d9d6..ee37fe6 100644 --- a/node/identity.go +++ b/node/identity.go @@ -19,7 +19,7 @@ type identityFile struct { // loadIdentity reads the key at path, creating a new one (mode 0600) if the // file does not exist. A sandbox that starts with an empty home therefore -// gets a fresh key, as section 13 recommends. +// gets a fresh key, as section 16 recommends. func loadIdentity(path string) (ed25519.PrivateKey, error) { b, err := os.ReadFile(path) if errors.Is(err, os.ErrNotExist) { diff --git a/node/inbound.go b/node/inbound.go index 90aceaf..cf24554 100644 --- a/node/inbound.go +++ b/node/inbound.go @@ -5,7 +5,7 @@ import ( "time" "github.com/agentwireprotocol/awp/store" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) @@ -60,7 +60,7 @@ func (n *Node) handleLine(c *Conn, line []byte) error { case "": return c.fatal(wire.ErrBadFrame, env.ID, "missing message type") } - // Unknown types are ignored (section 5). Replying unsupported is + // Unknown types are ignored (section 8). Replying unsupported is // optional; staying quiet keeps a newer peer's chatter harmless. return nil } @@ -386,11 +386,9 @@ func (n *Node) onIntroduce(c *Conn, line []byte) error { grant = g } } - addr := in.Peer.Address - if a, err := transport.Parse(addr); err == nil { + addr := "" + if a, err := tunnel.ParseAddress(in.Peer.Address); err == nil && a.KeyString() == in.Peer.Key { addr = a.String() - } else { - addr = "" } err := n.st.Tx(func(q store.Q) error { id := in.ID diff --git a/node/node.go b/node/node.go index 1001b2b..0b0df06 100644 --- a/node/node.go +++ b/node/node.go @@ -8,18 +8,16 @@ import ( "crypto/ed25519" "encoding/json" "errors" - "fmt" - "net" "os" "path/filepath" - "slices" "strings" "sync" "time" "github.com/agentwireprotocol/awp/store" - "github.com/agentwireprotocol/awp/transport" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" + "tailscale.com/types/logger" ) // Config configures a Node. Zero values get the defaults from the spec. @@ -48,13 +46,14 @@ type Config struct { // remembered, since the next start may come from another harness. DetectedHarness string - // Listen lists addresses to accept connections on: "tailcat", - // "tcp:host:port" or "unix:/path". + // Listen lists the carriers to accept connections on (tunnel.Config): + // "tailcat", "udp:HOST:PORT", "unix:/path", "ws:HOST:PORT[=URL]" or + // "cloudflare". Listen []string - // Advertise is the address sent in hello.addr so the peer can dial us - // back. Empty means the tailcat address if there is one, otherwise the - // first listen address. "none" disables it. + // Advertise controls hello.addr, the address the peer can dial us back + // at: empty sends this node's address without its pre-shared key, + // "none" sends nothing. Advertise string PingInterval time.Duration // idle time before a ping; default 30s @@ -72,9 +71,8 @@ type Config struct { TailcatLogf func(format string, args ...any) // Trace logs every line sent and received. Trace bool - // AllowPlaintext permits plain TCP to public addresses (see - // transport.Transport.AllowPublicTCP). - AllowPlaintext bool + // Cloudflared is the cloudflared binary for the cloudflare carrier. + Cloudflared string // Presence publishes this node's signed presence to its peers, who // gossip it on (presence.go). Receiving and forwarding other agents' // presence happens either way. @@ -119,18 +117,15 @@ type Node struct { key string st *store.Store ids *wire.IDGen - tr *transport.Transport + tun *tunnel.Tunnel ctx context.Context cancel context.CancelFunc wg sync.WaitGroup - mu sync.Mutex - peers map[string]*peerState - listeners []*listener - tailcat *transport.TailcatListener - tailcatErr string - closed bool + mu sync.Mutex + peers map[string]*peerState + closed bool cmu sync.Mutex changeCh chan struct{} @@ -140,11 +135,6 @@ type Node struct { presDoc json.RawMessage // last signed presence document we published } -type listener struct { - addr string - ln net.Listener -} - // peerState is the in-memory state for one remote key. type peerState struct { key string @@ -184,9 +174,22 @@ func Open(cfg Config) (*Node, error) { if last, err := st.MaxOwnID(); err == nil && last != "" { n.ids.Observe(last) } - n.tr = &transport.Transport{AllowPublicTCP: cfg.AllowPlaintext} + var tlogf logger.Logf if cfg.TailcatLogf != nil { - n.tr.Logf = cfg.TailcatLogf + tlogf = logger.Logf(cfg.TailcatLogf) + } + n.tun, err = tunnel.New(tunnel.Config{ + Identity: priv, + StateDir: cfg.Home, + Listen: cfg.Listen, + Changed: n.bump, + Logf: cfg.Logf, + TailcatLogf: tlogf, + Cloudflared: cfg.Cloudflared, + }) + if err != nil { + st.Close() + return nil, err } // Name and about persist once set, so a daemon started on demand by // any command keeps the identity its user gave it. @@ -259,31 +262,16 @@ func (n *Node) logf(format string, args ...any) { n.cfg.Logf(format, args...) } -// Start opens the listeners and starts reconnecting to peers that have -// unfinished business. A tailcat listener starts in the background, since -// picking a DERP region takes a few seconds. +// Start opens the carriers and starts reconnecting to peers that have +// unfinished business. tailcat and cloudflare come up in the background, +// since they take seconds. func (n *Node) Start() error { - for _, l := range n.cfg.Listen { - if l == "tailcat" { - n.wg.Add(1) - go n.startTailcat() - continue - } - a, err := transport.Parse(l) - if err != nil { - return err - } - ln, err := transport.Listen(a) - if err != nil { - return fmt.Errorf("listen %s: %w", l, err) - } - addr := a - if a.Kind == transport.KindTCP { - addr.Target = ln.Addr().String() - } - n.addListener(&listener{addr: addr.String(), ln: ln}) + if err := n.tun.Start(); err != nil { + return err } n.wg.Add(1) + go n.acceptLoop() + n.wg.Add(1) go n.maintain() if n.cfg.Presence { n.wg.Add(1) @@ -292,63 +280,20 @@ func (n *Node) Start() error { return nil } -func (n *Node) addListener(l *listener) { - n.mu.Lock() - n.listeners = append(n.listeners, l) - n.mu.Unlock() - n.logf("listening on %s", l.addr) - n.wg.Add(1) - go n.acceptLoop(l) - n.bump() -} - -func (n *Node) startTailcat() { +func (n *Node) acceptLoop() { defer n.wg.Done() - backoff := 5 * time.Second for { - ctx, cancel := context.WithTimeout(n.ctx, 60*time.Second) - tl, err := transport.ListenTailcat(ctx, filepath.Join(n.cfg.Home, "tailcat.json"), n.tr.Logf) - cancel() - if err == nil { - n.mu.Lock() - if n.closed { - n.mu.Unlock() - tl.Close() - return - } - n.tailcat = tl - n.tailcatErr = "" - n.mu.Unlock() - n.addListener(&listener{addr: "tailcat:" + tl.Address(), ln: tl}) - return - } - n.mu.Lock() - n.tailcatErr = err.Error() - n.mu.Unlock() - n.logf("tailcat listener: %v (retrying in %v)", err, backoff) - select { - case <-n.ctx.Done(): - return - case <-time.After(backoff): - } - backoff = min(backoff*2, 5*time.Minute) - } -} - -func (n *Node) acceptLoop(l *listener) { - defer n.wg.Done() - for { - raw, err := l.ln.Accept() + s, err := n.tun.Accept() if err != nil { if n.ctx.Err() == nil { - n.logf("accept on %s: %v", l.addr, err) + n.logf("accept: %v", err) } return } n.wg.Add(1) go func() { defer n.wg.Done() - n.serveConn(raw, false, l.addr, nil) + n.serveConn(s, false, nil) }() } } @@ -368,64 +313,42 @@ func (n *Node) Close() error { conns = append(conns, p.conn) } } - ls := n.listeners n.mu.Unlock() n.cancel() - for _, l := range ls { - l.ln.Close() - } for _, c := range conns { c.Close("shutdown") } + n.tun.Close() n.wg.Wait() - n.tr.Close() return n.st.Close() } -// Addresses returns the addresses this node listens on, tailcat first. -func (n *Node) Addresses() []string { - n.mu.Lock() - defer n.mu.Unlock() - var out []string - for _, l := range n.listeners { - out = append(out, l.addr) - } - slices.SortStableFunc(out, func(a, b string) int { - ta, tb := strings.HasPrefix(a, "tailcat:"), strings.HasPrefix(b, "tailcat:") - switch { - case ta && !tb: - return -1 - case tb && !ta: - return 1 - } - return 0 - }) - return out -} +// Address returns this node's address (section 6): its key, the listener's +// pre-shared key and every endpoint it listens on. It has no endpoints +// until a carrier is up. +func (n *Node) Address() tunnel.Address { return n.tun.Address() } -// TailcatAddress returns the bare tailcat address, if listening on tailcat. -func (n *Node) TailcatAddress() (addr, errMsg string) { - n.mu.Lock() - defer n.mu.Unlock() - if n.tailcat != nil { - return n.tailcat.Address(), "" - } - return "", n.tailcatErr +// Pending lists carriers still coming up, with their last error if any. +func (n *Node) Pending() map[string]string { return n.tun.Pending() } + +// RotatePSK replaces the listener's pre-shared key: every address shared +// before stops admitting peers not met yet. +func (n *Node) RotatePSK() error { + err := n.tun.RotatePSK() + n.bump() + return err } // advertised is the address sent in hello.addr. func (n *Node) advertised() string { - switch n.cfg.Advertise { - case "none": + if n.cfg.Advertise == "none" { return "" - case "": - addrs := n.Addresses() - if len(addrs) > 0 { - return addrs[0] - } + } + a := n.tun.Address() + if len(a.Endpoints) == 0 { return "" } - return n.cfg.Advertise + return a.Public().String() } // --- change notification --- @@ -623,7 +546,7 @@ func (n *Node) reconnectAll() { } } -// wantConnection is the rule from section 4.3: keep trying for as long as +// wantConnection is the rule from section 10.3: keep trying for as long as // there are unacked messages or open threads, unless the peer is parked // (a bye was exchanged and nothing new has been queued since). func (n *Node) wantConnection(key string) bool { diff --git a/node/node_test.go b/node/node_test.go index fc58bcf..32e1792 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -16,13 +16,14 @@ import ( "time" "github.com/agentwireprotocol/awp/store" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) // testNode starts a node listening on a Unix socket in its own home. func testNode(t *testing.T, name string, mod ...func(*Config)) *Node { t.Helper() - home := t.TempDir() + home := shortTemp(t) cfg := Config{ Home: home, Name: name, @@ -65,7 +66,19 @@ func restart(t *testing.T, n *Node) *Node { return m } -func addr(n *Node) string { return n.Addresses()[0] } +func addr(n *Node) string { return n.Address().String() } + +// shortTemp is a temporary directory with a short path, since Unix socket +// paths are limited to about 100 bytes. +func shortTemp(t *testing.T) string { + t.Helper() + d, err := os.MkdirTemp("", "awpn") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(d) }) + return d +} // waitFor polls cond at every state change of n until it holds. func waitFor(t *testing.T, n *Node, what string, cond func() bool) { @@ -327,21 +340,35 @@ func TestBlobSentOnAck(t *testing.T) { type rawPeer struct { t *testing.T c net.Conn + tun *tunnel.Tunnel r *wire.Reader priv ed25519.PrivateKey key string } +// dialRaw opens a tunnel stream to n from a fresh key and hands it to the +// test to speak NDJSON on directly. func dialRaw(t *testing.T, n *Node) *rawPeer { t.Helper() - c, err := net.Dial("unix", strings.TrimPrefix(addr(n), "unix:")) + pub, priv, _ := ed25519.GenerateKey(nil) + tu, err := tunnel.New(tunnel.Config{Identity: priv}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { tu.Close() }) + a, err := tunnel.ParseAddress(addr(n)) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := tu.Dial(ctx, a) if err != nil { t.Fatal(err) } t.Cleanup(func() { c.Close() }) - pub, priv, _ := ed25519.GenerateKey(nil) c.SetDeadline(time.Now().Add(10 * time.Second)) - return &rawPeer{t: t, c: c, r: wire.NewReader(c), priv: priv, key: wire.FormatKey(pub)} + return &rawPeer{t: t, c: c, tun: tu, r: wire.NewReader(c), priv: priv, key: wire.FormatKey(pub)} } func (p *rawPeer) send(line string) { @@ -387,44 +414,44 @@ func (p *rawPeer) expectErr(code string) { } } -func (p *rawPeer) hello(v int) string { - return fmt.Sprintf(`{"t":"hello","id":"%s","ts":"%s","v":%d,"key":"%s","name":"raw","nonce":"%s","caps":["chat"]}`, wire.NewIDGen().New(), wire.Now(), v, p.key, wire.Nonce(32)) +func (p *rawPeer) hello(v int) string { return p.helloAs(v, p.key) } + +func (p *rawPeer) helloAs(v int, key string) string { + return fmt.Sprintf(`{"t":"hello","id":"%s","ts":"%s","v":%d,"key":"%s","name":"raw","caps":["chat"]}`, wire.NewIDGen().New(), wire.Now(), v, key) } -// handshake completes hello, auth and resume like a real peer would. +// handshake completes hello and resume like a real peer would. func (p *rawPeer) handshake() { p.t.Helper() - my := p.hello(0) - p.send(my) - env, theirs := p.read() - if env.T != wire.THello { + p.send(p.hello(wire.Version)) + if env, _ := p.read(); env.T != wire.THello { p.t.Fatalf("first line %s", env.T) } - p.send(fmt.Sprintf(`{"t":"auth","id":"x2","ts":"%s","sig":"%s"}`, wire.Now(), wire.SignAuth(p.priv, []byte(my), theirs))) - if env, _ := p.read(); env.T != wire.TAuth { - p.t.Fatalf("expected auth, got %s", env.T) - } if env, _ := p.read(); env.T != wire.TResume { p.t.Fatalf("expected resume, got %s", env.T) } p.send(fmt.Sprintf(`{"t":"resume","id":"x3","ts":"%s","seen":{}}`, wire.Now())) } -func TestBadAuth(t *testing.T) { +// TestHelloKeyMismatch: hello must name the key on the other end of the +// tunnel (section 10.1), and never the receiver's own. +func TestHelloKeyMismatch(t *testing.T) { n := testNode(t, "n") - p := dialRaw(t, n) - p.send(p.hello(0)) - p.read() // their hello - // Sign the wrong transcript. - p.send(fmt.Sprintf(`{"t":"auth","id":"x","ts":"%s","sig":"%s"}`, wire.Now(), wire.SignAuth(p.priv, []byte("forged"), []byte("transcript")))) - p.read() // their auth - p.expectErr(wire.ErrAuth) + other, _, _ := ed25519.GenerateKey(nil) + for name, key := range map[string]string{"another key": wire.FormatKey(other), "its own key": n.Key()} { + t.Run(name, func(t *testing.T) { + p := dialRaw(t, n) + p.send(p.helloAs(wire.Version, key)) + p.read() // their hello + p.expectErr(wire.ErrAuth) + }) + } } func TestVersionMismatch(t *testing.T) { n := testNode(t, "n") p := dialRaw(t, n) - p.send(p.hello(1)) + p.send(p.hello(wire.Version + 1)) p.read() p.expectErr(wire.ErrVersion) } @@ -605,9 +632,14 @@ func TestIntroduce(t *testing.T) { } waitFor(t, c, "introduction", func() bool { return len(received(t, c, b.Key(), "introduce")) == 1 }) p, _ := store.GetPeer(c.Store().DB(), a.Key()) - if p == nil || len(p.Addrs) == 0 || p.Addrs[0].Addr != addr(a) { + if p == nil || len(p.Addrs) == 0 { t.Fatalf("introduced peer at c: %+v", p) } + // The introducer knew A's address with its pre-shared key (B dialed + // A), and handed that on. + if ia, err := tunnel.ParseAddress(p.Addrs[0].Addr); err != nil || ia.PSK == nil || ia.KeyString() != a.Key() { + t.Fatalf("introduced address %q: %v", p.Addrs[0].Addr, err) + } // C dials the introduced address and presents the grant. if _, err := c.Connect(ctx, p.Addrs[0].Addr); err != nil { t.Fatalf("introduced connect: %v", err) @@ -673,7 +705,8 @@ func TestDuplicateConnectionReplaced(t *testing.T) { } // A raw second connection from a's key is not possible without a's // private key, so exercise replacement by dialing again directly. - if _, err := a.dial(context.Background(), addr(b)); err != nil { + ab, _ := tunnel.ParseAddress(addr(b)) + if _, err := a.dial(context.Background(), ab); err != nil { t.Fatal(err) } waitFor(t, b, "one active connection", func() bool { return b.Connected(a.Key()) && a.Connected(b.Key()) }) @@ -683,23 +716,17 @@ func TestDuplicateConnectionReplaced(t *testing.T) { waitFor(t, b, "msg", func() bool { return len(received(t, b, a.Key(), "msg")) == 1 }) } -func TestPlainTCPRefusedToPublicAddresses(t *testing.T) { +func TestConnectRejectsBadAddresses(t *testing.T) { a := testNode(t, "a") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _, err := a.Connect(ctx, "tcp:1.1.1.1:7777") - if err == nil || !strings.Contains(err.Error(), "refusing plain TCP") { - t.Fatalf("public TCP dial: %v", err) - } - // Loopback is fine (the dial fails only because nothing listens). - _, err = a.Connect(ctx, "tcp:127.0.0.1:1") - if err == nil || strings.Contains(err.Error(), "refusing") { - t.Fatalf("loopback TCP dial: %v", err) + for _, bad := range []string{"tcp:127.0.0.1:1", "unix:/tmp/x", "tcomFwWCCcjS5nKNqAod034nWoJZW0LZqDhhC8U_dKdnDRYQ8uNGFpGQEu", addr(a)} { + if _, err := a.Connect(ctx, bad); err == nil { + t.Errorf("connected to %q", bad) + } } } -// An explicit harness is remembered and beats what the environment of a -// later start suggests; a detected one is only a fallback, never remembered. func TestHarnessPrecedence(t *testing.T) { open := func(home string, cfg Config) *Node { t.Helper() diff --git a/node/outbound.go b/node/outbound.go index b606a73..539af6a 100644 --- a/node/outbound.go +++ b/node/outbound.go @@ -179,7 +179,7 @@ func deriveSubject(parts []wire.Part, files []string) string { return "" } -// SetState sends our view of a thread's state (section 8.1). +// SetState sends our view of a thread's state (section 11.1). func (n *Node) SetState(peer, th, state, note string) (*SendResult, error) { if state == "" { return nil, errors.New("state is required") @@ -210,7 +210,7 @@ func (n *Node) SetState(peer, th, state, note string) (*SendResult, error) { return &SendResult{ID: id, Th: th, Peer: peer, Connected: n.Connected(peer)}, nil } -// Grant mints a grant for sub (section 10.3), records it, and sends it if +// Grant mints a grant for sub (section 13.3), records it, and sends it if // the peer is connected; otherwise it goes out after the next resume. func (n *Node) Grant(sub string, caps []string, ttl time.Duration) (*store.GrantRow, error) { if _, err := wire.ParseKey(sub); err != nil { @@ -265,7 +265,8 @@ func (n *Node) Introduce(to, intro string, caps []string, ttl time.Duration, th if err != nil { return nil, err } - if ip == nil || len(ip.Addrs) == 0 { + addr, ok := n.addressOf(intro) + if ip == nil || !ok { return nil, fmt.Errorf("no known address for %s", wire.ShortKey(intro)) } if ttl <= 0 { @@ -278,7 +279,7 @@ func (n *Node) Introduce(to, intro string, caps []string, ttl time.Duration, th } msg := wire.Introduce{ Envelope: wire.Envelope{T: wire.TIntroduce, TS: wire.FormatTime(now), Th: th}, - Peer: wire.IntroPeer{Key: intro, Name: ip.Name, Address: ip.Addrs[0].Addr}, + Peer: wire.IntroPeer{Key: intro, Name: ip.Name, Address: addr.String()}, Grant: g.Raw, } if th == "" && !n.Connected(to) { @@ -308,7 +309,7 @@ func (n *Node) Introduce(to, intro string, caps []string, ttl time.Duration, th return &SendResult{ID: msg.ID, Th: th, Peer: to, Connected: n.Connected(to)}, nil } -// Bye closes the connection to a peer gracefully (section 9.6) and parks +// Bye closes the connection to a peer gracefully (section 12.6) and parks // it: no reconnection until something new is queued for it. func (n *Node) Bye(peer, reason string) error { n.st.SetParked(peer, true) diff --git a/node/policy.go b/node/policy.go index 95015c7..a2e0c33 100644 --- a/node/policy.go +++ b/node/policy.go @@ -8,10 +8,11 @@ import ( "time" "github.com/agentwireprotocol/awp/store" + "github.com/agentwireprotocol/awp/tunnel" "github.com/agentwireprotocol/awp/wire" ) -// Admission policies (section 7.3). +// Admission policies (section 10.2). const ( // AcceptAny admits any key with the default capabilities. It is what // the spec recommends for a coding agent on a sandbox. @@ -21,7 +22,7 @@ const ( AcceptAllowlist = "allowlist" ) -// Capabilities with a meaning in this implementation (section 10.1). +// Capabilities with a meaning in this implementation (section 13.1). const ( CapExec = "exec" CapFSRead = "fs:read" @@ -74,7 +75,7 @@ type honored struct { caps []string } -// honoredGrants applies section 10.2: a grant counts if we issued it, if a +// honoredGrants applies section 13.2: a grant counts if we issued it, if a // trusted key issued it, or if its issuer holds an introduce grant from us // or a trusted key (one level of delegation). A delegated grant is // attenuated: it confers at most what the introducer itself was granted, @@ -193,14 +194,18 @@ func (n *Node) acceptPresentedGrant(peer string, raw json.RawMessage, via string // grantsToPresent picks the grants we hold that concern the peer we are // talking to: ones it issued, and introductions meant for it. Other grants // stay private. -func (n *Node) grantsToPresent(peer string) []json.RawMessage { +func (n *Node) grantsToPresent(s *tunnel.Stream) []json.RawMessage { held, err := n.st.Grants(store.GrantQuery{Role: store.RoleHeld, Sub: n.key, ValidAt: time.Now()}) if err != nil { return nil } + is := func(key string) bool { + pub, err := wire.ParseKey(key) + return err == nil && s.Is(pub) + } var out []json.RawMessage for _, g := range held { - if g.Iss == peer || g.Audience == peer { + if is(g.Iss) || (g.Audience != "" && is(g.Audience)) { out = append(out, json.RawMessage(g.Raw)) } } diff --git a/node/presence_test.go b/node/presence_test.go index a02caf4..35fc449 100644 --- a/node/presence_test.go +++ b/node/presence_test.go @@ -164,9 +164,8 @@ func TestPresenceExpiry(t *testing.T) { // handshakeCaps is handshake with the given hello caps. func (p *rawPeer) handshakeCaps(caps string) { p.t.Helper() - my := fmt.Sprintf(`{"t":"hello","id":"%s","ts":"%s","v":0,"key":"%s","name":"raw","nonce":"%s","caps":%s}`, - wire.NewIDGen().New(), wire.Now(), p.key, wire.Nonce(32), caps) - p.send(my) + p.send(fmt.Sprintf(`{"t":"hello","id":"%s","ts":"%s","v":%d,"key":"%s","name":"raw","caps":%s}`, + wire.NewIDGen().New(), wire.Now(), wire.Version, p.key, caps)) env, theirs := p.read() if env.T != wire.THello { p.t.Fatalf("first line %s", env.T) @@ -176,10 +175,6 @@ func (p *rawPeer) handshakeCaps(caps string) { if !slices.Contains(h.Caps, wire.TPresence) { p.t.Fatalf("hello caps %v lack presence", h.Caps) } - p.send(fmt.Sprintf(`{"t":"auth","id":"x2","ts":"%s","sig":"%s"}`, wire.Now(), wire.SignAuth(p.priv, []byte(my), theirs))) - if env, _ := p.read(); env.T != wire.TAuth { - p.t.Fatalf("expected auth, got %s", env.T) - } if env, _ := p.read(); env.T != wire.TResume { p.t.Fatalf("expected resume, got %s", env.T) } diff --git a/plugin/awp/skills/awp/SKILL.md b/plugin/awp/skills/awp/SKILL.md index 75dbd56..4abd679 100644 --- a/plugin/awp/skills/awp/SKILL.md +++ b/plugin/awp/skills/awp/SKILL.md @@ -1,11 +1,11 @@ --- name: awp -description: Message other coding agents on other machines, peer to peer. Use it to delegate a task to another agent or take one on, to follow up or answer in a thread, and to send results and files back. Also use it when the user gives you an awp or tailcat address (it starts with "tc"), asks you to connect to, check on or reply to another agent, or when awp messages appear in your context. +description: Message other coding agents on other machines, peer to peer. Use it to delegate a task to another agent or take one on, to follow up or answer in a thread, and to send results and files back. Also use it when the user gives you an awp address (it starts with "awp1"), asks you to connect to, check on or reply to another agent, or when awp messages appear in your context. --- # awp -awp links you to another coding agent (Claude Code, Codex, Cursor, ...) on a different machine. Either side can start a thread, send messages and files, and report progress. Connections run over tailcat: WireGuard, peer to peer, with no accounts or servers to set up. awp speaks the Agent Wire Protocol (AWP), so the other side can be any AWP peer. +awp links you to another coding agent (Claude Code, Codex, Cursor, ...) on a different machine. Either side can start a thread, send messages and files, and report progress. Connections are WireGuard tunnels between the two agents' own keys, peer to peer, with no accounts or servers to set up. awp speaks the Agent Wire Protocol (AWP), so the other side can be any AWP peer. Run it as `awp`. Installed as a plugin, it is on PATH. If not, it sits next to this skill at `${CLAUDE_SKILL_DIR}/../../bin/awp`. If your harness shows `awp_*` MCP tools, they do the same things as the commands below. @@ -27,7 +27,7 @@ Run it as `awp`. Installed as a plugin, it is on PATH. If not, it sits next to t ## Connecting -- **Your address:** `awp up`. It prints `address tc...`. The address is a secret that lets someone reach you. Give it to your user to pass on, only to the agent they mean you to talk to. +- **Your address:** `awp up`. It prints `address awp1...`. The address is a secret that lets someone reach you. Give it to your user to pass on, only to the agent they mean you to talk to. - **Your model:** other agents and dashboards see which model you run on. Claude Code, Cursor and opencode report it automatically. Elsewhere, run `awp model ` once awp is up (for example `awp model gpt-5.5`), and again if you switch models. Check it with `awp model`. - **Their address:** `awp connect
`. After that, refer to the peer by the name it announced (see `awp peers`). @@ -114,5 +114,5 @@ If the peer's daemon does not serve the capability automatically, the request re - `reconnecting`: the daemon keeps retrying while there is unfinished business, and queued messages go out once the peer is back. - `offline`: nothing to do right now. - `said bye`: parked until you send it something. -- `awp status` shows your address. If tailcat is still starting, give it a few seconds. +- `awp status` shows your address. If a carrier (tailcat, cloudflare) is still starting, give it a few seconds. - The daemon log is at `~/.awp/daemon.log`. `awp down` stops the daemon. Queued messages stay on disk. diff --git a/python/awp_peer.py b/python/awp_peer.py deleted file mode 100644 index b7c4606..0000000 --- a/python/awp_peer.py +++ /dev/null @@ -1,2552 +0,0 @@ -#!/usr/bin/env python3 -# awp_peer.py -- an independent, single-file Python peer for the Agent Wire -# Protocol (AWP), SPEC.md "draft 1" (2026-09-25). -# -# Written from the spec text alone, to serve as an interop test partner for the -# Go reference implementation. It speaks the non-tailcat bindings of section -# 4.2: plain TCP and Unix stream sockets. Ed25519 comes from the `cryptography` -# package; if that is missing (or AWP_PURE_ED25519=1 is set) a pure-Python -# RFC 8032 implementation is used instead. -# -# CLI (test-driver contract): -# python3 awp_peer.py --state DIR [--name NAME] [--ping-interval SECS] listen ADDR -# python3 awp_peer.py --state DIR [--name NAME] [--ping-interval SECS] connect ADDR -# ADDR is tcp:HOST:PORT or unix:/path/to/socket -# Optional extras: --max-blob BYTES (receive limit, default 50 MiB), -# --trust KEY (repeatable; grant issuers to trust), --about TEXT, -v (debug log). -# Environment: AWP_FSYNC=0 skips fsync (still safe against kill -9, only -# power loss needs fsync); AWP_PURE_ED25519=1 forces the fallback crypto. -# -# stdin: one JSON command per line -# {"cmd":"send","th":"t1","subject":"optional","text":"hello","re":"optional id"} -# extension: "parts":[...] appends raw parts (code/data/blob) after the text part -# {"cmd":"state","th":"t1","state":"working","note":"optional"} -# {"cmd":"blob","th":"t1","path":"/some/file","text":"optional"} chunks + msg with a blob part -# {"cmd":"grant","sub":"ed25519:...","caps":["exec"],"ttl":3600} mint + send a `grant` message -# {"cmd":"bye","reason":"done"} -# {"cmd":"quit"} exit immediately, no bye -# any command may add "peer":"ed25519:..." to pick the target peer (extension) -# stdin EOF: keep running until killed. -# stdout: one JSON event per line, flushed immediately: -# identity, listening, connected, recv, blob, sent, acked, disconnected, error -# stderr: logs and diagnostics only. -# -# State directory (--state DIR): -# identity.json Ed25519 seed (mode 0600) -# default_peer.json peer that commands target by default -# grants_issued.json grants we minted; grants_held.json: grants whose sub is us -# peers// one directory per peer key, fp = hex(sha256(raw key))[:32] -# outbox.log append-only log of unacked outbound msg/state/chunk -# inbox.log received ids (dedup set, per-thread "last seen"), blob progress -# threads.json sendonce.json grants.json peer.json -# peers/_pending/ outbox for commands issued before any peer ever authenticated -# outgoing/ immutable copy of each blob being sent (chunk data source) -# blobs// completed received blobs (blobs//.partial/ while in flight) -# -# ============================================================================ -# SPEC AMBIGUITIES (and how this implementation resolves them) -# ============================================================================ -# Framing and envelope (sections 5, 6) -# A1. "Maximum line length: 1 MiB": read as at most 1,048,576 bytes of content, -# excluding the terminating "\n". Longer: send `err too_large`, close. -# Outbound: a send/state command whose line would exceed 1 MiB is rejected -# locally with an `error` event (send it as a blob). -# A2. `too_large` has no stated close behaviour (9.7 says "stated per code above", -# but it never is). It is not in the non-closing list, and after an -# over-long line the framing cannot be trusted, so we close. Closing codes -# are therefore bad_frame, version, auth, too_large. An `err` with a code we -# do not know is treated as non-closing. When we receive a closing err we -# close too. -# A3. Empty or whitespace-only lines are skipped instead of being `bad_frame` -# (liberal). A "\r" before "\n" stays part of the line bytes (it matters -# for the hello bytes that get signed); JSON parsing tolerates it. -# A4. Invalid UTF-8, invalid JSON, or valid JSON that is not an object -> -# `bad_frame` + close. A JSON object with a missing or non-string `t` is -# treated as an unknown message type and ignored, which is the literal reading -# of section 5. -# A5. We never send `unsupported`. Unknown types are logged, emitted as `recv` -# events and otherwise ignored. -# A6. Ids: ULIDs, monotonic per process: within one millisecond the random part -# is incremented. The generator is seeded from the highest id in our outbox -# logs at startup, so a restart with a clock that went backwards still gives -# increasing ids. "ids greater than the seen id" is plain string comparison. -# That works for fixed-length ULIDs, and we only ever compare our own ids. -# Note: resume silently depends on each sender's ids growing in send order, -# which is a hard requirement although the spec only "recommends" ULID/UUIDv7. -# A7. Timestamps: examples use whole seconds; we send UTC with milliseconds -# ("2026-09-25T17:03:11.123Z"). Incoming `ts` is never interpreted. Grant -# `exp` is emitted at whole seconds (rounded up) and any RFC 3339 form is -# accepted: any number of fraction digits, "Z" or a numeric offset. -# A8. msg/state that lack `id` or `th` are still delivered (a `recv` event) but -# cannot be acked (an ack needs both) or deduplicated. -# -# Handshake (section 7) -# B1. base64url padding is not specified. We send unpadded base64url (43 chars -# for keys and nonces, 86 for signatures). We accept padded or unpadded input, -# either alphabet, and ignore embedded whitespace, everywhere. -# B2. "Its byte encoding is the peer identity": the identity is the 32 raw key -# bytes, so two spellings of the same key are the same peer. The "ed25519:" -# prefix is matched case-insensitively. The `connected` event echoes the -# key string exactly as the peer sent it. -# B3. `caps`: the text says "chat and resume are mandatory and not listed", but -# both examples list "chat". We send ["chat","blob","grant"] as in the -# examples. On receipt we treat chat and resume as always supported whether -# listed or not. We do not stop sending chunks to a peer that omits "blob"; -# we only log it. -# B4. `v` missing -> treated as 0. `v` must be the number 0 (true/"0" are -# rejected). The version is checked when hello arrives, before we send auth. -# B5. Signature bytes: "awp-auth-v0" 0x00 0x00 . Lines are the exact bytes as written or read, -# without the "\n". -# B6. The transcript is symmetric. If a peer echoes our own hello back, the -# signed string for its side is byte-identical to ours, so it could replay -# our auth and pass. We reject (with `err auth`) any hello that carries our -# own key or is byte-identical to our hello. -# B7. Before the handshake completes: msg/state/ack/chunk/resume/grant/introduce -# -> `err auth` + close; ping/pong and unknown types are ignored; `bye` just -# closes. A repeated hello or auth is ignored. -# B8. `connected` is emitted once we have sent our auth and verified theirs. We -# cannot see the peer's verification of ours; if it fails, its err closes us. -# B9. Grants in `auth` are verified and stored; a bad grant does not fail auth. -# We present all unexpired grants we hold (sub == our key) in our own auth. -# B10. Policy (7.3): accept any key and log it. Handshake timeout 20 s. Each -# --state dir gets its own fresh key, and its fingerprint goes in `about` -# (section 13). -# -# Threads and messages (sections 8, 9) -# C1. "Open thread" (4.3 reconnect rule) is undefined. A thread is open once any -# msg/state has gone either way in it, until the most recent state message in -# either direction is "closed". A later msg or state reopens it. done and -# failed stay open because they are one side's view. -# C2. We do not make up a `subject` when the send command gives none. -# C3. Acks: sent for every msg/state, including re-acks of duplicates; ack.th -# copies the message's th. An ack for an unknown id is ignored. Acks for -# chunk ids are accepted and prune the chunk. We never ack chunks, since acks -# are mandatory only for msg and state. -# C4. "Durably received": for msg and state we emit `recv`, then append the id to -# inbox.log (fsync), then ack. So kill -9 can at worst cause one repeated -# `recv` after restart (at-least-once), never a loss. -# C5. `recv` events go out for every inbound message except ping/pong (including -# hello/auth/resume/ack/err/bye/grant/unknown types), but NOT for -# deduplicated replays of msg/state/chunk, so consumers see each message once. -# -# Chunks and blobs (9.3) -# D1. Base64 alphabet and padding for `data` are unspecified ("base64", not -# "base64url"): we send standard base64 with padding and accept any variant. -# D2. The 9.3 example chunk has no `th`, yet 12.1 shows a chunk replayed from the -# outbox, and replay is per thread. We put the `th` of the carrying msg on -# every chunk we send (the envelope allows `th` on any type). Chunks live in -# the outbox and are replayed by the thread rule. We accept chunks with or -# without `th`. -# D3. Chunks are never acked, but "a sender ... MUST keep it until [acked]". -# Our outbox drops chunk entries when (a) the msg with the blob part is acked -# (we send the chunks before that msg, so on an ordered stream its ack means -# they were all processed), (b) the peer's resume `seen` covers them, or -# (c) the peer sends blob_refused. -# D4. Order: we send chunks first, then the msg with the blob part. Both orders -# are accepted on receipt ("before or after this message"). -# D5. Receiver: dedup by id first. Then per ref: n < expected is a duplicate and -# is skipped; n > expected is a gap: `err internal` (non-closing), chunk -# dropped. A missing `n` is taken as the expected one, and a missing `last` -# as false. Partial blobs are kept on disk, so a restarted receiver resumes -# mid-blob. After a crash, a partial file longer than the logged progress is -# truncated back to it. -# D6. The blob limit (default 50 MiB) is checked against the declared `size` of -# a blob part when the msg arrives first, and against the decoded bytes -# received. A refusal is `err` code blob_refused with `ref`, and `re` set to -# the offending id. Chunk data that is not valid base64 is refused the same -# way, since no non-closing code fits better. Further chunks of a refused ref -# are ignored quietly. -# D7. Completed blobs go to DIR/blobs//. Refs are scoped -# to the sending peer. A ref that already finished ignores later chunks. -# -# Ping (9.4) -# E1. "Idle" means no inbound frame for ping-interval seconds. Any inbound frame -# counts as liveness. A ping is "missed" when nothing arrives within one -# interval. After the 2nd miss the connection is dead, so a silent peer is -# dropped about 3 intervals after its last frame. --ping-interval 0 turns -# pinging off. -# -# Resume (9.5, 4.3) -# F1. The seen map we send lists, per thread, the last replayable id (msg, state, -# or chunk with th) received, in arrival order. The 12.1 example shows an -# ACK id ("a9") used as a seen value. We accept that, but we do not record -# acks ourselves: an ack that overtakes an older queued msg in the same thread -# would otherwise make its sender skip that msg on replay. Recording less -# only causes harmless extra replays. -# F2. Since a peer may record acks (F1), our own sending never lets a newer id -# overtake an older replayable one in a thread. One FIFO writer carries -# outbox entries and acks. Acks created before the peer's resume arrives -# are held until our replay has been queued. ping/pong/err have no th and -# skip the queue. -# F3. Seen as an implicit ack: outbox entries with id <= seen[th] are treated as -# acknowledged. They are pruned and an `acked` event is emitted. The replay -# rule would never resend them, so if their ack was lost they could otherwise -# never be pruned. Safety valve: a seen value that is not a ULID cannot -# name one of our ids, so for that thread we skip pruning and replay the -# whole thread; receivers dedup. Anything left in the outbox after this -# is replayed, which is exactly the entries with id > seen. -# F4. Resume is processed once per connection. If the peer sends none within -# 10 s we act as if seen were {} and replay everything; duplicates are safe. -# F5. Entries without th would be replayed under "threads the other side did not -# list". Our outbox never has any: grant messages are not thread messages and -# are never acked, so they sit in a separate durable queue and are sent once -# after the next resume. -# F6. Retention: outbox entries stay until acked, however long that takes. The -# "until closed or 7 days" idea in open question 2 is not adopted. -# -# Reconnection (4.3) -# G1. The connecting side always retries until its first successful handshake. -# After that it reconnects only while it has unacked outbox entries, queued -# grants or open threads. Backoff: 0.5 s doubling to a 60 s cap, with +/-10% -# jitter, reset after a successful handshake. No give-up timeout. -# G2. After a `bye` in either direction we do not reconnect automatically until -# new outbound work is queued. bye is an explicit end of the session; the -# state stays on disk and the next connection resumes it. -# -# bye (9.6) -# H1. Our bye queues behind writes already queued (graceful). After writing it -# we send nothing else, not even acks or pongs. We close on the peer's bye or -# after 5 s. Messages that arrive in the meantime are delivered but not acked, -# so they will be replayed and deduplicated later. When the peer says bye -# first we answer bye at once, echoing its reason, and close. -# -# Grants (section 10) -# I1. Canonical JSON: keys sorted by code point (recursively), separators "," and -# ":" with no whitespace, non-ASCII sent raw as UTF-8, minimal escaping -# (Python json). When verifying we also accept the ASCII-escaped form and the -# Go encoding/json HTML-escaped form (\u003c \u003e \u0026 \u2028 \u2029). -# Numbers are not canonicalised; grants should not contain any. Every field -# except `sig` is signed, unknown ones included. -# I2. exp: RFC 3339 UTC. Grants with a missing, unparsable or past exp are not -# honoured. nonce: 32 random bytes, unpadded base64url. ttl defaults to 3600. -# I3. Honouring (10.2): iss is our key, or a --trust key, or iss holds a valid -# unexpired grant containing "introduce" whose iss is our key or a --trust -# key. That supporting grant can come from the same auth, from any grant the -# peer sent, or from grants we issued. Only one level of delegation. `sub` -# must match the peer's key by raw bytes. -# I4. A received grant whose sub is our key is kept in grants_held.json and shown -# in our future auth. One whose sub is the peer's key is a presented grant. -# A grant whose sub is someone else is kept only as delegation evidence. -# I5. Enforcement: the only request shape the spec defines is exec (a data part -# with mime application/vnd.awp.exec+json). Without an honoured exec grant -# we reply `err forbidden` (re = msg id), and we still ack the msg because -# receipt is not permission. errs skip the FIFO, so on the wire the err comes -# before the ack. With a grant we only log it. This peer never runs anything. -# I6. `introduce` is received and logged, and its grant stored. We never connect -# to introduced peers on our own. -# -# Multiple peers and transports -# J1. The outbox, seen map, dedup set and threads are kept per peer key ("for that -# peer"). Commands go to the peer that most recently authenticated, or to -# "peer" if given. Before any peer has ever authenticated they go to a -# pending queue that the first authenticated peer adopts, with the original -# ids and timestamps. A new connection from a key that is already connected -# replaces the old one. -# J2. 4.2 says peers SHOULD refuse plaintext TCP across untrusted networks. We -# warn on stderr for non-loopback TCP but do not refuse (this is a test peer). -# J3. `disconnected` is emitted only for connections that reached `connected`. -# Failed handshakes produce `error` events instead. -# J4. Commands issued while disconnected get `sent` at once (the message is -# durably queued). bye with no live connection -> `error` event. -# J5. `sent` is emitted for msg, state, grant and bye. The blob command's `sent` -# names its msg; chunks get no events. `acked` is emitted for msg/state only, -# also when the ack is implied by a resume seen map (F3). -# -# Places where the spec looks wrong or self-contradictory: B3 (caps text vs -# examples), A2 (too_large close rule missing), D2/D3 (chunks have no th and no -# ack, yet sit in a per-thread, ack-pruned outbox), F1 (example uses an ack id as -# seen), B6 (symmetric transcript allows reflection), A6 (resume needs monotonic -# ids but only "recommends" them), 7.2 (auth binds only the hellos, so on -# plaintext bindings later lines are unauthenticated), and section 15's -# subsections being numbered 16.x. -# ============================================================================ - -from __future__ import annotations - -import argparse -import asyncio -import base64 -import binascii -import collections -import datetime as _dt -import fcntl -import hashlib -import json -import math -import mimetypes -import os -import random -import re as _re -import secrets -import shutil -import socket -import stat -import sys -import threading -import time -import traceback - -# ---------------------------------------------------------------- constants - -PROTOCOL_VERSION = 0 -AUTH_CONTEXT = b"awp-auth-v0" -MAX_LINE = 1 << 20 # bytes, excluding the "\n" -CHUNK_SIZE = 256 * 1024 # payload bytes per chunk, before base64 -DEFAULT_BLOB_LIMIT = 50 * 1024 * 1024 -DEFAULT_PING_INTERVAL = 30.0 -BACKOFF_INITIAL = 0.5 -BACKOFF_CAP = 60.0 -BYE_TIMEOUT = 5.0 -HANDSHAKE_TIMEOUT = 20.0 -RESUME_TIMEOUT = 10.0 -DEFAULT_GRANT_TTL = 3600 -MY_CAPS = ["chat", "blob", "grant"] -EXEC_MIME = "application/vnd.awp.exec+json" -CLOSING_ERR_CODES = frozenset({"bad_frame", "version", "auth", "too_large"}) -ACKED_TYPES = frozenset({"msg", "state"}) -SEEN_TYPES = frozenset({"msg", "state", "chunk"}) -PRE_AUTH_FORBIDDEN = frozenset({"msg", "state", "ack", "chunk", "resume", "grant", "introduce"}) -PENDING = "_pending" -READ_SIZE = 256 * 1024 - -FSYNC = os.environ.get("AWP_FSYNC", "1") != "0" -DEBUG = os.environ.get("AWP_DEBUG", "") not in ("", "0") -_LOG_NAME = "awp-py" - - -# ---------------------------------------------------------------- logging - -def log(msg: str) -> None: - try: - sys.stderr.write(f"{time.strftime('%H:%M:%S')} [{_LOG_NAME}] {msg}\n") - sys.stderr.flush() - except Exception: - pass - - -def debug(msg: str) -> None: - if DEBUG: - log(msg) - - -class CommandError(Exception): - pass - - -class StartupError(Exception): - pass - - -# ---------------------------------------------------------------- encodings - -def b64url(data: bytes) -> str: - """Unpadded base64url (RFC 4648 section 5).""" - return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") - - -def b64std(data: bytes) -> str: - """Standard padded base64 (RFC 4648 section 4).""" - return base64.b64encode(data).decode("ascii") - - -_B64_TO_STD = str.maketrans("-_", "+/") - - -def b64decode_any(s) -> bytes: - """Liberal base64 decoder: either alphabet, padded or not, whitespace ignored.""" - if not isinstance(s, str): - raise ValueError("not a string") - try: - return base64.b64decode(s, validate=True) # fast path: std + padding - except (binascii.Error, ValueError): - pass - t = "".join(s.split()).translate(_B64_TO_STD).rstrip("=") - if len(t) % 4 == 1: - raise ValueError("invalid base64 length") - t += "=" * (-len(t) % 4) - try: - return base64.b64decode(t, validate=True) - except (binascii.Error, ValueError) as e: - raise ValueError(f"invalid base64: {e}") from None - - -def parse_key(s) -> bytes: - """'ed25519:' -> 32 raw bytes (the identity).""" - if not isinstance(s, str): - raise ValueError("key is not a string") - prefix, sep, rest = s.partition(":") - if not sep or prefix.lower() != "ed25519": - raise ValueError("key must look like ed25519:") - raw = b64decode_any(rest) - if len(raw) != 32: - raise ValueError(f"ed25519 key must be 32 bytes, got {len(raw)}") - return raw - - -def format_key(raw: bytes) -> str: - return "ed25519:" + b64url(raw) - - -def key_fp(raw: bytes) -> str: - return hashlib.sha256(raw).hexdigest()[:32] - - -def key_matches(s, raw) -> bool: - if raw is None: - return False - try: - return parse_key(s) == raw - except ValueError: - return False - - -def safe_name(s: str) -> str: - """Make a sender-chosen string safe to use as a file name.""" - clean = _re.sub(r"[^A-Za-z0-9._-]", "_", s)[:120].lstrip(".") - if clean != s or not clean: - digest = hashlib.sha256(s.encode("utf-8", "surrogatepass")).hexdigest()[:12] - clean = f"{clean or 'x'}_{digest}" - return clean - - -def dumps_line(obj) -> bytes: - """One compact JSON line (no newline), UTF-8.""" - try: - return json.dumps(obj, ensure_ascii=False, separators=(",", ":"), - allow_nan=False).encode("utf-8") - except UnicodeEncodeError: # lone surrogates: fall back to \u escapes - return json.dumps(obj, ensure_ascii=True, separators=(",", ":"), - allow_nan=False).encode("ascii") - - -def canonical_json(obj) -> bytes: - """Canonical JSON for grant signatures: sorted keys, no whitespace, UTF-8.""" - return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False, - allow_nan=False).encode("utf-8") - - -def canonical_json_variants(obj): - """Byte strings a signer might reasonably have produced for `obj` (see I1).""" - out = [] - try: - base = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - except ValueError: - return out - try: - out.append(base.encode("utf-8")) - except UnicodeEncodeError: - pass - asc = json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("ascii") - if asc not in out: - out.append(asc) - go = (base.replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026") - .replace("\u2028", "\\u2028").replace("\u2029", "\\u2029")) - try: - gob = go.encode("utf-8") - if gob not in out: - out.append(gob) - except UnicodeEncodeError: - pass - return out - - -def blob_refs(obj) -> list: - parts = obj.get("parts") if isinstance(obj, dict) else None - if not isinstance(parts, list): - return [] - return [p["ref"] for p in parts - if isinstance(p, dict) and p.get("k") == "blob" and isinstance(p.get("ref"), str)] - - -# ---------------------------------------------------------------- time - -def fmt_ts(t: float, millis: bool = True) -> str: - d = _dt.datetime.fromtimestamp(t, _dt.timezone.utc) - s = d.strftime("%Y-%m-%dT%H:%M:%S") - if millis: - s += ".%03d" % (d.microsecond // 1000) - return s + "Z" - - -def now_ts() -> str: - return fmt_ts(time.time()) - - -_RFC3339 = _re.compile( - r"^(\d{4})-(\d{2})-(\d{2})[Tt ](\d{2}):(\d{2}):(\d{2})(\.\d+)?([Zz]|[+-]\d{2}:?\d{2})$") - - -def parse_rfc3339(s) -> float: - m = _RFC3339.match(s) if isinstance(s, str) else None - if not m: - raise ValueError(f"not an RFC 3339 timestamp: {s!r}") - y, mo, d, hh, mi, ss = (int(m.group(i)) for i in range(1, 7)) - ss = min(ss, 59) # leap second - t = _dt.datetime(y, mo, d, hh, mi, ss, tzinfo=_dt.timezone.utc).timestamp() - if m.group(7): - t += float(m.group(7)) - tz = m.group(8) - if tz not in ("Z", "z"): - digits = tz[1:].replace(":", "") - off = int(digits[:2]) * 3600 + int(digits[2:]) * 60 - t -= off if tz[0] == "+" else -off - return t - - -# ---------------------------------------------------------------- ULID ids - -_CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" -_CROCKFORD_INDEX = {c: i for i, c in enumerate(_CROCKFORD)} - - -def ulid_encode(v: int) -> str: - out = [] - for _ in range(26): - out.append(_CROCKFORD[v & 31]) - v >>= 5 - return "".join(reversed(out)) - - -def ulid_decode(s): - if not isinstance(s, str) or len(s) != 26: - return None - v = 0 - for ch in s.upper(): - i = _CROCKFORD_INDEX.get(ch) - if i is None: - return None - v = (v << 5) | i - return v if v < (1 << 128) else None - - -class UlidGen: - """Monotonic ULIDs: strictly increasing even within one millisecond.""" - - def __init__(self): - self.last_ms = 0 - self.last_rand = 0 - - def observe(self, ulid) -> None: - v = ulid_decode(ulid) - if v is None: - return - ms, rnd = v >> 80, v & ((1 << 80) - 1) - if (ms, rnd) > (self.last_ms, self.last_rand): - self.last_ms, self.last_rand = ms, rnd - - def new(self) -> str: - ms = time.time_ns() // 1_000_000 - if ms > self.last_ms: - rnd = secrets.randbits(80) - else: - ms = self.last_ms - rnd = self.last_rand + 1 - if rnd >> 80: - ms += 1 - rnd = secrets.randbits(80) - self.last_ms, self.last_rand = ms, rnd - return ulid_encode((ms << 80) | rnd) - - -# ---------------------------------------------------------------- Ed25519 -# Pure-Python fallback, after the RFC 8032 section 6 reference code. - -_P = 2 ** 255 - 19 -_L = 2 ** 252 + 27742317777372353535851937790883648493 -_D = -121665 * pow(121666, _P - 2, _P) % _P -_SQRT_M1 = pow(2, (_P - 1) // 4, _P) - - -def _padd(a, b): - A = (a[1] - a[0]) * (b[1] - b[0]) % _P - B = (a[1] + a[0]) * (b[1] + b[0]) % _P - C = 2 * a[3] * b[3] * _D % _P - D = 2 * a[2] * b[2] % _P - E, F, G, H = B - A, D - C, D + C, B + A - return (E * F % _P, G * H % _P, F * G % _P, E * H % _P) - - -def _pmul(s, pt): - q = (0, 1, 1, 0) - while s > 0: - if s & 1: - q = _padd(q, pt) - pt = _padd(pt, pt) - s >>= 1 - return q - - -def _pequal(a, b): - return ((a[0] * b[2] - b[0] * a[2]) % _P == 0 and - (a[1] * b[2] - b[1] * a[2]) % _P == 0) - - -def _recover_x(y, sign): - if y >= _P: - return None - x2 = (y * y - 1) * pow(_D * y * y + 1, _P - 2, _P) - if x2 % _P == 0: - return None if sign else 0 - x = pow(x2, (_P + 3) // 8, _P) - if (x * x - x2) % _P != 0: - x = x * _SQRT_M1 % _P - if (x * x - x2) % _P != 0: - return None - if (x & 1) != sign: - x = _P - x - return x - - -_GY = 4 * pow(5, _P - 2, _P) % _P -_GX = _recover_x(_GY, 0) -_G = (_GX, _GY, 1, _GX * _GY % _P) - - -def _compress(pt) -> bytes: - zinv = pow(pt[2], _P - 2, _P) - x = pt[0] * zinv % _P - y = pt[1] * zinv % _P - return int.to_bytes(y | ((x & 1) << 255), 32, "little") - - -def _decompress(s): - if len(s) != 32: - return None - y = int.from_bytes(s, "little") - sign = y >> 255 - y &= (1 << 255) - 1 - x = _recover_x(y, sign) - if x is None: - return None - return (x, y, 1, x * y % _P) - - -def _sha512_modl(data: bytes) -> int: - return int.from_bytes(hashlib.sha512(data).digest(), "little") % _L - - -def _expand(seed: bytes): - h = hashlib.sha512(seed).digest() - a = int.from_bytes(h[:32], "little") - a &= (1 << 254) - 8 - a |= 1 << 254 - return a, h[32:] - - -def pure_ed25519_public(seed: bytes) -> bytes: - a, _ = _expand(seed) - return _compress(_pmul(a, _G)) - - -def pure_ed25519_sign(seed: bytes, msg: bytes) -> bytes: - a, prefix = _expand(seed) - pub = _compress(_pmul(a, _G)) - r = _sha512_modl(prefix + msg) - rs = _compress(_pmul(r, _G)) - h = _sha512_modl(rs + pub + msg) - s = (r + h * a) % _L - return rs + int.to_bytes(s, 32, "little") - - -def pure_ed25519_verify(pub: bytes, sig: bytes, msg: bytes) -> bool: - if len(pub) != 32 or len(sig) != 64: - return False - a_pt = _decompress(pub) - if a_pt is None: - return False - rs = sig[:32] - r_pt = _decompress(rs) - if r_pt is None: - return False - s = int.from_bytes(sig[32:], "little") - if s >= _L: - return False - h = _sha512_modl(rs + pub + msg) - return _pequal(_pmul(s, _G), _padd(r_pt, _pmul(h, a_pt))) - - -try: - if os.environ.get("AWP_PURE_ED25519", "") not in ("", "0"): - raise ImportError("pure-Python Ed25519 forced by AWP_PURE_ED25519") - from cryptography.exceptions import InvalidSignature as _InvalidSignature - from cryptography.hazmat.primitives import serialization as _ser - from cryptography.hazmat.primitives.asymmetric.ed25519 import ( - Ed25519PrivateKey as _SK, Ed25519PublicKey as _PK) - ED25519_BACKEND = "cryptography" -except ImportError: - ED25519_BACKEND = "pure-python" - - -def ed25519_public(seed: bytes) -> bytes: - if ED25519_BACKEND == "cryptography": - return _SK.from_private_bytes(seed).public_key().public_bytes( - _ser.Encoding.Raw, _ser.PublicFormat.Raw) - return pure_ed25519_public(seed) - - -def ed25519_sign(seed: bytes, msg: bytes) -> bytes: - if ED25519_BACKEND == "cryptography": - return _SK.from_private_bytes(seed).sign(msg) - return pure_ed25519_sign(seed, msg) - - -def ed25519_verify(pub: bytes, sig: bytes, msg: bytes) -> bool: - if not isinstance(sig, (bytes, bytearray)) or len(sig) != 64 or len(pub) != 32: - return False - if ED25519_BACKEND == "cryptography": - try: - _PK.from_public_bytes(bytes(pub)).verify(bytes(sig), msg) - return True - except (_InvalidSignature, ValueError): - return False - return pure_ed25519_verify(bytes(pub), bytes(sig), msg) - - -class Identity: - def __init__(self, seed: bytes): - self.seed = seed - self.pub = ed25519_public(seed) - self.key = format_key(self.pub) - - def sign(self, msg: bytes) -> bytes: - return ed25519_sign(self.seed, msg) - - @classmethod - def load_or_create(cls, path: str) -> "Identity": - for _ in range(3): - try: - with open(path, "r", encoding="utf-8") as f: - rec = json.load(f) - seed = b64decode_any(rec["seed"]) - if len(seed) != 32: - raise ValueError("seed must be 32 bytes") - return cls(seed) - except FileNotFoundError: - pass - except (ValueError, KeyError, TypeError) as e: - raise StartupError(f"identity file {path} is unreadable: {e}") from None - seed = os.urandom(32) - ident = cls(seed) - rec = {"alg": "ed25519", "seed": b64url(seed), "key": ident.key, "created": now_ts()} - try: - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - except FileExistsError: - time.sleep(0.05) - continue - with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(rec, f) - f.flush() - os.fsync(f.fileno()) - return ident - raise StartupError(f"could not create identity at {path}") - - -# ---------------------------------------------------------------- grants - -def verify_grant(g): - """Signature, shape and expiry check. Returns (ok, reason).""" - if not isinstance(g, dict): - return False, "grant is not an object" - try: - iss = parse_key(g.get("iss")) - parse_key(g.get("sub")) - sig = b64decode_any(g.get("sig")) - except ValueError as e: - return False, f"malformed grant: {e}" - caps = g.get("caps") - if not isinstance(caps, list) or not all(isinstance(c, str) for c in caps): - return False, "caps must be a list of strings" - body = {k: v for k, v in g.items() if k != "sig"} - if not any(ed25519_verify(iss, sig, m) for m in canonical_json_variants(body)): - return False, "bad signature" - try: - exp = parse_rfc3339(g.get("exp")) - except ValueError as e: - return False, f"bad exp: {e}" - if exp <= time.time(): - return False, "expired" - return True, "ok" - - -def mint_grant(identity: Identity, sub: str, caps, ttl: float) -> dict: - g = {"iss": identity.key, "sub": sub, "caps": list(caps), - "exp": fmt_ts(math.ceil(time.time() + ttl), millis=False), - "nonce": b64url(os.urandom(32))} - g["sig"] = b64url(identity.sign(canonical_json(g))) - return g - - -# ---------------------------------------------------------------- persistence - -def load_json(path: str, default): - try: - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - except FileNotFoundError: - return default - except (ValueError, OSError) as e: - log(f"warning: cannot read {path}: {e}; starting from empty") - return default - - -def atomic_write_json(path: str, obj, durable: bool = False) -> None: - tmp = f"{path}.tmp{os.getpid()}" - with open(tmp, "w", encoding="utf-8") as f: - json.dump(obj, f, ensure_ascii=True, separators=(",", ":")) - f.flush() - if durable and FSYNC: - os.fsync(f.fileno()) - os.replace(tmp, path) - - -def read_jsonl(path: str) -> list: - """Read an append-only JSONL log, dropping a torn final record.""" - try: - with open(path, "rb") as f: - data = f.read() - except FileNotFoundError: - return [] - end = data.rfind(b"\n") + 1 - if end < len(data): - log(f"warning: {path} ends with a partial record (killed mid-write?); truncating it") - with open(path, "r+b") as f: - f.truncate(end) - out = [] - for ln in data[:end].split(b"\n"): - if not ln.strip(): - continue - try: - rec = json.loads(ln) - except ValueError: - log(f"warning: skipping a corrupt record in {path}") - continue - if isinstance(rec, dict): - out.append(rec) - return out - - -def _rec_bytes(rec) -> bytes: - return json.dumps(rec, ensure_ascii=True, separators=(",", ":")).encode("ascii") + b"\n" - - -class AppendLog: - def __init__(self, path: str): - self.path = path - self.f = open(path, "ab") - - def append(self, recs, durable: bool = True, raw: bytes = None) -> None: - if isinstance(recs, dict): - recs = [recs] - self.f.write(raw if raw is not None else b"".join(_rec_bytes(r) for r in recs)) - self.f.flush() - if durable and FSYNC: - os.fsync(self.f.fileno()) - - def close(self) -> None: - try: - self.f.close() - except OSError: - pass - - -class Outbox: - """Unacked outbound msg/state/chunk entries, in original send order. - - entry = {"obj": wire object (chunks without "data")} - + {"src": file in outgoing/, "off": int, "len": int} for chunks - """ - - def __init__(self, path: str, outgoing_dir: str): - self.path = path - self.outgoing_dir = outgoing_dir - self.entries: dict = {} - self.by_ref: dict = {} - self.src_count: dict = {} - self.max_id = None - self.dels = 0 - for rec in read_jsonl(path): - op = rec.get("op") - if op == "add": - e = rec.get("e") - if self._valid(e) and e["obj"]["id"] not in self.entries: - self._mem_add(e) - elif op == "del": - if self._mem_del(rec.get("id")) is not None: - self.dels += 1 - elif op == "mark": - self._note_id(rec.get("id")) - self.log = AppendLog(path) - if self.dels: - self.compact() - - @staticmethod - def _valid(e) -> bool: - return (isinstance(e, dict) and isinstance(e.get("obj"), dict) - and isinstance(e["obj"].get("id"), str) and isinstance(e["obj"].get("t"), str)) - - def _note_id(self, i) -> None: - if isinstance(i, str) and (self.max_id is None or i > self.max_id): - self.max_id = i - - def _mem_add(self, e) -> None: - obj = e["obj"] - eid = obj["id"] - self.entries[eid] = e - self._note_id(eid) - if obj["t"] == "chunk": - self.by_ref.setdefault(obj.get("ref"), set()).add(eid) - src = e.get("src") - if src: - self.src_count[src] = self.src_count.get(src, 0) + 1 - - def _mem_del(self, eid): - e = self.entries.pop(eid, None) if isinstance(eid, str) else None - if e is None: - return None - obj = e["obj"] - if obj["t"] == "chunk": - ids = self.by_ref.get(obj.get("ref")) - if ids is not None: - ids.discard(eid) - if not ids: - del self.by_ref[obj.get("ref")] - src = e.get("src") - if src: - n = self.src_count.get(src, 0) - 1 - if n <= 0: - self.src_count.pop(src, None) - else: - self.src_count[src] = n - return e - - def add_many(self, entries) -> list: - fresh = [e for e in entries if e["obj"]["id"] not in self.entries] - if fresh: - self.log.append([{"op": "add", "e": e} for e in fresh], durable=True) - for e in fresh: - self._mem_add(e) - return fresh - - def remove(self, eid, delete_files: bool = True): - e = self._mem_del(eid) - if e is None: - return None - self.log.append({"op": "del", "id": eid}, durable=False) - self.dels += 1 - src = e.get("src") - if delete_files and src and src not in self.src_count: - try: - os.remove(os.path.join(self.outgoing_dir, src)) - except OSError: - pass - if self.dels > 512 and self.dels > 4 * len(self.entries): - self.compact() - return e - - def compact(self) -> None: - tmp = self.path + ".compact" - with open(tmp, "wb") as f: - if self.max_id: - f.write(_rec_bytes({"op": "mark", "id": self.max_id})) - for e in self.entries.values(): - f.write(_rec_bytes({"op": "add", "e": e})) - f.flush() - if FSYNC: - os.fsync(f.fileno()) - self.log.close() - os.replace(tmp, self.path) - self.log = AppendLog(self.path) - self.dels = 0 - - def reset(self) -> None: - """Forget everything (after another outbox adopted our entries).""" - self.entries.clear() - self.by_ref.clear() - self.src_count.clear() - self.compact() - - -class Inbox: - """What we have durably received from one peer: dedup set, per-thread seen, - blob progress. Everything is rebuilt from an append-only log at startup.""" - - def __init__(self, path: str, blob_dir: str): - self.path = path - self.blob_dir = blob_dir - self.partial_dir = os.path.join(blob_dir, ".partial") - self.dedup: set = set() - self.seen: dict = {} - self.partial: dict = {} # ref -> {"next": n, "size": bytes, "last": bool} - self.done: dict = {} - self.refused: set = set() - self.declared: dict = {} # ref -> size from a blob part (memory only) - self.to_finish: list = [] - for rec in read_jsonl(path): - self._apply(rec) - self.log = AppendLog(path) - self._reconcile() - - def partial_path(self, ref: str) -> str: - return os.path.join(self.partial_dir, safe_name(ref)) - - def final_path(self, ref: str) -> str: - return os.path.join(self.blob_dir, safe_name(ref)) - - def _apply(self, rec) -> None: - op = rec.get("op") - ref = rec.get("ref") - if op == "blob_done": - self.done[ref] = rec - self.partial.pop(ref, None) - return - if op == "blob_refused": - self.refused.add(ref) - self.partial.pop(ref, None) - return - mid, t, th = rec.get("id"), rec.get("t"), rec.get("th") - if isinstance(mid, str): - self.dedup.add(mid) - if isinstance(th, str) and t in SEEN_TYPES: - self.seen[th] = mid - if t == "chunk" and rec.get("stored"): - st = self.partial.get(ref) - if st is None: - st = self.partial[ref] = {"next": 0, "size": 0, "last": False} - st["next"] = int(rec.get("n", 0)) + 1 - st["size"] += int(rec.get("len", 0)) - st["last"] = bool(rec.get("last")) - - def record(self, rec, durable: bool = True, raw: bytes = None) -> None: - self.log.append(rec, durable=durable, raw=raw) - self._apply(rec) - - def _reconcile(self) -> None: - for ref, st in list(self.partial.items()): - p, final = self.partial_path(ref), self.final_path(ref) - if not os.path.exists(p): - if st["last"] and os.path.exists(final) and os.path.getsize(final) == st["size"]: - self.to_finish.append(ref) # renamed but not yet logged - else: - log(f"warning: data for partial blob {ref!r} is missing; it cannot complete") - del self.partial[ref] - continue - size = os.path.getsize(p) - if size > st["size"]: - with open(p, "r+b") as f: # data written, record not: roll back - f.truncate(st["size"]) - elif size < st["size"]: - log(f"warning: partial blob {ref!r} is shorter than its log; discarding it") - del self.partial[ref] - os.remove(p) - continue - if st["last"]: - self.to_finish.append(ref) - - -class Threads: - def __init__(self, path: str): - self.path = path - data = load_json(path, {}) - self.data = data if isinstance(data, dict) else {} - - def touch(self, th: str, obj: dict, outgoing: bool) -> None: - rec = self.data.get(th) - changed = False - if not isinstance(rec, dict): - rec = self.data[th] = {"closed": False} - subj = obj.get("subject") - if isinstance(subj, str): - rec["subject"] = subj - changed = True - closed = False - if obj.get("t") == "state": - st = obj.get("state") - closed = st == "closed" - side = "mine" if outgoing else "theirs" - if rec.get(side) != st: - rec[side] = st - changed = True - if rec.get("closed") != closed: - rec["closed"] = closed - changed = True - if changed: - self.save() - - def any_open(self) -> bool: - return any(isinstance(r, dict) and not r.get("closed") for r in self.data.values()) - - def merge(self, other: "Threads") -> None: - for th, rec in other.data.items(): - if th not in self.data: - self.data[th] = rec - self.save() - - def clear(self) -> None: - self.data = {} - self.save() - - def save(self) -> None: - atomic_write_json(self.path, self.data) - - -class SendOnce: - """Messages that are neither thread messages nor acked (grant): sent once.""" - - def __init__(self, path: str): - self.path = path - lst = load_json(path, []) - self.items = {o["id"]: o for o in lst if isinstance(o, dict) and isinstance(o.get("id"), str)} \ - if isinstance(lst, list) else {} - - def add(self, obj) -> None: - self.items[obj["id"]] = obj - self.save() - - def remove(self, oid) -> None: - if self.items.pop(oid, None) is not None: - self.save() - - def clear(self) -> None: - self.items = {} - self.save() - - def save(self) -> None: - atomic_write_json(self.path, list(self.items.values()), durable=True) - - -class GrantList: - def __init__(self, path: str): - self.path = path - lst = load_json(path, []) - self.items = [g for g in lst if isinstance(g, dict)] if isinstance(lst, list) else [] - self.prune() - - def prune(self) -> None: - keep = [] - for g in self.items: - try: - if parse_rfc3339(g.get("exp")) > time.time(): - keep.append(g) - except ValueError: - pass - if len(keep) != len(self.items): - self.items = keep - self.save() - - def add(self, g) -> None: - if any(x.get("sig") == g.get("sig") for x in self.items): - return - self.items.append(g) - self.save() - - def valid(self) -> list: - self.prune() - return list(self.items) - - def save(self) -> None: - atomic_write_json(self.path, self.items) - - -class PeerState: - """All durable state we keep about one remote key (or the pending queue).""" - - def __init__(self, node: "Node", fp: str, key_raw=None, key_str=None): - self.node = node - self.fp = fp - self.dir = os.path.join(node.state_dir, "peers", fp) - os.makedirs(self.dir, exist_ok=True) - self.meta_path = os.path.join(self.dir, "peer.json") - meta = load_json(self.meta_path, {}) - self.meta = meta if isinstance(meta, dict) else {} - if key_raw is None and fp != PENDING: - try: - key_raw = parse_key(self.meta.get("key")) - key_str = self.meta.get("key_as_sent") or self.meta.get("key") - except ValueError: - key_raw = None - self.key_raw = key_raw - self.key_str = key_str - self.outbox = Outbox(os.path.join(self.dir, "outbox.log"), node.outgoing_dir) - self.inbox = Inbox(os.path.join(self.dir, "inbox.log"), - os.path.join(node.state_dir, "blobs", fp)) - self.threads = Threads(os.path.join(self.dir, "threads.json")) - self.sendonce = SendOnce(os.path.join(self.dir, "sendonce.json")) - self.grants = GrantList(os.path.join(self.dir, "grants.json")) - if key_raw is not None and not self.meta.get("key"): - self.save_meta(key_str or format_key(key_raw), None) - - def save_meta(self, key_str: str, hello) -> None: - self.key_str = key_str - self.meta.update({"key": format_key(self.key_raw), "key_as_sent": key_str}) - if isinstance(hello, dict): - for k in ("name", "about", "caps"): - if k in hello: - self.meta[k] = hello[k] - self.meta["last_connected"] = now_ts() - atomic_write_json(self.meta_path, self.meta) - - def has_queued(self) -> bool: - return bool(self.outbox.entries or self.sendonce.items or self.threads.data) - - def has_work(self) -> bool: - return bool(self.outbox.entries or self.sendonce.items) or self.threads.any_open() - - def adopt(self, other: "PeerState") -> None: - if not other.has_queued(): - return - log(f"peer {self.key_str}: adopting {len(other.outbox.entries)} message(s) queued " - "before any peer had connected") - self.outbox.add_many(list(other.outbox.entries.values())) - for obj in list(other.sendonce.items.values()): - self.sendonce.add(obj) - self.threads.merge(other.threads) - other.outbox.reset() - other.sendonce.clear() - other.threads.clear() - - -# ---------------------------------------------------------------- addresses - -def parse_addr(s: str): - if s.startswith("tcp:"): - host, sep, port = s[4:].rpartition(":") - if not sep or not port.isdigit(): - raise StartupError(f"bad tcp address {s!r}; expected tcp:HOST:PORT") - return ("tcp", host.strip("[]"), int(port)) - if s.startswith("unix:"): - path = s[5:] - if not path: - raise StartupError(f"bad unix address {s!r}; expected unix:/path") - return ("unix", path) - if "/" in s: - return ("unix", s) - host, sep, port = s.rpartition(":") - if sep and port.isdigit(): - return ("tcp", host.strip("[]"), int(port)) - raise StartupError(f"unrecognised address {s!r}; use tcp:HOST:PORT or unix:/path") - - -def fmt_host(host: str) -> str: - return f"[{host}]" if ":" in host else host - - -def is_loopback(host: str) -> bool: - return host in ("localhost", "::1") or host.startswith("127.") - - -async def open_stream(target): - if target[0] == "tcp": - return await asyncio.open_connection(target[1], target[2], limit=2 * MAX_LINE) - return await asyncio.open_unix_connection(target[1], limit=2 * MAX_LINE) - - -def make_tcp_listener(host: str, port: int) -> socket.socket: - infos = socket.getaddrinfo(host or None, port, type=socket.SOCK_STREAM, - flags=socket.AI_PASSIVE) - if not infos: - raise StartupError(f"cannot resolve {host!r}") - fam, typ, proto, _, sa = infos[0] - s = socket.socket(fam, typ, proto) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - s.bind(sa) - except OSError as e: - s.close() - raise StartupError(f"cannot listen on tcp:{fmt_host(host)}:{port}: {e}") from None - s.listen(128) - s.setblocking(False) - return s - - -def prepare_unix_path(path: str) -> None: - try: - st = os.lstat(path) - except FileNotFoundError: - d = os.path.dirname(path) - if d: - os.makedirs(d, exist_ok=True) - return - if not stat.S_ISSOCK(st.st_mode): - raise StartupError(f"{path} exists and is not a socket") - probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - probe.connect(path) - except OSError: - os.unlink(path) # stale socket from a dead process - return - finally: - probe.close() - raise StartupError(f"{path}: another process is already listening there") - - -# ---------------------------------------------------------------- connection - -class Connection: - """One transport connection: handshake, reader, FIFO writer, timers.""" - - def __init__(self, node: "Node", reader, writer, label: str): - self.node = node - self.reader = reader - self.writer = writer - self.label = label - self.loop = asyncio.get_running_loop() - now = self.loop.time() - self.started = now - self.done = asyncio.Event() - self.closing = False - self.close_reason = None - self.linger = False - self.hello = node.make_hello() - self.my_hello_line = dumps_line(self.hello) - self.peer_hello_line = None - self.peer_hello = None - self.peer_key_raw = None - self.peer_key_str = None - self.established = False - self.established_at = 0.0 - self.got_resume = False - self.live = False - self.ps = None - self.queue = collections.deque() - self.qevent = asyncio.Event() - self.held = [] - self.bye_queued = False - self.bye_sent = False - self.bye_sent_at = 0.0 - self.last_inbound = now - self.ping_id = None - self.ping_at = 0.0 - self.missed = 0 - - # -- lifecycle --------------------------------------------------------- - - async def run(self) -> bool: - self.node.all_conns.add(self) - tasks = [] - try: - self._write(self.my_hello_line) - tasks = [asyncio.create_task(self._read_loop()), - asyncio.create_task(self._pump()), - asyncio.create_task(self._timer())] - await self.done.wait() - finally: - for t in tasks: - t.cancel() - await asyncio.gather(*tasks, return_exceptions=True) - await self._shutdown() - self.node.on_conn_closed(self) - return self.established - - def close(self, reason: str) -> None: - if self.closing: - return - self.closing = True - self.close_reason = reason - self.done.set() - self.qevent.set() - - async def _shutdown(self) -> None: - w = self.writer - try: - await asyncio.wait_for(w.drain(), 2.0) - except Exception: - pass - if self.linger and not w.is_closing(): - # We sent a closing err: half-close and drain input briefly so the - # kernel does not answer unread data with a RST that could destroy - # the err line before the peer reads it. - try: - if w.can_write_eof(): - w.write_eof() - deadline = self.loop.time() + 1.0 - while True: - left = deadline - self.loop.time() - if left <= 0: - break - data = await asyncio.wait_for(self.reader.read(READ_SIZE), left) - if not data: - break - except Exception: - pass - try: - w.close() - await asyncio.wait_for(w.wait_closed(), 2.0) - except Exception: - pass - - # -- writing ------------------------------------------------------------- - - def _write(self, line: bytes) -> bool: - if self.writer.is_closing(): - return False - try: - self.writer.write(line + b"\n") - return True - except Exception as e: - self.close(f"write failed: {e}") - return False - - def send_now(self, obj) -> bool: - """Write a control message immediately (bypasses the FIFO).""" - if self.bye_sent or self.closing: - return False - return self._write(dumps_line(obj)) - - def send_err(self, code: str, detail: str, re=None, ref=None) -> None: - log(f"{self.label}: sending err {code}: {detail}") - self.send_now(self.node.make_err(code, detail, re=re, ref=ref)) - - def fail(self, code: str, detail: str, re=None) -> None: - """Send a closing err and close.""" - if self.closing: - return - if not self.bye_sent: - self._write(dumps_line(self.node.make_err(code, detail, re=re))) - self.node.emit_error(f"{self.label}: sent err {code}: {detail}") - self.linger = True - self.close(f"err {code}: {detail}") - - def queue_entry(self, eid: str) -> None: - if self.live and not self.bye_queued and not self.closing: - self.queue.append(("entry", eid)) - self.qevent.set() - # otherwise the replay after the peer's resume picks it up - - def queue_once(self, oid: str) -> None: - if self.live and not self.bye_queued and not self.closing: - self.queue.append(("once", oid)) - self.qevent.set() - - def queue_ack(self, th: str, re_id: str) -> None: - if self.bye_queued or self.closing: - return - item = ("line", dumps_line(self.node.envelope("ack", th=th, re=re_id))) - if self.live: - self.queue.append(item) - self.qevent.set() - else: - self.held.append(item) # see F2: never overtake our own replay - - def initiate_bye(self, reason: str): - if self.bye_queued or self.bye_sent or self.closing: - return None - obj = self.node.envelope("bye", reason=reason) - line = dumps_line(obj) - self.bye_queued = True - if self.live: - self.queue.append(("bye", line)) - self.qevent.set() - else: - self._write(line) - self.bye_sent = True - self.bye_sent_at = self.loop.time() - return obj["id"] - - def go_live(self, seen: dict) -> None: - """Peer's resume arrived: replay, then flush held acks, then go FIFO.""" - if self.live or self.closing: - return - ps = self.ps - self.node.apply_seen(ps, seen) - replay = list(ps.outbox.entries.keys()) - for eid in replay: - self.queue.append(("entry", eid)) - for oid in list(ps.sendonce.items.keys()): - self.queue.append(("once", oid)) - self.queue.extend(self.held) - self.held.clear() - self.live = True - if replay: - log(f"{self.label}: replaying {len(replay)} outbox message(s)") - self.qevent.set() - - async def _pump(self) -> None: - try: - while not self.closing: - if not self.queue or self.bye_sent: - self.qevent.clear() - await self.qevent.wait() - continue - kind, val = self.queue.popleft() - if kind == "entry": - e = self.ps.outbox.entries.get(val) - if e is None: - continue # pruned while queued - try: - line = self.node.entry_line(e) - except (OSError, ValueError) as ex: - self.node.emit_error(f"cannot read outgoing blob data for {val}: {ex}; dropping it") - self.node.prune(self.ps, val, "unreadable") - continue - elif kind == "once": - obj = self.ps.sendonce.items.get(val) - if obj is None: - continue - line = dumps_line(obj) - else: - line = val - if not self._write(line): - break - if kind == "once": - self.ps.sendonce.remove(val) - elif kind == "bye": - self.bye_sent = True - self.bye_sent_at = self.loop.time() - self.queue.clear() - await self.writer.drain() - except asyncio.CancelledError: - raise - except Exception as ex: - self.close(f"write failed: {ex}") - - # -- timers -------------------------------------------------------------- - - async def _timer(self) -> None: - iv = self.node.ping_interval - tick = 0.25 if iv <= 0 else max(0.02, min(0.5, iv / 5.0)) - while not self.closing: - await asyncio.sleep(tick) - if self.closing: - return - now = self.loop.time() - if not self.established: - if now - self.started > HANDSHAKE_TIMEOUT: - self.node.emit_error(f"{self.label}: handshake timed out") - self.close("handshake timeout") - return - continue - if not self.got_resume and now - self.established_at > RESUME_TIMEOUT: - log(f"{self.label}: no resume from peer within {RESUME_TIMEOUT:.0f}s; replaying everything") - self.got_resume = True - self.go_live({}) - if self.bye_sent: - if now - self.bye_sent_at >= BYE_TIMEOUT: - self.close("bye (peer did not answer within 5s)") - return - continue - if iv <= 0: - continue - if self.ping_id is None: - if now - self.last_inbound >= iv: - self._send_ping(now) - elif now - self.ping_at >= iv: - self.missed += 1 - if self.missed >= 2: - self.close(f"dead connection: {self.missed} missed pongs") - return - self._send_ping(now) - - def _send_ping(self, now: float) -> None: - ping = self.node.envelope("ping") - self.ping_id = ping["id"] - self.ping_at = now - self.send_now(ping) - - # -- reading ------------------------------------------------------------- - - async def _read_loop(self) -> None: - buf = bytearray() - scan = 0 - try: - while not self.closing: - data = await self.reader.read(READ_SIZE) - if not data: - self.close("connection closed by peer") - return - buf += data - while not self.closing: - i = buf.find(b"\n", scan) - if i < 0: - scan = len(buf) - if len(buf) > MAX_LINE: - self.fail("too_large", f"line exceeds {MAX_LINE} bytes") - return - break - line = bytes(buf[:i]) - del buf[:i + 1] - scan = 0 - if len(line) > MAX_LINE: - self.fail("too_large", f"line of {len(line)} bytes exceeds {MAX_LINE}") - return - self._handle_line(line) - except asyncio.CancelledError: - raise - except (ConnectionError, OSError) as e: - self.close(f"read failed: {e}") - except Exception as e: - log(f"{self.label}: internal error in reader: {e}\n{traceback.format_exc()}") - self.close(f"internal error: {e}") - - def _handle_line(self, line: bytes) -> None: - self.last_inbound = self.loop.time() - self.ping_id = None - self.missed = 0 - if not line.strip(): - return # A3 - try: - obj = json.loads(line.decode("utf-8")) - except (UnicodeDecodeError, ValueError) as e: - self.fail("bad_frame", f"line is not valid UTF-8 JSON: {e}") - return - if not isinstance(obj, dict): - self.fail("bad_frame", "line is not a JSON object") - return - t = obj.get("t") - try: - if not self.established: - self._handshake(line, obj, t) - else: - self._dispatch(obj, t) - except Exception as e: - log(f"{self.label}: internal error handling {t!r}: {e}\n{traceback.format_exc()}") - oid = obj.get("id") - self.send_err("internal", f"internal error handling {t!r}: {e}", - re=oid if isinstance(oid, str) else None) - - # -- handshake ----------------------------------------------------------- - - def _handshake(self, line: bytes, obj: dict, t) -> None: - node = self.node - oid = obj.get("id") if isinstance(obj.get("id"), str) else None - if t == "hello": - if self.peer_hello is not None: - log(f"{self.label}: ignoring repeated hello") - return - node.emit_recv(obj) - v = obj.get("v", 0) - if isinstance(v, bool) or not isinstance(v, (int, float)) or v != PROTOCOL_VERSION: - self.fail("version", f"unsupported protocol version {v!r}; this peer speaks {PROTOCOL_VERSION}", re=oid) - return - try: - raw = parse_key(obj.get("key")) - except ValueError as e: - self.fail("auth", f"invalid key in hello: {e}", re=oid) - return - if raw == node.identity.pub or line == self.my_hello_line: - self.fail("auth", "hello carries our own key (reflected handshake)", re=oid) - return - self.peer_hello_line = line - self.peer_hello = obj - self.peer_key_raw = raw - self.peer_key_str = obj.get("key") - log(f"{self.label}: hello from {self.peer_key_str} name={obj.get('name')!r}") - sig = node.identity.sign(AUTH_CONTEXT + b"\x00" + self.my_hello_line + b"\x00" + line) - held = node.held.valid() - auth = node.envelope("auth", sig=b64url(sig), grants=held if held else None) - self._write(dumps_line(auth)) - return - if t == "auth": - if self.peer_hello is None: - self.fail("auth", "auth received before hello", re=oid) - return - node.emit_recv(obj) - try: - sig = b64decode_any(obj.get("sig")) - except ValueError: - sig = b"" - transcript = AUTH_CONTEXT + b"\x00" + self.peer_hello_line + b"\x00" + self.my_hello_line - if not ed25519_verify(self.peer_key_raw, sig, transcript): - self.fail("auth", "auth signature does not verify", re=oid) - return - self._on_established(obj) - return - if t == "err": - node.emit_recv(obj) - code = obj.get("code") - node.emit_error(f"{self.label}: peer sent err {code} during handshake: {obj.get('detail')}") - if code in CLOSING_ERR_CODES: - self.close(f"peer sent err {code}") - return - if t in ("ping", "pong"): - return - if t == "bye": - node.emit_recv(obj) - self.close("bye before handshake completed") - return - if t in PRE_AUTH_FORBIDDEN: - self.fail("auth", f"{t!r} received before the handshake completed", re=oid) - return - node.emit_recv(obj) - log(f"{self.label}: ignoring unknown message type {t!r} during handshake") - - def _on_established(self, auth: dict) -> None: - node = self.node - self.established = True - self.established_at = self.loop.time() - self.ps = node.bind_peer(self.peer_key_raw, self.peer_key_str, self.peer_hello) - node.register_connection(self) - grants = auth.get("grants") - if isinstance(grants, list): - for g in grants: - node.receive_grant(self, g, "auth") - h = self.peer_hello - name = h.get("name") if isinstance(h.get("name"), str) else "" - caps = h.get("caps") if isinstance(h.get("caps"), list) else [] - about = h.get("about") if isinstance(h.get("about"), str) else "" - if "blob" not in caps: - log(f"{self.label}: peer does not list the 'blob' capability") - log(f"{self.label}: connected to {self.peer_key_str} ({name})") - node.emit({"event": "connected", "key": self.peer_key_str, "name": name, - "caps": caps, "about": about}) - self.send_now(node.envelope("resume", seen=dict(self.ps.inbox.seen))) - - # -- established --------------------------------------------------------- - - def _dispatch(self, obj: dict, t) -> None: - node = self.node - ps = self.ps - if t == "ping": - pid = obj.get("id") - self.send_now(node.envelope("pong", re=pid if isinstance(pid, str) else None)) - return - if t == "pong": - return - if t in ACKED_TYPES: - node.on_msg_or_state(self, obj, t) - return - if t == "chunk": - node.on_chunk(self, obj) - return - node.emit_recv(obj) - if t == "ack": - rid = obj.get("re") - if isinstance(rid, str): - node.prune(ps, rid, "ack") - elif t == "resume": - if self.got_resume: - log(f"{self.label}: ignoring repeated resume") - return - self.got_resume = True - seen = obj.get("seen") - clean = {} - if isinstance(seen, dict): - clean = {k: v for k, v in seen.items() if isinstance(k, str) and isinstance(v, str)} - self.go_live(clean) - elif t == "grant": - node.receive_grant(self, obj.get("grant"), "grant message") - elif t == "introduce": - node.on_introduce(self, obj) - elif t == "bye": - self._on_bye(obj) - elif t == "err": - self._on_err(obj) - elif t in ("hello", "auth"): - log(f"{self.label}: ignoring {t} after the handshake") - else: - debug(f"{self.label}: ignoring unknown message type {t!r}") - - def _on_bye(self, obj: dict) -> None: - self.node.on_bye_exchange() - if not self.bye_sent: - reason = obj.get("reason") if isinstance(obj.get("reason"), str) else "bye" - self._write(dumps_line(self.node.envelope("bye", reason=reason))) - self.bye_sent = True - self.bye_sent_at = self.loop.time() - self.close("bye") - - def _on_err(self, obj: dict) -> None: - code = obj.get("code") - detail = obj.get("detail") - if code == "blob_refused": - ref = obj.get("ref") - if not isinstance(ref, str): - ref = self.node.ref_for_entry(self.ps, obj.get("re")) - if ref: - self.node.stop_blob(self.ps, ref) - if code in CLOSING_ERR_CODES: - self.node.emit_error(f"{self.label}: peer sent err {code}: {detail}") - self.close(f"peer sent err {code}: {detail}") - else: - log(f"{self.label}: peer sent err {code}: {detail}") - - -# ---------------------------------------------------------------- node - -class Node: - def __init__(self, args): - global _LOG_NAME, DEBUG - self.args = args - self.mode = args.mode - if args.verbose: - DEBUG = True - self.state_dir = os.path.abspath(os.path.expanduser(args.state)) - os.makedirs(self.state_dir, exist_ok=True) - self._lock_fd = os.open(os.path.join(self.state_dir, "lock"), os.O_RDWR | os.O_CREAT, 0o600) - try: - fcntl.flock(self._lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - raise StartupError(f"state directory {self.state_dir} is in use by another process") from None - self.outgoing_dir = os.path.join(self.state_dir, "outgoing") - os.makedirs(self.outgoing_dir, exist_ok=True) - os.makedirs(os.path.join(self.state_dir, "peers"), exist_ok=True) - self.name = args.name or f"awp-py@{socket.gethostname()}" - _LOG_NAME = self.name - self.ping_interval = float(args.ping_interval) - if 0 < self.ping_interval < 0.05: - self.ping_interval = 0.05 - self.blob_limit = int(args.max_blob) - self.ids = UlidGen() - self.identity = Identity.load_or_create(os.path.join(self.state_dir, "identity.json")) - self.key_str = self.identity.key - self.about = args.about or ( - f"awp-py independent Python test peer; key fingerprint " - f"sha256:{key_fp(self.identity.pub)[:16]}") - self.trusted = set() - extra_trust = load_json(os.path.join(self.state_dir, "trust.json"), []) - for k in list(args.trust or []) + (extra_trust if isinstance(extra_trust, list) else []): - try: - self.trusted.add(parse_key(k)) - except ValueError as e: - log(f"warning: ignoring bad --trust key {k!r}: {e}") - self.issued = GrantList(os.path.join(self.state_dir, "grants_issued.json")) - self.held = GrantList(os.path.join(self.state_dir, "grants_held.json")) - self.peers: dict = {} - peers_dir = os.path.join(self.state_dir, "peers") - for fp in sorted(os.listdir(peers_dir)): - if fp != PENDING and os.path.isdir(os.path.join(peers_dir, fp)): - self.peers[fp] = PeerState(self, fp) - self.pending = PeerState(self, PENDING) - dp = load_json(os.path.join(self.state_dir, "default_peer.json"), {}) - fp = dp.get("fp") if isinstance(dp, dict) else None - self.default_fp = fp if fp in self.peers else None - if self.default_fp and self.pending.has_queued(): # crash during adoption - self.peers[self.default_fp].adopt(self.pending) - for ps in self.all_peer_states(): - if ps.outbox.max_id: - self.ids.observe(ps.outbox.max_id) - self._cleanup_outgoing() - self.connections: dict = {} - self.all_conns: set = set() - self.bye_suppressed = False - self.ever_established = False - self.wake = asyncio.Event() - self.conn_counter = 0 - - def all_peer_states(self): - return list(self.peers.values()) + [self.pending] - - def _cleanup_outgoing(self) -> None: - used = set() - for ps in self.all_peer_states(): - used.update(ps.outbox.src_count.keys()) - for name in os.listdir(self.outgoing_dir): - if name not in used: - try: - os.remove(os.path.join(self.outgoing_dir, name)) - except OSError: - pass - - def start(self) -> None: - log(f"identity {self.key_str} (ed25519 via {ED25519_BACKEND}), state {self.state_dir}") - self.emit({"event": "identity", "key": self.key_str, "name": self.name}) - for ps in list(self.peers.values()): - for ref in ps.inbox.to_finish: - self.finish_blob(ps, ref) - ps.inbox.to_finish = [] - - # -- output --------------------------------------------------------------- - - def emit(self, ev: dict) -> None: - try: - s = json.dumps(ev, ensure_ascii=False, separators=(",", ":")) - except ValueError: - s = json.dumps(ev, ensure_ascii=True, separators=(",", ":"), default=str) - try: - try: - sys.stdout.write(s + "\n") - except UnicodeEncodeError: - sys.stdout.write(json.dumps(ev, ensure_ascii=True, separators=(",", ":")) + "\n") - sys.stdout.flush() - except (BrokenPipeError, OSError, ValueError): - pass - - def emit_error(self, detail: str) -> None: - log(f"error: {detail}") - self.emit({"event": "error", "detail": detail}) - - def emit_recv(self, obj: dict) -> None: - if obj.get("t") == "chunk" and "data" in obj: - obj = {k: v for k, v in obj.items() if k != "data"} - self.emit({"event": "recv", "msg": obj}) - - # -- message construction ------------------------------------------------- - - def envelope(self, t: str, **fields) -> dict: - obj = {"t": t, "id": self.ids.new(), "ts": now_ts()} - for k, v in fields.items(): - if v is not None: - obj[k] = v - return obj - - def make_hello(self) -> dict: - return self.envelope("hello", v=PROTOCOL_VERSION, key=self.key_str, name=self.name, - nonce=b64url(os.urandom(32)), caps=list(MY_CAPS), about=self.about) - - def make_err(self, code: str, detail: str, re=None, ref=None) -> dict: - return self.envelope("err", re=re, code=code, detail=detail, ref=ref) - - def entry_line(self, e: dict) -> bytes: - obj = e["obj"] - if obj.get("t") == "chunk" and "src" in e: - with open(os.path.join(self.outgoing_dir, e["src"]), "rb") as f: - f.seek(int(e["off"])) - data = f.read(int(e["len"])) - if len(data) != int(e["len"]): - raise ValueError("outgoing blob copy is shorter than expected") - obj = dict(obj) - obj["data"] = b64std(data) - return dumps_line(obj) - - # -- peers and connections ---------------------------------------------- - - def peer_state_for(self, raw: bytes, key_str: str) -> PeerState: - fp = key_fp(raw) - ps = self.peers.get(fp) - if ps is None: - ps = self.peers[fp] = PeerState(self, fp, raw, key_str) - return ps - - def bind_peer(self, raw: bytes, key_str: str, hello: dict) -> PeerState: - ps = self.peer_state_for(raw, key_str) - ps.save_meta(key_str, hello) - if self.default_fp is None: - ps.adopt(self.pending) - if self.default_fp != ps.fp: - self.default_fp = ps.fp - atomic_write_json(os.path.join(self.state_dir, "default_peer.json"), - {"fp": ps.fp, "key": format_key(raw)}, durable=True) - return ps - - def register_connection(self, conn: Connection) -> None: - self.ever_established = True - old = self.connections.get(conn.ps.fp) - if old is not None and old is not conn: - log(f"{old.label}: superseded by {conn.label} (same key)") - old.close("superseded by a newer connection from the same key") - self.connections[conn.ps.fp] = conn - - def on_conn_closed(self, conn: Connection) -> None: - self.all_conns.discard(conn) - if conn.ps is not None and self.connections.get(conn.ps.fp) is conn: - del self.connections[conn.ps.fp] - reason = conn.close_reason or "closed" - log(f"{conn.label}: closed ({reason})") - if conn.established: - self.emit({"event": "disconnected", "reason": reason}) - - def target(self, c: dict) -> PeerState: - pk = c.get("peer") - if pk is not None: - try: - raw = parse_key(pk) - except ValueError as e: - raise CommandError(f"bad peer key: {e}") from None - return self.peer_state_for(raw, pk) - if self.default_fp: - return self.peers[self.default_fp] - return self.pending - - def has_work(self) -> bool: - if self.pending.has_work(): - return True - ps = self.peers.get(self.default_fp) if self.default_fp else None - return bool(ps and ps.has_work()) - - def new_work(self) -> None: - self.bye_suppressed = False - self.wake.set() - - def on_bye_exchange(self) -> None: - if self.mode == "connect": - self.bye_suppressed = True - - # -- outbox --------------------------------------------------------------- - - def queue_outbox(self, ps: PeerState, entries: list) -> None: - fresh = ps.outbox.add_many(entries) # durable before sending - for e in fresh: - obj = e["obj"] - if obj["t"] in ACKED_TYPES and isinstance(obj.get("th"), str): - ps.threads.touch(obj["th"], obj, outgoing=True) - conn = self.connections.get(ps.fp) - if conn is not None: - for e in fresh: - conn.queue_entry(e["obj"]["id"]) - self.new_work() - - def prune(self, ps: PeerState, eid: str, via: str) -> None: - e = ps.outbox.entries.get(eid) - if e is None: - return - obj = e["obj"] - t = obj.get("t") - if t in ACKED_TYPES: # emit first: a kill in between repeats, never loses - debug(f"{t} {eid} acknowledged ({via})") - self.emit({"event": "acked", "id": eid}) - ps.outbox.remove(eid) - if t == "msg": - for ref in blob_refs(obj): - for cid in list(ps.outbox.by_ref.get(ref, ())): - ps.outbox.remove(cid) - - def apply_seen(self, ps: PeerState, seen: dict) -> None: - """Outbox entries covered by the peer's seen map count as acked (F3).""" - usable = {} - for th, sid in seen.items(): - if ulid_decode(sid) is None: - # Every id we send is a ULID, so this cannot name one of ours; - # comparing would be meaningless. Replay that thread in full. - log(f"warning: peer's seen id {sid!r} for thread {th!r} is not one of our ids; " - "replaying the whole thread") - else: - usable[th] = sid.upper() - if not usable: - return - for eid in list(ps.outbox.entries.keys()): - e = ps.outbox.entries.get(eid) - if e is None: - continue - th = e["obj"].get("th") - if isinstance(th, str) and th in usable and eid <= usable[th]: - self.prune(ps, eid, "resume seen") - - def stop_blob(self, ps: PeerState, ref: str) -> None: - ids = list(ps.outbox.by_ref.get(ref, ())) - for cid in ids: - ps.outbox.remove(cid) - log(f"peer refused blob {ref!r}; dropped {len(ids)} queued chunk(s)") - - def ref_for_entry(self, ps: PeerState, eid): - e = ps.outbox.entries.get(eid) if isinstance(eid, str) else None - if e is None: - return None - if e["obj"].get("t") == "chunk": - return e["obj"].get("ref") - refs = blob_refs(e["obj"]) - return refs[0] if len(refs) == 1 else None - - # -- inbound -------------------------------------------------------------- - - def on_msg_or_state(self, conn: Connection, obj: dict, t: str) -> None: - ps = conn.ps - mid = obj.get("id") if isinstance(obj.get("id"), str) and obj.get("id") else None - th = obj.get("th") if isinstance(obj.get("th"), str) and obj.get("th") else None - if mid is not None and mid in ps.inbox.dedup: - debug(f"{conn.label}: duplicate {t} {mid}; re-acking") - if th is not None: - conn.queue_ack(th, mid) - return - rec = {"id": mid, "t": t, "th": th} - raw = _rec_bytes(rec) # prepared first: keeps emit->persist gap tiny - self.emit_recv(obj) # C4: emit, persist, ack - if mid is None: - log(f"{conn.label}: warning: {t} without id cannot be acked or deduplicated") - else: - ps.inbox.record(rec, durable=True, raw=raw) - if th is not None: - ps.threads.touch(th, obj, outgoing=False) - elif mid is not None: - log(f"{conn.label}: warning: {t} {mid} has no th; delivered but not acked") - if t == "msg": - self._inspect_parts(conn, obj, mid) - if mid is not None and th is not None: - conn.queue_ack(th, mid) - - def _inspect_parts(self, conn: Connection, obj: dict, mid) -> None: - ps = conn.ps - parts = obj.get("parts") - if not isinstance(parts, list): - return - needs = set() - for p in parts: - if not isinstance(p, dict): - continue - k = p.get("k") - if k == "blob": - ref, size = p.get("ref"), p.get("size") - if isinstance(ref, str) and isinstance(size, int) and not isinstance(size, bool): - ps.inbox.declared[ref] = size - if (size > self.blob_limit and ref not in ps.inbox.done - and ref not in ps.inbox.refused): - self.refuse_blob(conn, ps, ref, mid, - f"declared blob size {size} exceeds the local limit of {self.blob_limit} bytes") - elif k == "data" and p.get("mime") == EXEC_MIME: - needs.add("exec") - if needs: - missing = sorted(needs - self.honored_caps(ps)) - if missing: - conn.send_err("forbidden", f"request needs capability {', '.join(missing)} " - "and this peer holds no honoured grant for it", re=mid) - else: - log(f"{conn.label}: exec request {mid} is covered by a grant " - "(this test peer never executes anything)") - - def on_chunk(self, conn: Connection, obj: dict) -> None: - ps = conn.ps - inbox = ps.inbox - cid = obj.get("id") if isinstance(obj.get("id"), str) and obj.get("id") else None - th = obj.get("th") if isinstance(obj.get("th"), str) and obj.get("th") else None - if cid is not None and cid in inbox.dedup: - debug(f"{conn.label}: duplicate chunk {cid}") - return - self.emit_recv(obj) - ref = obj.get("ref") - if not isinstance(ref, str) or not ref: - log(f"{conn.label}: warning: chunk {cid} has no ref; ignored") - return - base = {"id": cid, "t": "chunk", "th": th, "ref": ref} - if ref in inbox.refused or ref in inbox.done: - if cid: - inbox.record(dict(base, stored=False), durable=False) - return - st = inbox.partial.get(ref) - expected = st["next"] if st else 0 - n = obj.get("n") - if isinstance(n, bool) or not isinstance(n, int): - n = expected - if n < expected: - if cid: - inbox.record(dict(base, n=n, stored=False), durable=False) - return - if n > expected: - conn.send_err("internal", f"blob {ref}: got chunk {n} but chunk {expected} was expected; " - "chunk ignored", re=cid) - return - try: - data = b64decode_any(obj.get("data", "")) - except ValueError as e: - self.refuse_blob(conn, ps, ref, cid, f"chunk data is not valid base64: {e}") - if cid: - inbox.record(dict(base, n=n, stored=False), durable=False) - return - total = (st["size"] if st else 0) + len(data) - if total > self.blob_limit: - self.refuse_blob(conn, ps, ref, cid, - f"blob {ref} exceeds the local size limit of {self.blob_limit} bytes") - if cid: - inbox.record(dict(base, n=n, stored=False), durable=False) - return - last = obj.get("last") is True - os.makedirs(inbox.partial_dir, exist_ok=True) - with open(inbox.partial_path(ref), "ab" if st else "wb") as f: - f.write(data) - f.flush() - if FSYNC: - os.fsync(f.fileno()) - inbox.record(dict(base, n=n, len=len(data), last=last, stored=True), durable=False) - if last: - self.finish_blob(ps, ref) - - def refuse_blob(self, conn: Connection, ps: PeerState, ref: str, re_id, detail: str) -> None: - inbox = ps.inbox - if ref in inbox.refused: - return - conn.send_err("blob_refused", detail, re=re_id, ref=ref) - inbox.record({"op": "blob_refused", "ref": ref}, durable=False) - try: - os.remove(inbox.partial_path(ref)) - except OSError: - pass - - def finish_blob(self, ps: PeerState, ref: str) -> None: - inbox = ps.inbox - src, dst = inbox.partial_path(ref), inbox.final_path(ref) - if os.path.exists(src): - os.replace(src, dst) - h = hashlib.sha256() - size = 0 - with open(dst, "rb") as f: - for block in iter(lambda: f.read(1 << 20), b""): - h.update(block) - size += len(block) - declared = inbox.declared.get(ref) - if declared is not None and declared != size: - log(f"warning: blob {ref!r} is {size} bytes but its blob part declared {declared}") - digest = h.hexdigest() - log(f"blob {ref!r} complete: {size} bytes -> {dst}") - self.emit({"event": "blob", "ref": ref, "path": dst, "size": size, "sha256": digest}) - inbox.record({"op": "blob_done", "ref": ref, "path": dst, "size": size, "sha256": digest}, - durable=True) - - # -- grants ----------------------------------------------------------------- - - def is_root(self, raw: bytes) -> bool: - return raw == self.identity.pub or raw in self.trusted - - def supporting_grants(self, ps: PeerState) -> list: - return list(ps.grants.items) + list(self.issued.items) + list(self.held.items) - - def grant_honored(self, g: dict, ps: PeerState) -> bool: - ok, _ = verify_grant(g) - if not ok: - return False - iss = parse_key(g["iss"]) - if self.is_root(iss): - return True - for s in self.supporting_grants(ps): # one level of delegation - if s is g or s.get("sig") == g.get("sig"): - continue - if not verify_grant(s)[0]: - continue - try: - if (parse_key(s["sub"]) == iss and "introduce" in s["caps"] - and self.is_root(parse_key(s["iss"]))): - return True - except (ValueError, KeyError, TypeError): - continue - return False - - def honored_caps(self, ps: PeerState) -> set: - caps = set() - cands = list(ps.grants.items) + [g for g in self.issued.items - if key_matches(g.get("sub"), ps.key_raw)] - for g in cands: - if key_matches(g.get("sub"), ps.key_raw) and self.grant_honored(g, ps): - caps.update(c for c in g.get("caps", []) if isinstance(c, str)) - return caps - - def receive_grant(self, conn: Connection, g, source: str) -> None: - ok, why = verify_grant(g) - if not ok: - log(f"{conn.label}: ignoring invalid grant ({source}): {why}") - return - sub, iss = parse_key(g["sub"]), parse_key(g["iss"]) - conn.ps.grants.add(g) - caps = g.get("caps") - if sub == self.identity.pub: - self.held.add(g) - log(f"{conn.label}: now holding grant {caps} from {g['iss']} ({source})") - elif sub == conn.peer_key_raw: - verdict = "honoured" if self.grant_honored(g, conn.ps) else "not honoured (issuer not trusted)" - log(f"{conn.label}: peer presented grant {caps} issued by {g['iss']} ({source}): {verdict}") - else: - log(f"{conn.label}: stored grant {caps} for third party {g['sub']} ({source})") - if iss == self.identity.pub: - debug("grant was issued by us") - - def on_introduce(self, conn: Connection, obj: dict) -> None: - g = obj.get("grant") - if g is not None: - self.receive_grant(conn, g, "introduce") - peer = obj.get("peer") if isinstance(obj.get("peer"), dict) else None - rec = {"from": conn.peer_key_str, "at": now_ts(), "th": obj.get("th"), "peer": peer} - with open(os.path.join(self.state_dir, "introductions.jsonl"), "ab") as f: - f.write(_rec_bytes(rec)) - log(f"{conn.label}: introduced to {peer} (not connecting automatically)") - - # -- commands --------------------------------------------------------------- - - def on_command_line(self, raw: bytes) -> None: - s = raw.strip() - if not s: - return - try: - c = json.loads(s.decode("utf-8")) - except (UnicodeDecodeError, ValueError) as e: - self.emit_error(f"bad command line (not JSON): {e}") - return - if not isinstance(c, dict): - self.emit_error("bad command line: not a JSON object") - return - handler = {"send": self.cmd_send, "state": self.cmd_state, "blob": self.cmd_blob, - "grant": self.cmd_grant, "bye": self.cmd_bye, "quit": self.cmd_quit}.get(c.get("cmd")) - if handler is None: - self.emit_error(f"unknown command {c.get('cmd')!r}") - return - try: - handler(c) - except CommandError as e: - self.emit_error(str(e)) - except Exception as e: - log(f"command {c.get('cmd')!r} failed: {e}\n{traceback.format_exc()}") - self.emit_error(f"command {c.get('cmd')!r} failed: {e}") - - def on_stdin_eof(self) -> None: - log("stdin closed; running until killed") - - @staticmethod - def _req_str(c: dict, k: str) -> str: - v = c.get(k) - if not isinstance(v, str) or not v: - raise CommandError(f"{c.get('cmd')}: {k!r} must be a non-empty string") - return v - - @staticmethod - def _opt_str(c: dict, k: str): - v = c.get(k) - if v is None or v == "": - return None - if not isinstance(v, str): - raise CommandError(f"{c.get('cmd')}: {k!r} must be a string") - return v - - @staticmethod - def _check_size(obj: dict) -> None: - if len(dumps_line(obj)) > MAX_LINE: - raise CommandError("message would exceed the 1 MiB line limit; send it as a blob") - - def cmd_send(self, c: dict) -> None: - th = self._req_str(c, "th") - parts = [] - text = c.get("text") - if text is not None: - parts.append({"k": "text", "text": text if isinstance(text, str) else json.dumps(text)}) - extra = c.get("parts") - if extra is not None: - if not isinstance(extra, list) or not all(isinstance(p, dict) for p in extra): - raise CommandError("send: 'parts' must be a list of objects") - parts.extend(extra) - if not parts: - raise CommandError("send: needs 'text' or 'parts'") - ps = self.target(c) - obj = self.envelope("msg", th=th, re=self._opt_str(c, "re"), - subject=self._opt_str(c, "subject"), parts=parts) - self._check_size(obj) - self.queue_outbox(ps, [{"obj": obj}]) - self.emit({"event": "sent", "id": obj["id"], "t": "msg"}) - - def cmd_state(self, c: dict) -> None: - th = self._req_str(c, "th") - st = self._req_str(c, "state") - ps = self.target(c) - obj = self.envelope("state", th=th, re=self._opt_str(c, "re"), state=st, - note=self._opt_str(c, "note")) - self._check_size(obj) - self.queue_outbox(ps, [{"obj": obj}]) - self.emit({"event": "sent", "id": obj["id"], "t": "state"}) - - def cmd_blob(self, c: dict) -> None: - th = self._req_str(c, "th") - path = self._req_str(c, "path") - text = c.get("text") - ps = self.target(c) - try: - st = os.stat(path) - except OSError as e: - raise CommandError(f"blob: cannot read {path}: {e}") from None - if not stat.S_ISREG(st.st_mode): - raise CommandError(f"blob: {path} is not a regular file") - ref = "blob_" + self.ids.new().lower() - src = safe_name(ref) - dst = os.path.join(self.outgoing_dir, src) - shutil.copyfile(path, dst) # immutable, durable copy - with open(dst, "rb") as f: - if FSYNC: - os.fsync(f.fileno()) - size = os.path.getsize(dst) - name = self._opt_str(c, "name") or os.path.basename(path) - mime = self._opt_str(c, "mime") or mimetypes.guess_type(name)[0] or "application/octet-stream" - nchunks = max(1, -(-size // CHUNK_SIZE)) - entries = [] - for n in range(nchunks): # D4: chunks first ... - off = n * CHUNK_SIZE - cobj = self.envelope("chunk", th=th, ref=ref, n=n, last=(n == nchunks - 1)) - entries.append({"obj": cobj, "src": src, "off": off, "len": min(CHUNK_SIZE, size - off)}) - parts = [] - if text is not None: - parts.append({"k": "text", "text": text if isinstance(text, str) else json.dumps(text)}) - parts.append({"k": "blob", "ref": ref, "name": name, "mime": mime, "size": size}) - mobj = self.envelope("msg", th=th, re=self._opt_str(c, "re"), # ... then the msg - subject=self._opt_str(c, "subject"), parts=parts) - entries.append({"obj": mobj}) - self.queue_outbox(ps, entries) - log(f"blob {ref!r}: {size} bytes in {nchunks} chunk(s) queued, msg {mobj['id']}") - self.emit({"event": "sent", "id": mobj["id"], "t": "msg"}) - - def cmd_grant(self, c: dict) -> None: - sub = self._opt_str(c, "sub") - if c.get("peer") is not None: - ps = self.target(c) - else: - ps = None - if sub is not None: - try: - ps = self.peers.get(key_fp(parse_key(sub))) - except ValueError as e: - raise CommandError(f"grant: bad sub key: {e}") from None - ps = ps or self.target(c) - if sub is None: - if ps.key_raw is None: - raise CommandError("grant: 'sub' is required when no peer is known yet") - sub = ps.key_str or format_key(ps.key_raw) - try: - parse_key(sub) - except ValueError as e: - raise CommandError(f"grant: bad sub key: {e}") from None - caps = c.get("caps", []) - if not isinstance(caps, list) or not all(isinstance(x, str) for x in caps): - raise CommandError("grant: 'caps' must be a list of strings") - ttl = c.get("ttl", DEFAULT_GRANT_TTL) - if isinstance(ttl, bool) or not isinstance(ttl, (int, float)) or ttl <= 0: - raise CommandError("grant: 'ttl' must be a positive number of seconds") - g = mint_grant(self.identity, sub, caps, ttl) - self.issued.add(g) - obj = self.envelope("grant", grant=g) - ps.sendonce.add(obj) - conn = self.connections.get(ps.fp) - if conn is not None: - conn.queue_once(obj["id"]) - self.new_work() - log(f"minted grant {caps} for {sub}, expires {g['exp']}") - self.emit({"event": "sent", "id": obj["id"], "t": "grant"}) - - def cmd_bye(self, c: dict) -> None: - reason = self._opt_str(c, "reason") or "done" - conns = [x for x in self.all_conns if x.established and not x.closing] - if c.get("peer") is not None: - want = self.target(c) - conns = [x for x in conns if x.ps is want] - self.on_bye_exchange() - if not conns: - self.emit_error("bye: not connected") - return - for conn in conns: - bid = conn.initiate_bye(reason) - if bid: - self.emit({"event": "sent", "id": bid, "t": "bye"}) - - def cmd_quit(self, c: dict) -> None: - log("quit") - try: - sys.stdout.flush() - sys.stderr.flush() - finally: - os._exit(0) - - # -- run modes --------------------------------------------------------------- - - async def run_listen(self, addr: str) -> None: - target = parse_addr(addr) - if target[0] == "tcp": - _, host, port = target - if not is_loopback(host): - log("warning: plaintext TCP on a non-loopback address; awp over TCP has no " - "transport encryption (spec 4.2, 13), use only on trusted networks") - sock = make_tcp_listener(host, port) - server = await asyncio.start_server(self._accept, sock=sock, limit=2 * MAX_LINE) - shown = f"tcp:{fmt_host(host)}:{sock.getsockname()[1]}" - else: - path = target[1] - prepare_unix_path(path) - server = await asyncio.start_unix_server(self._accept, path=path, limit=2 * MAX_LINE) - shown = f"unix:{path}" - log(f"listening on {shown}") - self.emit({"event": "listening", "addr": shown, "key": self.key_str}) - async with server: - await server.serve_forever() - - async def _accept(self, reader, writer) -> None: - self.conn_counter += 1 - peer = writer.get_extra_info("peername") - where = f"{peer[0]}:{peer[1]}" if isinstance(peer, tuple) and len(peer) >= 2 else "unix peer" - conn = Connection(self, reader, writer, f"conn#{self.conn_counter} from {where}") - try: - await conn.run() - except Exception as e: - log(f"{conn.label}: crashed: {e}\n{traceback.format_exc()}") - - async def run_connect(self, addr: str) -> None: - target = parse_addr(addr) - if target[0] == "tcp" and not is_loopback(target[1]): - log("warning: plaintext TCP to a non-loopback address; use only on trusted networks") - delay = BACKOFF_INITIAL - while True: - if self.ever_established and (self.bye_suppressed or not self.has_work()): - self.wake.clear() - if self.bye_suppressed or not self.has_work(): - log("idle: nothing unacked and no open threads; waiting for new work") - await self.wake.wait() - continue - try: - reader, writer = await asyncio.wait_for(open_stream(target), timeout=15) - except (OSError, asyncio.TimeoutError) as e: - d = delay * random.uniform(0.9, 1.1) - log(f"connect {addr} failed: {e or type(e).__name__}; retrying in {d:.2f}s") - await asyncio.sleep(d) - delay = min(delay * 2, BACKOFF_CAP) - continue - self.conn_counter += 1 - conn = Connection(self, reader, writer, f"conn#{self.conn_counter} to {addr}") - established = await conn.run() - if established: - delay = BACKOFF_INITIAL - if self.bye_suppressed or not self.has_work(): - continue - d = delay * random.uniform(0.9, 1.1) - log(f"reconnecting to {addr} in {d:.2f}s") - await asyncio.sleep(d) - delay = min(delay * 2, BACKOFF_CAP) - - -# ---------------------------------------------------------------- main - -def start_stdin_thread(loop, node: Node) -> None: - def reader(): - try: - stream = sys.stdin.buffer if sys.stdin is not None else None - if stream is not None: - for raw in iter(stream.readline, b""): - loop.call_soon_threadsafe(node.on_command_line, raw) - except Exception as e: - log(f"stdin reader stopped: {e}") - try: - loop.call_soon_threadsafe(node.on_stdin_eof) - except RuntimeError: - pass - - threading.Thread(target=reader, name="awp-stdin", daemon=True).start() - - -async def amain(args) -> int: - try: - node = Node(args) - node.start() - except StartupError as e: - log(f"fatal: {e}") - print(json.dumps({"event": "error", "detail": str(e)}), flush=True) - return 1 - start_stdin_thread(asyncio.get_running_loop(), node) - try: - if args.mode == "listen": - await node.run_listen(args.addr) - else: - await node.run_connect(args.addr) - except StartupError as e: - node.emit_error(str(e)) - return 1 - return 0 - - -def main(argv=None) -> int: - ap = argparse.ArgumentParser( - prog="awp_peer.py", - description="Independent Python peer for the Agent Wire Protocol (draft 1) over TCP or Unix sockets.") - ap.add_argument("--state", default=os.path.join(os.path.expanduser("~"), ".awp"), - help="state directory (identity, outbox, seen map, blobs)") - ap.add_argument("--name", default=None, help="name to announce in hello") - ap.add_argument("--ping-interval", type=float, default=DEFAULT_PING_INTERVAL, - help="seconds of inbound silence before pinging (0 disables; default 30)") - ap.add_argument("--max-blob", type=int, default=DEFAULT_BLOB_LIMIT, - help="largest blob accepted, in bytes (default 50 MiB)") - ap.add_argument("--trust", action="append", default=[], - help="trust grants issued by this key (repeatable)") - ap.add_argument("--about", default=None, help="free text for hello.about") - ap.add_argument("-v", "--verbose", action="store_true", help="debug logging on stderr") - sub = ap.add_subparsers(dest="mode", required=True, metavar="{listen,connect}") - for mode in ("listen", "connect"): - sp = sub.add_parser(mode, help=f"{mode} on ADDR (tcp:HOST:PORT or unix:/path)") - sp.add_argument("addr") - args = ap.parse_args(argv) - try: - sys.stdout.reconfigure(encoding="utf-8") - except (AttributeError, ValueError): - pass - try: - return asyncio.run(amain(args)) - except KeyboardInterrupt: - return 130 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/python/interop_test.py b/python/interop_test.py deleted file mode 100644 index 97612c9..0000000 --- a/python/interop_test.py +++ /dev/null @@ -1,322 +0,0 @@ -"""Interop tests: the Go reference daemon against the independent Python peer. - -The two implementations were written separately from SPEC.md, so these tests -check the spec, not one implementation's reading of it. Run from this -directory with the Go binary built: - - go build -o /tmp/awp ../cmd/awp - AWP_BIN=/tmp/awp python3 -m unittest -v interop_test -""" - -import hashlib -import json -import os -import queue -import shutil -import signal -import socket -import subprocess -import sys -import tempfile -import threading -import time -import unittest - -HERE = os.path.dirname(os.path.abspath(__file__)) -PEER = os.path.join(HERE, "awp_peer.py") -BIN = os.environ.get("AWP_BIN", "awp") - - -def free_port(): - s = socket.socket() - s.bind(("127.0.0.1", 0)) - port = s.getsockname()[1] - s.close() - return port - - -class PyPeer: - """The Python peer as a subprocess: JSON commands in, JSON events out.""" - - def __init__(self, state, mode, addr, name="py"): - self.args = [sys.executable, PEER, "--state", state, "--name", name, "--ping-interval", "5", mode, addr] - self.events = queue.Queue() - self.log = [] - self.start() - - def start(self): - self.proc = subprocess.Popen(self.args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, text=True, bufsize=1) - threading.Thread(target=self._read, args=(self.proc,), daemon=True).start() - - def _read(self, proc): - for line in proc.stdout: - try: - ev = json.loads(line) - except ValueError: - continue - self.log.append(ev) - self.events.put(ev) - - def cmd(self, **c): - self.proc.stdin.write(json.dumps(c) + "\n") - self.proc.stdin.flush() - - def expect(self, pred, what, timeout=20): - deadline = time.time() + timeout - while time.time() < deadline: - try: - ev = self.events.get(timeout=max(0.05, deadline - time.time())) - except queue.Empty: - break - if pred(ev): - return ev - raise AssertionError("python peer: timed out waiting for " + what) - - def kill9(self): - self.proc.send_signal(signal.SIGKILL) - self.proc.wait() - self._close_pipes() - - def stop(self): - if self.proc.poll() is None: - self.proc.kill() - self.proc.wait() - self._close_pipes() - - def _close_pipes(self): - for f in (self.proc.stdin, self.proc.stdout): - try: - f.close() - except (OSError, ValueError): - pass - - -class GoPeer: - """The Go daemon in the foreground, plus CLI calls against it.""" - - def __init__(self, home, listen, name="go", extra_env=None): - self.home = home - self.env = dict(os.environ, AWP_HOME=home, AWP_NAME=name, AWP_LISTEN=listen, AWP_PING_INTERVAL="5s") - self.env.update(extra_env or {}) - self.start() - - def start(self): - self.log = open(os.path.join(self.home, "daemon.log"), "a") - self.proc = subprocess.Popen([BIN, "daemon"], env=self.env, stdout=self.log, stderr=self.log) - deadline = time.time() + 20 - while time.time() < deadline: - if subprocess.run([BIN, "status"], env=self.env, capture_output=True).returncode == 0: - return - time.sleep(0.1) - raise AssertionError("go daemon did not start") - - def run(self, *args, check=True, timeout=60): - r = subprocess.run([BIN, *args], env=self.env, capture_output=True, text=True, timeout=timeout) - if check and r.returncode != 0: - raise AssertionError(f"awp {' '.join(args)} failed ({r.returncode}): {r.stderr}{r.stdout}") - return r.stdout - - def json(self, *args, **kw): - return json.loads(self.run(*args, "--json", **kw)) - - def key(self): - return self.json("status")["key"] - - def kill9(self): - self.proc.send_signal(signal.SIGKILL) - self.proc.wait() - self.log.close() - - def stop(self): - if self.proc.poll() is None: - self.proc.terminate() - try: - self.proc.wait(10) - except subprocess.TimeoutExpired: - self.proc.kill() - self.proc.wait() - self.log.close() - - -def sha256(path): - with open(path, "rb") as f: - return hashlib.sha256(f.read()).hexdigest() - - -class Interop(unittest.TestCase): - def setUp(self): - self.dir = tempfile.mkdtemp(prefix="hi") - self.cleanup = [] - - def tearDown(self): - for c in reversed(self.cleanup): - c() - shutil.rmtree(self.dir, ignore_errors=True) - - def path(self, *p): - return os.path.join(self.dir, *p) - - def go(self, name="go", **kw): - home = self.path(name) - os.makedirs(home, exist_ok=True) - g = GoPeer(home, "unix:" + self.path(name + ".sock"), name=name, **kw) - self.cleanup.append(g.stop) - return g - - def py(self, mode, addr, name="py"): - p = PyPeer(self.path(name + "-state"), mode, addr, name=name) - self.cleanup.append(p.stop) - return p - - def test_go_dials_python_full_conversation(self): - port = free_port() - py = self.py("listen", f"tcp:127.0.0.1:{port}") - py.expect(lambda e: e.get("event") == "listening", "listening") - go = self.go(extra_env={"AWP_SERVE": "exec", "AWP_ROOT": self.dir}) - gokey = go.key() - - out = go.run("connect", f"tcp:127.0.0.1:{port}") - self.assertIn("connected to py", out) - ev = py.expect(lambda e: e.get("event") == "connected", "connected") - self.assertEqual(ev["key"], gokey) - - # Go -> Python: every part kind but blob, acked. - res = go.json("send", "py", "--subject", "Run integration suite", "--data", '{"repo":"fly-apps/foo"}', - "--wait-ack", "10s", "Please run `make integration`.") - self.assertTrue(res["acked"], res) - th = res["th"] - m = py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "msg", "msg")["msg"] - self.assertEqual(m["th"], th) - self.assertEqual(m["subject"], "Run integration suite") - self.assertEqual([p["k"] for p in m["parts"]], ["text", "data"]) - self.assertEqual(m["parts"][1]["data"], {"repo": "fly-apps/foo"}) - - # Python -> Go: reply and state in the same thread. - py.cmd(cmd="state", th=th, state="working", note="cloning") - py.cmd(cmd="send", th=th, text="3 of 42 failing", re=m["id"]) - py.expect(lambda e: e.get("event") == "acked", "ack from go") - out = go.run("wait", "--thread", th, "--timeout", "20s") - if "3 of 42 failing" not in out: - out += go.run("wait", "--thread", th, "--timeout", "20s") - self.assertIn("3 of 42 failing", out) - t = [t for t in go.json("threads") if t["th"] == th][0] - self.assertEqual((t["their_state"], t["their_note"]), ("working", "cloning")) - - # Blob Go -> Python, five chunks. - big = self.path("go-big.bin") - with open(big, "wb") as f: - f.write(os.urandom(1_200_000)) - go.run("send", "--thread", th, "--file", big, "log attached") - ev = py.expect(lambda e: e.get("event") == "blob", "blob from go") - self.assertEqual(ev["sha256"], sha256(big)) - - # Blob Python -> Go. - pyblob = self.path("py-big.bin") - with open(pyblob, "wb") as f: - f.write(os.urandom(900_000)) - py.cmd(cmd="blob", th=th, path=pyblob, text="here is mine") - deadline = time.time() + 20 - while time.time() < deadline: - blobs = [b for b in go.json("blobs") if b["dir"] == "in" and b["status"] == "complete"] - if blobs: - break - time.sleep(0.2) - self.assertTrue(blobs, "go never completed the python blob") - self.assertEqual(sha256(blobs[0]["path"]), sha256(pyblob)) - - # A grant minted by Python verifies in Go (canonical JSON and - # signatures agree across implementations). - py.cmd(cmd="grant", sub=gokey, caps=["fs:read"], ttl=3600) - deadline = time.time() + 20 - held = [] - while time.time() < deadline and not held: - held = [g for g in go.json("grants") if g["role"] == "held"] - time.sleep(0.2) - self.assertTrue(held, "go did not accept python's grant") - self.assertEqual(held[0]["caps"], ["fs:read"]) - - # An exec request without a grant is refused with err forbidden. - exec_part = {"k": "data", "mime": "application/vnd.awp.exec+json", "data": {"cmd": ["echo", "interop"]}} - py.cmd(cmd="send", th=th, parts=[exec_part]) - e = py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "err", "err forbidden")["msg"] - self.assertEqual(e["code"], "forbidden") - - # A grant minted by Go verifies in Python; then exec is served. - pykey = [p for p in go.json("peers") if p["name"] == "py"][0]["key"] - go.run("grant", pykey, "exec", "--ttl", "10m") - py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "grant", "grant from go") - py.cmd(cmd="send", th=th, parts=[exec_part]) - - def exec_result(e): - if e.get("event") != "recv" or e["msg"]["t"] != "msg": - return False - return any(p.get("mime") == "application/vnd.awp.exec-result+json" for p in e["msg"]["parts"]) - r = py.expect(exec_result, "exec result")["msg"] - data = [p for p in r["parts"] if p.get("mime") == "application/vnd.awp.exec-result+json"][0]["data"] - self.assertEqual((data["exit"], data["stdout"]), (0, "interop\n")) - - # Bye from Python closes cleanly on both sides. - py.cmd(cmd="bye", reason="done") - py.expect(lambda e: e.get("event") == "disconnected", "disconnect") - deadline = time.time() + 10 - while time.time() < deadline and any(p["connected"] for p in go.json("peers")): - time.sleep(0.2) - self.assertFalse(any(p["connected"] for p in go.json("peers"))) - - def test_python_dials_go(self): - go = self.go() - py = self.py("connect", "unix:" + self.path("go.sock")) - py.expect(lambda e: e.get("event") == "connected", "connected") - py.cmd(cmd="send", th="t-py-1", subject="From the python side", text="hi go") - py.expect(lambda e: e.get("event") == "acked", "ack") - out = go.run("tail", "--once") - self.assertIn("hi go", out) - self.assertIn("From the python side", out) - go.run("send", "--thread", "t-py-1", "hi python") - m = py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "msg", "reply")["msg"] - self.assertEqual((m["th"], m["parts"][0]["text"]), ("t-py-1", "hi python")) - - def test_resume_when_python_is_killed(self): - port = free_port() - py = self.py("listen", f"tcp:127.0.0.1:{port}") - py.expect(lambda e: e.get("event") == "listening", "listening") - go = self.go() - go.run("connect", f"tcp:127.0.0.1:{port}") - th = go.json("send", "py", "--subject", "resume", "0")["th"] - py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "msg", "first") - py.kill9() - for i in range(1, 30): - go.run("send", "--thread", th, str(i)) - py.start() # same state dir, same port: the go side reconnects - py.expect(lambda e: e.get("event") == "recv" and e["msg"]["t"] == "msg" and e["msg"]["parts"][0]["text"] == "29", "last message", timeout=60) - time.sleep(1) # let any stray duplicate show up before counting - texts = [ev["msg"]["parts"][0]["text"] for ev in py.log if ev.get("event") == "recv" and ev["msg"]["t"] == "msg"] - self.assertEqual(sorted(set(texts), key=int), [str(i) for i in range(30)]) - self.assertEqual(len(texts), 30, "duplicates delivered: %r" % texts) - self.assertEqual(texts, [str(i) for i in range(30)], "out of order") - - def test_resume_when_go_is_killed(self): - go = self.go() - py = self.py("connect", "unix:" + self.path("go.sock")) - py.expect(lambda e: e.get("event") == "connected", "connected") - py.cmd(cmd="send", th="t1", text="0") - py.expect(lambda e: e.get("event") == "acked", "ack") - go.kill9() - for i in range(1, 30): - py.cmd(cmd="send", th="t1", text=str(i)) - time.sleep(0.5) - go.start() - deadline = time.time() + 60 - texts = [] - while time.time() < deadline: - res = go.json("read", "t1", "--last", "100") - texts = [e["msg"]["parts"][0]["text"] for e in res["events"] if e["dir"] == "in" and e["type"] == "msg"] - if len(texts) >= 30: - break - time.sleep(0.3) - self.assertEqual(texts, [str(i) for i in range(30)]) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/test_awp_peer.py b/python/test_awp_peer.py deleted file mode 100644 index 491aade..0000000 --- a/python/test_awp_peer.py +++ /dev/null @@ -1,969 +0,0 @@ -#!/usr/bin/env python3 -"""Tests for awp_peer.py (awp draft 1, independent Python peer). - -Runs real peer processes against each other over TCP and Unix sockets, and uses -raw sockets (with an independent handshake written against the spec) for the -negative cases. - - cd python && python3 -m unittest -v test_awp_peer -""" - -import base64 -import hashlib -import json -import os -import shutil -import socket -import subprocess -import sys -import tempfile -import threading -import time -import unittest -from datetime import datetime, timezone - -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric.ed25519 import (Ed25519PrivateKey, - Ed25519PublicKey) - -HERE = os.path.dirname(os.path.abspath(__file__)) -PEER = os.path.join(HERE, "awp_peer.py") -sys.path.insert(0, HERE) -import awp_peer as hp # noqa: E402 (unit-level checks only) - -EXEC_MIME = "application/vnd.awp.exec+json" - - -# ---------------------------------------------------------------- helpers - -def b64url(b: bytes) -> str: - return base64.urlsafe_b64encode(b).rstrip(b"=").decode() - - -def b64url_dec(s: str) -> bytes: - return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4)) - - -def raw_pub(key: str) -> bytes: - assert key.startswith("ed25519:"), key - return b64url_dec(key[len("ed25519:"):]) - - -def ts() -> str: - return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def free_port() -> int: - s = socket.socket() - s.bind(("127.0.0.1", 0)) - port = s.getsockname()[1] - s.close() - return port - - -def is_recv(t, **match): - def pred(e): - if e.get("event") != "recv" or e["msg"].get("t") != t: - return False - return all(e["msg"].get(k) == v for k, v in match.items()) - return pred - - -def text_of(msg) -> str: - for p in msg.get("parts", []): - if p.get("k") == "text": - return p.get("text") - return None - - -class PeerProc: - """A awp_peer.py subprocess with stdout events collected on a thread.""" - - def __init__(self, tmp, state, mode, addr, name, ping=None, extra=()): - self.name = name - self.stderr_path = os.path.join(tmp, f"{name}-{time.monotonic_ns()}.stderr") - args = [sys.executable, PEER, "--state", state, "--name", name] - if ping is not None: - args += ["--ping-interval", str(ping)] - args += list(extra) + [mode, addr] - self.errf = open(self.stderr_path, "wb") - self.proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=self.errf) - self.events = [] - self.cv = threading.Condition() - self.eof = False - self.thread = threading.Thread(target=self._reader, daemon=True) - self.thread.start() - - def _reader(self): - for line in self.proc.stdout: - try: - ev = json.loads(line) - except ValueError: - ev = {"event": "_not_json", "raw": line.decode("utf-8", "replace")} - with self.cv: - self.events.append(ev) - self.cv.notify_all() - with self.cv: - self.eof = True - self.cv.notify_all() - - def cmd(self, **c): - self.proc.stdin.write((json.dumps(c) + "\n").encode()) - self.proc.stdin.flush() - - def mark(self) -> int: - with self.cv: - return len(self.events) - - def find(self, pred, start=0): - with self.cv: - return [e for e in self.events[start:] if pred(e)] - - def wait_for(self, pred, timeout=15.0, start=0, what="event"): - deadline = time.time() + timeout - with self.cv: - i = start - while True: - while i < len(self.events): - if pred(self.events[i]): - return self.events[i] - i += 1 - left = deadline - time.time() - if left <= 0 or self.eof: - break - self.cv.wait(min(left, 0.5)) - raise AssertionError(f"{self.name}: timed out waiting for {what}\n" - f"--- events:\n" + "\n".join(json.dumps(e)[:300] for e in self.events[-30:]) + - f"\n--- stderr tail:\n{self.stderr_tail()}") - - def wait_event(self, kind, timeout=15.0, start=0, **match): - return self.wait_for(lambda e: e.get("event") == kind and - all(e.get(k) == v for k, v in match.items()), - timeout=timeout, start=start, what=f"{kind} {match}") - - def wait_count(self, pred, n, timeout=15.0, what="events"): - deadline = time.time() + timeout - with self.cv: - while True: - got = [e for e in self.events if pred(e)] - if len(got) >= n: - return got - left = deadline - time.time() - if left <= 0 or self.eof: - break - self.cv.wait(min(left, 0.5)) - raise AssertionError(f"{self.name}: wanted {n} {what}, got {len(got)}\n" - f"--- stderr tail:\n{self.stderr_tail()}") - - def stderr_tail(self, n=40) -> str: - try: - with open(self.stderr_path, "rb") as f: - lines = f.read().decode("utf-8", "replace").splitlines() - return "\n".join(lines[-n:]) - except OSError: - return "" - - def kill9(self): - if self.proc.poll() is None: - self.proc.kill() - self.proc.wait(5) - self.thread.join(5) - - def stop(self): - if self.proc.poll() is None: - try: - self.cmd(cmd="quit") - self.proc.wait(3) - except Exception: - self.proc.kill() - self.proc.wait(5) - self.thread.join(5) - for f in (self.proc.stdin, self.proc.stdout, self.errf): - try: - f.close() - except Exception: - pass - - -class RawPeer: - """A minimal hand-rolled awp client used to poke at the peer's edges.""" - - def __init__(self, addr, sk=None, timeout=10.0): - kind, rest = addr.split(":", 1) - if kind == "tcp": - host, port = rest.rsplit(":", 1) - self.sock = socket.create_connection((host, int(port)), timeout=timeout) - else: - self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self.sock.settimeout(timeout) - self.sock.connect(rest) - self.buf = b"" - self.sk = sk or Ed25519PrivateKey.generate() - self.pub = self.sk.public_key().public_bytes(serialization.Encoding.Raw, - serialization.PublicFormat.Raw) - self.key = "ed25519:" + b64url(self.pub) - self.n = 0 - - def next_id(self) -> str: - self.n += 1 - return f"RAW{time.time_ns():020d}{self.n:04d}" - - def send_raw(self, data: bytes): - self.sock.sendall(data) - - def send(self, obj) -> bytes: - line = json.dumps(obj, separators=(",", ":")).encode() - self.sock.sendall(line + b"\n") - return line - - def read_line(self, timeout=10.0): - """Next line without the newline, or None on EOF.""" - self.sock.settimeout(timeout) - while b"\n" not in self.buf: - chunk = self.sock.recv(65536) - if not chunk: - return None - self.buf += chunk - line, _, self.buf = self.buf.partition(b"\n") - return line - - def read(self, timeout=10.0): - line = self.read_line(timeout) - return None if line is None else json.loads(line) - - def read_until(self, pred, timeout=10.0, seen=None): - deadline = time.time() + timeout - while True: - obj = self.read(max(0.05, deadline - time.time())) - if obj is None: - raise AssertionError("EOF while waiting for a matching message") - if seen is not None: - seen.append(obj) - if pred(obj): - return obj - - def read_to_eof(self, timeout=10.0): - out = [] - deadline = time.time() + timeout - while True: - obj = self.read(max(0.05, deadline - time.time())) - if obj is None: - return out - out.append(obj) - - def hello_obj(self, v=0): - return {"t": "hello", "id": self.next_id(), "ts": ts(), "v": v, "key": self.key, - "name": "raw-test-client", "nonce": b64url(os.urandom(32)), "caps": ["chat"], - "about": "unit test"} - - def handshake(self, seen=None, tamper=False, grants=None): - """Full hello/auth/resume exchange. Returns (peer_hello, peer_auth, peer_resume).""" - peer_hello_line = self.read_line() - assert peer_hello_line is not None, "no hello from peer" - peer_hello = json.loads(peer_hello_line) - assert peer_hello["t"] == "hello" - my_line = self.send(self.hello_obj()) - peer_auth = self.read() - assert peer_auth and peer_auth["t"] == "auth", peer_auth - # Independently verify the peer's signature (section 7.2). - Ed25519PublicKey.from_public_bytes(raw_pub(peer_hello["key"])).verify( - b64url_dec(peer_auth["sig"]), - b"awp-auth-v0\x00" + peer_hello_line + b"\x00" + my_line) - sig = bytearray(self.sk.sign(b"awp-auth-v0\x00" + my_line + b"\x00" + peer_hello_line)) - if tamper: - sig[5] ^= 0x01 - auth = {"t": "auth", "id": self.next_id(), "ts": ts(), "sig": b64url(bytes(sig))} - if grants: - auth["grants"] = grants - self.send(auth) - if tamper: - return peer_hello, peer_auth, None - peer_resume = self.read_until(lambda o: o.get("t") == "resume") - self.send({"t": "resume", "id": self.next_id(), "ts": ts(), "seen": seen or {}}) - return peer_hello, peer_auth, peer_resume - - def ping_roundtrip(self, timeout=10.0, seen=None): - pid = self.next_id() - self.send({"t": "ping", "id": pid, "ts": ts()}) - return self.read_until(lambda o: o.get("t") == "pong" and o.get("re") == pid, - timeout=timeout, seen=seen) - - def close(self): - try: - self.sock.close() - except OSError: - pass - - -# ---------------------------------------------------------------- base class - -class PeerTestBase(unittest.TestCase): - def setUp(self): - self.tmp = tempfile.mkdtemp(prefix="hp-") - if len(os.path.join(self.tmp, "listen.sock")) > 100: # AF_UNIX path limit - shutil.rmtree(self.tmp) - self.tmp = tempfile.mkdtemp(prefix="hp-", dir="/tmp") - self.procs = [] - self.raws = [] - - def tearDown(self): - for r in self.raws: - r.close() - for p in self.procs: - p.stop() - shutil.rmtree(self.tmp, ignore_errors=True) - - def state(self, name) -> str: - return os.path.join(self.tmp, f"state-{name}") - - def unix_addr(self, name="listen") -> str: - return "unix:" + os.path.join(self.tmp, f"{name}.sock") - - def spawn(self, state, mode, addr, name, **kw) -> PeerProc: - p = PeerProc(self.tmp, state, mode, addr, name, **kw) - self.procs.append(p) - return p - - def listener(self, addr, name="alice", state=None, **kw): - p = self.spawn(state or self.state(name), "listen", addr, name, **kw) - ev = p.wait_event("listening") - return p, ev["addr"] - - def raw(self, addr, **kw) -> RawPeer: - r = RawPeer(addr, **kw) - self.raws.append(r) - return r - - def pair(self, addr, **kw): - """alice listens on addr, bob connects; both report connected.""" - a, real = self.listener(addr, "alice", **kw) - b = self.spawn(self.state("bob"), "connect", real, "bob", **kw) - ida = a.wait_event("identity") - idb = b.wait_event("identity") - ca = a.wait_event("connected") - cb = b.wait_event("connected") - self.assertEqual(ca["key"], idb["key"]) - self.assertEqual(cb["key"], ida["key"]) - return a, b, real - - -# ---------------------------------------------------------------- two peers - -class TwoPeerTests(PeerTestBase): - - def _roundtrip(self, addr): - a, b, real = self.pair(addr) - self.assertTrue(a.wait_event("listening")["addr"].startswith(addr.split(":")[0])) - # First connection: both resumes carry an empty seen map. - self.assertEqual(a.wait_for(is_recv("resume"))["msg"]["seen"], {}) - self.assertEqual(b.wait_for(is_recv("resume"))["msg"]["seen"], {}) - # bob -> alice, acked - b.cmd(cmd="send", th="t1", subject="Port the middleware", text="hello alice") - mid = b.wait_event("sent", t="msg")["id"] - got = a.wait_for(is_recv("msg", id=mid))["msg"] - self.assertEqual(got["th"], "t1") - self.assertEqual(got["subject"], "Port the middleware") - self.assertEqual(text_of(got), "hello alice") - for f in ("id", "ts", "t"): - self.assertIn(f, got) - b.wait_event("acked", id=mid) - ack = b.wait_for(is_recv("ack", re=mid))["msg"] - self.assertEqual(ack["th"], "t1") - # alice -> bob, reply with re, acked - a.cmd(cmd="send", th="t1", text="hello bob", re=mid) - rid = a.wait_event("sent", t="msg")["id"] - reply = b.wait_for(is_recv("msg", id=rid))["msg"] - self.assertEqual(reply["re"], mid) - a.wait_event("acked", id=rid) - # state is acked too - a.cmd(cmd="state", th="t1", state="working", note="running tests") - sid = a.wait_event("sent", t="state")["id"] - st = b.wait_for(is_recv("state", id=sid))["msg"] - self.assertEqual((st["state"], st["note"], st["th"]), ("working", "running tests", "t1")) - a.wait_event("acked", id=sid) - # ids from one sender sort by time - ids = [e["id"] for e in a.find(lambda e: e.get("event") == "sent")] - self.assertEqual(ids, sorted(ids)) - - def test_tcp_handshake_and_message_roundtrip(self): - self._roundtrip("tcp:127.0.0.1:0") - - def test_unix_handshake_and_message_roundtrip(self): - self._roundtrip(self.unix_addr()) - - def test_queued_while_disconnected_delivered_in_order(self): - addr = self.unix_addr() - a, b, _ = self.pair(addr) - b.cmd(cmd="send", th="t1", subject="queue test", text="first") - first = b.wait_event("sent")["id"] - a.wait_for(is_recv("msg", id=first)) - b.wait_event("acked", id=first) - a.kill9() - b.wait_event("disconnected") - mark = b.mark() - texts = [f"queued-{i}" for i in range(6)] - for i, t in enumerate(texts): - b.cmd(cmd="send", th="t1" if i % 2 == 0 else "t2", text=t) - b.cmd(cmd="state", th="t1", state="waiting", note="while you were away") - sent = b.wait_count(lambda e: e.get("event") == "sent", 1 + len(texts) + 1) - queued_ids = [e["id"] for e in sent[1:]] - time.sleep(0.3) - self.assertEqual(b.find(lambda e: e.get("event") == "acked", start=mark), []) - a2 = self.spawn(self.state("alice"), "listen", addr, "alice") # same state + address - a2.wait_event("connected") - # alice's resume after restart remembers what it saw before the kill - resume = b.wait_for(is_recv("resume"), start=mark)["msg"] - self.assertEqual(resume["seen"], {"t1": first}) - got = a2.wait_count(lambda e: e.get("event") == "recv" and e["msg"].get("t") in ("msg", "state"), - len(queued_ids)) - self.assertEqual([e["msg"]["id"] for e in got], queued_ids) - self.assertEqual([text_of(e["msg"]) for e in got[:-1]], texts) - for q in queued_ids: - b.wait_event("acked", id=q) - - def test_messages_queued_before_first_connection(self): - addr = self.unix_addr() - b = self.spawn(self.state("bob"), "connect", addr, "bob") # nobody listening yet - b.wait_event("identity") - for i in range(3): - b.cmd(cmd="send", th="early", text=f"early-{i}") - ids = [e["id"] for e in b.wait_count(lambda e: e.get("event") == "sent", 3)] - time.sleep(0.7) # a few failed attempts - a, _ = self.listener(addr) - got = a.wait_count(is_recv("msg"), 3) - self.assertEqual([e["msg"]["id"] for e in got], ids) - for i in ids: - b.wait_event("acked", id=i) - - def test_kill9_receiver_resumes_without_loss_or_duplicates(self): - addr = f"tcp:127.0.0.1:{free_port()}" - a1, b, _ = self.pair(addr) - n1, n2 = 10, 30 - # A paced conversation first. The kill lands between messages: the peer - # emits `recv` a few microseconds before it persists the id, so a kill - # inside that gap could legitimately repeat one event (at-least-once). - for i in range(n1): - b.cmd(cmd="send", th="t1" if i % 3 else "t2", text=f"m{i}") - time.sleep(0.01) - a1.wait_for(lambda e: is_recv("msg")(e) and text_of(e["msg"]) == f"m{n1 - 1}", what="paced") - time.sleep(0.005) - a1.kill9() - # Burst immediately: the first of these are written into the dying - # connection before bob notices, the rest queue in the outbox. - for i in range(n1, n1 + n2): - b.cmd(cmd="send", th="t1" if i % 3 else "t2", text=f"m{i}") - sent = b.wait_count(lambda e: e.get("event") == "sent", n1 + n2) - sent_ids = [e["id"] for e in sent] - b.wait_event("disconnected") - a2 = self.spawn(self.state("alice"), "listen", addr, "alice") - a2.wait_event("connected") - a2.wait_for(lambda e: is_recv("msg")(e) and text_of(e["msg"]) == f"m{n1 + n2 - 1}", - what="last message") - for i in sent_ids: - b.wait_event("acked", id=i) - received = [e["msg"] for p in (a1, a2) for e in p.find(is_recv("msg"))] - rid = [m["id"] for m in received] - self.assertEqual(len(rid), len(set(rid)), "duplicate delivery") - self.assertEqual(set(rid), set(sent_ids), "lost messages") - # per-thread order preserved across the crash - by_text = {f"m{i}": sent_ids[i] for i in range(n1 + n2)} - for th in ("t1", "t2"): - expect = [by_text[f"m{i}"] for i in range(n1 + n2) if (("t1" if i % 3 else "t2") == th)] - self.assertEqual([m["id"] for m in received if m["th"] == th], expect) - - def test_kill9_sender_replays_from_durable_outbox(self): - addr = self.unix_addr() - a, b1, _ = self.pair(addr) - n = 30 - for i in range(n): - b1.cmd(cmd="send", th="t1", text=f"s{i}") - ids = [e["id"] for e in b1.wait_count(lambda e: e.get("event") == "sent", n)] - b1.kill9() # some of those are still unacked - a.wait_event("disconnected") - b2 = self.spawn(self.state("bob"), "connect", addr, "bob") - b2.wait_event("connected") - a.wait_for(lambda e: is_recv("msg")(e) and text_of(e["msg"]) == f"s{n - 1}", what="last") - got = [e["msg"]["id"] for e in a.find(is_recv("msg"))] - self.assertEqual(got, ids) # exactly once, in order - deadline = time.time() + 10 - while time.time() < deadline: - acked = {e["id"] for p in (b1, b2) for e in p.find(lambda e: e.get("event") == "acked")} - if acked >= set(ids): - break - time.sleep(0.1) - self.assertTrue(acked >= set(ids), f"unacked: {set(ids) - acked}") - - def test_multichunk_blob_arrives_byte_identical(self): - a, b, _ = self.pair("tcp:127.0.0.1:0") - data = os.urandom(700 * 1024 + 123) - path = os.path.join(self.tmp, "integration.log") - with open(path, "wb") as f: - f.write(data) - b.cmd(cmd="blob", th="t1", path=path, text="Log attached.") - mid = b.wait_event("sent", t="msg")["id"] - ev = a.wait_event("blob", timeout=30) - with open(ev["path"], "rb") as f: - self.assertEqual(f.read(), data) - self.assertEqual(ev["size"], len(data)) - self.assertEqual(ev["sha256"], hashlib.sha256(data).hexdigest()) - chunks = [e["msg"] for e in a.find(is_recv("chunk"))] - self.assertEqual([c["n"] for c in chunks], [0, 1, 2]) - self.assertEqual([c["last"] for c in chunks], [False, False, True]) - self.assertTrue(all("data" not in c and c["ref"] == ev["ref"] for c in chunks)) - msg = a.wait_for(is_recv("msg", id=mid))["msg"] - blob_part = [p for p in msg["parts"] if p["k"] == "blob"][0] - self.assertEqual((blob_part["ref"], blob_part["size"], blob_part["name"]), - (ev["ref"], len(data), "integration.log")) - self.assertEqual(text_of(msg), "Log attached.") - b.wait_event("acked", id=mid) - # acking the msg prunes its chunks and the outgoing copy - outgoing = os.path.join(self.state("bob"), "outgoing") - deadline = time.time() + 5 - while os.listdir(outgoing) and time.time() < deadline: - time.sleep(0.05) - self.assertEqual(os.listdir(outgoing), []) - - def test_kill9_receiver_mid_blob_completes_after_restart(self): - addr = f"tcp:127.0.0.1:{free_port()}" - a1, b, _ = self.pair(addr) - data = os.urandom(11 * 256 * 1024 + 999) # 12 chunks - path = os.path.join(self.tmp, "big.tar") - with open(path, "wb") as f: - f.write(data) - b.cmd(cmd="blob", th="b1", path=path, text="tarball") - mid = b.wait_event("sent", t="msg")["id"] - a1.wait_for(lambda e: is_recv("chunk")(e) and e["msg"]["n"] == 1, what="chunk 1") - a1.kill9() - self.assertEqual(a1.find(lambda e: e.get("event") == "blob"), []) - b.wait_event("disconnected") - a2 = self.spawn(self.state("alice"), "listen", addr, "alice") - ev = a2.wait_event("blob", timeout=30) - with open(ev["path"], "rb") as f: - self.assertEqual(hashlib.sha256(f.read()).hexdigest(), hashlib.sha256(data).hexdigest()) - self.assertEqual(ev["sha256"], hashlib.sha256(data).hexdigest()) - b.wait_event("acked", id=mid) - # the restarted receiver resumed mid-blob instead of starting over - ns = [e["msg"]["n"] for e in a2.find(is_recv("chunk"))] - self.assertTrue(ns and ns[0] >= 1, ns) - self.assertEqual(ns, list(range(ns[0], 12))) - - def test_blob_over_limit_is_refused(self): - a, _ = self.listener("tcp:127.0.0.1:0", extra=["--max-blob", "100000"]) - addr = a.wait_event("listening")["addr"] - b = self.spawn(self.state("bob"), "connect", addr, "bob") - b.wait_event("connected") - path = os.path.join(self.tmp, "big.bin") - with open(path, "wb") as f: - f.write(os.urandom(300 * 1024)) - b.cmd(cmd="blob", th="t1", path=path) - mid = b.wait_event("sent", t="msg")["id"] - err = b.wait_for(is_recv("err", code="blob_refused"))["msg"] - self.assertTrue(err["ref"].startswith("blob_")) - b.wait_event("acked", id=mid) # the msg itself is still delivered - time.sleep(0.3) - self.assertEqual(a.find(lambda e: e.get("event") == "blob"), []) - self.assertEqual(len(b.find(is_recv("err"))), 1) - self.assertEqual(os.listdir(os.path.join(self.state("bob"), "outgoing")), []) - - def test_grant_gates_exec_requests(self): - a, b, _ = self.pair(self.unix_addr()) - exec_part = {"k": "data", "mime": EXEC_MIME, "data": {"argv": ["make", "test"]}} - b.cmd(cmd="send", th="x1", text="please run", parts=[exec_part]) - m1 = b.wait_event("sent")["id"] - b.wait_for(is_recv("err", code="forbidden", re=m1)) - b.wait_event("acked", id=m1) - # alice grants bob exec; verify the grant independently - bkey = b.wait_event("identity")["key"] - akey = a.wait_event("identity")["key"] - a.cmd(cmd="grant", sub=bkey, caps=["exec"], ttl=600) - a.wait_event("sent", t="grant") - g = b.wait_for(is_recv("grant"))["msg"]["grant"] - self.assertEqual((g["iss"], g["sub"], g["caps"]), (akey, bkey, ["exec"])) - body = {k: v for k, v in g.items() if k != "sig"} - canon = json.dumps(body, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() - Ed25519PublicKey.from_public_bytes(raw_pub(akey)).verify(b64url_dec(g["sig"]), canon) - exp = datetime.strptime(g["exp"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) - self.assertTrue(500 < (exp - datetime.now(timezone.utc)).total_seconds() <= 601) - # bob keeps the grant (it will present it in future auth messages) - with open(os.path.join(self.state("bob"), "grants_held.json")) as f: - self.assertEqual(json.load(f)[0]["sig"], g["sig"]) - # the same request is now allowed: acked with no forbidden err before the ack - mark = b.mark() - b.cmd(cmd="send", th="x1", text="please run again", parts=[exec_part]) - m2 = b.wait_event("sent", start=mark, t="msg")["id"] - b.wait_event("acked", id=m2) - self.assertEqual(b.find(is_recv("err", re=m2)), []) - # after a restart bob presents the held grant in its auth message - b.kill9() - a.wait_event("disconnected") - mark = a.mark() - b2 = self.spawn(self.state("bob"), "connect", self.unix_addr(), "bob") - b2.wait_event("connected") - auth = a.wait_for(is_recv("auth"), start=mark)["msg"] - self.assertEqual([x["sig"] for x in auth.get("grants", [])], [g["sig"]]) - - def test_bye_closes_both_sides_and_suppresses_reconnect(self): - a, b, _ = self.pair(self.unix_addr()) - b.cmd(cmd="send", th="t1", text="one thing") # t1 is now an open thread - mid = b.wait_event("sent")["id"] - b.wait_event("acked", id=mid) - b.cmd(cmd="bye", reason="done") - b.wait_event("sent", t="bye") - self.assertEqual(a.wait_for(is_recv("bye"))["msg"]["reason"], "done") - b.wait_for(is_recv("bye")) # alice answered with bye - a.wait_event("disconnected") - b.wait_event("disconnected") - time.sleep(1.5) - self.assertEqual(len(b.find(lambda e: e.get("event") == "connected")), 1) - # new outbound work brings the connection back - b.cmd(cmd="send", th="t1", text="after bye") - a.wait_for(lambda e: is_recv("msg")(e) and text_of(e["msg"]) == "after bye") - self.assertEqual(len(b.wait_count(lambda e: e.get("event") == "connected", 2)), 2) - - -# ---------------------------------------------------------------- raw sockets - -class RawSocketTests(PeerTestBase): - - def test_tampered_auth_signature_rejected(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.handshake(tamper=True) - rest = r.read_to_eof() - errs = [o for o in rest if o.get("t") == "err"] - self.assertEqual([e["code"] for e in errs], ["auth"]) - self.assertEqual([o for o in rest if o.get("t") == "resume"], []) - a.wait_event("error") - self.assertEqual(a.find(lambda e: e.get("event") == "connected"), []) - - def test_version_1_rejected(self): - a, addr = self.listener("tcp:127.0.0.1:0") - r = self.raw(addr) - self.assertEqual(json.loads(r.read_line())["t"], "hello") - r.send(r.hello_obj(v=1)) - rest = r.read_to_eof() - self.assertEqual([o["t"] for o in rest], ["err"]) # no auth after a bad version - self.assertEqual(rest[0]["code"], "version") - - def test_garbage_line_gets_bad_frame(self): - a, addr = self.listener(self.unix_addr()) - for garbage in (b"this is not json\n", b"[1,2,3]\n"): - r = self.raw(addr) - r.read_line() - r.send_raw(garbage) - rest = r.read_to_eof() - self.assertEqual([(o["t"], o["code"]) for o in rest], [("err", "bad_frame")]) - # also after a completed handshake - r = self.raw(addr) - r.handshake() - r.send_raw(b"{\"t\":\"msg\",oops}\n") - rest = r.read_to_eof() - self.assertEqual([(o["t"], o.get("code")) for o in rest if o["t"] == "err"], [("err", "bad_frame")]) - - def test_line_over_1mib_gets_too_large(self): - a, addr = self.listener("tcp:127.0.0.1:0") - r = self.raw(addr) - r.read_line() - r.send_raw(b"x" * ((1 << 20) + 100)) - rest = r.read_to_eof() - self.assertEqual([(o["t"], o["code"]) for o in rest], [("err", "too_large")]) - - def test_unknown_type_and_unknown_fields_ignored(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.handshake() - a.wait_event("connected") - r.send({"t": "frobnicate", "id": r.next_id(), "ts": ts(), "th": "t9", "payload": {"x": [1, 2]}}) - r.send({"t": "hello-again-v9", "ts": ts()}) - r.send({"no_t_at_all": True}) - mid = r.next_id() - r.send({"t": "msg", "id": mid, "ts": ts(), "th": "t9", "subject": "extensible", - "x-priority": "high", "future": {"nested": [1, {"a": None}]}, - "parts": [{"k": "text", "text": "hi", "lang_hint": "en"}, - {"k": "hologram", "frames": 3}, - {"k": "code", "lang": "go", "text": "package main"}, - {"k": "data", "mime": "application/json", "data": {"z": 1}}]}) - seen = [] - ack = r.read_until(lambda o: o.get("t") == "ack", seen=seen) - self.assertEqual((ack["re"], ack["th"]), (mid, "t9")) - r.ping_roundtrip(seen=seen) - self.assertEqual([o for o in seen if o.get("t") == "err"], []) - got = a.wait_for(is_recv("msg", id=mid))["msg"] - self.assertEqual(got["future"], {"nested": [1, {"a": None}]}) - self.assertEqual(len(got["parts"]), 4) - a.wait_for(is_recv("frobnicate")) - self.assertEqual(a.find(lambda e: e.get("event") in ("error", "disconnected")), []) - - def test_duplicate_id_is_delivered_once_and_reacked(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.handshake() - mid = r.next_id() - m = {"t": "msg", "id": mid, "ts": ts(), "th": "d1", "parts": [{"k": "text", "text": "once"}]} - r.send(m) - ack1 = r.read_until(lambda o: o.get("t") == "ack") - r.send(m) # replayed after it was received - ack2 = r.read_until(lambda o: o.get("t") == "ack") - self.assertEqual((ack1["re"], ack2["re"]), (mid, mid)) - self.assertNotEqual(ack1["id"], ack2["id"]) - r.ping_roundtrip() - self.assertEqual(len(a.find(is_recv("msg", id=mid))), 1) - - def test_resume_replays_unacked_in_original_order_with_original_ids(self): - a, addr = self.listener(self.unix_addr()) - for i in range(3): # queued before anyone connects - a.cmd(cmd="send", th="t1", text=f"r{i}", subject="replay" if i == 0 else None) - ids = [e["id"] for e in a.wait_count(lambda e: e.get("event") == "sent", 3)] - r = self.raw(addr) - r.handshake() - first = [r.read_until(lambda o: o.get("t") == "msg") for _ in range(3)] - self.assertEqual([m["id"] for m in first], ids) - mine = r.next_id() # something for alice to remember - r.send({"t": "msg", "id": mine, "ts": ts(), "th": "t7", "parts": [{"k": "text", "text": "x"}]}) - r.read_until(lambda o: o.get("t") == "ack" and o.get("re") == mine) - r.send({"t": "ack", "id": r.next_id(), "ts": ts(), "th": "t1", "re": ids[0]}) - a.wait_event("acked", id=ids[0]) - r.close() - a.wait_event("disconnected") - # same key again, claiming to have durably seen up to ids[1] - r2 = self.raw(addr, sk=r.sk) - _, _, resume = r2.handshake(seen={"t1": ids[1]}) - self.assertEqual(resume["seen"], {"t7": mine}) # alice's own seen map - a.wait_event("acked", id=ids[1]) # covered by our seen: implicit ack - seen = [] - m = r2.read_until(lambda o: o.get("t") == "msg", seen=seen) - self.assertEqual(m, first[2]) # original id, ts and content - r2.send({"t": "ack", "id": r2.next_id(), "ts": ts(), "th": "t1", "re": ids[2]}) - a.wait_event("acked", id=ids[2]) - r2.ping_roundtrip(seen=seen) - self.assertEqual([o["id"] for o in seen if o.get("t") == "msg"], [ids[2]]) - - def test_silent_peer_is_dropped_after_two_missed_pongs(self): - a, addr = self.listener(self.unix_addr(), ping=0.3) - r = self.raw(addr) - r.handshake() - t0 = time.time() - rest = r.read_to_eof(timeout=10) - elapsed = time.time() - t0 - pings = [o for o in rest if o.get("t") == "ping"] - self.assertGreaterEqual(len(pings), 2) - self.assertLess(elapsed, 3.0) - ev = a.wait_event("disconnected") - self.assertIn("missed pongs", ev["reason"]) - - def test_go_style_encodings_and_omitted_fields_accepted(self): - """Padded std-alphabet key/sig, seen:null (Go nil map), msg-before-chunks, - chunks without th and with omitempty-dropped n/last, unpadded/url data.""" - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - peer_hello_line = r.read_line() - hello = r.hello_obj() - hello["key"] = "ed25519:" + base64.b64encode(r.pub).decode() # padded, std - my_line = r.send(hello) - r.read_until(lambda o: o.get("t") == "auth") - sig = r.sk.sign(b"awp-auth-v0\x00" + my_line + b"\x00" + peer_hello_line) - r.send({"t": "auth", "id": r.next_id(), "ts": ts(), "sig": base64.b64encode(sig).decode()}) - r.read_until(lambda o: o.get("t") == "resume") - r.send({"t": "resume", "id": r.next_id(), "ts": ts(), "seen": None}) - self.assertEqual(a.wait_event("connected")["key"], hello["key"]) # echoed as sent - data = os.urandom(300 * 1024) - c0, c1 = data[:200 * 1024], data[200 * 1024:] - mid = r.next_id() - r.send({"t": "msg", "id": mid, "ts": ts(), "th": "g1", - "parts": [{"k": "blob", "ref": "L1", "name": "x.bin", - "mime": "application/octet-stream", "size": len(data)}]}) - r.read_until(lambda o: o.get("t") == "ack" and o.get("re") == mid) - r.send({"t": "chunk", "id": r.next_id(), "ts": ts(), "ref": "L1", - "data": base64.b64encode(c0).decode().rstrip("=")}) # no n, no last - r.send({"t": "chunk", "id": r.next_id(), "ts": ts(), "ref": "L1", "n": 1, "last": True, - "data": base64.urlsafe_b64encode(c1).decode()}) - ev = a.wait_event("blob") - with open(ev["path"], "rb") as f: - self.assertEqual(f.read(), data) - self.assertEqual(ev["ref"], "L1") - - def test_bye_is_answered_with_bye(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.handshake() - r.send({"t": "bye", "id": r.next_id(), "ts": ts(), "reason": "done"}) - rest = r.read_to_eof() - self.assertEqual([(o["t"], o.get("reason")) for o in rest], [("bye", "done")]) - a.wait_event("disconnected", reason="bye") - - def test_err_close_semantics(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.handshake() - for code in ("unsupported", "forbidden", "internal", "blob_refused", "some_future_code"): - r.send({"t": "err", "id": r.next_id(), "ts": ts(), "code": code, "detail": "x"}) - r.ping_roundtrip() # still open - r.send({"t": "err", "id": r.next_id(), "ts": ts(), "code": "auth", "detail": "go away"}) - self.assertEqual(r.read_to_eof(), []) - a.wait_event("disconnected") - - def test_one_level_grant_delegation_via_introduce(self): - a, addr = self.listener(self.unix_addr()) - akey = a.wait_event("identity")["key"] - - def mint(sk, sub, caps): - iss = "ed25519:" + b64url(sk.public_key().public_bytes( - serialization.Encoding.Raw, serialization.PublicFormat.Raw)) - g = {"iss": iss, "sub": sub, "caps": caps, "exp": "2099-01-01T00:00:00Z", - "nonce": b64url(os.urandom(16))} - g["sig"] = b64url(sk.sign(json.dumps(g, sort_keys=True, separators=(",", ":")).encode())) - return g - - def exec_forbidden(r): - mid = r.next_id() - r.send({"t": "msg", "id": mid, "ts": ts(), "th": "e1", - "parts": [{"k": "data", "mime": EXEC_MIME, "data": {"argv": ["id"]}}]}) - seen = [] - r.read_until(lambda o: o.get("t") == "ack" and o.get("re") == mid, seen=seen) - r.ping_roundtrip(seen=seen) # a forbidden err would precede this - return any(o.get("t") == "err" and o.get("code") == "forbidden" and o.get("re") == mid - for o in seen) - - # alice gives X the right to introduce - x = self.raw(addr) - x.handshake() - self.assertTrue(exec_forbidden(x)) # introduce alone is not exec - a.cmd(cmd="grant", sub=x.key, caps=["introduce"], ttl=600) - g_ax = x.read_until(lambda o: o.get("t") == "grant")["grant"] - self.assertEqual((g_ax["iss"], g_ax["caps"]), (akey, ["introduce"])) - # X introduces itself with a grant for alice (stored as held by alice) - g_xa = mint(x.sk, akey, ["chat-extra"]) - x.send({"t": "introduce", "id": x.next_id(), "ts": ts(), "th": "i1", - "peer": {"key": x.key, "name": "x", "address": "unix:/nowhere"}, "grant": g_xa}) - x.ping_roundtrip() - with open(os.path.join(self.state("alice"), "grants_held.json")) as f: - self.assertIn(g_xa["sig"], [g["sig"] for g in json.load(f)]) - x.close() - # X delegates exec to Y; alice honours it because X holds introduce from alice - ysk = Ed25519PrivateKey.generate() - ykey = "ed25519:" + b64url(ysk.public_key().public_bytes( - serialization.Encoding.Raw, serialization.PublicFormat.Raw)) - y = self.raw(addr, sk=ysk) - y.handshake(grants=[mint(x.sk, ykey, ["exec"])]) - self.assertFalse(exec_forbidden(y)) - # an issuer holding no introduce grant is not honoured - wsk = Ed25519PrivateKey.generate() - z = self.raw(addr) - z.handshake(grants=[mint(wsk, z.key, ["exec"])]) - self.assertTrue(exec_forbidden(z)) - - def test_messages_before_auth_are_rejected(self): - a, addr = self.listener(self.unix_addr()) - r = self.raw(addr) - r.read_line() - r.send(r.hello_obj()) - r.send({"t": "msg", "id": r.next_id(), "ts": ts(), "th": "t1", - "parts": [{"k": "text", "text": "sneaky"}]}) - rest = r.read_to_eof() - self.assertEqual([o.get("code") for o in rest if o["t"] == "err"], ["auth"]) - self.assertEqual(a.find(is_recv("msg")), []) - - -# ---------------------------------------------------------------- unit level - -class UnitTests(unittest.TestCase): - - def test_pure_python_ed25519_matches_cryptography(self): - for i in range(4): - seed = os.urandom(32) - msg = os.urandom(i * 37) - sk = Ed25519PrivateKey.from_private_bytes(seed) - pub = sk.public_key().public_bytes(serialization.Encoding.Raw, - serialization.PublicFormat.Raw) - self.assertEqual(hp.pure_ed25519_public(seed), pub) - sig = hp.pure_ed25519_sign(seed, msg) - self.assertEqual(sig, sk.sign(msg)) # Ed25519 is deterministic - self.assertTrue(hp.pure_ed25519_verify(pub, sig, msg)) - bad = bytearray(sig) - bad[0] ^= 1 - self.assertFalse(hp.pure_ed25519_verify(pub, bytes(bad), msg)) - self.assertFalse(hp.pure_ed25519_verify(pub, sig, msg + b"x")) - - def test_pure_python_peer_interoperates(self): - """A peer forced onto the fallback crypto still handshakes with a raw client.""" - tmp = tempfile.mkdtemp(prefix="hp-") - try: - env = dict(os.environ, AWP_PURE_ED25519="1") - sock = os.path.join(tmp, "p.sock") - if len(sock) > 100: - shutil.rmtree(tmp) - tmp = tempfile.mkdtemp(prefix="hp-", dir="/tmp") - sock = os.path.join(tmp, "p.sock") - p = subprocess.Popen([sys.executable, PEER, "--state", os.path.join(tmp, "s"), - "listen", "unix:" + sock], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, env=env) - try: - for line in p.stdout: - if json.loads(line).get("event") == "listening": - break - r = RawPeer("unix:" + sock) - r.handshake() # verifies the peer's signature too - r.ping_roundtrip() - r.close() - finally: - p.kill() - p.wait() - p.stdin.close() - p.stdout.close() - finally: - shutil.rmtree(tmp, ignore_errors=True) - - def test_ulids_are_monotonic(self): - g = hp.UlidGen() - ids = [g.new() for _ in range(5000)] - self.assertEqual(ids, sorted(ids)) - self.assertEqual(len(set(ids)), len(ids)) - self.assertTrue(all(len(i) == 26 for i in ids)) - g2 = hp.UlidGen() - future = hp.ulid_encode(((time.time_ns() // 1_000_000 + 60_000) << 80) | 5) - g2.observe(future) # e.g. clock went backwards - self.assertGreater(g2.new(), future) - - def test_liberal_base64(self): - raw = os.urandom(32) - for s in (base64.urlsafe_b64encode(raw).decode(), - base64.urlsafe_b64encode(raw).decode().rstrip("="), - base64.b64encode(raw).decode(), - base64.b64encode(raw).decode().rstrip("=")): - self.assertEqual(hp.b64decode_any(s), raw) - self.assertEqual(hp.parse_key("ed25519:" + s), raw) - with self.assertRaises(ValueError): - hp.parse_key("ed25519:" + b64url(raw[:31])) - with self.assertRaises(ValueError): - hp.parse_key("rsa:" + b64url(raw)) - - def test_rfc3339_parsing(self): - base = hp.parse_rfc3339("2026-09-25T20:00:00Z") - self.assertEqual(hp.parse_rfc3339("2026-09-25T20:00:00.123456789Z"), base + 0.123456789) - self.assertEqual(hp.parse_rfc3339("2026-09-25T22:00:00+02:00"), base) - self.assertEqual(hp.parse_rfc3339("2026-09-25T15:30:00-04:30"), base) - - def test_grant_mint_and_verify(self): - seed = os.urandom(32) - ident = hp.Identity(seed) - sub = "ed25519:" + b64url(os.urandom(32)) - g = hp.mint_grant(ident, sub, ["exec", "fs:read"], 60) - self.assertEqual(hp.verify_grant(g), (True, "ok")) - tampered = dict(g, caps=["exec", "fs:read", "admin"]) - self.assertFalse(hp.verify_grant(tampered)[0]) - expired = hp.mint_grant(ident, sub, ["exec"], -10) - self.assertEqual(hp.verify_grant(expired), (False, "expired")) - # a Go-style (HTML-escaped) canonical encoding also verifies - sk = Ed25519PrivateKey.from_private_bytes(seed) - g2 = {"iss": ident.key, "sub": sub, "caps": ["a&c"], "exp": "2099-01-01T00:00:00Z", - "nonce": "n"} - go_bytes = json.dumps(g2, sort_keys=True, separators=(",", ":")).replace( - "<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026").encode() - g2["sig"] = b64url(sk.sign(go_bytes)) - self.assertTrue(hp.verify_grant(g2)[0]) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/schema/schema.go b/schema/schema.go index e7352e3..6141bad 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -1,8 +1,8 @@ // Package schema holds the published JSON Schema of the Agent Wire Protocol -// (v0/awp.schema.json) and validates lines against it. +// (v1/awp.schema.json) and validates lines against it. // // The schema is generated from the wire package by go generate ./wire; it -// is the same document that https://agentwireprotocol.com/schema/v0/awp.schema.json +// is the same document that https://agentwireprotocol.com/schema/v1/awp.schema.json // serves. A test fails when the file is older than the package. package schema @@ -22,11 +22,11 @@ import ( ) // ID is the schema's $id. -const ID = "https://agentwireprotocol.com/schema/v0/awp.schema.json" +const ID = "https://agentwireprotocol.com/schema/v1/awp.schema.json" // V0 is the schema document. // -//go:embed v0/awp.schema.json +//go:embed v1/awp.schema.json var V0 []byte // Validator checks lines and objects against the schema. It is safe for @@ -99,7 +99,7 @@ func (v *Validator) Def(name string) (*jsonschema.Schema, error) { // ErrUnknownType is wrapped by Line for a line whose type the schema does // not know. Such a line is still checked against the envelope, since -// receivers must ignore unknown types (section 5) after reading it. +// receivers must ignore unknown types (section 8) after reading it. var ErrUnknownType = errors.New("unknown message type") // Line validates one line as it travels: a JSON object whose type selects diff --git a/schema/schema_test.go b/schema/schema_test.go index 3f68cad..2cd84ba 100644 --- a/schema/schema_test.go +++ b/schema/schema_test.go @@ -124,9 +124,8 @@ func TestGoMessages(t *testing.T) { }} msgLine, _ := wire.Encode(msg) messages := map[string]any{ - wire.THello: wire.Hello{Envelope: env("hello", "", ""), V: wire.Version, Key: key, Name: "go@test", Nonce: wire.Nonce(32), - Caps: []string{"chat", "blob"}, About: "test", Addr: "unix:/tmp/x", Shares: []string{otherKey}}, - wire.TAuth: wire.Auth{Envelope: env("auth", "", ""), Sig: wire.SignAuth(priv, []byte("a"), []byte("b")), Grants: []json.RawMessage{grant.Raw}}, + wire.THello: wire.Hello{Envelope: env("hello", "", ""), V: wire.Version, Key: key, Name: "go@test", + Caps: []string{"chat", "blob"}, About: "test", Addr: "awp1o2NrZXlYICz4lGVLe_5MyFCpuS1-L-EVBF8KHDCr9bieP7E5semX", Grants: []json.RawMessage{grant.Raw}, Shares: []string{otherKey}}, wire.TResume: wire.Resume{Envelope: env("resume", "", ""), Seen: map[string]string{"t1": ids.New()}}, wire.TMsg: msg, wire.TState: wire.State{Envelope: env("state", "t1", ""), State: "working", Note: "n"}, @@ -137,7 +136,7 @@ func TestGoMessages(t *testing.T) { wire.TBye: wire.Bye{Envelope: env("bye", "", ""), Reason: "done"}, wire.TErr: wire.Err{Envelope: env("err", "", ""), Code: wire.ErrBlobRefused, Detail: "too big", Ref: "b1"}, wire.TGrant: wire.GrantMsg{Envelope: env("grant", "", ""), Grant: grant.Raw}, - wire.TIntroduce: wire.Introduce{Envelope: env("introduce", "t1", ""), Peer: wire.IntroPeer{Key: otherKey, Name: "o", Address: "tailcat:tc0"}, Grant: grant.Raw}, + wire.TIntroduce: wire.Introduce{Envelope: env("introduce", "t1", ""), Peer: wire.IntroPeer{Key: otherKey, Name: "o", Address: "awp1o2NrZXlYICz4lGVLe_5MyFCpuS1-L-EVBF8KHDCr9bieP7E5semX"}, Grant: grant.Raw}, wire.TPresence: wire.PresenceMsg{Envelope: env("presence", "", ""), Doc: presence, Hops: 1}, wire.TMirror: wire.Mirror{Envelope: env("mirror", "t1", ""), Of: otherKey, OfName: "o", Subject: "s", Dir: "out", Line: msgLine}, wire.TPrivate: wire.Private{Envelope: env("private", "t1", "")}, @@ -188,8 +187,8 @@ func TestRejects(t *testing.T) { "unknown part": fmt.Sprintf(`{"t":"msg","id":"x","ts":"%s","th":"t","parts":[{"k":"hologram"}]}`, ts), "text part empty": fmt.Sprintf(`{"t":"msg","id":"x","ts":"%s","th":"t","parts":[{"k":"text"}]}`, ts), "blob no size": fmt.Sprintf(`{"t":"msg","id":"x","ts":"%s","th":"t","parts":[{"k":"blob","ref":"r"}]}`, ts), - "hello bad key": fmt.Sprintf(`{"t":"hello","id":"x","ts":"%s","v":0,"key":"rsa:abc","nonce":"abc"}`, ts), - "hello v string": fmt.Sprintf(`{"t":"hello","id":"x","ts":"%s","v":"0","key":"ed25519:%s","nonce":"abc"}`, ts, strings.Repeat("A", 43)), + "hello bad key": fmt.Sprintf(`{"t":"hello","id":"x","ts":"%s","v":1,"key":"rsa:abc"}`, ts), + "hello v string": fmt.Sprintf(`{"t":"hello","id":"x","ts":"%s","v":"1","key":"ed25519:%s"}`, ts, strings.Repeat("A", 43)), "err bad code": fmt.Sprintf(`{"t":"err","id":"x","ts":"%s","code":"oops"}`, ts), "chunk bad data": fmt.Sprintf(`{"t":"chunk","id":"x","ts":"%s","ref":"r","n":0,"last":true,"data":"not base64!"}`, ts), "chunk n negative": fmt.Sprintf(`{"t":"chunk","id":"x","ts":"%s","ref":"r","n":-1,"last":true,"data":"aGk="}`, ts), diff --git a/schema/v0/awp.schema.json b/schema/v1/awp.schema.json similarity index 89% rename from schema/v0/awp.schema.json rename to schema/v1/awp.schema.json index f3aa15b..1857364 100644 --- a/schema/v0/awp.schema.json +++ b/schema/v1/awp.schema.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://agentwireprotocol.com/schema/v0/awp.schema.json", + "$id": "https://agentwireprotocol.com/schema/v1/awp.schema.json", "$ref": "#/$defs/Message", "$defs": { "Ack": { @@ -40,54 +40,6 @@ "title": "ack", "description": "Ack says the msg or state named by re is durably received (section 9.2). Acks drive outbox pruning and are not acked themselves." }, - "Auth": { - "properties": { - "t": { - "type": "string", - "const": "auth", - "description": "`t` is the message type." - }, - "id": { - "type": "string", - "minLength": 1, - "description": "`id` is unique per sender. A ULID or UUIDv7 is recommended, so that ids sort by time; resume compares ids as strings." - }, - "ts": { - "type": "string", - "format": "date-time", - "description": "`ts` is an RFC 3339 UTC timestamp." - }, - "th": { - "type": "string", - "description": "`th` is the thread id. Required for msg, state and ack." - }, - "re": { - "type": "string", - "description": "`re` is the id of the message this one responds to." - }, - "sig": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]+$", - "description": "`sig` is the Ed25519 signature, base64url, over \"awp-auth-v0\" || 0x00 || my hello line || 0x00 || peer hello line, the lines as sent and received without the newline." - }, - "grants": { - "items": { - "$ref": "#/$defs/Grant" - }, - "type": "array", - "description": "`grants` are grant objects (section 10.2) the sender presents." - } - }, - "type": "object", - "required": [ - "t", - "id", - "ts", - "sig" - ], - "title": "auth", - "description": "Auth proves possession of the key sent in hello (section 7.2)." - }, "BlobPart": { "properties": { "k": { @@ -159,7 +111,7 @@ "ts" ], "title": "bye", - "description": "Bye is a graceful close (section 9.6). After sending it a peer sends nothing else and closes after the other side's bye or after 5 seconds." + "description": "Bye is a graceful close (section 12.6). After sending it a peer sends nothing else and closes after the other side's bye or after 5 seconds." }, "Chunk": { "properties": { @@ -216,7 +168,7 @@ "data" ], "title": "chunk", - "description": "Chunk carries part of a blob (section 9.3). Chunks of one blob arrive in order; chunks of different blobs may interleave. A chunk carries th when the blob belongs to a thread, so that resume replays it like any other threaded message." + "description": "Chunk carries part of a blob (section 12.3). Chunks of one blob arrive in order; chunks of different blobs may interleave. A chunk carries th when the blob belongs to a thread, so that resume replays it like any other threaded message." }, "CodePart": { "properties": { @@ -263,7 +215,7 @@ "data" ], "title": "data part", - "description": "Inline JSON, with its MIME type. Requests (section 10.1) are data parts of a vnd.awp type." + "description": "Inline JSON, with its MIME type. Requests (section 13.1) are data parts of a vnd.awp type." }, "Envelope": { "properties": { @@ -296,7 +248,7 @@ "id", "ts" ], - "description": "Envelope holds the fields every line carries (section 6). The type-specific fields sit beside them at the top level." + "description": "Envelope holds the fields every line carries (section 9). The type-specific fields sit beside them at the top level." }, "Err": { "properties": { @@ -327,12 +279,13 @@ "type": "string", "enum": [ "bad_frame", + "too_large", "version", "auth", + "refused", "unsupported", "forbidden", "blob_refused", - "too_large", "internal" ], "description": "`code` is the error code." @@ -354,7 +307,7 @@ "code" ], "title": "err", - "description": "Err reports a problem (section 9.7). Whether it closes the connection depends on the code: bad_frame, version, auth and too_large do, unsupported, forbidden, blob_refused and internal do not." + "description": "Err reports a problem (section 12.7). Whether it closes the connection depends on the code: bad_frame, too_large, version, auth and refused do, unsupported, forbidden, blob_refused and internal do not." }, "Grant": { "properties": { @@ -373,7 +326,7 @@ "type": "string" }, "type": "array", - "description": "`caps` are the capability strings granted (section 10.1)." + "description": "`caps` are the capability strings granted (section 13.1)." }, "exp": { "type": "string", @@ -405,7 +358,7 @@ "nonce", "sig" ], - "description": "GrantObject is the wire form of a grant (section 10.2): a signed statement that sub may use caps until exp, honored by iss and by whoever trusts iss. sig is iss's Ed25519 signature over the canonical JSON of the object without sig: keys sorted, no whitespace, UTF-8." + "description": "GrantObject is the wire form of a grant (section 13.2): a signed statement that sub may use caps until exp, honored by iss and by whoever trusts iss. sig is iss's Ed25519 signature over the canonical JSON of the object without sig: keys sorted, no whitespace, UTF-8." }, "GrantMsg": { "properties": { @@ -434,7 +387,7 @@ }, "grant": { "$ref": "#/$defs/Grant", - "description": "`grant` is the grant object (section 10.2)." + "description": "`grant` is the grant object (section 13.2)." } }, "type": "object", @@ -445,7 +398,7 @@ "grant" ], "title": "grant", - "description": "GrantMsg delivers a grant after the handshake (section 10.3)." + "description": "GrantMsg delivers a grant after the handshake (section 13.3)." }, "Hello": { "properties": { @@ -475,28 +428,23 @@ "v": { "type": "integer", "minimum": 0, - "description": "`v` is the protocol major version. A peer that sees a version it does not speak sends err version and closes." + "description": "`v` is the protocol major version, 1 for draft 2. A peer that sees a version it does not speak sends err version and closes." }, "key": { "type": "string", "pattern": "^ed25519:[A-Za-z0-9_-]{43}$", - "description": "`key` is the long-term Ed25519 public key. Its bytes are the peer's identity." + "description": "`key` is the sender's Ed25519 identity key. It must be the key on the other end of the tunnel (section 13.1), or the receiver sends err auth and closes." }, "name": { "type": "string", "description": "`name` is how the peer calls itself, \"harness@host\" by convention." }, - "nonce": { - "type": "string", - "pattern": "^[A-Za-z0-9_-]+$", - "description": "`nonce` is 32 random bytes, base64url. The auth signature covers the whole hello line, nonce included." - }, "caps": { "items": { "type": "string" }, "type": "array", - "description": "`caps` lists the supported message families beyond the mandatory chat and resume: blob, grant, introduce, and any extension.", + "description": "`caps` lists the supported message families beyond the mandatory core: blob, grant, introduce, and any extension.", "examples": [ [ "chat", @@ -508,11 +456,19 @@ }, "about": { "type": "string", - "description": "`about` is free text for the other agent's context. It is not authenticated until auth completes." + "description": "`about` is free text for the other agent's context." }, "addr": { "type": "string", - "description": "`addr` is an extension: an address at which the sender can be reached, so that either side can reconnect (section 4.3). Peers that do not know it ignore it, as section 5 requires." + "pattern": "^awp1[A-Za-z0-9_-]+$", + "description": "`addr` is the sender's own address without its pre-shared key, so the other side can reconnect to it later (section 13.3)." + }, + "grants": { + "items": { + "$ref": "#/$defs/Grant" + }, + "type": "array", + "description": "`grants` are grant objects (section 13.2) the sender presents." }, "shares": { "items": { @@ -528,11 +484,10 @@ "id", "ts", "v", - "key", - "nonce" + "key" ], "title": "hello", - "description": "Hello is the first line each side sends, at once, without waiting for the other (section 7.1)." + "description": "Hello is the first line each side sends, at once, without waiting for the other (section 10.1)." }, "IntroPeer": { "properties": { @@ -547,7 +502,8 @@ }, "address": { "type": "string", - "description": "Address is where the introduced peer listens, a hint: the key is the identity." + "pattern": "^awp1[A-Za-z0-9_-]+$", + "description": "Address is the introduced peer's address (section 9). It carries the pre-shared key only when the introducer holds one." } }, "type": "object", @@ -598,16 +554,13 @@ "peer" ], "title": "introduce", - "description": "Introduce hands the recipient another peer's identity and address plus a grant issued by the introducer (section 10.4). The introduced peer honors the grant only if it trusts the introducer with introduce." + "description": "Introduce hands the recipient another peer's identity and address plus a grant issued by the introducer (section 13.4). The introduced peer honors the grant only if it trusts the introducer with introduce." }, "Message": { "oneOf": [ { "$ref": "#/$defs/Hello" }, - { - "$ref": "#/$defs/Auth" - }, { "$ref": "#/$defs/Resume" }, @@ -651,7 +604,7 @@ "$ref": "#/$defs/Private" } ], - "description": "One line of a connection: a message of SPEC.md sections 7 to 10, or one of the extensions marked x-extension. A receiver ignores lines of a type it does not know, after checking the envelope." + "description": "One line of a connection: a message of SPEC.md sections 10 to 13, or one of the extensions marked x-extension. A receiver ignores lines of a type it does not know, after checking the envelope." }, "Mirror": { "properties": { @@ -773,7 +726,7 @@ "th" ], "title": "msg", - "description": "Msg is one turn in a thread (section 9.1). The first msg with a new th creates the thread." + "description": "Msg is one turn in a thread (section 12.1). The first msg with a new th creates the thread." }, "Part": { "oneOf": [ @@ -825,7 +778,7 @@ "ts" ], "title": "ping", - "description": "Ping is a liveness probe (section 9.4): sent when idle for 30 seconds by convention; two missed pongs mean the connection is dead." + "description": "Ping is a liveness probe (section 12.4): sent when idle for 30 seconds by convention; two missed pongs mean the connection is dead." }, "Pong": { "properties": { @@ -1152,7 +1105,7 @@ "seen" ], "title": "resume", - "description": "Resume is sent by both sides after every handshake (section 9.5). The receiver replays its outbox messages the sender has not seen." + "description": "Resume is sent by both sides after every handshake (section 12.5). The receiver replays its outbox messages the sender has not seen." }, "State": { "properties": { @@ -1205,7 +1158,7 @@ "th" ], "title": "state", - "description": "State is the sender's view of a thread's soft state (section 8.1). The two sides can disagree." + "description": "State is the sender's view of a thread's soft state (section 11.1). The two sides can disagree." }, "TextPart": { "properties": { @@ -1229,6 +1182,6 @@ } }, "$comment": "Generated from the wire package of github.com/agentwireprotocol/awp by go generate ./wire. Do not edit by hand.", - "title": "Agent Wire Protocol v0", + "title": "Agent Wire Protocol v1", "description": "The messages of the Agent Wire Protocol, one JSON object per line (SPEC.md). Every message is an envelope (t, id, ts, and th or re where they apply) with the fields of its type beside it." } diff --git a/schema/v0/schema.mdx b/schema/v1/schema.mdx similarity index 81% rename from schema/v0/schema.mdx rename to schema/v1/schema.mdx index c6279f7..345f4f9 100644 --- a/schema/v0/schema.mdx +++ b/schema/v1/schema.mdx @@ -5,13 +5,13 @@ description: Every message of the protocol and its fields, generated from the re {/* Generated from the wire package of github.com/agentwireprotocol/awp by go generate ./wire. Do not edit by hand. */} -Every line of a connection is one JSON object: the [envelope](#envelope) plus the fields of its message type, selected by `t`. This page is generated from the same source as the [JSON Schema](https://agentwireprotocol.com/schema/v0/awp.schema.json) (draft 2020-12), the `wire` package of the reference implementation, and lists the same constraints. Where it disagrees with [the specification](https://agentwireprotocol.com/spec), the specification wins and the schema has a bug. +Every line of a connection is one JSON object: the [envelope](#envelope) plus the fields of its message type, selected by `t`. This page is generated from the same source as the [JSON Schema](https://agentwireprotocol.com/schema/v1/awp.schema.json) (draft 2020-12), the `wire` package of the reference implementation, and lists the same constraints. Where it disagrees with [the specification](https://agentwireprotocol.com/spec), the specification wins and the schema has a bug. Receivers ignore fields they do not know, and lines whose `t` they do not know, so no object here is closed. Messages marked *extension* are spoken by the reference implementation and not in the specification. ## Envelope -Envelope holds the fields every line carries (section 6). The type-specific fields sit beside them at the top level. +Envelope holds the fields every line carries (section 9). The type-specific fields sit beside them at the top level. | Field | Type | Required | Description | |---|---|---|---| @@ -29,35 +29,24 @@ Each message carries the envelope; the tables list the fields beside it. ### hello -Hello is the first line each side sends, at once, without waiting for the other (section 7.1). +Hello is the first line each side sends, at once, without waiting for the other (section 10.1). Envelope: `t` is `"hello"`. | Field | Type | Required | Description | |---|---|---|---| -| `v` | integer, ≥ 0 | yes | `v` is the protocol major version. A peer that sees a version it does not speak sends err version and closes. | -| `key` | string, `^ed25519:[A-Za-z0-9_-]{43}$` | yes | `key` is the long-term Ed25519 public key. Its bytes are the peer's identity. | +| `v` | integer, ≥ 0 | yes | `v` is the protocol major version, 1 for draft 2. A peer that sees a version it does not speak sends err version and closes. | +| `key` | string, `^ed25519:[A-Za-z0-9_-]{43}$` | yes | `key` is the sender's Ed25519 identity key. It must be the key on the other end of the tunnel (section 13.1), or the receiver sends err auth and closes. | | `name` | string | | `name` is how the peer calls itself, "harness@host" by convention. | -| `nonce` | string, `^[A-Za-z0-9_-]+$` | yes | `nonce` is 32 random bytes, base64url. The auth signature covers the whole hello line, nonce included. | -| `caps` | array of string | | `caps` lists the supported message families beyond the mandatory chat and resume: blob, grant, introduce, and any extension. | -| `about` | string | | `about` is free text for the other agent's context. It is not authenticated until auth completes. | -| `addr` | string | | `addr` is an extension: an address at which the sender can be reached, so that either side can reconnect (section 4.3). Peers that do not know it ignore it, as section 5 requires. | +| `caps` | array of string | | `caps` lists the supported message families beyond the mandatory core: blob, grant, introduce, and any extension. | +| `about` | string | | `about` is free text for the other agent's context. | +| `addr` | string, `^awp1[A-Za-z0-9_-]+$` | | `addr` is the sender's own address without its pre-shared key, so the other side can reconnect to it later (section 13.3). | +| `grants` | array of [Grant](#grant-object) | | `grants` are grant objects (section 13.2) the sender presents. | | `shares` | array of string | | `shares` is an extension (mirror.go): the keys of the hosts this agent mirrors its conversations to, so the other party knows. | -### auth - -Auth proves possession of the key sent in hello (section 7.2). - -Envelope: `t` is `"auth"`. - -| Field | Type | Required | Description | -|---|---|---|---| -| `sig` | string, `^[A-Za-z0-9_-]+$` | yes | `sig` is the Ed25519 signature, base64url, over "awp-auth-v0" \|\| 0x00 \|\| my hello line \|\| 0x00 \|\| peer hello line, the lines as sent and received without the newline. | -| `grants` | array of [Grant](#grant-object) | | `grants` are grant objects (section 10.2) the sender presents. | - ### resume -Resume is sent by both sides after every handshake (section 9.5). The receiver replays its outbox messages the sender has not seen. +Resume is sent by both sides after every handshake (section 12.5). The receiver replays its outbox messages the sender has not seen. Envelope: `t` is `"resume"`. @@ -67,7 +56,7 @@ Envelope: `t` is `"resume"`. ### state -State is the sender's view of a thread's soft state (section 8.1). The two sides can disagree. +State is the sender's view of a thread's soft state (section 11.1). The two sides can disagree. Envelope: `t` is `"state"`; `th` is required. @@ -78,7 +67,7 @@ Envelope: `t` is `"state"`; `th` is required. ### msg -Msg is one turn in a thread (section 9.1). The first msg with a new th creates the thread. +Msg is one turn in a thread (section 12.1). The first msg with a new th creates the thread. Envelope: `t` is `"msg"`; `th` is required. @@ -95,7 +84,7 @@ Envelope: `t` is `"ack"`; `th` and `re` are required. ### chunk -Chunk carries part of a blob (section 9.3). Chunks of one blob arrive in order; chunks of different blobs may interleave. A chunk carries th when the blob belongs to a thread, so that resume replays it like any other threaded message. +Chunk carries part of a blob (section 12.3). Chunks of one blob arrive in order; chunks of different blobs may interleave. A chunk carries th when the blob belongs to a thread, so that resume replays it like any other threaded message. Envelope: `t` is `"chunk"`. @@ -108,7 +97,7 @@ Envelope: `t` is `"chunk"`. ### ping -Ping is a liveness probe (section 9.4): sent when idle for 30 seconds by convention; two missed pongs mean the connection is dead. +Ping is a liveness probe (section 12.4): sent when idle for 30 seconds by convention; two missed pongs mean the connection is dead. Envelope: `t` is `"ping"`. @@ -120,7 +109,7 @@ Envelope: `t` is `"pong"`; `re` is required. ### bye -Bye is a graceful close (section 9.6). After sending it a peer sends nothing else and closes after the other side's bye or after 5 seconds. +Bye is a graceful close (section 12.6). After sending it a peer sends nothing else and closes after the other side's bye or after 5 seconds. Envelope: `t` is `"bye"`. @@ -130,29 +119,29 @@ Envelope: `t` is `"bye"`. ### err -Err reports a problem (section 9.7). Whether it closes the connection depends on the code: bad_frame, version, auth and too_large do, unsupported, forbidden, blob_refused and internal do not. +Err reports a problem (section 12.7). Whether it closes the connection depends on the code: bad_frame, too_large, version, auth and refused do, unsupported, forbidden, blob_refused and internal do not. Envelope: `t` is `"err"`. | Field | Type | Required | Description | |---|---|---|---| -| `code` | string, one of `bad_frame`, `version`, `auth`, `unsupported`, `forbidden`, `blob_refused`, `too_large`, `internal` | yes | `code` is the error code. | +| `code` | string, one of `bad_frame`, `too_large`, `version`, `auth`, `refused`, `unsupported`, `forbidden`, `blob_refused`, `internal` | yes | `code` is the error code. | | `detail` | string | | `detail` is human readable. | | `ref` | string | | `ref` names the blob a blob_refused is about. | ### grant -GrantMsg delivers a grant after the handshake (section 10.3). +GrantMsg delivers a grant after the handshake (section 13.3). Envelope: `t` is `"grant"`. | Field | Type | Required | Description | |---|---|---|---| -| `grant` | [Grant](#grant-object) | yes | `grant` is the grant object (section 10.2). | +| `grant` | [Grant](#grant-object) | yes | `grant` is the grant object (section 13.2). | ### introduce -Introduce hands the recipient another peer's identity and address plus a grant issued by the introducer (section 10.4). The introduced peer honors the grant only if it trusts the introducer with introduce. +Introduce hands the recipient another peer's identity and address plus a grant issued by the introducer (section 13.4). The introduced peer honors the grant only if it trusts the introducer with introduce. Envelope: `t` is `"introduce"`. @@ -218,7 +207,7 @@ Fenced code without the fence; lang names the language. ### data part -Inline JSON, with its MIME type. Requests (section 10.1) are data parts of a vnd.awp type. +Inline JSON, with its MIME type. Requests (section 13.1) are data parts of a vnd.awp type. | Field | Type | Required | Description | |---|---|---|---| @@ -242,13 +231,13 @@ Objects carried inside messages. ### Grant object -GrantObject is the wire form of a grant (section 10.2): a signed statement that sub may use caps until exp, honored by iss and by whoever trusts iss. sig is iss's Ed25519 signature over the canonical JSON of the object without sig: keys sorted, no whitespace, UTF-8. +GrantObject is the wire form of a grant (section 13.2): a signed statement that sub may use caps until exp, honored by iss and by whoever trusts iss. sig is iss's Ed25519 signature over the canonical JSON of the object without sig: keys sorted, no whitespace, UTF-8. | Field | Type | Required | Description | |---|---|---|---| | `iss` | string, `^ed25519:[A-Za-z0-9_-]{43}$` | yes | `iss` is the granting key. | | `sub` | string, `^ed25519:[A-Za-z0-9_-]{43}$` | yes | `sub` is the receiving key. | -| `caps` | array of string | yes | `caps` are the capability strings granted (section 10.1). | +| `caps` | array of string | yes | `caps` are the capability strings granted (section 13.1). | | `exp` | string, date-time | yes | `exp` is when the grant expires, RFC 3339. | | `nonce` | string, `^[A-Za-z0-9_-]+$` | yes | `nonce` makes each grant distinct, base64url. | | `sig` | string, `^[A-Za-z0-9_-]+$` | yes | `sig` is the issuer's signature, base64url. | @@ -262,7 +251,7 @@ IntroPeer identifies the introduced peer. |---|---|---|---| | `key` | string, `^ed25519:[A-Za-z0-9_-]{43}$` | yes | Key is the introduced peer's public key. | | `name` | string | | Name is what the introduced peer calls itself. | -| `address` | string | | Address is where the introduced peer listens, a hint: the key is the identity. | +| `address` | string, `^awp1[A-Za-z0-9_-]+$` | | Address is the introduced peer's address (section 9). It carries the pre-shared key only when the introducer holds one. | ### Presence document diff --git a/scripts/conformance.sh b/scripts/conformance.sh deleted file mode 100755 index 1404aed..0000000 --- a/scripts/conformance.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# Run the conformance suite against the Python peer, both ways: with the -# peer listening (every scenario, each on a connection of its own), then -# with the peer dialing the runner (the scenarios that share a connection). -set -euo pipefail - -root=$(cd "$(dirname "$0")/.." && pwd) -bin=${AWP_BIN:-$root/bin/awp} -peer="$root/python/awp_peer.py" -tmp=$(mktemp -d) -listener="" -cleanup() { - [ -n "$listener" ] && kill "$listener" 2>/dev/null || true - rm -rf "${tmp:?}" -} -trap cleanup EXIT - -port=$(python3 -c 'import socket; s = socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') -python3 "$peer" --state "$tmp/listen" --name py-listen listen "tcp:127.0.0.1:$port" >"$tmp/listen.log" 2>&1 /dev/null; then - break - fi - sleep 0.1 -done - -echo "== the Python peer listens; the runner connects" -"$bin" conform "tcp:127.0.0.1:$port" -kill "$listener" -wait "$listener" 2>/dev/null || true -listener="" - -echo -echo "== the runner listens; the Python peer connects" -"$bin" conform --listen tcp:127.0.0.1:0 \ - --run "python3 $peer --state $tmp/connect --name py-connect connect {addr}" 0 { diff --git a/transport/transport.go b/transport/transport.go deleted file mode 100644 index 9c5f286..0000000 --- a/transport/transport.go +++ /dev/null @@ -1,330 +0,0 @@ -// Package transport provides the byte-stream bindings AWP runs over -// (SPEC.md section 4): tailcat, embedded as a library, plus plain TCP and -// Unix sockets for Fly 6PN, local use and tests. -package transport - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net" - "net/netip" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/tailscale/tailcat" - "tailscale.com/types/logger" -) - -// TailcatPort is the port awp listens on inside the tailcat tunnel. It -// is the port tailcat's own pipe mode uses, so `tailcat ` connects a -// terminal straight to an awp peer: you can type NDJSON at it. -const TailcatPort = 1 - -// Kinds of address. -const ( - KindTailcat = "tailcat" - KindTCP = "tcp" - KindUnix = "unix" -) - -// Addr is a parsed awp address. -type Addr struct { - Kind string - Target string // tailcat address, host:port or socket path -} - -// String returns the canonical form: "tailcat:tc...", "tcp:host:port" or -// "unix:/path". -func (a Addr) String() string { return a.Kind + ":" + a.Target } - -// Parse accepts: -// -// tc... a bare tailcat address, as `tailcat` prints it -// tailcat:tc... -// tcp:host:port or tcp://host:port -// unix:/path or unix:///path -func Parse(s string) (Addr, error) { - s = strings.TrimSpace(s) - switch { - case strings.HasPrefix(s, "tailcat:"): - t := strings.TrimPrefix(strings.TrimPrefix(s, "tailcat:"), "//") - if !looksTailcat(t) { - return Addr{}, fmt.Errorf("address %q: not a tailcat address", s) - } - return Addr{KindTailcat, t}, nil - case looksTailcat(s): - return Addr{KindTailcat, s}, nil - case strings.HasPrefix(s, "tcp:"): - t := strings.TrimPrefix(strings.TrimPrefix(s, "tcp:"), "//") - if _, _, err := net.SplitHostPort(t); err != nil { - return Addr{}, fmt.Errorf("address %q: %v", s, err) - } - return Addr{KindTCP, t}, nil - case strings.HasPrefix(s, "unix:"): - t := strings.TrimPrefix(s, "unix:") - if strings.HasPrefix(t, "//") { - t = strings.TrimPrefix(t, "//") - } - if t == "" { - return Addr{}, fmt.Errorf("address %q: empty socket path", s) - } - return Addr{KindUnix, t}, nil - } - return Addr{}, fmt.Errorf("address %q: want tc..., tailcat:tc..., tcp:host:port or unix:/path", s) -} - -func looksTailcat(s string) bool { - if len(s) < 40 || !strings.HasPrefix(s, "tc") { - return false - } - for _, c := range s { - ok := c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' || c == '_' - if !ok { - return false - } - } - return true -} - -// Transport dials awp addresses. It keeps one tailcat client per -// tailcat address, since each client is a whole userspace WireGuard node. -type Transport struct { - // Logf receives tailcat's diagnostics. Nil discards them. - Logf logger.Logf - - // AllowPublicTCP permits plain TCP to public addresses. By default - // awp refuses, as section 4.2 asks: plain TCP has no encryption, so - // it is only for loopback and private networks such as Fly's 6PN. - AllowPublicTCP bool - - mu sync.Mutex - clients map[string]*tailcat.Client -} - -// Dial opens a byte stream to a. -func (t *Transport) Dial(ctx context.Context, a Addr) (net.Conn, error) { - switch a.Kind { - case KindTCP: - if !t.AllowPublicTCP { - if err := checkPrivate(ctx, a.Target); err != nil { - return nil, err - } - } - var d net.Dialer - return d.DialContext(ctx, "tcp", a.Target) - case KindUnix: - var d net.Dialer - return d.DialContext(ctx, "unix", a.Target) - case KindTailcat: - c := t.client(a.Target) - conn, err := c.DialTCPPort(ctx, TailcatPort) - if err != nil { - return nil, fmt.Errorf("tailcat dial: %w", err) - } - return conn, nil - } - return nil, fmt.Errorf("unsupported address kind %q", a.Kind) -} - -var cgnat = netip.MustParsePrefix("100.64.0.0/10") // Tailscale and carrier-grade NAT - -// checkPrivate refuses plain TCP to any address outside loopback, private -// (RFC 1918 and IPv6 ULA, which includes Fly's fdaa::/16 6PN), link-local -// and CGNAT ranges. -func checkPrivate(ctx context.Context, target string) error { - host, _, err := net.SplitHostPort(target) - if err != nil { - return err - } - ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) - if err != nil { - return err - } - for _, ip := range ips { - ip = ip.Unmap() - if !(ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || cgnat.Contains(ip)) { - return fmt.Errorf("refusing plain TCP to public address %s: awp over TCP is unencrypted (use a tailcat address, or set AWP_ALLOW_PLAINTEXT=1)", ip) - } - } - return nil -} - -func (t *Transport) client(addr string) *tailcat.Client { - t.mu.Lock() - defer t.mu.Unlock() - if c, ok := t.clients[addr]; ok { - return c - } - if t.clients == nil { - t.clients = map[string]*tailcat.Client{} - } - c := &tailcat.Client{Server: tailcat.Addr(addr), Logf: t.logf()} - t.clients[addr] = c - return c -} - -func (t *Transport) logf() logger.Logf { - if t.Logf == nil { - return logger.Discard - } - return t.Logf -} - -// Reset drops the cached tailcat client for a, so the next dial starts from -// a fresh WireGuard node. Used after repeated failures, and when the peer at -// that address is no longer wanted. -func (t *Transport) Reset(a Addr) { - if a.Kind != KindTailcat { - return - } - t.mu.Lock() - c := t.clients[a.Target] - delete(t.clients, a.Target) - t.mu.Unlock() - if c != nil { - go c.Close() - } -} - -// Close shuts down all tailcat clients. -func (t *Transport) Close() { - t.mu.Lock() - clients := t.clients - t.clients = nil - t.mu.Unlock() - for _, c := range clients { - c.Close() - } -} - -// Listen opens a listener for a TCP or Unix address. A stale Unix socket -// file is removed first. -func Listen(a Addr) (net.Listener, error) { - switch a.Kind { - case KindTCP: - return net.Listen("tcp", a.Target) - case KindUnix: - if fi, err := os.Stat(a.Target); err == nil && fi.Mode()&os.ModeSocket != 0 { - if c, err := net.DialTimeout("unix", a.Target, time.Second); err == nil { - c.Close() - return nil, fmt.Errorf("%s: already in use", a.Target) - } - os.Remove(a.Target) - } - return net.Listen("unix", a.Target) - } - return nil, fmt.Errorf("cannot listen on %s addresses this way", a.Kind) -} - -// TailcatListener is an awp listener inside a tailcat tunnel. -type TailcatListener struct { - net.Listener - srv *tailcat.Server - addr string -} - -// Address returns the listener's shareable tailcat address. -func (l *TailcatListener) Address() string { return l.addr } - -// Close stops the listener and the tailcat server. -func (l *TailcatListener) Close() error { - l.Listener.Close() - return l.srv.Close() -} - -// ListenTailcat starts a tailcat server whose key, pre-shared key and DERP -// region persist in keyFile, so the address stays the same across restarts -// (a sandbox waking from sleep comes back at the address its peers know). -// The first run picks the nearest region and writes the file. -func ListenTailcat(ctx context.Context, keyFile string, logf logger.Logf) (*TailcatListener, error) { - if logf == nil { - logf = logger.Discard - } - pk, err := loadOrCreateKey(keyFile) - if err != nil { - return nil, err - } - if pk.Public.RegionID <= 0 { - // Pick the nearest DERP region once and remember it, like - // `tailcat genkey --fixed-region`: the same key, pre-shared key and - // region give the same address on every start. (Reading the region - // back out of the address does not work: embedded regions are - // renumbered.) - dm, err := tailcat.FetchDERPMap(ctx, tailcat.ExpandForServer) - if err != nil { - return nil, fmt.Errorf("fetching DERP map: %w", err) - } - id, err := tailcat.PickBestRegion(ctx, dm) - if err != nil { - return nil, fmt.Errorf("picking DERP region: %w", err) - } - if id == 0 { - return nil, errors.New("no reachable DERP region") - } - pk.Public.RegionID = id - if err := saveKey(keyFile, pk); err != nil { - return nil, err - } - } - srv := &tailcat.Server{ - Key: pk.Private, - PresharedKey: pk.Public.PresharedKey, - RegionID: pk.Public.RegionID, - Logf: logf, - } - ln, err := srv.Listen(ctx, "tcp", fmt.Sprintf(":%d", TailcatPort)) - if err != nil { - srv.Close() - if strings.Contains(err.Error(), "no such region") { - // The DERP map dropped our region: forget it so the next - // attempt picks a new one (the address changes; peers learn - // it from hello). - pk.Public.RegionID = 0 - saveKey(keyFile, pk) - } - return nil, err - } - return &TailcatListener{Listener: ln, srv: srv, addr: string(srv.TailcatAddr())}, nil -} - -func loadOrCreateKey(path string) (*tailcat.PrivateKey, error) { - b, err := os.ReadFile(path) - if err == nil { - var pk tailcat.PrivateKey - if err := json.Unmarshal(b, &pk); err != nil { - return nil, fmt.Errorf("%s: %w", path, err) - } - if pk.Private.IsZero() || pk.Public.PresharedKey.IsZero() { - return nil, fmt.Errorf("%s: incomplete tailcat key", path) - } - return &pk, nil - } - if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - pk := tailcat.NewPrivateKey() - if err := saveKey(path, pk); err != nil { - return nil, err - } - return pk, nil -} - -func saveKey(path string, pk *tailcat.PrivateKey) error { - b, err := json.MarshalIndent(pk, "", "\t") - if err != nil { - return err - } - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return err - } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, b, 0o600); err != nil { - return err - } - return os.Rename(tmp, path) -} diff --git a/tunnel/address.go b/tunnel/address.go new file mode 100644 index 0000000..f5f07e3 --- /dev/null +++ b/tunnel/address.go @@ -0,0 +1,136 @@ +package tunnel + +import ( + "crypto/ed25519" + "encoding/base64" + "errors" + "fmt" + "strings" + + "github.com/fxamacker/cbor/v2" +) + +// AddressPrefix starts every AWP address. +const AddressPrefix = "awp1" + +// Carrier kinds (section 7). +const ( + KindUDP = "udp" + KindTailcat = "tailcat" + KindWS = "ws" + KindUnix = "unix" +) + +// Endpoint is where a peer can be reached on one carrier. +type Endpoint struct { + Kind string `cbor:"k" json:"k"` + Value string `cbor:"v" json:"v"` +} + +func (e Endpoint) String() string { return e.Kind + ":" + e.Value } + +// Address is a peer's key, optional pre-shared key and endpoints: everything +// a dialer needs (section 6). +type Address struct { + Key ed25519.PublicKey + PSK []byte // 32 bytes, or nil + Endpoints []Endpoint +} + +type wireAddress struct { + Key []byte `cbor:"key"` + PSK []byte `cbor:"psk,omitempty"` + EP []Endpoint `cbor:"ep,omitempty"` +} + +var encMode = func() cbor.EncMode { + m, err := cbor.CoreDetEncOptions().EncMode() + if err != nil { + panic(err) + } + return m +}() + +// String encodes the address: "awp1" and the unpadded base64url of its +// deterministic CBOR encoding. +func (a Address) String() string { + b, err := encMode.Marshal(wireAddress{Key: a.Key, PSK: a.PSK, EP: a.Endpoints}) + if err != nil { + return "" + } + return AddressPrefix + base64.RawURLEncoding.EncodeToString(b) +} + +// KeyString is the address's key in the protocol's ed25519: form. +func (a Address) KeyString() string { + return "ed25519:" + base64.RawURLEncoding.EncodeToString(a.Key) +} + +// Public is the address without its pre-shared key: what a peer may say +// about itself in hello, where the admission secret has no place. +func (a Address) Public() Address { + a.PSK = nil + return a +} + +// IsAddress reports whether s looks like an AWP address. +func IsAddress(s string) bool { return strings.HasPrefix(strings.TrimSpace(s), AddressPrefix) } + +// ParseAddress decodes an address. Unknown endpoint kinds are kept (a dialer +// skips kinds it has no carrier for); unknown CBOR keys are ignored. +func ParseAddress(s string) (Address, error) { + s = strings.TrimSpace(s) + if !strings.HasPrefix(s, AddressPrefix) { + return Address{}, fmt.Errorf("address %q: want %s...", clip(s), AddressPrefix) + } + b, err := base64.RawURLEncoding.DecodeString(s[len(AddressPrefix):]) + if err != nil { + return Address{}, fmt.Errorf("address %q: %v", clip(s), err) + } + var w wireAddress + if err := cbor.Unmarshal(b, &w); err != nil { + return Address{}, fmt.Errorf("address %q: %v", clip(s), err) + } + if len(w.Key) != ed25519.PublicKeySize { + return Address{}, fmt.Errorf("address %q: key is %d bytes, want 32", clip(s), len(w.Key)) + } + if w.PSK != nil && len(w.PSK) != 32 { + return Address{}, fmt.Errorf("address %q: pre-shared key is %d bytes, want 32", clip(s), len(w.PSK)) + } + if _, err := X25519Public(w.Key); err != nil { + return Address{}, fmt.Errorf("address %q: key is not a valid Ed25519 point", clip(s)) + } + return Address{Key: ed25519.PublicKey(w.Key), PSK: w.PSK, Endpoints: w.EP}, nil +} + +// Merge combines addresses of one key: endpoints in first-seen order without +// duplicates, and the first pre-shared key given. +func Merge(as ...Address) (Address, error) { + if len(as) == 0 { + return Address{}, errors.New("no address") + } + out := Address{Key: as[0].Key} + seen := map[Endpoint]bool{} + for _, a := range as { + if !a.Key.Equal(out.Key) { + return Address{}, fmt.Errorf("addresses for different keys: %s and %s", out.KeyString(), a.KeyString()) + } + if out.PSK == nil && a.PSK != nil { + out.PSK = a.PSK + } + for _, e := range a.Endpoints { + if !seen[e] { + seen[e] = true + out.Endpoints = append(out.Endpoints, e) + } + } + } + return out, nil +} + +func clip(s string) string { + if len(s) > 24 { + return s[:24] + "…" + } + return s +} diff --git a/tunnel/bind.go b/tunnel/bind.go new file mode 100644 index 0000000..4fb284b --- /dev/null +++ b/tunnel/bind.go @@ -0,0 +1,315 @@ +package tunnel + +import ( + "encoding/hex" + "errors" + "net" + "net/netip" + "strings" + "sync" + "time" + + "github.com/tailscale/wireguard-go/conn" +) + +// bind is the WireGuard conn.Bind over every carrier. WireGuard sees one +// endpoint per peer (peerEP); the bind decides which carrier path a packet +// takes. A peer's current path is the one its last authenticated packet +// came from; handshake packets also go to every candidate path, so a peer +// that moved or slept is found again (section 7). +type bind struct { + t *Tunnel + + mu sync.Mutex + peers map[Key]*peerEP + open bool + closed chan struct{} + recv chan inPacket +} + +type inPacket struct { + data []byte + from *pathEP +} + +type path struct{ kind, value string } + +func (p path) String() string { return p.kind + ":" + p.value } + +var errNoPath = errors.New("no path to peer") + +func newBind(t *Tunnel) *bind { + return &bind{t: t, peers: map[Key]*peerEP{}, recv: make(chan inPacket, 1024), closed: make(chan struct{})} +} + +// deliver hands a received datagram to WireGuard. The bind owns pkt from +// here. A full queue drops the packet, as a UDP socket would. +func (b *bind) deliver(kind, from string, pkt []byte) { + select { + case b.recv <- inPacket{pkt, &pathEP{b: b, p: path{kind, from}}}: + default: + } +} + +func (b *bind) peer(k Key) *peerEP { + b.mu.Lock() + defer b.mu.Unlock() + e := b.peers[k] + if e == nil { + e = &peerEP{b: b, key: k} + b.peers[k] = e + } + return e +} + +func (b *bind) Open(port uint16) ([]conn.ReceiveFunc, uint16, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.open { + return nil, 0, conn.ErrBindAlreadyOpen + } + b.open = true + closed := make(chan struct{}) + b.closed = closed + fn := func(slab []byte, pkts []conn.ReceivedPacket) (int, error) { + select { + case p := <-b.recv: + n := copy(slab, p.data) + pkts[0] = conn.ReceivedPacket{Offset: 0, Size: n, Endpoint: p.from} + return 1, nil + case <-closed: + return 0, net.ErrClosed + } + } + return []conn.ReceiveFunc{fn}, port, nil +} + +func (b *bind) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.open { + close(b.closed) + b.open = false + } + return nil +} + +func (b *bind) SetMark(uint32) error { return nil } +func (b *bind) BatchSize() int { return 1 } + +func (b *bind) Send(bufs [][]byte, ep conn.Endpoint, offset int) error { + var err error + for _, buf := range bufs { + pkt := buf[offset:] + switch e := ep.(type) { + case *peerEP: + err = e.send(pkt) + case *pathEP: + err = b.sendPath(e.p, pkt) + default: + return conn.ErrWrongEndpointType + } + } + return err +} + +func (b *bind) sendPath(p path, pkt []byte) error { + c := b.t.carrier(p.kind) + if c == nil { + return errors.New("no carrier " + p.kind) + } + return c.send(p.value, pkt) +} + +// ParseEndpoint accepts "peer:", the form the tunnel uses +// to configure peers, and "kind:value" for a single path. +func (b *bind) ParseEndpoint(s string) (conn.Endpoint, error) { + if h, ok := strings.CutPrefix(s, "peer:"); ok { + raw, err := hex.DecodeString(h) + if err != nil || len(raw) != 32 { + return nil, errors.New("bad peer endpoint") + } + var k Key + copy(k[:], raw) + return b.peer(k), nil + } + kind, value, ok := strings.Cut(s, ":") + if !ok { + return nil, errors.New("bad endpoint " + s) + } + return &pathEP{b: b, p: path{kind, value}}, nil +} + +// maxCands bounds the candidate paths kept per peer. +const maxCands = 8 + +// peerEP is the endpoint WireGuard holds for one peer. +type peerEP struct { + b *bind + key Key + + mu sync.Mutex + cur *path // where authenticated packets come from, the preferred path + heard time.Time // when cur last delivered one + cands []path // endpoints from addresses, and paths initiations came from + init *path // where the latest initiation came from: the response goes back that way + initAt time.Time +} + +// rank orders carrier kinds by preference: direct before relayed. +func rank(kind string) int { + switch kind { + case KindUnix: + return 0 + case KindUDP: + return 1 + case KindWS: + return 2 + case KindTailcat: + return 3 + } + return 4 +} + +// stale is how long the current path may be silent before a less +// preferred one takes over. +const stale = 3 * time.Second + +// addCands adds candidate paths, newest first. +func (e *peerEP) addCands(ps ...path) { + e.mu.Lock() + defer e.mu.Unlock() + out := append([]path{}, ps...) + for _, c := range e.cands { + dup := false + for _, p := range ps { + if p == c { + dup = true + break + } + } + if !dup { + out = append(out, c) + } + } + if len(out) > maxCands { + out = out[:maxCands] + } + e.cands = out +} + +// heardFrom records an authenticated packet on p. p becomes the current +// path if it is the current one, as good a kind (a peer that moved, on the +// same carrier), or the current one has gone quiet. +func (e *peerEP) heardFrom(p path) { + now := time.Now() + e.mu.Lock() + defer e.mu.Unlock() + if e.cur == nil || *e.cur == p || rank(p.kind) <= rank(e.cur.kind) || now.Sub(e.heard) > stale { + e.cur = &p + e.heard = now + } +} + +func (e *peerEP) current() string { + e.mu.Lock() + defer e.mu.Unlock() + if e.cur == nil { + return "" + } + return e.cur.String() +} + +// initiatedFrom records the path an initiation arrived on. A peer that +// sends one initiation down several paths reaches us on each; the +// response goes back on the most preferred of them. +func (e *peerEP) initiatedFrom(p path) { + now := time.Now() + e.mu.Lock() + defer e.mu.Unlock() + if e.init == nil || rank(p.kind) <= rank(e.init.kind) || now.Sub(e.initAt) > time.Second { + e.init = &p + e.initAt = now + } +} + +// WireGuard message types, the first byte of every datagram. +const ( + msgInitiation = 1 + msgResponse = 2 +) + +func (e *peerEP) send(pkt []byte) error { + e.mu.Lock() + cur, init := e.cur, e.init + cands := append([]path{}, e.cands...) + e.mu.Unlock() + if len(pkt) > 0 && pkt[0] == msgResponse && init != nil { + if err := e.b.sendPath(*init, pkt); err == nil { + return nil + } + } + handshake := len(pkt) > 0 && pkt[0] == msgInitiation + if cur != nil { + err := e.b.sendPath(*cur, pkt) + if err == nil && !handshake { + return nil + } + if err != nil && len(cands) == 0 { + return err + } + } + sent := cur != nil + err := errNoPath + for _, c := range cands { + if cur != nil && c == *cur { + continue + } + if e2 := e.b.sendPath(c, pkt); e2 == nil { + sent = true + } else { + err = e2 + } + } + if sent { + return nil + } + return err +} + +func (e *peerEP) ClearSrc() {} +func (e *peerEP) SrcToString() string { return "" } +func (e *peerEP) DstToString() string { return "peer:" + e.key.String() } +func (e *peerEP) DstToBytes() []byte { return e.key[:] } +func (e *peerEP) DstIP() netip.Addr { return netip.Addr{} } +func (e *peerEP) SrcIP() netip.Addr { return netip.Addr{} } + +// pathEP is the endpoint of one received datagram. WireGuard tells it which +// peer the datagram was from: before verification for an initiation (the +// path becomes a candidate, so the response can reach it), after +// decryption for everything else (the path becomes current). +type pathEP struct { + b *bind + p path +} + +func (e *pathEP) InitiationMessagePublicKey(pk [32]byte) { + ep := e.b.peer(Key(pk)) + ep.addCands(e.p) + ep.initiatedFrom(e.p) +} + +func (e *pathEP) FromPeer(pk [32]byte) { e.b.peer(Key(pk)).heardFrom(e.p) } + +func (e *pathEP) ClearSrc() {} +func (e *pathEP) SrcToString() string { return "" } +func (e *pathEP) DstToString() string { return e.p.String() } +func (e *pathEP) DstToBytes() []byte { return []byte(e.p.String()) } +func (e *pathEP) DstIP() netip.Addr { return netip.Addr{} } +func (e *pathEP) SrcIP() netip.Addr { return netip.Addr{} } + +var ( + _ conn.Bind = (*bind)(nil) + _ conn.InitiationAwareEndpoint = (*pathEP)(nil) + _ conn.PeerAwareEndpoint = (*pathEP)(nil) +) diff --git a/tunnel/carrier.go b/tunnel/carrier.go new file mode 100644 index 0000000..b330560 --- /dev/null +++ b/tunnel/carrier.go @@ -0,0 +1,296 @@ +package tunnel + +import ( + "bufio" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "strings" + "sync" + "time" +) + +// carrier moves WireGuard datagrams (section 7). send must not block for +// long: WireGuard calls it from its send path. +type carrier interface { + send(to string, pkt []byte) error + close() error +} + +// pconn is one carrier connection that preserves datagram boundaries. +type pconn interface { + readPacket() ([]byte, error) + writePacket([]byte) error + Close() error +} + +// lenConn carries datagrams on a byte stream, each prefixed by its length +// as a 16-bit big-endian integer (sections 7.2 and 7.4). +type lenConn struct { + c net.Conn + r *bufio.Reader + wmu sync.Mutex +} + +func newLenConn(c net.Conn) *lenConn { return &lenConn{c: c, r: bufio.NewReaderSize(c, 64<<10)} } + +func (l *lenConn) readPacket() ([]byte, error) { + var h [2]byte + if _, err := io.ReadFull(l.r, h[:]); err != nil { + return nil, err + } + p := make([]byte, binary.BigEndian.Uint16(h[:])) + if _, err := io.ReadFull(l.r, p); err != nil { + return nil, err + } + return p, nil +} + +func (l *lenConn) writePacket(p []byte) error { + if len(p) > 0xffff { + return errors.New("datagram too large") + } + buf := make([]byte, 2+len(p)) + binary.BigEndian.PutUint16(buf, uint16(len(p))) + copy(buf[2:], p) + l.wmu.Lock() + defer l.wmu.Unlock() + l.c.SetWriteDeadline(time.Now().Add(10 * time.Second)) + _, err := l.c.Write(buf) + return err +} + +func (l *lenConn) Close() error { return l.c.Close() } + +// queued gives a pconn a bounded send queue drained by its own goroutine, +// so a slow connection (a rate-limited relay, a congested proxy) drops +// datagrams like a full UDP socket instead of blocking WireGuard. +type queued struct { + pconn + q chan []byte + done chan struct{} + once sync.Once +} + +const sendQueue = 256 + +func newQueued(pc pconn) *queued { + q := &queued{pconn: pc, q: make(chan []byte, sendQueue), done: make(chan struct{})} + go func() { + for { + select { + case p := <-q.q: + if err := q.pconn.writePacket(p); err != nil { + q.Close() + return + } + case <-q.done: + return + } + } + }() + return q +} + +func (q *queued) writePacket(p []byte) error { + select { + case <-q.done: + return net.ErrClosed + default: + } + select { + case q.q <- append([]byte(nil), p...): + default: // full: drop, as UDP would + } + return nil +} + +func (q *queued) Close() error { + q.once.Do(func() { close(q.done) }) + return q.pconn.Close() +} + +// mux is a carrier made of connections: accepted ones are addressed as +// "@n", dialed ones by their endpoint value. A send to an endpoint with no +// connection dials in the background and queues a few datagrams meanwhile; +// WireGuard retransmits anything that does not make it. +type mux struct { + b *bind + kind string + dial func(ctx context.Context, to string) (pconn, error) + // failed, if set, is told about each failed dial. + failed func(to string, err error) + + mu sync.Mutex + next int + accepted map[string]pconn + dialed map[string]*slot + closers []io.Closer + done bool +} + +type slot struct { + c pconn + dialing bool + queue [][]byte +} + +const maxQueued = 16 + +func newMux(b *bind, kind string, dial func(context.Context, string) (pconn, error)) *mux { + return &mux{b: b, kind: kind, dial: dial, accepted: map[string]pconn{}, dialed: map[string]*slot{}} +} + +// serveListener accepts connections until ln closes. +func (m *mux) serveListener(ln net.Listener, wrap func(net.Conn) pconn) { + m.addCloser(ln) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + m.accept(wrap(c)) + } + }() +} + +func (m *mux) addCloser(c io.Closer) { + m.mu.Lock() + defer m.mu.Unlock() + if m.done { + c.Close() + return + } + m.closers = append(m.closers, c) +} + +func (m *mux) accept(pc pconn) { + m.mu.Lock() + if m.done { + m.mu.Unlock() + pc.Close() + return + } + m.next++ + id := fmt.Sprintf("@%d", m.next) + pc = newQueued(pc) + m.accepted[id] = pc + m.mu.Unlock() + go m.read(pc, id, func() { + m.mu.Lock() + delete(m.accepted, id) + m.mu.Unlock() + }) +} + +func (m *mux) read(pc pconn, from string, end func()) { + defer end() + defer pc.Close() + for { + p, err := pc.readPacket() + if err != nil { + return + } + m.b.deliver(m.kind, from, p) + } +} + +func (m *mux) send(to string, pkt []byte) error { + if strings.HasPrefix(to, "@") { + m.mu.Lock() + pc := m.accepted[to] + m.mu.Unlock() + if pc == nil { + return errors.New(m.kind + ": connection gone") + } + return pc.writePacket(pkt) + } + m.mu.Lock() + if m.done { + m.mu.Unlock() + return net.ErrClosed + } + s := m.dialed[to] + if s == nil { + s = &slot{} + m.dialed[to] = s + } + if pc := s.c; pc != nil { + m.mu.Unlock() + if err := pc.writePacket(pkt); err == nil { + return nil + } + pc.Close() + m.mu.Lock() + if s.c == pc { + s.c = nil + } + } + if len(s.queue) < maxQueued { + s.queue = append(s.queue, append([]byte(nil), pkt...)) + } + if !s.dialing { + s.dialing = true + go m.dialSlot(to, s) + } + m.mu.Unlock() + return nil +} + +func (m *mux) dialSlot(to string, s *slot) { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + pc, err := m.dial(ctx, to) + cancel() + m.mu.Lock() + s.dialing = false + if err != nil || m.done { + s.queue = nil + m.mu.Unlock() + if pc != nil { + pc.Close() + } + if err != nil && m.failed != nil { + m.failed(to, err) + } + return + } + pc = newQueued(pc) + s.c = pc + q := s.queue + s.queue = nil + m.mu.Unlock() + for _, p := range q { + pc.writePacket(p) + } + go m.read(pc, to, func() { + m.mu.Lock() + if s.c == pc { + s.c = nil + } + m.mu.Unlock() + }) +} + +func (m *mux) close() error { + m.mu.Lock() + m.done = true + var cs []io.Closer + cs = append(cs, m.closers...) + for _, pc := range m.accepted { + cs = append(cs, pc) + } + for _, s := range m.dialed { + if s.c != nil { + cs = append(cs, s.c) + } + } + m.mu.Unlock() + for _, c := range cs { + c.Close() + } + return nil +} diff --git a/tunnel/carrier_stream.go b/tunnel/carrier_stream.go new file mode 100644 index 0000000..26e9ecd --- /dev/null +++ b/tunnel/carrier_stream.go @@ -0,0 +1,381 @@ +package tunnel + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + "github.com/tailscale/tailcat" + "tailscale.com/types/logger" +) + +// --- unix (section 7.4) --- + +func newUnix(b *bind) *mux { + return newMux(b, KindUnix, func(ctx context.Context, to string) (pconn, error) { + var d net.Dialer + c, err := d.DialContext(ctx, "unix", to) + if err != nil { + return nil, err + } + return newLenConn(c), nil + }) +} + +// listenUnix listens on a socket path, removing a stale socket first. +func listenUnix(m *mux, p string) ([]string, error) { + if fi, err := os.Stat(p); err == nil && fi.Mode()&os.ModeSocket != 0 { + if c, err := net.DialTimeout("unix", p, time.Second); err == nil { + c.Close() + return nil, fmt.Errorf("%s: already in use", p) + } + os.Remove(p) + } + ln, err := net.Listen("unix", p) + if err != nil { + return nil, err + } + m.serveListener(ln, func(c net.Conn) pconn { return newLenConn(c) }) + abs, err := filepath.Abs(p) + if err != nil { + abs = p + } + return []string{abs}, nil +} + +// --- tailcat (section 7.2) --- + +// tailcatPort is the port inside the tailcat tunnel that carries WireGuard +// datagrams, length-prefixed like the unix carrier. +const tailcatPort = 1 + +type tailcatCarrier struct { + *mux + logf logger.Logf + + mu sync.Mutex + clients map[string]*tailcat.Client + failures map[string]int + servers []*tailcat.Server +} + +func newTailcat(b *bind, logf logger.Logf) *tailcatCarrier { + if logf == nil { + logf = logger.Discard + } + t := &tailcatCarrier{logf: logf, clients: map[string]*tailcat.Client{}, failures: map[string]int{}} + t.mux = newMux(b, KindTailcat, func(ctx context.Context, to string) (pconn, error) { + c, err := t.client(to).DialTCPPort(ctx, tailcatPort) + if err != nil { + return nil, fmt.Errorf("tailcat dial: %w", err) + } + t.mu.Lock() + delete(t.failures, to) + t.mu.Unlock() + return newLenConn(c), nil + }) + t.mux.failed = func(to string, err error) { + t.mu.Lock() + t.failures[to]++ + reset := t.failures[to]%3 == 0 + t.mu.Unlock() + if reset { + t.reset(to) // the next dial starts from a fresh tailcat node + } + } + return t +} + +func (t *tailcatCarrier) client(addr string) *tailcat.Client { + t.mu.Lock() + defer t.mu.Unlock() + c := t.clients[addr] + if c == nil { + c = &tailcat.Client{Server: tailcat.Addr(addr), Logf: t.logf} + t.clients[addr] = c + } + return c +} + +func (t *tailcatCarrier) reset(addr string) { + t.mu.Lock() + c := t.clients[addr] + delete(t.clients, addr) + t.mu.Unlock() + if c != nil { + go c.Close() + } +} + +// listen starts a tailcat server whose key, pre-shared key and DERP region +// persist in keyFile (empty: a fresh one each time), so the endpoint +// survives restarts. +func (t *tailcatCarrier) listen(ctx context.Context, keyFile string) ([]string, error) { + pk, err := loadOrCreateTailcatKey(keyFile) + if err != nil { + return nil, err + } + if pk.Public.RegionID <= 0 { + // Pick the nearest DERP region once and remember it, like + // `tailcat genkey --fixed-region`. + dm, err := tailcat.FetchDERPMap(ctx, tailcat.ExpandForServer) + if err != nil { + return nil, fmt.Errorf("fetching DERP map: %w", err) + } + id, err := tailcat.PickBestRegion(ctx, dm) + if err != nil { + return nil, fmt.Errorf("picking DERP region: %w", err) + } + if id == 0 { + return nil, errors.New("no reachable DERP region") + } + pk.Public.RegionID = id + if err := saveTailcatKey(keyFile, pk); err != nil { + return nil, err + } + } + srv := &tailcat.Server{Key: pk.Private, PresharedKey: pk.Public.PresharedKey, RegionID: pk.Public.RegionID, Logf: t.logf} + ln, err := srv.Listen(ctx, "tcp", fmt.Sprintf(":%d", tailcatPort)) + if err != nil { + srv.Close() + if strings.Contains(err.Error(), "no such region") { + // The DERP map dropped our region: pick a new one next time. + pk.Public.RegionID = 0 + saveTailcatKey(keyFile, pk) + } + return nil, err + } + t.mu.Lock() + t.servers = append(t.servers, srv) + t.mu.Unlock() + t.serveListener(ln, func(c net.Conn) pconn { return newLenConn(c) }) + return []string{string(srv.TailcatAddr())}, nil +} + +func (t *tailcatCarrier) close() error { + t.mux.close() + t.mu.Lock() + cs, ss := t.clients, t.servers + t.clients, t.servers = map[string]*tailcat.Client{}, nil + t.mu.Unlock() + for _, c := range cs { + c.Close() + } + for _, s := range ss { + s.Close() + } + return nil +} + +func loadOrCreateTailcatKey(path string) (*tailcat.PrivateKey, error) { + if path == "" { + return tailcat.NewPrivateKey(), nil + } + b, err := os.ReadFile(path) + if err == nil { + var pk tailcat.PrivateKey + if err := json.Unmarshal(b, &pk); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if pk.Private.IsZero() || pk.Public.PresharedKey.IsZero() { + return nil, fmt.Errorf("%s: incomplete tailcat key", path) + } + return &pk, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + pk := tailcat.NewPrivateKey() + return pk, saveTailcatKey(path, pk) +} + +func saveTailcatKey(path string, pk *tailcat.PrivateKey) error { + if path == "" { + return nil + } + b, err := json.MarshalIndent(pk, "", "\t") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// --- ws (section 7.3) --- + +// WSSubprotocol is the WebSocket subprotocol of the ws carrier. +const WSSubprotocol = "awp.wg.1" + +type wsConn struct{ c *websocket.Conn } + +func (w wsConn) readPacket() ([]byte, error) { + for { + typ, p, err := w.c.Read(context.Background()) + if err != nil { + return nil, err + } + if typ == websocket.MessageBinary { + return p, nil + } + } +} + +func (w wsConn) writePacket(p []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return w.c.Write(ctx, websocket.MessageBinary, p) +} + +func (w wsConn) Close() error { return w.c.Close(websocket.StatusNormalClosure, "") } + +type wsCarrier struct { + *mux + mu sync.Mutex + procs []*exec.Cmd +} + +func newWS(b *bind) *wsCarrier { + return &wsCarrier{mux: newMux(b, KindWS, func(ctx context.Context, to string) (pconn, error) { + c, _, err := websocket.Dial(ctx, to, &websocket.DialOptions{Subprotocols: []string{WSSubprotocol}}) + if err != nil { + return nil, err + } + c.SetReadLimit(1 << 16) + return wsConn{c}, nil + })} +} + +// listen serves WebSocket upgrades on hostport, at any path. The endpoint +// is public if given (the URL a tunnel provider forwards to hostport), +// otherwise ws://hostport/awp. +func (w *wsCarrier) listen(hostport, public string) ([]string, error) { + ln, err := net.Listen("tcp", hostport) + if err != nil { + return nil, err + } + srv := &http.Server{Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + c, err := websocket.Accept(rw, r, &websocket.AcceptOptions{ + Subprotocols: []string{WSSubprotocol}, + // Origin checks protect cookies; the payload is WireGuard and + // carries its own authentication, and browsers are welcome. + InsecureSkipVerify: true, + }) + if err != nil { + return + } + if c.Subprotocol() != WSSubprotocol { + c.Close(websocket.StatusPolicyViolation, "want subprotocol "+WSSubprotocol) + return + } + c.SetReadLimit(1 << 16) + cn := &closeNotify{wsConn: wsConn{c}, done: make(chan struct{})} + w.accept(cn) + <-cn.done + }), ReadHeaderTimeout: 10 * time.Second} + w.addCloser(srv) + go srv.Serve(ln) + if public == "" { + public = "ws://" + ln.Addr().String() + "/awp" + } + return []string{public}, nil +} + +// closeNotify keeps an HTTP handler alive for as long as its WebSocket is. +type closeNotify struct { + wsConn + once sync.Once + done chan struct{} +} + +func (c *closeNotify) Close() error { + err := c.wsConn.Close() + c.once.Do(func() { close(c.done) }) + return err +} + +var quickTunnelURL = regexp.MustCompile(`https://[a-z0-9-]+\.trycloudflare\.com`) + +// listenCloudflare serves WebSockets on a loopback port and exposes it +// through a Cloudflare quick tunnel: `cloudflared tunnel --url`, no account. +// The endpoint is the wss:// URL it prints. +func (w *wsCarrier) listenCloudflare(ctx context.Context, cloudflared string) ([]string, error) { + if cloudflared == "" { + cloudflared = "cloudflared" + } + bin, err := exec.LookPath(cloudflared) + if err != nil { + return nil, fmt.Errorf("cloudflare carrier needs cloudflared on PATH: %w", err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + local := ln.Addr().String() + ln.Close() + if _, err := w.listen(local, "unused"); err != nil { + return nil, err + } + cmd := exec.Command(bin, "tunnel", "--no-autoupdate", "--url", "http://"+local) + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + w.mu.Lock() + w.procs = append(w.procs, cmd) + w.mu.Unlock() + found := make(chan string, 1) + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + if u := quickTunnelURL.FindString(sc.Text()); u != "" { + select { + case found <- u: + default: + } + } + } + }() + go cmd.Wait() + select { + case u := <-found: + return []string{"wss://" + strings.TrimPrefix(u, "https://") + "/awp"}, nil + case <-ctx.Done(): + cmd.Process.Kill() + return nil, fmt.Errorf("cloudflared printed no quick tunnel URL: %w", ctx.Err()) + } +} + +func (w *wsCarrier) close() error { + w.mux.close() + w.mu.Lock() + ps := w.procs + w.procs = nil + w.mu.Unlock() + for _, p := range ps { + if p.Process != nil { + p.Process.Kill() + } + } + return nil +} diff --git a/tunnel/carrier_udp.go b/tunnel/carrier_udp.go new file mode 100644 index 0000000..5bf4d26 --- /dev/null +++ b/tunnel/carrier_udp.go @@ -0,0 +1,179 @@ +package tunnel + +import ( + "context" + "errors" + "net" + "net/netip" + "strconv" + "sync" + "time" +) + +// udpCarrier is WireGuard's native carrier (section 7.1). It has one socket +// per listen address, plus an ephemeral one for dialing when it listens on +// none. +type udpCarrier struct { + b *bind + + mu sync.Mutex + socks []*net.UDPConn + dialer *net.UDPConn + resolve map[string]netip.AddrPort + done bool +} + +func newUDP(b *bind) *udpCarrier { + return &udpCarrier{b: b, resolve: map[string]netip.AddrPort{}} +} + +// listen binds hostport and returns the endpoints to advertise. An +// unspecified host advertises every global and private unicast address of +// this machine. +func (u *udpCarrier) listen(hostport string) ([]string, error) { + host, port, err := net.SplitHostPort(hostport) + if err != nil { + return nil, err + } + c, err := net.ListenUDP("udp", mustUDPAddr(hostport)) + if err != nil { + return nil, err + } + u.mu.Lock() + if u.done { + u.mu.Unlock() + c.Close() + return nil, net.ErrClosed + } + u.socks = append(u.socks, c) + u.mu.Unlock() + go u.read(c) + if port == "0" { + port = strconv.Itoa(c.LocalAddr().(*net.UDPAddr).Port) + } + if ip, err := netip.ParseAddr(host); err == nil && !ip.IsUnspecified() { + return []string{net.JoinHostPort(host, port)}, nil + } + if host != "" && net.ParseIP(host) == nil { + return []string{net.JoinHostPort(host, port)}, nil // a name + } + return localEndpoints(port), nil +} + +func mustUDPAddr(hostport string) *net.UDPAddr { + a, err := net.ResolveUDPAddr("udp", hostport) + if err != nil { + return &net.UDPAddr{} + } + return a +} + +func localEndpoints(port string) []string { + var out []string + addrs, _ := net.InterfaceAddrs() + for _, a := range addrs { + pfx, err := netip.ParsePrefix(a.String()) + if err != nil { + continue + } + ip := pfx.Addr() + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsMulticast() { + continue + } + out = append(out, net.JoinHostPort(ip.String(), port)) + } + if len(out) == 0 { + out = []string{net.JoinHostPort("127.0.0.1", port)} + } + return out +} + +func (u *udpCarrier) read(c *net.UDPConn) { + buf := make([]byte, 65536) + for { + n, from, err := c.ReadFromUDPAddrPort(buf) + if err != nil { + if errors.Is(err, net.ErrClosed) { + return + } + continue + } + from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port()) + u.b.deliver(KindUDP, from.String(), append([]byte(nil), buf[:n]...)) + } +} + +func (u *udpCarrier) sock() (*net.UDPConn, error) { + u.mu.Lock() + defer u.mu.Unlock() + if u.done { + return nil, net.ErrClosed + } + if len(u.socks) > 0 { + return u.socks[0], nil + } + if u.dialer == nil { + c, err := net.ListenUDP("udp", nil) + if err != nil { + return nil, err + } + u.dialer = c + go u.read(c) + } + return u.dialer, nil +} + +func (u *udpCarrier) addr(to string) (netip.AddrPort, error) { + if ap, err := netip.ParseAddrPort(to); err == nil { + return ap, nil + } + u.mu.Lock() + ap, ok := u.resolve[to] + u.mu.Unlock() + if ok { + return ap, nil + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + host, port, err := net.SplitHostPort(to) + if err != nil { + return netip.AddrPort{}, err + } + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil || len(ips) == 0 { + return netip.AddrPort{}, errors.New("udp: cannot resolve " + host) + } + p, _ := strconv.Atoi(port) + ap = netip.AddrPortFrom(ips[0].Unmap(), uint16(p)) + u.mu.Lock() + u.resolve[to] = ap + u.mu.Unlock() + return ap, nil +} + +func (u *udpCarrier) send(to string, pkt []byte) error { + ap, err := u.addr(to) + if err != nil { + return err + } + c, err := u.sock() + if err != nil { + return err + } + _, err = c.WriteToUDPAddrPort(pkt, ap) + return err +} + +func (u *udpCarrier) close() error { + u.mu.Lock() + u.done = true + socks := append([]*net.UDPConn{}, u.socks...) + if u.dialer != nil { + socks = append(socks, u.dialer) + } + u.mu.Unlock() + for _, c := range socks { + c.Close() + } + return nil +} diff --git a/tunnel/keys.go b/tunnel/keys.go new file mode 100644 index 0000000..fb44b7c --- /dev/null +++ b/tunnel/keys.go @@ -0,0 +1,76 @@ +// Package tunnel is AWP's connection layer (SPEC.md sections 4 to 7): a +// WireGuard tunnel between two peers' identity keys, carried by any mix of +// carriers (udp, tailcat, ws, unix), with a TCP stream on port 1 inside it. +// +// A peer's Ed25519 identity key, converted to X25519, is its WireGuard +// static key, so the tunnel's authentication is the protocol's: a Stream's +// remote key is the key that completed the WireGuard handshake. +package tunnel + +import ( + "crypto/ed25519" + "crypto/sha256" + "crypto/sha512" + "encoding/base64" + "errors" + "net/netip" + + "filippo.io/edwards25519" +) + +// Port is the TCP port AWP listens on inside the tunnel. +const Port = 1 + +// MTU is the tunnel's inner MTU: the IPv6 minimum, so nothing inside ever +// needs to know about the path. +const MTU = 1280 + +// Key is a WireGuard (X25519) public key. +type Key [32]byte + +func (k Key) String() string { return base64.RawURLEncoding.EncodeToString(k[:]) } + +// X25519Private returns the WireGuard private key for an Ed25519 identity: +// the first half of SHA-512 of the seed, clamped (section 4.2). It is the +// scalar Ed25519 itself signs with. +func X25519Private(priv ed25519.PrivateKey) [32]byte { + h := sha512.Sum512(priv.Seed()) + var k [32]byte + copy(k[:], h[:32]) + k[0] &= 248 + k[31] &= 127 + k[31] |= 64 + return k +} + +// X25519Public converts an Ed25519 public key to its WireGuard public key, +// the Montgomery form of the same point (RFC 7748 section 4.1). +// +// Two Ed25519 keys that differ only in the sign bit map to the same X25519 +// key. Whoever holds the private key of one holds the other's (its +// negation), so this is not a way to claim someone else's identity; peers +// compare identities by the full Ed25519 key. +func X25519Public(pub ed25519.PublicKey) (Key, error) { + if len(pub) != ed25519.PublicKeySize { + return Key{}, errors.New("tunnel: bad Ed25519 public key length") + } + p, err := new(edwards25519.Point).SetBytes(pub) + if err != nil { + return Key{}, err + } + var k Key + copy(k[:], p.BytesMontgomery()) + return k, nil +} + +// IP is a tunnel key's IPv6 address inside the tunnel (section 4.3): 0xfd +// followed by the first 15 bytes of SHA-256("awp-ula-v1" || key). +func IP(k Key) netip.Addr { + h := sha256.New() + h.Write([]byte("awp-ula-v1")) + h.Write(k[:]) + var a [16]byte + a[0] = 0xfd + copy(a[1:], h.Sum(nil)[:15]) + return netip.AddrFrom16(a) +} diff --git a/tunnel/netstack.go b/tunnel/netstack.go new file mode 100644 index 0000000..4b3cb90 --- /dev/null +++ b/tunnel/netstack.go @@ -0,0 +1,156 @@ +package tunnel + +// A userspace TCP/IP stack as a WireGuard TUN device, adapted from +// wireguard-go's tun/netstack (MIT licensed, Copyright (C) 2017-2023 +// WireGuard LLC) for the gVisor version this module pins, and trimmed to +// what the tunnel needs: one IPv6 address, TCP. + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "syscall" + "time" + + "github.com/tailscale/wireguard-go/tun" + "gvisor.dev/gvisor/pkg/buffer" + glog "gvisor.dev/gvisor/pkg/log" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv6" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/icmp" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" +) + +// gVisor logs warnings (segments for connections already gone, and the +// like) straight to stderr. They are noise for a peer's user. +func init() { glog.SetTarget(discard{}) } + +type discard struct{} + +func (discard) Emit(int, glog.Level, time.Time, string, ...any) {} + +type netTun struct { + ep *channel.Endpoint + stack *stack.Stack + events chan tun.Event + incoming chan *buffer.View + closed chan struct{} + mtu int +} + +func newNetTun(addr netip.Addr, mtu int) (*netTun, error) { + t := &netTun{ + ep: channel.New(1024, uint32(mtu), ""), + stack: stack.New(stack.Options{ + NetworkProtocols: []stack.NetworkProtocolFactory{ipv6.NewProtocol}, + TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, icmp.NewProtocol6}, + HandleLocal: true, + }), + events: make(chan tun.Event, 10), + incoming: make(chan *buffer.View), + closed: make(chan struct{}), + mtu: mtu, + } + sack := tcpip.TCPSACKEnabled(true) + if err := t.stack.SetTransportProtocolOption(tcp.ProtocolNumber, &sack); err != nil { + return nil, fmt.Errorf("enable TCP SACK: %v", err) + } + t.ep.AddNotify(t) + if err := t.stack.CreateNIC(1, t.ep); err != nil { + return nil, fmt.Errorf("CreateNIC: %v", err) + } + pa := tcpip.ProtocolAddress{ + Protocol: ipv6.ProtocolNumber, + AddressWithPrefix: tcpip.AddrFromSlice(addr.AsSlice()).WithPrefix(), + } + if err := t.stack.AddProtocolAddress(1, pa, stack.AddressProperties{}); err != nil { + return nil, fmt.Errorf("AddProtocolAddress(%v): %v", addr, err) + } + t.stack.AddRoute(tcpip.Route{Destination: header.IPv6EmptySubnet, NIC: 1}) + t.events <- tun.EventUp + return t, nil +} + +func (t *netTun) Name() (string, error) { return "awp", nil } +func (t *netTun) File() *os.File { return nil } +func (t *netTun) Events() <-chan tun.Event { return t.events } +func (t *netTun) MTU() (int, error) { return t.mtu, nil } +func (t *netTun) BatchSize() int { return 1 } + +func (t *netTun) Read(slab []byte, packets []tun.ReadPacket) (int, error) { + var view *buffer.View + select { + case view = <-t.incoming: + case <-t.closed: + return 0, os.ErrClosed + } + n, err := view.Read(slab[tun.ReadPacketSpacing : len(slab)-tun.ReadPacketSpacing]) + if err != nil { + return 0, err + } + packets[0].Size = n + packets[0].Offset = tun.ReadPacketSpacing + return 1, nil +} + +func (t *netTun) Write(bufs [][]byte, offset int) (int, error) { + for _, b := range bufs { + p := b[offset:] + if len(p) == 0 { + continue + } + if p[0]>>4 != 6 { + return 0, syscall.EAFNOSUPPORT + } + pkb := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(p)}) + t.ep.InjectInbound(header.IPv6ProtocolNumber, pkb) + } + return len(bufs), nil +} + +// WriteNotify is called by the channel endpoint when the stack has a packet +// to send. +func (t *netTun) WriteNotify() { + pkt := t.ep.Read() + if pkt == nil { + return + } + view := pkt.ToView() + pkt.DecRef() + select { + case t.incoming <- view: + case <-t.closed: + } +} + +func (t *netTun) Close() error { + select { + case <-t.closed: + return nil + default: + } + close(t.closed) + t.stack.RemoveNIC(1) + t.stack.Close() + close(t.events) + t.ep.Close() + return nil +} + +func fullAddr(ap netip.AddrPort) tcpip.FullAddress { + return tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFromSlice(ap.Addr().AsSlice()), Port: ap.Port()} +} + +func (t *netTun) dialTCP(ctx context.Context, ap netip.AddrPort) (*gonet.TCPConn, error) { + return gonet.DialContextTCP(ctx, t.stack, fullAddr(ap), ipv6.ProtocolNumber) +} + +func (t *netTun) listenTCP(ap netip.AddrPort) (net.Listener, error) { + return gonet.ListenTCP(t.stack, fullAddr(ap), ipv6.ProtocolNumber) +} diff --git a/tunnel/state.go b/tunnel/state.go new file mode 100644 index 0000000..4d13715 --- /dev/null +++ b/tunnel/state.go @@ -0,0 +1,119 @@ +package tunnel + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" +) + +// state is what the tunnel persists: the listener's pre-shared key, and the +// pre-shared key agreed with each peer (section 5.2). It lives in +// tunnel.json in the state directory, or only in memory. +type state struct { + path string + + mu sync.Mutex + PSK string `json:"psk"` + Pairs map[string]string `json:"pairs"` // tunnel key (base64url) -> pre-shared key +} + +func loadState(dir string) (*state, error) { + s := &state{Pairs: map[string]string{}} + if dir != "" { + s.path = filepath.Join(dir, "tunnel.json") + b, err := os.ReadFile(s.path) + switch { + case err == nil: + if err := json.Unmarshal(b, s); err != nil { + return nil, fmt.Errorf("%s: %w", s.path, err) + } + if s.Pairs == nil { + s.Pairs = map[string]string{} + } + case !errors.Is(err, os.ErrNotExist): + return nil, err + } + } + if _, err := decode32(s.PSK); err != nil { + s.PSK = random32() + if err := s.saveLocked(); err != nil { + return nil, err + } + } + return s, nil +} + +func random32() string { + var b [32]byte + rand.Read(b[:]) + return base64.RawURLEncoding.EncodeToString(b[:]) +} + +func decode32(s string) ([32]byte, error) { + var out [32]byte + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil || len(b) != 32 { + return out, errors.New("want 32 bytes, base64url") + } + copy(out[:], b) + return out, nil +} + +func (s *state) saveLocked() error { + if s.path == "" { + return nil + } + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil { + return err + } + tmp := s.path + ".tmp" + if err := os.WriteFile(tmp, append(b, '\n'), 0o600); err != nil { + return err + } + return os.Rename(tmp, s.path) +} + +func (s *state) listenerPSK() [32]byte { + s.mu.Lock() + defer s.mu.Unlock() + k, _ := decode32(s.PSK) + return k +} + +func (s *state) rotate() error { + s.mu.Lock() + defer s.mu.Unlock() + s.PSK = random32() + return s.saveLocked() +} + +func (s *state) pair(k Key) ([32]byte, bool) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.Pairs[k.String()] + if !ok { + return [32]byte{}, false + } + p, err := decode32(v) + return p, err == nil +} + +func (s *state) setPair(k Key, psk [32]byte) { + s.mu.Lock() + defer s.mu.Unlock() + v := base64.RawURLEncoding.EncodeToString(psk[:]) + if s.Pairs[k.String()] == v { + return + } + s.Pairs[k.String()] = v + s.saveLocked() +} diff --git a/tunnel/tunnel.go b/tunnel/tunnel.go new file mode 100644 index 0000000..3333552 --- /dev/null +++ b/tunnel/tunnel.go @@ -0,0 +1,541 @@ +package tunnel + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "errors" + "fmt" + "net" + "net/netip" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/tailscale/wireguard-go/device" + "tailscale.com/types/logger" +) + +// Config configures a Tunnel. +type Config struct { + // Identity is the peer's Ed25519 key; its X25519 form is the WireGuard key. + Identity ed25519.PrivateKey + + // StateDir holds tunnel.json (pre-shared keys) and tailcat.json (the + // tailcat carrier's key). Empty keeps everything in memory. + StateDir string + + // Listen lists carriers to accept on: + // + // tailcat tailcat, the default for anything behind NAT + // udp:HOST:PORT plain UDP; an empty or unspecified host advertises every local address + // unix:/path a local socket + // ws:HOST:PORT WebSockets, advertised as ws://HOST:PORT/awp + // ws:HOST:PORT=URL WebSockets behind a tunnel or proxy, advertised as URL + // cloudflare WebSockets through a Cloudflare quick tunnel (needs cloudflared) + Listen []string + + // Admit decides whether an unknown key may start a handshake. Nil admits + // every key that holds the listener's pre-shared key (section 5.2). + Admit func(Key) bool + + // Changed is called when the address changes (a carrier came up). + Changed func() + + Logf func(format string, args ...any) // the tunnel's own log + TailcatLogf logger.Logf // tailcat's diagnostics + Cloudflared string // the cloudflared binary; default "cloudflared" on PATH + Verbose bool // WireGuard's debug log +} + +// Tunnel is one peer's WireGuard device, its carriers, and the TCP stack +// inside. +type Tunnel struct { + cfg Config + pub Key + ip netip.Addr + st *state + dev *device.Device + tun *netTun + bind *bind + ln net.Listener + + udp *udpCarrier + unix *mux + tailcat *tailcatCarrier + ws *wsCarrier + + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + endpoints map[int][]Endpoint // by index in cfg.Listen + errs map[string]string // listen spec -> last error + keys map[netip.Addr]Key + admitted map[Key][32]byte // psk each lookup-admitted peer was given + closed bool +} + +// New creates the device and the stack. Start begins listening. +func New(cfg Config) (*Tunnel, error) { + if len(cfg.Identity) != ed25519.PrivateKeySize { + return nil, errors.New("tunnel: no identity") + } + if cfg.Logf == nil { + cfg.Logf = func(string, ...any) {} + } + st, err := loadState(cfg.StateDir) + if err != nil { + return nil, err + } + pub, err := X25519Public(cfg.Identity.Public().(ed25519.PublicKey)) + if err != nil { + return nil, err + } + t := &Tunnel{cfg: cfg, pub: pub, ip: IP(pub), st: st, endpoints: map[int][]Endpoint{}, errs: map[string]string{}, keys: map[netip.Addr]Key{}, admitted: map[Key][32]byte{}} + t.ctx, t.cancel = context.WithCancel(context.Background()) + t.bind = newBind(t) + t.udp = newUDP(t.bind) + t.unix = newUnix(t.bind) + t.tailcat = newTailcat(t.bind, cfg.TailcatLogf) + t.ws = newWS(t.bind) + + t.tun, err = newNetTun(t.ip, MTU) + if err != nil { + return nil, err + } + lg := device.NewLogger(device.LogLevelSilent, "") + if cfg.Verbose { + lg = &device.Logger{Verbosef: func(f string, a ...any) { cfg.Logf("wg: "+f, a...) }, Errorf: func(f string, a ...any) { cfg.Logf("wg: "+f, a...) }} + } + t.dev = device.NewDevice(t.tun, t.bind, lg) + if err := t.dev.SetPrivateKey(device.NoisePrivateKey(X25519Private(cfg.Identity))); err != nil { + t.dev.Close() + return nil, err + } + t.dev.SetPeerLookupFunc(t.lookup) + if err := t.dev.Up(); err != nil { + t.dev.Close() + return nil, err + } + t.ln, err = t.tun.listenTCP(netip.AddrPortFrom(t.ip, Port)) + if err != nil { + t.dev.Close() + return nil, err + } + return t, nil +} + +func inUse(err error) bool { + return errors.Is(err, syscall.EADDRINUSE) || strings.Contains(err.Error(), "already in use") +} + +// Key is this tunnel's WireGuard public key. +func (t *Tunnel) Key() Key { return t.pub } + +func (t *Tunnel) carrier(kind string) carrier { + switch kind { + case KindUDP: + return t.udp + case KindUnix: + return t.unix + case KindTailcat: + return t.tailcat + case KindWS: + return t.ws + } + return nil +} + +// lookup admits a handshake from a key WireGuard does not know yet. +func (t *Tunnel) lookup(pk device.NoisePublicKey) (*device.NewPeerConfig, bool) { + k := Key(pk) + psk, known := t.st.pair(k) + if !known { + if !t.listening() { + return nil, false // only peers we have met may reach a dialer + } + if t.cfg.Admit != nil && !t.cfg.Admit(k) { + return nil, false + } + psk = t.st.listenerPSK() + } + t.noteKey(k) + t.mu.Lock() + t.admitted[k] = psk + t.mu.Unlock() + return &device.NewPeerConfig{ + AllowedIPs: []netip.Prefix{netip.PrefixFrom(IP(k), 128)}, + PresharedKey: device.NoisePresharedKey(psk), + Endpoint: t.bind.peer(k), + }, true +} + +func (t *Tunnel) noteKey(k Key) { + t.mu.Lock() + t.keys[IP(k)] = k + t.mu.Unlock() +} + +func (t *Tunnel) listening() bool { + t.mu.Lock() + defer t.mu.Unlock() + for _, es := range t.endpoints { + if len(es) > 0 { + return true + } + } + return false +} + +// Start opens every carrier in cfg.Listen. tailcat and cloudflare come up in +// the background, since they take seconds; Changed is called when they do. +// Start fails only if a local carrier (udp, unix, ws) cannot listen. +func (t *Tunnel) Start() error { + t.mu.Lock() + specs := t.cfg.Listen + t.cfg.Listen = nil + t.mu.Unlock() + for _, spec := range specs { + if err := t.Listen(spec); err != nil { + return err + } + } + return nil +} + +// Listen adds a carrier to listen on, as in Config.Listen. A local carrier +// is up when Listen returns; tailcat and cloudflare come up in the +// background (Pending says how they are doing). Listening on a carrier +// already listened on does nothing. +func (t *Tunnel) Listen(spec string) error { + t.mu.Lock() + for _, s := range t.cfg.Listen { + if s == spec { + t.mu.Unlock() + return nil + } + } + i := len(t.cfg.Listen) + t.cfg.Listen = append(t.cfg.Listen, spec) + t.mu.Unlock() + kind, _, _ := strings.Cut(spec, ":") + switch kind { + case KindTailcat, "cloudflare": + go t.listenRetry(i, spec) + return nil + } + // A restarted peer may find its socket or port still held for a moment + // by the process it replaces: try for a few seconds. + es, err := t.listenOne(t.ctx, spec) + for deadline := time.Now().Add(3 * time.Second); err != nil && inUse(err) && time.Now().Before(deadline); { + time.Sleep(100 * time.Millisecond) + es, err = t.listenOne(t.ctx, spec) + } + if err != nil { + t.mu.Lock() + t.errs[spec] = err.Error() + t.mu.Unlock() + return fmt.Errorf("listen %s: %w", spec, err) + } + t.setEndpoints(i, es) + return nil +} + +func (t *Tunnel) listenOne(ctx context.Context, spec string) ([]Endpoint, error) { + kind, arg, _ := strings.Cut(spec, ":") + var vals []string + var err error + switch kind { + case KindUDP: + vals, err = t.udp.listen(arg) + case KindUnix: + if arg == "" { + return nil, errors.New("unix: empty path") + } + vals, err = listenUnix(t.unix, arg) + case KindWS: + hostport, public, _ := strings.Cut(arg, "=") + vals, err = t.ws.listen(hostport, public) + kind = KindWS + case KindTailcat: + keyFile := "" + if t.cfg.StateDir != "" { + keyFile = filepath.Join(t.cfg.StateDir, "tailcat.json") + } + vals, err = t.tailcat.listen(ctx, keyFile) + case "cloudflare": + vals, err = t.ws.listenCloudflare(ctx, t.cfg.Cloudflared) + kind = KindWS + default: + return nil, fmt.Errorf("unknown carrier %q (want tailcat, udp:HOST:PORT, unix:PATH, ws:HOST:PORT or cloudflare)", kind) + } + if err != nil { + return nil, err + } + var out []Endpoint + for _, v := range vals { + out = append(out, Endpoint{Kind: kind, Value: v}) + } + return out, nil +} + +func (t *Tunnel) listenRetry(i int, spec string) { + backoff := 5 * time.Second + for { + ctx, cancel := context.WithTimeout(t.ctx, 60*time.Second) + es, err := t.listenOne(ctx, spec) + cancel() + if err == nil { + t.setEndpoints(i, es) + return + } + t.mu.Lock() + t.errs[spec] = err.Error() + t.mu.Unlock() + t.cfg.Logf("%s listener: %v (retrying in %v)", spec, err, backoff) + select { + case <-t.ctx.Done(): + return + case <-time.After(backoff): + } + backoff = min(backoff*2, 5*time.Minute) + } +} + +func (t *Tunnel) setEndpoints(i int, es []Endpoint) { + t.mu.Lock() + t.endpoints[i] = es + delete(t.errs, t.cfg.Listen[i]) + t.mu.Unlock() + for _, e := range es { + t.cfg.Logf("listening on %s", e) + } + if t.cfg.Changed != nil { + t.cfg.Changed() + } +} + +// Address is this peer's address: key, the listener's pre-shared key, and +// every endpoint it listens on, in the order of Config.Listen. +func (t *Tunnel) Address() Address { + t.mu.Lock() + defer t.mu.Unlock() + a := Address{Key: t.cfg.Identity.Public().(ed25519.PublicKey)} + for i := range t.cfg.Listen { + a.Endpoints = append(a.Endpoints, t.endpoints[i]...) + } + if len(a.Endpoints) > 0 { + psk := t.st.listenerPSK() + a.PSK = psk[:] + } + return a +} + +// Pending lists carriers still coming up, with their last error if any. +func (t *Tunnel) Pending() map[string]string { + t.mu.Lock() + defer t.mu.Unlock() + out := map[string]string{} + for i, spec := range t.cfg.Listen { + if len(t.endpoints[i]) == 0 { + out[spec] = t.errs[spec] + } + } + return out +} + +// RotatePSK replaces the listener's pre-shared key. Every address shared +// before stops working for peers not met yet; peers already met keep the +// key agreed with them. +func (t *Tunnel) RotatePSK() error { + if err := t.st.rotate(); err != nil { + return err + } + // Drop peers admitted under the old key that never completed a stream. + t.dev.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool { + _, known := t.st.pair(Key(pk)) + return !known + }) + return nil +} + +// Stream is a connection to port 1 inside the tunnel. +type Stream struct { + net.Conn + // Remote is the WireGuard key that completed the handshake. + Remote Key + t *Tunnel +} + +// Is reports whether the Ed25519 key s ("ed25519:...") is the one on the +// other end of the stream (section 10.1). +func (s *Stream) Is(pub ed25519.PublicKey) bool { + k, err := X25519Public(pub) + return err == nil && k == s.Remote +} + +// Via describes the carrier path the peer was last heard on. +func (s *Stream) Via() string { + if v := s.t.bind.peer(s.Remote).current(); v != "" { + return v + } + return "tunnel" +} + +// Accept waits for the next stream a peer opens. +func (t *Tunnel) Accept() (*Stream, error) { + for { + c, err := t.ln.Accept() + if err != nil { + return nil, err + } + ra, ok := c.RemoteAddr().(*net.TCPAddr) + if !ok { + c.Close() + continue + } + ip, _ := netip.AddrFromSlice(ra.IP) + t.mu.Lock() + k, known := t.keys[ip.Unmap()] + psk, admitted := t.admitted[k] + delete(t.admitted, k) + t.mu.Unlock() + if !known { + c.Close() + continue + } + if admitted { + // A stream proves the handshake held: the pair's key is agreed. + t.st.setPair(k, psk) + } + return &Stream{Conn: c, Remote: k, t: t}, nil + } +} + +// Dial opens a stream to the peer the addresses describe (all for one key). +// The pre-shared key tried first is the one agreed with that peer before, +// then the address's; the one that works is remembered (section 5.2). +func (t *Tunnel) Dial(ctx context.Context, addrs ...Address) (*Stream, error) { + a, err := Merge(addrs...) + if err != nil { + return nil, err + } + k, err := X25519Public(a.Key) + if err != nil { + return nil, err + } + if k == t.pub { + return nil, errors.New("that address is this peer's own") + } + var cands []path + for _, e := range a.Endpoints { + if t.carrier(e.Kind) != nil { + cands = append(cands, path{e.Kind, e.Value}) + } + } + ep := t.bind.peer(k) + if len(cands) == 0 && ep.current() == "" { + return nil, fmt.Errorf("no endpoint this peer can use in %s", a.KeyString()) + } + ep.addCands(cands...) + t.noteKey(k) + + var psks [][32]byte + if p, ok := t.st.pair(k); ok { + psks = append(psks, p) + } + if len(a.PSK) == 32 { + var p [32]byte + copy(p[:], a.PSK) + if len(psks) == 0 || psks[0] != p { + psks = append(psks, p) + } + } + if len(psks) == 0 { + psks = append(psks, [32]byte{}) + } + var lastErr error + for i, psk := range psks { + if err := t.configure(k, psk); err != nil { + return nil, err + } + actx, cancel := ctx, context.CancelFunc(func() {}) + if i < len(psks)-1 { + actx, cancel = context.WithTimeout(ctx, 12*time.Second) + } + c, err := t.dialStream(actx, k) + cancel() + if err == nil { + t.st.setPair(k, psk) + return &Stream{Conn: c, Remote: k, t: t}, nil + } + lastErr = err + if ctx.Err() != nil { + break + } + } + return nil, fmt.Errorf("no tunnel to %s: %w", a.KeyString(), lastErr) +} + +// dialStream opens the TCP stream to k's port 1. It first rides whatever +// session WireGuard holds; if that goes nowhere for a few seconds (the +// peer restarted and lost it, or moved), it forces a fresh handshake +// rather than wait the 15 seconds WireGuard takes to notice by itself. +func (t *Tunnel) dialStream(ctx context.Context, k Key) (net.Conn, error) { + to := netip.AddrPortFrom(IP(k), Port) + quick, cancel := context.WithTimeout(ctx, 3*time.Second) + c, err := t.tun.dialTCP(quick, to) + cancel() + if err == nil { + return c, nil + } + if ctx.Err() != nil { + return nil, err + } + if p, ok := t.dev.LookupActivePeer(device.NoisePublicKey(k)); ok { + p.ExpireCurrentKeypairs() + } + return t.tun.dialTCP(ctx, to) +} + +// configure adds or updates the WireGuard peer for k. +func (t *Tunnel) configure(k Key, psk [32]byte) error { + var b strings.Builder + fmt.Fprintf(&b, "public_key=%s\n", hex.EncodeToString(k[:])) + fmt.Fprintf(&b, "preshared_key=%s\n", hex.EncodeToString(psk[:])) + fmt.Fprintf(&b, "endpoint=peer:%s\n", hex.EncodeToString(k[:])) + b.WriteString("replace_allowed_ips=true\n") + fmt.Fprintf(&b, "allowed_ip=%s/128\n", IP(k)) + return t.dev.IpcSet(b.String()) +} + +// Close stops the device and every carrier. +func (t *Tunnel) Close() error { + t.mu.Lock() + if t.closed { + t.mu.Unlock() + return nil + } + t.closed = true + t.mu.Unlock() + t.cancel() + t.ln.Close() + t.udp.close() + t.unix.close() + t.tailcat.close() + t.ws.close() + t.dev.Close() + return nil +} + +// CloseWrite half-closes the stream: the peer reads EOF, and can still +// write. +func (s *Stream) CloseWrite() error { + if cw, ok := s.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return s.Conn.Close() +} diff --git a/tunnel/tunnel_test.go b/tunnel/tunnel_test.go new file mode 100644 index 0000000..bdaa734 --- /dev/null +++ b/tunnel/tunnel_test.go @@ -0,0 +1,383 @@ +package tunnel + +import ( + "bufio" + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "golang.org/x/crypto/curve25519" +) + +func exampleKey(name string) ed25519.PrivateKey { + seed := sha256.Sum256([]byte("awp draft 2 example key: " + name)) + return ed25519.NewKeyFromSeed(seed[:]) +} + +func b64(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } + +// TestSpecVectors pins the worked example in SPEC.md section 4.2. +func TestSpecVectors(t *testing.T) { + priv := exampleKey("B") + if got := b64(priv.Seed()); got != "n91wt2HNOBdumTc7UI0QeqJw8wML6cYWcGzmd0k8Tbo" { + t.Fatalf("seed %s", got) + } + xpriv := X25519Private(priv) + if got := b64(xpriv[:]); got != "AFrT5h44MRTokzB0U6z4Jesw-ISu-CTuXvFnIMZbzUQ" { + t.Errorf("X25519 private %s", got) + } + xpub, err := X25519Public(priv.Public().(ed25519.PublicKey)) + if err != nil { + t.Fatal(err) + } + if got := xpub.String(); got != "O7KNWfmBVF214-z17qtor-Hl4UWIDOerTw_nad9Vn1Q" { + t.Errorf("X25519 public %s", got) + } + if got := IP(xpub).String(); got != "fdea:9171:7e2f:c57a:5d92:7d7:169b:67fb" { + t.Errorf("tunnel IP %s", got) + } + // The converted private key's public key is the converted public key. + check, _ := curve25519.X25519(xpriv[:], curve25519.Basepoint) + if string(check) != string(xpub[:]) { + t.Error("conversion mismatch") + } + for _, n := range []string{"A", "B", "C"} { + p := exampleKey(n) + k, _ := X25519Public(p.Public().(ed25519.PublicKey)) + t.Logf("%s ed25519:%s X25519 %s ip %s", n, b64(p.Public().(ed25519.PublicKey)), k, IP(k)) + } +} + +// TestSpecAddresses decodes every address in SPEC.md. +func TestSpecAddresses(t *testing.T) { + src, err := os.ReadFile("../SPEC.md") + if err != nil { + t.Fatal(err) + } + found := regexp.MustCompile(`awp1[A-Za-z0-9_-]{40,}`).FindAllString(string(src), -1) + if len(found) < 3 { + t.Fatalf("found %d addresses in SPEC.md", len(found)) + } + for _, s := range found { + a, err := ParseAddress(s) + if err != nil { + t.Errorf("%s: %v", s, err) + continue + } + if a.String() != s { + t.Errorf("%s does not re-encode to itself (not deterministic CBOR)", s) + } + } +} + +func TestAddress(t *testing.T) { + priv := exampleKey("B") + psk := sha256.Sum256([]byte("awp draft 2 example psk")) + a := Address{Key: priv.Public().(ed25519.PublicKey), PSK: psk[:], Endpoints: []Endpoint{ + {KindUDP, "[2a09:8280:1::4:1b2c]:41641"}, + {KindTailcat, "tcomFwWCCcjS5nKNqAod034nWoJZW0LZqDhhC8U_dKdnDRYQ8uNGFpGQEu"}, + {KindWS, "wss://quiet-otter-7f3a.trycloudflare.com/awp"}, + }} + s := a.String() + t.Logf("example address %s", s) + if !strings.HasPrefix(s, "awp1") || s != a.String() { + t.Fatal("not deterministic") + } + b, err := ParseAddress(" " + s + "\n") + if err != nil { + t.Fatal(err) + } + if b.String() != s || !b.Key.Equal(a.Key) || len(b.Endpoints) != 3 || b.Endpoints[2] != a.Endpoints[2] { + t.Fatalf("round trip: %+v", b) + } + if p := b.Public(); p.PSK != nil || p.String() == s { + t.Fatal("Public kept the psk") + } + m, err := Merge(a.Public(), Address{Key: a.Key, Endpoints: []Endpoint{{KindUnix, "/x"}, a.Endpoints[0]}}, a) + if err != nil || len(m.Endpoints) != 4 || m.PSK == nil { + t.Fatalf("merge: %+v %v", m, err) + } + if _, err := Merge(a, Address{Key: exampleKey("A").Public().(ed25519.PublicKey)}); err == nil { + t.Fatal("merged different keys") + } + for _, bad := range []string{"", "awp1", "awp1!!", "tcfoo", "awp1" + b64([]byte{0xa1, 0x63, 'k', 'e', 'y', 0x41, 0})} { + if _, err := ParseAddress(bad); err == nil { + t.Errorf("parsed %q", bad) + } + } +} + +type peer struct { + priv ed25519.PrivateKey + t *Tunnel +} + +func newPeer(t *testing.T, dir string, listen ...string) *peer { + t.Helper() + var priv ed25519.PrivateKey + if dir != "" { + priv = exampleKey(dir) + dir = filepath.Join(tmp(t), dir) + } else { + _, priv, _ = ed25519.GenerateKey(nil) + } + return startPeer(t, priv, dir, listen...) +} + +var tmpDir string + +func tmp(t *testing.T) string { + if tmpDir == "" { + // Short, since unix socket paths are limited to ~100 bytes. + d, err := os.MkdirTemp("", "awptun") + if err != nil { + t.Fatal(err) + } + tmpDir = d + } + return tmpDir +} + +func TestMain(m *testing.M) { + code := m.Run() + if tmpDir != "" { + os.RemoveAll(tmpDir) + } + os.Exit(code) +} + +func startPeer(t *testing.T, priv ed25519.PrivateKey, dir string, listen ...string) *peer { + t.Helper() + tu, err := New(Config{Identity: priv, StateDir: dir, Listen: listen, Logf: t.Logf, Verbose: os.Getenv("WGV") != ""}) + if err != nil { + t.Fatal(err) + } + if err := tu.Start(); err != nil { + tu.Close() + t.Fatal(err) + } + t.Cleanup(func() { tu.Close() }) + return &peer{priv: priv, t: tu} +} + +// echo serves one stream at a time, replying to each line with its key and +// the line. +func echo(p *peer) { + go func() { + for { + s, err := p.t.Accept() + if err != nil { + return + } + go func() { + defer s.Close() + sc := bufio.NewScanner(s) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + fmt.Fprintf(s, "%s %s\n", s.Remote, sc.Text()) + } + }() + } + }() +} + +func roundTrip(t *testing.T, s *Stream, from *peer, msg string) { + t.Helper() + s.SetDeadline(time.Now().Add(20 * time.Second)) + if _, err := fmt.Fprintln(s, msg); err != nil { + t.Fatal(err) + } + line, err := bufio.NewReader(s).ReadString('\n') + if err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("%s %s\n", from.t.Key(), msg) + if line != want { + t.Fatalf("got %q, want %q", line, want) + } +} + +func dial(t *testing.T, from *peer, addrs ...Address) *Stream { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + s, err := from.t.Dial(ctx, addrs...) + if err != nil { + t.Fatal(err) + } + return s +} + +func TestCarriers(t *testing.T) { + for _, tc := range []struct{ name, spec string }{ + {"unix", "unix:" + filepath.Join(tmp(t), "c.sock")}, + {"udp", "udp:127.0.0.1:0"}, + {"ws", "ws:127.0.0.1:0"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := newPeer(t, "", tc.spec) + echo(b) + a := newPeer(t, "") + addr := b.t.Address() + if len(addr.Endpoints) != 1 || addr.PSK == nil { + t.Fatalf("address %+v", addr) + } + s := dial(t, a, addr) + defer s.Close() + if !s.Is(b.priv.Public().(ed25519.PublicKey)) || s.Is(a.priv.Public().(ed25519.PublicKey)) { + t.Fatal("Is") + } + roundTrip(t, s, a, "hello over "+tc.name) + // A large write crosses many datagrams. + big := strings.Repeat("x", 200000) + roundTrip(t, s, a, big) + t.Logf("via %s", s.Via()) + }) + } +} + +func TestAdmission(t *testing.T) { + sock := filepath.Join(tmp(t), "adm.sock") + b := newPeer(t, "adm-b", "unix:"+sock) + echo(b) + addr := b.t.Address() + + // A wrong pre-shared key gets nowhere. + c := newPeer(t, "") + wrong := addr + wrong.PSK = make([]byte, 32) + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + if s, err := c.t.Dial(ctx, wrong); err == nil { + s.Close() + t.Fatal("dialed with the wrong psk") + } + cancel() + + // The right one works, and the pair is remembered on both sides. + a := newPeer(t, "adm-a") + s := dial(t, a, addr) + roundTrip(t, s, a, "first") + s.Close() + + // B dials back with no endpoint: it knows where A is from the tunnel. + echo(a) + back := dial(t, b, Address{Key: a.priv.Public().(ed25519.PublicKey)}) + roundTrip(t, back, b, "back") + back.Close() + + // Rotating B's psk locks out new peers holding the old address, but not A. + if err := b.t.RotatePSK(); err != nil { + t.Fatal(err) + } + d := newPeer(t, "") + ctx, cancel = context.WithTimeout(context.Background(), 8*time.Second) + if s, err := d.t.Dial(ctx, addr); err == nil { + s.Close() + t.Fatal("old address still admits new peers") + } + cancel() + s = dial(t, a, addr) + roundTrip(t, s, a, "after rotation") + s.Close() + + // B restarts from its state directory; A reaches it at the same socket. + b.t.Close() + b2 := startPeer(t, b.priv, filepath.Join(tmp(t), "adm-b"), "unix:"+sock) + echo(b2) + s = dial(t, a, b2.t.Address()) + roundTrip(t, s, a, "after restart") + s.Close() +} + +func TestNotListening(t *testing.T) { + // A peer that listens on nothing admits only peers it has met. + a := newPeer(t, "") + b := newPeer(t, "", "udp:127.0.0.1:0") + echo(a) + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + if s, err := b.t.Dial(ctx, Address{Key: a.priv.Public().(ed25519.PublicKey), Endpoints: []Endpoint{{KindUDP, "127.0.0.1:9"}}}); err == nil { + s.Close() + t.Fatal("reached a non-listening stranger") + } +} + +func TestTailcat(t *testing.T) { + if os.Getenv("AWP_TEST_TAILCAT") == "" { + t.Skip("set AWP_TEST_TAILCAT=1 to test through tailcat's DERP relays") + } + carrierLive(t, "tailcat") +} + +func TestCloudflare(t *testing.T) { + if os.Getenv("AWP_TEST_CLOUDFLARE") == "" { + t.Skip("set AWP_TEST_CLOUDFLARE=1 to test through a Cloudflare quick tunnel (needs cloudflared)") + } + carrierLive(t, "cloudflare") +} + +func carrierLive(t *testing.T, spec string) { + b := newPeer(t, "", spec) + echo(b) + deadline := time.Now().Add(90 * time.Second) + for len(b.t.Address().Endpoints) == 0 { + if time.Now().After(deadline) { + t.Fatalf("%s did not come up: %v", spec, b.t.Pending()) + } + time.Sleep(200 * time.Millisecond) + } + t.Logf("address %s", b.t.Address()) + a := newPeer(t, "") + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + s, err := a.t.Dial(ctx, b.t.Address()) + if err != nil { + t.Fatal(err) + } + defer s.Close() + roundTrip(t, s, a, "through "+spec) + roundTrip(t, s, a, strings.Repeat("y", 100000)) + io.WriteString(s, "") +} + +// TestTwoPaths: a listener on two carriers. Repeated dials are quick (no +// replayed handshakes, one response per initiation), and when the +// preferred path dies mid-stream the stream carries on over the other. +func TestTwoPaths(t *testing.T) { + b := newPeer(t, "", "udp:127.0.0.1:0", "ws:127.0.0.1:0") + echo(b) + a := newPeer(t, "") + for i := 0; i < 6; i++ { + start := time.Now() + s := dial(t, a, b.t.Address()) + roundTrip(t, s, a, fmt.Sprintf("dial %d", i)) + if d := time.Since(start); d > 2*time.Second { + t.Errorf("dial %d took %v", i, d) + } + if i < 5 { + s.Close() + continue + } + if !strings.HasPrefix(s.Via(), "udp:") { + t.Fatalf("via %s, want the udp path preferred", s.Via()) + } + // The udp path dies; the ws path takes over. + b.t.udp.close() + start = time.Now() + roundTrip(t, s, a, "after udp died") + t.Logf("failover took %v, now via %s", time.Since(start).Round(time.Millisecond), s.Via()) + if !strings.HasPrefix(s.Via(), "ws:") { + t.Errorf("via %s after udp died", s.Via()) + } + s.Close() + } +} diff --git a/wire/canon.go b/wire/canon.go index d9eb0b5..e9e204e 100644 --- a/wire/canon.go +++ b/wire/canon.go @@ -8,7 +8,7 @@ import ( "unicode/utf8" ) -// Canonical re-encodes a JSON document the way section 10.2 defines +// Canonical re-encodes a JSON document the way section 13.2 defines // canonical JSON: object keys sorted, no whitespace, UTF-8. Strings escape // only what JSON requires (quote, backslash and control characters, using // the short forms \b \f \n \r \t), so the output matches Python's diff --git a/wire/grant.go b/wire/grant.go index aeea278..7d1a861 100644 --- a/wire/grant.go +++ b/wire/grant.go @@ -12,7 +12,7 @@ import ( "time" ) -// GrantObject is the wire form of a grant (section 10.2): a signed +// GrantObject is the wire form of a grant (section 13.2): a signed // statement that sub may use caps until exp, honored by iss and by whoever // trusts iss. sig is iss's Ed25519 signature over the canonical JSON of the // object without sig: keys sorted, no whitespace, UTF-8. @@ -21,7 +21,7 @@ type GrantObject struct { Iss string `json:"iss" jsonschema:"pattern=^ed25519:[A-Za-z0-9_-]{43}$"` // Sub is the receiving key. Sub string `json:"sub" jsonschema:"pattern=^ed25519:[A-Za-z0-9_-]{43}$"` - // Caps are the capability strings granted (section 10.1). + // Caps are the capability strings granted (section 13.1). Caps []string `json:"caps"` // Exp is when the grant expires, RFC 3339. Exp string `json:"exp" jsonschema:"format=date-time"` diff --git a/wire/keys.go b/wire/keys.go index 9e14573..8e90db4 100644 --- a/wire/keys.go +++ b/wire/keys.go @@ -44,7 +44,7 @@ func ShortKey(key string) string { return body } -// Fingerprint is the SHA-256 fingerprint of a public key, the way section 13 +// Fingerprint is the SHA-256 fingerprint of a public key, the way section 16 // suggests agents print it in their about text. func Fingerprint(pub ed25519.PublicKey) string { sum := sha256.Sum256(pub) @@ -75,39 +75,9 @@ func Nonce(n int) string { return EncodeB64(b) } -// AuthPayload is the byte string signed in auth: -// "awp-auth-v0" || 0x00 || my_hello_line || 0x00 || peer_hello_line. -func AuthPayload(myHello, peerHello []byte) []byte { - b := make([]byte, 0, len(AuthContext)+2+len(myHello)+len(peerHello)) - b = append(b, AuthContext...) - b = append(b, 0) - b = append(b, myHello...) - b = append(b, 0) - b = append(b, peerHello...) - return b -} - -// SignAuth signs the auth transcript from the signer's point of view. -func SignAuth(priv ed25519.PrivateKey, myHello, peerHello []byte) string { - return EncodeB64(ed25519.Sign(priv, AuthPayload(myHello, peerHello))) -} - // ErrBadSignature means a signature did not verify. var ErrBadSignature = errors.New("signature does not verify") -// VerifyAuth checks the peer's auth signature. From the verifier's side the -// peer's hello comes first, since the peer signed its own hello first. -func VerifyAuth(peer ed25519.PublicKey, sig string, peerHello, myHello []byte) error { - raw, err := DecodeB64(sig) - if err != nil { - return fmt.Errorf("sig: %v", err) - } - if !ed25519.Verify(peer, AuthPayload(peerHello, myHello), raw) { - return ErrBadSignature - } - return nil -} - func abbrev(s string) string { if len(s) > 24 { return s[:24] + "…" diff --git a/wire/wire.go b/wire/wire.go index d302681..990a307 100644 --- a/wire/wire.go +++ b/wire/wire.go @@ -1,5 +1,5 @@ // Package wire implements the wire format of the Agent Wire Protocol -// (AWP): NDJSON framing, the message envelope, the typed messages of SPEC.md sections 6 to 10, Ed25519 +// (AWP): NDJSON framing, the message envelope, the typed messages of SPEC.md sections 9 to 13, Ed25519 // key encoding, canonical JSON and signed grants. // // The package has no I/O policy of its own. It is shared by the daemon, the @@ -15,8 +15,8 @@ import ( "time" ) -// The JSON Schema in schema/v0/awp.schema.json and its reference page -// schema/v0/schema.mdx are generated from this package. `make schema` +// The JSON Schema in schema/v1/awp.schema.json and its reference page +// schema/v1/schema.mdx are generated from this package. `make schema` // regenerates them; a test fails when they are stale. // //go:generate go run ../internal/schemagen @@ -24,22 +24,18 @@ import ( // Protocol constants. const ( // Version is the protocol major version sent in hello.v. - Version = 0 + Version = 1 // MaxLine is the largest line a peer accepts, excluding the newline. MaxLine = 1 << 20 // ChunkSize is the recommended chunk payload size before encoding. ChunkSize = 256 << 10 - - // AuthContext prefixes the byte string signed in auth. - AuthContext = "awp-auth-v0" ) // Message types. const ( THello = "hello" - TAuth = "auth" TResume = "resume" TMsg = "msg" TState = "state" @@ -65,7 +61,6 @@ type Message struct { func Messages() []Message { return []Message{ {THello, reflect.TypeFor[Hello]()}, - {TAuth, reflect.TypeFor[Auth]()}, {TResume, reflect.TypeFor[Resume]()}, {TState, reflect.TypeFor[State]()}, {TMsg, reflect.TypeFor[Msg]()}, @@ -89,11 +84,12 @@ func Extensions() []string { return []string{TPresence, TMirror, TPrivate} } -// Error codes from section 9.7. +// Error codes from section 12.7. const ( ErrBadFrame = "bad_frame" ErrVersion = "version" ErrAuth = "auth" + ErrRefused = "refused" ErrUnsupported = "unsupported" ErrForbidden = "forbidden" ErrBlobRefused = "blob_refused" @@ -101,9 +97,9 @@ const ( ErrInternal = "internal" ) -// ErrCodes lists the error codes of section 9.7, in the spec's order. +// ErrCodes lists the error codes of section 12.7, in the spec's order. func ErrCodes() []string { - return []string{ErrBadFrame, ErrVersion, ErrAuth, ErrUnsupported, ErrForbidden, ErrBlobRefused, ErrTooLarge, ErrInternal} + return []string{ErrBadFrame, ErrTooLarge, ErrVersion, ErrAuth, ErrRefused, ErrUnsupported, ErrForbidden, ErrBlobRefused, ErrInternal} } // ErrCloses reports whether an err with this code closes the connection. @@ -112,7 +108,7 @@ func ErrCodes() []string { // cannot tear down a connection just by inventing a code. func ErrCloses(code string) bool { switch code { - case ErrBadFrame, ErrVersion, ErrAuth, ErrTooLarge: + case ErrBadFrame, ErrVersion, ErrAuth, ErrRefused, ErrTooLarge: return true } return false @@ -126,7 +122,7 @@ const ( PartBlob = "blob" ) -// PartKind says which fields a part of one kind carries (section 9.1): +// PartKind says which fields a part of one kind carries (section 12.1): // Required are always present, Optional only when set. Part.MarshalJSON // follows this table, and the JSON Schema is generated from it. type PartKind struct { @@ -136,17 +132,17 @@ type PartKind struct { Optional []string } -// PartKinds lists the part kinds of section 9.1. +// PartKinds lists the part kinds of section 12.1. func PartKinds() []PartKind { return []PartKind{ {PartText, "Text, markdown by convention.", []string{"text"}, nil}, {PartCode, "Fenced code without the fence; lang names the language.", []string{"text"}, []string{"lang"}}, - {PartData, "Inline JSON, with its MIME type. Requests (section 10.1) are data parts of a vnd.awp type.", []string{"data"}, []string{"mime"}}, + {PartData, "Inline JSON, with its MIME type. Requests (section 13.1) are data parts of a vnd.awp type.", []string{"data"}, []string{"mime"}}, {PartBlob, "A blob sent in chunk messages before or after this message, named by ref.", []string{"ref", "size"}, []string{"name", "mime"}}, } } -// Recommended thread states from section 8.1. +// Recommended thread states from section 11.1. const ( StateOpen = "open" StateWorking = "working" @@ -156,23 +152,25 @@ const ( StateClosed = "closed" ) -// ThreadStates lists the recommended states of section 8.1. Peers may use +// ThreadStates lists the recommended states of section 11.1. Peers may use // others. func ThreadStates() []string { return []string{StateOpen, StateWorking, StateWaiting, StateDone, StateFailed, StateClosed} } // Patterns for the encoded fields, as the JSON Schema states them: keys are -// "ed25519:" and 43 characters of unpadded base64url (section 7.1); nonces +// "ed25519:" and 43 characters of unpadded base64url (section 10.1); nonces // and signatures are unpadded base64url; chunk data is standard base64 -// (section 9.3). Receivers are more liberal (DecodeB64). +// (section 12.3). Receivers are more liberal (DecodeB64). const ( KeyPattern = "^ed25519:[A-Za-z0-9_-]{43}$" B64URLPattern = "^[A-Za-z0-9_-]+$" B64Pattern = "^[A-Za-z0-9+/]*={0,2}$" + // AddressPattern matches an address (section 6): "awp1" and base64url. + AddressPattern = "^awp1[A-Za-z0-9_-]+$" ) -// Envelope holds the fields every line carries (section 6). The +// Envelope holds the fields every line carries (section 9). The // type-specific fields sit beside them at the top level. type Envelope struct { // T is the message type. @@ -189,49 +187,35 @@ type Envelope struct { } // Hello is the first line each side sends, at once, without waiting for -// the other (section 7.1). +// the other (section 10.1). type Hello struct { Envelope - // V is the protocol major version. A peer that sees a version it does - // not speak sends err version and closes. + // V is the protocol major version, 1 for draft 2. A peer that sees a + // version it does not speak sends err version and closes. V int `json:"v" jsonschema:"minimum=0"` - // Key is the long-term Ed25519 public key. Its bytes are the peer's - // identity. + // Key is the sender's Ed25519 identity key. It must be the key on the + // other end of the tunnel (section 13.1), or the receiver sends err + // auth and closes. Key string `json:"key" jsonschema:"pattern=^ed25519:[A-Za-z0-9_-]{43}$"` // Name is how the peer calls itself, "harness@host" by convention. Name string `json:"name,omitempty"` - // Nonce is 32 random bytes, base64url. The auth signature covers the - // whole hello line, nonce included. - Nonce string `json:"nonce" jsonschema:"pattern=^[A-Za-z0-9_-]+$"` - // Caps lists the supported message families beyond the mandatory chat - // and resume: blob, grant, introduce, and any extension. + // Caps lists the supported message families beyond the mandatory + // core: blob, grant, introduce, and any extension. Caps []string `json:"caps,omitempty"` - // About is free text for the other agent's context. It is not - // authenticated until auth completes. + // About is free text for the other agent's context. About string `json:"about,omitempty"` - - // Addr is an extension: an address at which the sender can be reached, - // so that either side can reconnect (section 4.3). Peers that do not - // know it ignore it, as section 5 requires. - Addr string `json:"addr,omitempty"` + // Addr is the sender's own address without its pre-shared key, so the + // other side can reconnect to it later (section 13.3). + Addr string `json:"addr,omitempty" jsonschema:"pattern=^awp1[A-Za-z0-9_-]+$"` + // Grants are grant objects (section 13.2) the sender presents. + Grants []json.RawMessage `json:"grants,omitempty"` // Shares is an extension (mirror.go): the keys of the hosts this agent // mirrors its conversations to, so the other party knows. Shares []string `json:"shares,omitempty"` } -// Auth proves possession of the key sent in hello (section 7.2). -type Auth struct { - Envelope - // Sig is the Ed25519 signature, base64url, over "awp-auth-v0" || 0x00 || - // my hello line || 0x00 || peer hello line, the lines as sent and - // received without the newline. - Sig string `json:"sig" jsonschema:"pattern=^[A-Za-z0-9_-]+$"` - // Grants are grant objects (section 10.2) the sender presents. - Grants []json.RawMessage `json:"grants,omitempty"` -} - -// Resume is sent by both sides after every handshake (section 9.5). The +// Resume is sent by both sides after every handshake (section 12.5). The // receiver replays its outbox messages the sender has not seen. type Resume struct { Envelope @@ -240,7 +224,7 @@ type Resume struct { Seen map[string]string `json:"seen"` } -// Msg is one turn in a thread (section 9.1). The first msg with a new th +// Msg is one turn in a thread (section 12.1). The first msg with a new th // creates the thread. type Msg struct { Envelope @@ -329,7 +313,7 @@ func writeJSON(buf *bytes.Buffer, v any) { buf.Truncate(buf.Len() - 1) // Encode appends a newline } -// State is the sender's view of a thread's soft state (section 8.1). The +// State is the sender's view of a thread's soft state (section 11.1). The // two sides can disagree. type State struct { Envelope @@ -346,7 +330,7 @@ type Ack struct { Envelope } -// Chunk carries part of a blob (section 9.3). Chunks of one blob arrive in +// Chunk carries part of a blob (section 12.3). Chunks of one blob arrive in // order; chunks of different blobs may interleave. A chunk carries th when // the blob belongs to a thread, so that resume replays it like any other // threaded message. @@ -363,7 +347,7 @@ type Chunk struct { Data string `json:"data"` } -// Ping is a liveness probe (section 9.4): sent when idle for 30 seconds by +// Ping is a liveness probe (section 12.4): sent when idle for 30 seconds by // convention; two missed pongs mean the connection is dead. type Ping struct { Envelope @@ -374,7 +358,7 @@ type Pong struct { Envelope } -// Bye is a graceful close (section 9.6). After sending it a peer sends +// Bye is a graceful close (section 12.6). After sending it a peer sends // nothing else and closes after the other side's bye or after 5 seconds. type Bye struct { Envelope @@ -382,8 +366,8 @@ type Bye struct { Reason string `json:"reason,omitempty"` } -// Err reports a problem (section 9.7). Whether it closes the connection -// depends on the code: bad_frame, version, auth and too_large do, +// Err reports a problem (section 12.7). Whether it closes the connection +// depends on the code: bad_frame, too_large, version, auth and refused do, // unsupported, forbidden, blob_refused and internal do not. type Err struct { Envelope @@ -395,15 +379,15 @@ type Err struct { Ref string `json:"ref,omitempty"` } -// GrantMsg delivers a grant after the handshake (section 10.3). +// GrantMsg delivers a grant after the handshake (section 13.3). type GrantMsg struct { Envelope - // Grant is the grant object (section 10.2). + // Grant is the grant object (section 13.2). Grant json.RawMessage `json:"grant"` } // Introduce hands the recipient another peer's identity and address plus a -// grant issued by the introducer (section 10.4). The introduced peer honors +// grant issued by the introducer (section 13.4). The introduced peer honors // the grant only if it trusts the introducer with introduce. type Introduce struct { Envelope @@ -420,9 +404,9 @@ type IntroPeer struct { Key string `json:"key" jsonschema:"pattern=^ed25519:[A-Za-z0-9_-]{43}$"` // Name is what the introduced peer calls itself. Name string `json:"name,omitempty"` - // Address is where the introduced peer listens, a hint: the key is the - // identity. - Address string `json:"address,omitempty"` + // Address is the introduced peer's address (section 9). It carries the + // pre-shared key only when the introducer holds one. + Address string `json:"address,omitempty" jsonschema:"pattern=^awp1[A-Za-z0-9_-]+$"` } // Encode marshals a message to one line, without the trailing newline and diff --git a/wire/wire_test.go b/wire/wire_test.go index c347b4f..270a83d 100644 --- a/wire/wire_test.go +++ b/wire/wire_test.go @@ -87,33 +87,6 @@ func TestKeys(t *testing.T) { } } -func TestAuthSignature(t *testing.T) { - pubA, privA, _ := ed25519.GenerateKey(nil) - helloA := []byte(`{"t":"hello","id":"a1","ts":"x","v":0,"key":"A","nonce":"nA"}`) - helloB := []byte(`{"t":"hello","id":"b1","ts":"x","v":0,"key":"B","nonce":"nB"}`) - sig := SignAuth(privA, helloA, helloB) - // B verifies with A's key, peer (A) hello first. - if err := VerifyAuth(pubA, sig, helloA, helloB); err != nil { - t.Fatal(err) - } - // Swapped order, a different hello, or a different key must fail. - if VerifyAuth(pubA, sig, helloB, helloA) == nil { - t.Fatal("verified with swapped hellos") - } - if VerifyAuth(pubA, sig, helloA, append(helloB, ' ')) == nil { - t.Fatal("verified with modified hello") - } - pubC, _, _ := ed25519.GenerateKey(nil) - if VerifyAuth(pubC, sig, helloA, helloB) == nil { - t.Fatal("verified with wrong key") - } - payload := AuthPayload(helloA, helloB) - want := "awp-auth-v0\x00" + string(helloA) + "\x00" + string(helloB) - if string(payload) != want { - t.Fatalf("payload = %q", payload) - } -} - const canonInput = `{"z":1,"a":{"y":[3,"x",null,true],"b":"q\"uo\\te"},"m":"<&> é ✓ 
 \u0001 \t\n","caps":["exec","fs:read"],"n":-1.50e3}` func TestCanonical(t *testing.T) { @@ -260,7 +233,7 @@ func TestEncode(t *testing.T) { t.Fatalf("decode: %v %+v", err, back) } // Hello always carries v, even when it is zero; chunks always carry n and last. - hl, _ := Encode(Hello{Envelope: Envelope{T: THello}, Key: "k", Nonce: "n"}) + hl, _ := Encode(Hello{Envelope: Envelope{T: THello}, Key: "k"}) if !strings.Contains(string(hl), `"v":0`) { t.Fatalf("hello without v: %s", hl) } @@ -347,7 +320,8 @@ func TestPatterns(t *testing.T) { for _, c := range []struct{ pattern, value string }{ {KeyPattern, FormatKey(pub)}, {B64URLPattern, Nonce(32)}, - {B64URLPattern, SignAuth(priv, []byte("a"), []byte("b"))}, + {B64URLPattern, EncodeB64(ed25519.Sign(priv, []byte("a")))}, + {AddressPattern, "awp1o2NrZXlYICz4lGVLe_5MyFCpuS1-L-EVBF8KHDCr9bieP7E5semX"}, {B64Pattern, base64.StdEncoding.EncodeToString([]byte("hello, world"))}, {B64Pattern, ""}, } { @@ -355,7 +329,7 @@ func TestPatterns(t *testing.T) { t.Errorf("%q does not match %s", c.value, c.pattern) } } - known := map[string]bool{KeyPattern: true, B64URLPattern: true, B64Pattern: true} + known := map[string]bool{KeyPattern: true, B64URLPattern: true, B64Pattern: true, AddressPattern: true} seen := map[reflect.Type]bool{} var walk func(rt reflect.Type) walk = func(rt reflect.Type) {