From b842a4990ac7f356ccd7d1276f0c64eee85b05b7 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:56:27 +0000 Subject: [PATCH 01/10] fork-kit: pc-agents walls and their check (part 1 of 2) A WSL2 distro on a Windows PC as the wall around T3's agents: the files the kit owns (wsl.conf, resolv.conf, managed policy, uid-matched nft table, a root system unit with User=agent and a sandbox, a daily check timer), rendered from the pc-agents section of the machine file, and `pc-agents check [--enforce]`: the fleet's wsl2 walls (run, not copied) plus the kit's own. The unit runs the check before every start. A server test pins that the managed policy locks the fork's settings (agentSlots, linkedEnvironmentsEnabled, automaticUpdates). Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + .../policy/managedSettings.forkKeys.test.ts | 85 +++++++ fork-kit/pc-agents/README.md | 94 ++++++++ fork-kit/pc-agents/lib/pc_agents.py | 219 ++++++++++++++++++ fork-kit/pc-agents/sbin/pc-agents | 115 +++++++++ fork-kit/pc-agents/tests/check.test.py | 76 ++++++ fork-kit/pc-agents/tests/fake_distro.py | 157 +++++++++++++ 7 files changed, 747 insertions(+) create mode 100644 apps/server/src/policy/managedSettings.forkKeys.test.ts create mode 100644 fork-kit/pc-agents/README.md create mode 100644 fork-kit/pc-agents/lib/pc_agents.py create mode 100755 fork-kit/pc-agents/sbin/pc-agents create mode 100644 fork-kit/pc-agents/tests/check.test.py create mode 100644 fork-kit/pc-agents/tests/fake_distro.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 58e58018f94aa..0e2b9a7491353 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -56,6 +56,7 @@ fork-kit/heavy/tests/install.test.py fork-kit/heavy/tests/sweep.test.py # A real systemd user manager (the step starts one for the runner user). fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml +fork-kit/pc-agents/tests/check.test.py # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml diff --git a/apps/server/src/policy/managedSettings.forkKeys.test.ts b/apps/server/src/policy/managedSettings.forkKeys.test.ts new file mode 100644 index 0000000000000..99f818179d0ab --- /dev/null +++ b/apps/server/src/policy/managedSettings.forkKeys.test.ts @@ -0,0 +1,85 @@ +// Fork: a managed policy locks the fork's own settings like upstream's. The pc-agents host kit +// (fork-kit/pc-agents) writes a policy that locks the beta switch, automatic updates and the +// child ceiling (agentSlots, one field per dotted path); this pins that each of them is accepted +// and held, and that a partial object is refused (the kit refuses object values for that reason). + +import { DEFAULT_SERVER_SETTINGS, type ServerSettings } from "@t3tools/contracts"; +import { assert, it } from "@effect/vitest"; + +import { makeManagedSettings } from "./managedSettings.ts"; + +const KIT_POLICY = { + linkedEnvironmentsEnabled: { value: true, locked: true }, + automaticUpdates: { value: false, locked: true }, + "agentSlots.enabled": { value: true, locked: true }, + "agentSlots.maxPerMachine": { value: 2, locked: true }, +}; + +const managed = () => { + const result = makeManagedSettings(KIT_POLICY, "full-access"); + if (result._tag !== "Valid") assert.fail(result.reason); + return result.settings; +}; + +const userSettings: ServerSettings = { + ...DEFAULT_SERVER_SETTINGS, + linkedEnvironmentsEnabled: false, + automaticUpdates: true, + agentSlots: { ...DEFAULT_SERVER_SETTINGS.agentSlots, enabled: false, maxPerMachine: 9 }, +}; + +it("accepts the pc-agents kit's locks on fork settings", () => { + assert.deepEqual([...managed().lockedPaths].toSorted(), [ + "agentSlots.enabled", + "agentSlots.maxPerMachine", + "automaticUpdates", + "linkedEnvironmentsEnabled", + ]); +}); + +it("lays the locked fork values over the user's", () => { + const applied = managed().applyLocks(userSettings); + assert.strictEqual(applied.linkedEnvironmentsEnabled, true); + assert.strictEqual(applied.automaticUpdates, false); + assert.strictEqual(applied.agentSlots.enabled, true); + assert.strictEqual(applied.agentSlots.maxPerMachine, 2); +}); + +it("refuses a write that raises the locked child ceiling or turns the beta off", () => { + const settings = managed(); + const current = settings.applyLocks(userSettings); + const raise = settings.vetWrite( + current, + { ...current, agentSlots: { ...current.agentSlots, maxPerMachine: 5 } }, + { agentSlots: { maxPerMachine: 5 } }, + ); + assert.deepEqual(raise, { _tag: "Locked", path: "agentSlots.maxPerMachine" }); + const off = settings.vetWrite( + current, + { ...current, linkedEnvironmentsEnabled: false }, + { linkedEnvironmentsEnabled: false }, + ); + assert.deepEqual(off, { _tag: "Locked", path: "linkedEnvironmentsEnabled" }); +}); + +it("locks one field of a fork setting and leaves the others to the user", () => { + const result = makeManagedSettings({ "agentSlots.maxPerMachine": { value: 2, locked: true } }); + if (result._tag !== "Valid") assert.fail(result.reason); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); + assert.strictEqual(result.settings.applyLocks(userSettings).agentSlots.enabled, false); +}); + +it("takes an object only as a fork setting's complete value", () => { + const partial = makeManagedSettings({ + agentSlots: { value: { maxPerMachine: 2 }, locked: true }, + }); + assert.strictEqual(partial._tag, "Invalid"); + const whole = makeManagedSettings({ + agentSlots: { + value: { ...DEFAULT_SERVER_SETTINGS.agentSlots, maxPerMachine: 2 }, + locked: true, + }, + }); + if (whole._tag !== "Valid") assert.fail(whole.reason); + assert.strictEqual(whole.settings.applyLocks(userSettings).agentSlots.maxPerMachine, 2); +}); diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md new file mode 100644 index 0000000000000..b825f0888c34a --- /dev/null +++ b/fork-kit/pc-agents/README.md @@ -0,0 +1,94 @@ +# pc-agents: run T3's agents on a Windows PC, walled in a WSL2 distro of their own + +A PC can take agent work from another T3 (a linked environment, with **Full access**). Full access runs as the PC +T3's user, so the link is not the wall: the distro is. This kit makes a fresh WSL2 distro into that wall and runs +T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, +so the agent cannot undo it: + +| Wall | What the kit does | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own; cron, at and ssh masked | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | nftables, for the agent's uid only: private, link-local, CGNAT (Tailscale) and multicast ranges dropped, so the Windows host (the NAT gateway), the home network and the tailnet are out of reach. Public addresses (GitHub, Anthropic, package mirrors) and replies to the linked T3 go through | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and daily (a failed wall stops T3) | + +`pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with +`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: every file above is root's, +mode 644 and exactly what the kit writes, nothing overrides the unit (no drop-in), the masks are in place, `agent` is +the only login user, the kit's code is root's, no Windows drive is mounted, the firewall table is loaded, and the +default gateway (the Windows host) is in a blocked range. The fleet's `mounts`, `sockets` and `userns` walls are +about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must +be the kit's file, byte for byte, with no drop-in). + +Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. This folder has the walls and their +check so far; `pc-agents setup`, which writes them into a fresh distro, and its owner steps are still to come. + +## Config + +The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine.json` (root-owned, see +[common](../common/README.md)). Example (made-up values): + +```json +{ + "version": 1, + "pc-agents": { + "organization": "Example home lab", + "t3Version": "0.0.30", + "t3ReleaseBaseUrl": "https://github.com/example/t3code/releases/download", + "port": 3773, + "maxRuntimeMode": "full-access", + "lockedSettings": { "agentSlots.enabled": true, "agentSlots.maxPerMachine": 2 }, + "memoryMax": "8G", + "cpuQuota": "400%", + "nameservers": ["1.1.1.1", "9.9.9.9"], + "tailscaleTag": "tag:pc-agents" + } +} +``` + +| Key | What | +| ----------------------- | ----------------------------------------------------------------------------------------------------------- | +| `organization` | the name the managed policy shows in T3's settings | +| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | +| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | +| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | +| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | +| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`, one field per dotted path: the child ceiling (`agentSlots.…`) | +| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | +| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default | +| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | + +Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, +`usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's +(`apps/server/src/policy/managedSettings.ts`), and a test proves it for the fork's keys. Lock one field per dotted +path (`agentSlots.maxPerMachine`): T3 takes an object only as a setting's complete value and refuses to start on a +policy it cannot apply, so the kit refuses object values. A field inside a setting that can be empty must be locked +as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and does not start. + +## Limits + +- **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. + The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from + the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. + Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. +- **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's + and Anthropic's addresses are many and change). Add an outbound proxy if that matters. +- **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not + run; the distro is the sandbox, so use full access. +- **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, + but the code files are not hashed. + +## Tests + +``` +python3 -I -B fork-kit/pc-agents/tests/check.test.py +``` + +A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and +`systemctl`: every wall passes on a walled distro, and each one fails on its own when its part is taken away. +`apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py new file mode 100644 index 0000000000000..29000cb399cb1 --- /dev/null +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -0,0 +1,219 @@ +"""pc-agents: the facts, the files this kit owns and the checks, shared by `pc-agents setup` and `pc-agents check`. + +Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where +PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" +import hashlib, ipaddress, json, os, pwd, re, subprocess + +import machine_file + +KIT = 'pc-agents' +ROOT = os.environ.get('PC_AGENTS_ROOT', '') +if ROOT and os.geteuid() == 0: + raise SystemExit('pc-agents: PC_AGENTS_ROOT is for the tests only, never for root; unset it') +OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes +HOME, USER = '/home/agent', 'agent' +OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' +RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote +UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' +CHECK = OPT + '/current/sbin/pc-agents' +WALLS_PY = OPT + '/current/walls/fork-fleet-walls.py' +# Walls of fork-fleet-walls/wsl2 that are about the fleet launcher's job units. Here the job unit is T3's own unit, +# whose sandbox this kit writes and checks itself (the unit file must be exactly the kit's, with no drop-in). +UNIT_WALLS = {'mounts', 'sockets', 'userns'} +MASKS = ['/etc/binfmt.d/WSLInterop.conf'] + ['/etc/systemd/system/' + u for u in + ('cron.service', 'atd.service', 'ssh.service', 'ssh.socket')] +UNIT_DIRS = ('/etc/systemd/system', '/run/systemd/system', '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', + '/lib/systemd/system', '/run/systemd/transient', '/run/systemd/generator.early') +BLOCKED4 = ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', + '224.0.0.0/4', '240.0.0.0/4') +BLOCKED6 = ('fc00::/7', 'fe80::/10', 'ff00::/8') +MODES = ('approval-required', 'auto-accept-edits', 'auto', 'full-access') +KIT_SETTINGS = {'linkedEnvironmentsEnabled': True, 'automaticUpdates': False} +VERSION = re.compile(r'(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?') +HEAD = '# pc-agents kit: written by `pc-agents setup`; an edit is refused at its next run and fails the walls check\n' + +# What the kit runs from a release: source path -> (path in the version folder, mode). The walls are the merged +# fleet walls of the same release, not a copy: fork-fleet-walls.py reads its folder and the launcher beside it. +CODE = {'fork-kit/pc-agents/sbin/pc-agents': ('sbin/pc-agents', 0o755), + 'fork-kit/pc-agents/lib/pc_agents.py': ('lib/pc_agents.py', 0o644), + 'fork-kit/common/machine_file.py': ('lib/machine_file.py', 0o644), + '.github/scripts/fork-fleet-walls.py': ('walls/fork-fleet-walls.py', 0o755), + '.github/scripts/fork-fleet-launch.sh': ('walls/fork-fleet-launch.sh', 0o755), + '.github/scripts/fork-fleet-job-paths.txt': ('walls/fork-fleet-job-paths.txt', 0o644), + 'scripts/install.sh': ('share/install.sh', 0o644)} +for _w, _m in (('linux', 0o755), ('wsl2', 0o755), ('linux.fixture.json', 0o644), ('wsl2.fixture.json', 0o644)): + CODE['.github/scripts/fork-fleet-walls/' + _w] = ('walls/fork-fleet-walls/' + _w, _m) + + +def is_ip_list(value, net, glob=False, most=64): + try: + ips = [(ipaddress.ip_network if net else ipaddress.ip_address)(v) for v in value] + except (TypeError, ValueError) as e: + return f'not a list of IP {"networks" if net else "addresses"} ({e})' + if not 0 < len(ips) <= most and not (net and not ips): + return f'1 to {most} entries' + return 'only public addresses (a private one is blocked)' if glob and not all(i.is_global for i in ips) else None + + +SPEC = { + 'organization': (str, machine_file.REQUIRED, lambda v: None if 0 < len(v) <= 80 else '1 to 80 characters'), + 't3Version': (str, machine_file.REQUIRED, lambda v: None if VERSION.fullmatch(v) else 'an exact version, no v'), + 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, + lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), + 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), + 'agentUid': (int, 1100, lambda v: None if 1000 <= v < 60000 else '1000 to 59999'), + 'maxRuntimeMode': (str, machine_file.REQUIRED, lambda v: None if v in MODES else 'one of ' + ', '.join(MODES)), + 'lockedSettings': (dict, {}, lambda v: next((f'{k}: set by the kit itself, or not a settings path' for k in v + if k in KIT_SETTINGS or not re.fullmatch(r'[a-z]\w*(\.\w+)*', k)), None) + or next((f'{k}: lock one field per dotted path ({k}.field), not an object (T3 takes an object ' + 'only as the whole setting and does not start on a policy it cannot apply)' + for k, x in v.items() if isinstance(x, dict)), None)), + 'memoryMax': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*[KMGT]', v) else 'like 8G'), + 'cpuQuota': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*%', v) else 'like 400%'), + 'nameservers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, False, True, 3)), + 'allow': (list, [], lambda v: is_ip_list(v, True)), + 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), +} + + +def config(): + section = machine_file.read_section(KIT, ROOT + machine_file.default_path() if ROOT else None, OWNER) + return machine_file.fields(section, SPEC, f'{machine_file.default_path()}: {KIT}') + + +def runtime(c): + return f'{HOME}/.t3/runtime/versions/{c["t3Version"]}/t3' + + +def render(c): + """{path: (text, mode)} of every file the kit owns, from the config alone (the same every run).""" + settings = {**KIT_SETTINGS, **c['lockedSettings']} + policy = {'schemaVersion': 1, 'organization': c['organization'], + 'settings': {k: {'value': v, 'locked': True} for k, v in settings.items()}, + 'limits': {'maxRuntimeMode': c['maxRuntimeMode'], 'connect': False, 'webhookTasks': False, + 'browserCookieImport': False, 'updates': {'mode': 'off'}}} + allow4 = [a for a in c['allow'] if ':' not in a] + allow6 = [a for a in c['allow'] if ':' in a] + nft = HEAD + 'table inet pc_agents\ndelete table inet pc_agents\ntable inet pc_agents {\n\tchain agent_out {\n' + \ + '\t\ttype filter hook output priority filter; policy accept;\n' + \ + f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n\t\toifname "lo" accept\n' + \ + (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ + (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ + f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' + unit = HEAD + f'''[Unit] +Description=T3 Code for agents, walled (pc-agents kit) +Wants=network-online.target +After=network-online.target + +[Service] +User={USER} +Group={USER} +Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true +Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin +ExecStartPre=+/usr/sbin/nft -f {STATE}/firewall.nft +ExecStartPre=+{CHECK} check +ExecStart={runtime(c)} serve +Restart=on-failure +RestartSec=30 +UMask=0077 +MemoryMax={c["memoryMax"]} +CPUQuota={c["cpuQuota"]} +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=read-only +ReadWritePaths={HOME} +PrivateTmp=yes +PrivateDevices=yes +PrivateIPC=yes +ProtectProc=invisible +InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS +SystemCallArchitectures=native +RestrictNamespaces=yes +RestrictSUIDSGID=yes +RestrictRealtime=yes +LockPersonality=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectKernelLogs=yes +ProtectControlGroups=yes +ProtectClock=yes +ProtectHostname=yes + +[Install] +WantedBy=multi-user.target +''' + wsl = HEAD + ('[boot]\nsystemd=true\n[automount]\nenabled=false\nmountFsTab=false\n[interop]\nenabled=false\n' + f'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault={USER}\n') + sd = '/etc/systemd/system/' + return { + '/etc/wsl.conf': (wsl, 0o644), + '/etc/resolv.conf': (HEAD + ''.join(f'nameserver {n}\n' for n in c['nameservers']), 0o644), + '/etc/t3code/policy.json': (json.dumps(policy, indent=2, sort_keys=True) + '\n', 0o644), + STATE + '/firewall.nft': (nft, 0o644), + sd + UNIT: (unit, 0o644), + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents daily walls check\n\n[Service]\nType=oneshot\n' + f'ExecStart={CHECK} check --enforce\n', 0o644), + sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents daily walls check\n\n[Timer]\nOnCalendar=daily\n' + 'Persistent=true\n\n[Install]\nWantedBy=timers.target\n', 0o644), + } + + +def sha(text): + return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() + + +def record(): + try: + with open(ROOT + RECORD, encoding='utf-8') as f: + return dict(reversed(line.rstrip('\n').split('\t', 1)) for line in f if '\t' in line) + except FileNotFoundError: + return {} + + +def lstat(path): + try: + return os.lstat(ROOT + path) + except FileNotFoundError: + return None + + +def safe(path, st, kind): + """Why path (stat st) is not what kind says (S_ISREG etc.), owned by OWNER, writable by it alone; or None.""" + if st is None: + return f'{path} is missing' + if not kind(st.st_mode) or st.st_uid != OWNER or st.st_mode & 0o022: + return f'{path} is not owned by uid {OWNER} and writable by it alone (or is the wrong kind of file)' + return None + + +def users(): + """{name: (uid, gid)} of the passwd file, and {group: [members]}.""" + def rows(path): + try: + with open(ROOT + path, encoding='utf-8') as f: + return [line.rstrip('\n').split(':') for line in f if line.count(':') >= 3] + except FileNotFoundError: + return [] + return ({r[0]: (int(r[2]), int(r[3])) for r in rows('/etc/passwd') if r[2].isdigit() and r[3].isdigit()}, + {r[0]: [m for m in r[3].split(',') if m] for r in rows('/etc/group')}) + + +def user_problem(c, need_agent): + """Why this distro is someone else's (another login user, an agent that is not the kit's), or None.""" + people, groups = users() + others = sorted(n for n, (uid, _) in people.items() if 1000 <= uid < 60000 and n != USER) + if others: + return f'this distro already has login users ({", ".join(others)}): use a fresh distro for pc-agents' + if USER not in people: + return f'no user {USER}' if need_agent else None + if people[USER][0] != c['agentUid']: + return f'user {USER} has uid {people[USER][0]}, not {c["agentUid"]}: not the kit\'s' + extra = sorted(g for g, members in groups.items() if USER in members) + return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None + + +def run(argv, **kw): + return subprocess.run([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=120, **kw) diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents new file mode 100755 index 0000000000000..449635b942ef4 --- /dev/null +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -0,0 +1,115 @@ +#!/usr/bin/python3 -I +"""pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). + + pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also + stops T3 when one fails (the daily timer). T3's unit runs it before + every start, so T3 never starts on a failed wall. +Standard library only.""" +import os, re, stat, sys + +HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +COMMON = os.path.join(os.path.dirname(HERE), 'common') +sys.path[:0] = [os.path.join(HERE, 'lib')] + ([COMMON] if os.path.isdir(COMMON) else []) +sys.dont_write_bytecode = True +import pc_agents as k # noqa: E402 + + +def say(text): + print(f'pc-agents: {text}', flush=True) + + +def own_walls(c): + """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2.""" + out, done = [], k.record() + for path, (text, mode) in k.render(c).items(): + st = k.lstat(path) + why = k.safe(path, st, stat.S_ISREG) + if why is None: + with open(k.ROOT + path, 'rb') as f: + why = None if f.read() == text.encode() and done.get(path) == k.sha(text) else \ + f'{path} differs from what the kit writes (pc-agents setup)' + out.append(('file:' + os.path.basename(path), f'{path} is the kit\'s, root-owned', why)) + masks = [p for p in k.MASKS if not (k.lstat(p) and os.path.islink(k.ROOT + p) + and os.readlink(k.ROOT + p) == '/dev/null')] + out.append(('masks', 'WSL interop, cron, at and ssh are masked', masks and 'not masked: ' + ', '.join(masks))) + extra = [f'{d}/{n}' for d in k.UNIT_DIRS for n in (k.UNIT + '.d', k.WALLS + '.service.d') + ( + ('service.d',) if d in k.UNIT_DIRS[:2] else ()) + ((k.UNIT,) if d != k.UNIT_DIRS[0] else ()) + if k.lstat(f'{d}/{n}')] + out.append(('drop-ins', 'nothing overrides the unit', extra and 'found ' + ', '.join(extra))) + out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) + code = [] + for top, dirs, names in os.walk(k.ROOT + k.OPT + '/current/'): + for n in dirs + names: + p = os.path.join(top, n)[len(k.ROOT):] + code += [why] if (why := k.safe(p, k.lstat(p), lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] + current = k.lstat(k.OPT + '/current') + code += [] if current and current.st_uid == k.OWNER else [f'{k.OPT}/current is not the kit\'s'] + out.append(('kit-code', f'{k.OPT}/current is root-owned', code[:3] and '; '.join(code[:3]))) + mounts = [] + try: + with open(k.ROOT + '/proc/self/mountinfo', encoding='utf-8', errors='replace') as f: + for line in f: + fields = line.split(' - ') + where, fstype = fields[0].split()[4], (fields[1].split() or [''])[0] if len(fields) > 1 else '' + if fstype in ('9p', 'drvfs', 'virtiofs') and not where.startswith('/usr/lib/wsl'): + mounts.append(f'{where} ({fstype})') + except (OSError, IndexError) as e: + mounts.append(f'cannot read /proc/self/mountinfo ({e})') + out.append(('automount', 'no Windows drive is mounted', mounts and 'mounted: ' + ', '.join(mounts[:3]))) + r = k.run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) + nft_ok = r.returncode == 0 and f'skuid != {c["agentUid"]}' in r.stdout and ' drop' in r.stdout + out.append(('firewall', 'the agent firewall is loaded', None if nft_ok else 'table inet pc_agents is not loaded')) + gateway = None + try: + with open(k.ROOT + '/proc/net/route', encoding='utf-8') as f: + row = next((line.split() for line in f if line.split()[1:2] == ['00000000']), None) + gateway = row and k.ipaddress.ip_address(bytes.fromhex(row[2])[::-1]) + except (OSError, ValueError) as e: + gateway = f'unknown ({e})' + blocked = isinstance(gateway, k.ipaddress.IPv4Address) and \ + any(gateway in k.ipaddress.ip_network(n) for n in k.BLOCKED4) + out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range', + None if blocked else f'the gateway is {gateway}, not in a blocked range')) + return out + + +def wsl2_walls(): + """(wall, what, why or None) of the fleet's wsl2 walls (fork-fleet-walls.py, its contract enforced there).""" + env = {'PATH': '/usr/sbin:/usr/bin:/sbin:/bin', 'LC_ALL': 'C'} + if k.ROOT: + env['FORK_FLEET_WALLS_ROOT'] = k.ROOT + try: + p = k.subprocess.run([k.ROOT + k.WALLS_PY, 'describe'], env=env, stdin=k.subprocess.DEVNULL, + capture_output=True, text=True, timeout=120, cwd='/') + except (OSError, k.subprocess.TimeoutExpired) as e: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'cannot run them ({e})')] + lines = [m.groups() for m in re.finditer(r'^(PASS|FAIL) (\S+) (.*)$', p.stdout, re.M)] + if p.returncode not in (0, 1) or not lines: + return [('wsl2', 'the fleet\'s wsl2 walls ran', f'exit {p.returncode}: {p.stderr.strip()[-300:]}')] + return [(f'wsl2:{w}', what, None if mark == 'PASS' else what) for mark, w, what in lines if w not in k.UNIT_WALLS] + + +def check(enforce): + try: + c = k.config() + results = wsl2_walls() + own_walls(c) + except Exception as e: # a wall that cannot be told does not hold + results = [('check', 'the walls can be read', f'{e.__class__.__name__}: {e}')] + for wall, what, why in results: + print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) + failed = any(why for _, _, why in results) + if failed and enforce: + k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) + say(f'a wall does not hold: stopped {k.UNIT}') + return 1 if failed else 0 + + +def main(argv): + if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): + return check(argv[1:] == ['--enforce']) + print(__doc__, file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py new file mode 100644 index 0000000000000..03fc4bb708724 --- /dev/null +++ b/fork-kit/pc-agents/tests/check.test.py @@ -0,0 +1,76 @@ +"""pc-agents check on a fake WSL2 distro (fake_distro.py): every wall passes on a walled distro, and each one fails +on its own when its part of the wall is taken away; --enforce stops T3. + + python3 -I -B fork-kit/pc-agents/tests/check.test.py +""" +import os, shutil, sys, unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fake_distro import Case, DRVFS, SHA1, k # noqa: E402 + + +class Walls(Case): + def setUp(self): + super().setUp() + self.walled() + + def fails(self, wall, *also): + """check fails on wall, and on no other wall but those in also (which the same change takes away).""" + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertEqual({line.split()[1] for line in out.splitlines() if line.startswith('FAIL')}, {wall, *also}, out) + + def test_all_hold(self): + code, out = self.check() + self.assertEqual(code, 0, out) + walls = [line.split()[1] for line in out.splitlines()] + for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', + 'drop-ins', 'agent', 'kit-code', 'automount', 'firewall', 'gateway', 'file:policy.json', + 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf'): + self.assertIn(wall, walls) + for wall in k.UNIT_WALLS: # the launcher's job units: not T3's unit + self.assertNotIn('wsl2:' + wall, walls) + + def test_enforce_stops_t3(self): + os.unlink(self.path('/log/nft-loaded')) + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stop t3code-agents.service', self.log()) + + def test_each_wall(self): + def edit(p, old, new): + return lambda: self.write(p, self.read(p).replace(old, new)) + cases = { + 'file:policy.json': edit('/etc/t3code/policy.json', '"full-access"', '"auto"'), + 'file:t3code-agents.service': lambda: os.chmod(self.path('/etc/systemd/system/t3code-agents.service'), + 0o666), + 'file:resolv.conf': lambda: os.unlink(self.path('/etc/resolv.conf')), + 'masks': lambda: os.unlink(self.path('/etc/systemd/system/ssh.socket')), + 'drop-ins': lambda: os.makedirs(self.path('/etc/systemd/system/t3code-agents.service.d')), + 'agent': edit('/etc/group', 'sudo:x:27:', 'sudo:x:27:agent'), + 'kit-code': lambda: os.chmod(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py'), 0o666), + 'automount': lambda: self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + DRVFS), + 'firewall': lambda: os.unlink(self.path('/log/nft-loaded')), + 'gateway': lambda: self.write('/proc/net/route', 'Iface\tDestination\tGateway\nx\t00000000\t08080808\n'), + 'wsl2:interop': lambda: self.write('/proc/sys/fs/binfmt_misc/WSLInterop', + 'enabled\ninterpreter /init\nflags: PF\noffset 0\nmagic 4d5a\n'), + 'wsl2:networking': edit('/usr/bin/wslinfo', 'echo nat', 'echo mirrored'), + } + clean = {p: v[4] for p, v in self.snapshot().items()} + for wall, change in cases.items(): + with self.subTest(wall=wall): + shutil.rmtree(self.root) + self.setUp() + self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, clean) + change() + self.fails(wall, *(['wsl2:resolv'] if wall == 'file:resolv.conf' else [])) + + def test_no_config_fails(self): + os.unlink(self.path('/etc/fork-kit/machine.json')) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py new file mode 100644 index 0000000000000..abe99cb8ce8ea --- /dev/null +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -0,0 +1,157 @@ +"""A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the +current user as root, fake useradd, runuser, systemctl and nft (each logs to /log in the fake root). No root, no +systemd, no network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest + +KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +REPO = os.path.dirname(os.path.dirname(KIT)) +WSL2 = os.path.join(REPO, '.github', 'scripts', 'fork-fleet-walls', 'wsl2.fixture.json') +SHA1, SHA2 = 'a' * 40, 'b' * 40 +CONFIG = {'organization': 'Example lab', 't3Version': '1.2.3', 't3ReleaseBaseUrl': 'https://example.com/releases', + 'port': 3800, 'maxRuntimeMode': 'full-access', 'lockedSettings': {'agentSlots.maxPerMachine': 2}, + 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'tailscaleTag': 'tag:pc-agents'} +TOOLS = { # fake programs, under the fake root; each finds the root from its own path + 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' + 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' + 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', + 'usr/sbin/runuser': 'R="${0%/usr/sbin/runuser}"; echo "$*" >>"$R/log/runuser"\n[ -e "$R/log/runuser-fail" ] && exit 1\n' + 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' + 'd="$R/home/agent/.t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n" >"$d/t3"\n', + 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"\n', + 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' + 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', +} +DRVFS = '43 29 0:43 / /mnt/c rw,noatime - 9p drvfs rw\n44 29 0:44 / /c rw,noatime - 9p drvfs rw\n' + + +def blob(path): + with open(path, 'rb') as f: + return f.read() + + +def load(): + os.environ['PC_AGENTS_ROOT'] = '/nonexistent' + loader = importlib.machinery.SourceFileLoader('pc_agents_cli', os.path.join(KIT, 'sbin', 'pc-agents')) + mod = importlib.util.module_from_spec(importlib.util.spec_from_loader('pc_agents_cli', loader)) + loader.exec_module(mod) + return mod + + +cli = load() +k = cli.k + + +class Case(unittest.TestCase): + def setUp(self): + self.umask = os.umask(0o022) + self.root = os.path.realpath(tempfile.mkdtemp(prefix='pc-agents-test-')) + os.chmod(self.root, 0o755) + self.addCleanup(self.close) + k.ROOT, k.OWNER = self.root, os.geteuid() + with open(WSL2, encoding='utf-8') as f: + files = json.load(f)['pass'] + for rel in ('etc/wsl.conf', 'etc/resolv.conf', 'etc/binfmt.d/WSLInterop.conf'): + del files[rel] # a fresh distro: the kit writes these + files['etc/wsl.conf'] = '[boot]\nsystemd=true\n' + files['etc/passwd'] = 'root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534::/nonexistent:/usr/sbin/nologin\n' + files['etc/group'] = 'root:x:0:\nsudo:x:27:\nnogroup:x:65534:\n' + files['proc/net/route'] = 'Iface\tDestination\tGateway\tFlags\nboth\t00000000\t011014AC\t0003\n' + files['log/'] = '' + files.update({rel: {'text': '#!/bin/sh\n' + text, 'mode': '755'} for rel, text in TOOLS.items()}) + for rel, value in files.items(): + self.write('/' + rel, value) + os.symlink('/mnt/wsl/resolv.conf', self.path('/etc/resolv.conf')) + self.machine(CONFIG) + + def close(self): + os.umask(self.umask) + shutil.rmtree(self.root, ignore_errors=True) + + def path(self, p): + return self.root + p + + def write(self, p, value): + full = self.path(p) + if os.path.islink(full): + os.unlink(full) + if p.endswith('/'): + return os.makedirs(full, exist_ok=True) + os.makedirs(os.path.dirname(full), exist_ok=True) + text, mode = (value['text'], int(value['mode'], 8)) if isinstance(value, dict) else (value, 0o644) + with open(full, 'w', encoding='utf-8') as f: + f.write(text) + os.chmod(full, mode) + + def read(self, p): + with open(self.path(p), encoding='utf-8') as f: + return f.read() + + def machine(self, section): + self.write('/etc/fork-kit/machine.json', json.dumps({'version': 1, 'pc-agents': section})) + + def terminate(self): + """What `wsl --terminate` does once wsl.conf turned automount off: Windows' drives are no longer mounted.""" + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo').replace(DRVFS, '')) + + def call(self, fn, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = fn(*argv) + return code, out.getvalue() + + def setup_run(self, sha=SHA1): + return self.call(cli.main, ['setup', '--source', REPO, '--sha', sha]) + + def check(self, *flags): + return self.call(cli.main, ['check', *flags]) + + def log(self, name='systemctl'): + try: + return self.read('/log/' + name).splitlines() + except FileNotFoundError: + return [] + + def ready(self): + """A distro set up all the way: first run, `wsl --terminate`, second run.""" + code, out = self.setup_run() + self.assertEqual(code, 3, out) + self.terminate() + code, out = self.setup_run() + self.assertEqual(code, 0, out) + + def walled(self, sha=SHA1): + """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" + files = k.render(k.config()) + for p, (text, mode) in files.items(): + self.write(p, {'text': text, 'mode': f'{mode:o}'}) + self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items()))) + for p in k.MASKS: + os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) + os.symlink('/dev/null', self.path(p)) + with open(self.path('/etc/passwd'), 'a') as f: + f.write('agent:x:1100:1100::/home/agent:/bin/bash\n') + with open(self.path('/etc/group'), 'a') as f: + f.write('agent:x:1100:\n') + for src, (dst, mode) in k.CODE.items(): + with open(os.path.join(REPO, src), encoding='utf-8') as f: + self.write(f'/opt/pc-agents/{sha}/{dst}', {'text': f.read(), 'mode': f'{mode:o}'}) + os.symlink(sha, self.path('/opt/pc-agents/current')) + shutil.copy(self.path(k.STATE + '/firewall.nft'), self.path('/log/nft-loaded')) + self.terminate() + + def snapshot(self): + """Every path under the fake root but the fake tools' logs: kind, mode, owner, mtime, inode, link, content.""" + out = {} + for top, dirs, names in os.walk(self.root): + for n in dirs + names: + full = os.path.join(top, n) + rel = full[len(self.root):] + if rel.startswith('/log'): + continue + st = os.lstat(full) + body = os.readlink(full) if stat.S_ISLNK(st.st_mode) else \ + blob(full) if stat.S_ISREG(st.st_mode) else None + out[rel] = (st.st_mode, st.st_uid, st.st_mtime_ns, st.st_ino, body) + return out + + From 9d46a340200f0ffddf21f3a233f94f5851c68cab Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:56:41 +0000 Subject: [PATCH 02/10] fork-kit: pc-agents setup, bootstrap and owner steps (part 2 of 2) `pc-agents setup` makes a fresh WSL2 distro into the wall: refuses a distro that is not fresh with nothing changed, adds the agent user, installs the kit's release under /opt/pc-agents/, writes and records every file atomically, masks interop/cron/at/ssh, installs the configured T3 version for the agent, loads the firewall, and enables T3 only after every wall holds. A second run changes nothing; a run stopped at any write finishes on the next. The bootstrap fetches a ci-release commit; the README has the owner steps, the Tailscale tag and ACL, and the state answers. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + fork-kit/README.md | 11 +- fork-kit/common/machine.example.json | 12 ++ fork-kit/pc-agents/README.md | 76 ++++++++- fork-kit/pc-agents/bootstrap.sh | 51 ++++++ fork-kit/pc-agents/sbin/pc-agents | 120 ++++++++++++++ fork-kit/pc-agents/tests/setup.test.py | 214 +++++++++++++++++++++++++ 7 files changed, 474 insertions(+), 11 deletions(-) create mode 100755 fork-kit/pc-agents/bootstrap.sh create mode 100644 fork-kit/pc-agents/tests/setup.test.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 0e2b9a7491353..d1480724893b1 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -57,6 +57,7 @@ fork-kit/heavy/tests/sweep.test.py # A real systemd user manager (the step starts one for the runner user). fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml fork-kit/pc-agents/tests/check.test.py +fork-kit/pc-agents/tests/setup.test.py # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml diff --git a/fork-kit/README.md b/fork-kit/README.md index eac02830f821d..17fbea35fff66 100644 --- a/fork-kit/README.md +++ b/fork-kit/README.md @@ -13,11 +13,12 @@ Every kit follows the same pattern: - safe to run again, and safe to stop at any step; - an uninstall that leaves nothing. -| Kit | What | -| ---------------------------- | ------------------------------------------------------------------------------------------------------- | -| [gh-apps](gh-apps/README.md) | GitHub App tokens for agents through `gh-as`, without giving agents the keys (Linux) | -| [golive](golive/README.md) | Move a T3 boot service to a new build with a cold copy first; core and `check` so far (Linux) | -| [heavy](heavy/README.md) | `heavy `: run builds and tests in a capped slice, stop what they leave behind (Linux, systemd) | +| Kit | What | +| -------------------------------- | ------------------------------------------------------------------------------------------------------- | +| [gh-apps](gh-apps/README.md) | GitHub App tokens for agents through `gh-as`, without giving agents the keys (Linux) | +| [golive](golive/README.md) | Move a T3 boot service to a new build with a cold copy first; core and `check` so far (Linux) | +| [heavy](heavy/README.md) | `heavy `: run builds and tests in a capped slice, stop what they leave behind (Linux, systemd) | +| [pc-agents](pc-agents/README.md) | Run T3's agents on a Windows PC in a walled WSL2 distro, checked before every start (WSL2) | [common](common/README.md) is code the kits share, copied into each kit's version folder: the loader of the one private machine file (`/etc/fork-kit/machine.json`) where the facts of one machine live. diff --git a/fork-kit/common/machine.example.json b/fork-kit/common/machine.example.json index f2173a8f51d48..f287401f037bc 100644 --- a/fork-kit/common/machine.example.json +++ b/fork-kit/common/machine.example.json @@ -25,5 +25,17 @@ "backupDir": "/srv/example/golive/backups", "stateDir": "/srv/example/golive/state", "freeFloorMb": 2048 + }, + "pc-agents": { + "organization": "Example home lab", + "t3Version": "0.0.30", + "t3ReleaseBaseUrl": "https://github.com/example/t3code/releases/download", + "port": 3773, + "maxRuntimeMode": "full-access", + "lockedSettings": { "agentSlots.enabled": true, "agentSlots.maxPerMachine": 2 }, + "memoryMax": "8G", + "cpuQuota": "400%", + "nameservers": ["1.1.1.1", "9.9.9.9"], + "tailscaleTag": "tag:pc-agents" } } diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index b825f0888c34a..a8b949b8777f2 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -23,8 +23,7 @@ default gateway (the Windows host) is in a blocked range. The fleet's `mounts`, about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must be the kit's file, byte for byte, with no drop-in). -Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. This folder has the walls and their -check so far; `pc-agents setup`, which writes them into a fresh distro, and its owner steps are still to come. +Linux (WSL2, systemd, Debian or Ubuntu) only. Python 3 standard library only. ## Config @@ -70,6 +69,67 @@ path (`agentSlots.maxPerMachine`): T3 takes an object only as a setting's comple policy it cannot apply, so the kit refuses object values. A field inside a setting that can be empty must be locked as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and does not start. +## Owner steps (once per PC) + +Commands are for Windows PowerShell unless a step says otherwise. `` is a name of your choice (for example +`t3-agents`); `` is the fork's `OWNER/NAME`, `` a commit of its `ci-release` branch. + +1. **Make a fresh distro** (not the CI one, not your own): `wsl --install -d Ubuntu-24.04 --name --no-launch`. +2. **Write the machine file** inside it as root: `wsl -d -u root`, then + `mkdir -p /etc/fork-kit && nano /etc/fork-kit/machine.json` (the example above, your values), and leave with `exit`. +3. **Run the setup** as root: + `wsl -d -u root -- sh -c 'curl -fsSL https://raw.githubusercontent.com///fork-kit/pc-agents/bootstrap.sh -o /root/pc-agents-bootstrap.sh && sh /root/pc-agents-bootstrap.sh --repo --sha '`. + The first run writes everything, then says a wall does not hold (Windows' drives are still mounted) and does + not start T3. +4. **Restart the distro** so `wsl.conf` applies: `wsl --terminate `. +5. **Run the setup again** (the same command as step 3). Now every wall holds and T3 starts. It prints the next step. +6. **Join Tailscale** with the tag (as root in the distro): install it + (`curl -fsSL https://tailscale.com/install.sh | sh`), then `tailscale up --advertise-tags=` and open the + link it prints in a browser on any machine. The ACL is below. +7. **Log in to Claude as the agent:** install Claude Code for `agent` and log in + (`wsl -d -u agent`, then the install command from Claude Code's docs, then `claude`); open the link it + prints in your browser. Codex: the same with `codex login`. +8. **Log in to GitHub as the agent:** install `gh` (as root, from GitHub CLI's docs), then + `wsl -d -u agent -- gh auth login --with-token` and paste a fine-grained token for the agents' repos into + that window only. +9. **Link from the other T3:** on this PC's T3 (`http://:`), Settings → Sharing & access → + make a pairing code. In the other T3: Settings → Environments → Linked environments → Link, the same address + and the code, **Full access**. + +Run steps 3 to 5 again for a new release or a changed machine file; T3 restarts only when something changed. To +remove everything: `wsl --unregister `. + +## Tailscale tag and ACL + +The PC's node is tagged, so it is not you: it gets no access of its own, only what the ACL grants. Grant one thing, +the linking T3 reaching T3's port here, and nothing from the PC: + +```json +{ + "tagOwners": { "tag:pc-agents": ["autogroup:admin"], "tag:orchestrator": ["autogroup:admin"] }, + "grants": [{ "src": ["tag:orchestrator"], "dst": ["tag:pc-agents"], "ip": ["tcp:3773"] }] +} +``` + +With no grant whose `src` is `tag:pc-agents`, the PC starts no connection into the tailnet; and the agent's uid cannot +reach any tailnet address anyway (100.64.0.0/10 and `fc00::/7` are dropped for it). Replies to the linking T3 are +allowed (connection tracking). + +## Runs again, stops half-way, meets old or empty state + +- **Again:** every file is compared before it is written; an unchanged file is not touched, and T3 restarts only + when a file, the release or the T3 version changed. The tests run the setup twice and find no change. +- **Stopped half-way:** each file is written whole (a new file, then one rename) and recorded at once in + `/etc/pc-agents/installed.tsv`. T3 is enabled only at the end, after every wall holds; a run that stops anywhere + (the tests stop it at every write) leaves T3 off, and the next run finishes the job. +- **A distro that is not fresh:** refused before anything is written, with the reasons: another login user, an + `agent` with another uid or in another group, a file the kit would write that it did not write (or that was + edited since, for example a hand-written `wsl.conf` or `policy.json`), a cron, at or ssh unit where the kit masks + one. A fresh distro's own `wsl.conf` (systemd only) and WSL's generated `resolv.conf` are replaced. +- **Old releases:** each release has its own folder under `/opt/pc-agents/`, switched by the `current` link; old ones + stay (small) until the distro is removed. +- **No or a broken machine file:** setup and check refuse, naming the key; the unit does not start. + ## Limits - **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. @@ -80,15 +140,19 @@ as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and and Anthropic's addresses are many and change). Add an outbound proxy if that matters. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. +- **Tailscale, Claude Code and `gh` are yours to install** (steps 6 to 8): their installers and logins need you. +- **No uninstall** but removing the distro; the distro holds nothing else. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, - but the code files are not hashed. + but the code files are not hashed. The bootstrap duplicates the heavy kit's (a shared bootstrap is a follow-up). ## Tests ``` +python3 -I -B fork-kit/pc-agents/tests/setup.test.py python3 -I -B fork-kit/pc-agents/tests/check.test.py ``` -A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and -`systemctl`: every wall passes on a walled distro, and each one fails on its own when its part is taken away. -`apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. +A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root), with fake `useradd`, `runuser`, `systemctl` +and `nft`: every file and mode, a second run changes nothing, a run stopped at every write finishes on the next, +each kind of foreign distro is refused, and each wall fails on its own. `apps/server/src/policy/managedSettings.forkKeys.test.ts` +proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/bootstrap.sh b/fork-kit/pc-agents/bootstrap.sh new file mode 100755 index 0000000000000..334edad3f1f04 --- /dev/null +++ b/fork-kit/pc-agents/bootstrap.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# pc-agents bootstrap: runs `pc-agents setup` from one release of this fork. As root, in a fresh WSL2 distro: +# +# sh pc-agents-bootstrap.sh --repo FORK_OWNER/FORK --sha RELEASE_SHA +# +# It refuses a sha that is not a commit on the fork's protected release branch (ci-release), downloads exactly that +# commit, prints the sha and the hash of what it downloaded, and runs its setup (safe to run again). The setup reads +# the "pc-agents" section of /etc/fork-kit/machine.json (fork-kit/pc-agents/README.md). Needs curl, tar, python3. +set -eu +umask 022 +PATH=/usr/sbin:/usr/bin:/sbin:/bin +export PATH +unset TMPDIR CURL_HOME SSL_CERT_FILE SSL_CERT_DIR PC_AGENTS_ROOT || true + +die() { echo "pc-agents-bootstrap: $*" >&2; exit 1; } + +repo='' sha='' branch='ci-release' api='https://api.github.com' codeload='https://codeload.github.com' +while [ $# -gt 0 ]; do + case "$1" in + --repo) repo="${2-}"; shift 2 ;; + --sha) sha="${2-}"; shift 2 ;; + *) die "unknown option $1 (see the top of this file)" ;; + esac +done +printf '%s\n' "$repo" | grep -Eqx '[A-Za-z0-9][A-Za-z0-9-]{0,38}/[A-Za-z0-9._-]{1,100}' || die "--repo must be OWNER/NAME" +printf '%s\n' "$sha" | grep -Eqx '[0-9a-f]{40}' || die "--sha must be the full 40-character release commit" +[ "$(id -u)" = 0 ] || die "run this as root" + +work="$(mktemp -d /root/pc-agents-bootstrap.XXXXXXXX)" +trap 'rm -rf "${work:?}"' EXIT +trap 'exit 1' INT TERM HUP +fetch() { curl -fsSL --proto '=https' --tlsv1.2 --max-redirs 0 -H 'Accept: application/vnd.github+json' -o "$2" "$1"; } + +fetch "$api/repos/$repo/branches/$branch" "$work/branch.json" || die "cannot read branch $branch of $repo" +fetch "$api/repos/$repo/compare/$sha...$branch" "$work/compare.json" || die "cannot compare $sha with $branch" +python3 -I -B - "$work/branch.json" "$work/compare.json" "$sha" <<'PY' || die "$sha is not a release: it must be a commit on the protected branch $branch" +import json, sys +branch, compare = (json.load(open(p, encoding='utf-8')) for p in sys.argv[1:3]) +ok = (branch.get('protected') is True and compare.get('status') in ('ahead', 'identical') + and (compare.get('merge_base_commit') or {}).get('sha') == sys.argv[3]) +sys.exit(0 if ok else 1) +PY + +fetch "$codeload/$repo/tar.gz/$sha" "$work/kit.tar.gz" || die "cannot download $repo at $sha" +mkdir "$work/src" +tar -xzf "$work/kit.tar.gz" -C "$work/src" --no-same-owner --no-same-permissions +top="$(find "$work/src" -mindepth 1 -maxdepth 1 -type d)" +[ "$(printf '%s\n' "$top" | wc -l)" = 1 ] && [ -n "$top" ] || die "the download does not have one top folder" +[ -f "$top/fork-kit/pc-agents/sbin/pc-agents" ] || die "release $sha has no fork-kit/pc-agents" +echo "pc-agents-bootstrap: release $sha of $repo, download sha256 $(sha256sum "$work/kit.tar.gz" | cut -d' ' -f1)" +python3 -I -B "$top/fork-kit/pc-agents/sbin/pc-agents" setup --source "$top" --sha "$sha" diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 449635b942ef4..151df2453808b 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -1,6 +1,7 @@ #!/usr/bin/python3 -I """pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). + pc-agents setup --source DIR --sha SHA as root, in a fresh distro: make or repair everything; safe to run again pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also stops T3 when one fails (the daily timer). T3's unit runs it before every start, so T3 never starts on a failed wall. @@ -11,6 +12,7 @@ HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) COMMON = os.path.join(os.path.dirname(HERE), 'common') sys.path[:0] = [os.path.join(HERE, 'lib')] + ([COMMON] if os.path.isdir(COMMON) else []) sys.dont_write_bytecode = True +import machine_file # noqa: E402 import pc_agents as k # noqa: E402 @@ -18,6 +20,122 @@ def say(text): print(f'pc-agents: {text}', flush=True) +def put(path, data, mode, link=False): + """Write a file (or a link) whole: a new file beside it, then one rename. Returns True when it changed.""" + real = k.ROOT + path + st = k.lstat(path) + if link and st and stat.S_ISLNK(st.st_mode) and os.readlink(real) == data: + return False + if not link and st and stat.S_ISREG(st.st_mode) and stat.S_IMODE(st.st_mode) == mode and st.st_uid == k.OWNER: + with open(real, 'rb') as f: + if f.read() == (data.encode() if isinstance(data, str) else data): + return False + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + tmp = f'{real}.pc-agents-new' + if os.path.lexists(tmp): + os.unlink(tmp) + if link: + os.symlink(data, tmp) + else: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as f: + f.write(data.encode() if isinstance(data, str) else data) + os.fchmod(f.fileno(), mode) + if not k.ROOT: + os.fchown(f.fileno(), 0, 0) + os.fsync(f.fileno()) + os.replace(tmp, real) + return True + + +def fresh(path, st): + """A file a fresh distro has where the kit writes its own: replaced without asking.""" + if path == '/etc/resolv.conf': + return stat.S_ISLNK(st.st_mode) or 'generated by WSL' in open(k.ROOT + path, errors='replace').read() + if path == '/etc/wsl.conf' and stat.S_ISREG(st.st_mode): + text = re.sub(r'(?m)^\s*[#;].*\n?|\s+', '', open(k.ROOT + path, errors='replace').read()).lower() + return text in ('', '[boot]systemd=true') + return False + + +def preflight(c, files): + """Every reason not to touch this distro, before anything is written.""" + bad = [why] if (why := k.user_problem(c, False)) else [] + done = k.record() + for path, (text, _) in files.items(): + st = k.lstat(path) + if st is None or fresh(path, st): + continue + if not stat.S_ISREG(st.st_mode): + bad.append(f'{path} is not a plain file') + continue + with open(k.ROOT + path, 'rb') as f: + now = k.sha(f.read()) + if now != k.sha(text) and done.get(path) != now: + bad.append(f'{path} was not written by this kit, or was edited since (move it away to go on)') + for path in k.MASKS: + st = k.lstat(path) + if st and not (stat.S_ISLNK(st.st_mode) and os.readlink(k.ROOT + path) == '/dev/null'): + bad.append(f'{path} exists and is not a mask (a link to /dev/null)') + return bad + + +def setup(source, sha): + if not re.fullmatch(r'[0-9a-f]{40}', sha) or not os.path.isdir(source): + return say('usage: pc-agents setup --source DIR --sha 40-HEX-SHA') or 2 + if not k.ROOT and os.geteuid() != 0: + return say('run this as root') or 2 + os.umask(0o022) + try: + c = k.config() + except machine_file.MachineFileError as e: + return say(f'{e} (fork-kit/pc-agents/README.md has an example)') or 2 + files = k.render(c) + bad = preflight(c, files) + if bad: + return say('refused, nothing changed:\n ' + '\n '.join(bad)) or 1 + if k.USER not in k.users()[0]: + r = k.run(['/usr/sbin/useradd', '--create-home', '--home-dir', k.HOME, '--shell', '/bin/bash', + '--uid', str(c['agentUid']), '--user-group', k.USER]) + if r.returncode: + return say(f'useradd failed: {r.stderr.strip()[-300:]}') or 1 + if k.lstat('/usr/sbin/nft') is None and k.run(['/usr/bin/apt-get', 'install', '-y', 'nftables']).returncode: + return say('cannot install nftables (apt-get install -y nftables)') or 1 + version = f'{k.OPT}/{sha}' + for src, (dst, mode) in k.CODE.items(): + with open(os.path.join(source, src), 'rb') as f: + put(f'{version}/{dst}', f.read(), mode) + changed = put(f'{k.OPT}/current', sha, None, link=True) + done = k.record() + for path, (text, mode) in files.items(): # each one recorded as soon as it is written: a stop leaves no stranger + changed |= put(path, text, mode) + done[path] = k.sha(text) + put(k.RECORD, ''.join(f'{h}\t{p}\n' for p, h in sorted(done.items())), 0o644) + for path in k.MASKS: + put(path, '/dev/null', None, link=True) + if k.lstat(k.runtime(c)) is None: + say(f'installing T3 {c["t3Version"]} for {k.USER}') + r = k.run(['/usr/sbin/runuser', '-u', k.USER, '--', '/usr/bin/env', '-i', f'HOME={k.HOME}', + 'PATH=/usr/bin:/bin', f'T3CODE_HOME={k.HOME}/.t3', f'T3CODE_VERSION={c["t3Version"]}', + f'T3CODE_RELEASE_BASE_URL={c["t3ReleaseBaseUrl"]}', '/bin/sh', f'{version}/share/install.sh']) + if r.returncode or k.lstat(k.runtime(c)) is None: + return say(f'T3 install failed: {(r.stderr or r.stdout).strip()[-400:]}') or 1 + k.run(['/usr/bin/systemctl', 'daemon-reload']) + k.run(['/usr/sbin/nft', '-f', k.STATE + '/firewall.nft']) # as T3's unit does before each start + units = [k.UNIT, k.WALLS + '.timer'] + if check(False) != 0: + k.run(['/usr/bin/systemctl', 'disable', '--now'] + units) + return say('a wall does not hold, so T3 is not enabled. After a first run: wsl --terminate ' + 'from Windows, then run setup again') or 3 + r = k.run(['/usr/bin/systemctl', 'enable', '--now'] + units) + if r.returncode == 0 and changed: + r = k.run(['/usr/bin/systemctl', 'restart', k.UNIT]) + if r.returncode: + return say(f'systemctl failed: {r.stderr.strip()[-300:]}') or 1 + say(f'done: T3 serves on port {c["port"]}. Next: tailscale up --advertise-tags={c["tailscaleTag"]}') + return 0 + + def own_walls(c): """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2.""" out, done = [], k.record() @@ -107,6 +225,8 @@ def check(enforce): def main(argv): if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): return check(argv[1:] == ['--enforce']) + if argv[:1] == ['setup'] and len(argv) == 5 and argv[1] == '--source' and argv[3] == '--sha': + return setup(argv[2], argv[4]) print(__doc__, file=sys.stderr) return 2 diff --git a/fork-kit/pc-agents/tests/setup.test.py b/fork-kit/pc-agents/tests/setup.test.py new file mode 100644 index 0000000000000..688dc8f16ae26 --- /dev/null +++ b/fork-kit/pc-agents/tests/setup.test.py @@ -0,0 +1,214 @@ +"""pc-agents setup on a fake WSL2 distro (fake_distro.py): every file and mode, T3 enabled only after every wall +holds, a second run changes nothing, a run stopped at any write finishes on the next, and a distro that is not +fresh is refused with nothing changed. + + python3 -I -B fork-kit/pc-agents/tests/setup.test.py +""" +import json, os, re, shutil, stat, sys, unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from fake_distro import CONFIG, KIT, REPO, SHA1, SHA2, Case, blob, cli, k # noqa: E402 + + +class Setup(Case): + def test_fresh_distro_serves_only_after_the_walls_hold(self): + code, out = self.setup_run() + self.assertEqual(code, 3, out) + self.assertIn('FAIL automount', out) + self.assertIn('wsl --terminate', out) + self.assertFalse([line for line in self.log() if line.startswith('enable')]) + self.assertIn('disable --now t3code-agents.service pc-agents-walls.timer', self.log()) + self.terminate() + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertNotIn('FAIL', out) + self.assertIn('enable --now t3code-agents.service pc-agents-walls.timer', self.log()) + self.assertIn('tailscale up --advertise-tags=tag:pc-agents', out) + + def test_every_file_and_permission(self): + self.ready() + c = k.config() + record = k.record() + for p, (text, mode) in k.render(c).items(): + st = os.lstat(self.path(p)) + self.assertTrue(stat.S_ISREG(st.st_mode), p) + self.assertEqual((stat.S_IMODE(st.st_mode), st.st_uid), (mode, os.geteuid()), p) + self.assertEqual(self.read(p), text, p) + self.assertEqual(record[p], k.sha(text), p) + for p in k.MASKS: + self.assertEqual(os.readlink(self.path(p)), '/dev/null', p) + self.assertEqual(os.readlink(self.path('/opt/pc-agents/current')), SHA1) + for src, (dst, mode) in k.CODE.items(): + p = self.path(f'/opt/pc-agents/{SHA1}/{dst}') + self.assertEqual(stat.S_IMODE(os.stat(p).st_mode), mode, dst) + self.assertEqual(blob(p), blob(os.path.join(REPO, src)), dst) + self.assertIn('agent:x:1100:1100::/home/agent:/bin/bash', self.read('/etc/passwd')) + self.assertTrue(os.path.isfile(self.path('/home/agent/.t3/runtime/versions/1.2.3/t3'))) + self.assertIn('T3CODE_VERSION=1.2.3', self.log('runuser')[0]) + self.assertIn('T3CODE_RELEASE_BASE_URL=https://example.com/releases', self.log('runuser')[0]) + + def test_what_the_files_say(self): + self.ready() + policy = json.loads(self.read('/etc/t3code/policy.json')) + self.assertEqual(policy['settings'], {'agentSlots.maxPerMachine': {'value': 2, 'locked': True}, + 'automaticUpdates': {'value': False, 'locked': True}, + 'linkedEnvironmentsEnabled': {'value': True, 'locked': True}}) + self.assertEqual(policy['limits']['maxRuntimeMode'], 'full-access') + self.assertEqual((policy['limits']['connect'], policy['limits']['webhookTasks']), (False, False)) + wsl = self.read('/etc/wsl.conf') + for line in ('[automount]\nenabled=false\nmountFsTab=false', '[interop]\nenabled=false\nappendWindowsPath=false', + '[boot]\nsystemd=true', 'generateResolvConf=false', 'default=agent'): + self.assertIn(line, wsl) + unit = self.read('/etc/systemd/system/t3code-agents.service') + for line in ('User=agent', 'ExecStartPre=+/opt/pc-agents/current/sbin/pc-agents check', 'ProtectSystem=strict', + 'InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix', 'MemoryMax=8G', 'CPUQuota=400%', + 'ExecStart=/home/agent/.t3/runtime/versions/1.2.3/t3 serve', 'T3CODE_PORT=3800', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK', 'SystemCallFilter=~io_uring_setup'): + self.assertIn(line, unit) + self.assertNotIn('AF_VSOCK', unit) + nft = self.read('/etc/pc-agents/firewall.nft') + for line in ('meta skuid != 1100 accept', '172.16.0.0/12', '10.0.0.0/8', '192.168.0.0/16', '100.64.0.0/10', + 'fc00::/7'): + self.assertIn(line, nft) + self.assertEqual(self.read('/etc/resolv.conf').splitlines()[1:], ['nameserver 1.1.1.1', 'nameserver 9.9.9.9']) + + def test_second_run_changes_nothing(self): + self.ready() + before = self.snapshot() + os.unlink(self.path('/log/systemctl')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertEqual(self.snapshot(), before) + self.assertFalse([line for line in self.log() if line.startswith('restart')]) + self.assertEqual(len(self.log('runuser')), 1) # T3 downloaded once + + def test_new_release_or_config_restarts_t3(self): + self.ready() + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run(SHA2)[0], 0) + self.assertEqual(os.readlink(self.path('/opt/pc-agents/current')), SHA2) + self.assertIn('restart t3code-agents.service', self.log()) + os.unlink(self.path('/log/systemctl')) + self.machine({**CONFIG, 'memoryMax': '4G'}) + self.assertEqual(self.setup_run(SHA2)[0], 0) + self.assertIn('MemoryMax=4G', self.read('/etc/systemd/system/t3code-agents.service')) + self.assertIn('restart t3code-agents.service', self.log()) + + def test_stopped_half_way_then_run_again(self): + self.terminate() + self.assertEqual(self.setup_run()[0], 0) + whole = {p: v[4] for p, v in self.snapshot().items()} + real_put, writes = cli.put, [] + + def counting(*a, **kw): + writes.append(a[0]) + return real_put(*a, **kw) + cli.put = counting + self.addCleanup(setattr, cli, 'put', real_put) + shutil.rmtree(self.root) + self.setUp() + self.terminate() + self.setup_run() + total = len(writes) + self.assertGreater(total, 20) + for stop in range(1, total + 1): + shutil.rmtree(self.root) + self.setUp() + self.terminate() + + def stopping(*a, n=[0], **kw): + n[0] += 1 + if n[0] == stop: + raise KeyboardInterrupt + return real_put(*a, **kw) + cli.put = stopping + with self.assertRaises(KeyboardInterrupt): + self.setup_run() + cli.put = real_put + self.assertFalse([line for line in self.log() if line.startswith('enable')], f'stopped at write {stop}') + code, out = self.setup_run() + self.assertEqual(code, 0, f'stopped at write {stop} ({writes[stop - 1]}): {out}') + self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, whole, f'stopped at write {stop}') + self.assertIn('enable --now t3code-agents.service pc-agents-walls.timer', self.log()) + + def test_t3_install_fails_then_works(self): + self.terminate() + self.write('/log/runuser-fail', '') + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('T3 install failed', out) + self.assertFalse([line for line in self.log() if line.startswith('enable')]) + os.unlink(self.path('/log/runuser-fail')) + self.assertEqual(self.setup_run()[0], 0) + + +class Refused(Case): + def refused(self, why): + before = self.snapshot() + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('refused, nothing changed', out) + self.assertIn(why, out) + self.assertEqual(self.snapshot(), before) + self.assertEqual(self.log(), []) + + def test_a_distro_with_its_own_user(self): + with open(self.path('/etc/passwd'), 'a') as f: + f.write('someone:x:1000:1000::/home/someone:/bin/bash\n') + self.refused('already has login users (someone)') + + def test_a_wsl_conf_of_its_own(self): + self.write('/etc/wsl.conf', '[boot]\nsystemd=true\n[user]\ndefault=someone\n') + self.refused('/etc/wsl.conf was not written by this kit') + + def test_a_policy_of_its_own(self): + self.write('/etc/t3code/policy.json', '{"schemaVersion": 1}\n') + self.refused('/etc/t3code/policy.json was not written by this kit') + + def test_an_agent_that_is_not_the_kits(self): + with open(self.path('/etc/passwd'), 'a') as f: + f.write('agent:x:1200:1200::/home/agent:/bin/bash\n') + self.refused('user agent has uid 1200, not 1100') + + def test_an_agent_with_sudo(self): + self.ready() + self.write('/etc/group', self.read('/etc/group').replace('sudo:x:27:', 'sudo:x:27:agent')) + os.unlink(self.path('/log/systemctl')) + self.refused('user agent is in other groups (sudo)') + + def test_an_unmasked_ssh(self): + self.write('/etc/systemd/system/ssh.service', '[Service]\nExecStart=/usr/sbin/sshd\n') + self.refused('ssh.service exists and is not a mask') + + def test_a_file_edited_after_setup(self): + self.ready() + with open(self.path('/etc/systemd/system/t3code-agents.service'), 'a') as f: + f.write('[Service]\nUser=root\n') + os.unlink(self.path('/log/systemctl')) + self.refused('t3code-agents.service was not written by this kit, or was edited since') + + def test_the_examples_are_valid_and_the_same(self): + with open(os.path.join(REPO, 'fork-kit', 'common', 'machine.example.json'), encoding='utf-8') as f: + example = json.load(f)['pc-agents'] + with open(os.path.join(KIT, 'README.md'), encoding='utf-8') as f: + readme = re.search(r'```json\n(\{\n "version".*?)\n```', f.read(), re.S).group(1) + self.assertEqual(json.loads(readme)['pc-agents'], example) + self.machine(example) + self.assertEqual(k.config()['agentUid'], 1100) + self.assertEqual(k.config()['lockedSettings'], {'agentSlots.enabled': True, 'agentSlots.maxPerMachine': 2}) + + def test_bad_config(self): + for change, why in (({'lockedSettings': {'linkedEnvironmentsEnabled': False}}, 'set by the kit itself'), + ({'lockedSettings': {'agentSlots': {'maxPerMachine': 2}}}, 'one field per dotted path'), + ({'nameservers': ['192.168.1.1']}, 'only public addresses'), + ({'t3Version': 'latest'}, 'an exact version'), + ({'maxRuntimeMode': 'root'}, 'one of')): + self.machine({**CONFIG, **change}) + code, out = self.setup_run() + self.assertEqual(code, 2, out) + self.assertIn(why, out) + self.assertFalse(os.path.lexists(self.path('/etc/t3code/policy.json'))) + + +if __name__ == '__main__': + unittest.main() From 364edb63fe5f325607d807f7a23c3b2aab2ab625 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:10:16 +0000 Subject: [PATCH 03/10] fork-kit: pc-agents fix round 1 (walls) - Agent code runs only in T3's unit: default distro user root, the agent's user manager masked, walls for the mask, linger and user units. - T3's runtime is root's under /opt/pc-agents/t3; the unit runs it from there; a wall checks its owner, mode and recorded hash. - A check that cannot run or crashes stops T3 (ExecStopPost on the walls service); a failed stop is reported. - Second network wall in the unit (IPAddressDeny/IPAddressAllow with the new peers key); the firewall wall compares the loaded table with the file; check timer every 2 minutes. - Units wall asks systemctl show; the file scan covers system.control, system.attached, generators, /run, /usr/lib and prefix drop-ins. - Gateway wall fails when allow or peers cover the gateway. - lockedSettings takes whole objects and record entries; blank organization refused; StartLimit for a policy T3 cannot apply. - run() kills the whole process group on a timeout. Co-Authored-By: Claude Opus 5.5 --- fork-kit/pc-agents/README.md | 70 ++++++--- fork-kit/pc-agents/lib/pc_agents.py | 154 +++++++++++++++--- fork-kit/pc-agents/sbin/pc-agents | 119 ++++++++++---- fork-kit/pc-agents/tests/check.test.py | 198 +++++++++++++++++++++++- fork-kit/pc-agents/tests/fake_distro.py | 59 +++++-- 5 files changed, 505 insertions(+), 95 deletions(-) diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index b825f0888c34a..c16182d0de384 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -5,21 +5,34 @@ T3's user, so the link is not the wall: the distro is. This kit makes a fresh WS T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, so the agent cannot undo it: -| Wall | What the kit does | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` | -| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | -| No way out of the user | one login user (`agent`), in no group but its own; cron, at and ssh masked | -| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | -| No LAN, no Windows host | nftables, for the agent's uid only: private, link-local, CGNAT (Tailscale) and multicast ranges dropped, so the Windows host (the NAT gateway), the home network and the tailnet are out of reach. Public addresses (GitHub, Anthropic, package mirrors) and replies to the linked T3 go through | -| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked | -| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and daily (a failed wall stops T3) | +| Wall | What the kit does | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the Windows host, the home network and the tailnet are out of reach; public addresses go through | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | `pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with -`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: every file above is root's, -mode 644 and exactly what the kit writes, nothing overrides the unit (no drop-in), the masks are in place, `agent` is -the only login user, the kit's code is root's, no Windows drive is mounted, the firewall table is loaded, and the -default gateway (the Windows host) is in a blocked range. The fleet's `mounts`, `sockets` and `userns` walls are +`fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: + +- every file above is root's, mode 644 and exactly what the kit writes (and recorded in the root-owned + `/etc/pc-agents/installed.tsv`); +- nothing overrides the kit's units: no drop-in or other copy in any folder systemd reads (`system.control`, + `system.attached`, the generators, `/run`, `/usr/lib`, prefix drop-ins like `t3code-.service.d`), and + `systemctl show` agrees (the kit's file, no drop-in, no pending reload, T3 and the timer enabled); +- the masks are in place, no user lingers, the agent has no systemd user units; +- `agent` is the only login user; +- the kit's code (`/opt/pc-agents`, the release folder it runs) and T3's runtime are root's, and T3's `t3` is the one + setup installed (its hash is in the record); +- no Windows drive is mounted; +- the loaded firewall table is the one from the kit's `firewall.nft`: `pc-agents load-firewall` saves the file's hash + and what the kernel then lists, and any later change, or an older file left loaded, differs; +- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through. + +The fleet's `mounts`, `sockets` and `userns` walls are about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must be the kit's file, byte for byte, with no drop-in). @@ -44,6 +57,7 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine "memoryMax": "8G", "cpuQuota": "400%", "nameservers": ["1.1.1.1", "9.9.9.9"], + "peers": ["100.101.102.103/32"], "tailscaleTag": "tag:pc-agents" } } @@ -57,31 +71,44 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine | `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | | `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | | `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | -| `lockedSettings` | more T3 settings to lock, `{path: value}`, one field per dotted path: the child ceiling (`agentSlots.…`) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | | `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | | `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | -| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | +| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | | `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, `usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's (`apps/server/src/policy/managedSettings.ts`), and a test proves it for the fork's keys. Lock one field per dotted -path (`agentSlots.maxPerMachine`): T3 takes an object only as a setting's complete value and refuses to start on a -policy it cannot apply, so the kit refuses object values. A field inside a setting that can be empty must be locked -as the whole setting; T3 says so in its log (`journalctl -u t3code-agents`) and does not start. +path (`agentSlots.maxPerMachine`), or give a whole setting (`agentSlots`) or one record entry +(`providerInstances.codex-work`) as an object holding every field: T3 takes an object only as a complete value. The kit +refuses what is cheap to tell (a field of a provider instance, an object deeper than a record entry, the kit's own +two settings, a blank `organization`); T3 refuses the rest, like a field inside a setting that can be empty (lock the +whole setting), says why in its log (`journalctl -u t3code-agents`) and does not start. T3 exits 1 on a policy it +cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); +it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). ## Limits - **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. - Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. + Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. Another + distro's root can drop or change the global nft table: the check sees it within 2 minutes and stops T3, and the + unit's own `IPAddressDeny=` (per cgroup, not global) holds meanwhile. Abstract unix sockets are shared by every + distro too (the unit cannot hide them; `RestrictAddressFamilies` keeps `AF_UNIX` for T3 itself), so a program in + another distro listening on one is reachable. +- **The policy binds an honest T3:** it is T3 that applies the locks. The kit makes sure the T3 that runs is the one + root installed (root-owned under `/opt/pc-agents/t3`, its hash recorded and checked), not one the agent could + change. An agent can still start its own programs (even its own copy of T3) inside the unit: those ignore the + policy, but stay inside every other wall. - **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's and Anthropic's addresses are many and change). Add an outbound proxy if that matters. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, - but the code files are not hashed. + but the code files are not hashed (T3's `t3` is). ## Tests @@ -90,5 +117,6 @@ python3 -I -B fork-kit/pc-agents/tests/check.test.py ``` A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake `nft` and -`systemctl`: every wall passes on a walled distro, and each one fails on its own when its part is taken away. +`systemctl` (it keeps which units are enabled and running): every wall passes on a walled distro, and each one +fails on its own when its part is taken away. `apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 29000cb399cb1..703fbab0a5062 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -2,7 +2,7 @@ Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" -import hashlib, ipaddress, json, os, pwd, re, subprocess +import hashlib, ipaddress, json, os, re, signal, subprocess import machine_file @@ -13,8 +13,11 @@ OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes HOME, USER = '/home/agent', 'agent' OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' -RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote +RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) +LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded +T3 = OPT + '/t3' # T3's runtime, installed by root (T3CODE_HOME of scripts/install.sh): the agent cannot change it UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' +UNITS = (UNIT, WALLS + '.service', WALLS + '.timer') CHECK = OPT + '/current/sbin/pc-agents' WALLS_PY = OPT + '/current/walls/fork-fleet-walls.py' # Walls of fork-fleet-walls/wsl2 that are about the fleet launcher's job units. Here the job unit is T3's own unit, @@ -22,8 +25,12 @@ UNIT_WALLS = {'mounts', 'sockets', 'userns'} MASKS = ['/etc/binfmt.d/WSLInterop.conf'] + ['/etc/systemd/system/' + u for u in ('cron.service', 'atd.service', 'ssh.service', 'ssh.socket')] -UNIT_DIRS = ('/etc/systemd/system', '/run/systemd/system', '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', - '/lib/systemd/system', '/run/systemd/transient', '/run/systemd/generator.early') +# Every folder systemd reads system units and drop-ins from (systemd.unit(5)); the kit's units live in the first. +UNIT_DIRS = ('/etc/systemd/system', '/etc/systemd/system.control', '/run/systemd/system.control', + '/run/systemd/transient', '/run/systemd/generator.early', '/etc/systemd/system.attached', + '/run/systemd/system', '/run/systemd/system.attached', '/run/systemd/generator', + '/usr/local/lib/systemd/system', '/usr/lib/systemd/system', '/lib/systemd/system', + '/run/systemd/generator.late') BLOCKED4 = ('0.0.0.0/8', '10.0.0.0/8', '100.64.0.0/10', '169.254.0.0/16', '172.16.0.0/12', '192.168.0.0/16', '224.0.0.0/4', '240.0.0.0/4') BLOCKED6 = ('fc00::/7', 'fe80::/10', 'ff00::/8') @@ -56,33 +63,60 @@ def is_ip_list(value, net, glob=False, most=64): SPEC = { - 'organization': (str, machine_file.REQUIRED, lambda v: None if 0 < len(v) <= 80 else '1 to 80 characters'), + 'organization': (str, machine_file.REQUIRED, + lambda v: None if v.strip() and len(v) <= 80 else '1 to 80 characters, not blank'), 't3Version': (str, machine_file.REQUIRED, lambda v: None if VERSION.fullmatch(v) else 'an exact version, no v'), 't3ReleaseBaseUrl': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'https://[A-Za-z0-9./_-]+', v) else 'an https URL'), 'port': (int, machine_file.REQUIRED, lambda v: None if 1024 <= v <= 65535 else '1024 to 65535'), 'agentUid': (int, 1100, lambda v: None if 1000 <= v < 60000 else '1000 to 59999'), 'maxRuntimeMode': (str, machine_file.REQUIRED, lambda v: None if v in MODES else 'one of ' + ', '.join(MODES)), - 'lockedSettings': (dict, {}, lambda v: next((f'{k}: set by the kit itself, or not a settings path' for k in v - if k in KIT_SETTINGS or not re.fullmatch(r'[a-z]\w*(\.\w+)*', k)), None) - or next((f'{k}: lock one field per dotted path ({k}.field), not an object (T3 takes an object ' - 'only as the whole setting and does not start on a policy it cannot apply)' - for k, x in v.items() if isinstance(x, dict)), None)), + 'lockedSettings': (dict, {}, lambda v: next(filter(None, (locked_problem(p, x) for p, x in v.items())), None)), 'memoryMax': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*[KMGT]', v) else 'like 8G'), 'cpuQuota': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'[1-9]\d*%', v) else 'like 400%'), 'nameservers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, False, True, 3)), 'allow': (list, [], lambda v: is_ip_list(v, True)), + 'peers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, True, most=16) if v else '1 to 16 entries'), 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), } +def locked_problem(path, value): + """Why T3 would refuse to lock path to value, where that is cheap to tell (T3 checks the rest and does not start + on a policy it cannot apply: managedSettings.ts), or None. An object is a complete value: a whole setting + (`agentSlots`) or one entry of a record (`providerInstances.codex-work`), never a part of one.""" + segments = path.split('.') + if not re.fullmatch(r'[a-z]\w*(\.[A-Za-z0-9_-]+)*', path) or segments[0] in KIT_SETTINGS: + return f'{path}: set by the kit itself, or not a settings path' + if segments[0] == 'providerInstances' and len(segments) > 2: + return f'{path}: lock providerInstances.{segments[1]} as a whole (T3 refuses one field of a provider instance)' + if isinstance(value, dict) and len(segments) > 2: + return (f'{path}: an object only as a whole setting ({segments[0]}) or a record entry; lock one field per ' + 'dotted path instead (T3 does not start on a partial object)') + return None + + def config(): section = machine_file.read_section(KIT, ROOT + machine_file.default_path() if ROOT else None, OWNER) return machine_file.fields(section, SPEC, f'{machine_file.default_path()}: {KIT}') def runtime(c): - return f'{HOME}/.t3/runtime/versions/{c["t3Version"]}/t3' + """T3's version folder (root's, from scripts/install.sh with T3CODE_HOME=T3); the unit runs its `t3`.""" + return f'{T3}/runtime/versions/{c["t3Version"]}' + + +def masks(c): + """Units the kit masks: WSL's binfmt handler, cron, at, ssh and the agent's systemd user manager (with it masked, + no login of the agent, no linger and no user unit runs anything outside T3's unit).""" + return MASKS + [f'/etc/systemd/system/user@{c["agentUid"]}.service'] + + +def dropin_dirs(unit): + """Every drop-in folder name systemd reads for unit: its own, its type's and each dash prefix's.""" + name, kind = unit.rsplit('.', 1) + parts = name.split('-') + return [unit + '.d', kind + '.d'] + ['-'.join(parts[:i]) + f'-.{kind}.d' for i in range(1, len(parts))] def render(c): @@ -104,15 +138,18 @@ def render(c): Description=T3 Code for agents, walled (pc-agents kit) Wants=network-online.target After=network-online.target +# T3 exits 1 on a policy it cannot apply, like on any other failure: give up after 5 starts in 10 minutes. +StartLimitIntervalSec=600 +StartLimitBurst=5 [Service] User={USER} Group={USER} Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin -ExecStartPre=+/usr/sbin/nft -f {STATE}/firewall.nft +ExecStartPre=+{CHECK} load-firewall ExecStartPre=+{CHECK} check -ExecStart={runtime(c)} serve +ExecStart={runtime(c)}/t3 serve Restart=on-failure RestartSec=30 UMask=0077 @@ -126,7 +163,9 @@ def render(c): PrivateDevices=yes PrivateIPC=yes ProtectProc=invisible -InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl +InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale +IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} +IPAddressAllow={" ".join(c["peers"] + c["allow"])} RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS SystemCallArchitectures=native @@ -145,7 +184,7 @@ def render(c): WantedBy=multi-user.target ''' wsl = HEAD + ('[boot]\nsystemd=true\n[automount]\nenabled=false\nmountFsTab=false\n[interop]\nenabled=false\n' - f'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault={USER}\n') + 'appendWindowsPath=false\n[network]\ngenerateResolvConf=false\n[user]\ndefault=root\n') sd = '/etc/systemd/system/' return { '/etc/wsl.conf': (wsl, 0o644), @@ -153,10 +192,14 @@ def render(c): '/etc/t3code/policy.json': (json.dumps(policy, indent=2, sort_keys=True) + '\n', 0o644), STATE + '/firewall.nft': (nft, 0o644), sd + UNIT: (unit, 0o644), - sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents daily walls check\n\n[Service]\nType=oneshot\n' - f'ExecStart={CHECK} check --enforce\n', 0o644), - sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents daily walls check\n\n[Timer]\nOnCalendar=daily\n' - 'Persistent=true\n\n[Install]\nWantedBy=timers.target\n', 0o644), + # A check that does not succeed, even one that cannot start or crashes before it can stop T3, stops T3. + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\n\n[Service]\nType=oneshot\n' + f'ExecStart={CHECK} check --enforce\n' + 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' + f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), + sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' + 'OnActiveSec=1min\nOnUnitActiveSec=2min\nAccuracySec=10s\n\n[Install]\n' + 'WantedBy=timers.target\n', 0o644), } @@ -164,12 +207,58 @@ def sha(text): return hashlib.sha256(text.encode() if isinstance(text, str) else text).hexdigest() +def file_sha(path): + h = hashlib.sha256() + with open(ROOT + path, 'rb') as f: + for block in iter(lambda: f.read(1 << 20), b''): + h.update(block) + return h.hexdigest() + + def record(): + """{path: {sha256, ...}} of the record: one hash per path, two while setup replaces a file (old and new).""" + out = {} try: with open(ROOT + RECORD, encoding='utf-8') as f: - return dict(reversed(line.rstrip('\n').split('\t', 1)) for line in f if '\t' in line) + for line in f: + if '\t' in line: + h, path = line.rstrip('\n').split('\t', 1) + out.setdefault(path, set()).add(h) except FileNotFoundError: - return {} + pass + return out + + +def save(path, text): + """Write a small state file whole (new file, one rename) unless it already says text.""" + real = ROOT + path + try: + with open(real, encoding='utf-8') as f: + if f.read() == text: + return + except (FileNotFoundError, UnicodeDecodeError): + pass + os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) + with open(real + '.new', 'w', encoding='utf-8') as f: + f.write(text) + os.replace(real + '.new', real) + + +def nft_listing(): + r = run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) + return r.stdout if r.returncode == 0 else None + + +def load_firewall(): + """Load firewall.nft and save its hash with what the kernel then lists, so the check can tell that the loaded + table is this file's (any change by anyone, or an older file left loaded, differs). Returns why not, or None.""" + body = file_sha(STATE + '/firewall.nft') + r = run(['/usr/sbin/nft', '-f', STATE + '/firewall.nft']) + listing = nft_listing() if r.returncode == 0 else None + if listing is None: + return f'cannot load {STATE}/firewall.nft: {(r.stderr or r.stdout).strip()[-300:]}' + save(LOADED, f'{body}\n{listing}') + return None def lstat(path): @@ -214,6 +303,23 @@ def user_problem(c, need_agent): return f'user {USER} is in other groups ({", ".join(extra)}): it must have none' if extra else None -def run(argv, **kw): - return subprocess.run([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, capture_output=True, text=True, - timeout=120, **kw) +def run(argv, timeout=120): + """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops, the + whole group is killed (a download's curl too); a timeout returns exit 124 and says so in stderr.""" + p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + out, err = p.communicate(timeout=timeout) + except BaseException as e: + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: # something left the group but holds the pipes: stop reading + out, err = '', '' + if not isinstance(e, subprocess.TimeoutExpired): + raise + return subprocess.CompletedProcess(argv, 124, out, f'{err}\n{argv[0]} took more than {timeout} s; stopped it') + return subprocess.CompletedProcess(argv, p.returncode, out, err) diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 449635b942ef4..73710c4c1ead4 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -2,14 +2,16 @@ """pc-agents: wall a WSL2 distro in, then run T3 Code in it as an unprivileged user (fork-kit/pc-agents/README.md). pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also - stops T3 when one fails (the daily timer). T3's unit runs it before - every start, so T3 never starts on a failed wall. + stops T3 when one fails (the timer, every 2 minutes). T3's unit runs + it before every start, so T3 never starts on a failed wall. + pc-agents load-firewall as root: load the agent firewall and note what the kernel lists Standard library only.""" import os, re, stat, sys HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -COMMON = os.path.join(os.path.dirname(HERE), 'common') -sys.path[:0] = [os.path.join(HERE, 'lib')] + ([COMMON] if os.path.isdir(COMMON) else []) +LIB = os.path.join(HERE, 'lib') +COMMON = os.path.join(os.path.dirname(HERE), 'common') # the repository's; an installed kit has its own copy in LIB +sys.path[:0] = [LIB] + ([] if os.path.exists(os.path.join(LIB, 'machine_file.py')) else [COMMON]) sys.dont_write_bytecode = True import pc_agents as k # noqa: E402 @@ -18,47 +20,91 @@ def say(text): print(f'pc-agents: {text}', flush=True) -def own_walls(c): - """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2.""" +def tree(top, inside=None): + """Why something in the folder top (itself included) is not root's alone, or a link out of inside; or None.""" + bad = [why] if (why := k.safe(top, k.lstat(top), stat.S_ISDIR)) else [] + for here, dirs, names in os.walk(k.ROOT + top): + for n in dirs + names: + p = os.path.join(here, n)[len(k.ROOT):] + st = k.lstat(p) + if stat.S_ISLNK(st.st_mode) and inside and st.st_uid == k.OWNER and \ + os.path.realpath(k.ROOT + p).startswith(os.path.realpath(k.ROOT + inside) + '/'): + continue + bad += [why] if (why := k.safe(p, st, lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] + return bad[:3] and '; '.join(bad[:3]) + + +def own_walls(c, before_enable=False): + """(wall, what, why or None) for the walls this kit adds to fork-fleet-walls/wsl2. before_enable: setup asks + before it enables T3 and the timer, so they need not be enabled yet.""" out, done = [], k.record() for path, (text, mode) in k.render(c).items(): st = k.lstat(path) why = k.safe(path, st, stat.S_ISREG) if why is None: with open(k.ROOT + path, 'rb') as f: - why = None if f.read() == text.encode() and done.get(path) == k.sha(text) else \ + why = None if f.read() == text.encode() and k.sha(text) in done.get(path, ()) else \ f'{path} differs from what the kit writes (pc-agents setup)' out.append(('file:' + os.path.basename(path), f'{path} is the kit\'s, root-owned', why)) - masks = [p for p in k.MASKS if not (k.lstat(p) and os.path.islink(k.ROOT + p) - and os.readlink(k.ROOT + p) == '/dev/null')] - out.append(('masks', 'WSL interop, cron, at and ssh are masked', masks and 'not masked: ' + ', '.join(masks))) - extra = [f'{d}/{n}' for d in k.UNIT_DIRS for n in (k.UNIT + '.d', k.WALLS + '.service.d') + ( - ('service.d',) if d in k.UNIT_DIRS[:2] else ()) + ((k.UNIT,) if d != k.UNIT_DIRS[0] else ()) - if k.lstat(f'{d}/{n}')] - out.append(('drop-ins', 'nothing overrides the unit', extra and 'found ' + ', '.join(extra))) + out.append(('record', f'{k.RECORD} is root-owned', k.safe(k.RECORD, k.lstat(k.RECORD), stat.S_ISREG))) + masks = [p for p in k.masks(c) if not (k.lstat(p) and os.path.islink(k.ROOT + p) + and os.readlink(k.ROOT + p) == '/dev/null')] + out.append(('masks', 'WSL interop, cron, at, ssh and the agent\'s user manager are masked', + masks and 'not masked: ' + ', '.join(masks))) + linger = sorted(os.listdir(k.ROOT + '/var/lib/systemd/linger')) if k.lstat('/var/lib/systemd/linger') else [] + out.append(('linger', 'no user lingers', linger and 'lingering: ' + ', '.join(linger[:5]))) + units = [p for p in ('.config/systemd/user', '.config/systemd/user.control', '.local/share/systemd/user') + if os.path.isdir(f'{k.ROOT}{k.HOME}/{p}') and os.listdir(f'{k.ROOT}{k.HOME}/{p}')] + out.append(('user-units', f'{k.USER} has no systemd user units', units and 'found ~/' + ', ~/'.join(units))) + extra = [f'{d}/{n}' for d in k.UNIT_DIRS for u in k.UNITS + for n in k.dropin_dirs(u) + ([u] if d != k.UNIT_DIRS[0] else []) if k.lstat(f'{d}/{n}')] + out.append(('drop-ins', 'no drop-in or other copy of the kit\'s units', extra and 'found ' + ', '.join(extra[:5]))) + seen = [] + for u in k.UNITS: # what systemd has loaded, not only what is on disk + r = k.run(['/usr/bin/systemctl', 'show', '-p', + 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload', u]) + got = dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line) + want = {'LoadState': 'loaded', 'FragmentPath': f'{k.UNIT_DIRS[0]}/{u}', 'DropInPaths': '', + 'NeedDaemonReload': 'no'} + if u != k.WALLS + '.service' and not before_enable: + want['UnitFileState'] = 'enabled' + seen += [f'{u}: {key}={got.get(key)!r}' for key, v in want.items() if r.returncode or got.get(key) != v] + out.append(('units', 'systemd runs the kit\'s units as written, enabled', seen and '; '.join(seen[:3]))) out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) - code = [] - for top, dirs, names in os.walk(k.ROOT + k.OPT + '/current/'): - for n in dirs + names: - p = os.path.join(top, n)[len(k.ROOT):] - code += [why] if (why := k.safe(p, k.lstat(p), lambda m: stat.S_ISDIR(m) or stat.S_ISREG(m))) else [] current = k.lstat(k.OPT + '/current') - code += [] if current and current.st_uid == k.OWNER else [f'{k.OPT}/current is not the kit\'s'] - out.append(('kit-code', f'{k.OPT}/current is root-owned', code[:3] and '; '.join(code[:3]))) + target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' + code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ + [f'{k.OPT}/current is not the kit\'s link to a release folder'] + code += [why for d in ('/', '/opt', k.OPT) if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] + code += [why] if target and (why := tree(f'{k.OPT}/{target}')) else [] + out.append(('kit-code', f'{k.OPT} and the release it runs are root\'s alone', code[:3] and '; '.join(code[:3]))) + folder, rt = k.runtime(c), [] + if k.lstat(folder): # else not installed (yet): T3 cannot start + rt = [why for d in (k.T3, k.T3 + '/runtime', k.T3 + '/runtime/versions') + if (why := k.safe(d, k.lstat(d), stat.S_ISDIR))] or [why for why in [tree(folder, folder)] if why] + if not rt and k.lstat(folder + '/t3') and k.file_sha(folder + '/t3') not in done.get(folder + '/t3', ()): + rt = [f'{folder}/t3 is not the one pc-agents setup installed'] + out.append(('runtime', f'T3 {c["t3Version"]} is root\'s alone, as installed', rt and '; '.join(rt))) mounts = [] try: with open(k.ROOT + '/proc/self/mountinfo', encoding='utf-8', errors='replace') as f: for line in f: fields = line.split(' - ') where, fstype = fields[0].split()[4], (fields[1].split() or [''])[0] if len(fields) > 1 else '' - if fstype in ('9p', 'drvfs', 'virtiofs') and not where.startswith('/usr/lib/wsl'): + if fstype in ('9p', 'drvfs', 'virtiofs') and not (where + '/').startswith('/usr/lib/wsl/'): mounts.append(f'{where} ({fstype})') except (OSError, IndexError) as e: mounts.append(f'cannot read /proc/self/mountinfo ({e})') out.append(('automount', 'no Windows drive is mounted', mounts and 'mounted: ' + ', '.join(mounts[:3]))) - r = k.run(['/usr/sbin/nft', 'list', 'table', 'inet', 'pc_agents']) - nft_ok = r.returncode == 0 and f'skuid != {c["agentUid"]}' in r.stdout and ' drop' in r.stdout - out.append(('firewall', 'the agent firewall is loaded', None if nft_ok else 'table inet pc_agents is not loaded')) + saved = None + if k.safe(k.LOADED, k.lstat(k.LOADED), stat.S_ISREG) is None: + with open(k.ROOT + k.LOADED, encoding='utf-8', errors='replace') as f: + saved = f.read() + listing = k.nft_listing() + fw = 'table inet pc_agents is not loaded' if listing is None else None if saved == \ + f'{k.sha(k.render(c)[k.STATE + "/firewall.nft"][0])}\n{listing}' else \ + 'the loaded table inet pc_agents is not the one from the kit\'s firewall.nft (pc-agents load-firewall)' + out.append(('firewall', 'the kit\'s agent firewall is what is loaded', fw)) gateway = None try: with open(k.ROOT + '/proc/net/route', encoding='utf-8') as f: @@ -66,10 +112,13 @@ def own_walls(c): gateway = row and k.ipaddress.ip_address(bytes.fromhex(row[2])[::-1]) except (OSError, ValueError) as e: gateway = f'unknown ({e})' - blocked = isinstance(gateway, k.ipaddress.IPv4Address) and \ - any(gateway in k.ipaddress.ip_network(n) for n in k.BLOCKED4) - out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range', - None if blocked else f'the gateway is {gateway}, not in a blocked range')) + + def nets(names): + return [n for n in names if isinstance(gateway, k.ipaddress.IPv4Address) and gateway in k.ipaddress.ip_network(n)] + opened = nets(c['allow'] + c['peers']) + out.append(('gateway', 'the Windows host (the default gateway) is in a blocked range and not let through', + f'allow or peers let the gateway {gateway} through: {", ".join(opened)}' if opened else + None if nets(k.BLOCKED4) else f'the gateway is {gateway}, not in a blocked range')) return out @@ -89,24 +138,28 @@ def wsl2_walls(): return [(f'wsl2:{w}', what, None if mark == 'PASS' else what) for mark, w, what in lines if w not in k.UNIT_WALLS] -def check(enforce): +def check(enforce, before_enable=False): try: c = k.config() - results = wsl2_walls() + own_walls(c) + results = wsl2_walls() + own_walls(c, before_enable) except Exception as e: # a wall that cannot be told does not hold results = [('check', 'the walls can be read', f'{e.__class__.__name__}: {e}')] for wall, what, why in results: print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) failed = any(why for _, _, why in results) if failed and enforce: - k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) - say(f'a wall does not hold: stopped {k.UNIT}') + r = k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) + say(f'a wall does not hold: stopped {k.UNIT}' if r.returncode == 0 else + f'a wall does not hold, and stopping {k.UNIT} FAILED: {r.stderr.strip()[-300:]}') return 1 if failed else 0 def main(argv): if argv[:1] == ['check'] and argv[1:] in ([], ['--enforce']): return check(argv[1:] == ['--enforce']) + if argv == ['load-firewall']: + why = k.load_firewall() + return say(why) or 1 if why else 0 print(__doc__, file=sys.stderr) return 2 diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index 03fc4bb708724..1f69c0b77b80b 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -3,10 +3,10 @@ python3 -I -B fork-kit/pc-agents/tests/check.test.py """ -import os, shutil, sys, unittest +import os, shutil, subprocess, sys, time, unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from fake_distro import Case, DRVFS, SHA1, k # noqa: E402 +from fake_distro import CONFIG, Case, DRVFS, SHA1, k # noqa: E402 class Walls(Case): @@ -25,8 +25,9 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'drop-ins', 'agent', 'kit-code', 'automount', 'firewall', 'gateway', 'file:policy.json', - 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf'): + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'kit-code', 'runtime', + 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', + 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) for wall in k.UNIT_WALLS: # the launcher's job units: not T3's unit self.assertNotIn('wsl2:' + wall, walls) @@ -36,6 +37,15 @@ def test_enforce_stops_t3(self): code, out = self.check('--enforce') self.assertEqual(code, 1, out) self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + self.assertIn('stopped t3code-agents.service', out) + + def test_enforce_says_when_the_stop_fails(self): + os.unlink(self.path('/log/nft-loaded')) + self.write('/log/systemctl-fail', '') + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stopping t3code-agents.service FAILED', out) def test_each_wall(self): def edit(p, old, new): @@ -55,6 +65,12 @@ def edit(p, old, new): 'wsl2:interop': lambda: self.write('/proc/sys/fs/binfmt_misc/WSLInterop', 'enabled\ninterpreter /init\nflags: PF\noffset 0\nmagic 4d5a\n'), 'wsl2:networking': edit('/usr/bin/wslinfo', 'echo nat', 'echo mirrored'), + 'record': lambda: os.chmod(self.path(k.RECORD), 0o666), + 'linger': lambda: self.write('/var/lib/systemd/linger/agent', ''), + 'user-units': lambda: self.write('/home/agent/.config/systemd/user/x.service', '[Service]\n'), + 'units': lambda: self.write('/log/units/t3code-agents.service.dropins', '/run/systemd/system/x.conf'), + 'runtime': lambda: os.chmod(self.path(k.runtime(CONFIG) + '/t3'), 0o777), + 'file:pc-agents-walls.timer': edit('/etc/systemd/system/pc-agents-walls.timer', '2min', '1d'), } clean = {p: v[4] for p, v in self.snapshot().items()} for wall, change in cases.items(): @@ -65,6 +81,78 @@ def edit(p, old, new): change() self.fails(wall, *(['wsl2:resolv'] if wall == 'file:resolv.conf' else [])) + def again(self, **section): + """machine.json changed and set up again: the new files, recorded, their firewall loaded.""" + self.machine({**CONFIG, **section}) + self.files() + self.assertIsNone(k.load_firewall()) + + def test_agent_user_manager_masked(self): + os.unlink(self.path('/etc/systemd/system/user@1100.service')) + self.fails('masks') + + def test_unit_copies_and_prefix_drop_ins(self): + for p in ('/usr/lib/systemd/system/t3code-agents.service', '/run/systemd/generator.late/pc-agents-walls.timer', + '/etc/systemd/system/t3code-.service.d/', '/etc/systemd/system/pc-agents-.timer.d/', + '/run/systemd/system.attached/t3code-agents.service.d/', '/etc/systemd/system/service.d/'): + with self.subTest(path=p): + self.write(p, '[Service]\n') + self.fails('drop-ins') + (os.rmdir if p.endswith('/') else os.unlink)(self.path(p.rstrip('/'))) + + def test_units_as_systemd_loaded_them(self): + for name, change in (('timer disabled', lambda: os.unlink(self.path('/log/units/pc-agents-walls.timer.enabled'))), + ('T3 disabled', lambda: os.unlink(self.path('/log/units/t3code-agents.service.enabled'))), + ('not reloaded', lambda: self.write('/log/units/reload', '')), + ('drop-in', lambda: self.write('/log/units/pc-agents-walls.service.dropins', '/x.conf'))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + change() + self.fails('units') + + def test_firewall_changed_after_it_was_loaded(self): + self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace('accept', 'accept\n', 1)) + self.fails('firewall') + + def test_firewall_file_newer_than_the_loaded_table(self): + self.machine({**CONFIG, 'allow': ['192.168.1.0/24']}) + self.files() + self.fails('firewall') + self.assertIsNone(k.load_firewall()) + self.assertEqual(self.check()[0], 0) + + def test_allow_or_peers_covering_the_gateway(self): + for key in ('allow', 'peers'): + with self.subTest(key): + self.again(**{key: ['172.16.0.0/12']}) + self.fails('gateway') + self.again() + self.assertEqual(self.check()[0], 0) + + def test_runtime_changed_or_missing(self): + t3 = k.runtime(CONFIG) + '/t3' + self.write(t3, {'text': '#!/bin/sh\nexec /home/agent/x\n', 'mode': '755'}) + self.fails('runtime') + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o777) + self.fails('runtime') + shutil.rmtree(self.path(k.runtime(CONFIG))) # not installed: T3 cannot start, nothing to wall + os.chmod(self.path('/opt/pc-agents/t3/runtime'), 0o755) + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_kit_code_folders(self): + os.chmod(self.path('/opt/pc-agents'), 0o775) + self.fails('kit-code') + os.chmod(self.path('/opt/pc-agents'), 0o755) + os.symlink('/etc', self.path(f'/opt/pc-agents/{SHA1}/lib/etc')) + self.fails('kit-code') + + def test_automount_only_under_usr_lib_wsl(self): + self.write('/proc/self/mountinfo', self.read('/proc/self/mountinfo') + + '45 29 0:45 / /usr/lib/wslfoo rw - 9p drvfs rw\n') + self.fails('automount') + def test_no_config_fails(self): os.unlink(self.path('/etc/fork-kit/machine.json')) code, out = self.check() @@ -72,5 +160,107 @@ def test_no_config_fails(self): self.assertIn('FAIL check ', out) +class Holes(Case): + """Ways the agent could run code outside T3's walled unit, or the walls be changed under systemd.""" + def setUp(self): + super().setUp() + self.walled() + + def test_agent_user_unit(self): + self.write('/home/agent/.config/systemd/user/x.service', '[Service]\nExecStart=/home/agent/x\n') + os.makedirs(self.path('/home/agent/.config/systemd/user/default.target.wants')) + os.symlink('../x.service', self.path('/home/agent/.config/systemd/user/default.target.wants/x.service')) + self.write('/var/lib/systemd/linger/agent', '') + self.assertEqual(self.check()[0], 1) + + def test_system_control_drop_in(self): + self.write('/etc/systemd/system.control/t3code-agents.service.d/50-x.conf', '[Service]\nRestrictAddressFamilies=\n') + self.assertEqual(self.check()[0], 1) + + def test_timer_drop_in(self): + self.write('/etc/systemd/system/pc-agents-walls.timer.d/off.conf', '[Timer]\nOnCalendar=\n') + self.assertEqual(self.check()[0], 1) + + def test_prefix_drop_in(self): + self.write('/etc/systemd/system/t3code-.service.d/x.conf', '[Service]\nUser=root\n') + self.assertEqual(self.check()[0], 1) + + +class Crash(Case): + """A check that cannot run, or crashes, still stops T3 (the walls service's ExecStopPost).""" + def setUp(self): + super().setUp() + self.walled() + + def stop_post(self, result): + line = next(x for x in k.render(k.config())['/etc/systemd/system/pc-agents-walls.service'][0].splitlines() + if x.startswith('ExecStopPost=')) + cmd = line.split('=', 1)[1].replace('$$', '$').replace('/usr/bin/systemctl', self.path('/usr/bin/systemctl')) + subprocess.run(cmd, shell=True, check=True, env={'SERVICE_RESULT': result, 'PATH': '/usr/bin:/bin'}) + + def test_missing_lib_stops_t3(self): + os.unlink(self.path(f'/opt/pc-agents/{SHA1}/lib/pc_agents.py')) + p = subprocess.run([sys.executable, '-I', self.path(f'/opt/pc-agents/{SHA1}/sbin/pc-agents'), 'check', + '--enforce'], env={'PC_AGENTS_ROOT': self.root, 'PATH': '/usr/bin:/bin'}, + capture_output=True, text=True, timeout=60) + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertNotIn('stop t3code-agents.service', self.log()) # the check itself could not stop T3 + self.stop_post('exit-code') + self.assertIn('stop t3code-agents.service', self.log()) + self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) + + def test_success_does_not_stop_t3(self): + self.stop_post('success') + self.assertEqual(self.log(), []) + + +class Unit(Case): + """What the kit's files say: things no wall can see from outside the unit.""" + def test_unit(self): + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + self.assertIn('\nExecStart=/opt/pc-agents/t3/runtime/versions/1.2.3/t3 serve\n', unit) + self.assertIn(' -/run/tailscale', unit) + deny = next(x for x in unit.splitlines() if x.startswith('IPAddressDeny=')).split('=', 1)[1].split() + self.assertEqual(set(deny), set(k.BLOCKED4 + k.BLOCKED6)) + self.assertIn('\nIPAddressAllow=100.101.102.103/32\n', unit) + self.assertIn('\nStartLimitBurst=5\n', unit) + self.assertIn('\n[user]\ndefault=root\n', files['/etc/wsl.conf'][0]) + self.assertIn('\nOnUnitActiveSec=2min\n', files['/etc/systemd/system/pc-agents-walls.timer'][0]) + + def test_config(self): + # The kit passes these; T3 itself tells a complete object from a partial one and does not start on a partial. + ok = {'agentSlots': {'maxPerMachine': 2}, 'providerInstances.codex-work': {'driver': 'codex'}, + 'agentSlots.maxPerMachine': 2, 'providerInstances': {}} + self.assertEqual([p for p, v in ok.items() if k.locked_problem(p, v)], []) + bad = {'providerInstances.codex-work.enabled': True, 'agentSlots.a.b': {'x': 1}, 'automaticUpdates': True, + 'linkedEnvironmentsEnabled.x': 1, 'Agent': 1, 'a..b': 1} + self.assertEqual([p for p, v in bad.items() if not k.locked_problem(p, v)], []) + for section in ({**CONFIG, 'organization': ' '}, {**CONFIG, 'peers': []}, + {key: v for key, v in CONFIG.items() if key != 'peers'}): + with self.subTest(section=section): + self.machine(section) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL check ', out) + + +class Run(Case): + def test_timeout_kills_the_whole_group(self): + self.write('/x', {'text': f'#!/bin/sh\nsleep 60 &\necho $! >{self.root}/pid\nwait\n', 'mode': '755'}) + r = k.run(['/x'], timeout=1) + self.assertEqual(r.returncode, 124) + self.assertIn('took more than 1 s', r.stderr) + pid = int(self.read('/pid')) + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f'sleep {pid} still runs') + + if __name__ == '__main__': unittest.main() diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index abe99cb8ce8ea..79f79362ac740 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -1,6 +1,7 @@ """A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the -current user as root, fake useradd, runuser, systemctl and nft (each logs to /log in the fake root). No root, no -systemd, no network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" +current user as root, fake useradd, env (T3's installer), systemctl, systemd-run and nft (each logs to /log in the +fake root; systemctl keeps which units are enabled and which ExecStart runs in /log/units). No root, no systemd, no +network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -9,15 +10,36 @@ SHA1, SHA2 = 'a' * 40, 'b' * 40 CONFIG = {'organization': 'Example lab', 't3Version': '1.2.3', 't3ReleaseBaseUrl': 'https://example.com/releases', 'port': 3800, 'maxRuntimeMode': 'full-access', 'lockedSettings': {'agentSlots.maxPerMachine': 2}, - 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'tailscaleTag': 'tag:pc-agents'} + 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'peers': ['100.101.102.103/32'], + 'tailscaleTag': 'tag:pc-agents'} TOOLS = { # fake programs, under the fake root; each finds the root from its own path 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', - 'usr/sbin/runuser': 'R="${0%/usr/sbin/runuser}"; echo "$*" >>"$R/log/runuser"\n[ -e "$R/log/runuser-fail" ] && exit 1\n' - 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' - 'd="$R/home/agent/.t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n" >"$d/t3"\n', - 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"\n', + # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) + 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' + 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' + 'd="$R/opt/pc-agents/t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n# t3 $v\\n" >"$d/t3"\n' + 'chmod 755 "$d/t3"\n', + 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n', + 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"; S="$R/log/units"; mkdir -p "$S"\n' + '[ -e "$R/log/systemctl-fail" ] && { echo "systemctl: failed (test)" >&2; exit 1; }\n' + 'run() { grep "^ExecStart=" "$R/etc/systemd/system/$1" >"$S/$1.active" 2>/dev/null || :; }\n' + 'case "$1" in\n' + 'enable|disable) v=$1; shift; now=; [ "$1" = --now ] && { now=1; shift; }\n' + ' for u; do if [ $v = enable ]; then : >"$S/$u.enabled"; [ -n "$now" ] && ' + '{ [ -e "$S/$u.active" ] || run "$u"; }\n' + ' else rm -f "$S/$u.enabled"; [ -z "$now" ] || rm -f "$S/$u.active"; fi; done;;\n' + 'restart) shift; for u; do run "$u"; done;;\n' + 'stop) shift; for u; do rm -f "$S/$u.active"; done;;\n' + 'is-active) [ -e "$S/$2.active" ] && echo active || { echo inactive; exit 3; };;\n' + 'show) u=$4; f=; [ -e "$R/etc/systemd/system/$u" ] && f=/etc/systemd/system/$u\n' + ' st=disabled; [ -e "$S/$u.enabled" ] && st=enabled\n' + ' [ "$u" = pc-agents-walls.service ] && st=static\n' + ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' + ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' + ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no;;\n' + 'esac\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', } @@ -119,13 +141,22 @@ def ready(self): code, out = self.setup_run() self.assertEqual(code, 0, out) - def walled(self, sha=SHA1): - """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" - files = k.render(k.config()) + def files(self): + """The kit's files from machine.json, T3's runtime and the record of both, as setup writes them.""" + c = k.config() + files = k.render(c) for p, (text, mode) in files.items(): self.write(p, {'text': text, 'mode': f'{mode:o}'}) - self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items()))) - for p in k.MASKS: + t3 = k.runtime(c) + '/t3' + self.write(t3, {'text': '#!/bin/sh\n', 'mode': '755'}) + self.write(k.RECORD, ''.join(f'{k.sha(t)}\t{p}\n' for p, (t, _) in sorted(files.items())) + + f'{k.file_sha(t3)}\t{t3}\n') + return c + + def walled(self, sha=SHA1): + """What `pc-agents setup` leaves after `wsl --terminate` and a second run, written here directly.""" + c = self.files() + for p in k.masks(c): os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) os.symlink('/dev/null', self.path(p)) with open(self.path('/etc/passwd'), 'a') as f: @@ -136,7 +167,9 @@ def walled(self, sha=SHA1): with open(os.path.join(REPO, src), encoding='utf-8') as f: self.write(f'/opt/pc-agents/{sha}/{dst}', {'text': f.read(), 'mode': f'{mode:o}'}) os.symlink(sha, self.path('/opt/pc-agents/current')) - shutil.copy(self.path(k.STATE + '/firewall.nft'), self.path('/log/nft-loaded')) + self.assertIsNone(k.load_firewall()) + k.run(['/usr/bin/systemctl', 'enable', '--now', k.UNIT, k.WALLS + '.timer']) + os.unlink(self.path('/log/systemctl')) self.terminate() def snapshot(self): From b6ae1ea9bc3619ab51d49e1425a07428f9477c9b Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 07:44:55 +0000 Subject: [PATCH 04/10] fork-kit: pc-agents fix round 2 (walls) Loopback only to T3's port and a devPorts range (unit SocketBindDeny/Allow, nft drop on lo); walls service ordered after T3 with a start timeout; agent login shell nologin, sudoers and running-timer walls; README limits (hairpin NAT, D-Bus, shm, sock_diag, resolved stub, OSC 52) and how to restart T3 after a wall stopped it. Co-Authored-By: Claude Opus 5.5 --- fork-kit/pc-agents/README.md | 78 ++++++++++++++++--------- fork-kit/pc-agents/lib/pc_agents.py | 63 ++++++++++++++++++-- fork-kit/pc-agents/sbin/pc-agents | 13 ++++- fork-kit/pc-agents/tests/check.test.py | 65 ++++++++++++++++++++- fork-kit/pc-agents/tests/fake_distro.py | 10 ++-- 5 files changed, 190 insertions(+), 39 deletions(-) diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index c16182d0de384..73213ec8991d4 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -5,15 +5,16 @@ T3's user, so the link is not the wall: the distro is. This kit makes a fresh WS T3 in it as `agent`, a user with no sudo, under a root-owned system unit. Everything that holds the wall is root's, so the agent cannot undo it: -| Wall | What the kit does | -| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | -| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | -| No way out of the user | one login user (`agent`), in no group but its own; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | -| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | -| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the Windows host, the home network and the tailnet are out of reach; public addresses go through | -| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | -| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | +| Wall | What the kit does | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No Windows files | `/etc/wsl.conf`: `[automount] enabled=false`, `mountFsTab=false`; the unit also hides `/mnt`, `/run/WSL`, `/tmp/.X11-unix`, `/usr/lib/wsl` and `/run/tailscale` (the Tailscale daemon's socket) | +| No Windows programs | `[interop] enabled=false`, `appendWindowsPath=false`; WSL's binfmt handler masked | +| No way out of the user | one login user (`agent`), in no group but its own, no sudo rule, login shell `/usr/sbin/nologin`; the distro's default user is `root`, not `agent`; cron, at, ssh and the agent's systemd user manager (`user@.service`) masked, so no login, linger or user unit of the agent runs anything outside T3's unit | +| A sandboxed unit | `t3code-agents.service` (system unit, `User=agent`): `ProtectSystem=strict`, only `/home/agent` writable, no `AF_VSOCK` (Hyper-V sockets to Windows), io_uring off, no namespaces, `PrivateDevices`, `MemoryMax`, `CPUQuota` | +| No LAN, no Windows host | twice, nftables for the agent's uid and the unit's `IPAddressDeny=`: private, link-local, CGNAT and multicast ranges blocked but for `allow` and the linking `peers`, so the private addresses of the Windows host, the home network and the tailnet are out of reach; public addresses go through (the home's own public address too: see Limits) | +| Localhost only for T3 | the unit's `SocketBindDeny=any` with `SocketBindAllow=` T3's port and `devPorts`, and nftables dropping the agent's other localhost traffic: the agent listens on, and reaches on `localhost`, only T3's port and `devPorts`, so servers of other distros on `localhost` are out of reach | +| Locked T3 settings | `/etc/t3code/policy.json` (managed policy): linked environments on, automatic updates off, Connect, webhook tasks and browser cookie import off, `maxRuntimeMode` and your `lockedSettings` locked; T3 itself is root's (under `/opt/pc-agents/t3`) | +| Checked, not trusted | `pc-agents check` runs before every start of T3 (the unit refuses to start on a failed wall) and every 2 minutes (a failed wall stops T3; a check that cannot run or crashes stops it too) | `pc-agents check` is the fleet's WSL2 walls check (`.github/scripts/fork-fleet-walls.py` with `fork-fleet-walls/wsl2`, from the same release, not a copy) plus this kit's own walls: @@ -23,8 +24,9 @@ so the agent cannot undo it: - nothing overrides the kit's units: no drop-in or other copy in any folder systemd reads (`system.control`, `system.attached`, the generators, `/run`, `/usr/lib`, prefix drop-ins like `t3code-.service.d`), and `systemctl show` agrees (the kit's file, no drop-in, no pending reload, T3 and the timer enabled); -- the masks are in place, no user lingers, the agent has no systemd user units; -- `agent` is the only login user; +- the masks are in place, no user lingers, the agent has no systemd user units, the timer is running; +- `agent` is the only login user, its login shell is `/usr/sbin/nologin`, and no sudo rule (`/etc/sudoers`, + `/etc/sudoers.d`) names it, its group or uid, or every user; - the kit's code (`/opt/pc-agents`, the release folder it runs) and T3's runtime are root's, and T3's `t3` is the one setup installed (its hash is in the record); - no Windows drive is mounted; @@ -63,20 +65,21 @@ The kit reads the `pc-agents` section of the machine file `/etc/fork-kit/machine } ``` -| Key | What | -| ----------------------- | ----------------------------------------------------------------------------------------------------------- | -| `organization` | the name the managed policy shows in T3's settings | -| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | -| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | -| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | -| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | -| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | -| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | -| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | -| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | -| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | -| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | -| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | +| Key | What | +| ----------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `organization` | the name the managed policy shows in T3's settings | +| `t3Version` | the exact T3 version to run (the same as the T3 that links to this PC); installed with `scripts/install.sh` | +| `t3ReleaseBaseUrl` | where that version's release files are (`T3CODE_RELEASE_BASE_URL` of `scripts/install.sh`) | +| `port` | T3's port (`T3CODE_PORT`); the Tailscale ACL opens it to the linking T3 only | +| `agentUid` | the `agent` user's uid (default 1100; not 1000, the first user of every other distro, see Limits) | +| `maxRuntimeMode` | the highest runtime mode T3 allows (`full-access` for a linked PC) | +| `lockedSettings` | more T3 settings to lock, `{path: value}`: the child ceiling (`agentSlots.…`), see below | +| `memoryMax`, `cpuQuota` | the unit's `MemoryMax` and `CPUQuota` | +| `nameservers` | 1 to 3 public DNS servers for `/etc/resolv.conf` (WSL's own one is on the blocked Windows host) | +| `allow` | optional: private networks the agent may still reach (CIDR list); empty by default; never the Windows host | +| `devPorts` | optional: the TCP ports the agent's dev servers may listen on and be reached on, `LOW-HIGH` (default `4100-4199`) | +| `peers` | the linking T3's Tailscale address(es) (CIDR list, 1 to 16): the only private addresses the unit talks to | +| `tailscaleTag` | the tag this PC's Tailscale node gets (`tag:name`) | Every T3 setting can be locked through `lockedSettings`, the fork's own settings (`agentSlots`, `usagePacingGate`, `usagePacing`, `waitFor`, ...) too: the managed policy locks any key of the server settings, upstream's or the fork's @@ -89,11 +92,19 @@ whole setting), says why in its log (`journalctl -u t3code-agents`) and does not cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). +A wall that fails stops T3, and nothing starts it again by itself: fix what `pc-agents check` names (or run +`pc-agents setup` again), then start it as root: `systemctl start t3code-agents.service` (its own start runs the +check first and refuses on a failed wall). + ## Limits - **One VM for all distros.** WSL2 runs every distro in one VM: one kernel, one network namespace, one memory cap. The firewall table is therefore global and matches the agent's uid in every distro (keep `agentUid` apart from - the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable. + the other distros' users); `localhost` is shared, so a server another distro runs on localhost is reachable on + T3's port or `devPorts` (the agent's other localhost traffic is dropped). **Run no localhost servers in other + distros** of this PC on those ports. What the agent serves on those ports is forwarded to Windows' `localhost` by + WSL: open it in a browser only as you would an unknown site (a page on `localhost` shares cookies with every other + `localhost` page, whatever the port). Global kernel switches (user namespaces, sysctls) are left alone: the unit closes them for T3 instead. Another distro's root can drop or change the global nft table: the check sees it within 2 minutes and stops T3, and the unit's own `IPAddressDeny=` (per cgroup, not global) holds meanwhile. Abstract unix sockets are shared by every @@ -104,7 +115,20 @@ it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). change. An agent can still start its own programs (even its own copy of T3) inside the unit: those ignore the policy, but stay inside every other wall. - **Public internet is open** to the agent: the firewall blocks private ranges, not the public internet (GitHub's - and Anthropic's addresses are many and change). Add an outbound proxy if that matters. + and Anthropic's addresses are many and change). Add an outbound proxy if that matters. That includes the home's + own public address: a router that loops it back (hairpin NAT) lets the agent reach its admin page, if the router + serves it there, and every port-forwarded device of the home. Forward no ports to devices the agent must not + reach, and keep the router's admin page off the public side. +- **Servers in the agent's dev range only:** a test or dev server that listens on a port outside `devPorts` cannot + be reached over `localhost` (port 0, any free port, may be bound, but the agent cannot connect to it there). + Point them at `devPorts`, or widen it. +- **Shared with root's programs in this distro:** the system D-Bus (`/run/dbus`, its own access rules decide), + `/dev/shm` (mode 1777; others' files keep their modes), and `NETLINK_SOCK_DIAG`, which lists every distro's + sockets (addresses and ports, not their data). +- **DNS** goes to the public `nameservers` only. A `systemd-resolved` stub (`127.0.0.53`), which may resolve LAN + names through Windows, is out of reach: the agent's localhost traffic outside T3's port and `devPorts` is dropped. +- **Terminals:** what the agent prints in a terminal can use escape codes, such as OSC 52, which some terminals turn + into a write to the Windows clipboard. Do not paste from the clipboard blindly after watching agent output. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. - **No code hash manifest yet:** the kit's code is checked for root ownership and its written files byte for byte, diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 703fbab0a5062..930ddf4e4c1e6 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -12,6 +12,7 @@ raise SystemExit('pc-agents: PC_AGENTS_ROOT is for the tests only, never for root; unset it') OWNER = os.geteuid() if ROOT else 0 # who must own every file the kit writes HOME, USER = '/home/agent', 'agent' +NOLOGIN = '/usr/sbin/nologin' # the agent's login shell: no shell for `wsl -u agent` or su; T3 gets SHELL from its unit OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded @@ -78,9 +79,24 @@ def is_ip_list(value, net, glob=False, most=64): 'allow': (list, [], lambda v: is_ip_list(v, True)), 'peers': (list, machine_file.REQUIRED, lambda v: is_ip_list(v, True, most=16) if v else '1 to 16 entries'), 'tailscaleTag': (str, machine_file.REQUIRED, lambda v: None if re.fullmatch(r'tag:[a-z0-9-]{1,50}', v) else 'tag:name'), + 'devPorts': (str, '4100-4199', lambda v: None if port_range(v) else 'LOW-HIGH, 1024 to 65535, like 4100-4199'), } +def port_range(text): + """(low, high) of a 'LOW-HIGH' port range of 1024..65535, or None.""" + m = re.fullmatch(r'([1-9]\d{3,4})-([1-9]\d{3,4})', text) + low, high = (int(m[1]), int(m[2])) if m else (0, 0) + return (low, high) if 1024 <= low <= high <= 65535 else None + + +def loopback_ports(c): + """The TCP ports the agent may listen on and reach on localhost: T3's port and devPorts (nft and the unit's + SocketBindAllow=). Written apart unless the range holds T3's port (nft refuses overlapping set elements).""" + low, high = port_range(c['devPorts']) + return ([] if low <= c['port'] <= high else [str(c['port'])]) + [f'{low}-{high}'] + + def locked_problem(path, value): """Why T3 would refuse to lock path to value, where that is cheap to tell (T3 checks the rest and does not start on a policy it cannot apply: managedSettings.ts), or None. An object is a complete value: a whole setting @@ -130,7 +146,8 @@ def render(c): allow6 = [a for a in c['allow'] if ':' in a] nft = HEAD + 'table inet pc_agents\ndelete table inet pc_agents\ntable inet pc_agents {\n\tchain agent_out {\n' + \ '\t\ttype filter hook output priority filter; policy accept;\n' + \ - f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n\t\toifname "lo" accept\n' + \ + f'\t\tmeta skuid != {c["agentUid"]} accept\n\t\tct state established,related accept\n' + \ + f'\t\toifname "lo" tcp dport {{ {", ".join(loopback_ports(c))} }} accept\n\t\toifname "lo" drop\n' + \ (f'\t\tip daddr {{ {", ".join(allow4)} }} accept\n' if allow4 else '') + \ (f'\t\tip6 daddr {{ {", ".join(allow6)} }} accept\n' if allow6 else '') + \ f'\t\tip daddr {{ {", ".join(BLOCKED4)} }} drop\n\t\tip6 daddr {{ {", ".join(BLOCKED6)} }} drop\n\t}}\n}}\n' @@ -146,7 +163,7 @@ def render(c): User={USER} Group={USER} Environment=HOME={HOME} T3CODE_HOME={HOME}/.t3 T3CODE_PORT={c["port"]} T3CODE_LAN=true -Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin +Environment=PATH={HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin SHELL=/bin/bash ExecStartPre=+{CHECK} load-firewall ExecStartPre=+{CHECK} check ExecStart={runtime(c)}/t3 serve @@ -166,6 +183,8 @@ def render(c): InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale IPAddressDeny={" ".join(BLOCKED4 + BLOCKED6)} IPAddressAllow={" ".join(c["peers"] + c["allow"])} +SocketBindDeny=any +{chr(10).join("SocketBindAllow=tcp:" + p for p in loopback_ports(c))} RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK SystemCallFilter=~io_uring_setup:ENOSYS io_uring_enter:ENOSYS io_uring_register:ENOSYS SystemCallArchitectures=native @@ -193,8 +212,10 @@ def render(c): STATE + '/firewall.nft': (nft, 0o644), sd + UNIT: (unit, 0o644), # A check that does not succeed, even one that cannot start or crashes before it can stop T3, stops T3. - sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\n\n[Service]\nType=oneshot\n' - f'ExecStart={CHECK} check --enforce\n' + # After T3's unit: a check started while T3 is still starting (slow network at boot) waits for that start, + # instead of failing on a firewall not loaded yet and stopping T3 before it ran. + sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\nAfter={UNIT}\n\n[Service]\n' + f'Type=oneshot\nTimeoutStartSec=5min\nExecStart={CHECK} check --enforce\n' 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' @@ -289,6 +310,40 @@ def rows(path): {r[0]: [m for m in r[3].split(',') if m] for r in rows('/etc/group')}) +def login_shell(name): + """name's login shell in the passwd file, or None.""" + try: + with open(ROOT + '/etc/passwd', encoding='utf-8') as f: + return next((r[6] for r in (line.rstrip('\n').split(':') for line in f) if r[0] == name and len(r) > 6), + None) + except FileNotFoundError: + return None + + +def sudo_problem(c): + """Why sudo could give the agent more (a sudoers line naming it, its group or uid, or every user), or None. + Harmless inside the unit (NoNewPrivileges), but a way out from anywhere else.""" + who = re.compile(rf'(^|[\s,=:!(])(%?{USER}|%?#{c["agentUid"]}|ALL)($|[\s,=:)])') + found = [] + names = ['/etc/sudoers'] + (['/etc/sudoers.d/' + n for n in sorted(os.listdir(ROOT + '/etc/sudoers.d'))] + if os.path.isdir(ROOT + '/etc/sudoers.d') else []) + for path in names: + try: + with open(ROOT + path, encoding='utf-8', errors='replace') as f: + lines = f.read().replace('\\\n', ' ').splitlines() + except (FileNotFoundError, IsADirectoryError): + continue + for line in lines: + line = re.sub(r'#(?!include)(?!\d).*', '', line).strip() + if line.startswith('Defaults') or line.startswith(('@include', '#include')): + continue + first = line.split(None, 1)[0] if line else '' + # a user list (first field) or an alias naming the agent; `root ALL=(ALL) ALL` names root, not the agent + if line and (who.search(first) or re.match(r'User_Alias\b', line) and who.search(line.split('=', 1)[-1])): + found.append(f'{path}: {line[:80]}') + return found and 'sudo rules that cover the agent: ' + '; '.join(found[:3]) + + def user_problem(c, need_agent): """Why this distro is someone else's (another login user, an agent that is not the kit's), or None.""" people, groups = users() diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 73710c4c1ead4..188d680d13a44 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -62,15 +62,22 @@ def own_walls(c, before_enable=False): seen = [] for u in k.UNITS: # what systemd has loaded, not only what is on disk r = k.run(['/usr/bin/systemctl', 'show', '-p', - 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload', u]) + 'LoadState,FragmentPath,DropInPaths,UnitFileState,NeedDaemonReload,ActiveState', u]) got = dict(line.split('=', 1) for line in r.stdout.splitlines() if '=' in line) want = {'LoadState': 'loaded', 'FragmentPath': f'{k.UNIT_DIRS[0]}/{u}', 'DropInPaths': '', 'NeedDaemonReload': 'no'} if u != k.WALLS + '.service' and not before_enable: want['UnitFileState'] = 'enabled' + if u == k.WALLS + '.timer' and not before_enable: # enabled is not enough: a stopped timer checks nothing + want['ActiveState'] = 'active' seen += [f'{u}: {key}={got.get(key)!r}' for key, v in want.items() if r.returncode or got.get(key) != v] - out.append(('units', 'systemd runs the kit\'s units as written, enabled', seen and '; '.join(seen[:3]))) - out.append(('agent', f'one login user, {k.USER}, in no group but its own', k.user_problem(c, True))) + out.append(('units', 'systemd runs the kit\'s units as written, enabled, the timer running', + seen and '; '.join(seen[:3]))) + shell = k.login_shell(k.USER) + out.append(('agent', f'one login user, {k.USER}, in no group but its own, with no login shell', + k.user_problem(c, True) or (None if shell == k.NOLOGIN else + f'{k.USER}\'s login shell is {shell}, not {k.NOLOGIN}'))) + out.append(('sudoers', f'no sudo rule covers {k.USER}', k.sudo_problem(c))) current = k.lstat(k.OPT + '/current') target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index 1f69c0b77b80b..de9717f11eec8 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -25,7 +25,7 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'kit-code', 'runtime', + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'kit-code', 'runtime', 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) @@ -71,6 +71,7 @@ def edit(p, old, new): 'units': lambda: self.write('/log/units/t3code-agents.service.dropins', '/run/systemd/system/x.conf'), 'runtime': lambda: os.chmod(self.path(k.runtime(CONFIG) + '/t3'), 0o777), 'file:pc-agents-walls.timer': edit('/etc/systemd/system/pc-agents-walls.timer', '2min', '1d'), + 'sudoers': lambda: self.write('/etc/sudoers.d/90-agent', 'agent ALL=(ALL) NOPASSWD: ALL\n'), } clean = {p: v[4] for p, v in self.snapshot().items()} for wall, change in cases.items(): @@ -87,6 +88,28 @@ def again(self, **section): self.files() self.assertIsNone(k.load_firewall()) + def test_agent_has_no_login_shell(self): + self.write('/etc/passwd', self.read('/etc/passwd').replace('/home/agent:/usr/sbin/nologin', '/home/agent:/bin/bash')) + self.fails('agent') + + def test_sudo_rules_that_cover_the_agent(self): + base = 'Defaults env_reset\nroot ALL=(ALL:ALL) ALL\n%sudo ALL=(ALL:ALL) ALL\n@includedir /etc/sudoers.d\n' + self.write('/etc/sudoers', base + '# agent ALL=(ALL) ALL is only a comment\n') + self.write('/etc/sudoers.d/README', '# agent\n') + self.assertEqual(self.check()[0], 0) + for rule in ('agent ALL=(ALL) NOPASSWD: ALL', '%agent ALL=(ALL) ALL', '#1100 ALL=(ALL) ALL', 'ALL ALL=(ALL) ALL', + 'bob,agent ALL=(ALL) ALL', 'User_Alias AGENTS = bob, agent', 'agent \\\n ALL=(ALL) ALL'): + with self.subTest(rule=rule): + self.write('/etc/sudoers.d/x', rule + '\n') + self.fails('sudoers') + os.unlink(self.path('/etc/sudoers.d/x')) + self.write('/etc/sudoers', base + 'agent ALL=(ALL) ALL\n') + self.fails('sudoers') + + def test_timer_stopped_but_enabled(self): + os.unlink(self.path('/log/units/pc-agents-walls.timer.active')) + self.fails('units') + def test_agent_user_manager_masked(self): os.unlink(self.path('/etc/systemd/system/user@1100.service')) self.fails('masks') @@ -227,6 +250,46 @@ def test_unit(self): self.assertIn('\nStartLimitBurst=5\n', unit) self.assertIn('\n[user]\ndefault=root\n', files['/etc/wsl.conf'][0]) self.assertIn('\nOnUnitActiveSec=2min\n', files['/etc/systemd/system/pc-agents-walls.timer'][0]) + self.assertIn(' SHELL=/bin/bash\n', unit) # the agent's login shell is nologin; T3's shells are bash + walls = files['/etc/systemd/system/pc-agents-walls.service'][0] + for line in ('After=t3code-agents.service', 'TimeoutStartSec=5min', 'Type=oneshot'): + self.assertIn('\n' + line + '\n', walls) + + def test_loopback(self): + """The agent listens on and reaches over localhost only T3's port and devPorts: other distros' localhost + servers are out of reach, and what the agent serves is not forwarded to Windows' localhost on any port.""" + files = k.render(k.config()) + unit = files['/etc/systemd/system/t3code-agents.service'][0] + nft = files['/etc/pc-agents/firewall.nft'][0] + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3800\nSocketBindAllow=tcp:4100-4199\n', unit) + self.assertIn('\t\toifname "lo" tcp dport { 3800, 4100-4199 } accept\n\t\toifname "lo" drop\n', nft) + self.assertNotIn('oifname "lo" accept', nft) + self.assertLess(nft.index('oifname "lo" drop'), nft.index('ip daddr {')) + self.machine({**CONFIG, 'devPorts': '3700-3899'}) # T3's port inside the range: written once (nft) + files = k.render(k.config()) + self.assertIn('\nSocketBindDeny=any\nSocketBindAllow=tcp:3700-3899\nRestrict', files['/etc/systemd/system/t3code-agents.service'][0]) + self.assertIn('tcp dport { 3700-3899 } accept', files['/etc/pc-agents/firewall.nft'][0]) + for bad in ('80-90', '5000-4000', '4100', '4100-70000', ' 4100-4199', 4100): + with self.subTest(devPorts=bad): + self.machine({**CONFIG, 'devPorts': bad}) + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('devPorts', out) + + def test_loopback_rules_are_checked(self): + self.walled() + for name, change in (('unit', lambda: self.write('/etc/systemd/system/t3code-agents.service', self.read( + '/etc/systemd/system/t3code-agents.service').replace('SocketBindDeny=any\n', ''))), + ('loaded table', lambda: self.write('/log/nft-loaded', self.read('/log/nft-loaded').replace( + '\t\toifname "lo" drop\n', '')))): + with self.subTest(name): + shutil.rmtree(self.root) + self.setUp() + self.walled() + change() + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL ' + ('file:t3code-agents.service' if name == 'unit' else 'firewall'), out) def test_config(self): # The kit passes these; T3 itself tells a complete object from a partial one and does not start on a partial. diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index 79f79362ac740..eab2f08c38cc6 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -13,8 +13,9 @@ 'memoryMax': '8G', 'cpuQuota': '400%', 'nameservers': ['1.1.1.1', '9.9.9.9'], 'peers': ['100.101.102.103/32'], 'tailscaleTag': 'tag:pc-agents'} TOOLS = { # fake programs, under the fake root; each finds the root from its own path - 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; prev=; for a; do [ "$prev" = --uid ] && uid=$a; prev=$a; ' - 'done\necho "$a:x:$uid:$uid::/home/$a:/bin/bash" >>"$R/etc/passwd"\n' + 'usr/sbin/useradd': 'R="${0%/usr/sbin/useradd}"; uid=; sh=/bin/sh; prev=; for a; do [ "$prev" = --uid ] && uid=$a; ' + '[ "$prev" = --shell ] && sh=$a; prev=$a; done\n' + 'echo "$a:x:$uid:$uid::/home/$a:$sh" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' @@ -38,7 +39,8 @@ ' [ "$u" = pc-agents-walls.service ] && st=static\n' ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' - ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no;;\n' + ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no\n' + ' [ -e "$S/$u.active" ] && echo ActiveState=active || echo ActiveState=inactive;;\n' 'esac\n', 'usr/sbin/nft': 'R="${0%/usr/sbin/nft}"\ncase "$1" in -f) cp "$R$2" "$R/log/nft-loaded";;\n' 'list) cat "$R/log/nft-loaded" 2>/dev/null || exit 1;; esac\n', @@ -160,7 +162,7 @@ def walled(self, sha=SHA1): os.makedirs(os.path.dirname(self.path(p)), exist_ok=True) os.symlink('/dev/null', self.path(p)) with open(self.path('/etc/passwd'), 'a') as f: - f.write('agent:x:1100:1100::/home/agent:/bin/bash\n') + f.write('agent:x:1100:1100::/home/agent:/usr/sbin/nologin\n') with open(self.path('/etc/group'), 'a') as f: f.write('agent:x:1100:\n') for src, (dst, mode) in k.CODE.items(): From 315aa3beecd0c3356e8f8507a9a8718e5d5636d1 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:06:20 +0000 Subject: [PATCH 05/10] fork-ci-tests: run test files in parallel so the step fits its 10-minute limit The serial run had grown to about 500 s of tests on a 4-core runner and was cancelled at the job limit. Test files now run FORK_CI_TEST_JOBS at once (default: the CPU count), each one's output printed whole in its own group. A process a test leaves behind holding its output no longer keeps the run waiting. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 85 +++++++++++++++++++----- .github/scripts/fork-ci-tests.test.cjs | 90 +++++++++++++++++++++++++- 2 files changed, 159 insertions(+), 16 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 9b4d31a5a5991..07ac124c3c848 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -2,8 +2,11 @@ // A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, // which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less // tests fail; a listed test that is gone fails once the tested history had it, a note before. +// Test files run in parallel, FORK_CI_TEST_JOBS at once (default: the CPU count); each one's output +// is printed whole, in its own group, when it ends. const childProcess = require("node:child_process"); const fs = require("node:fs"); +const os = require("node:os"); const path = require("node:path"); const ROOT = path.resolve(__dirname, "../.."); @@ -119,24 +122,72 @@ function readFile(file) { } } -function main() { +// Calls start(item) for every item, at most `jobs` at once, each as soon as a slot is free. +// start returns a promise; the results come back in the items' order. +async function runAll(items, jobs, start) { + const results = new Array(items.length); + let next = 0; + async function worker() { + while (next < items.length) { + const index = next++; + results[index] = await start(items[index]); + } + } + await Promise.all(Array.from({ length: Math.max(1, Math.min(jobs, items.length)) }, worker)); + return results; +} + +function jobCount(env = process.env) { + const jobs = Number(env.FORK_CI_TEST_JOBS); + return Number.isInteger(jobs) && jobs > 0 ? jobs : os.availableParallelism(); +} + +// Runs one test file, its stdout and stderr kept together, and prints them as one group when it ends. +function runTest(file) { + const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; + const started = Date.now(); + return new Promise((resolve) => { + const child = childProcess.spawn(command, [...args, file], { + cwd: ROOT, + stdio: ["ignore", "pipe", "pipe"], + }); + const output = []; + child.stdout.on("data", (chunk) => output.push(chunk)); + child.stderr.on("data", (chunk) => output.push(chunk)); + let ended = false; + const done = (status, signal) => { + if (ended) return; + ended = true; + // Stop reading pipes a left-behind process still holds, so they do not keep this script alive. + child.stdout.destroy(); + child.stderr.destroy(); + const seconds = ((Date.now() - started) / 1000).toFixed(1); + process.stdout.write(`::group::${file} (${seconds} s)\n`); + process.stdout.write(Buffer.concat(output)); + process.stdout.write("\n::endgroup::\n"); + if (status !== 0) + console.log(`::error file=${file}::${file} failed (exit ${status ?? signal})`); + resolve({ file, status, seconds }); + }; + child.on("error", (error) => { + output.push(Buffer.from(`cannot start ${command}: ${error.message}\n`)); + done(null, "spawn error"); + }); + child.on("close", done); + // A process the test left behind may hold the pipes open: the test is done when it exits. + child.on("exit", (status, signal) => setTimeout(() => done(status, signal), 5000).unref()); + }); +} + +async function main() { const entries = readList(fs.readFileSync(path.join(ROOT, LIST), "utf8")); const result = plan(entries, presentTests(), everInHistory, readFile); const rows = []; let failed = result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; - for (const file of result.run) { - const [command, ...args] = RUNNER[file.match(TEST_FILE)[1]]; - const started = Date.now(); - console.log(`::group::${file}`); - const run = childProcess.spawnSync(command, [...args, file], { cwd: ROOT, stdio: "inherit" }); - console.log("::endgroup::"); - const seconds = ((Date.now() - started) / 1000).toFixed(1); - if (run.status !== 0) { - failed += 1; - console.log(`::error file=${file}::${file} failed (exit ${run.status ?? run.signal})`); - } - rows.push(`| \`${file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${seconds} s |`); + for (const run of await runAll(result.run, jobCount(), runTest)) { + if (run.status !== 0) failed += 1; + rows.push(`| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`); } for (const file of result.unlisted) console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); @@ -175,6 +226,10 @@ function main() { if (failed) process.exitCode = 1; } -module.exports = { plan, readList, findTests, presentTests }; +module.exports = { plan, readList, findTests, presentTests, runAll, jobCount }; -if (require.main === module) main(); +if (require.main === module) + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index da0290dc92daa..341da49ffe258 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -5,7 +5,7 @@ const os = require("node:os"); const path = require("node:path"); const test = require("node:test"); -const { plan, readList, findTests, presentTests } = require("./fork-ci-tests.cjs"); +const { plan, readList, findTests, presentTests, runAll, jobCount } = require("./fork-ci-tests.cjs"); const never = () => false; const here = (...files) => files.map((file) => ({ file, runBy: null })); @@ -143,3 +143,91 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => { fs.rmSync(root, { recursive: true, force: true }); } }); + +test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => { + let running = 0; + let most = 0; + const started = []; + const results = await runAll([30, 5, 5, 5, 5], 2, async (ms) => { + started.push(ms); + running += 1; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, ms)); + running -= 1; + return ms * 2; + }); + assert.equal(most, 2); + assert.deepEqual(results, [60, 10, 10, 10, 10]); + assert.equal(started.length, 5); + assert.deepEqual(await runAll([], 4, async () => 1), []); +}); + +test("jobCount reads FORK_CI_TEST_JOBS, else the CPU count", () => { + assert.equal(jobCount({ FORK_CI_TEST_JOBS: "3" }), 3); + for (const bad of [undefined, "", "0", "-2", "1.5", "many"]) + assert.equal(jobCount({ FORK_CI_TEST_JOBS: bad }), os.availableParallelism(), String(bad)); +}); + +// Runs the real script in a scratch tree whose list names `tests` ({ file: shell text }). +function runScript(tests, jobs) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "fork-ci-tests-")); + const write = (file, text) => { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), text); + }; + write(".github/scripts/fork-ci-tests.cjs", fs.readFileSync(require.resolve("./fork-ci-tests.cjs"))); + write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n"); + for (const [file, text] of Object.entries(tests)) write(file, text); + const env = { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid") }; + delete env.GITHUB_STEP_SUMMARY; + const started = Date.now(); + const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { + cwd: root, + env, + encoding: "utf8", + timeout: 60000, + }); + return { root, run, seconds: (Date.now() - started) / 1000 }; +} + +test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => { + const { root, run, seconds } = runScript( + { + "fork-ci/a.test.sh": "echo a-start; sleep 2; echo a-end\n", + "fork-ci/b.test.sh": "echo b-start; sleep 2; echo b-end\n", + "fork-ci/c.test.sh": "echo c-start; sleep 2; echo c-end; exit 3\n", + }, + 3, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + // One after another they take 6 s. + assert.ok(seconds < 5, `took ${seconds} s`); + for (const name of ["a", "b", "c"]) + assert.match( + run.stdout, + new RegExp(`::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`), + ); + assert.match(run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test("a process a test leaves behind holding its output does not hang the run", () => { + const { root, run, seconds } = runScript( + { "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' }, + 1, + ); + try { + assert.equal(run.error, undefined, String(run.error)); + assert.equal(run.status, 0, run.stdout + run.stderr); + assert.ok(seconds < 20, `took ${seconds} s`); + assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/); + } finally { + try { + process.kill(Number(fs.readFileSync(path.join(root, "left.pid"), "utf8"))); + } catch {} + fs.rmSync(root, { recursive: true, force: true }); + } +}); From 519449e8c462e7ebfd6c546c5b174f7f43e73dba Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:12:01 +0000 Subject: [PATCH 06/10] fork-ci-tests: format with vp fmt Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 4 +++- .github/scripts/fork-ci-tests.test.cjs | 29 +++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 07ac124c3c848..2f42fe040fdf1 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -187,7 +187,9 @@ async function main() { result.unlisted.length + result.deleted.length + result.noRunner.length + result.notRun.length; for (const run of await runAll(result.run, jobCount(), runTest)) { if (run.status !== 0) failed += 1; - rows.push(`| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`); + rows.push( + `| \`${run.file}\` | ${run.status === 0 ? "passed" : "**failed**"} | ${run.seconds} s |`, + ); } for (const file of result.unlisted) console.log(`::error file=${file}::${file} is not in ${LIST}; add it there`); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 341da49ffe258..c064d1dd994fb 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -5,7 +5,14 @@ const os = require("node:os"); const path = require("node:path"); const test = require("node:test"); -const { plan, readList, findTests, presentTests, runAll, jobCount } = require("./fork-ci-tests.cjs"); +const { + plan, + readList, + findTests, + presentTests, + runAll, + jobCount, +} = require("./fork-ci-tests.cjs"); const never = () => false; const here = (...files) => files.map((file) => ({ file, runBy: null })); @@ -175,10 +182,17 @@ function runScript(tests, jobs) { fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); fs.writeFileSync(path.join(root, file), text); }; - write(".github/scripts/fork-ci-tests.cjs", fs.readFileSync(require.resolve("./fork-ci-tests.cjs"))); + write( + ".github/scripts/fork-ci-tests.cjs", + fs.readFileSync(require.resolve("./fork-ci-tests.cjs")), + ); write(".github/fork-ci-tests.txt", Object.keys(tests).join("\n") + "\n"); for (const [file, text] of Object.entries(tests)) write(file, text); - const env = { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid") }; + const env = { + ...process.env, + FORK_CI_TEST_JOBS: String(jobs), + LEFT: path.join(root, "left.pid"), + }; delete env.GITHUB_STEP_SUMMARY; const started = Date.now(); const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { @@ -206,9 +220,14 @@ test("test files run in parallel, each one's output in its own group, and a fail for (const name of ["a", "b", "c"]) assert.match( run.stdout, - new RegExp(`::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`), + new RegExp( + `::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`, + ), ); - assert.match(run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/); + assert.match( + run.stdout, + /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/, + ); } finally { fs.rmSync(root, { recursive: true, force: true }); } From f446ea1264c32294fe7096ca793f5eac651380af Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:30:52 +0000 Subject: [PATCH 07/10] fork-ci-tests: prove parallel runs without clocks; shorter header The runAll test drives controlled promises and checks start order and the most running at once; the script test has two files that each wait for the other's marker file, so they pass only when run side by side (and fail with FORK_CI_TEST_JOBS=1). No timers or Date.now in the tests. Header cut to 3 lines. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.cjs | 9 ++-- .github/scripts/fork-ci-tests.test.cjs | 68 +++++++++++++++++++------- 2 files changed, 52 insertions(+), 25 deletions(-) diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 2f42fe040fdf1..696b307ef5188 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -1,9 +1,6 @@ -// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit must be listed in .github/fork-ci-tests.txt. -// A plain entry runs here by its extension (RUNNER); ` run by ` is run by that file, -// which must name the test and cannot be a (disabled) upstream workflow. Unlisted or runner-less -// tests fail; a listed test that is gone fails once the tested history had it, a note before. -// Test files run in parallel, FORK_CI_TEST_JOBS at once (default: the CPU count); each one's output -// is printed whole, in its own group, when it ends. +// Fork only: every `*.test.*` under .github/scripts, fork-ci and fork-kit is listed in .github/fork-ci-tests.txt and +// runs here by extension (RUNNER), FORK_CI_TEST_JOBS at once (default: CPU count), or `run by ` (not an upstream +// workflow; it names the test). Unlisted or runner-less tests fail; a listed one that is gone fails once history had it. const childProcess = require("node:child_process"); const fs = require("node:fs"); const os = require("node:os"); diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index c064d1dd994fb..1171f8153d5ac 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -152,20 +152,33 @@ test("an unlisted test under .github/scripts or fork-ci fails the run", () => { }); test("runAll runs at most `jobs` at once, starts the next as soon as a slot frees, keeps the order", async () => { + const runs = new Map(); // item -> resolve of its run, in start order + const waiting = []; // [count, resolve]: resolved once that many runs started + const started = (count) => + new Promise((resolve) => (runs.size >= count ? resolve() : waiting.push([count, resolve]))); let running = 0; let most = 0; - const started = []; - const results = await runAll([30, 5, 5, 5, 5], 2, async (ms) => { - started.push(ms); + const all = runAll(["a", "b", "c", "d"], 2, (item) => { running += 1; most = Math.max(most, running); - await new Promise((resolve) => setTimeout(resolve, ms)); - running -= 1; - return ms * 2; + const done = new Promise((resolve) => runs.set(item, resolve)).then(() => { + running -= 1; + return item.toUpperCase(); + }); + for (const [count, resolve] of waiting) if (runs.size >= count) resolve(); + return done; }); + await started(2); + assert.deepEqual([...runs.keys()], ["a", "b"]); // two at once, c waits for a free slot + runs.get("b")(); // the second ends first: its slot takes c + await started(3); + assert.deepEqual([...runs.keys()], ["a", "b", "c"]); + runs.get("a")(); + await started(4); + runs.get("d")(); + runs.get("c")(); + assert.deepEqual(await all, ["A", "B", "C", "D"]); assert.equal(most, 2); - assert.deepEqual(results, [60, 10, 10, 10, 10]); - assert.equal(started.length, 5); assert.deepEqual(await runAll([], 4, async () => 1), []); }); @@ -192,31 +205,35 @@ function runScript(tests, jobs) { ...process.env, FORK_CI_TEST_JOBS: String(jobs), LEFT: path.join(root, "left.pid"), + MARKS: root, }; delete env.GITHUB_STEP_SUMMARY; - const started = Date.now(); const run = childProcess.spawnSync("node", [".github/scripts/fork-ci-tests.cjs"], { cwd: root, env, encoding: "utf8", timeout: 60000, }); - return { root, run, seconds: (Date.now() - started) / 1000 }; + return { root, run }; } +// A test file that passes only when `other` starts while it runs (it waits up to 5 s for the other's mark). +const meets = (name, other) => + `echo ${name}-start; : > "$MARKS/${name}"\n` + + `for i in $(seq 50); do [ -e "$MARKS/${other}" ] && { echo ${name}-end; exit 0; }; sleep 0.1; done\n` + + `echo "${other} did not run beside ${name}"; exit 1\n`; + test("test files run in parallel, each one's output in its own group, and a failure fails the run", () => { - const { root, run, seconds } = runScript( + const { root, run } = runScript( { - "fork-ci/a.test.sh": "echo a-start; sleep 2; echo a-end\n", - "fork-ci/b.test.sh": "echo b-start; sleep 2; echo b-end\n", - "fork-ci/c.test.sh": "echo c-start; sleep 2; echo c-end; exit 3\n", + "fork-ci/a.test.sh": meets("a", "b"), + "fork-ci/b.test.sh": meets("b", "a"), + "fork-ci/c.test.sh": "echo c-start; echo c-end; exit 3\n", }, 3, ); try { assert.equal(run.status, 1, run.stdout + run.stderr); - // One after another they take 6 s. - assert.ok(seconds < 5, `took ${seconds} s`); for (const name of ["a", "b", "c"]) assert.match( run.stdout, @@ -224,6 +241,7 @@ test("test files run in parallel, each one's output in its own group, and a fail `::group::fork-ci/${name}\\.test\\.sh \\([0-9.]+ s\\)\\n${name}-start\\n${name}-end\\n`, ), ); + assert.doesNotMatch(run.stdout, /::error file=fork-ci\/[ab]\.test\.sh/); assert.match( run.stdout, /::error file=fork-ci\/c\.test\.sh::fork-ci\/c\.test\.sh failed \(exit 3\)/, @@ -233,15 +251,27 @@ test("test files run in parallel, each one's output in its own group, and a fail } }); +test("FORK_CI_TEST_JOBS=1 runs one file at a time", () => { + const { root, run } = runScript( + { "fork-ci/a.test.sh": meets("a", "b"), "fork-ci/b.test.sh": meets("b", "a") }, + 1, + ); + try { + assert.equal(run.status, 1, run.stdout + run.stderr); + assert.match(run.stdout, /b did not run beside a/); // b started only after a ended + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + test("a process a test leaves behind holding its output does not hang the run", () => { - const { root, run, seconds } = runScript( + const { root, run } = runScript( { "fork-ci/left.test.sh": 'sleep 60 & echo $! > "$LEFT"; echo done\n' }, 1, ); try { - assert.equal(run.error, undefined, String(run.error)); + assert.equal(run.error, undefined, String(run.error)); // not killed at the 60 s limit: the run ended assert.equal(run.status, 0, run.stdout + run.stderr); - assert.ok(seconds < 20, `took ${seconds} s`); assert.match(run.stdout, /::group::fork-ci\/left\.test\.sh[^\n]*\ndone\n/); } finally { try { From d6f9e208521046a4dd415a8ac6921490db125a41 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:45:07 +0000 Subject: [PATCH 08/10] fork tests: no pipe into a reader that stops early (SIGPIPE race) `sed ... | grep -q` under pipefail fails when grep exits at the match while sed still writes: a race (5 in 2000 runs idle, 45 under load) that the parallel runner made show up in fork-follow.test.sh. Ten such pipes (grep -q, head) now read the whole output first; two of them failed open (`! ... | grep -q`). fork-ci-tests.test.cjs fails on any new one in a listed shell test. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-ci-tests.test.cjs | 41 +++++++++++++++++++ .github/scripts/fork-follow.test.sh | 14 +++---- .../scripts/fork-interaction-limit.test.sh | 4 +- .github/scripts/fork-secret-scan.test.sh | 2 +- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 1171f8153d5ac..3fbda8b020161 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -280,3 +280,44 @@ test("a process a test leaves behind holding its output does not hang the run", fs.rmSync(root, { recursive: true, force: true }); } }); + +// A pipe into a reader that stops early (`grep -q`, `grep -m`, `head`) kills the writer with SIGPIPE +// whenever it has more to write; under `pipefail` that reads as a failed check, more often under load. +const earlyExitPipe = + /(? + text + .split("\n") + .map((line, index) => ({ line, number: index + 1 })) + .filter(({ line }) => !/^\s*#/.test(line) && earlyExitPipe.test(line)); + +test("a pipe into grep -q, grep -m or head is found; reading the whole output first is not", () => { + const found = earlyExitPipes( + [ + 'sed -n "/a/,$p" f | grep -q x', + "! calls | grep -vqE y", + "git show c | grep -qx z", + "seq 9 | grep -m1 3", + 'sed p <<< "$s" | head -n 1', + 'grep -q x <<< "$(sed -n p f)"', + "a || grep -q x f", + "# a comment: `x | grep -q` stops early", + '[[ -z "$(grep -E a f | grep -vE b)" ]]', + ].join("\n"), + ).map(({ number }) => number); + assert.deepEqual(found, [1, 2, 3, 4, 5]); +}); + +test("no listed shell test pipes into a reader that stops early", () => { + const root = path.resolve(__dirname, "../.."); + const shellTests = readList(fs.readFileSync(path.join(root, ".github/fork-ci-tests.txt"), "utf8")) + .map(({ file }) => file) + .filter((file) => file.endsWith(".sh") && fs.existsSync(path.join(root, file))); + assert.ok(shellTests.length > 0); + const found = shellTests.flatMap((file) => + earlyExitPipes(fs.readFileSync(path.join(root, file), "utf8")).map( + ({ line, number }) => `${file}:${number}: ${line.trim()}`, + ), + ); + assert.deepEqual(found, []); +}); diff --git a/.github/scripts/fork-follow.test.sh b/.github/scripts/fork-follow.test.sh index 72eafc76779d3..cb5a252047a79 100755 --- a/.github/scripts/fork-follow.test.sh +++ b/.github/scripts/fork-follow.test.sh @@ -441,7 +441,7 @@ plan check "plan: upstream edits a deleted listed file -> the conflict is resolved by deleting it, rebase" \ eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.11 ]] && ! in_candidate ci.yml && [[ ! -e "$root/report.md" ]]' check "plan: the resolved rebase keeps the fork's patches and its list" \ - eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -qx .github/workflows/ci.yml' + eval 'action_is rebase && git -C "$root/work" cat-file -e "$(out candidate):feature.txt" && grep -qx .github/workflows/ci.yml <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Re-run: an earlier run published this candidate, then stopped before the branch moved. The # next run rebuilds the same files and advances to that release instead of building again. released="$(out candidate)" @@ -470,7 +470,7 @@ new_fork v0.1.11 fork_commit "fork: list ci.yml" 'echo .github/workflows/ci.yml >> .github/fork-removed-workflows.txt' plan check "plan: upstream brings a listed file back -> deleted again on the candidate, rebase" \ - eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && git -C "$root/work" log -1 --format=%s "$(out candidate)" | grep -q "fork: delete upstream workflows listed"' + eval 'action_is rebase && [[ "$(out base_tag)" == v0.1.12 ]] && ! in_candidate ci.yml && grep -q "fork: delete upstream workflows listed" <<< "$(git -C "$root/work" log -1 --format=%s "$(out candidate)")"' new_upstream new_fork v0.1.10 @@ -506,7 +506,7 @@ new_fork v0.1.10 fork_commit "fork: delete ci.yml, CRLF list" "printf '# deleted\\r\\n.github/workflows/ci.yml\\r\\n' > .github/fork-removed-workflows.txt && rm .github/workflows/ci.yml" plan check "plan: a list with CRLF line ends is read like one with LF, the listed file stays deleted" \ - eval 'action_is rebase && ! in_candidate ci.yml && git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt" | grep -q $'"'"'ci.yml\r$'"'"'' + eval 'action_is rebase && ! in_candidate ci.yml && grep -q $'"'"'ci.yml\r$'"'"' <<< "$(git -C "$root/work" show "$(out candidate):.github/fork-removed-workflows.txt")"' # Upstream renames a listed file: git reports a rename/delete conflict on the new name, which the # list does not name, so the plan stops and names it rather than shipping it under that name. @@ -1054,9 +1054,9 @@ check "workflow: advance's if: is a conjunction (no top-level ||) of the plan's check "workflow: the move step runs only when the checks before it passed (no if:)" \ eval 'awk "/- name: Move the series branch/ { s = 1; next } s && /^ - / { exit } s && /^ if:/ { bad = 1 } END { exit bad }" <<< "$advance_job"' check "workflow: action none deletes a leftover candidate branch" \ - eval 'grep -B2 -A3 "action == .none." "$workflow" | grep -q "fork-follow-push.sh cleanup"' + eval 'grep -q "fork-follow-push.sh cleanup" <<< "$(grep -B2 -A3 "action == .none." "$workflow")"' check "workflow: the last sweep reports runs since the run began" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "SINCE: \${{ needs.guard.outputs.started }}"' + eval 'grep -q "SINCE: \${{ needs.guard.outputs.started }}" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' # Each job that reads FORK_PUSH_TOKEN runs in the Environment that holds it, named as in env. token_jobs="$(grep -vF "$presence" "$workflow" | awk '/^ [a-z_-]+:$/ { job = $1 } /secrets\.FORK_PUSH_TOKEN/ { print job }' | sort -u)" env_jobs="$(awk '/^ [a-z_-]+:$/ { job = $1 } /^ environment: fork-push$/ { print job }' "$workflow" | sort -u)" @@ -1067,9 +1067,9 @@ check "workflow: the jobs that read FORK_PUSH_TOKEN, and only they, run in the E check "workflow: no job uses actions/checkout; four plain-git checkouts, none with a token" \ eval '! grep -q "actions/checkout@" "$workflow" && [[ $(grep -c "^ git init -q$" "$workflow") == 4 ]]' check "workflow: every action is pinned by commit" \ - eval '! grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vqE "@[0-9a-f]{40}( |$)"' + eval '[[ -z "$(grep -E "^\s*-?\s*uses: [^.][^@]*@" "$workflow" | grep -vE "@[0-9a-f]{40}( |$)")" ]]' check "workflow: the report runs on every run (also action none)" \ - eval 'sed -n "/^ report:/,\$p" "$workflow" | grep -q "^ if: always()$"' + eval 'grep -q "^ if: always()$" <<< "$(sed -n "/^ report:/,\$p" "$workflow")"' check "workflow: upstream workflows are swept after each push" \ eval '[[ $(grep -c "fork-workflows.sh" "$workflow") -ge 3 ]]' diff --git a/.github/scripts/fork-interaction-limit.test.sh b/.github/scripts/fork-interaction-limit.test.sh index 63424f684b844..0c6a499be68c3 100644 --- a/.github/scripts/fork-interaction-limit.test.sh +++ b/.github/scripts/fork-interaction-limit.test.sh @@ -107,8 +107,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } # --- check ------------------------------------------------------------------------------------- reset; limit collaborators_only 180; run check diff --git a/.github/scripts/fork-secret-scan.test.sh b/.github/scripts/fork-secret-scan.test.sh index fdddfabb60641..675a9713b543f 100755 --- a/.github/scripts/fork-secret-scan.test.sh +++ b/.github/scripts/fork-secret-scan.test.sh @@ -61,7 +61,7 @@ elif [[ -n "${GITLEAKS_CONFIG:-}" ]]; then settings="$(cat "$GITLEAKS_CONFIG")" elif [[ -n "${GITLEAKS_CONFIG_TOML:-}" ]]; then settings="$GITLEAKS_CONFIG_TOML" elif [[ -f "$source/.gitleaks.toml" ]]; then settings="$(cat "$source/.gitleaks.toml")" fi -extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | head -n 1)" +extend_path="$(sed -nE "s/^[[:space:]]*path[[:space:]]*=[[:space:]]*[\"']([^\"']*)[\"'].*/\1/p" <<< "$settings" | sed -n 1p)" if [[ "$settings" == *"[extend]"* && -n "$extend_path" && ! -f "$extend_path" ]]; then echo "FTL Failed to load config error=\"failed to load extended config, err: open $extend_path: no such file or directory\"" exit 1 From 4f70d86f459e247b0cafc8d7ad529f6b77c143cc Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:47:05 +0000 Subject: [PATCH 09/10] pc-agents setup: restart wall, stays-up wait, env lookup, round 2 fixes - restart wall: setup marks T3 for a restart (boot id + monotonic time) before and after every change; check fails, and --enforce stops T3 and as-agent units, while a T3 that started before the marker runs. - after the restart setup watches T3 for 15 s before "done" and removes the marker only then; reset-failed first, so a rerun within the start limit still starts T3. - as-agent runs the command through /usr/bin/env with the agent's PATH (~/.local/bin first); the agent gets /usr/sbin/nologin (usermod for an agent an older kit made). - SIGTERM/SIGHUP kill the running child's process group too. - README: the restart wall, Windows logon task to keep the distro up, limits, step 6 wording. Co-Authored-By: Claude Opus 5.5 --- fork-kit/pc-agents/README.md | 73 +++++++-- fork-kit/pc-agents/lib/pc_agents.py | 64 +++++++- fork-kit/pc-agents/sbin/pc-agents | 76 ++++++--- fork-kit/pc-agents/tests/check.test.py | 72 ++++++++- fork-kit/pc-agents/tests/fake_distro.py | 43 +++-- fork-kit/pc-agents/tests/setup.test.py | 201 +++++++++++++++++++++++- 6 files changed, 469 insertions(+), 60 deletions(-) diff --git a/fork-kit/pc-agents/README.md b/fork-kit/pc-agents/README.md index 2c0f0b6d21c8b..3fb5f6048ed5b 100644 --- a/fork-kit/pc-agents/README.md +++ b/fork-kit/pc-agents/README.md @@ -32,7 +32,11 @@ so the agent cannot undo it: - no Windows drive is mounted; - the loaded firewall table is the one from the kit's `firewall.nft`: `pc-agents load-firewall` saves the file's hash and what the kernel then lists, and any later change, or an older file left loaded, differs; -- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through. +- the default gateway (the Windows host) is in a blocked range, and no `allow` or `peers` entry lets it through; +- no T3 runs that started before setup's last change: while setup's marker `/etc/pc-agents/restart-pending` is there + (see "Runs again" below), a T3 process started before the time in it fails this wall, and so does a marker or a + start time it cannot read. So the 2-minute check stops a T3 that a stopped or failed setup left running with old + files, and `pc-agents as-agent` refuses meanwhile. Setup's own first check skips it. The fleet's `mounts`, `sockets` and `userns` walls are about the CI launcher's job units, so they are not reported here; the unit's own sandbox is checked instead (it must @@ -89,11 +93,13 @@ refuses what is cheap to tell (a field of a provider instance, an object deeper two settings, a blank `organization`); T3 refuses the rest, like a field inside a setting that can be empty (lock the whole setting), says why in its log (`journalctl -u t3code-agents`) and does not start. T3 exits 1 on a policy it cannot apply, the same code as any other failure, so the unit cannot tell it apart (`RestartPreventExitStatus=`); -it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). +it gives up after 5 starts in 10 minutes instead (`StartLimitBurst=5`). Setup clears that count +(`systemctl reset-failed`) before it starts T3, so running it again within the 10 minutes still starts T3. A wall that fails stops T3, and nothing starts it again by itself: fix what `pc-agents check` names (or run `pc-agents setup` again), then start it as root: `systemctl start t3code-agents.service` (its own start runs the -check first and refuses on a failed wall). +check first and refuses on a failed wall). If it says `start request repeated too quickly`, run +`systemctl reset-failed t3code-agents.service` first. ## Owner steps (once per PC) @@ -117,19 +123,35 @@ Commands are for Windows PowerShell unless a step says otherwise. `` is 6. **Join Tailscale** with the tag (as root in the distro): install it (`curl -fsSL https://tailscale.com/install.sh | sh`), then `tailscale up --advertise-tags= --accept-dns=false` (keep the kit's public nameservers; MagicDNS is on the tailnet, which the agent cannot reach) and open the link it - prints in a browser on any machine. Put the linking T3's Tailscale address in `peers` (step 2's file) and run step 3 - again. The ACL is below. + prints in a browser on any machine. The linking T3's Tailscale address is already in `peers` (step 2's file needs + it; setup refuses a file without it); if that address changes, edit the file and run step 3 again. The ACL is + below. 7. **Log in to Claude as the agent:** never open a shell as `agent` (`wsl -u agent` would start the agent's own session, outside T3's walls). Run each command through the kit instead, which runs it as `agent` under the same walls as T3 (a transient system unit, `systemd-run`): `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- sh -c ''`, - then the same with `-- claude`; open the link it prints in your browser. Codex: the same with `-- codex login`. + then the same with `-- claude`; open the link it prints in your browser. The kit runs the command through `env` + with the agent's `PATH` (`/home/agent/.local/bin` first), so a bare `claude` is found there, and with only + `HOME`, `T3CODE_HOME`, `PATH` and `SHELL` set. Codex: its browser login waits on `localhost` port 1455, outside + `devPorts`, so the walls refuse it: use its device-code login instead, the same with `-- codex login --device-auth` + (`codex login --help` names it). 8. **Log in to GitHub as the agent:** install `gh` (as root, from GitHub CLI's docs), then `wsl -d -u root -- /opt/pc-agents/current/sbin/pc-agents as-agent -- gh auth login --with-token` and paste a fine-grained token for the agents' repos into that window only. 9. **Link from the other T3:** on this PC's T3 (`http://:`), Settings → Sharing & access → make a pairing code. In the other T3: Settings → Environments → Linked environments → Link, the same address and the code, **Full access**. +10. **Keep the distro running:** WSL stops a distro soon after its last window closes, and does not start one when + Windows starts. A logon task of your own (no administrator needed) keeps this one running while you are logged + in. In PowerShell, as yourself (the same task as the desk starter's, `fork-kit/desk/windows/start-desk.ps1`): + ```powershell + $a = New-ScheduledTaskAction -Execute 'conhost.exe' -Argument '--headless wsl.exe -d -u root --exec sleep infinity' + Register-ScheduledTask -TaskName 'pc-agents ' -Action $a -Trigger (New-ScheduledTaskTrigger -AtLogOn -User ([Security.Principal.WindowsIdentity]::GetCurrent().Name)) -Settings (New-ScheduledTaskSettingsSet -ExecutionTimeLimit 0 -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries) -Force + Start-ScheduledTask -TaskName 'pc-agents ' + ``` + `conhost.exe --headless` shows no window. Running it again replaces the task. A distro set up before this step + only needs these lines; nothing in the distro changes. To remove it: + `Unregister-ScheduledTask -TaskName 'pc-agents ' -Confirm:$false`. Run step 3 again for a new release or a changed machine file (and steps 4 and 5 if it asks); T3 restarts only when something changed. To remove everything: `wsl --unregister `. @@ -154,15 +176,26 @@ allowed (connection tracking). - **Again:** every file is compared before it is written; an unchanged file is not touched, and T3 restarts only when a file, the release or the T3 version changed. The tests run the setup twice and find no change. -- **Stopped half-way:** each file is written whole (a new file, then one rename). Before the first change, setup - writes the root-owned marker `/etc/pc-agents/restart-pending`; it is removed only after T3 restarted and is - running. So a run stopped anywhere, or a failed T3 install, leaves the marker, and the next good run restarts T3 - even when it has nothing left to write (an old T3 never keeps running under new files). The record +- **Stopped half-way:** each file is written whole (a new file, then one rename). Before and after every change + (and after a T3 install, and once more right before its restart), setup writes the root-owned marker + `/etc/pc-agents/restart-pending` with this boot's id and the time (`CLOCK_MONOTONIC`, the clock of systemd's + `ExecMainStartTimestampMonotonic`, which no clock change moves). It is removed only after T3 restarted and stayed + up for 15 seconds (the same process, active, no automatic restart). So a run stopped anywhere, or a failed T3 + install, leaves the marker, and the next good run restarts T3 even when it has nothing left to write. Meanwhile + the `restart` wall fails while the T3 that runs started before the marker's time, so the 2-minute check stops it + (an old T3 never keeps running under new files and a passing check). A run stopped during setup's own restart or + its 15 seconds leaves a T3 started after the marker: the wall passes, and the next run restarts it once more. A + marker of an earlier boot passes (every T3 now started after it); one an older kit wrote, with no time in it, + fails until setup runs again. A T3 that starts and then exits within the 15 seconds is reported (`T3 did not + start or stay up`), and the marker stays. The record (`/etc/pc-agents/installed.tsv`) holds a file's old and new hash while it is replaced, so a run stopped right after a write, followed by an edit of the machine file, is not mistaken for a hand edit. A new T3 version is installed by root after the new firewall is loaded, so a slow or failed download never leaves the old firewall in place. -- **Timeouts:** every program setup runs has a time limit (30 minutes for T3's download, 15 for `apt-get`); past it - the whole process group is stopped and setup says so. Run it again. +- **Timeouts and signals:** every program setup runs has a time limit (30 minutes for T3's download, 15 for + `apt-get`); past it the whole process group is stopped and setup says so. Run it again. Ctrl-C, `SIGTERM` or + `SIGHUP` (a closed window) stop setup the same way, the running download's whole group too. +- **Start limit:** a rerun within 10 minutes of five failed starts still starts T3 (`systemctl reset-failed` first). +- **An older kit's distro:** an `agent` made with a login shell gets `/usr/sbin/nologin` (`usermod`), once. - **A distro that is not fresh:** refused before anything is written, with the reasons: another login user, an `agent` with another uid or in another group, a file the kit would write that it did not write (or that was edited since, for example a hand-written `wsl.conf` or `policy.json`), a cron, at or ssh unit where the kit masks @@ -206,7 +239,12 @@ allowed (connection tracking). - **DNS** goes to the public `nameservers` only. A `systemd-resolved` stub (`127.0.0.53`), which may resolve LAN names through Windows, is out of reach: the agent's localhost traffic outside T3's port and `devPorts` is dropped. - **Terminals:** what the agent prints in a terminal can use escape codes, such as OSC 52, which some terminals turn - into a write to the Windows clipboard. Do not paste from the clipboard blindly after watching agent output. + into a write to the Windows clipboard. `pc-agents as-agent` runs with `--pty`, so the owner steps' output reaches + your terminal that way too. Do not paste from the clipboard blindly after watching agent output. +- **Runs only while you are logged in:** the logon task (step 10) keeps the distro running while you are logged in + and the PC is awake. Logging out, sleep, a Windows restart or `wsl --shutdown` stop T3 (the linking T3 sees this + PC offline); the task starts it again at your next logon. Without the task, T3 stops soon after the last WSL + window closes. - **No namespaces in the unit** (`RestrictNamespaces=yes`): provider sandboxes that need them (bubblewrap) do not run; the distro is the sandbox, so use full access. - **Tailscale, Claude Code and `gh` are yours to install** (steps 6 to 8): their installers and logins need you. @@ -224,7 +262,10 @@ python3 -I -B fork-kit/pc-agents/tests/check.test.py ``` A fake distro in a temporary folder (`PC_AGENTS_ROOT`, refused for root; `tests/fake_distro.py`), with fake -`useradd`, T3's installer, `systemctl` (it keeps which units are enabled and running), `systemd-run` and `nft`: every -file and mode, a second run changes nothing, a run stopped at every write finishes on the next and restarts T3, each -kind of foreign distro is refused, and each wall fails on its own when its part is taken away. +`useradd`, `usermod`, T3's installer, `systemctl` (it keeps which units are enabled and running, their main process +and start time, a start limit, and a T3 that dies a moment after it starts), `systemd-run` (a bare command is looked +up in the caller's `PATH`, as the real one does) and `nft`: every file and mode, a second run changes nothing, a run +stopped at every write finishes on the next and restarts T3, an upgrade stopped at every write fails `check` until +setup runs again, `SIGTERM` and `SIGHUP` end the download too, each kind of foreign distro is refused, and each wall +fails on its own when its part is taken away. `apps/server/src/policy/managedSettings.forkKeys.test.ts` proves the managed policy locks the fork's settings. diff --git a/fork-kit/pc-agents/lib/pc_agents.py b/fork-kit/pc-agents/lib/pc_agents.py index 5a9ce7922304d..59b45718cc599 100644 --- a/fork-kit/pc-agents/lib/pc_agents.py +++ b/fork-kit/pc-agents/lib/pc_agents.py @@ -2,7 +2,7 @@ Every path is read and written under ROOT, which is '' (the real machine) except in the tests, where PC_AGENTS_ROOT names a fake one; root never gets a fake one. Standard library only.""" -import hashlib, ipaddress, json, os, re, signal, subprocess +import hashlib, ipaddress, json, os, re, signal, subprocess, time import machine_file @@ -16,6 +16,9 @@ OPT, STATE = '/opt/pc-agents', '/etc/pc-agents' RECORD = STATE + '/installed.tsv' # sha256 path of every file this kit wrote (a path may have two mid-write) PENDING = STATE + '/restart-pending' # setup changed something T3 runs with: restart T3 before removing this +BOOT_ID = '/proc/sys/kernel/random/boot_id' # the marker holds this boot's id and CLOCK_MONOTONIC, like systemd's +AS_AGENT = 'pc-agents-as-agent' # name prefix of `pc-agents as-agent`'s transient units (check --enforce stops them) +SETTLE, SETTLE_STEP = 15, 1 # after a (re)start, setup watches T3 this many seconds, every SETTLE_STEP, before done LOADED = '/run/pc-agents/firewall.loaded' # sha256 of the firewall file and `nft list` of it, saved when loaded T3 = OPT + '/t3' # T3's runtime, installed by root (T3CODE_HOME of scripts/install.sh): the agent cannot change it UNIT, WALLS = 't3code-agents.service', 'pc-agents-walls' @@ -226,7 +229,7 @@ def render(c): sd + WALLS + '.service': (HEAD + f'[Unit]\nDescription=pc-agents walls check\nAfter={UNIT}\n\n[Service]\n' f'Type=oneshot\nTimeoutStartSec=5min\nExecStart={CHECK} check --enforce\n' 'ExecStopPost=/bin/sh -c \'[ "$$SERVICE_RESULT" = success ] || ' - f'exec /usr/bin/systemctl stop {UNIT}\'\n', 0o644), + f'exec /usr/bin/systemctl stop {UNIT} "{AS_AGENT}-*.service"\'\n', 0o644), sd + WALLS + '.timer': (HEAD + '[Unit]\nDescription=pc-agents walls check every 2 minutes\n\n[Timer]\n' 'OnActiveSec=1min\nOnUnitActiveSec=2min\nAccuracySec=10s\n\n[Install]\n' 'WantedBy=timers.target\n', 0o644), @@ -329,6 +332,58 @@ def login_shell(name): return None +def clock(): + """(this boot's id, CLOCK_MONOTONIC in microseconds): systemd's *TimestampMonotonic clock, which no clock change + or WSL time sync moves.""" + with open(ROOT + BOOT_ID, encoding='ascii') as f: + return f.read().strip(), time.monotonic_ns() // 1000 + + +def marker(): + """The restart marker's text: when setup last changed something T3 runs with (boot id, monotonic microseconds).""" + boot, now = clock() + return f'boot_id={boot} monotonic_us={now}\nsetup changed what T3 runs with; the next good run restarts T3 and ' \ + 'removes this. Until then `pc-agents check` fails while a T3 started before this time runs.\n' + + +def props(text): + """`systemctl show` output as a dict.""" + return dict(line.split('=', 1) for line in text.splitlines() if '=' in line) + + +def restart_problem(): + """Why a T3 runs that started before setup's last change (a restart is pending), or None. No marker, or no T3 + process (stopped, or still in its ExecStartPre check): nothing stale runs. Anything unreadable fails.""" + if lstat(PENDING) is None: + return None + r = run(['/usr/bin/systemctl', 'show', '-p', 'MainPID,ExecMainStartTimestampMonotonic', UNIT]) + got = props(r.stdout) + if r.returncode or not re.fullmatch(r'\d+', got.get('MainPID', '')): + return f'cannot tell whether T3 runs ({(r.stderr or r.stdout).strip()[-200:]})' + if got['MainPID'] == '0': + return None + try: + with open(ROOT + PENDING, encoding='utf-8', errors='replace') as f: + m = re.match(r'boot_id=(\S+) monotonic_us=(\d+)\n', f.read()) + boot = clock()[0] + except OSError as e: + return f'cannot read {PENDING} or the boot id ({e})' + started = got.get('ExecMainStartTimestampMonotonic', '') + if not m or not re.fullmatch(r'[1-9]\d*', started): + return f'a restart is pending ({PENDING}) and T3\'s start time or the marker\'s cannot be read' + if m[1] == boot and int(started) <= int(m[2]): # a marker of an earlier boot is older than any T3 now + return f'T3 runs as started before setup\'s last change ({PENDING}): run pc-agents setup again' + return None + + +def stop_on_signals(): + """SIGTERM and SIGHUP end this process like Ctrl-C does, through run()'s clean-up (its child's group is killed).""" + def stop(signum, frame): + raise SystemExit(128 + signum) + for s in (signal.SIGTERM, signal.SIGHUP): + signal.signal(s, stop) + + def sudo_problem(c): """Why sudo could give the agent more (a sudoers line naming it, its group or uid, or every user), or None. Harmless inside the unit (NoNewPrivileges), but a way out from anywhere else.""" @@ -368,8 +423,9 @@ def user_problem(c, need_agent): def run(argv, timeout=120): - """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops, the - whole group is killed (a download's curl too); a timeout returns exit 124 and says so in stderr.""" + """Run argv (argv[0] under ROOT) in a process group of its own. Past timeout, or when this process stops (Ctrl-C, + or SIGTERM and SIGHUP once stop_on_signals() ran), the whole group is killed (a download's curl too); a timeout + returns exit 124 and says so in stderr.""" p = subprocess.Popen([ROOT + argv[0]] + argv[1:], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) try: diff --git a/fork-kit/pc-agents/sbin/pc-agents b/fork-kit/pc-agents/sbin/pc-agents index 55aa34880a8c4..d30404ed22481 100755 --- a/fork-kit/pc-agents/sbin/pc-agents +++ b/fork-kit/pc-agents/sbin/pc-agents @@ -3,12 +3,13 @@ pc-agents setup --source DIR --sha SHA as root, in a fresh distro: make or repair everything; safe to run again pc-agents check [--enforce] every wall, PASS or FAIL; exit 0 only when all hold. --enforce also - stops T3 when one fails (the timer, every 2 minutes). T3's unit runs - it before every start, so T3 never starts on a failed wall. + stops T3 and every as-agent unit when one fails (the timer, every 2 + minutes). T3's unit runs it before every start, so T3 never starts on + a failed wall. pc-agents load-firewall as root: load the agent firewall and note what the kernel lists pc-agents as-agent -- CMD... as root: run CMD as the agent under T3's unit walls (the owner steps) Standard library only.""" -import os, re, stat, sys +import os, re, stat, sys, time HERE = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) LIB = os.path.join(HERE, 'lib') @@ -36,10 +37,12 @@ def same(path, data, mode, link=False): def put(path, data, mode, link=False): """Write a file (or a link) whole: a new file beside it, then one rename. Returns True when it changed. Every - change but to the record marks T3 for a restart first, so a run stopped after it still restarts T3 next time.""" + change but to the record marks T3 for a restart before and after it, so a run stopped after it still restarts T3 + next time, and `check` fails on a T3 that started before it.""" if same(path, data, mode, link): return False - if path not in (k.RECORD, k.PENDING): + marks = path not in (k.RECORD, k.PENDING) + if marks: mark() real = k.ROOT + path os.makedirs(os.path.dirname(real), mode=0o755, exist_ok=True) @@ -57,13 +60,15 @@ def put(path, data, mode, link=False): os.fchown(f.fileno(), 0, 0) os.fsync(f.fileno()) os.replace(tmp, real) + if marks: + mark() return True def mark(): - """Note that T3 must restart (root's marker, removed only once T3 restarted and runs).""" - if not k.lstat(k.PENDING): - put(k.PENDING, 'setup changed what T3 runs with; the next good run restarts T3 and removes this\n', 0o644) + """Note that T3 must restart, and when setup last changed something (root's marker, rewritten on every change and + removed only once T3 restarted and stayed up). A T3 started before that time is stale: `check` fails on it.""" + put(k.PENDING, k.marker(), 0o644) def keep(done): @@ -126,6 +131,7 @@ def install_t3(c, version, done): timeout=1800) if r.returncode or k.lstat(folder + '/t3') is None: return f'T3 install failed: {(r.stderr or r.stdout).strip()[-400:]}' + mark() if not done.get(folder + '/t3'): # a mismatch is left to the runtime wall; only a missing hash is added done[folder + '/t3'] = {k.file_sha(folder + '/t3')} keep(done) @@ -147,10 +153,14 @@ def setup(source, sha): if bad: return say('refused, nothing changed:\n ' + '\n '.join(bad)) or 1 if k.USER not in k.users()[0]: - r = k.run(['/usr/sbin/useradd', '--create-home', '--home-dir', k.HOME, '--shell', '/bin/bash', + r = k.run(['/usr/sbin/useradd', '--create-home', '--home-dir', k.HOME, '--shell', k.NOLOGIN, '--uid', str(c['agentUid']), '--user-group', k.USER]) if r.returncode: return say(f'useradd failed: {r.stderr.strip()[-300:]}') or 1 + elif k.login_shell(k.USER) != k.NOLOGIN: # an agent an older kit made with a login shell + r = k.run(['/usr/sbin/usermod', '--shell', k.NOLOGIN, k.USER]) + if r.returncode: + return say(f'usermod failed: {r.stderr.strip()[-300:]}') or 1 if k.lstat('/usr/sbin/nft') is None: for argv in (['/usr/bin/apt-get', 'update'], ['/usr/bin/apt-get', 'install', '-y', 'nftables']): r = k.run(argv, timeout=900) @@ -189,14 +199,18 @@ def setup(source, sha): k.run(['/usr/bin/systemctl', 'disable', '--now'] + units) return say('a wall does not hold, so T3 is not enabled. After a first run: wsl --terminate ' 'from Windows, then run setup again') or 3 + k.run(['/usr/bin/systemctl', 'reset-failed', k.UNIT]) # a rerun within the start limit's 10 minutes still starts + ran = k.run(['/usr/bin/systemctl', 'is-active', k.UNIT]).returncode == 0 r = k.run(['/usr/bin/systemctl', 'enable', '--now'] + units) if r.returncode == 0 and k.lstat(k.PENDING): + mark() # every change of this and earlier runs is before now; the T3 started next is after it r = k.run(['/usr/bin/systemctl', 'restart', k.UNIT]) if r.returncode == 0: r = k.run(['/usr/bin/systemctl', 'is-active', k.UNIT]) - if r.returncode: - return say(f'T3 did not start ({(r.stderr or r.stdout).strip()[-300:]}): see journalctl -u {k.UNIT}; ' - 'run setup again once fixed') or 1 + why = (r.stderr or r.stdout).strip()[-300:] if r.returncode else \ + stays_up() if k.lstat(k.PENDING) or not ran else None + if why: + return say(f'T3 did not start or stay up ({why}): see journalctl -u {k.UNIT}; run setup again once fixed') or 1 if k.lstat(k.PENDING): os.unlink(k.ROOT + k.PENDING) say(f'done: T3 serves on port {c["port"]}. Next: tailscale up --advertise-tags={c["tailscaleTag"]} ' @@ -204,6 +218,24 @@ def setup(source, sha): return 0 +def stays_up(): + """Why T3 did not stay up for k.SETTLE seconds after a start (it can exit on a policy it cannot apply a moment + after it started), or None: active all along, the same main process, no automatic restart.""" + first, end = None, time.monotonic() + k.SETTLE + while True: + r = k.run(['/usr/bin/systemctl', 'show', '-p', 'ActiveState,MainPID,NRestarts', k.UNIT]) + got = k.props(r.stdout) + now = (got.get('ActiveState'), got.get('MainPID'), got.get('NRestarts')) + if r.returncode or now[0] != 'active' or now[1] in (None, '', '0'): + return f'T3 is {now[0] or "unknown"}' + if first and now != first: + return f'T3 restarted (main process {first[1]} then {now[1]}, restarts {first[2]} then {now[2]})' + first = first or now + if time.monotonic() >= end: + return None + time.sleep(k.SETTLE_STEP) + + def as_agent(cmd): """Run cmd as the agent under the unit's walls (a transient system unit): never a login shell, never the agent's user manager. For the owner steps (a provider's login), as root.""" @@ -215,10 +247,12 @@ def as_agent(cmd): return say(str(e)) or 2 if check(False) != 0: return say('a wall does not hold: not running anything as the agent') or 1 - argv = [k.ROOT + '/usr/bin/systemd-run', '--quiet', '--collect', '--pty', f'--uid={k.USER}', f'--gid={k.USER}', - f'--working-directory={k.HOME}', f'--setenv=HOME={k.HOME}', f'--setenv=T3CODE_HOME={k.HOME}/.t3', - f'--setenv=PATH={k.HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin'] - argv += [f'--property={line}' for line in k.sandbox(c)] + ['--'] + cmd + argv = [k.ROOT + '/usr/bin/systemd-run', '--quiet', '--collect', '--pty', f'--unit={k.AS_AGENT}-{os.getpid()}', + f'--uid={k.USER}', f'--gid={k.USER}', f'--working-directory={k.HOME}', f'--setenv=HOME={k.HOME}', + f'--setenv=T3CODE_HOME={k.HOME}/.t3', f'--setenv=PATH={k.HOME}/.local/bin:/usr/local/bin:/usr/bin:/bin', + '--setenv=SHELL=/bin/bash'] + # systemd-run looks a bare name up in root's PATH, not the unit's: env looks it up in the agent's (~/.local/bin) + argv += [f'--property={line}' for line in k.sandbox(c)] + ['--', '/usr/bin/env', '--'] + cmd return k.subprocess.call(argv) @@ -280,6 +314,8 @@ def own_walls(c, before_enable=False): k.user_problem(c, True) or (None if shell == k.NOLOGIN else f'{k.USER}\'s login shell is {shell}, not {k.NOLOGIN}'))) out.append(('sudoers', f'no sudo rule covers {k.USER}', k.sudo_problem(c))) + if not before_enable: # setup asks before its own restart, while the marker is there by design + out.append(('restart', 'no T3 runs that started before setup\'s last change', k.restart_problem())) current = k.lstat(k.OPT + '/current') target = os.readlink(k.ROOT + k.OPT + '/current') if current and stat.S_ISLNK(current.st_mode) else '' code = [] if current and current.st_uid == k.OWNER and re.fullmatch(r'[0-9a-f]{40}', target) else \ @@ -357,9 +393,10 @@ def check(enforce, before_enable=False): print(f'FAIL {wall} {what}: {why}'[:500] if why else f'PASS {wall} {what}', flush=True) failed = any(why for _, _, why in results) if failed and enforce: - r = k.run(['/usr/bin/systemctl', 'stop', k.UNIT]) - say(f'a wall does not hold: stopped {k.UNIT}' if r.returncode == 0 else - f'a wall does not hold, and stopping {k.UNIT} FAILED: {r.stderr.strip()[-300:]}') + for unit in (k.UNIT, f'{k.AS_AGENT}-*.service'): # T3, then every running `pc-agents as-agent` + r = k.run(['/usr/bin/systemctl', 'stop', unit]) + say(f'a wall does not hold: stopped {unit}' if r.returncode == 0 else + f'a wall does not hold, and stopping {unit} FAILED: {r.stderr.strip()[-300:]}') return 1 if failed else 0 @@ -378,4 +415,5 @@ def main(argv): if __name__ == '__main__': + k.stop_on_signals() sys.exit(main(sys.argv[1:])) diff --git a/fork-kit/pc-agents/tests/check.test.py b/fork-kit/pc-agents/tests/check.test.py index de9717f11eec8..155974180e039 100644 --- a/fork-kit/pc-agents/tests/check.test.py +++ b/fork-kit/pc-agents/tests/check.test.py @@ -6,7 +6,7 @@ import os, shutil, subprocess, sys, time, unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from fake_distro import CONFIG, Case, DRVFS, SHA1, k # noqa: E402 +from fake_distro import CONFIG, Case, DRVFS, SHA1, cli, k # noqa: E402 class Walls(Case): @@ -25,7 +25,8 @@ def test_all_hold(self): self.assertEqual(code, 0, out) walls = [line.split()[1] for line in out.splitlines()] for wall in ('wsl2:interop', 'wsl2:resolv', 'wsl2:windows-path', 'wsl2:networking', 'wsl2:os', 'masks', - 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'kit-code', 'runtime', + 'record', 'linger', 'user-units', 'drop-ins', 'units', 'agent', 'sudoers', 'restart', 'kit-code', + 'runtime', 'automount', 'firewall', 'gateway', 'file:policy.json', 'file:t3code-agents.service', 'file:firewall.nft', 'file:wsl.conf', 'file:pc-agents-walls.timer'): self.assertIn(wall, walls) @@ -37,6 +38,7 @@ def test_enforce_stops_t3(self): code, out = self.check('--enforce') self.assertEqual(code, 1, out) self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop pc-agents-as-agent-*.service', self.log()) # and every `pc-agents as-agent` run self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) self.assertIn('stopped t3code-agents.service', out) @@ -229,7 +231,7 @@ def test_missing_lib_stops_t3(self): self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) self.assertNotIn('stop t3code-agents.service', self.log()) # the check itself could not stop T3 self.stop_post('exit-code') - self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop t3code-agents.service pc-agents-as-agent-*.service', self.log()) self.assertFalse(os.path.exists(self.path('/log/units/t3code-agents.service.active'))) def test_success_does_not_stop_t3(self): @@ -237,6 +239,70 @@ def test_success_does_not_stop_t3(self): self.assertEqual(self.log(), []) +class Restart(Case): + """The restart wall: while setup's marker is there, a T3 that started before it runs stale files.""" + def setUp(self): + super().setUp() + self.walled() + + def mark(self, text=None): + self.write(k.PENDING, k.marker() if text is None else text) + + def restart_fails(self, why): + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertEqual({line.split()[1] for line in out.splitlines() if line.startswith('FAIL')}, {'restart'}, out) + self.assertIn(why, out) + + def test_t3_started_before_the_marker_fails_and_enforce_stops_it(self): + self.mark() + self.restart_fails('T3 runs as started before setup\'s last change') + code, out = self.check('--enforce') + self.assertEqual(code, 1, out) + self.assertIn('stop t3code-agents.service', self.log()) + self.assertIn('stop pc-agents-as-agent-*.service', self.log()) + self.assertEqual(self.check()[0], 0) # T3 stopped: nothing stale runs; the next setup starts it + + def test_t3_started_after_the_marker_passes(self): # setup's own restart, before it removes the marker + self.mark() + k.run(['/usr/bin/systemctl', 'restart', k.UNIT]) + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_marker_of_an_earlier_boot_passes(self): # every T3 now started after it + self.mark('boot_id=00000000-0000-0000-0000-000000000000 monotonic_us=999999999999999\n') + code, out = self.check() + self.assertEqual(code, 0, out) + + def test_marker_of_an_older_kit_fails(self): # it had no time in it + self.mark('setup changed what T3 runs with; the next good run restarts T3 and removes this\n') + self.restart_fails('cannot be read') + + def test_no_start_time_fails(self): + self.mark() + os.unlink(self.path('/log/units/t3code-agents.service.start')) + self.restart_fails('cannot be read') + + def test_unreadable_marker_fails(self): + if os.geteuid() == 0: + self.skipTest('root reads any file') + self.mark() + os.chmod(self.path(k.PENDING), 0) + self.restart_fails('cannot read') + + def test_systemctl_show_fails(self): + self.mark() + self.write('/log/systemctl-fail', '') + code, out = self.check() + self.assertEqual(code, 1, out) + self.assertIn('FAIL restart', out) + + def test_skipped_before_the_first_enable(self): # setup's own check, while its marker is there by design + self.mark() + code, out = self.call(cli.check, False, True) + self.assertNotIn('restart', [line.split()[1] for line in out.splitlines() if line.startswith(('PASS', 'FAIL'))]) + + class Unit(Case): """What the kit's files say: things no wall can see from outside the unit.""" def test_unit(self): diff --git a/fork-kit/pc-agents/tests/fake_distro.py b/fork-kit/pc-agents/tests/fake_distro.py index 4b2db61733523..1831a88a036f8 100644 --- a/fork-kit/pc-agents/tests/fake_distro.py +++ b/fork-kit/pc-agents/tests/fake_distro.py @@ -1,8 +1,8 @@ """A fake WSL2 distro in a temporary folder for the pc-agents tests: every fixed path under PC_AGENTS_ROOT, the -current user as root, fake useradd, env (T3's installer), apt-get, systemctl, systemd-run and nft (each logs to /log -in the fake root; systemctl keeps which units are enabled, and the ExecStart and policy a started one runs with, in -/log/units). No root, no systemd, no network, no real /etc. Fresh as `wsl --install` leaves one, from the fleet's -wsl2 walls fixture.""" +current user as root, fake useradd, usermod, env (T3's installer), apt-get, systemctl, systemd-run and nft (each logs +to /log in the fake root; systemctl keeps which units are enabled, and the ExecStart and policy a started one runs +with, its main PID and its start time (CLOCK_MONOTONIC, as systemd's) in /log/units). No root, no systemd, no network, +no real /etc. Fresh as `wsl --install` leaves one, from the fleet's wsl2 walls fixture.""" import contextlib, importlib.machinery, importlib.util, io, json, os, shutil, stat, sys, tempfile, unittest KIT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -18,33 +18,52 @@ '[ "$prev" = --shell ] && sh=$a; prev=$a; done\n' 'echo "$a:x:$uid:$uid::/home/$a:$sh" >>"$R/etc/passwd"\n' 'echo "$a:x:$uid:" >>"$R/etc/group"\nmkdir -p "$R/home/$a"\n', + 'usr/sbin/usermod': 'R="${0%/usr/sbin/usermod}"; echo "$*" >>"$R/log/usermod"\n' + '[ "$1" = --shell ] && sed -i "s#^$3:\\(\\([^:]*:\\)\\{5\\}\\)[^:]*\\$#$3:\\1$2#" "$R/etc/passwd"\n', # T3's installer, run by root through env -i (scripts/install.sh with T3CODE_HOME=/opt/pc-agents/t3) 'usr/bin/env': 'R="${0%/usr/bin/env}"; echo "$*" >>"$R/log/install"\n[ -e "$R/log/install-fail" ] && exit 1\n' + # install-hang: a download that never ends, its pid noted (the group kill must end it) + '[ -e "$R/log/install-hang" ] && { sleep 60 & echo $! >"$R/log/hang.pid"; wait; }\n' 'for a; do case "$a" in T3CODE_VERSION=*) v=${a#*=};; esac; done\n' 'd="$R/opt/pc-agents/t3/runtime/versions/$v"; mkdir -p "$d"; printf "#!/bin/sh\\n# t3 $v\\n" >"$d/t3"\n' 'chmod 755 "$d/t3"\n', - 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n', + # systemd-run: like the real one, a bare command name is looked up in the CALLER's PATH, before anything runs + 'usr/bin/systemd-run': 'R="${0%/usr/bin/systemd-run}"; printf "%s\\n" "$@" >"$R/log/systemd-run"\n' + 'while [ $# -gt 0 ] && [ "$1" != -- ]; do shift; done; cmd=$2\n' + 'case "$cmd" in */*) [ -x "$R$cmd" ] || [ -x "$cmd" ];; *) command -v "$cmd" >/dev/null;; esac' + ' || { echo "Failed to find executable $cmd: No such file or directory" >&2; exit 1; }\n', 'usr/bin/systemctl': 'R="${0%/usr/bin/systemctl}"; echo "$*" >>"$R/log/systemctl"; S="$R/log/units"; mkdir -p "$S"\n' '[ -e "$R/log/systemctl-fail" ] && { echo "systemctl: failed (test)" >&2; exit 1; }\n' # what a started unit runs with: its ExecStart and T3's policy - 'run() { [ "$1" = t3code-agents.service ] && [ -e "$R/log/t3-crash" ] && ' - '{ rm -f "$S/$1.active"; return; }\n{ grep "^ExecStart=" "$R/etc/systemd/system/$1"; ' - 'cat "$R/etc/t3code/policy.json"; } >"$S/$1.active" 2>/dev/null || :; }\n' # t3-crash: T3 exits + # a start: refused while the start limit is hit ($S/U.limit, until reset-failed); with + # /log/t3-crash T3 exits at once, with /log/t3-dies after the first look (show) at it + 'run() { [ -e "$S/$1.limit" ] && { echo "$1: start request repeated too quickly" >&2; ' + 'return 1; }\nrm -f "$S/$1.seen"\n' + '[ "$1" = t3code-agents.service ] && [ -e "$R/log/t3-crash" ] && { rm -f "$S/$1.active"; return; }\n' + '{ grep "^ExecStart=" "$R/etc/systemd/system/$1"; ' + 'cat "$R/etc/t3code/policy.json"; } >"$S/$1.active" 2>/dev/null || :\n' + 'echo $$ >"$S/$1.pid"; python3 -I -c "import time; print(time.monotonic_ns() // 1000)" ' + '>"$S/$1.start"; }\n' 'case "$1" in\n' 'enable|disable) v=$1; shift; now=; [ "$1" = --now ] && { now=1; shift; }\n' ' for u; do if [ $v = enable ]; then : >"$S/$u.enabled"; [ -n "$now" ] && ' - '{ [ -e "$S/$u.active" ] || run "$u"; }\n' + '{ [ -e "$S/$u.active" ] || run "$u" || exit 1; }\n' ' else rm -f "$S/$u.enabled"; [ -z "$now" ] || rm -f "$S/$u.active"; fi; done;;\n' - 'restart) shift; for u; do run "$u"; done;;\n' + 'restart) shift; for u; do run "$u" || exit 1; done;;\n' + 'reset-failed) shift; for u; do rm -f "$S/$u.limit"; done;;\n' 'stop) shift; for u; do rm -f "$S/$u.active"; done;;\n' 'is-active) [ -e "$S/$2.active" ] && echo active || { echo inactive; exit 3; };;\n' 'show) u=$4; f=; [ -e "$R/etc/systemd/system/$u" ] && f=/etc/systemd/system/$u\n' + ' [ -e "$R/log/t3-dies" ] && [ -e "$S/$u.active" ] && { [ -e "$S/$u.seen" ] && ' + 'rm -f "$S/$u.active"; : >"$S/$u.seen"; }\n' ' st=disabled; [ -e "$S/$u.enabled" ] && st=enabled\n' ' [ "$u" = pc-agents-walls.service ] && st=static\n' ' echo "LoadState=${f:+loaded}"; echo "FragmentPath=$f"\n' ' echo "DropInPaths=$(cat "$S/$u.dropins" 2>/dev/null)"; echo "UnitFileState=$st"\n' ' [ -e "$S/reload" ] && echo NeedDaemonReload=yes || echo NeedDaemonReload=no\n' - ' [ -e "$S/$u.active" ] && echo ActiveState=active || echo ActiveState=inactive;;\n' + ' if [ -e "$S/$u.active" ]; then echo ActiveState=active; echo "MainPID=$(cat "$S/$u.pid")"\n' + ' else echo ActiveState=inactive; echo MainPID=0; fi; echo NRestarts=0\n' + ' echo "ExecMainStartTimestampMonotonic=$(cat "$S/$u.start" 2>/dev/null || echo 0)";;\n' 'esac\n', # apt-get: update, or install nftables (the fake nft kept in /log/nft-package); /log/apt-fail makes it fail 'usr/bin/apt-get': 'R="${0%/usr/bin/apt-get}"; echo "$*" >>"$R/log/apt-get"\n' @@ -75,6 +94,7 @@ def load(): class Case(unittest.TestCase): def setUp(self): + k.SETTLE, k.SETTLE_STEP = 0.3, 0.05 # setup watches T3 0.3 s here, not 15 self.umask = os.umask(0o022) self.root = os.path.realpath(tempfile.mkdtemp(prefix='pc-agents-test-')) os.chmod(self.root, 0o755) @@ -87,6 +107,7 @@ def setUp(self): files['etc/wsl.conf'] = '[boot]\nsystemd=true\n' files['etc/passwd'] = 'root:x:0:0:root:/root:/bin/bash\nnobody:x:65534:65534::/nonexistent:/usr/sbin/nologin\n' files['etc/group'] = 'root:x:0:\nsudo:x:27:\nnogroup:x:65534:\n' + files['proc/sys/kernel/random/boot_id'] = '0f8e4f9c-1b6a-4d2e-9a57-3c1d2e4f5a6b\n' files['proc/net/route'] = 'Iface\tDestination\tGateway\tFlags\nboth\t00000000\t011014AC\t0003\n' files['log/'] = files['opt/'] = files['etc/systemd/system/'] = '' # as in a fresh Ubuntu files.update({rel: {'text': '#!/bin/sh\n' + text, 'mode': '755'} for rel, text in TOOLS.items()}) diff --git a/fork-kit/pc-agents/tests/setup.test.py b/fork-kit/pc-agents/tests/setup.test.py index 82bcc1cffef36..dbd4e25024801 100644 --- a/fork-kit/pc-agents/tests/setup.test.py +++ b/fork-kit/pc-agents/tests/setup.test.py @@ -5,7 +5,7 @@ python3 -I -B fork-kit/pc-agents/tests/setup.test.py """ -import json, os, re, shutil, stat, sys, unittest +import json, os, re, shutil, signal, stat, subprocess, sys, time, unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from fake_distro import CONFIG, KIT, REPO, SHA1, SHA2, Case, blob, cli, k # noqa: E402 @@ -45,7 +45,7 @@ def test_every_file_and_permission(self): p = self.path(f'/opt/pc-agents/{SHA1}/{dst}') self.assertEqual(stat.S_IMODE(os.stat(p).st_mode), mode, dst) self.assertEqual(blob(p), blob(os.path.join(REPO, src)), dst) - self.assertIn('agent:x:1100:1100::/home/agent:/bin/bash', self.read('/etc/passwd')) + self.assertIn('agent:x:1100:1100::/home/agent:/usr/sbin/nologin', self.read('/etc/passwd')) self.assertIn('/etc/systemd/system/user@1100.service', k.masks(c)) t3 = self.path('/opt/pc-agents/t3/runtime/versions/1.2.3/t3') self.assertEqual(os.stat(t3).st_uid, os.geteuid()) @@ -137,7 +137,8 @@ def stopping(*a, n=[0], **kw): with self.assertRaises(KeyboardInterrupt): self.setup_run() cli.put = real_put - self.assertFalse([line for line in self.log() if line.startswith('enable')], f'stopped at write {stop}') + if stop < total: # the last write is the marker right before the restart, after the enable + self.assertFalse([line for line in self.log() if line.startswith('enable')], f'stopped at write {stop}') code, out = self.setup_run() self.assertEqual(code, 0, f'stopped at write {stop} ({writes[stop - 1]}): {out}') self.assertEqual({p: v[4] for p, v in self.snapshot().items()}, whole, f'stopped at write {stop}') @@ -242,6 +243,186 @@ def test_t3_that_does_not_start_is_reported(self): self.assertFalse(os.path.lexists(self.path(k.PENDING))) + def test_t3_that_dies_a_moment_after_the_restart_is_reported(self): + self.ready() + self.write('/log/t3-dies', '') # T3 starts, then exits on something it cannot apply + self.machine({**CONFIG, 'memoryMax': '4G'}) + code, out = self.setup_run() + self.assertEqual(code, 1, out) + self.assertIn('T3 did not start or stay up (T3 is inactive)', out) + self.assertNotIn('done:', out) + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + os.unlink(self.path('/log/t3-dies')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + def test_rerun_within_the_start_limit_still_restarts_t3(self): + self.ready() + self.write('/log/units/t3code-agents.service.limit', '') # systemd: start request repeated too quickly + self.machine({**CONFIG, 'memoryMax': '4G'}) + os.unlink(self.path('/log/systemctl')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + log = self.log() + self.assertLess(log.index('reset-failed t3code-agents.service'), log.index('restart t3code-agents.service')) + + def test_agent_of_an_older_kit_gets_no_login_shell(self): + self.ready() + self.write('/etc/passwd', self.read('/etc/passwd').replace('/home/agent:/usr/sbin/nologin', '/home/agent:/bin/bash')) + code, out = self.setup_run() + self.assertEqual(code, 0, out) + self.assertIn('agent:x:1100:1100::/home/agent:/usr/sbin/nologin', self.read('/etc/passwd')) + self.assertEqual(self.log('usermod'), ['--shell /usr/sbin/nologin agent']) + os.unlink(self.path('/log/usermod')) + self.assertEqual(self.setup_run()[0], 0) + self.assertEqual(self.log('usermod'), []) # once + + +class RestartPending(Case): + """The restart wall with setup: a T3 that started before setup's last change fails `check`.""" + stop_after = Setup.stop_after + + def test_interrupted_upgrade_keeps_old_t3_and_check_fails(self): # lenspcas1 #1 + self.ready() + self.machine({**CONFIG, 't3Version': '1.2.4', 'maxRuntimeMode': 'approval-required'}) + self.write('/log/install-fail', '') + self.assertEqual(self.setup_run()[0], 1) + running = self.read('/log/units/t3code-agents.service.active') + self.assertIn('versions/1.2.3/t3 serve', running) + self.assertIn('full-access', running) + code, out = self.call(cli.main, ['as-agent', '--', 'sh']) + self.assertEqual(code, 1, out) + self.assertEqual(self.log('systemd-run'), []) + os.unlink(self.path('/log/systemctl')) + code, out = self.check('--enforce') + self.assertNotEqual(code, 0, out) + self.assertIn('FAIL restart', out) + self.assertIn('stop t3code-agents.service', self.log()) + os.unlink(self.path('/log/install-fail')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('versions/1.2.4/t3 serve', self.read('/log/units/t3code-agents.service.active')) + self.assertEqual(self.check('--enforce')[0], 0) + + def test_killed_after_the_policy_is_written_check_fails(self): # lenspcac1 #3 + self.ready() + self.machine({**CONFIG, 'maxRuntimeMode': 'approval-required'}) + self.stop_after('/etc/t3code/policy.json') + self.assertNotIn('approval-required', self.read('/log/units/t3code-agents.service.active')) + self.assertNotEqual(self.check('--enforce')[0], 0) + + def test_upgrade_killed_at_any_write_fails_check_until_run_again(self): + def files(): + return {p: v[4] for p, v in self.snapshot().items() if p not in (k.PENDING, k.RECORD)} + upgrade = {**CONFIG, 'memoryMax': '4G', 'maxRuntimeMode': 'approval-required'} + real_put, writes = cli.put, [] + self.addCleanup(setattr, cli, 'put', real_put) + self.ready() + self.machine(upgrade) + cli.put = lambda *a, **kw: writes.append(a[0]) or real_put(*a, **kw) + self.assertEqual(self.setup_run(SHA2)[0], 0) + cli.put = real_put + self.assertGreater(len(writes), 10) + for stop in range(1, len(writes) + 1): + shutil.rmtree(self.root) + self.setUp() + self.ready() + self.machine(upgrade) + before = files() + + def stopping(*a, n=[0], **kw): + n[0] += 1 + if n[0] == stop: + raise KeyboardInterrupt + return real_put(*a, **kw) + cli.put = stopping + with self.assertRaises(KeyboardInterrupt): + self.setup_run(SHA2) + cli.put = real_put + code, out = self.check() + if files() != before: # something T3 runs with changed + self.assertIn('FAIL restart', out, f'stopped at write {stop} ({writes[stop - 1]})') + code, out = self.setup_run(SHA2) + self.assertEqual(code, 0, f'stopped at write {stop}: {out}') + code, out = self.check() + self.assertEqual(code, 0, f'stopped at write {stop}: {out}') + + def test_check_passes_on_its_own_restart_and_a_kill_while_watching_restarts_again(self): + self.ready() + self.machine({**CONFIG, 'memoryMax': '4G'}) + real, seen = cli.stays_up, [] + + def watching(): # killed while setup watches the restarted T3 + seen.append(self.check()) + raise KeyboardInterrupt + cli.stays_up = watching + try: + with self.assertRaises(KeyboardInterrupt): + self.setup_run() + finally: + cli.stays_up = real + self.assertEqual(seen[0][0], 0, seen[0][1]) # the T3 setup just restarted is not stale + self.assertTrue(os.path.isfile(self.path(k.PENDING))) + self.assertEqual(self.check()[0], 0) + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('restart t3code-agents.service', self.log()) # once more, then the marker goes + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + + def test_marker_of_an_older_kit_fails_until_setup_runs_again(self): + self.ready() + self.write(k.PENDING, 'setup changed what T3 runs with; the next good run restarts T3 and removes this\n') + self.assertIn('FAIL restart', self.check()[1]) + os.unlink(self.path('/log/systemctl')) + self.assertEqual(self.setup_run()[0], 0) + self.assertIn('restart t3code-agents.service', self.log()) + self.assertFalse(os.path.lexists(self.path(k.PENDING))) + self.assertEqual(self.check()[0], 0) + + +class Signals(Case): + def test_sigterm_or_sighup_kills_the_running_download_too(self): + for sig in (signal.SIGTERM, signal.SIGHUP): + with self.subTest(signal=sig.name): + self.write('/log/install-hang', '') + p = subprocess.Popen([sys.executable, '-I', os.path.join(KIT, 'sbin', 'pc-agents'), 'setup', + '--source', REPO, '--sha', SHA1], env={'PC_AGENTS_ROOT': self.root, + 'PATH': '/usr/bin:/bin'}, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + hang = self.path('/log/hang.pid') + try: + end = time.monotonic() + 60 + while not os.path.exists(hang) and time.monotonic() < end and p.poll() is None: + time.sleep(0.05) + self.assertTrue(os.path.exists(hang), 'the fake download did not start') + time.sleep(0.2) + pid = int(self.read('/log/hang.pid')) + p.send_signal(sig) + self.assertEqual(p.wait(timeout=30), 128 + sig) + end = time.monotonic() + 10 + while time.monotonic() < end and alive(pid): + time.sleep(0.05) + self.assertFalse(alive(pid), 'the download outlived pc-agents') + finally: + if p.poll() is None: + p.kill() + p.wait() + if os.path.exists(hang): + try: + os.kill(int(self.read('/log/hang.pid')), signal.SIGKILL) + except (ProcessLookupError, ValueError): + pass + os.unlink(hang) + + +def alive(pid): + """pid runs (a zombie, ended but not yet reaped, does not count).""" + try: + with open(f'/proc/{pid}/stat', encoding='ascii') as f: + return f.read().rsplit(')', 1)[1].split()[0] != 'Z' + except FileNotFoundError: + return False + + class StaleFirewall(Case): def test_tightened_allow_with_failed_install(self): self.machine({**CONFIG, 'allow': ['192.168.1.0/24']}) @@ -251,7 +432,10 @@ def test_tightened_allow_with_failed_install(self): self.write('/log/install-fail', '') self.assertEqual(self.setup_run()[0], 1) self.assertNotIn('192.168.1.0/24', self.read('/log/nft-loaded')) # new table already loaded - self.assertEqual(self.check('--enforce')[0], 0) # and check agrees with the file + status, out = self.check() + fails = [line for line in out.splitlines() if line.startswith('FAIL')] + self.assertEqual(status, 1) # the old T3 still runs: only the restart wall fails, + self.assertEqual([line.split()[1] for line in fails], ['restart'], out) # the firewall agrees with the file def test_a_loaded_table_that_is_not_the_files_fails(self): self.ready() @@ -266,15 +450,18 @@ def test_a_loaded_table_that_is_not_the_files_fails(self): class AsAgent(Case): def test_runs_as_the_agent_under_the_units_walls(self): self.ready() - code, out = self.call(cli.main, ['as-agent', '--', 'claude', 'login']) + tool = 'pc-agents-test-tool-only-in-the-agents-path' # like claude in ~/.local/bin: not on root's PATH + code, out = self.call(cli.main, ['as-agent', '--', tool, 'login']) self.assertEqual(code, 0, out) argv = self.log('systemd-run') - for arg in ('--pty', '--uid=agent', '--gid=agent', '--setenv=HOME=/home/agent', + self.assertEqual(argv[-5:], ['--', '/usr/bin/env', '--', tool, 'login']) # env looks it up in PATH below + self.assertTrue([a for a in argv if re.fullmatch(r'--unit=pc-agents-as-agent-\d+', a)], argv) + for arg in ('--pty', '--uid=agent', '--gid=agent', '--setenv=HOME=/home/agent', '--setenv=SHELL=/bin/bash', + '--setenv=PATH=/home/agent/.local/bin:/usr/local/bin:/usr/bin:/bin', '--property=InaccessiblePaths=-/mnt -/run/WSL -/tmp/.X11-unix -/usr/lib/wsl -/run/tailscale'): self.assertIn(arg, argv) for line in k.sandbox(k.config()): self.assertIn('--property=' + line, argv) - self.assertEqual(argv[-3:], ['--', 'claude', 'login']) def test_refused_when_a_wall_does_not_hold(self): self.ready() From d9ce01555815a02ed77f3dbc0e090c726f21b699 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:51:45 +0000 Subject: [PATCH 10/10] fork-run-alert test: read calls fully before grep -q Main's fork-run-alert.test.sh had the same early-exit pipe the new guard refuses; read the whole output first, as in the other shell tests. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/fork-run-alert.test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/fork-run-alert.test.sh b/.github/scripts/fork-run-alert.test.sh index ea5e77c6751dc..ca4311e340760 100644 --- a/.github/scripts/fork-run-alert.test.sh +++ b/.github/scripts/fork-run-alert.test.sh @@ -83,8 +83,8 @@ expect() { # expect LABEL 'CONDITION' (shell, evaluated) } rc_is() { [[ "$rc" == "$1" ]]; } says() { grep -qF -- "$1" <<< "$out"; } -called() { calls | grep -qF -- "$1"; } -not_called() { ! calls | grep -qF -- "$1"; } +called() { grep -qF -- "$1" <<< "$(calls)"; } +not_called() { ! grep -qF -- "$1" <<< "$(calls)"; } issues() { jq length "$root/state/issues.json" 2> /dev/null || echo 0; } reset; run failure --message "The daily secret sweep failed"