diff --git a/Makefile b/Makefile index 78cb05f1..2789cb02 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPONENTS := pull-config mie-opensource-landing create-a-container +COMPONENTS := agent mie-opensource-landing create-a-container PACKAGER ?= deb # Forwarded to every component Makefile. diff --git a/README.md b/README.md index de6b424c..52ea6bef 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Full documentation lives in [`mie-opensource-landing/docs/`](mie-opensource-land | Path | Purpose | |---|---| | [`create-a-container/`](create-a-container/) | Manager web application (Node.js + Express + Sequelize) | -| [`pull-config/`](pull-config/) | Cron-driven config distribution for nginx and dnsmasq on agents — see [pull-config docs](mie-opensource-landing/docs/developers/pull-config.md) | +| [`agent/`](agent/) | Site agent: systemd-timer check-in that reports status and applies nginx/dnsmasq config — see [agent docs](mie-opensource-landing/docs/developers/agent.md) | | [`images/`](images/) | Docker Bake definitions for the `base`, `nodejs`, `agent`, and `manager` images — see [Docker Images](mie-opensource-landing/docs/developers/docker-images.md) | | [`manager-control-program/`](manager-control-program/) | MCP server for AI-assisted container management — see [MCP Server](mie-opensource-landing/docs/users/mcp-server.md) | | [`mie-opensource-landing/`](mie-opensource-landing/) | Documentation site source | diff --git a/pull-config/.fpm b/agent/.fpm similarity index 57% rename from pull-config/.fpm rename to agent/.fpm index eb462614..50e677dc 100644 --- a/pull-config/.fpm +++ b/agent/.fpm @@ -5,13 +5,13 @@ --vendor "Medical Informatics Engineering" --url "https://github.com/mieweb/opensource-server" --category admin ---description "MIE opensource-server edge agent (pull-config): cron-driven distribution of nginx and dnsmasq configuration to edge nodes. Also ships the static error pages referenced by the manager-rendered nginx config." +--description "MIE opensource-server site agent: checks in with the manager every 30 seconds, reports host and service status, and renders and applies nginx and dnsmasq configuration. Also ships the static error pages referenced by the rendered nginx config." +--depends nodejs --depends nginx --depends libnginx-mod-stream --depends libnginx-mod-http-modsecurity --depends modsecurity-crs --depends ssl-cert --depends dnsmasq ---depends curl ---depends cron --deb-no-default-config-files +--after-install contrib/postinstall.sh diff --git a/agent/.gitignore b/agent/.gitignore new file mode 100644 index 00000000..0d22d8e5 --- /dev/null +++ b/agent/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +dist/ +.pkg/ +*.deb +*.rpm +*.apk diff --git a/pull-config/Makefile b/agent/Makefile similarity index 55% rename from pull-config/Makefile rename to agent/Makefile index 2d5f5be3..20a4abdd 100644 --- a/pull-config/Makefile +++ b/agent/Makefile @@ -2,44 +2,56 @@ PREFIX ?= /opt/opensource-server DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/agent +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system +# Staging directory for packaging STAGE := $(CURDIR)/.pkg/buildroot INSTALL := install INSTALL_DATA := $(INSTALL) -m 0644 -INSTALL_PROG := $(INSTALL) -m 0755 ERROR_PAGES := ../error-pages .PHONY: help deps build dev test install deb rpm apk package clean help: - @echo "pull-config — builds the opensource-agent package." + @echo "agent — builds the opensource-agent package." @echo "" @echo "Targets:" - @echo " deps install dependencies (none; plain bash)" - @echo " build build the component (nothing to compile)" - @echo " test run tests (none; plain bash)" + @echo " deps install dependencies (npm ci)" + @echo " build compile the TypeScript sources" + @echo " test run tests (none yet)" @echo " install stage files into DESTDIR (default /)" @echo " deb build the .deb package" @echo " rpm build the .rpm package" @echo " apk build the .apk package" - @echo " clean remove the staging dir and built packages" + @echo " clean remove build output, staging dir and built packages" @echo " help show this message" @echo "" @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." -# Plain bash; nothing to install, compile, test, or watch. deps: + npm ci + +# Compile, then drop devDependencies so only the runtime node_modules +# (ejs) is staged into the package. build: deps -dev: deps + npm run build + npm prune --omit=dev + +dev: build + +# No test suite yet; kept as a no-op so the repo-wide `make test` passes. test: install: build - $(INSTALL) -D -m 0755 bin/pull-config $(DESTDIR)$(PREFIX)/pull-config/bin/pull-config - $(INSTALL) -d $(DESTDIR)/etc/pull-config.d - $(INSTALL_PROG) etc/pull-config.d/* $(DESTDIR)/etc/pull-config.d/ - $(INSTALL) -D -m 0644 etc/cron.d/pull-config $(DESTDIR)/etc/cron.d/pull-config + $(INSTALL) -d $(DESTBIN) + $(INSTALL_DATA) package.json $(DESTBIN)/ + cp -a dist templates node_modules $(DESTBIN)/ + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages $(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/ $(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache @@ -59,5 +71,5 @@ apk: $(MAKE) package PACKAGER=apk clean: - rm -rf $(STAGE) + rm -rf $(STAGE) node_modules dist rm -f *.deb *.rpm *.apk diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 00000000..fd3cb243 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,6 @@ +# agent + +Site agent for opensource-server. A oneshot Node.js script, launched every 30 seconds by a systemd timer, that checks in with the manager (`POST /api/v1/agents`), reports host and service status, and renders, tests and applies nginx and dnsmasq configuration from the returned snapshot. + +- **Admin guide:** [Deploying Agents](https://mieweb.github.io/opensource-server/docs/admins/deploying-agents) — installation and configuration +- **Developer reference:** [agent](https://mieweb.github.io/opensource-server/docs/developers/agent) — check-in protocol, apply/rollback flow, managed services diff --git a/agent/contrib/postinstall.sh b/agent/contrib/postinstall.sh new file mode 100644 index 00000000..c243f72d --- /dev/null +++ b/agent/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +UNITS="opensource-agent.timer" + +# Nothing to do without systemctl (non-systemd container/chroot). +command -v systemctl >/dev/null 2>&1 || exit 0 + +# `systemctl enable` only creates static symlinks, so it works during an image +# build too +systemctl enable $UNITS + +# daemon-reload and restart need a running systemd; skip them at build time. +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + systemctl restart $UNITS +fi + +exit 0 diff --git a/agent/contrib/systemd/opensource-agent.service b/agent/contrib/systemd/opensource-agent.service new file mode 100644 index 00000000..ff2d2814 --- /dev/null +++ b/agent/contrib/systemd/opensource-agent.service @@ -0,0 +1,14 @@ +[Unit] +Description=MIE opensource-server agent check-in +Wants=network-online.target +# environment.service (container images) writes the runtime env into +# /etc/environment; harmless ordering hint when the unit doesn't exist. +After=network-online.target environment.service + +[Service] +Type=oneshot +# Runtime config (SITE_ID, MANAGER_URL, API_KEY) lives in /etc/environment. +EnvironmentFile=-/etc/environment +# systemd creates /var/lib/opensource-agent and passes it as $STATE_DIRECTORY. +StateDirectory=opensource-agent +ExecStart=/usr/bin/node /opt/opensource-server/agent/dist/index.js diff --git a/agent/contrib/systemd/opensource-agent.timer b/agent/contrib/systemd/opensource-agent.timer new file mode 100644 index 00000000..d89a7623 --- /dev/null +++ b/agent/contrib/systemd/opensource-agent.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run the opensource-server agent check-in every 30 seconds + +[Timer] +OnBootSec=30s +OnUnitActiveSec=30s +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/agent/package-lock.json b/agent/package-lock.json new file mode 100644 index 00000000..b1d8fe22 --- /dev/null +++ b/agent/package-lock.json @@ -0,0 +1,299 @@ +{ + "name": "opensource-agent", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "opensource-agent", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "@particle/dbus-next": "^0.11.4", + "ejs": "^3.1.10" + }, + "devDependencies": { + "@types/ejs": "^3.1.5", + "@types/node": "^24.0.0", + "typescript": "^5.7.0" + } + }, + "node_modules/@nornagon/put": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@nornagon/put/-/put-0.0.8.tgz", + "integrity": "sha512-ugvXJjwF5ldtUpa7D95kruNJ41yFQDEKyF5CW4TgKJnh+W/zmlBzXXeKTyqIgwMFrkePN2JqOBqcF0M0oOunow==", + "license": "MIT/X11", + "engines": { + "node": ">=0.3.0" + } + }, + "node_modules/@particle/dbus-next": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/@particle/dbus-next/-/dbus-next-0.11.4.tgz", + "integrity": "sha512-oJsiPMMYA+DlVbaVV1Ab3xRiurlGtAxdaGvLrIHOjPSyyVvsq3NNMwarF0pvxUnzSwo9t+Hwt69I96ZfRjro0Q==", + "license": "MIT", + "dependencies": { + "@nornagon/put": "0.0.8", + "event-stream": "4.0.1", + "jsbi": "^4.3.0", + "long": "^4.0.0", + "safe-buffer": "^5.2.1", + "xml2js": "^0.6.2" + } + }, + "node_modules/@types/ejs": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/ejs/-/ejs-3.1.5.tgz", + "integrity": "sha512-nv+GSx77ZtXiJzwKdsASqi+YQ5Z7vwHsTP0JY2SiQgjGckkBRKZnk8nIM+7oUZ1VCtuTz0+By4qVR7fqzp/Dfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", + "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "license": "MIT", + "dependencies": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, + "node_modules/filelist": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", + "integrity": "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", + "license": "MIT" + }, + "node_modules/jake": { + "version": "10.9.4", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", + "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.6", + "filelist": "^1.0.4", + "picocolors": "^1.1.1" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jsbi": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", + "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", + "license": "Apache-2.0" + }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0" + }, + "node_modules/map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==", + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "license": [ + "MIT", + "Apache2" + ], + "dependencies": { + "through": "~2.3" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "license": "MIT", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha512-6yHMqgLYDzQDcAkL+tjJDC5nSNuNIx0vZtRZeiPh7Saef7VHX9H5Ijn9l2VIol2zaNYlYEX6KyuT/237A58qEQ==", + "license": "MIT", + "dependencies": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + } + } +} diff --git a/agent/package.json b/agent/package.json new file mode 100644 index 00000000..afa3d143 --- /dev/null +++ b/agent/package.json @@ -0,0 +1,20 @@ +{ + "name": "opensource-agent", + "version": "0.0.0", + "private": true, + "description": "MIE opensource-server site agent: checks in with the manager, reports host/service status, and renders and applies nginx and dnsmasq configuration.", + "license": "Apache-2.0", + "scripts": { + "build": "tsc", + "start": "node dist/index.js" + }, + "dependencies": { + "@particle/dbus-next": "^0.11.4", + "ejs": "^3.1.10" + }, + "devDependencies": { + "@types/ejs": "^3.1.5", + "@types/node": "^24.0.0", + "typescript": "^5.7.0" + } +} diff --git a/agent/src/api.ts b/agent/src/api.ts new file mode 100644 index 00000000..ef1b6b0f --- /dev/null +++ b/agent/src/api.ts @@ -0,0 +1,39 @@ +/** Check-in client for the manager's POST /api/v1/agents endpoint. */ + +import type { AgentConfig } from './config'; +import type { CheckinRequest, SiteConfig } from './types'; + +export type CheckinResult = + | { notModified: true } + | { notModified: false; etag?: string; config: SiteConfig }; + +// Fail fast if the manager is unreachable or the connection stalls, so the +// oneshot unit exits predictably instead of hanging and queueing timer runs. +const CHECKIN_TIMEOUT_MS = 30_000; + +export async function checkin( + cfg: AgentConfig, + body: CheckinRequest, + etag?: string, +): Promise { + const headers: Record = { 'Content-Type': 'application/json' }; + if (etag) headers['If-None-Match'] = etag; + if (cfg.apiKey) headers['Authorization'] = `Bearer ${cfg.apiKey}`; + + const res = await fetch(`${cfg.managerUrl}/api/v1/agents`, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), + }); + + if (res.status === 304) return { notModified: true }; + if (!res.ok) throw new Error(`Check-in failed: HTTP ${res.status}`); + + const { data } = (await res.json()) as { data: SiteConfig }; + return { + notModified: false, + etag: res.headers.get('etag') ?? undefined, + config: data, + }; +} diff --git a/agent/src/apply.ts b/agent/src/apply.ts new file mode 100644 index 00000000..4a518d53 --- /dev/null +++ b/agent/src/apply.ts @@ -0,0 +1,149 @@ +/** + * Managed services: how to render, test, apply and reload each service's + * configuration. Files are written atomically (temp file + rename) with + * backup/rollback — if the test command rejects the new config, the + * previous files are restored and the apply is reported as a failure at the + * next check-in. + */ + +import fs from 'fs'; +import path from 'path'; +import { execFileSync } from 'child_process'; +import ejs from 'ejs'; +import { reloadOrRestartService, restartService, sighupService } from './system'; +import type { ApplyResult, SiteConfig } from './types'; + +const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); + +interface RenderedFile { + dest: string; + content: string; +} + +export interface ManagedService { + /** systemd unit name (also the key reported at check-in). */ + unit: string; + /** Render all managed files. Returns null when there is nothing to manage + * yet (e.g. dnsmasq before the site exists). */ + render(config: SiteConfig): Promise; + /** Command that validates the staged config before it is kept. */ + test?: string[]; + /** Reload/restart after a successful apply. */ + reload(changedFiles: string[]): Promise; +} + +function renderTemplate(template: string, data: object): Promise { + return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); +} + +function run(cmd: string[]): void { + execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe' }); +} + +export const services: ManagedService[] = [ + { + unit: 'nginx', + async render(config) { + return [{ + dest: '/etc/nginx/nginx.conf', + content: await renderTemplate('nginx.conf.ejs', config.nginx), + }]; + }, + test: ['nginx', '-t'], + reload() { + return reloadOrRestartService('nginx'); + }, + }, + { + unit: 'dnsmasq', + async render(config) { + const site = config.site; + // Skip dnsmasq management until the site's DHCP/DNS settings are fully + // configured — the templates need all of these fields. + if (!site?.internalDomain || !site.dhcpRange || !site.subnetMask + || !site.gateway || !site.dnsForwarders) { + return null; + } + const data = { site }; + return [ + { dest: '/etc/dnsmasq.conf', content: await renderTemplate('dnsmasq/conf.ejs', data) }, + { dest: '/var/lib/dnsmasq/dhcp-hosts', content: await renderTemplate('dnsmasq/dhcp-hosts.ejs', data) }, + { dest: '/var/lib/dnsmasq/hosts', content: await renderTemplate('dnsmasq/hosts.ejs', data) }, + { dest: '/var/lib/dnsmasq/dhcp-opts', content: await renderTemplate('dnsmasq/dhcp-opts.ejs', data) }, + { dest: '/var/lib/dnsmasq/servers', content: await renderTemplate('dnsmasq/servers.ejs', data) }, + ]; + }, + test: ['dnsmasq', '--test'], + reload(changedFiles) { + // The main config requires a full restart; the auxiliary files under + // /var/lib/dnsmasq are re-read on SIGHUP. + if (changedFiles.includes('/etc/dnsmasq.conf')) { + return restartService('dnsmasq'); + } + return sighupService('dnsmasq'); + }, + }, +]; + +function readIfExists(file: string): string | null { + try { + return fs.readFileSync(file, 'utf8'); + } catch { + return null; + } +} + +// Write via temp file + rename so a crash mid-write can never leave a +// truncated config on disk. +function writeFileAtomic(dest: string, content: string): void { + const tmp = `${dest}.tmp-${process.pid}`; + fs.writeFileSync(tmp, content); + fs.renameSync(tmp, dest); +} + +export async function applyService(svc: ManagedService, config: SiteConfig): Promise { + const files = await svc.render(config); + if (!files) return 'success'; + + const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); + const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); + if (changed.length === 0) return 'success'; + + // Stage the new files (previous contents kept in memory for rollback). + for (const f of files) { + fs.mkdirSync(path.dirname(f.dest), { recursive: true }); + writeFileAtomic(f.dest, f.content); + } + + const rollback = () => { + for (const [dest, prev] of current) { + if (prev === null) fs.rmSync(dest, { force: true }); + else writeFileAtomic(dest, prev); + } + }; + + if (svc.test) { + try { + run(svc.test); + } catch (err) { + rollback(); + console.error(`${svc.unit}: config test failed, rolled back: ${(err as Error).message}`); + return 'failure'; + } + } + + try { + await svc.reload(changed); + } catch (err) { + // The new config passed its test but the service couldn't pick it up: + // restore the previous files and reload again (best effort) so the + // service keeps running the last known-good config. + rollback(); + await svc.reload(changed).catch(() => { /* reported via service state at next check-in */ }); + console.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); + return 'failure'; + } + + console.log(`${svc.unit}: applied ${changed.length} file(s)`); + return 'success'; +} diff --git a/agent/src/config.ts b/agent/src/config.ts new file mode 100644 index 00000000..d01c91f3 --- /dev/null +++ b/agent/src/config.ts @@ -0,0 +1,24 @@ +/** Agent configuration, read from the environment (systemd passes + * /etc/environment through EnvironmentFile=). */ + +export interface AgentConfig { + siteId: number; + managerUrl: string; + apiKey?: string; + stateDir: string; +} + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { + const siteId = parseInt(env.SITE_ID ?? '', 10); + const managerUrl = env.MANAGER_URL; + if (!Number.isInteger(siteId) || !managerUrl) { + throw new Error('SITE_ID and MANAGER_URL must be set in the environment'); + } + return { + siteId, + managerUrl: managerUrl.replace(/\/+$/, ''), + apiKey: env.API_KEY || undefined, + // Set by systemd from StateDirectory=; fallback for manual runs. + stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', + }; +} diff --git a/agent/src/index.ts b/agent/src/index.ts new file mode 100644 index 00000000..5d2a2d9b --- /dev/null +++ b/agent/src/index.ts @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/** + * opensource-agent: oneshot check-in with the manager, launched every 30s by + * a systemd timer. + * + * Each pass POSTs system info + service states to /api/v1/agents with the + * last applied config ETag in If-None-Match. A 304 means nothing changed and + * the agent exits. A 200 carries the site's config snapshot: the agent + * renders, tests and applies it, then checks in again to report the apply + * results — repeating until it gets a 304. + */ + +import os from 'os'; +import { loadConfig, type AgentConfig } from './config'; +import { State } from './state'; +import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; +import { checkin } from './api'; +import { services, applyService } from './apply'; +import type { CheckinRequest, ServiceStatus } from './types'; + +// Safety cap: a flapping server-side config can't keep a single run alive +// forever; the timer starts a fresh run 30s later anyway. +const MAX_PASSES = 5; + +async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { + const serviceStatus: Record = {}; + for (const svc of services) { + serviceStatus[svc.unit] = { + state: await getServiceState(svc.unit), + lastApply: state.lastApply[svc.unit] ?? 'unknown', + }; + } + return { + siteId: cfg.siteId, + hostname: os.hostname(), + currentTime: Math.floor(Date.now() / 1000), + ipv4Address: getPrimaryIpv4(), + services: serviceStatus, + }; +} + +async function main(): Promise { + const cfg = loadConfig(); + const state = State.load(cfg.stateDir); + + for (let pass = 0; pass < MAX_PASSES; pass++) { + const result = await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); + if (result.notModified) return; + + for (const svc of services) { + state.lastApply[svc.unit] = await applyService(svc, result.config); + } + + // The ETag is saved even after a failed apply: a rejected config won't + // fix itself without a server-side change (which changes the ETag), and + // the failure has been reported via lastApply. + state.etag = result.etag; + state.save(); + } + + // MAX_PASSES exhausted (flapping server-side config): check in once more so + // the final pass' apply results reach the manager instead of going stale + // until the next timer run. + await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); +} + +main() + .then(() => disconnectSystemBus()) + .catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + }); diff --git a/agent/src/state.ts b/agent/src/state.ts new file mode 100644 index 00000000..3e3cdbf4 --- /dev/null +++ b/agent/src/state.ts @@ -0,0 +1,42 @@ +/** Persistent agent state: last applied config ETag + per-service apply + * results, stored as JSON under the state dir. */ + +import fs from 'fs'; +import path from 'path'; +import type { ApplyResult } from './types'; + +export class State { + etag?: string; + lastApply: Record = {}; + + private constructor(private readonly file: string) {} + + static load(stateDir: string): State { + const state = new State(path.join(stateDir, 'state.json')); + let raw: string; + try { + raw = fs.readFileSync(state.file, 'utf8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return state; // first run + throw err; + } + try { + const data = JSON.parse(raw) as { etag?: string; lastApply?: Record }; + state.etag = data.etag; + state.lastApply = data.lastApply ?? {}; + } catch (err) { + if (!(err instanceof SyntaxError)) throw err; + // A corrupt state file just means a full re-apply on this run. + console.error(`Ignoring unparsable state file ${state.file}: ${err.message}`); + } + return state; + } + + save(): void { + fs.mkdirSync(path.dirname(this.file), { recursive: true }); + fs.writeFileSync( + this.file, + JSON.stringify({ etag: this.etag, lastApply: this.lastApply }, null, 2), + ); + } +} diff --git a/agent/src/system.ts b/agent/src/system.ts new file mode 100644 index 00000000..5a3fda70 --- /dev/null +++ b/agent/src/system.ts @@ -0,0 +1,142 @@ +/** Host system info and systemd unit control via the systemd D-Bus API. */ + +import os from 'os'; +import dbus, { type MessageBus, type ProxyObject, type Variant } from '@particle/dbus-next'; + +/** First non-internal IPv4 address, or null when none is configured. */ +export function getPrimaryIpv4(): string | null { + for (const ifaces of Object.values(os.networkInterfaces())) { + for (const iface of ifaces ?? []) { + if (iface.family === 'IPv4' && !iface.internal) return iface.address; + } + } + return null; +} + +// --- systemd (org.freedesktop.systemd1 on the system bus) ------------------- + +const SYSTEMD = 'org.freedesktop.systemd1'; +const SYSTEMD_PATH = '/org/freedesktop/systemd1'; +const MANAGER_IFACE = 'org.freedesktop.systemd1.Manager'; +const UNIT_IFACE = 'org.freedesktop.systemd1.Unit'; +const PROPERTIES_IFACE = 'org.freedesktop.DBus.Properties'; + +const SIGHUP = 1; + +/** The subset of org.freedesktop.DBus.Properties the agent uses. */ +interface DbusProperties { + Get(iface: string, prop: string): Promise>; +} + +/** The subset of org.freedesktop.systemd1.Manager the agent uses. */ +interface SystemdManager { + LoadUnit(name: string): Promise; + ReloadOrRestartUnit(name: string, mode: string): Promise; + RestartUnit(name: string, mode: string): Promise; + KillUnit(name: string, who: string, signal: number): Promise; + Subscribe(): Promise; + Unsubscribe(): Promise; + on(event: 'JobRemoved', listener: (id: number, job: unknown, unit: string, result: string) => void): void; + removeListener(event: 'JobRemoved', listener: (...args: never[]) => void): void; +} + +let bus: MessageBus | null = null; +let manager: SystemdManager | null = null; + +function getBus(): MessageBus { + if (!bus) bus = dbus.systemBus(); + return bus; +} + +async function getManager(): Promise { + if (!manager) { + const obj: ProxyObject = await getBus().getProxyObject(SYSTEMD, SYSTEMD_PATH); + manager = obj.getInterface(MANAGER_IFACE) as unknown as SystemdManager; + } + return manager; +} + +/** Close the D-Bus connection so the oneshot process can exit. */ +export function disconnectSystemBus(): void { + bus?.disconnect(); + bus = null; + manager = null; +} + +/** systemctl-style shorthand: bare names get a .service suffix. */ +function unitName(unit: string): string { + return unit.includes('.') ? unit : `${unit}.service`; +} + +/** systemd ActiveState for a unit (active, inactive, failed, ...). */ +export async function getServiceState(unit: string): Promise { + try { + const mgr = await getManager(); + const path = await mgr.LoadUnit(unitName(unit)); + const obj = await getBus().getProxyObject(SYSTEMD, path); + const props = obj.getInterface(PROPERTIES_IFACE) as unknown as DbusProperties; + const state = await props.Get(UNIT_IFACE, 'ActiveState'); + return state.value || 'unknown'; + } catch { + // Any D-Bus failure just means the state can't be determined right now. + return 'unknown'; + } +} + +/** Enqueue a systemd job and wait for it to finish; throws unless the job + * completes with result "done". */ +async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit', unit: string): Promise { + const mgr = await getManager(); + // JobRemoved is only broadcast to subscribed clients. + await mgr.Subscribe(); + try { + await new Promise((resolve, reject) => { + // The job can finish before the method call returns its path, so + // removals seen in the meantime are buffered. + const removedEarly = new Map(); + let jobPath: string | undefined; + + const finish = (result: string) => { + mgr.removeListener('JobRemoved', onRemoved); + if (result === 'done') resolve(); + else reject(new Error(`systemd ${method} job for ${unit} finished with result "${result}"`)); + }; + const onRemoved = (_id: number, job: unknown, _unit: string, result: string) => { + const path = String(job); + if (jobPath === undefined) removedEarly.set(path, result); + else if (path === jobPath) finish(result); + }; + + mgr.on('JobRemoved', onRemoved); + mgr[method](unitName(unit), 'replace').then( + (job) => { + jobPath = String(job); + const result = removedEarly.get(jobPath); + if (result !== undefined) finish(result); + }, + (err: Error) => { + mgr.removeListener('JobRemoved', onRemoved); + reject(err); + }, + ); + }); + } finally { + await mgr.Unsubscribe(); + } +} + +/** Equivalent of `systemctl reload-or-restart `. */ +export function reloadOrRestartService(unit: string): Promise { + return runJob('ReloadOrRestartUnit', unit); +} + +/** Equivalent of `systemctl restart `. */ +export function restartService(unit: string): Promise { + return runJob('RestartUnit', unit); +} + +/** Equivalent of `systemctl kill --signal=SIGHUP ` (main process). */ +export async function sighupService(unit: string): Promise { + const mgr = await getManager(); + await mgr.KillUnit(unitName(unit), 'main', SIGHUP); +} diff --git a/agent/src/types.ts b/agent/src/types.ts new file mode 100644 index 00000000..4918e55d --- /dev/null +++ b/agent/src/types.ts @@ -0,0 +1,77 @@ +/** + * Shared types: check-in request/response shapes exchanged with the manager's + * POST /api/v1/agents endpoint. + */ + +export type ApplyResult = 'success' | 'failure'; + +export interface ServiceStatus { + /** systemd ActiveState: active, inactive, failed, ... */ + state: string; + /** Outcome of the last config apply for this service. */ + lastApply: ApplyResult | 'unknown'; +} + +export interface CheckinRequest { + siteId: number; + hostname: string; + /** Current agent time, epoch seconds UTC. */ + currentTime: number; + ipv4Address: string | null; + services: Record; +} + +export interface SiteContainer { + hostname: string; + ipv4Address: string; + macAddress: string | null; +} + +export interface SiteNode { + name: string; + ipv4Address: string | null; + containers: SiteContainer[]; +} + +/** Mirrors the manager's Site model, where every field except id is + * nullable — a site can be partially configured. Consumers must guard + * before using these values (see the dnsmasq render skip in apply.ts). */ +export interface SiteInfo { + id: number; + name: string | null; + internalDomain: string | null; + dhcpRange: string | null; + subnetMask: string | null; + gateway: string | null; + dnsForwarders: string | null; + nodes: SiteNode[]; +} + +export interface HttpService { + internalPort: number; + container: { ipv4Address: string }; + externalHostname: string; + backendProtocol: string; + authRequired: boolean; + externalDomain: { name: string; authServer: string | null }; +} + +export interface StreamService { + internalPort: number; + container: { ipv4Address: string }; + externalPort: number; + protocol: string; +} + +export interface NginxConfig { + httpServices: HttpService[]; + streamServices: StreamService[]; + externalDomains: { name: string }[]; +} + +/** Config snapshot for the whole site. `site` is null before the first site + * exists (bootstrap); only the fallback nginx config is rendered then. */ +export interface SiteConfig { + site: SiteInfo | null; + nginx: NginxConfig; +} diff --git a/create-a-container/views/dnsmasq/conf.ejs b/agent/templates/dnsmasq/conf.ejs similarity index 100% rename from create-a-container/views/dnsmasq/conf.ejs rename to agent/templates/dnsmasq/conf.ejs diff --git a/create-a-container/views/dnsmasq/dhcp-hosts.ejs b/agent/templates/dnsmasq/dhcp-hosts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/dhcp-hosts.ejs rename to agent/templates/dnsmasq/dhcp-hosts.ejs diff --git a/create-a-container/views/dnsmasq/dhcp-opts.ejs b/agent/templates/dnsmasq/dhcp-opts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/dhcp-opts.ejs rename to agent/templates/dnsmasq/dhcp-opts.ejs diff --git a/create-a-container/views/dnsmasq/hosts.ejs b/agent/templates/dnsmasq/hosts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/hosts.ejs rename to agent/templates/dnsmasq/hosts.ejs diff --git a/create-a-container/views/dnsmasq/servers.ejs b/agent/templates/dnsmasq/servers.ejs similarity index 100% rename from create-a-container/views/dnsmasq/servers.ejs rename to agent/templates/dnsmasq/servers.ejs diff --git a/create-a-container/views/nginx-conf.ejs b/agent/templates/nginx.conf.ejs similarity index 94% rename from create-a-container/views/nginx-conf.ejs rename to agent/templates/nginx.conf.ejs index db703bca..dad4b500 100644 --- a/create-a-container/views/nginx-conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -170,8 +170,8 @@ http { } <%_ httpServices.forEach((service, index) => { _%> - <%_ const authRequired = service.httpService.authRequired; _%> - <%_ const authServer = service.httpService.externalDomain.authServer; _%> + <%_ const authRequired = service.authRequired; _%> + <%_ const authServer = service.externalDomain.authServer; _%> server { listen 443 ssl; listen [::]:443 ssl; @@ -180,11 +180,11 @@ http { http2 on; http3 on; - server_name <%= service.httpService.externalHostname %>.<%= service.httpService.externalDomain.name %>; + server_name <%= service.externalHostname %>.<%= service.externalDomain.name %>; <%_ /* SSL certificates */ -%> - ssl_certificate /etc/ssl/certs/<%= service.httpService.externalDomain.name %>.crt; - ssl_certificate_key /etc/ssl/private/<%= service.httpService.externalDomain.name %>.key; + ssl_certificate /etc/ssl/certs/<%= service.externalDomain.name %>.crt; + ssl_certificate_key /etc/ssl/private/<%= service.externalDomain.name %>.key; error_page 403 @403; location @403 { @@ -281,7 +281,7 @@ http { proxy_set_header X-Email $auth_email; proxy_set_header X-Access-Token $auth_token; <%_ } _%> - proxy_pass <%= service.httpService.backendProtocol %>://<%= service.Container.ipv4Address %>:<%= service.internalPort %>; + proxy_pass <%= service.backendProtocol %>://<%= service.container.ipv4Address %>:<%= service.internalPort %>; proxy_http_version 1.1; <%_ /* Proxy headers */ -%> @@ -408,8 +408,8 @@ stream { <%_ streamServices.forEach((service, index) => { _%> server { - listen <%= service.transportService.externalPort %><%= service.transportService.protocol === 'udp' ? ' udp' : '' %>; - proxy_pass <%= service.Container.ipv4Address %>:<%= service.internalPort %>; + listen <%= service.externalPort %><%= service.protocol === 'udp' ? ' udp' : '' %>; + proxy_pass <%= service.container.ipv4Address %>:<%= service.internalPort %>; } <%_ }) _%> } diff --git a/agent/tsconfig.json b/agent/tsconfig.json new file mode 100644 index 00000000..a853fb00 --- /dev/null +++ b/agent/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "moduleResolution": "node", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src"] +} diff --git a/compose.yml b/compose.yml index 97461bc6..f904fb26 100644 --- a/compose.yml +++ b/compose.yml @@ -28,19 +28,20 @@ services: skopeo copy docker://$${IMAGE_REF} oci-archive:$${FILENAME} fi -# This service runs once when bringing the compose stack up to ensure the -# node_modules are populated in the user's create-a-container directory. This -# is usually handled by the manager's Dockerfile, but we're mounting over that -# directory so we need to be sure that the user has them. + # This service runs once when bringing the compose stack up to ensure the + # node_modules are populated in the user's create-a-container directory. This + # is usually handled by the manager's Dockerfile, but we're mounting over that + # directory so we need to be sure that the user has them. node: image: node:24-trixie volumes: - ./:/opt/opensource-server - working_dir: /opt/opensource-server/create-a-container + working_dir: /opt/opensource-server entrypoint: ["/bin/sh", "-c"] command: - | - exec npm ci --no-audit --no-fund + npm -C create-a-container ci --no-audit --no-fund + make -C agent build # Watches the React client and rebuilds the production bundle on change. The # Manager LXC inside Proxmox serves create-a-container/client/dist statically diff --git a/create-a-container/Makefile b/create-a-container/Makefile index 26d8b31e..76334078 100644 --- a/create-a-container/Makefile +++ b/create-a-container/Makefile @@ -15,7 +15,7 @@ INSTALL_DATA := $(INSTALL) -m 0644 # metadata are intentionally excluded; only the built client/dist ships. APP_FILES := server.js app.js job-runner.js package.json package-lock.json openapi.v1.yaml APP_DIRS := bin config data middlewares migrations models node_modules \ - public resources routers seeders utils views + public resources routers seeders utils .PHONY: help deps build dev test install deb rpm apk package clean diff --git a/create-a-container/app.js b/create-a-container/app.js index dc2a609a..3f714236 100644 --- a/create-a-container/app.js +++ b/create-a-container/app.js @@ -34,9 +34,6 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { const app = express(); - // setup views (still used by templates router for nginx-conf / dnsmasq files) - app.set('views', path.join(__dirname, 'views')); - app.set('view engine', 'ejs'); app.set('trust proxy', 1); // Parse query strings with qs so bracket notation (e.g. `user[0]=alice`) // yields real arrays. Express 5 defaults to the 'simple' parser. @@ -101,8 +98,8 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { // js/missing-token-validation). Behavior-preserving: csrfGuard skips // GET/HEAD/OPTIONS and Bearer-only requests, and every state-changing // route lives under /api/v1 where the v1 router already applies it — this - // additionally covers the GET-only surfaces (templates, swagger, SPA, - // static) and rejects mutations to unknown paths early. Mounted after the + // additionally covers the GET-only surfaces (swagger, SPA, static) and + // rejects mutations to unknown paths early. Mounted after the // rate limiter so CSRF 403s still count against the failure budget; the // v1 router keeps its own csrfGuard so it stays safe mounted standalone. const { csrfGuard, jsonErrorHandler } = require('./middlewares/api'); @@ -115,10 +112,8 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { // --- Mount Routers --- const apiV1Router = require('./routers/api/v1'); - const templatesRouter = require('./routers/templates'); app.use('/api/v1', apiV1Router); - app.use('/', templatesRouter); // serves /sites/:siteId/nginx and /sites/:siteId/dnsmasq/:file // --- API Documentation (Swagger UI) --- // Swagger UI at /api documents the versioned v1 API (the spec also served at /api/v1/openapi.*). @@ -134,7 +129,7 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { // --- SPA: serve compiled React app for everything else --- const clientDist = path.join(__dirname, 'client', 'dist'); app.use(express.static(clientDist)); - app.get(/^\/(?!api(\/|$)|sites\/[^/]+\/(nginx$|dnsmasq\/)).*$/, (req, res) => { + app.get(/^\/(?!api(\/|$)).*$/, (req, res) => { res.sendFile(path.join(clientDist, 'index.html')); }); diff --git a/create-a-container/client/src/app/Sidebar.tsx b/create-a-container/client/src/app/Sidebar.tsx index ae6fefdd..33fe8dce 100644 --- a/create-a-container/client/src/app/Sidebar.tsx +++ b/create-a-container/client/src/app/Sidebar.tsx @@ -19,6 +19,7 @@ import { Container as ContainerIcon, Globe, KeyRound, + Radio, Server, Settings, Users, @@ -57,6 +58,7 @@ const ADMIN: NavLink[] = [ icon: , adminOnly: true, }, + { to: '/agents', label: 'Agents', icon: , adminOnly: true }, { to: '/apikeys', label: 'API Keys', icon: }, { to: '/settings', label: 'Settings', icon: , adminOnly: true }, ]; diff --git a/create-a-container/client/src/app/router.tsx b/create-a-container/client/src/app/router.tsx index 00cd3b1e..50e2d7cc 100644 --- a/create-a-container/client/src/app/router.tsx +++ b/create-a-container/client/src/app/router.tsx @@ -16,6 +16,7 @@ import { NodeFormPage } from '@/pages/nodes/NodeFormPage'; import { NodeImportPage } from '@/pages/nodes/NodeImportPage'; import { ExternalDomainsListPage } from '@/pages/external-domains/ExternalDomainsListPage'; import { ExternalDomainFormPage } from '@/pages/external-domains/ExternalDomainFormPage'; +import { AgentsListPage } from '@/pages/agents/AgentsListPage'; import { JobDetailPage } from '@/pages/jobs/JobDetailPage'; import { UsersListPage } from '@/pages/users/UsersListPage'; import { UserFormPage } from '@/pages/users/UserFormPage'; @@ -65,6 +66,8 @@ export const router = createBrowserRouter([ { path: '/external-domains/new', element: }, { path: '/external-domains/:id/edit', element: }, + { path: '/agents', element: }, + { path: '/jobs/:id', element: }, { path: '/users', element: }, diff --git a/create-a-container/client/src/lib/queries.ts b/create-a-container/client/src/lib/queries.ts index 1177f3f5..29b75e56 100644 --- a/create-a-container/client/src/lib/queries.ts +++ b/create-a-container/client/src/lib/queries.ts @@ -4,6 +4,7 @@ */ import { api } from './api'; import type { + Agent, ApiKey, Container, ContainerMetadata, @@ -35,6 +36,7 @@ export const keys = { containerMetadata: (image: string) => ['container-metadata', image] as const, externalDomains: () => ['external-domains'] as const, externalDomain: (id: number | string) => ['external-domains', String(id)] as const, + agents: () => ['agents'] as const, users: () => ['users'] as const, user: (uid: number | string) => ['users', String(uid)] as const, groups: () => ['groups'] as const, @@ -60,6 +62,9 @@ export const queries = { getNode: (siteId: number | string, id: number | string) => api.get(`/api/v1/sites/${siteId}/nodes/${id}`), + // Agents + listAgents: () => api.get('/api/v1/agents'), + // Containers listContainers: ( siteId: number | string, diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index dab1c503..4b228867 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -31,6 +31,25 @@ export interface Node { hasSecret: boolean; } +export interface AgentServiceStatus { + /** systemd ActiveState: active, inactive, failed, ... */ + state: string; + /** Outcome of the agent's last config apply for this service. */ + lastApply: 'success' | 'failure' | 'unknown'; +} + +export interface Agent { + id: number; + siteId: number; + siteName: string | null; + hostname: string; + ipv4Address: string | null; + services: Record | null; + lastCheckinAt: string | null; + /** Server-computed, so it is immune to client clock drift. */ + secondsSinceCheckin: number | null; +} + export interface ExternalDomain { id: number; name: string; diff --git a/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx b/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx new file mode 100644 index 00000000..cb565266 --- /dev/null +++ b/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx @@ -0,0 +1,32 @@ +import { Badge } from '@mieweb/ui'; +import type { AgentServiceStatus } from '@/lib/types'; + +function badgeVariant(status: AgentServiceStatus): 'success' | 'danger' | 'warning' { + if (status.state !== 'active') return 'danger'; + if (status.lastApply === 'failure') return 'warning'; + return 'success'; +} + +/** One badge per managed service, e.g. "nginx: active" — red when the unit + * isn't active, amber when the last config apply failed. */ +export function AgentServiceBadges({ + services, +}: { + services: Record | null; +}) { + const entries = Object.entries(services ?? {}); + if (entries.length === 0) return —; + return ( +
+ {entries.map(([name, status]) => ( + + {name}: {status.state} + + ))} +
+ ); +} diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx new file mode 100644 index 00000000..96bbfbb1 --- /dev/null +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -0,0 +1,99 @@ +import { useQuery } from '@tanstack/react-query'; +import { + Alert, + AlertDescription, + PageHeader, + Spinner, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@mieweb/ui'; +import { Radio } from 'lucide-react'; +import { ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; +import type { Agent } from '@/lib/types'; +import { useDocumentTitle } from '@/lib/useDocumentTitle'; +import { AgentServiceBadges } from './AgentServiceBadges'; +import { OnlineBadge } from './OnlineBadge'; + +function formatLastCheckin(agent: Agent): string { + const seconds = agent.secondsSinceCheckin; + if (seconds === null || !agent.lastCheckinAt) return 'never'; + if (seconds < 60) return `${seconds}s ago`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + return new Date(agent.lastCheckinAt).toLocaleString(); +} + +export function AgentsListPage() { + useDocumentTitle('Agents'); + const { data, isLoading, error } = useQuery({ + queryKey: keys.agents(), + queryFn: queries.listAgents, + // Agents check in every 30s; keep the health view current. + refetchInterval: 30000, + }); + + const hasAgents = !!data && data.length > 0; + + return ( +
+ } + /> + {error && ( + + {(error as ApiError).message} + + )} + {isLoading && ( +
+ +
+ )} + {data && data.length === 0 && ( + + + No agents have checked in yet. Agents register themselves at their first check-in. + + + )} + + {hasAgents && ( + + + + Hostname + Site + IPv4 + Status + Services + Last check-in + + + + {data.map((agent: Agent) => ( + + {agent.hostname} + {agent.siteName || agent.siteId} + {agent.ipv4Address || '—'} + + + + + + + {formatLastCheckin(agent)} + + ))} + +
+ )} +
+ ); +} diff --git a/create-a-container/client/src/pages/agents/OnlineBadge.tsx b/create-a-container/client/src/pages/agents/OnlineBadge.tsx new file mode 100644 index 00000000..b39c0a22 --- /dev/null +++ b/create-a-container/client/src/pages/agents/OnlineBadge.tsx @@ -0,0 +1,15 @@ +import { Badge } from '@mieweb/ui'; +import type { Agent } from '@/lib/types'; + +// Agents check in every 30 seconds; three missed intervals means offline. +const OFFLINE_AFTER_SECONDS = 90; + +export function OnlineBadge({ agent }: { agent: Agent }) { + const online = + agent.secondsSinceCheckin !== null && agent.secondsSinceCheckin <= OFFLINE_AFTER_SECONDS; + return online ? ( + Online + ) : ( + Offline + ); +} diff --git a/create-a-container/middlewares/index.js b/create-a-container/middlewares/index.js index 05ebb897..b2f8654a 100644 --- a/create-a-container/middlewares/index.js +++ b/create-a-container/middlewares/index.js @@ -73,10 +73,9 @@ function requireAdmin(req, res, next) { return res.status(403).send('Forbidden: Admin access required'); } -// Localhost-or-admin middleware -// Allows localhost requests through without auth. Remote requests must authenticate -// as an admin user (via session or API key). -function requireLocalhostOrAdmin(req, res, next) { +// True when the request comes directly from localhost (and was not proxied +// on behalf of a remote client, per X-Real-IP / X-Forwarded-For). +function isLocalhostRequest(req) { const isLocalhost = (ip) => { return ip === '127.0.0.1' || ip === '::1' || @@ -89,26 +88,22 @@ function requireLocalhostOrAdmin(req, res, next) { req.ip; const realIp = req.get('X-Real-IP'); + // First hop of X-Forwarded-For is the original client (covers proxies that + // set it without also setting X-Real-IP). + const forwardedFor = (req.get('X-Forwarded-For') || '').split(',')[0].trim(); - // If direct connection is from localhost and no non-localhost X-Real-IP, allow through - if (isLocalhost(directIp) && (!realIp || isLocalhost(realIp))) { - return next(); - } - - // Not localhost — require auth + admin - requireAuth(req, res, (err) => { - if (err) return next(err); - requireAdmin(req, res, next); - }); + return isLocalhost(directIp) + && (!realIp || isLocalhost(realIp)) + && (!forwardedFor || isLocalhost(forwardedFor)); } const { setCurrentSite, loadSites } = require('./currentSite'); module.exports = { isApiRequest, + isLocalhostRequest, requireAuth, requireAdmin, - requireLocalhostOrAdmin, setCurrentSite, loadSites }; diff --git a/create-a-container/migrations/20260714120000-create-agents.js b/create-a-container/migrations/20260714120000-create-agents.js new file mode 100644 index 00000000..3f4db170 --- /dev/null +++ b/create-a-container/migrations/20260714120000-create-agents.js @@ -0,0 +1,51 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('Agents', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER + }, + siteId: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: 'Sites', key: 'id' }, + onDelete: 'CASCADE' + }, + hostname: { + type: Sequelize.STRING, + allowNull: false + }, + ipv4Address: { + type: Sequelize.STRING, + allowNull: true + }, + services: { + type: Sequelize.JSON, + allowNull: true + }, + lastCheckinAt: { + type: Sequelize.DATE, + allowNull: true + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }); + await queryInterface.addIndex('Agents', ['siteId', 'hostname'], { + unique: true, + name: 'agents_site_id_hostname_unique' + }); + }, + async down(queryInterface) { + await queryInterface.dropTable('Agents'); + } +}; diff --git a/create-a-container/models/agent.js b/create-a-container/models/agent.js new file mode 100644 index 00000000..16d36740 --- /dev/null +++ b/create-a-container/models/agent.js @@ -0,0 +1,43 @@ +'use strict'; +const { Model } = require('sequelize'); +module.exports = (sequelize, DataTypes) => { + class Agent extends Model { + static associate(models) { + Agent.belongsTo(models.Site, { + foreignKey: 'siteId', + as: 'site' + }); + } + } + Agent.init({ + siteId: { + type: DataTypes.INTEGER, + allowNull: false + }, + hostname: { + type: DataTypes.STRING, + allowNull: false + }, + ipv4Address: { + type: DataTypes.STRING, + allowNull: true + }, + services: { + // Per-service status as reported by the agent at check-in: + // { nginx: { state: 'active', lastApply: 'success' }, ... } + type: DataTypes.JSON, + allowNull: true + }, + lastCheckinAt: { + type: DataTypes.DATE, + allowNull: true + } + }, { + sequelize, + modelName: 'Agent', + indexes: [ + { unique: true, fields: ['siteId', 'hostname'] } + ] + }); + return Agent; +}; diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index 01a1820e..e44204a6 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -32,6 +32,7 @@ tags: - name: Sites - name: Containers - name: Nodes + - name: Agents - name: Jobs - name: External Domains - name: Users @@ -537,6 +538,50 @@ paths: description: SSE stream — events `log` and `status` content: { text/event-stream: {} } + /agents: + post: + tags: [Agents] + summary: Agent check-in + description: | + Site agents POST their system info and per-service status every 30 + seconds. The response carries the site's config snapshot with a strong + `ETag`; send it back via `If-None-Match` to receive `304 Not Modified` + when nothing changed. Allowed from localhost without credentials + (manager bootstrap) or with an admin API key. + parameters: + - in: header + name: If-None-Match + required: false + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [siteId, hostname] + properties: + siteId: { type: integer } + hostname: { type: string } + currentTime: { type: integer, description: Agent time, epoch seconds UTC } + ipv4Address: { type: string, nullable: true } + services: + type: object + additionalProperties: + type: object + properties: + state: { type: string, description: 'systemd ActiveState: active, inactive, failed, ...' } + lastApply: { type: string, enum: [success, failure, unknown] } + responses: + '200': + description: Config snapshot (`{ data: { site, nginx } }`) with `ETag` header + '304': { description: Config unchanged since If-None-Match } + '422': { description: Missing siteId/hostname } + get: + tags: [Agents] + summary: Current status of all agents (admin) + responses: { '200': { description: 'List of agents with lastCheckinAt, services and server-computed secondsSinceCheckin' } } + /external-domains: get: { tags: [External Domains], responses: { '200': { description: List } } } post: { tags: [External Domains], responses: { '201': { description: Created (admin) } } } diff --git a/create-a-container/package-lock.json b/create-a-container/package-lock.json index 61bf89fe..880f51ba 100644 --- a/create-a-container/package-lock.json +++ b/create-a-container/package-lock.json @@ -11,7 +11,6 @@ "cookie-parser": "^1.4.7", "csrf-sync": "^4.2.1", "dotenv": "^17.2.3", - "ejs": "^3.1.10", "express": "^5.2.1", "express-rate-limit": "^8.5.1", "express-session": "^1.19.0", @@ -1750,12 +1749,6 @@ "dev": true, "license": "MIT" }, - "node_modules/async": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", - "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "license": "MIT" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1980,6 +1973,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -2725,21 +2719,6 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", "license": "MIT" }, - "node_modules/ejs": { - "version": "3.1.10", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", - "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", - "license": "Apache-2.0", - "dependencies": { - "jake": "^10.8.5" - }, - "bin": { - "ejs": "bin/cli.js" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/electron-to-chromium": { "version": "1.5.389", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.389.tgz", @@ -3083,15 +3062,6 @@ "bser": "2.1.1" } }, - "node_modules/filelist": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", - "integrity": "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==", - "license": "Apache-2.0", - "dependencies": { - "minimatch": "^5.0.1" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3924,23 +3894,6 @@ "@pkgjs/parseargs": "^0.11.0" } }, - "node_modules/jake": { - "version": "10.9.4", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", - "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", - "license": "Apache-2.0", - "dependencies": { - "async": "^3.2.6", - "filelist": "^1.0.4", - "picocolors": "^1.1.1" - }, - "bin": { - "jake": "bin/cli.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/jest": { "version": "30.4.2", "resolved": "https://registry.npmjs.org/jest/-/jest-30.4.2.tgz", @@ -4864,18 +4817,6 @@ "node": ">=6" } }, - "node_modules/minimatch": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", - "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -5642,6 +5583,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { diff --git a/create-a-container/package.json b/create-a-container/package.json index c008a9f0..ee3a338c 100644 --- a/create-a-container/package.json +++ b/create-a-container/package.json @@ -25,7 +25,6 @@ "cookie-parser": "^1.4.7", "csrf-sync": "^4.2.1", "dotenv": "^17.2.3", - "ejs": "^3.1.10", "express": "^5.2.1", "express-rate-limit": "^8.5.1", "express-session": "^1.19.0", diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js new file mode 100644 index 00000000..84b6b45a --- /dev/null +++ b/create-a-container/routers/api/v1/agents.js @@ -0,0 +1,85 @@ +/** + * /api/v1/agents — site agent check-in and status. + * + * POST / agent check-in: records system info + service states, responds with + * the site's config snapshot (or 304 when the agent's If-None-Match + * ETag still matches). + * GET / admin: current status of all agents. + */ + +const express = require('express'); +const { Agent, Site } = require('../../../models'); +const { isLocalhostRequest } = require('../../../middlewares'); +const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); +const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); + +const router = express.Router(); + +// Check-in auth: the manager's own agent checks in over localhost without +// credentials (bootstrap: no site, no API key exist yet); remote agents +// authenticate with an admin API key via apiAuth/apiAdmin so error +// responses follow the v1 JSON envelope. +function checkinAuth(req, res, next) { + if (isLocalhostRequest(req)) return next(); + return apiAuth(req, res, (err) => { + if (err) return next(err); + return apiAdmin(req, res, next); + }); +} + +router.post('/', checkinAuth, asyncHandler(async (req, res) => { + const { siteId, hostname, ipv4Address, services } = req.body || {}; + const parsedSiteId = typeof siteId === 'number' ? siteId : Number(siteId); + if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { + return fail(res, 422, 'validation_failed', 'siteId and hostname are required'); + } + + // Record the check-in. Skipped during bootstrap (the site row doesn't exist + // yet) since the foreign key has nothing to point at. + const site = await Site.findByPk(parsedSiteId); + if (site) { + const [agent] = await Agent.findOrCreate({ + where: { siteId: parsedSiteId, hostname }, + }); + await agent.update({ + ipv4Address: ipv4Address || null, + services: services || null, + lastCheckinAt: new Date(), + }); + } + + const config = await buildAgentConfig(parsedSiteId); + // Manual conditional-request handling: Express's built-in ETag/fresh logic + // (res.send + req.fresh) only produces 304s for GET/HEAD, and the check-in + // is a POST. + const etag = computeConfigEtag(config); + res.set('ETag', etag); + if (req.get('If-None-Match') === etag) { + return res.status(304).end(); + } + return ok(res, config); +})); + +router.get('/', apiAuth, apiAdmin, asyncHandler(async (req, res) => { + const agents = await Agent.findAll({ + include: [{ model: Site, as: 'site', attributes: ['id', 'name'] }], + order: [['siteId', 'ASC'], ['hostname', 'ASC']], + }); + const now = Date.now(); + return ok(res, agents.map((a) => ({ + id: a.id, + siteId: a.siteId, + siteName: a.site?.name || null, + hostname: a.hostname, + ipv4Address: a.ipv4Address, + services: a.services, + lastCheckinAt: a.lastCheckinAt, + // Computed server-side so UI staleness judgments don't depend on the + // client's clock. + secondsSinceCheckin: a.lastCheckinAt + ? Math.max(0, Math.round((now - new Date(a.lastCheckinAt).getTime()) / 1000)) + : null, + }))); +})); + +module.exports = router; diff --git a/create-a-container/routers/api/v1/index.js b/create-a-container/routers/api/v1/index.js index 3cf43a38..815f0e8c 100644 --- a/create-a-container/routers/api/v1/index.js +++ b/create-a-container/routers/api/v1/index.js @@ -51,6 +51,11 @@ router.get('/openapi.yaml', (_req, res) => { res.type('text/yaml').sendFile(path.join(__dirname, '..', '..', '..', 'openapi.v1.yaml')); }); +// Agent check-in — mounted before the CSRF guard because the manager's own +// agent posts from localhost with neither a session cookie nor a Bearer token +// (auth is handled inside the router: localhost or admin API key). +router.use('/agents', require('./agents')); + // CSRF guard before any state-changing route below router.use(csrfGuard); diff --git a/create-a-container/routers/templates.js b/create-a-container/routers/templates.js deleted file mode 100644 index 938ce756..00000000 --- a/create-a-container/routers/templates.js +++ /dev/null @@ -1,103 +0,0 @@ -const express = require('express'); -const { Op } = require('sequelize'); -const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain } = require('../models'); -const { requireLocalhostOrAdmin } = require('../middlewares'); - -const router = express.Router(); - -async function loadDnsmasqSite(siteId) { - return Site.findByPk(siteId, { - include: [{ - model: Node, - as: 'nodes', - include: [{ - model: Container, - as: 'containers', - where: { ipv4Address: { [Op.ne]: null } }, - required: false, - attributes: ['macAddress', 'ipv4Address', 'hostname'], - }], - }], - }); -} - -const DNSMASQ_TEMPLATES = ['conf', 'dhcp-hosts', 'hosts', 'dhcp-opts', 'servers']; - -router.get('/sites/:siteId/dnsmasq/:file', requireLocalhostOrAdmin, async (req, res) => { - const { file } = req.params; - if (!DNSMASQ_TEMPLATES.includes(file)) return res.status(404).send('Not found'); - const site = await loadDnsmasqSite(parseInt(req.params.siteId, 10)); - if (!site) return res.status(404).send('Site not found'); - res.set('Content-Type', 'text/plain'); - return res.render(`dnsmasq/${file}`, { site }); -}); - -router.get('/sites/:siteId/nginx', requireLocalhostOrAdmin, async (req, res) => { - const siteId = parseInt(req.params.siteId, 10); - const site = await Site.findByPk(siteId, { - include: [{ - model: Node, - as: 'nodes', - include: [{ - model: Container, - as: 'containers', - where: { ipv4Address: { [Op.ne]: null } }, - required: false, - include: [{ - model: Service, - as: 'services', - include: [ - { - model: HTTPService, - as: 'httpService', - include: [{ model: ExternalDomain, as: 'externalDomain' }], - }, - { model: TransportService, as: 'transportService' }, - ], - }], - }], - }, { - model: ExternalDomain, - as: 'externalDomains', - }], - }); - - // Bootstrap fallback: if the site does not exist yet, still render an - // empty nginx config so the manager API remains reachable (over TLS) to - // create the first site. Without this, bootstrapping would require - // plaintext HTTP access, which breaks our security requirements for - // registering nodes and creating the first site. - if (!site) { - res.set('Content-Type', 'text/plain'); - return res.render('nginx-conf', { - httpServices: [], - streamServices: [], - externalDomains: [], - }); - } - - const allServices = []; - site?.nodes?.forEach((node) => { - node?.containers?.forEach((container) => { - container?.services?.forEach((service) => { - service.Container = container; - allServices.push(service); - }); - }); - }); - - const httpServices = allServices.filter((s) => s.type === 'http'); - const streamServices = allServices.filter((s) => s.type === 'transport'); - - const usedDomainIds = new Set(); - httpServices.forEach((s) => { - if (s.httpService?.externalDomain?.id) usedDomainIds.add(s.httpService.externalDomain.id); - }); - (site?.externalDomains || []).forEach((d) => usedDomainIds.add(d.id)); - const externalDomains = await ExternalDomain.findAll({ where: { id: [...usedDomainIds] } }); - - res.set('Content-Type', 'text/plain'); - return res.render('nginx-conf', { httpServices, streamServices, externalDomains }); -}); - -module.exports = router; diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js new file mode 100644 index 00000000..781d16f5 --- /dev/null +++ b/create-a-container/utils/agent-config.js @@ -0,0 +1,150 @@ +/** + * Builds the site configuration snapshot returned to agents at check-in. + * + * The snapshot is plain JSON with a deterministic shape so a strong ETag can + * be computed over it: agents send the ETag back via If-None-Match and get a + * 304 when nothing changed. The shape mirrors what the agent's EJS templates + * (agent/templates/) expect. + */ + +const crypto = require('crypto'); +const { Op } = require('sequelize'); +const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain } = require('../models'); + +/** + * Load a site with everything the agent templates need, serialized to plain + * JSON. Returns `{ site: null, nginx: { ... empty ... } }` when the site does + * not exist yet — a bootstrap fallback so a fresh manager's own agent can + * render a minimal nginx config (manager reachable over TLS) before the first + * site is created. + * + * @param {number} siteId + * @returns {Promise} `{ site, nginx: { httpServices, streamServices, externalDomains } }` + */ +async function buildAgentConfig(siteId) { + const site = await Site.findByPk(siteId, { + include: [{ + model: Node, + as: 'nodes', + include: [{ + model: Container, + as: 'containers', + where: { ipv4Address: { [Op.ne]: null } }, + required: false, + include: [{ + model: Service, + as: 'services', + include: [ + { + model: HTTPService, + as: 'httpService', + include: [{ model: ExternalDomain, as: 'externalDomain' }], + }, + { model: TransportService, as: 'transportService' }, + ], + }], + }], + }, { + model: ExternalDomain, + as: 'externalDomains', + }], + order: [ + [{ model: Node, as: 'nodes' }, 'id', 'ASC'], + [{ model: Node, as: 'nodes' }, { model: Container, as: 'containers' }, 'id', 'ASC'], + [{ model: Node, as: 'nodes' }, { model: Container, as: 'containers' }, { model: Service, as: 'services' }, 'id', 'ASC'], + [{ model: ExternalDomain, as: 'externalDomains' }, 'id', 'ASC'], + ], + }); + + if (!site) { + return { + site: null, + nginx: { httpServices: [], streamServices: [], externalDomains: [] }, + }; + } + + const httpServices = []; + const streamServices = []; + for (const node of site.nodes || []) { + for (const container of node.containers || []) { + for (const service of container.services || []) { + const base = { + internalPort: service.internalPort, + container: { ipv4Address: container.ipv4Address }, + }; + if (service.type === 'http' && service.httpService && service.httpService.externalDomain) { + httpServices.push({ + ...base, + externalHostname: service.httpService.externalHostname, + backendProtocol: service.httpService.backendProtocol, + authRequired: !!service.httpService.authRequired, + externalDomain: { + name: service.httpService.externalDomain.name, + authServer: service.httpService.externalDomain.authServer || null, + }, + }); + } else if (service.type === 'transport' && service.transportService) { + streamServices.push({ + ...base, + externalPort: service.transportService.externalPort, + protocol: service.transportService.protocol, + }); + } + } + } + } + + // Domains needing a TLS server block: any domain referenced by an HTTP + // service plus the site's own external domains. + const usedDomainIds = new Set(); + for (const node of site.nodes || []) { + for (const container of node.containers || []) { + for (const service of container.services || []) { + const id = service.httpService?.externalDomain?.id; + if (id) usedDomainIds.add(id); + } + } + } + for (const d of site.externalDomains || []) usedDomainIds.add(d.id); + const externalDomains = await ExternalDomain.findAll({ + where: { id: [...usedDomainIds] }, + order: [['id', 'ASC']], + }); + + return { + site: { + id: site.id, + name: site.name, + internalDomain: site.internalDomain, + dhcpRange: site.dhcpRange, + subnetMask: site.subnetMask, + gateway: site.gateway, + dnsForwarders: site.dnsForwarders, + nodes: (site.nodes || []).map((node) => ({ + name: node.name, + ipv4Address: node.ipv4Address, + containers: (node.containers || []).map((c) => ({ + hostname: c.hostname, + ipv4Address: c.ipv4Address, + macAddress: c.macAddress, + })), + })), + }, + nginx: { + httpServices, + streamServices, + externalDomains: externalDomains.map((d) => ({ name: d.name })), + }, + }; +} + +/** + * Strong ETag over a config snapshot. Deterministic because buildAgentConfig + * constructs the object with stable key/array ordering. + */ +function computeConfigEtag(config) { + const hash = crypto.createHash('sha256').update(JSON.stringify(config)).digest('hex'); + return `"${hash}"`; +} + +module.exports = { buildAgentConfig, computeConfigEtag }; diff --git a/images/agent/Dockerfile b/images/agent/Dockerfile index c46446d9..c68d219a 100644 --- a/images/agent/Dockerfile +++ b/images/agent/Dockerfile @@ -6,7 +6,7 @@ FROM nodejs COPY images/opensource-server.sources /etc/apt/sources.list.d/opensource-server.sources # Install the agent package (and its dependencies: nginx with stream + -# ModSecurity modules, dnsmasq, cron, ...) built by the builder image. +# ModSecurity modules, dnsmasq, ...) built by the builder image. RUN --mount=from=builder,source=/dist,target=/dist \ apt-get update && \ apt-get install -y /dist/opensource-agent_*.deb && \ @@ -29,7 +29,7 @@ COPY ./images/agent/nginx.logrotate /etc/logrotate.d/nginx COPY ./images/agent/crs-setup.conf /etc/modsecurity/crs/crs-setup.conf COPY ./images/agent/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf /etc/modsecurity/crs/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf -# Configure DNSMasq to only get its config from our pull-config +# Configure DNSMasq to only get its config from our agent RUN sed -i \ -e 's/^CONFIG_DIR=\(.*\)$/#CONFIG_DIR=\1/' \ -e 's/^#IGNORE_RESOLVCONF=\(.*\)$/IGNORE_RESOLVCONF=\1/' \ diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index 4eb624eb..b76bdfc4 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -1,8 +1,9 @@ # syntax=docker/dockerfile:1 FROM agent -# Default environment for pull-config. The manager runs the app on localhost -# so it doesn't need an API key. Remote agents must set these explicitly. +# Default environment for the site agent. The manager runs the app on +# localhost so it doesn't need an API key. Remote agents must set these +# explicitly. ENV SITE_ID=1 ENV MANAGER_URL=http://localhost:3000 diff --git a/mie-opensource-landing/docs/admins/deploying-agents.md b/mie-opensource-landing/docs/admins/deploying-agents.md index 89f45d21..d16adb39 100644 --- a/mie-opensource-landing/docs/admins/deploying-agents.md +++ b/mie-opensource-landing/docs/admins/deploying-agents.md @@ -16,7 +16,7 @@ Agents are deployed **manually in Proxmox** (not through the manager UI) and sho ```mermaid graph LR - A[Configure site
in manager] --> B[Generate
admin API key] --> C[Deploy agent
on Proxmox node] --> D[Verify agent
pulls config] --> E[Import node
in manager] + A[Configure site
in manager] --> B[Generate
admin API key] --> C[Deploy agent
on Proxmox node] --> D[Verify agent
checks in] --> E[Import node
in manager] classDef step fill:#f9f9f9,stroke:#333,stroke-width:1px class A,B,C,D,E step @@ -69,7 +69,7 @@ lxc.environment = API_KEY= |----------|-------------| | `SITE_ID` | Numeric site ID from the manager (visible in the URL when viewing the site) | | `MANAGER_URL` | Base URL of the manager container (e.g., `http://192.168.1.10:3000`) | -| `API_KEY` | API key from an admin account. Used to authenticate config pulls. | +| `API_KEY` | API key from an admin account. Used to authenticate check-ins. | ## 4. Start and Verify @@ -77,25 +77,26 @@ lxc.environment = API_KEY= pct start ``` -Verify pull-config is working by checking that nginx and dnsmasq configs were fetched: +Verify the agent is checking in and applied its configs: ```bash # Enter the container pct enter -# Check that configs were pulled +# Timer and last runs +systemctl status opensource-agent.timer +journalctl -u opensource-agent.service + +# Check that configs were applied cat /etc/nginx/nginx.conf cat /etc/dnsmasq.conf -# Test manually -/etc/pull-config.d/nginx -/etc/pull-config.d/dnsmasq-conf -/etc/pull-config.d/dnsmasq-dhcp-hosts -/etc/pull-config.d/dnsmasq-hosts -/etc/pull-config.d/dnsmasq-dhcp-opts -/etc/pull-config.d/dnsmasq-servers +# Run a check-in manually +systemctl start opensource-agent.service ``` +The agent also appears on the manager's **Agents** page (`/agents`, admin only) with its last check-in time and service health. + ## 5. Forward Network Traffic Forward the following ports from the Proxmox host to the agent container: @@ -113,25 +114,24 @@ With the agent running, proceed to [import the node](core-concepts/nodes.md#impo ## How It Works -The agent uses **pull-config**, a cron-based system that fetches configuration files from the manager every minute. +A systemd timer launches the agent every 30 seconds. It checks in with the manager, reporting host info and service status, and receives the site's config snapshot in return — see the [agent developer reference](../developers/agent.md) for the protocol and apply/rollback details. ```mermaid sequenceDiagram - participant Cron - participant pull-config + participant Timer as systemd timer + participant Agent participant Manager - Cron->>pull-config: run-parts /etc/pull-config.d - pull-config->>Manager: GET /sites/{SITE_ID}/nginx
Authorization: Bearer {API_KEY}
If-None-Match: {etag} + Timer->>Agent: start (every 30s) + Agent->>Manager: POST /api/v1/agents
Authorization: Bearer {API_KEY}
If-None-Match: {etag} alt Config changed - Manager-->>pull-config: 200 OK + new config - pull-config->>pull-config: Validate (nginx -t) - pull-config->>pull-config: Apply + reload + Manager-->>Agent: 200 OK + config snapshot + ETag + Agent->>Agent: Render, validate (nginx -t), apply + reload + Agent->>Manager: POST again (reports apply results) + Manager-->>Agent: 304 Not Modified else No changes - Manager-->>pull-config: 304 Not Modified + Manager-->>Agent: 304 Not Modified end ``` -Each service (nginx, dnsmasq) has its own instance script(s) in `/etc/pull-config.d/` that run independently. ETag caching ensures configs are only downloaded when they change. - -Dnsmasq is split into 5 pull-config instances: the main config (`dnsmasq-conf`) triggers a full restart, while the auxiliary files (`dnsmasq-dhcp-hosts`, `dnsmasq-hosts`, `dnsmasq-dhcp-opts`, `dnsmasq-servers`) trigger a SIGHUP reload. This avoids restarting dnsmasq when only DHCP leases or host records change. +Each check-in is recorded by the manager, so admins can monitor agent health on the **Agents** page. ETag caching keeps unchanged configs to a single 304 round trip. Dnsmasq's main config triggers a full restart when it changes, while the auxiliary files (DHCP hosts, host records, options, upstream servers) only trigger a SIGHUP reload. diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md new file mode 100644 index 00000000..e00e7012 --- /dev/null +++ b/mie-opensource-landing/docs/developers/agent.md @@ -0,0 +1,83 @@ + +# agent + +{{ contributor_warning }} + +Node.js (TypeScript) check-in agent that reports host/service status to the manager and applies nginx and dnsmasq configuration. Installed on agent and manager containers. For deployment instructions, see [Deploying Agents](../admins/deploying-agents.md). + +## Architecture + +A systemd timer launches the oneshot agent every 30 seconds. Each run checks in with the manager (`POST /api/v1/agents`), applies any config changes, and exits. + +TypeScript sources live in `agent/src` (compiled to `dist/`), the locally rendered EJS templates in `agent/templates`, and the systemd service/timer units in `agent/contrib/systemd`. The `agent/Makefile` builds the `opensource-agent` package (see Release Pipeline). Service reloads and state queries go through the systemd D-Bus API; config validation uses `nginx -t` and `dnsmasq --test`. + +## Check-in Protocol + +```mermaid +sequenceDiagram + participant Timer as systemd timer + participant Agent + participant Manager + + Timer->>Agent: start (every 30s) + Agent->>Manager: POST /api/v1/agents
{siteId, hostname, ipv4Address, services}
If-None-Match: {etag} + alt Config changed + Manager-->>Agent: 200 OK + config snapshot + ETag + Agent->>Agent: Render templates, validate
(nginx -t, dnsmasq --test) + Agent->>Agent: Apply + reload (rollback on failure) + Agent->>Manager: POST again (reports apply results) + Manager-->>Agent: 304 Not Modified + else No changes + Manager-->>Agent: 304 Not Modified + end + Agent->>Timer: exit +``` + +The check-in body carries system info and per-service status: + +```json +{ + "siteId": 1, + "hostname": "agent", + "currentTime": 1771234560, + "ipv4Address": "172.17.0.2", + "services": { + "nginx": { "state": "active", "lastApply": "success" }, + "dnsmasq": { "state": "active", "lastApply": "success" } + } +} +``` + +The manager records every check-in in the `Agents` table (shown on the web client's `/agents` page) and responds with the site's config snapshot as JSON. A strong `ETag` covers the snapshot; the agent stores it in `/var/lib/opensource-agent/state.json` and sends it back via `If-None-Match`, so unchanged configs cost a single `304` round trip. + +## Environment Variables + +Read from the process environment (systemd loads `/etc/environment`). Set via container runtime (Docker `ENV`, Proxmox LXC config) — the base image's `environment.sh` service propagates them on boot. + +| Variable | Required | Description | +|----------|----------|-------------| +| `SITE_ID` | Yes | Numeric site ID from the manager | +| `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | +| `API_KEY` | No | Admin API key for remote agents. Not needed on the manager (localhost is trusted). | +| `STATE_DIRECTORY` | No | State directory, set by systemd via `StateDirectory=` (default `/var/lib/opensource-agent`) | + +The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. + +## Managed Services + +| Service | Files | Test | Reload | +|---------|-------|------|--------| +| nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | +| dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | + +Apply flow per service: render templates, skip if nothing changed, stage new files, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. + +To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. + +## Running Manually + +```bash +systemctl status opensource-agent.timer # timer state +journalctl -u opensource-agent.service # past runs +node /opt/opensource-server/agent/dist/index.js # single run by hand +``` diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index 6b567db0..149fec58 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -11,6 +11,7 @@ The cluster management system uses Sequelize ORM with PostgreSQL. While Sequeliz erDiagram Sites ||--o{ Nodes : contains Sites ||--o{ ExternalDomains : "default site" + Sites ||--o{ Agents : "checked in by" Nodes ||--o{ Containers : hosts Containers ||--o{ Services : exposes Containers }o--o| Jobs : "created by" @@ -51,6 +52,15 @@ erDiagram int siteId FK } + Agents { + int id PK + int siteId FK + string hostname "unique per site" + string ipv4Address + json services "per-service state + lastApply" + date lastCheckinAt + } + Containers { int id PK string hostname UK @@ -179,6 +189,9 @@ Top-level organizational unit. Has many Nodes. Has many ExternalDomains (as defa ### Node Proxmox VE server within a site. `name` must match Proxmox hostname (unique). `imageStorage` defaults to `'local'` (CT templates). `volumeStorage` defaults to `'local-lvm'` (container rootfs). `networkBridge` defaults to `'vmbr0'` (Proxmox bridge used in container net0 config). `nvidiaAvailable` indicates the node has NVIDIA drivers and nvidia-container-toolkit configured for GPU passthrough. Belongs to Site, has many Containers. +### Agent +Site agent registered by its check-in (`POST /api/v1/agents`, every 30s). Unique composite index on `(siteId, hostname)`. `services` stores the per-service status reported by the agent (`{ nginx: { state, lastApply }, ... }`); `lastCheckinAt` drives the online/offline health shown on the web client's Agents page. Belongs to Site. See [agent](agent.md). + ### Container LXC container on a Proxmox node. Unique composite index on `(nodeId, containerId)`. `hostname`, `macAddress`, `ipv4Address` globally unique. `nvidiaRequested` indicates GPU passthrough was requested — the container is assigned to an NVIDIA-capable node and the nvidia hookscript is attached. Belongs to Node and optionally to a Job. diff --git a/mie-opensource-landing/docs/developers/docker-images.md b/mie-opensource-landing/docs/developers/docker-images.md index 2c94c5b0..98566ef7 100644 --- a/mie-opensource-landing/docs/developers/docker-images.md +++ b/mie-opensource-landing/docs/developers/docker-images.md @@ -36,7 +36,7 @@ The same Dockerfile as `docker`, built on top of the `nodejs` image instead of t ### Agent (`agent`) -Extends nodejs with the `opensource-agent` package (pull-config, nginx with ModSecurity/OWASP CRS, dnsmasq) and [acme.sh](https://github.com/acmesh-official/acme.sh) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). +Extends nodejs with the `opensource-agent` package (check-in agent, nginx with ModSecurity/OWASP CRS, dnsmasq) and [acme.sh](https://github.com/acmesh-official/acme.sh) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). **Registry:** `ghcr.io/mieweb/opensource-server/agent` · **Source:** [`images/agent/`](https://github.com/mieweb/opensource-server/tree/main/images/agent) diff --git a/mie-opensource-landing/docs/developers/pull-config.md b/mie-opensource-landing/docs/developers/pull-config.md deleted file mode 100644 index bc1dd43a..00000000 --- a/mie-opensource-landing/docs/developers/pull-config.md +++ /dev/null @@ -1,92 +0,0 @@ - -# pull-config - -{{ contributor_warning }} - -Cron-based configuration management that pulls config files from the manager API. Installed on agent and manager containers. For deployment instructions, see [Deploying Agents](../admins/deploying-agents.md). - -## Architecture - -Executable instance scripts in `/etc/pull-config.d/` are run every minute by cron via `run-parts`. Each script sources environment variables, sets service-specific config, and calls the main pull-config binary. - -``` -pull-config/ -├── bin/ -│ └── pull-config # Main script (called by instance scripts) -├── etc/ -│ ├── cron.d/ -│ │ └── pull-config # Single cron entry: run-parts /etc/pull-config.d -│ └── pull-config.d/ # Executable instance scripts -│ ├── nginx -│ ├── dnsmasq-conf -│ ├── dnsmasq-dhcp-hosts -│ ├── dnsmasq-hosts -│ ├── dnsmasq-dhcp-opts -│ └── dnsmasq-servers -└── Makefile # Builds the opensource-agent package (see Release Pipeline) -``` - -## Environment Variables - -Sourced from `/etc/environment` by each instance script. Set via container runtime (Docker `ENV`, Proxmox LXC config) — the base image's `environment.sh` service propagates them on boot. - -| Variable | Required | Description | -|----------|----------|-------------| -| `SITE_ID` | Yes | Numeric site ID from the manager | -| `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | -| `API_KEY` | No | Bearer token for remote agents. Not needed on the manager (localhost is trusted). | - -The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. - -## Instance Script Variables - -Each instance script exports these before calling `exec /opt/opensource-server/pull-config/bin/pull-config`: - -### Required (must be exported) - -| Variable | Description | -|----------|-------------| -| `CONF_FILE` | Target configuration file path | -| `CONF_URL` | URL to fetch configuration from | - -### Optional - -| Variable | Description | -|----------|-------------| -| `TEST_COMMAND` | Command to validate config before applying (e.g., `nginx -t`) | -| `RELOAD_COMMAND` | Custom command to reload the service | -| `SERVICE_NAME` | Service name for `systemctl reload-or-restart` fallback | - -## Adding an Instance - -Create an executable script in `/etc/pull-config.d/`: - -```bash -#!/usr/bin/env bash - -set -a; . /etc/environment; set +a - -export CONF_FILE=/etc/myservice/config.conf -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/myservice.conf -export TEST_COMMAND="myservice --validate-config" -export SERVICE_NAME="myservice" - -exec /opt/opensource-server/pull-config/bin/pull-config -``` - -```bash -sudo chmod +x /etc/pull-config.d/myservice -``` - -The cron job picks it up automatically on the next run. - -### File Naming - -`run-parts` only executes files matching: `a-zA-Z0-9_-` (no dots — use `nginx` not `nginx.sh`). - -## Behavior - -- **ETag caching** — only downloads when the manager reports a change (HTTP 304) -- **Validation** — runs `TEST_COMMAND` before applying; rolls back on failure -- **API key auth** — sends `Authorization: Bearer $API_KEY` when `API_KEY` is set -- **Reload** — uses `RELOAD_COMMAND` if set, otherwise `systemctl reload-or-restart $SERVICE_NAME` diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md index e7bb0115..df33f01b 100644 --- a/mie-opensource-landing/docs/developers/release-pipeline.md +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -12,11 +12,11 @@ are reused by local development, the container images, and CI. |---|---|---|---| | [`create-a-container/`](https://github.com/mieweb/opensource-server/tree/main/create-a-container) | `opensource-server` | amd64 | Manager web app, job runner, systemd units | | [`mie-opensource-landing/`](https://github.com/mieweb/opensource-server/tree/main/mie-opensource-landing) | `opensource-docs` | all | Prebuilt documentation site | -| [`pull-config/`](https://github.com/mieweb/opensource-server/tree/main/pull-config) | `opensource-agent` | all | pull-config engine, instances, error pages | +| [`agent/`](https://github.com/mieweb/opensource-server/tree/main/agent) | `opensource-agent` | all | Check-in agent, config templates, systemd timer, error pages | Everything installs under the `/opt/opensource-server` prefix, matching the -paths referenced by the systemd units, the pull-config instances, and the -manager-rendered nginx configuration. `opensource-server` depends on +paths referenced by the systemd units and the agent-rendered nginx +configuration. `opensource-server` depends on `opensource-agent` and `opensource-docs` because the manager's nginx config serves the agent's error pages and the docs site. @@ -51,7 +51,7 @@ is stripped if present (`v2026.6.3` and `2026.6.3` are equivalent). ```bash # Build and stage a component anywhere: -make -C pull-config install DESTDIR=/tmp/agent-root +make -C agent install DESTDIR=/tmp/agent-root # Build one package: make -C create-a-container deb # -> create-a-container/*.deb @@ -78,7 +78,7 @@ make -C mie-opensource-landing dev # docs live server Each component has a `.fpm` options file holding the static package metadata (name, architecture, dependencies, description, scripts, config files). The Makefile's `package` target stages the component into a `.pkg/buildroot` and runs [fpm](https://fpm.readthedocs.io/) with the dynamic options on the command line — output type, version (composed per format by `./package-version`), and the staging dir. fpm's `dir` input copies the staged tree verbatim from `-C .pkg/buildroot`, preserving symlinks (e.g. `node_modules/.bin/sequelize`) and the directory layout. The same definition produces deb, rpm, and apk, so `make rpm` and `make apk` also work. - `opensource-server` ships the `container-creator` and `job-runner` systemd units and enables them via an `after-install` script (`before-remove` disables them on real removal). The log directory is created on demand by the unit's `LogsDirectory`, not shipped in the package. The logrotate drop-in is the only config file. The `container-creator-init` unit (which provisions a *local* PostgreSQL) is **not** in the package — it is part of the manager image, since the package only suggests postgresql and works with a remote database too. -- `opensource-agent` ships the pull-config engine, instances, cron schedule and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. +- `opensource-agent` ships the compiled check-in agent, its config templates, the systemd service + 30s timer (enabled via an `after-install` script) and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. - `opensource-docs` ships content only. Config files are marked explicitly with `--config-files`; `--deb-no-default-config-files` stops fpm/dpkg from auto-marking everything under `/etc` as a conffile. diff --git a/mie-opensource-landing/zensical.toml b/mie-opensource-landing/zensical.toml index e83a4f30..9108783f 100644 --- a/mie-opensource-landing/zensical.toml +++ b/mie-opensource-landing/zensical.toml @@ -54,7 +54,7 @@ nav = [ { "Docker Images" = "developers/docker-images.md" }, { "Release Pipeline" = "developers/release-pipeline.md" }, { "Database Schema" = "developers/database-schema.md" }, - { "Pull Config" = "developers/pull-config.md" }, + { "Agent" = "developers/agent.md" }, { "Contributing" = "developers/contributing.md" }, ] }, ] }, diff --git a/pull-config/.gitignore b/pull-config/.gitignore deleted file mode 100644 index 1c09fed7..00000000 --- a/pull-config/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# packaging build artifacts -/.pkg/ -/*.deb -/*.rpm -/*.apk diff --git a/pull-config/README.md b/pull-config/README.md deleted file mode 100644 index 30c2ed60..00000000 --- a/pull-config/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# pull-config - -Cron-based configuration management that pulls config files from the manager API. Installed on agent and manager containers. - -- **Admin guide:** [Deploying Agents](https://mieweb.github.io/opensource-server/docs/admins/deploying-agents) — how to deploy and configure an agent container -- **Developer reference:** [pull-config](https://mieweb.github.io/opensource-server/docs/developers/pull-config) — architecture, instance script variables, adding new instances - - diff --git a/pull-config/bin/pull-config b/pull-config/bin/pull-config deleted file mode 100755 index 241d0482..00000000 --- a/pull-config/bin/pull-config +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Required variables (must be set by calling script) -if [[ -z "${CONF_FILE:-}" ]] || [[ -z "${CONF_URL:-}" ]]; then - echo "ERROR: CONF_FILE and CONF_URL must be set by the calling script" >&2 - exit 1 -fi - -# Set defaults for optional variables -TEST_COMMAND="${TEST_COMMAND:-}" -RELOAD_COMMAND="${RELOAD_COMMAND:-}" -SERVICE_NAME="${SERVICE_NAME:-}" - -# Derived file paths -ETAG_FILE="${CONF_FILE}.etag" -TEMP_FILE="${CONF_FILE}.tmp" -HEADERS_FILE="${CONF_FILE}.headers" -BACKUP_FILE="${CONF_FILE}.bak" - -# Cleanup function -cleanup() { - rm -f "${TEMP_FILE}" "${HEADERS_FILE}" "${BACKUP_FILE}" -} - -# Set trap to always cleanup on exit -trap cleanup EXIT - -# Function to download config and extract ETag -download_config() { - local url="$1" - - # Build curl command as array - local curl_cmd=( - curl - -w "%{http_code}" - -D "${HEADERS_FILE}" - -o "${TEMP_FILE}" - -sSL - ) - - # Add ETag header if it exists and conf file exists - if [[ -f "${CONF_FILE}" ]] && [[ -f "${ETAG_FILE}" ]]; then - local etag - etag=$(cat "${ETAG_FILE}") - curl_cmd+=(-H "If-None-Match: ${etag}") - fi - - # Add API key authentication if available - if [[ -n "${API_KEY:-}" ]]; then - curl_cmd+=(-H "Authorization: Bearer ${API_KEY}") - fi - - # Add URL - curl_cmd+=("${url}") - - # Execute curl and capture HTTP status code - local http_code - http_code=$("${curl_cmd[@]}" 2>/dev/null || echo "000") - - # Return the http_code - echo "${http_code}" -} - -# Download configuration -HTTP_CODE=$(download_config "${CONF_URL}") - -# Check if we got a 304 Not Modified -if [[ ${HTTP_CODE} -eq 304 ]]; then - # No changes, exit (cleanup handled by trap) - exit 0 -fi - -# Check if we got a successful response -if [[ ${HTTP_CODE} -ne 200 ]]; then - echo "Failed to download configuration (HTTP ${HTTP_CODE})" >&2 - exit 1 -fi - -# Extract new ETag from headers -NEW_ETAG="" -if [[ -f "${HEADERS_FILE}" ]]; then - NEW_ETAG=$(grep -i '^etag:' "${HEADERS_FILE}" | sed 's/^etag: *//i' | tr -d '\r\n' || echo "") -fi - -# Backup existing config if it exists -if [[ -f "${CONF_FILE}" ]]; then - mv "${CONF_FILE}" "${BACKUP_FILE}" -fi - -# Move new config into place -mv "${TEMP_FILE}" "${CONF_FILE}" - -# Test the new configuration if TEST_COMMAND is provided -if [[ -n "${TEST_COMMAND}" ]]; then - if ! eval "${TEST_COMMAND}" >/dev/null 2>&1; then - echo "Configuration test failed" >&2 - # Restore backup if it exists - if [[ -f "${BACKUP_FILE}" ]]; then - mv "${BACKUP_FILE}" "${CONF_FILE}" - else - # No backup, just remove the bad config - rm -f "${CONF_FILE}" - fi - exit 1 - fi -fi - -# Configuration is valid, save new ETag (cleanup handled by trap) -if [[ -n "${NEW_ETAG}" ]]; then - echo "${NEW_ETAG}" > "${ETAG_FILE}" -fi - -# Reload/restart the service -if [[ -n "${RELOAD_COMMAND}" ]]; then - # Use custom reload command - eval "${RELOAD_COMMAND}" >/dev/null 2>&1 -elif [[ -n "${SERVICE_NAME}" ]]; then - # Fall back to systemctl reload-or-restart - systemctl reload-or-restart "${SERVICE_NAME}" >/dev/null 2>&1 -fi diff --git a/pull-config/etc/cron.d/pull-config b/pull-config/etc/cron.d/pull-config deleted file mode 100644 index f4fb0396..00000000 --- a/pull-config/etc/cron.d/pull-config +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# Run all pull-config instances every minute -* * * * * root /bin/run-parts /etc/pull-config.d diff --git a/pull-config/etc/pull-config.d/dnsmasq-conf b/pull-config/etc/pull-config.d/dnsmasq-conf deleted file mode 100755 index 949c9668..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-conf +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq main configuration — requires full restart on change - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -export CONF_FILE=/etc/dnsmasq.conf -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/conf -export TEST_COMMAND="dnsmasq --test" -export RELOAD_COMMAND="systemctl restart dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts b/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts deleted file mode 100755 index b7b5f63f..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DHCP static leases — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/dhcp-hosts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/dhcp-hosts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts b/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts deleted file mode 100755 index 2d134cda..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DHCP options — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/dhcp-opts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/dhcp-opts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-hosts b/pull-config/etc/pull-config.d/dnsmasq-hosts deleted file mode 100755 index 0cd4f285..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-hosts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq host records (addn-hosts) — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/hosts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/hosts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-servers b/pull-config/etc/pull-config.d/dnsmasq-servers deleted file mode 100755 index e15af851..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-servers +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DNS forwarders — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/servers -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/servers -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/nginx b/pull-config/etc/pull-config.d/nginx deleted file mode 100755 index e68bfb1c..00000000 --- a/pull-config/etc/pull-config.d/nginx +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Nginx instance configuration for pull-config - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -# Configuration file to manage -export CONF_FILE=/etc/nginx/nginx.conf - -# URL to fetch configuration from -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/nginx - -# Command to test configuration validity before applying -export TEST_COMMAND="nginx -t" - -# Command to reload the service -export RELOAD_COMMAND="nginx -s reload" - -# Fallback service name for systemctl (if RELOAD_COMMAND fails) -export SERVICE_NAME="nginx" - -# Execute pull-config -exec /opt/opensource-server/pull-config/bin/pull-config -