From fece682892af011b0436c55a5ccde116d91b6b34 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Wed, 15 Jul 2026 07:44:31 -0700 Subject: [PATCH 1/6] Replace pull-config with TypeScript agent Resolves: https://github.com/mieweb/opensource-server/issues/357 Replaces the cron-driven bash pull-config system with a TypeScript Node.js agent that runs every 30 seconds via a systemd timer. Key changes: - New `agent/` package: TypeScript sources (config, state, api, system, apply), EJS templates moved from create-a-container/views/, systemd service+timer units - Manager: new `Agents` DB model/migration, `POST /api/v1/agents` check-in endpoint with ETag/304 support, `GET /api/v1/agents` admin list, `utils/agent-config.js` config snapshot builder - Client: AgentsListPage with service health badges, sidebar nav entry, query/types - Removes: pull-config bash scripts, EJS template rendering from manager (templates router, ejs dependency, views engine setup) - Updates docs, Dockerfiles, Makefiles, and OpenAPI spec --- Makefile | 2 +- README.md | 2 +- {pull-config => agent}/.fpm | 6 +- agent/.gitignore | 6 + {pull-config => agent}/Makefile | 35 ++-- agent/README.md | 6 + agent/contrib/postinstall.sh | 19 +++ .../contrib/systemd/opensource-agent.service | 12 ++ agent/contrib/systemd/opensource-agent.timer | 10 ++ agent/package-lock.json | 139 ++++++++++++++++ agent/package.json | 19 +++ agent/src/api.ts | 34 ++++ agent/src/apply.ts | 129 +++++++++++++++ agent/src/config.ts | 23 +++ agent/src/index.ts | 65 ++++++++ agent/src/state.ts | 29 ++++ agent/src/system.ts | 26 +++ agent/src/types.ts | 74 +++++++++ .../templates}/dnsmasq/conf.ejs | 0 .../templates}/dnsmasq/dhcp-hosts.ejs | 0 .../templates}/dnsmasq/dhcp-opts.ejs | 0 .../templates}/dnsmasq/hosts.ejs | 0 .../templates}/dnsmasq/servers.ejs | 0 .../templates/nginx.conf.ejs | 16 +- agent/tsconfig.json | 16 ++ create-a-container/Makefile | 2 +- create-a-container/client/src/app/Sidebar.tsx | 2 + create-a-container/client/src/app/router.tsx | 3 + create-a-container/client/src/lib/queries.ts | 5 + create-a-container/client/src/lib/types.ts | 18 +++ .../src/pages/agents/AgentServiceBadges.tsx | 32 ++++ .../src/pages/agents/AgentsListPage.tsx | 107 +++++++++++++ create-a-container/middlewares/index.js | 18 ++- .../20260714120000-create-agents.js | 51 ++++++ create-a-container/models/agent.js | 43 +++++ create-a-container/openapi.v1.yaml | 45 ++++++ create-a-container/package-lock.json | 70 +------- create-a-container/package.json | 1 - create-a-container/routers/api/v1/agents.js | 80 ++++++++++ create-a-container/routers/api/v1/index.js | 5 + create-a-container/routers/templates.js | 103 ------------ create-a-container/server.js | 7 +- create-a-container/utils/agent-config.js | 150 ++++++++++++++++++ images/agent/Dockerfile | 4 +- images/manager/Dockerfile | 5 +- .../docs/admins/deploying-agents.md | 46 +++--- .../docs/developers/agent.md | 99 ++++++++++++ .../docs/developers/database-schema.md | 13 ++ .../docs/developers/docker-images.md | 2 +- .../docs/developers/pull-config.md | 92 ----------- .../docs/developers/release-pipeline.md | 10 +- mie-opensource-landing/zensical.toml | 2 +- pull-config/.gitignore | 5 - pull-config/README.md | 8 - pull-config/bin/pull-config | 122 -------------- pull-config/etc/cron.d/pull-config | 5 - pull-config/etc/pull-config.d/dnsmasq-conf | 21 --- .../etc/pull-config.d/dnsmasq-dhcp-hosts | 23 --- .../etc/pull-config.d/dnsmasq-dhcp-opts | 23 --- pull-config/etc/pull-config.d/dnsmasq-hosts | 23 --- pull-config/etc/pull-config.d/dnsmasq-servers | 23 --- pull-config/etc/pull-config.d/nginx | 32 ---- 62 files changed, 1346 insertions(+), 622 deletions(-) rename {pull-config => agent}/.fpm (57%) create mode 100644 agent/.gitignore rename {pull-config => agent}/Makefile (56%) create mode 100644 agent/README.md create mode 100644 agent/contrib/postinstall.sh create mode 100644 agent/contrib/systemd/opensource-agent.service create mode 100644 agent/contrib/systemd/opensource-agent.timer create mode 100644 agent/package-lock.json create mode 100644 agent/package.json create mode 100644 agent/src/api.ts create mode 100644 agent/src/apply.ts create mode 100644 agent/src/config.ts create mode 100644 agent/src/index.ts create mode 100644 agent/src/state.ts create mode 100644 agent/src/system.ts create mode 100644 agent/src/types.ts rename {create-a-container/views => agent/templates}/dnsmasq/conf.ejs (100%) rename {create-a-container/views => agent/templates}/dnsmasq/dhcp-hosts.ejs (100%) rename {create-a-container/views => agent/templates}/dnsmasq/dhcp-opts.ejs (100%) rename {create-a-container/views => agent/templates}/dnsmasq/hosts.ejs (100%) rename {create-a-container/views => agent/templates}/dnsmasq/servers.ejs (100%) rename create-a-container/views/nginx-conf.ejs => agent/templates/nginx.conf.ejs (94%) create mode 100644 agent/tsconfig.json create mode 100644 create-a-container/client/src/pages/agents/AgentServiceBadges.tsx create mode 100644 create-a-container/client/src/pages/agents/AgentsListPage.tsx create mode 100644 create-a-container/migrations/20260714120000-create-agents.js create mode 100644 create-a-container/models/agent.js create mode 100644 create-a-container/routers/api/v1/agents.js delete mode 100644 create-a-container/routers/templates.js create mode 100644 create-a-container/utils/agent-config.js create mode 100644 mie-opensource-landing/docs/developers/agent.md delete mode 100644 mie-opensource-landing/docs/developers/pull-config.md delete mode 100644 pull-config/.gitignore delete mode 100644 pull-config/README.md delete mode 100755 pull-config/bin/pull-config delete mode 100644 pull-config/etc/cron.d/pull-config delete mode 100755 pull-config/etc/pull-config.d/dnsmasq-conf delete mode 100755 pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts delete mode 100755 pull-config/etc/pull-config.d/dnsmasq-dhcp-opts delete mode 100755 pull-config/etc/pull-config.d/dnsmasq-hosts delete mode 100755 pull-config/etc/pull-config.d/dnsmasq-servers delete mode 100755 pull-config/etc/pull-config.d/nginx diff --git a/Makefile b/Makefile index 4e53c609..f790bccd 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPONENTS := pull-config mie-opensource-landing create-a-container +COMPONENTS := agent mie-opensource-landing create-a-container PACKAGER ?= deb # Forwarded to every component Makefile. diff --git a/README.md b/README.md index de6b424c..52ea6bef 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Full documentation lives in [`mie-opensource-landing/docs/`](mie-opensource-land | Path | Purpose | |---|---| | [`create-a-container/`](create-a-container/) | Manager web application (Node.js + Express + Sequelize) | -| [`pull-config/`](pull-config/) | Cron-driven config distribution for nginx and dnsmasq on agents — see [pull-config docs](mie-opensource-landing/docs/developers/pull-config.md) | +| [`agent/`](agent/) | Site agent: systemd-timer check-in that reports status and applies nginx/dnsmasq config — see [agent docs](mie-opensource-landing/docs/developers/agent.md) | | [`images/`](images/) | Docker Bake definitions for the `base`, `nodejs`, `agent`, and `manager` images — see [Docker Images](mie-opensource-landing/docs/developers/docker-images.md) | | [`manager-control-program/`](manager-control-program/) | MCP server for AI-assisted container management — see [MCP Server](mie-opensource-landing/docs/users/mcp-server.md) | | [`mie-opensource-landing/`](mie-opensource-landing/) | Documentation site source | diff --git a/pull-config/.fpm b/agent/.fpm similarity index 57% rename from pull-config/.fpm rename to agent/.fpm index eb462614..50e677dc 100644 --- a/pull-config/.fpm +++ b/agent/.fpm @@ -5,13 +5,13 @@ --vendor "Medical Informatics Engineering" --url "https://github.com/mieweb/opensource-server" --category admin ---description "MIE opensource-server edge agent (pull-config): cron-driven distribution of nginx and dnsmasq configuration to edge nodes. Also ships the static error pages referenced by the manager-rendered nginx config." +--description "MIE opensource-server site agent: checks in with the manager every 30 seconds, reports host and service status, and renders and applies nginx and dnsmasq configuration. Also ships the static error pages referenced by the rendered nginx config." +--depends nodejs --depends nginx --depends libnginx-mod-stream --depends libnginx-mod-http-modsecurity --depends modsecurity-crs --depends ssl-cert --depends dnsmasq ---depends curl ---depends cron --deb-no-default-config-files +--after-install contrib/postinstall.sh diff --git a/agent/.gitignore b/agent/.gitignore new file mode 100644 index 00000000..0d22d8e5 --- /dev/null +++ b/agent/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +dist/ +.pkg/ +*.deb +*.rpm +*.apk diff --git a/pull-config/Makefile b/agent/Makefile similarity index 56% rename from pull-config/Makefile rename to agent/Makefile index e3cfd106..241e8938 100644 --- a/pull-config/Makefile +++ b/agent/Makefile @@ -2,45 +2,56 @@ PREFIX ?= /opt/opensource-server DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/agent +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system +# Staging directory for packaging STAGE := $(CURDIR)/.pkg/buildroot INSTALL := install INSTALL_DATA := $(INSTALL) -m 0644 -INSTALL_PROG := $(INSTALL) -m 0755 ERROR_PAGES := ../error-pages .PHONY: help deps build dev install deb rpm apk package clean help: - @echo "pull-config — builds the opensource-agent package." + @echo "agent — builds the opensource-agent package." @echo "" @echo "Targets:" - @echo " deps install dependencies (none; plain bash)" - @echo " build build the component (nothing to compile)" + @echo " deps install dependencies (npm ci)" + @echo " build compile the TypeScript sources" @echo " install stage files into DESTDIR (default /)" @echo " deb build the .deb package" @echo " rpm build the .rpm package" @echo " apk build the .apk package" - @echo " clean remove the staging dir and built packages" + @echo " clean remove build output, staging dir and built packages" @echo " help show this message" @echo "" @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." -# Plain bash; nothing to install, compile, or watch. deps: + npm ci + +# Compile, then drop devDependencies so only the runtime node_modules +# (ejs) is staged into the package. build: deps -dev: deps + npm run build + npm prune --omit=dev + +dev: build install: build - $(INSTALL) -D -m 0755 bin/pull-config $(DESTDIR)$(PREFIX)/pull-config/bin/pull-config - $(INSTALL) -d $(DESTDIR)/etc/pull-config.d - $(INSTALL_PROG) etc/pull-config.d/* $(DESTDIR)/etc/pull-config.d/ - $(INSTALL) -D -m 0644 etc/cron.d/pull-config $(DESTDIR)/etc/cron.d/pull-config + $(INSTALL) -d $(DESTBIN) + $(INSTALL_DATA) package.json $(DESTBIN)/ + cp -a dist templates node_modules $(DESTBIN)/ + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages $(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/ $(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache + $(INSTALL) -d -m 0755 $(DESTDIR)/var/lib/opensource-agent PACKAGER ?= deb package: @@ -57,5 +68,5 @@ apk: $(MAKE) package PACKAGER=apk clean: - rm -rf $(STAGE) + rm -rf $(STAGE) node_modules dist rm -f *.deb *.rpm *.apk diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 00000000..fd3cb243 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,6 @@ +# agent + +Site agent for opensource-server. A oneshot Node.js script, launched every 30 seconds by a systemd timer, that checks in with the manager (`POST /api/v1/agents`), reports host and service status, and renders, tests and applies nginx and dnsmasq configuration from the returned snapshot. + +- **Admin guide:** [Deploying Agents](https://mieweb.github.io/opensource-server/docs/admins/deploying-agents) — installation and configuration +- **Developer reference:** [agent](https://mieweb.github.io/opensource-server/docs/developers/agent) — check-in protocol, apply/rollback flow, managed services diff --git a/agent/contrib/postinstall.sh b/agent/contrib/postinstall.sh new file mode 100644 index 00000000..c243f72d --- /dev/null +++ b/agent/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +UNITS="opensource-agent.timer" + +# Nothing to do without systemctl (non-systemd container/chroot). +command -v systemctl >/dev/null 2>&1 || exit 0 + +# `systemctl enable` only creates static symlinks, so it works during an image +# build too +systemctl enable $UNITS + +# daemon-reload and restart need a running systemd; skip them at build time. +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + systemctl restart $UNITS +fi + +exit 0 diff --git a/agent/contrib/systemd/opensource-agent.service b/agent/contrib/systemd/opensource-agent.service new file mode 100644 index 00000000..5fa07ce4 --- /dev/null +++ b/agent/contrib/systemd/opensource-agent.service @@ -0,0 +1,12 @@ +[Unit] +Description=MIE opensource-server agent check-in +Wants=network-online.target +# environment.service (container images) writes the runtime env into +# /etc/environment; harmless ordering hint when the unit doesn't exist. +After=network-online.target environment.service + +[Service] +Type=oneshot +# Runtime config (SITE_ID, MANAGER_URL, API_KEY) lives in /etc/environment. +EnvironmentFile=-/etc/environment +ExecStart=/usr/bin/node /opt/opensource-server/agent/dist/index.js diff --git a/agent/contrib/systemd/opensource-agent.timer b/agent/contrib/systemd/opensource-agent.timer new file mode 100644 index 00000000..d89a7623 --- /dev/null +++ b/agent/contrib/systemd/opensource-agent.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run the opensource-server agent check-in every 30 seconds + +[Timer] +OnBootSec=30s +OnUnitActiveSec=30s +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/agent/package-lock.json b/agent/package-lock.json new file mode 100644 index 00000000..974435a7 --- /dev/null +++ b/agent/package-lock.json @@ -0,0 +1,139 @@ +{ + "name": "opensource-agent", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "opensource-agent", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "ejs": "^3.1.10" + }, + "devDependencies": { + "@types/ejs": "^3.1.5", + "@types/node": "^24.0.0", + "typescript": "^5.7.0" + } + }, + "node_modules/@types/ejs": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/@types/ejs/-/ejs-3.1.5.tgz", + "integrity": "sha512-nv+GSx77ZtXiJzwKdsASqi+YQ5Z7vwHsTP0JY2SiQgjGckkBRKZnk8nIM+7oUZ1VCtuTz0+By4qVR7fqzp/Dfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.13.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", + "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", + "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "license": "Apache-2.0", + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/filelist": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", + "integrity": "sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/jake": { + "version": "10.9.4", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", + "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.6", + "filelist": "^1.0.4", + "picocolors": "^1.1.1" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/agent/package.json b/agent/package.json new file mode 100644 index 00000000..371e34bd --- /dev/null +++ b/agent/package.json @@ -0,0 +1,19 @@ +{ + "name": "opensource-agent", + "version": "0.0.0", + "private": true, + "description": "MIE opensource-server site agent: checks in with the manager, reports host/service status, and renders and applies nginx and dnsmasq configuration.", + "license": "Apache-2.0", + "scripts": { + "build": "tsc", + "start": "node dist/index.js" + }, + "dependencies": { + "ejs": "^3.1.10" + }, + "devDependencies": { + "@types/ejs": "^3.1.5", + "@types/node": "^24.0.0", + "typescript": "^5.7.0" + } +} diff --git a/agent/src/api.ts b/agent/src/api.ts new file mode 100644 index 00000000..a15fc26f --- /dev/null +++ b/agent/src/api.ts @@ -0,0 +1,34 @@ +/** Check-in client for the manager's POST /api/v1/agents endpoint. */ + +import type { AgentConfig } from './config'; +import type { CheckinRequest, SiteConfig } from './types'; + +export type CheckinResult = + | { notModified: true } + | { notModified: false; etag?: string; config: SiteConfig }; + +export async function checkin( + cfg: AgentConfig, + body: CheckinRequest, + etag?: string, +): Promise { + const headers: Record = { 'Content-Type': 'application/json' }; + if (etag) headers['If-None-Match'] = etag; + if (cfg.apiKey) headers['Authorization'] = `Bearer ${cfg.apiKey}`; + + const res = await fetch(`${cfg.managerUrl}/api/v1/agents`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + + if (res.status === 304) return { notModified: true }; + if (!res.ok) throw new Error(`Check-in failed: HTTP ${res.status}`); + + const { data } = (await res.json()) as { data: SiteConfig }; + return { + notModified: false, + etag: res.headers.get('etag') ?? undefined, + config: data, + }; +} diff --git a/agent/src/apply.ts b/agent/src/apply.ts new file mode 100644 index 00000000..afe4f4d0 --- /dev/null +++ b/agent/src/apply.ts @@ -0,0 +1,129 @@ +/** + * Managed services: how to render, test, apply and reload each service's + * configuration. Files are written in place with backup/rollback — if the + * test command rejects the new config, the previous files are restored and + * the apply is reported as a failure at the next check-in. + */ + +import fs from 'fs'; +import path from 'path'; +import { execFileSync } from 'child_process'; +import ejs from 'ejs'; +import type { ApplyResult, SiteConfig } from './types'; + +const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); + +interface RenderedFile { + dest: string; + content: string; +} + +export interface ManagedService { + /** systemd unit name (also the key reported at check-in). */ + unit: string; + /** Render all managed files. Returns null when there is nothing to manage + * yet (e.g. dnsmasq before the site exists). */ + render(config: SiteConfig): Promise; + /** Command that validates the staged config before it is kept. */ + test?: string[]; + /** Reload/restart after a successful apply. */ + reload(changedFiles: string[]): void; +} + +function renderTemplate(template: string, data: object): Promise { + return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); +} + +function run(cmd: string[]): void { + execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe' }); +} + +export const services: ManagedService[] = [ + { + unit: 'nginx', + async render(config) { + return [{ + dest: '/etc/nginx/nginx.conf', + content: await renderTemplate('nginx.conf.ejs', config.nginx), + }]; + }, + test: ['nginx', '-t'], + reload() { + run(['systemctl', 'reload-or-restart', 'nginx']); + }, + }, + { + unit: 'dnsmasq', + async render(config) { + if (!config.site) return null; + const data = { site: config.site }; + return [ + { dest: '/etc/dnsmasq.conf', content: await renderTemplate('dnsmasq/conf.ejs', data) }, + { dest: '/var/lib/dnsmasq/dhcp-hosts', content: await renderTemplate('dnsmasq/dhcp-hosts.ejs', data) }, + { dest: '/var/lib/dnsmasq/hosts', content: await renderTemplate('dnsmasq/hosts.ejs', data) }, + { dest: '/var/lib/dnsmasq/dhcp-opts', content: await renderTemplate('dnsmasq/dhcp-opts.ejs', data) }, + { dest: '/var/lib/dnsmasq/servers', content: await renderTemplate('dnsmasq/servers.ejs', data) }, + ]; + }, + test: ['dnsmasq', '--test'], + reload(changedFiles) { + // The main config requires a full restart; the auxiliary files under + // /var/lib/dnsmasq are re-read on SIGHUP. + if (changedFiles.includes('/etc/dnsmasq.conf')) { + run(['systemctl', 'restart', 'dnsmasq']); + } else { + run(['systemctl', 'kill', '--signal=SIGHUP', 'dnsmasq']); + } + }, + }, +]; + +function readIfExists(file: string): string | null { + try { + return fs.readFileSync(file, 'utf8'); + } catch { + return null; + } +} + +export async function applyService(svc: ManagedService, config: SiteConfig): Promise { + const files = await svc.render(config); + if (!files) return 'success'; + + const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); + const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); + if (changed.length === 0) return 'success'; + + // Stage the new files (previous contents kept in memory for rollback). + for (const f of files) { + fs.mkdirSync(path.dirname(f.dest), { recursive: true }); + fs.writeFileSync(f.dest, f.content); + } + + const rollback = () => { + for (const [dest, prev] of current) { + if (prev === null) fs.rmSync(dest, { force: true }); + else fs.writeFileSync(dest, prev); + } + }; + + if (svc.test) { + try { + run(svc.test); + } catch (err) { + rollback(); + console.error(`${svc.unit}: config test failed, rolled back: ${(err as Error).message}`); + return 'failure'; + } + } + + try { + svc.reload(changed); + } catch (err) { + console.error(`${svc.unit}: reload failed: ${(err as Error).message}`); + return 'failure'; + } + + console.log(`${svc.unit}: applied ${changed.length} file(s)`); + return 'success'; +} diff --git a/agent/src/config.ts b/agent/src/config.ts new file mode 100644 index 00000000..e20ec53a --- /dev/null +++ b/agent/src/config.ts @@ -0,0 +1,23 @@ +/** Agent configuration, read from the environment (systemd passes + * /etc/environment through EnvironmentFile=). */ + +export interface AgentConfig { + siteId: number; + managerUrl: string; + apiKey?: string; + stateDir: string; +} + +export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { + const siteId = parseInt(env.SITE_ID ?? '', 10); + const managerUrl = env.MANAGER_URL; + if (!Number.isInteger(siteId) || !managerUrl) { + throw new Error('SITE_ID and MANAGER_URL must be set in the environment'); + } + return { + siteId, + managerUrl: managerUrl.replace(/\/+$/, ''), + apiKey: env.API_KEY || undefined, + stateDir: env.STATE_DIR || '/var/lib/opensource-agent', + }; +} diff --git a/agent/src/index.ts b/agent/src/index.ts new file mode 100644 index 00000000..68ce2c10 --- /dev/null +++ b/agent/src/index.ts @@ -0,0 +1,65 @@ +#!/usr/bin/env node +/** + * opensource-agent: oneshot check-in with the manager, launched every 30s by + * a systemd timer. + * + * Each pass POSTs system info + service states to /api/v1/agents with the + * last applied config ETag in If-None-Match. A 304 means nothing changed and + * the agent exits. A 200 carries the site's config snapshot: the agent + * renders, tests and applies it, then checks in again to report the apply + * results — repeating until it gets a 304. + */ + +import os from 'os'; +import { loadConfig, type AgentConfig } from './config'; +import { loadState, saveState, type AgentState } from './state'; +import { getPrimaryIpv4, getServiceState } from './system'; +import { checkin } from './api'; +import { services, applyService } from './apply'; +import type { CheckinRequest, ServiceStatus } from './types'; + +// Safety cap: a flapping server-side config can't keep a single run alive +// forever; the timer starts a fresh run 30s later anyway. +const MAX_PASSES = 5; + +function buildCheckinBody(cfg: AgentConfig, state: AgentState): CheckinRequest { + const serviceStatus: Record = {}; + for (const svc of services) { + serviceStatus[svc.unit] = { + state: getServiceState(svc.unit), + lastApply: state.lastApply[svc.unit] ?? 'unknown', + }; + } + return { + siteId: cfg.siteId, + hostname: os.hostname(), + currentTime: Math.floor(Date.now() / 1000), + ipv4Address: getPrimaryIpv4(), + services: serviceStatus, + }; +} + +async function main(): Promise { + const cfg = loadConfig(); + const state = loadState(cfg.stateDir); + + for (let pass = 0; pass < MAX_PASSES; pass++) { + const result = await checkin(cfg, buildCheckinBody(cfg, state), state.etag); + if (result.notModified) return; + + for (const svc of services) { + state.lastApply[svc.unit] = await applyService(svc, result.config); + } + + // The ETag is saved even after a failed apply: a rejected config won't + // fix itself without a server-side change (which changes the ETag), and + // the failure has been reported via lastApply. + state.etag = result.etag; + saveState(cfg.stateDir, state); + } +} + +main().catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/agent/src/state.ts b/agent/src/state.ts new file mode 100644 index 00000000..bd6a5ab0 --- /dev/null +++ b/agent/src/state.ts @@ -0,0 +1,29 @@ +/** Persistent agent state: last applied config ETag + per-service apply + * results, stored as JSON under the state dir. */ + +import fs from 'fs'; +import path from 'path'; +import type { ApplyResult } from './types'; + +export interface AgentState { + etag?: string; + lastApply: Record; +} + +function stateFile(stateDir: string): string { + return path.join(stateDir, 'state.json'); +} + +export function loadState(stateDir: string): AgentState { + try { + const raw = JSON.parse(fs.readFileSync(stateFile(stateDir), 'utf8')); + return { lastApply: {}, ...raw }; + } catch { + return { lastApply: {} }; + } +} + +export function saveState(stateDir: string, state: AgentState): void { + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(stateFile(stateDir), JSON.stringify(state, null, 2)); +} diff --git a/agent/src/system.ts b/agent/src/system.ts new file mode 100644 index 00000000..553667f6 --- /dev/null +++ b/agent/src/system.ts @@ -0,0 +1,26 @@ +/** Host system info: primary IPv4 address and systemd service states. */ + +import os from 'os'; +import { execFileSync } from 'child_process'; + +/** First non-internal IPv4 address, or null when none is configured. */ +export function getPrimaryIpv4(): string | null { + for (const ifaces of Object.values(os.networkInterfaces())) { + for (const iface of ifaces ?? []) { + if (iface.family === 'IPv4' && !iface.internal) return iface.address; + } + } + return null; +} + +/** systemd ActiveState for a unit (active, inactive, failed, ...). */ +export function getServiceState(unit: string): string { + try { + const out = execFileSync('systemctl', ['show', '--property=ActiveState', '--value', unit], { + encoding: 'utf8', + }); + return out.trim() || 'unknown'; + } catch { + return 'unknown'; + } +} diff --git a/agent/src/types.ts b/agent/src/types.ts new file mode 100644 index 00000000..6d80fd90 --- /dev/null +++ b/agent/src/types.ts @@ -0,0 +1,74 @@ +/** + * Shared types: check-in request/response shapes exchanged with the manager's + * POST /api/v1/agents endpoint. + */ + +export type ApplyResult = 'success' | 'failure'; + +export interface ServiceStatus { + /** systemd ActiveState: active, inactive, failed, ... */ + state: string; + /** Outcome of the last config apply for this service. */ + lastApply: ApplyResult | 'unknown'; +} + +export interface CheckinRequest { + siteId: number; + hostname: string; + /** Current agent time, epoch seconds UTC. */ + currentTime: number; + ipv4Address: string | null; + services: Record; +} + +export interface SiteContainer { + hostname: string; + ipv4Address: string; + macAddress: string | null; +} + +export interface SiteNode { + name: string; + ipv4Address: string | null; + containers: SiteContainer[]; +} + +export interface SiteInfo { + id: number; + name: string; + internalDomain: string; + dhcpRange: string; + subnetMask: string; + gateway: string; + dnsForwarders: string; + nodes: SiteNode[]; +} + +export interface HttpService { + internalPort: number; + container: { ipv4Address: string }; + externalHostname: string; + backendProtocol: string; + authRequired: boolean; + externalDomain: { name: string; authServer: string | null }; +} + +export interface StreamService { + internalPort: number; + container: { ipv4Address: string }; + externalPort: number; + protocol: string; +} + +export interface NginxConfig { + httpServices: HttpService[]; + streamServices: StreamService[]; + externalDomains: { name: string }[]; +} + +/** Config snapshot for the whole site. `site` is null before the first site + * exists (bootstrap); only the fallback nginx config is rendered then. */ +export interface SiteConfig { + site: SiteInfo | null; + nginx: NginxConfig; +} diff --git a/create-a-container/views/dnsmasq/conf.ejs b/agent/templates/dnsmasq/conf.ejs similarity index 100% rename from create-a-container/views/dnsmasq/conf.ejs rename to agent/templates/dnsmasq/conf.ejs diff --git a/create-a-container/views/dnsmasq/dhcp-hosts.ejs b/agent/templates/dnsmasq/dhcp-hosts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/dhcp-hosts.ejs rename to agent/templates/dnsmasq/dhcp-hosts.ejs diff --git a/create-a-container/views/dnsmasq/dhcp-opts.ejs b/agent/templates/dnsmasq/dhcp-opts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/dhcp-opts.ejs rename to agent/templates/dnsmasq/dhcp-opts.ejs diff --git a/create-a-container/views/dnsmasq/hosts.ejs b/agent/templates/dnsmasq/hosts.ejs similarity index 100% rename from create-a-container/views/dnsmasq/hosts.ejs rename to agent/templates/dnsmasq/hosts.ejs diff --git a/create-a-container/views/dnsmasq/servers.ejs b/agent/templates/dnsmasq/servers.ejs similarity index 100% rename from create-a-container/views/dnsmasq/servers.ejs rename to agent/templates/dnsmasq/servers.ejs diff --git a/create-a-container/views/nginx-conf.ejs b/agent/templates/nginx.conf.ejs similarity index 94% rename from create-a-container/views/nginx-conf.ejs rename to agent/templates/nginx.conf.ejs index db703bca..dad4b500 100644 --- a/create-a-container/views/nginx-conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -170,8 +170,8 @@ http { } <%_ httpServices.forEach((service, index) => { _%> - <%_ const authRequired = service.httpService.authRequired; _%> - <%_ const authServer = service.httpService.externalDomain.authServer; _%> + <%_ const authRequired = service.authRequired; _%> + <%_ const authServer = service.externalDomain.authServer; _%> server { listen 443 ssl; listen [::]:443 ssl; @@ -180,11 +180,11 @@ http { http2 on; http3 on; - server_name <%= service.httpService.externalHostname %>.<%= service.httpService.externalDomain.name %>; + server_name <%= service.externalHostname %>.<%= service.externalDomain.name %>; <%_ /* SSL certificates */ -%> - ssl_certificate /etc/ssl/certs/<%= service.httpService.externalDomain.name %>.crt; - ssl_certificate_key /etc/ssl/private/<%= service.httpService.externalDomain.name %>.key; + ssl_certificate /etc/ssl/certs/<%= service.externalDomain.name %>.crt; + ssl_certificate_key /etc/ssl/private/<%= service.externalDomain.name %>.key; error_page 403 @403; location @403 { @@ -281,7 +281,7 @@ http { proxy_set_header X-Email $auth_email; proxy_set_header X-Access-Token $auth_token; <%_ } _%> - proxy_pass <%= service.httpService.backendProtocol %>://<%= service.Container.ipv4Address %>:<%= service.internalPort %>; + proxy_pass <%= service.backendProtocol %>://<%= service.container.ipv4Address %>:<%= service.internalPort %>; proxy_http_version 1.1; <%_ /* Proxy headers */ -%> @@ -408,8 +408,8 @@ stream { <%_ streamServices.forEach((service, index) => { _%> server { - listen <%= service.transportService.externalPort %><%= service.transportService.protocol === 'udp' ? ' udp' : '' %>; - proxy_pass <%= service.Container.ipv4Address %>:<%= service.internalPort %>; + listen <%= service.externalPort %><%= service.protocol === 'udp' ? ' udp' : '' %>; + proxy_pass <%= service.container.ipv4Address %>:<%= service.internalPort %>; } <%_ }) _%> } diff --git a/agent/tsconfig.json b/agent/tsconfig.json new file mode 100644 index 00000000..a853fb00 --- /dev/null +++ b/agent/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "moduleResolution": "node", + "lib": ["ES2022"], + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "sourceMap": false, + "types": ["node"] + }, + "include": ["src"] +} diff --git a/create-a-container/Makefile b/create-a-container/Makefile index c7523dc9..f7cdd52e 100644 --- a/create-a-container/Makefile +++ b/create-a-container/Makefile @@ -15,7 +15,7 @@ INSTALL_DATA := $(INSTALL) -m 0644 # metadata are intentionally excluded; only the built client/dist ships. APP_FILES := server.js job-runner.js package.json package-lock.json openapi.v1.yaml APP_DIRS := bin config data middlewares migrations models node_modules \ - public routers seeders utils views + public routers seeders utils .PHONY: help deps build dev install deb rpm apk package clean diff --git a/create-a-container/client/src/app/Sidebar.tsx b/create-a-container/client/src/app/Sidebar.tsx index ae6fefdd..33fe8dce 100644 --- a/create-a-container/client/src/app/Sidebar.tsx +++ b/create-a-container/client/src/app/Sidebar.tsx @@ -19,6 +19,7 @@ import { Container as ContainerIcon, Globe, KeyRound, + Radio, Server, Settings, Users, @@ -57,6 +58,7 @@ const ADMIN: NavLink[] = [ icon: , adminOnly: true, }, + { to: '/agents', label: 'Agents', icon: , adminOnly: true }, { to: '/apikeys', label: 'API Keys', icon: }, { to: '/settings', label: 'Settings', icon: , adminOnly: true }, ]; diff --git a/create-a-container/client/src/app/router.tsx b/create-a-container/client/src/app/router.tsx index 00cd3b1e..50e2d7cc 100644 --- a/create-a-container/client/src/app/router.tsx +++ b/create-a-container/client/src/app/router.tsx @@ -16,6 +16,7 @@ import { NodeFormPage } from '@/pages/nodes/NodeFormPage'; import { NodeImportPage } from '@/pages/nodes/NodeImportPage'; import { ExternalDomainsListPage } from '@/pages/external-domains/ExternalDomainsListPage'; import { ExternalDomainFormPage } from '@/pages/external-domains/ExternalDomainFormPage'; +import { AgentsListPage } from '@/pages/agents/AgentsListPage'; import { JobDetailPage } from '@/pages/jobs/JobDetailPage'; import { UsersListPage } from '@/pages/users/UsersListPage'; import { UserFormPage } from '@/pages/users/UserFormPage'; @@ -65,6 +66,8 @@ export const router = createBrowserRouter([ { path: '/external-domains/new', element: }, { path: '/external-domains/:id/edit', element: }, + { path: '/agents', element: }, + { path: '/jobs/:id', element: }, { path: '/users', element: }, diff --git a/create-a-container/client/src/lib/queries.ts b/create-a-container/client/src/lib/queries.ts index 1177f3f5..29b75e56 100644 --- a/create-a-container/client/src/lib/queries.ts +++ b/create-a-container/client/src/lib/queries.ts @@ -4,6 +4,7 @@ */ import { api } from './api'; import type { + Agent, ApiKey, Container, ContainerMetadata, @@ -35,6 +36,7 @@ export const keys = { containerMetadata: (image: string) => ['container-metadata', image] as const, externalDomains: () => ['external-domains'] as const, externalDomain: (id: number | string) => ['external-domains', String(id)] as const, + agents: () => ['agents'] as const, users: () => ['users'] as const, user: (uid: number | string) => ['users', String(uid)] as const, groups: () => ['groups'] as const, @@ -60,6 +62,9 @@ export const queries = { getNode: (siteId: number | string, id: number | string) => api.get(`/api/v1/sites/${siteId}/nodes/${id}`), + // Agents + listAgents: () => api.get('/api/v1/agents'), + // Containers listContainers: ( siteId: number | string, diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 357da690..b217e295 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -30,6 +30,24 @@ export interface Node { hasSecret: boolean; } +export interface AgentServiceStatus { + /** systemd ActiveState: active, inactive, failed, ... */ + state: string; + /** Outcome of the agent's last config apply for this service. */ + lastApply: 'success' | 'failure' | 'unknown'; +} + +export interface Agent { + id: number; + siteId: number; + siteName: string | null; + hostname: string; + ipv4Address: string | null; + services: Record | null; + lastCheckinAt: string | null; + online: boolean; +} + export interface ExternalDomain { id: number; name: string; diff --git a/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx b/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx new file mode 100644 index 00000000..cb565266 --- /dev/null +++ b/create-a-container/client/src/pages/agents/AgentServiceBadges.tsx @@ -0,0 +1,32 @@ +import { Badge } from '@mieweb/ui'; +import type { AgentServiceStatus } from '@/lib/types'; + +function badgeVariant(status: AgentServiceStatus): 'success' | 'danger' | 'warning' { + if (status.state !== 'active') return 'danger'; + if (status.lastApply === 'failure') return 'warning'; + return 'success'; +} + +/** One badge per managed service, e.g. "nginx: active" — red when the unit + * isn't active, amber when the last config apply failed. */ +export function AgentServiceBadges({ + services, +}: { + services: Record | null; +}) { + const entries = Object.entries(services ?? {}); + if (entries.length === 0) return —; + return ( +
+ {entries.map(([name, status]) => ( + + {name}: {status.state} + + ))} +
+ ); +} diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx new file mode 100644 index 00000000..c973ec2f --- /dev/null +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -0,0 +1,107 @@ +import { useQuery } from '@tanstack/react-query'; +import { + Alert, + AlertDescription, + Badge, + PageHeader, + Spinner, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@mieweb/ui'; +import { Radio } from 'lucide-react'; +import { ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; +import type { Agent } from '@/lib/types'; +import { useDocumentTitle } from '@/lib/useDocumentTitle'; +import { AgentServiceBadges } from './AgentServiceBadges'; + +function formatLastCheckin(lastCheckinAt: string | null): string { + if (!lastCheckinAt) return 'never'; + const seconds = Math.max(0, Math.round((Date.now() - new Date(lastCheckinAt).getTime()) / 1000)); + if (seconds < 60) return `${seconds}s ago`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; + if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; + return new Date(lastCheckinAt).toLocaleString(); +} + +function OnlineBadge({ agent }: { agent: Agent }) { + return agent.online ? ( + Online + ) : ( + Offline + ); +} + +export function AgentsListPage() { + useDocumentTitle('Agents'); + const { data, isLoading, error } = useQuery({ + queryKey: keys.agents(), + queryFn: queries.listAgents, + // Agents check in every 30s; keep the health view current. + refetchInterval: 30000, + }); + + const hasAgents = !!data && data.length > 0; + + return ( +
+ } + /> + {error && ( + + {(error as ApiError).message} + + )} + {isLoading && ( +
+ +
+ )} + {data && data.length === 0 && ( + + + No agents have checked in yet. Agents register themselves at their first check-in. + + + )} + + {hasAgents && ( + + + + Hostname + Site + IPv4 + Status + Services + Last check-in + + + + {data.map((agent: Agent) => ( + + {agent.hostname} + {agent.siteName || agent.siteId} + {agent.ipv4Address || '—'} + + + + + + + {formatLastCheckin(agent.lastCheckinAt)} + + ))} + +
+ )} +
+ ); +} diff --git a/create-a-container/middlewares/index.js b/create-a-container/middlewares/index.js index 05ebb897..b1023ac2 100644 --- a/create-a-container/middlewares/index.js +++ b/create-a-container/middlewares/index.js @@ -73,10 +73,9 @@ function requireAdmin(req, res, next) { return res.status(403).send('Forbidden: Admin access required'); } -// Localhost-or-admin middleware -// Allows localhost requests through without auth. Remote requests must authenticate -// as an admin user (via session or API key). -function requireLocalhostOrAdmin(req, res, next) { +// True when the request comes directly from localhost (and was not proxied +// on behalf of a remote client, per X-Real-IP). +function isLocalhostRequest(req) { const isLocalhost = (ip) => { return ip === '127.0.0.1' || ip === '::1' || @@ -90,8 +89,14 @@ function requireLocalhostOrAdmin(req, res, next) { const realIp = req.get('X-Real-IP'); - // If direct connection is from localhost and no non-localhost X-Real-IP, allow through - if (isLocalhost(directIp) && (!realIp || isLocalhost(realIp))) { + return isLocalhost(directIp) && (!realIp || isLocalhost(realIp)); +} + +// Localhost-or-admin middleware +// Allows localhost requests through without auth. Remote requests must authenticate +// as an admin user (via session or API key). +function requireLocalhostOrAdmin(req, res, next) { + if (isLocalhostRequest(req)) { return next(); } @@ -106,6 +111,7 @@ const { setCurrentSite, loadSites } = require('./currentSite'); module.exports = { isApiRequest, + isLocalhostRequest, requireAuth, requireAdmin, requireLocalhostOrAdmin, diff --git a/create-a-container/migrations/20260714120000-create-agents.js b/create-a-container/migrations/20260714120000-create-agents.js new file mode 100644 index 00000000..3f4db170 --- /dev/null +++ b/create-a-container/migrations/20260714120000-create-agents.js @@ -0,0 +1,51 @@ +'use strict'; +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('Agents', { + id: { + allowNull: false, + autoIncrement: true, + primaryKey: true, + type: Sequelize.INTEGER + }, + siteId: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: 'Sites', key: 'id' }, + onDelete: 'CASCADE' + }, + hostname: { + type: Sequelize.STRING, + allowNull: false + }, + ipv4Address: { + type: Sequelize.STRING, + allowNull: true + }, + services: { + type: Sequelize.JSON, + allowNull: true + }, + lastCheckinAt: { + type: Sequelize.DATE, + allowNull: true + }, + createdAt: { + allowNull: false, + type: Sequelize.DATE + }, + updatedAt: { + allowNull: false, + type: Sequelize.DATE + } + }); + await queryInterface.addIndex('Agents', ['siteId', 'hostname'], { + unique: true, + name: 'agents_site_id_hostname_unique' + }); + }, + async down(queryInterface) { + await queryInterface.dropTable('Agents'); + } +}; diff --git a/create-a-container/models/agent.js b/create-a-container/models/agent.js new file mode 100644 index 00000000..16d36740 --- /dev/null +++ b/create-a-container/models/agent.js @@ -0,0 +1,43 @@ +'use strict'; +const { Model } = require('sequelize'); +module.exports = (sequelize, DataTypes) => { + class Agent extends Model { + static associate(models) { + Agent.belongsTo(models.Site, { + foreignKey: 'siteId', + as: 'site' + }); + } + } + Agent.init({ + siteId: { + type: DataTypes.INTEGER, + allowNull: false + }, + hostname: { + type: DataTypes.STRING, + allowNull: false + }, + ipv4Address: { + type: DataTypes.STRING, + allowNull: true + }, + services: { + // Per-service status as reported by the agent at check-in: + // { nginx: { state: 'active', lastApply: 'success' }, ... } + type: DataTypes.JSON, + allowNull: true + }, + lastCheckinAt: { + type: DataTypes.DATE, + allowNull: true + } + }, { + sequelize, + modelName: 'Agent', + indexes: [ + { unique: true, fields: ['siteId', 'hostname'] } + ] + }); + return Agent; +}; diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index ea9ec94f..3220102e 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -32,6 +32,7 @@ tags: - name: Sites - name: Containers - name: Nodes + - name: Agents - name: Jobs - name: External Domains - name: Users @@ -481,6 +482,50 @@ paths: description: SSE stream — events `log` and `status` content: { text/event-stream: {} } + /agents: + post: + tags: [Agents] + summary: Agent check-in + description: | + Site agents POST their system info and per-service status every 30 + seconds. The response carries the site's config snapshot with a strong + `ETag`; send it back via `If-None-Match` to receive `304 Not Modified` + when nothing changed. Allowed from localhost without credentials + (manager bootstrap) or with an admin API key. + parameters: + - in: header + name: If-None-Match + required: false + schema: { type: string } + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [siteId, hostname] + properties: + siteId: { type: integer } + hostname: { type: string } + currentTime: { type: integer, description: Agent time, epoch seconds UTC } + ipv4Address: { type: string, nullable: true } + services: + type: object + additionalProperties: + type: object + properties: + state: { type: string, description: 'systemd ActiveState: active, inactive, failed, ...' } + lastApply: { type: string, enum: [success, failure, unknown] } + responses: + '200': + description: Config snapshot (`{ data: { site, nginx } }`) with `ETag` header + '304': { description: Config unchanged since If-None-Match } + '422': { description: Missing siteId/hostname } + get: + tags: [Agents] + summary: Current status of all agents (admin) + responses: { '200': { description: 'List of agents with lastCheckinAt, services and online flag' } } + /external-domains: get: { tags: [External Domains], responses: { '200': { description: List } } } post: { tags: [External Domains], responses: { '201': { description: Created (admin) } } } diff --git a/create-a-container/package-lock.json b/create-a-container/package-lock.json index ca9a1c69..b9649ccc 100644 --- a/create-a-container/package-lock.json +++ b/create-a-container/package-lock.json @@ -11,7 +11,6 @@ "cookie-parser": "^1.4.7", "csrf-sync": "^4.2.1", "dotenv": "^17.2.3", - "ejs": "^3.1.10", "express": "^5.2.1", "express-rate-limit": "^8.5.1", "express-session": "^1.19.0", @@ -236,12 +235,6 @@ "sprintf-js": "~1.0.2" } }, - "node_modules/async": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", - "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", - "license": "MIT" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1079,21 +1072,6 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, - "node_modules/ejs": { - "version": "3.1.10", - "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", - "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", - "license": "Apache-2.0", - "dependencies": { - "jake": "^10.8.5" - }, - "bin": { - "ejs": "bin/cli.js" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -1281,36 +1259,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/filelist": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", - "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", - "license": "Apache-2.0", - "dependencies": { - "minimatch": "^5.0.1" - } - }, - "node_modules/filelist/node_modules/brace-expansion": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", - "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/filelist/node_modules/minimatch": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", - "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -1870,23 +1818,6 @@ "@pkgjs/parseargs": "^0.11.0" } }, - "node_modules/jake": { - "version": "10.9.4", - "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", - "integrity": "sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==", - "license": "Apache-2.0", - "dependencies": { - "async": "^3.2.6", - "filelist": "^1.0.4", - "picocolors": "^1.1.1" - }, - "bin": { - "jake": "bin/cli.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/jose": { "version": "4.15.9", "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", @@ -2567,6 +2498,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { diff --git a/create-a-container/package.json b/create-a-container/package.json index d69379e9..265fe393 100644 --- a/create-a-container/package.json +++ b/create-a-container/package.json @@ -24,7 +24,6 @@ "cookie-parser": "^1.4.7", "csrf-sync": "^4.2.1", "dotenv": "^17.2.3", - "ejs": "^3.1.10", "express": "^5.2.1", "express-rate-limit": "^8.5.1", "express-session": "^1.19.0", diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js new file mode 100644 index 00000000..030987ae --- /dev/null +++ b/create-a-container/routers/api/v1/agents.js @@ -0,0 +1,80 @@ +/** + * /api/v1/agents — site agent check-in and status. + * + * POST / agent check-in: records system info + service states, responds with + * the site's config snapshot (or 304 when the agent's If-None-Match + * ETag still matches). + * GET / admin: current status of all agents. + */ + +const express = require('express'); +const { Agent, Site } = require('../../../models'); +const { isLocalhostRequest } = require('../../../middlewares'); +const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); +const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); + +const router = express.Router(); + +// An agent is online if it checked in within three 30-second timer intervals. +const ONLINE_WINDOW_MS = 90 * 1000; + +// The manager's own agent checks in over localhost without credentials +// (bootstrap: no site, no API key exist yet). Remote agents authenticate +// with an admin API key. +function agentAuth(req, res, next) { + if (isLocalhostRequest(req)) return next(); + return apiAuth(req, res, (err) => { + if (err) return next(err); + return apiAdmin(req, res, next); + }); +} + +router.post('/', agentAuth, asyncHandler(async (req, res) => { + const { siteId, hostname, ipv4Address, services } = req.body || {}; + const parsedSiteId = parseInt(siteId, 10); + if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { + return fail(res, 422, 'validation_failed', 'siteId and hostname are required'); + } + + // Record the check-in. Skipped during bootstrap (the site row doesn't exist + // yet) since the foreign key has nothing to point at. + const site = await Site.findByPk(parsedSiteId); + if (site) { + const [agent] = await Agent.findOrCreate({ + where: { siteId: parsedSiteId, hostname }, + }); + await agent.update({ + ipv4Address: ipv4Address || null, + services: services || null, + lastCheckinAt: new Date(), + }); + } + + const config = await buildAgentConfig(parsedSiteId); + const etag = computeConfigEtag(config); + res.set('ETag', etag); + if (req.get('If-None-Match') === etag) { + return res.status(304).end(); + } + return ok(res, config); +})); + +router.get('/', apiAuth, apiAdmin, asyncHandler(async (req, res) => { + const agents = await Agent.findAll({ + include: [{ model: Site, as: 'site', attributes: ['id', 'name'] }], + order: [['siteId', 'ASC'], ['hostname', 'ASC']], + }); + const now = Date.now(); + return ok(res, agents.map((a) => ({ + id: a.id, + siteId: a.siteId, + siteName: a.site?.name || null, + hostname: a.hostname, + ipv4Address: a.ipv4Address, + services: a.services, + lastCheckinAt: a.lastCheckinAt, + online: !!a.lastCheckinAt && now - new Date(a.lastCheckinAt).getTime() <= ONLINE_WINDOW_MS, + }))); +})); + +module.exports = router; diff --git a/create-a-container/routers/api/v1/index.js b/create-a-container/routers/api/v1/index.js index f2a42623..9610555a 100644 --- a/create-a-container/routers/api/v1/index.js +++ b/create-a-container/routers/api/v1/index.js @@ -51,6 +51,11 @@ router.get('/openapi.yaml', (_req, res) => { res.type('text/yaml').sendFile(path.join(__dirname, '..', '..', '..', 'openapi.v1.yaml')); }); +// Agent check-in — mounted before the CSRF guard because the manager's own +// agent posts from localhost with neither a session cookie nor a Bearer token +// (auth is handled inside the router: localhost or admin API key). +router.use('/agents', require('./agents')); + // CSRF guard before any state-changing route below router.use(csrfGuard); diff --git a/create-a-container/routers/templates.js b/create-a-container/routers/templates.js deleted file mode 100644 index 938ce756..00000000 --- a/create-a-container/routers/templates.js +++ /dev/null @@ -1,103 +0,0 @@ -const express = require('express'); -const { Op } = require('sequelize'); -const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain } = require('../models'); -const { requireLocalhostOrAdmin } = require('../middlewares'); - -const router = express.Router(); - -async function loadDnsmasqSite(siteId) { - return Site.findByPk(siteId, { - include: [{ - model: Node, - as: 'nodes', - include: [{ - model: Container, - as: 'containers', - where: { ipv4Address: { [Op.ne]: null } }, - required: false, - attributes: ['macAddress', 'ipv4Address', 'hostname'], - }], - }], - }); -} - -const DNSMASQ_TEMPLATES = ['conf', 'dhcp-hosts', 'hosts', 'dhcp-opts', 'servers']; - -router.get('/sites/:siteId/dnsmasq/:file', requireLocalhostOrAdmin, async (req, res) => { - const { file } = req.params; - if (!DNSMASQ_TEMPLATES.includes(file)) return res.status(404).send('Not found'); - const site = await loadDnsmasqSite(parseInt(req.params.siteId, 10)); - if (!site) return res.status(404).send('Site not found'); - res.set('Content-Type', 'text/plain'); - return res.render(`dnsmasq/${file}`, { site }); -}); - -router.get('/sites/:siteId/nginx', requireLocalhostOrAdmin, async (req, res) => { - const siteId = parseInt(req.params.siteId, 10); - const site = await Site.findByPk(siteId, { - include: [{ - model: Node, - as: 'nodes', - include: [{ - model: Container, - as: 'containers', - where: { ipv4Address: { [Op.ne]: null } }, - required: false, - include: [{ - model: Service, - as: 'services', - include: [ - { - model: HTTPService, - as: 'httpService', - include: [{ model: ExternalDomain, as: 'externalDomain' }], - }, - { model: TransportService, as: 'transportService' }, - ], - }], - }], - }, { - model: ExternalDomain, - as: 'externalDomains', - }], - }); - - // Bootstrap fallback: if the site does not exist yet, still render an - // empty nginx config so the manager API remains reachable (over TLS) to - // create the first site. Without this, bootstrapping would require - // plaintext HTTP access, which breaks our security requirements for - // registering nodes and creating the first site. - if (!site) { - res.set('Content-Type', 'text/plain'); - return res.render('nginx-conf', { - httpServices: [], - streamServices: [], - externalDomains: [], - }); - } - - const allServices = []; - site?.nodes?.forEach((node) => { - node?.containers?.forEach((container) => { - container?.services?.forEach((service) => { - service.Container = container; - allServices.push(service); - }); - }); - }); - - const httpServices = allServices.filter((s) => s.type === 'http'); - const streamServices = allServices.filter((s) => s.type === 'transport'); - - const usedDomainIds = new Set(); - httpServices.forEach((s) => { - if (s.httpService?.externalDomain?.id) usedDomainIds.add(s.httpService.externalDomain.id); - }); - (site?.externalDomains || []).forEach((d) => usedDomainIds.add(d.id)); - const externalDomains = await ExternalDomain.findAll({ where: { id: [...usedDomainIds] } }); - - res.set('Content-Type', 'text/plain'); - return res.render('nginx-conf', { httpServices, streamServices, externalDomains }); -}); - -module.exports = router; diff --git a/create-a-container/server.js b/create-a-container/server.js index a349ad74..f7c7c121 100644 --- a/create-a-container/server.js +++ b/create-a-container/server.js @@ -38,9 +38,6 @@ async function main() { const app = express(); - // setup views (still used by templates router for nginx-conf / dnsmasq files) - app.set('views', path.join(__dirname, 'views')); - app.set('view engine', 'ejs'); app.set('trust proxy', 1); // Parse query strings with qs so bracket notation (e.g. `user[0]=alice`) // yields real arrays. Express 5 defaults to the 'simple' parser. @@ -98,10 +95,8 @@ async function main() { // --- Mount Routers --- const apiV1Router = require('./routers/api/v1'); - const templatesRouter = require('./routers/templates'); app.use('/api/v1', apiV1Router); - app.use('/', templatesRouter); // serves /sites/:siteId/nginx and /sites/:siteId/dnsmasq/:file // --- API Documentation (Swagger UI) --- // Swagger UI at /api documents the versioned v1 API (the spec also served at /api/v1/openapi.*). @@ -117,7 +112,7 @@ async function main() { // --- SPA: serve compiled React app for everything else --- const clientDist = path.join(__dirname, 'client', 'dist'); app.use(express.static(clientDist)); - app.get(/^\/(?!api(\/|$)|sites\/[^/]+\/(nginx$|dnsmasq\/)).*$/, (req, res) => { + app.get(/^\/(?!api(\/|$)).*$/, (req, res) => { res.sendFile(path.join(clientDist, 'index.html')); }); diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js new file mode 100644 index 00000000..781d16f5 --- /dev/null +++ b/create-a-container/utils/agent-config.js @@ -0,0 +1,150 @@ +/** + * Builds the site configuration snapshot returned to agents at check-in. + * + * The snapshot is plain JSON with a deterministic shape so a strong ETag can + * be computed over it: agents send the ETag back via If-None-Match and get a + * 304 when nothing changed. The shape mirrors what the agent's EJS templates + * (agent/templates/) expect. + */ + +const crypto = require('crypto'); +const { Op } = require('sequelize'); +const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain } = require('../models'); + +/** + * Load a site with everything the agent templates need, serialized to plain + * JSON. Returns `{ site: null, nginx: { ... empty ... } }` when the site does + * not exist yet — a bootstrap fallback so a fresh manager's own agent can + * render a minimal nginx config (manager reachable over TLS) before the first + * site is created. + * + * @param {number} siteId + * @returns {Promise} `{ site, nginx: { httpServices, streamServices, externalDomains } }` + */ +async function buildAgentConfig(siteId) { + const site = await Site.findByPk(siteId, { + include: [{ + model: Node, + as: 'nodes', + include: [{ + model: Container, + as: 'containers', + where: { ipv4Address: { [Op.ne]: null } }, + required: false, + include: [{ + model: Service, + as: 'services', + include: [ + { + model: HTTPService, + as: 'httpService', + include: [{ model: ExternalDomain, as: 'externalDomain' }], + }, + { model: TransportService, as: 'transportService' }, + ], + }], + }], + }, { + model: ExternalDomain, + as: 'externalDomains', + }], + order: [ + [{ model: Node, as: 'nodes' }, 'id', 'ASC'], + [{ model: Node, as: 'nodes' }, { model: Container, as: 'containers' }, 'id', 'ASC'], + [{ model: Node, as: 'nodes' }, { model: Container, as: 'containers' }, { model: Service, as: 'services' }, 'id', 'ASC'], + [{ model: ExternalDomain, as: 'externalDomains' }, 'id', 'ASC'], + ], + }); + + if (!site) { + return { + site: null, + nginx: { httpServices: [], streamServices: [], externalDomains: [] }, + }; + } + + const httpServices = []; + const streamServices = []; + for (const node of site.nodes || []) { + for (const container of node.containers || []) { + for (const service of container.services || []) { + const base = { + internalPort: service.internalPort, + container: { ipv4Address: container.ipv4Address }, + }; + if (service.type === 'http' && service.httpService && service.httpService.externalDomain) { + httpServices.push({ + ...base, + externalHostname: service.httpService.externalHostname, + backendProtocol: service.httpService.backendProtocol, + authRequired: !!service.httpService.authRequired, + externalDomain: { + name: service.httpService.externalDomain.name, + authServer: service.httpService.externalDomain.authServer || null, + }, + }); + } else if (service.type === 'transport' && service.transportService) { + streamServices.push({ + ...base, + externalPort: service.transportService.externalPort, + protocol: service.transportService.protocol, + }); + } + } + } + } + + // Domains needing a TLS server block: any domain referenced by an HTTP + // service plus the site's own external domains. + const usedDomainIds = new Set(); + for (const node of site.nodes || []) { + for (const container of node.containers || []) { + for (const service of container.services || []) { + const id = service.httpService?.externalDomain?.id; + if (id) usedDomainIds.add(id); + } + } + } + for (const d of site.externalDomains || []) usedDomainIds.add(d.id); + const externalDomains = await ExternalDomain.findAll({ + where: { id: [...usedDomainIds] }, + order: [['id', 'ASC']], + }); + + return { + site: { + id: site.id, + name: site.name, + internalDomain: site.internalDomain, + dhcpRange: site.dhcpRange, + subnetMask: site.subnetMask, + gateway: site.gateway, + dnsForwarders: site.dnsForwarders, + nodes: (site.nodes || []).map((node) => ({ + name: node.name, + ipv4Address: node.ipv4Address, + containers: (node.containers || []).map((c) => ({ + hostname: c.hostname, + ipv4Address: c.ipv4Address, + macAddress: c.macAddress, + })), + })), + }, + nginx: { + httpServices, + streamServices, + externalDomains: externalDomains.map((d) => ({ name: d.name })), + }, + }; +} + +/** + * Strong ETag over a config snapshot. Deterministic because buildAgentConfig + * constructs the object with stable key/array ordering. + */ +function computeConfigEtag(config) { + const hash = crypto.createHash('sha256').update(JSON.stringify(config)).digest('hex'); + return `"${hash}"`; +} + +module.exports = { buildAgentConfig, computeConfigEtag }; diff --git a/images/agent/Dockerfile b/images/agent/Dockerfile index c46446d9..c68d219a 100644 --- a/images/agent/Dockerfile +++ b/images/agent/Dockerfile @@ -6,7 +6,7 @@ FROM nodejs COPY images/opensource-server.sources /etc/apt/sources.list.d/opensource-server.sources # Install the agent package (and its dependencies: nginx with stream + -# ModSecurity modules, dnsmasq, cron, ...) built by the builder image. +# ModSecurity modules, dnsmasq, ...) built by the builder image. RUN --mount=from=builder,source=/dist,target=/dist \ apt-get update && \ apt-get install -y /dist/opensource-agent_*.deb && \ @@ -29,7 +29,7 @@ COPY ./images/agent/nginx.logrotate /etc/logrotate.d/nginx COPY ./images/agent/crs-setup.conf /etc/modsecurity/crs/crs-setup.conf COPY ./images/agent/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf /etc/modsecurity/crs/RESPONSE-999-EXCLUSION-RULES-AFTER-CRS.conf -# Configure DNSMasq to only get its config from our pull-config +# Configure DNSMasq to only get its config from our agent RUN sed -i \ -e 's/^CONFIG_DIR=\(.*\)$/#CONFIG_DIR=\1/' \ -e 's/^#IGNORE_RESOLVCONF=\(.*\)$/IGNORE_RESOLVCONF=\1/' \ diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index 4eb624eb..b76bdfc4 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -1,8 +1,9 @@ # syntax=docker/dockerfile:1 FROM agent -# Default environment for pull-config. The manager runs the app on localhost -# so it doesn't need an API key. Remote agents must set these explicitly. +# Default environment for the site agent. The manager runs the app on +# localhost so it doesn't need an API key. Remote agents must set these +# explicitly. ENV SITE_ID=1 ENV MANAGER_URL=http://localhost:3000 diff --git a/mie-opensource-landing/docs/admins/deploying-agents.md b/mie-opensource-landing/docs/admins/deploying-agents.md index 89f45d21..dbc599a6 100644 --- a/mie-opensource-landing/docs/admins/deploying-agents.md +++ b/mie-opensource-landing/docs/admins/deploying-agents.md @@ -16,7 +16,7 @@ Agents are deployed **manually in Proxmox** (not through the manager UI) and sho ```mermaid graph LR - A[Configure site
in manager] --> B[Generate
admin API key] --> C[Deploy agent
on Proxmox node] --> D[Verify agent
pulls config] --> E[Import node
in manager] + A[Configure site
in manager] --> B[Generate
admin API key] --> C[Deploy agent
on Proxmox node] --> D[Verify agent
checks in] --> E[Import node
in manager] classDef step fill:#f9f9f9,stroke:#333,stroke-width:1px class A,B,C,D,E step @@ -69,7 +69,7 @@ lxc.environment = API_KEY= |----------|-------------| | `SITE_ID` | Numeric site ID from the manager (visible in the URL when viewing the site) | | `MANAGER_URL` | Base URL of the manager container (e.g., `http://192.168.1.10:3000`) | -| `API_KEY` | API key from an admin account. Used to authenticate config pulls. | +| `API_KEY` | API key from an admin account. Used to authenticate check-ins. | ## 4. Start and Verify @@ -77,25 +77,26 @@ lxc.environment = API_KEY= pct start ``` -Verify pull-config is working by checking that nginx and dnsmasq configs were fetched: +Verify the agent is checking in and applied its configs: ```bash # Enter the container pct enter -# Check that configs were pulled +# Timer and last runs +systemctl status opensource-agent.timer +journalctl -u opensource-agent.service + +# Check that configs were applied cat /etc/nginx/nginx.conf cat /etc/dnsmasq.conf -# Test manually -/etc/pull-config.d/nginx -/etc/pull-config.d/dnsmasq-conf -/etc/pull-config.d/dnsmasq-dhcp-hosts -/etc/pull-config.d/dnsmasq-hosts -/etc/pull-config.d/dnsmasq-dhcp-opts -/etc/pull-config.d/dnsmasq-servers +# Run a check-in manually +node /opt/opensource-server/agent/dist/index.js ``` +The agent also appears on the manager's **Agents** page (`/agents`, admin only) with its last check-in time and service health. + ## 5. Forward Network Traffic Forward the following ports from the Proxmox host to the agent container: @@ -113,25 +114,24 @@ With the agent running, proceed to [import the node](core-concepts/nodes.md#impo ## How It Works -The agent uses **pull-config**, a cron-based system that fetches configuration files from the manager every minute. +A systemd timer launches the agent every 30 seconds. It checks in with the manager, reporting host info and service status, and receives the site's config snapshot in return — see the [agent developer reference](../developers/agent.md) for the protocol and apply/rollback details. ```mermaid sequenceDiagram - participant Cron - participant pull-config + participant Timer as systemd timer + participant Agent participant Manager - Cron->>pull-config: run-parts /etc/pull-config.d - pull-config->>Manager: GET /sites/{SITE_ID}/nginx
Authorization: Bearer {API_KEY}
If-None-Match: {etag} + Timer->>Agent: start (every 30s) + Agent->>Manager: POST /api/v1/agents
Authorization: Bearer {API_KEY}
If-None-Match: {etag} alt Config changed - Manager-->>pull-config: 200 OK + new config - pull-config->>pull-config: Validate (nginx -t) - pull-config->>pull-config: Apply + reload + Manager-->>Agent: 200 OK + config snapshot + ETag + Agent->>Agent: Render, validate (nginx -t), apply + reload + Agent->>Manager: POST again (reports apply results) + Manager-->>Agent: 304 Not Modified else No changes - Manager-->>pull-config: 304 Not Modified + Manager-->>Agent: 304 Not Modified end ``` -Each service (nginx, dnsmasq) has its own instance script(s) in `/etc/pull-config.d/` that run independently. ETag caching ensures configs are only downloaded when they change. - -Dnsmasq is split into 5 pull-config instances: the main config (`dnsmasq-conf`) triggers a full restart, while the auxiliary files (`dnsmasq-dhcp-hosts`, `dnsmasq-hosts`, `dnsmasq-dhcp-opts`, `dnsmasq-servers`) trigger a SIGHUP reload. This avoids restarting dnsmasq when only DHCP leases or host records change. +Each check-in is recorded by the manager, so admins can monitor agent health on the **Agents** page. ETag caching keeps unchanged configs to a single 304 round trip. Dnsmasq's main config triggers a full restart when it changes, while the auxiliary files (DHCP hosts, host records, options, upstream servers) only trigger a SIGHUP reload. diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md new file mode 100644 index 00000000..dc495fef --- /dev/null +++ b/mie-opensource-landing/docs/developers/agent.md @@ -0,0 +1,99 @@ + +# agent + +{{ contributor_warning }} + +Node.js (TypeScript) check-in agent that reports host/service status to the manager and applies nginx and dnsmasq configuration. Installed on agent and manager containers. For deployment instructions, see [Deploying Agents](../admins/deploying-agents.md). + +## Architecture + +A systemd timer launches the oneshot agent every 30 seconds. Each run checks in with the manager (`POST /api/v1/agents`), applies any config changes, and exits. + +``` +agent/ +├── src/ # TypeScript sources (compiled to dist/) +│ ├── index.ts # Oneshot entry point: check-in loop +│ ├── config.ts # Environment configuration +│ ├── state.ts # Persisted ETag + last apply results +│ ├── system.ts # Hostname/IP/systemd state collection +│ ├── api.ts # Check-in HTTP client +│ └── apply.ts # Managed services: render/test/apply/reload +├── templates/ # EJS templates rendered locally +│ ├── nginx.conf.ejs +│ └── dnsmasq/ # conf, dhcp-hosts, hosts, dhcp-opts, servers +├── contrib/ +│ ├── systemd/ # opensource-agent.service + .timer (30s) +│ └── postinstall.sh # Enables the timer on package install +└── Makefile # Builds the opensource-agent package (see Release Pipeline) +``` + +## Check-in Protocol + +```mermaid +sequenceDiagram + participant Timer as systemd timer + participant Agent + participant Manager + + Timer->>Agent: start (every 30s) + Agent->>Manager: POST /api/v1/agents
{siteId, hostname, ipv4Address, services}
If-None-Match: {etag} + alt Config changed + Manager-->>Agent: 200 OK + config snapshot + ETag + Agent->>Agent: Render templates, validate
(nginx -t, dnsmasq --test) + Agent->>Agent: Apply + reload (rollback on failure) + Agent->>Manager: POST again (reports apply results) + Manager-->>Agent: 304 Not Modified + else No changes + Manager-->>Agent: 304 Not Modified + end + Agent->>Timer: exit +``` + +The check-in body carries system info and per-service status: + +```json +{ + "siteId": 1, + "hostname": "agent", + "currentTime": 1771234560, + "ipv4Address": "172.17.0.2", + "services": { + "nginx": { "state": "active", "lastApply": "success" }, + "dnsmasq": { "state": "active", "lastApply": "success" } + } +} +``` + +The manager records every check-in in the `Agents` table (shown on the web client's `/agents` page) and responds with the site's config snapshot as JSON. A strong `ETag` covers the snapshot; the agent stores it in `/var/lib/opensource-agent/state.json` and sends it back via `If-None-Match`, so unchanged configs cost a single `304` round trip. + +## Environment Variables + +Read from the process environment (systemd loads `/etc/environment`). Set via container runtime (Docker `ENV`, Proxmox LXC config) — the base image's `environment.sh` service propagates them on boot. + +| Variable | Required | Description | +|----------|----------|-------------| +| `SITE_ID` | Yes | Numeric site ID from the manager | +| `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | +| `API_KEY` | No | Admin API key for remote agents. Not needed on the manager (localhost is trusted). | +| `STATE_DIR` | No | State directory (default `/var/lib/opensource-agent`) | + +The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. + +## Managed Services + +| Service | Files | Test | Reload | +|---------|-------|------|--------| +| nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | +| dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | + +Apply flow per service: render templates, skip if nothing changed, stage new files, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. + +To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. + +## Running Manually + +```bash +systemctl status opensource-agent.timer # timer state +journalctl -u opensource-agent.service # past runs +node /opt/opensource-server/agent/dist/index.js # single run by hand +``` diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index 6b567db0..149fec58 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -11,6 +11,7 @@ The cluster management system uses Sequelize ORM with PostgreSQL. While Sequeliz erDiagram Sites ||--o{ Nodes : contains Sites ||--o{ ExternalDomains : "default site" + Sites ||--o{ Agents : "checked in by" Nodes ||--o{ Containers : hosts Containers ||--o{ Services : exposes Containers }o--o| Jobs : "created by" @@ -51,6 +52,15 @@ erDiagram int siteId FK } + Agents { + int id PK + int siteId FK + string hostname "unique per site" + string ipv4Address + json services "per-service state + lastApply" + date lastCheckinAt + } + Containers { int id PK string hostname UK @@ -179,6 +189,9 @@ Top-level organizational unit. Has many Nodes. Has many ExternalDomains (as defa ### Node Proxmox VE server within a site. `name` must match Proxmox hostname (unique). `imageStorage` defaults to `'local'` (CT templates). `volumeStorage` defaults to `'local-lvm'` (container rootfs). `networkBridge` defaults to `'vmbr0'` (Proxmox bridge used in container net0 config). `nvidiaAvailable` indicates the node has NVIDIA drivers and nvidia-container-toolkit configured for GPU passthrough. Belongs to Site, has many Containers. +### Agent +Site agent registered by its check-in (`POST /api/v1/agents`, every 30s). Unique composite index on `(siteId, hostname)`. `services` stores the per-service status reported by the agent (`{ nginx: { state, lastApply }, ... }`); `lastCheckinAt` drives the online/offline health shown on the web client's Agents page. Belongs to Site. See [agent](agent.md). + ### Container LXC container on a Proxmox node. Unique composite index on `(nodeId, containerId)`. `hostname`, `macAddress`, `ipv4Address` globally unique. `nvidiaRequested` indicates GPU passthrough was requested — the container is assigned to an NVIDIA-capable node and the nvidia hookscript is attached. Belongs to Node and optionally to a Job. diff --git a/mie-opensource-landing/docs/developers/docker-images.md b/mie-opensource-landing/docs/developers/docker-images.md index 7f2fedec..3645304a 100644 --- a/mie-opensource-landing/docs/developers/docker-images.md +++ b/mie-opensource-landing/docs/developers/docker-images.md @@ -24,7 +24,7 @@ Extends base with Node.js 24 from NodeSource. Inherits LDAP authentication. ### Agent (`agent`) -Extends nodejs with the `opensource-agent` package (pull-config, nginx with ModSecurity/OWASP CRS, dnsmasq) and [acme.sh](https://github.com/acmesh-official/acme.sh) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). +Extends nodejs with the `opensource-agent` package (check-in agent, nginx with ModSecurity/OWASP CRS, dnsmasq) and [acme.sh](https://github.com/acmesh-official/acme.sh) for ACME certificate management. Used as the networking layer for each site — handles reverse proxy, DNS, and TLS. See [Deploying Agents](../admins/deploying-agents.md). **Registry:** `ghcr.io/mieweb/opensource-server/agent` · **Source:** [`images/agent/`](https://github.com/mieweb/opensource-server/tree/main/images/agent) diff --git a/mie-opensource-landing/docs/developers/pull-config.md b/mie-opensource-landing/docs/developers/pull-config.md deleted file mode 100644 index bc1dd43a..00000000 --- a/mie-opensource-landing/docs/developers/pull-config.md +++ /dev/null @@ -1,92 +0,0 @@ - -# pull-config - -{{ contributor_warning }} - -Cron-based configuration management that pulls config files from the manager API. Installed on agent and manager containers. For deployment instructions, see [Deploying Agents](../admins/deploying-agents.md). - -## Architecture - -Executable instance scripts in `/etc/pull-config.d/` are run every minute by cron via `run-parts`. Each script sources environment variables, sets service-specific config, and calls the main pull-config binary. - -``` -pull-config/ -├── bin/ -│ └── pull-config # Main script (called by instance scripts) -├── etc/ -│ ├── cron.d/ -│ │ └── pull-config # Single cron entry: run-parts /etc/pull-config.d -│ └── pull-config.d/ # Executable instance scripts -│ ├── nginx -│ ├── dnsmasq-conf -│ ├── dnsmasq-dhcp-hosts -│ ├── dnsmasq-hosts -│ ├── dnsmasq-dhcp-opts -│ └── dnsmasq-servers -└── Makefile # Builds the opensource-agent package (see Release Pipeline) -``` - -## Environment Variables - -Sourced from `/etc/environment` by each instance script. Set via container runtime (Docker `ENV`, Proxmox LXC config) — the base image's `environment.sh` service propagates them on boot. - -| Variable | Required | Description | -|----------|----------|-------------| -| `SITE_ID` | Yes | Numeric site ID from the manager | -| `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | -| `API_KEY` | No | Bearer token for remote agents. Not needed on the manager (localhost is trusted). | - -The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. - -## Instance Script Variables - -Each instance script exports these before calling `exec /opt/opensource-server/pull-config/bin/pull-config`: - -### Required (must be exported) - -| Variable | Description | -|----------|-------------| -| `CONF_FILE` | Target configuration file path | -| `CONF_URL` | URL to fetch configuration from | - -### Optional - -| Variable | Description | -|----------|-------------| -| `TEST_COMMAND` | Command to validate config before applying (e.g., `nginx -t`) | -| `RELOAD_COMMAND` | Custom command to reload the service | -| `SERVICE_NAME` | Service name for `systemctl reload-or-restart` fallback | - -## Adding an Instance - -Create an executable script in `/etc/pull-config.d/`: - -```bash -#!/usr/bin/env bash - -set -a; . /etc/environment; set +a - -export CONF_FILE=/etc/myservice/config.conf -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/myservice.conf -export TEST_COMMAND="myservice --validate-config" -export SERVICE_NAME="myservice" - -exec /opt/opensource-server/pull-config/bin/pull-config -``` - -```bash -sudo chmod +x /etc/pull-config.d/myservice -``` - -The cron job picks it up automatically on the next run. - -### File Naming - -`run-parts` only executes files matching: `a-zA-Z0-9_-` (no dots — use `nginx` not `nginx.sh`). - -## Behavior - -- **ETag caching** — only downloads when the manager reports a change (HTTP 304) -- **Validation** — runs `TEST_COMMAND` before applying; rolls back on failure -- **API key auth** — sends `Authorization: Bearer $API_KEY` when `API_KEY` is set -- **Reload** — uses `RELOAD_COMMAND` if set, otherwise `systemctl reload-or-restart $SERVICE_NAME` diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md index e7bb0115..df33f01b 100644 --- a/mie-opensource-landing/docs/developers/release-pipeline.md +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -12,11 +12,11 @@ are reused by local development, the container images, and CI. |---|---|---|---| | [`create-a-container/`](https://github.com/mieweb/opensource-server/tree/main/create-a-container) | `opensource-server` | amd64 | Manager web app, job runner, systemd units | | [`mie-opensource-landing/`](https://github.com/mieweb/opensource-server/tree/main/mie-opensource-landing) | `opensource-docs` | all | Prebuilt documentation site | -| [`pull-config/`](https://github.com/mieweb/opensource-server/tree/main/pull-config) | `opensource-agent` | all | pull-config engine, instances, error pages | +| [`agent/`](https://github.com/mieweb/opensource-server/tree/main/agent) | `opensource-agent` | all | Check-in agent, config templates, systemd timer, error pages | Everything installs under the `/opt/opensource-server` prefix, matching the -paths referenced by the systemd units, the pull-config instances, and the -manager-rendered nginx configuration. `opensource-server` depends on +paths referenced by the systemd units and the agent-rendered nginx +configuration. `opensource-server` depends on `opensource-agent` and `opensource-docs` because the manager's nginx config serves the agent's error pages and the docs site. @@ -51,7 +51,7 @@ is stripped if present (`v2026.6.3` and `2026.6.3` are equivalent). ```bash # Build and stage a component anywhere: -make -C pull-config install DESTDIR=/tmp/agent-root +make -C agent install DESTDIR=/tmp/agent-root # Build one package: make -C create-a-container deb # -> create-a-container/*.deb @@ -78,7 +78,7 @@ make -C mie-opensource-landing dev # docs live server Each component has a `.fpm` options file holding the static package metadata (name, architecture, dependencies, description, scripts, config files). The Makefile's `package` target stages the component into a `.pkg/buildroot` and runs [fpm](https://fpm.readthedocs.io/) with the dynamic options on the command line — output type, version (composed per format by `./package-version`), and the staging dir. fpm's `dir` input copies the staged tree verbatim from `-C .pkg/buildroot`, preserving symlinks (e.g. `node_modules/.bin/sequelize`) and the directory layout. The same definition produces deb, rpm, and apk, so `make rpm` and `make apk` also work. - `opensource-server` ships the `container-creator` and `job-runner` systemd units and enables them via an `after-install` script (`before-remove` disables them on real removal). The log directory is created on demand by the unit's `LogsDirectory`, not shipped in the package. The logrotate drop-in is the only config file. The `container-creator-init` unit (which provisions a *local* PostgreSQL) is **not** in the package — it is part of the manager image, since the package only suggests postgresql and works with a remote database too. -- `opensource-agent` ships the pull-config engine, instances, cron schedule and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. +- `opensource-agent` ships the compiled check-in agent, its config templates, the systemd service + 30s timer (enabled via an `after-install` script) and the static error pages. These are program code, not configuration (runtime config comes from `/etc/environment`), so nothing is marked as a config file. - `opensource-docs` ships content only. Config files are marked explicitly with `--config-files`; `--deb-no-default-config-files` stops fpm/dpkg from auto-marking everything under `/etc` as a conffile. diff --git a/mie-opensource-landing/zensical.toml b/mie-opensource-landing/zensical.toml index e83a4f30..9108783f 100644 --- a/mie-opensource-landing/zensical.toml +++ b/mie-opensource-landing/zensical.toml @@ -54,7 +54,7 @@ nav = [ { "Docker Images" = "developers/docker-images.md" }, { "Release Pipeline" = "developers/release-pipeline.md" }, { "Database Schema" = "developers/database-schema.md" }, - { "Pull Config" = "developers/pull-config.md" }, + { "Agent" = "developers/agent.md" }, { "Contributing" = "developers/contributing.md" }, ] }, ] }, diff --git a/pull-config/.gitignore b/pull-config/.gitignore deleted file mode 100644 index 1c09fed7..00000000 --- a/pull-config/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# packaging build artifacts -/.pkg/ -/*.deb -/*.rpm -/*.apk diff --git a/pull-config/README.md b/pull-config/README.md deleted file mode 100644 index 30c2ed60..00000000 --- a/pull-config/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# pull-config - -Cron-based configuration management that pulls config files from the manager API. Installed on agent and manager containers. - -- **Admin guide:** [Deploying Agents](https://mieweb.github.io/opensource-server/docs/admins/deploying-agents) — how to deploy and configure an agent container -- **Developer reference:** [pull-config](https://mieweb.github.io/opensource-server/docs/developers/pull-config) — architecture, instance script variables, adding new instances - - diff --git a/pull-config/bin/pull-config b/pull-config/bin/pull-config deleted file mode 100755 index 241d0482..00000000 --- a/pull-config/bin/pull-config +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Required variables (must be set by calling script) -if [[ -z "${CONF_FILE:-}" ]] || [[ -z "${CONF_URL:-}" ]]; then - echo "ERROR: CONF_FILE and CONF_URL must be set by the calling script" >&2 - exit 1 -fi - -# Set defaults for optional variables -TEST_COMMAND="${TEST_COMMAND:-}" -RELOAD_COMMAND="${RELOAD_COMMAND:-}" -SERVICE_NAME="${SERVICE_NAME:-}" - -# Derived file paths -ETAG_FILE="${CONF_FILE}.etag" -TEMP_FILE="${CONF_FILE}.tmp" -HEADERS_FILE="${CONF_FILE}.headers" -BACKUP_FILE="${CONF_FILE}.bak" - -# Cleanup function -cleanup() { - rm -f "${TEMP_FILE}" "${HEADERS_FILE}" "${BACKUP_FILE}" -} - -# Set trap to always cleanup on exit -trap cleanup EXIT - -# Function to download config and extract ETag -download_config() { - local url="$1" - - # Build curl command as array - local curl_cmd=( - curl - -w "%{http_code}" - -D "${HEADERS_FILE}" - -o "${TEMP_FILE}" - -sSL - ) - - # Add ETag header if it exists and conf file exists - if [[ -f "${CONF_FILE}" ]] && [[ -f "${ETAG_FILE}" ]]; then - local etag - etag=$(cat "${ETAG_FILE}") - curl_cmd+=(-H "If-None-Match: ${etag}") - fi - - # Add API key authentication if available - if [[ -n "${API_KEY:-}" ]]; then - curl_cmd+=(-H "Authorization: Bearer ${API_KEY}") - fi - - # Add URL - curl_cmd+=("${url}") - - # Execute curl and capture HTTP status code - local http_code - http_code=$("${curl_cmd[@]}" 2>/dev/null || echo "000") - - # Return the http_code - echo "${http_code}" -} - -# Download configuration -HTTP_CODE=$(download_config "${CONF_URL}") - -# Check if we got a 304 Not Modified -if [[ ${HTTP_CODE} -eq 304 ]]; then - # No changes, exit (cleanup handled by trap) - exit 0 -fi - -# Check if we got a successful response -if [[ ${HTTP_CODE} -ne 200 ]]; then - echo "Failed to download configuration (HTTP ${HTTP_CODE})" >&2 - exit 1 -fi - -# Extract new ETag from headers -NEW_ETAG="" -if [[ -f "${HEADERS_FILE}" ]]; then - NEW_ETAG=$(grep -i '^etag:' "${HEADERS_FILE}" | sed 's/^etag: *//i' | tr -d '\r\n' || echo "") -fi - -# Backup existing config if it exists -if [[ -f "${CONF_FILE}" ]]; then - mv "${CONF_FILE}" "${BACKUP_FILE}" -fi - -# Move new config into place -mv "${TEMP_FILE}" "${CONF_FILE}" - -# Test the new configuration if TEST_COMMAND is provided -if [[ -n "${TEST_COMMAND}" ]]; then - if ! eval "${TEST_COMMAND}" >/dev/null 2>&1; then - echo "Configuration test failed" >&2 - # Restore backup if it exists - if [[ -f "${BACKUP_FILE}" ]]; then - mv "${BACKUP_FILE}" "${CONF_FILE}" - else - # No backup, just remove the bad config - rm -f "${CONF_FILE}" - fi - exit 1 - fi -fi - -# Configuration is valid, save new ETag (cleanup handled by trap) -if [[ -n "${NEW_ETAG}" ]]; then - echo "${NEW_ETAG}" > "${ETAG_FILE}" -fi - -# Reload/restart the service -if [[ -n "${RELOAD_COMMAND}" ]]; then - # Use custom reload command - eval "${RELOAD_COMMAND}" >/dev/null 2>&1 -elif [[ -n "${SERVICE_NAME}" ]]; then - # Fall back to systemctl reload-or-restart - systemctl reload-or-restart "${SERVICE_NAME}" >/dev/null 2>&1 -fi diff --git a/pull-config/etc/cron.d/pull-config b/pull-config/etc/cron.d/pull-config deleted file mode 100644 index f4fb0396..00000000 --- a/pull-config/etc/cron.d/pull-config +++ /dev/null @@ -1,5 +0,0 @@ -SHELL=/bin/bash -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin - -# Run all pull-config instances every minute -* * * * * root /bin/run-parts /etc/pull-config.d diff --git a/pull-config/etc/pull-config.d/dnsmasq-conf b/pull-config/etc/pull-config.d/dnsmasq-conf deleted file mode 100755 index 949c9668..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-conf +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq main configuration — requires full restart on change - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -export CONF_FILE=/etc/dnsmasq.conf -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/conf -export TEST_COMMAND="dnsmasq --test" -export RELOAD_COMMAND="systemctl restart dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts b/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts deleted file mode 100755 index b7b5f63f..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-dhcp-hosts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DHCP static leases — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/dhcp-hosts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/dhcp-hosts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts b/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts deleted file mode 100755 index 2d134cda..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-dhcp-opts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DHCP options — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/dhcp-opts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/dhcp-opts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-hosts b/pull-config/etc/pull-config.d/dnsmasq-hosts deleted file mode 100755 index 0cd4f285..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-hosts +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq host records (addn-hosts) — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/hosts -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/hosts -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/dnsmasq-servers b/pull-config/etc/pull-config.d/dnsmasq-servers deleted file mode 100755 index e15af851..00000000 --- a/pull-config/etc/pull-config.d/dnsmasq-servers +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Dnsmasq DNS forwarders — reloadable via SIGHUP - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -mkdir -p /var/lib/dnsmasq - -export CONF_FILE=/var/lib/dnsmasq/servers -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/dnsmasq/servers -export TEST_COMMAND="" -export RELOAD_COMMAND="systemctl reload dnsmasq" -export SERVICE_NAME="" - -exec /opt/opensource-server/pull-config/bin/pull-config diff --git a/pull-config/etc/pull-config.d/nginx b/pull-config/etc/pull-config.d/nginx deleted file mode 100755 index e68bfb1c..00000000 --- a/pull-config/etc/pull-config.d/nginx +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Nginx instance configuration for pull-config - -# Export well-formed KEY=VALUE lines from /etc/environment (safe for unquoted values) -while IFS='=' read -r key value; do - [[ -z "$key" || "$key" =~ ^# ]] && continue - export "$key=$value" -done < /etc/environment - -if [[ -z "${SITE_ID:-}" ]] || [[ -z "${MANAGER_URL:-}" ]]; then - echo "ERROR: SITE_ID and MANAGER_URL must be set in /etc/environment" >&2 - exit 1 -fi - -# Configuration file to manage -export CONF_FILE=/etc/nginx/nginx.conf - -# URL to fetch configuration from -export CONF_URL=${MANAGER_URL}/sites/${SITE_ID}/nginx - -# Command to test configuration validity before applying -export TEST_COMMAND="nginx -t" - -# Command to reload the service -export RELOAD_COMMAND="nginx -s reload" - -# Fallback service name for systemctl (if RELOAD_COMMAND fails) -export SERVICE_NAME="nginx" - -# Execute pull-config -exec /opt/opensource-server/pull-config/bin/pull-config - From 15f0e4aecfb308f6398f9b505a16ef400aa2926e Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Tue, 21 Jul 2026 09:17:08 -0700 Subject: [PATCH 2/6] fix(agent): address PR #407 review comments - Use systemd StateDirectory=/STATE_DIRECTORY instead of Makefile-created state dir - Rework state.ts into a State class; only catch ENOENT/SyntaxError - Replace systemctl subprocess calls with the systemd D-Bus API (@jellybrick/dbus-next); keep nginx -t / dnsmasq --test exec validation - Roll back and best-effort re-reload when a service reload fails - Skip dnsmasq management until all required site fields are configured; mark SiteInfo fields nullable to match the manager model - Check in once more after MAX_PASSES so final apply results are reported - Use requireLocalhostOrAdmin for agent check-in auth; document why manual ETag handling is needed on POST - Replace boolean online flag with server-computed secondsSinceCheckin; client decides the offline threshold and renders drift-free timestamps - Docs: systemctl start for manual runs, drop the file tree listing, STATE_DIRECTORY env var --- agent/Makefile | 1 - .../contrib/systemd/opensource-agent.service | 2 + agent/package-lock.json | 134 ++++++++++++++++++ agent/package.json | 1 + agent/src/apply.ts | 29 ++-- agent/src/config.ts | 3 +- agent/src/index.ts | 29 ++-- agent/src/state.ts | 45 +++--- agent/src/system.ts | 130 ++++++++++++++++- agent/src/types.ts | 15 +- create-a-container/client/src/lib/types.ts | 3 +- .../src/pages/agents/AgentsListPage.tsx | 17 ++- create-a-container/openapi.v1.yaml | 2 +- create-a-container/routers/api/v1/agents.js | 30 ++-- .../docs/admins/deploying-agents.md | 2 +- .../docs/developers/agent.md | 20 +-- 16 files changed, 368 insertions(+), 95 deletions(-) diff --git a/agent/Makefile b/agent/Makefile index 241e8938..0832de84 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -51,7 +51,6 @@ install: build $(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages $(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/ $(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache - $(INSTALL) -d -m 0755 $(DESTDIR)/var/lib/opensource-agent PACKAGER ?= deb package: diff --git a/agent/contrib/systemd/opensource-agent.service b/agent/contrib/systemd/opensource-agent.service index 5fa07ce4..ff2d2814 100644 --- a/agent/contrib/systemd/opensource-agent.service +++ b/agent/contrib/systemd/opensource-agent.service @@ -9,4 +9,6 @@ After=network-online.target environment.service Type=oneshot # Runtime config (SITE_ID, MANAGER_URL, API_KEY) lives in /etc/environment. EnvironmentFile=-/etc/environment +# systemd creates /var/lib/opensource-agent and passes it as $STATE_DIRECTORY. +StateDirectory=opensource-agent ExecStart=/usr/bin/node /opt/opensource-server/agent/dist/index.js diff --git a/agent/package-lock.json b/agent/package-lock.json index 974435a7..0f8385f6 100644 --- a/agent/package-lock.json +++ b/agent/package-lock.json @@ -9,6 +9,7 @@ "version": "0.0.0", "license": "Apache-2.0", "dependencies": { + "@jellybrick/dbus-next": "^0.11.1", "ejs": "^3.1.10" }, "devDependencies": { @@ -17,6 +18,31 @@ "typescript": "^5.7.0" } }, + "node_modules/@jellybrick/dbus-next": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/@jellybrick/dbus-next/-/dbus-next-0.11.1.tgz", + "integrity": "sha512-MEN8boAd/XN3nTaRULNh/QDWPi1LzDXLLl83EV6AXsdWpZhBzagl8CjBC85g8769JMDhJFytVX8RKyVjUk9nAw==", + "license": "MIT", + "dependencies": { + "fast-xml-builder": "^1.2.0", + "fast-xml-parser": "^5.9.3" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@types/ejs": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/@types/ejs/-/ejs-3.1.5.tgz", @@ -34,6 +60,18 @@ "undici-types": "~7.18.0" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -70,6 +108,45 @@ "node": ">=0.10.0" } }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/filelist": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.6.tgz", @@ -79,6 +156,18 @@ "minimatch": "^5.0.1" } }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/jake": { "version": "10.9.4", "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.4.tgz", @@ -108,12 +197,42 @@ "node": ">=10" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "license": "ISC" }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -134,6 +253,21 @@ "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", "dev": true, "license": "MIT" + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } } } } diff --git a/agent/package.json b/agent/package.json index 371e34bd..dfd35eb0 100644 --- a/agent/package.json +++ b/agent/package.json @@ -9,6 +9,7 @@ "start": "node dist/index.js" }, "dependencies": { + "@jellybrick/dbus-next": "^0.11.1", "ejs": "^3.1.10" }, "devDependencies": { diff --git a/agent/src/apply.ts b/agent/src/apply.ts index afe4f4d0..9cadf53c 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -9,6 +9,7 @@ import fs from 'fs'; import path from 'path'; import { execFileSync } from 'child_process'; import ejs from 'ejs'; +import { reloadOrRestartService, restartService, sighupService } from './system'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); @@ -27,7 +28,7 @@ export interface ManagedService { /** Command that validates the staged config before it is kept. */ test?: string[]; /** Reload/restart after a successful apply. */ - reload(changedFiles: string[]): void; + reload(changedFiles: string[]): Promise; } function renderTemplate(template: string, data: object): Promise { @@ -49,14 +50,20 @@ export const services: ManagedService[] = [ }, test: ['nginx', '-t'], reload() { - run(['systemctl', 'reload-or-restart', 'nginx']); + return reloadOrRestartService('nginx'); }, }, { unit: 'dnsmasq', async render(config) { - if (!config.site) return null; - const data = { site: config.site }; + const site = config.site; + // Skip dnsmasq management until the site's DHCP/DNS settings are fully + // configured — the templates need all of these fields. + if (!site?.internalDomain || !site.dhcpRange || !site.subnetMask + || !site.gateway || !site.dnsForwarders) { + return null; + } + const data = { site }; return [ { dest: '/etc/dnsmasq.conf', content: await renderTemplate('dnsmasq/conf.ejs', data) }, { dest: '/var/lib/dnsmasq/dhcp-hosts', content: await renderTemplate('dnsmasq/dhcp-hosts.ejs', data) }, @@ -70,10 +77,9 @@ export const services: ManagedService[] = [ // The main config requires a full restart; the auxiliary files under // /var/lib/dnsmasq are re-read on SIGHUP. if (changedFiles.includes('/etc/dnsmasq.conf')) { - run(['systemctl', 'restart', 'dnsmasq']); - } else { - run(['systemctl', 'kill', '--signal=SIGHUP', 'dnsmasq']); + return restartService('dnsmasq'); } + return sighupService('dnsmasq'); }, }, ]; @@ -118,9 +124,14 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro } try { - svc.reload(changed); + await svc.reload(changed); } catch (err) { - console.error(`${svc.unit}: reload failed: ${(err as Error).message}`); + // The new config passed its test but the service couldn't pick it up: + // restore the previous files and reload again (best effort) so the + // service keeps running the last known-good config. + rollback(); + await svc.reload(changed).catch(() => { /* reported via service state at next check-in */ }); + console.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); return 'failure'; } diff --git a/agent/src/config.ts b/agent/src/config.ts index e20ec53a..d01c91f3 100644 --- a/agent/src/config.ts +++ b/agent/src/config.ts @@ -18,6 +18,7 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { siteId, managerUrl: managerUrl.replace(/\/+$/, ''), apiKey: env.API_KEY || undefined, - stateDir: env.STATE_DIR || '/var/lib/opensource-agent', + // Set by systemd from StateDirectory=; fallback for manual runs. + stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', }; } diff --git a/agent/src/index.ts b/agent/src/index.ts index 68ce2c10..5d2a2d9b 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -12,8 +12,8 @@ import os from 'os'; import { loadConfig, type AgentConfig } from './config'; -import { loadState, saveState, type AgentState } from './state'; -import { getPrimaryIpv4, getServiceState } from './system'; +import { State } from './state'; +import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; import { checkin } from './api'; import { services, applyService } from './apply'; import type { CheckinRequest, ServiceStatus } from './types'; @@ -22,11 +22,11 @@ import type { CheckinRequest, ServiceStatus } from './types'; // forever; the timer starts a fresh run 30s later anyway. const MAX_PASSES = 5; -function buildCheckinBody(cfg: AgentConfig, state: AgentState): CheckinRequest { +async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const serviceStatus: Record = {}; for (const svc of services) { serviceStatus[svc.unit] = { - state: getServiceState(svc.unit), + state: await getServiceState(svc.unit), lastApply: state.lastApply[svc.unit] ?? 'unknown', }; } @@ -41,10 +41,10 @@ function buildCheckinBody(cfg: AgentConfig, state: AgentState): CheckinRequest { async function main(): Promise { const cfg = loadConfig(); - const state = loadState(cfg.stateDir); + const state = State.load(cfg.stateDir); for (let pass = 0; pass < MAX_PASSES; pass++) { - const result = await checkin(cfg, buildCheckinBody(cfg, state), state.etag); + const result = await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); if (result.notModified) return; for (const svc of services) { @@ -55,11 +55,18 @@ async function main(): Promise { // fix itself without a server-side change (which changes the ETag), and // the failure has been reported via lastApply. state.etag = result.etag; - saveState(cfg.stateDir, state); + state.save(); } + + // MAX_PASSES exhausted (flapping server-side config): check in once more so + // the final pass' apply results reach the manager instead of going stale + // until the next timer run. + await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); } -main().catch((err) => { - console.error(err instanceof Error ? err.message : err); - process.exit(1); -}); +main() + .then(() => disconnectSystemBus()) + .catch((err) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + }); diff --git a/agent/src/state.ts b/agent/src/state.ts index bd6a5ab0..3e3cdbf4 100644 --- a/agent/src/state.ts +++ b/agent/src/state.ts @@ -5,25 +5,38 @@ import fs from 'fs'; import path from 'path'; import type { ApplyResult } from './types'; -export interface AgentState { +export class State { etag?: string; - lastApply: Record; -} + lastApply: Record = {}; -function stateFile(stateDir: string): string { - return path.join(stateDir, 'state.json'); -} + private constructor(private readonly file: string) {} -export function loadState(stateDir: string): AgentState { - try { - const raw = JSON.parse(fs.readFileSync(stateFile(stateDir), 'utf8')); - return { lastApply: {}, ...raw }; - } catch { - return { lastApply: {} }; + static load(stateDir: string): State { + const state = new State(path.join(stateDir, 'state.json')); + let raw: string; + try { + raw = fs.readFileSync(state.file, 'utf8'); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return state; // first run + throw err; + } + try { + const data = JSON.parse(raw) as { etag?: string; lastApply?: Record }; + state.etag = data.etag; + state.lastApply = data.lastApply ?? {}; + } catch (err) { + if (!(err instanceof SyntaxError)) throw err; + // A corrupt state file just means a full re-apply on this run. + console.error(`Ignoring unparsable state file ${state.file}: ${err.message}`); + } + return state; } -} -export function saveState(stateDir: string, state: AgentState): void { - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(stateFile(stateDir), JSON.stringify(state, null, 2)); + save(): void { + fs.mkdirSync(path.dirname(this.file), { recursive: true }); + fs.writeFileSync( + this.file, + JSON.stringify({ etag: this.etag, lastApply: this.lastApply }, null, 2), + ); + } } diff --git a/agent/src/system.ts b/agent/src/system.ts index 553667f6..5aca9fcf 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -1,7 +1,7 @@ -/** Host system info: primary IPv4 address and systemd service states. */ +/** Host system info and systemd unit control via the systemd D-Bus API. */ import os from 'os'; -import { execFileSync } from 'child_process'; +import dbus, { type MessageBus, type ProxyObject, type Variant } from '@jellybrick/dbus-next'; /** First non-internal IPv4 address, or null when none is configured. */ export function getPrimaryIpv4(): string | null { @@ -13,14 +13,130 @@ export function getPrimaryIpv4(): string | null { return null; } +// --- systemd (org.freedesktop.systemd1 on the system bus) ------------------- + +const SYSTEMD = 'org.freedesktop.systemd1'; +const SYSTEMD_PATH = '/org/freedesktop/systemd1'; +const MANAGER_IFACE = 'org.freedesktop.systemd1.Manager'; +const UNIT_IFACE = 'org.freedesktop.systemd1.Unit'; +const PROPERTIES_IFACE = 'org.freedesktop.DBus.Properties'; + +const SIGHUP = 1; + +/** The subset of org.freedesktop.DBus.Properties the agent uses. */ +interface DbusProperties { + Get(iface: string, prop: string): Promise>; +} + +/** The subset of org.freedesktop.systemd1.Manager the agent uses. */ +interface SystemdManager { + LoadUnit(name: string): Promise; + ReloadOrRestartUnit(name: string, mode: string): Promise; + RestartUnit(name: string, mode: string): Promise; + KillUnit(name: string, who: string, signal: number): Promise; + Subscribe(): Promise; + Unsubscribe(): Promise; + on(event: 'JobRemoved', listener: (id: number, job: unknown, unit: string, result: string) => void): void; + removeListener(event: 'JobRemoved', listener: (...args: never[]) => void): void; +} + +let bus: MessageBus | null = null; +let manager: SystemdManager | null = null; + +function getBus(): MessageBus { + if (!bus) bus = dbus.systemBus(); + return bus; +} + +async function getManager(): Promise { + if (!manager) { + const obj: ProxyObject = await getBus().getProxyObject(SYSTEMD, SYSTEMD_PATH); + manager = obj.getInterface(MANAGER_IFACE) as unknown as SystemdManager; + } + return manager; +} + +/** Close the D-Bus connection so the oneshot process can exit. */ +export function disconnectSystemBus(): void { + bus?.disconnect(); + bus = null; + manager = null; +} + +/** systemctl-style shorthand: bare names get a .service suffix. */ +function unitName(unit: string): string { + return unit.includes('.') ? unit : `${unit}.service`; +} + /** systemd ActiveState for a unit (active, inactive, failed, ...). */ -export function getServiceState(unit: string): string { +export async function getServiceState(unit: string): Promise { try { - const out = execFileSync('systemctl', ['show', '--property=ActiveState', '--value', unit], { - encoding: 'utf8', - }); - return out.trim() || 'unknown'; + const mgr = await getManager(); + const path = await mgr.LoadUnit(unitName(unit)); + const obj = await getBus().getProxyObject(SYSTEMD, path); + const props = obj.getInterface(PROPERTIES_IFACE) as unknown as DbusProperties; + const state = await props.Get(UNIT_IFACE, 'ActiveState'); + return state.value || 'unknown'; } catch { + // Any D-Bus failure just means the state can't be determined right now. return 'unknown'; } } + +/** Enqueue a systemd job and wait for it to finish; throws unless the job + * completes with result "done". */ +async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit', unit: string): Promise { + const mgr = await getManager(); + // JobRemoved is only broadcast to subscribed clients. + await mgr.Subscribe(); + try { + await new Promise((resolve, reject) => { + // The job can finish before the method call returns its path, so + // removals seen in the meantime are buffered. + const removedEarly = new Map(); + let jobPath: string | undefined; + + const finish = (result: string) => { + mgr.removeListener('JobRemoved', onRemoved); + if (result === 'done') resolve(); + else reject(new Error(`systemd ${method} job for ${unit} finished with result "${result}"`)); + }; + const onRemoved = (_id: number, job: unknown, _unit: string, result: string) => { + const path = String(job); + if (jobPath === undefined) removedEarly.set(path, result); + else if (path === jobPath) finish(result); + }; + + mgr.on('JobRemoved', onRemoved); + mgr[method](unitName(unit), 'replace').then( + (job) => { + jobPath = String(job); + const result = removedEarly.get(jobPath); + if (result !== undefined) finish(result); + }, + (err: Error) => { + mgr.removeListener('JobRemoved', onRemoved); + reject(err); + }, + ); + }); + } finally { + await mgr.Unsubscribe(); + } +} + +/** Equivalent of `systemctl reload-or-restart `. */ +export function reloadOrRestartService(unit: string): Promise { + return runJob('ReloadOrRestartUnit', unit); +} + +/** Equivalent of `systemctl restart `. */ +export function restartService(unit: string): Promise { + return runJob('RestartUnit', unit); +} + +/** Equivalent of `systemctl kill --signal=SIGHUP ` (main process). */ +export async function sighupService(unit: string): Promise { + const mgr = await getManager(); + await mgr.KillUnit(unitName(unit), 'main', SIGHUP); +} diff --git a/agent/src/types.ts b/agent/src/types.ts index 6d80fd90..4918e55d 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -33,14 +33,17 @@ export interface SiteNode { containers: SiteContainer[]; } +/** Mirrors the manager's Site model, where every field except id is + * nullable — a site can be partially configured. Consumers must guard + * before using these values (see the dnsmasq render skip in apply.ts). */ export interface SiteInfo { id: number; - name: string; - internalDomain: string; - dhcpRange: string; - subnetMask: string; - gateway: string; - dnsForwarders: string; + name: string | null; + internalDomain: string | null; + dhcpRange: string | null; + subnetMask: string | null; + gateway: string | null; + dnsForwarders: string | null; nodes: SiteNode[]; } diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index b217e295..97af36bd 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -45,7 +45,8 @@ export interface Agent { ipv4Address: string | null; services: Record | null; lastCheckinAt: string | null; - online: boolean; + /** Server-computed, so it is immune to client clock drift. */ + secondsSinceCheckin: number | null; } export interface ExternalDomain { diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx index c973ec2f..20056e94 100644 --- a/create-a-container/client/src/pages/agents/AgentsListPage.tsx +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -19,17 +19,22 @@ import type { Agent } from '@/lib/types'; import { useDocumentTitle } from '@/lib/useDocumentTitle'; import { AgentServiceBadges } from './AgentServiceBadges'; -function formatLastCheckin(lastCheckinAt: string | null): string { - if (!lastCheckinAt) return 'never'; - const seconds = Math.max(0, Math.round((Date.now() - new Date(lastCheckinAt).getTime()) / 1000)); +// Agents check in every 30 seconds; three missed intervals means offline. +const OFFLINE_AFTER_SECONDS = 90; + +function formatLastCheckin(agent: Agent): string { + const seconds = agent.secondsSinceCheckin; + if (seconds === null || !agent.lastCheckinAt) return 'never'; if (seconds < 60) return `${seconds}s ago`; if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`; if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`; - return new Date(lastCheckinAt).toLocaleString(); + return new Date(agent.lastCheckinAt).toLocaleString(); } function OnlineBadge({ agent }: { agent: Agent }) { - return agent.online ? ( + const online = + agent.secondsSinceCheckin !== null && agent.secondsSinceCheckin <= OFFLINE_AFTER_SECONDS; + return online ? ( Online ) : ( Offline @@ -96,7 +101,7 @@ export function AgentsListPage() { - {formatLastCheckin(agent.lastCheckinAt)} + {formatLastCheckin(agent)} ))} diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index 3220102e..ddbdd6cd 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -524,7 +524,7 @@ paths: get: tags: [Agents] summary: Current status of all agents (admin) - responses: { '200': { description: 'List of agents with lastCheckinAt, services and online flag' } } + responses: { '200': { description: 'List of agents with lastCheckinAt, services and server-computed secondsSinceCheckin' } } /external-domains: get: { tags: [External Domains], responses: { '200': { description: List } } } diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 030987ae..98a0bbd9 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -9,27 +9,16 @@ const express = require('express'); const { Agent, Site } = require('../../../models'); -const { isLocalhostRequest } = require('../../../middlewares'); +const { requireLocalhostOrAdmin } = require('../../../middlewares'); const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); const router = express.Router(); -// An agent is online if it checked in within three 30-second timer intervals. -const ONLINE_WINDOW_MS = 90 * 1000; - -// The manager's own agent checks in over localhost without credentials -// (bootstrap: no site, no API key exist yet). Remote agents authenticate -// with an admin API key. -function agentAuth(req, res, next) { - if (isLocalhostRequest(req)) return next(); - return apiAuth(req, res, (err) => { - if (err) return next(err); - return apiAdmin(req, res, next); - }); -} - -router.post('/', agentAuth, asyncHandler(async (req, res) => { +// Check-in auth: the manager's own agent checks in over localhost without +// credentials (bootstrap: no site, no API key exist yet); remote agents +// authenticate with an admin API key. +router.post('/', requireLocalhostOrAdmin, asyncHandler(async (req, res) => { const { siteId, hostname, ipv4Address, services } = req.body || {}; const parsedSiteId = parseInt(siteId, 10); if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { @@ -51,6 +40,9 @@ router.post('/', agentAuth, asyncHandler(async (req, res) => { } const config = await buildAgentConfig(parsedSiteId); + // Manual conditional-request handling: Express's built-in ETag/fresh logic + // (res.send + req.fresh) only produces 304s for GET/HEAD, and the check-in + // is a POST. const etag = computeConfigEtag(config); res.set('ETag', etag); if (req.get('If-None-Match') === etag) { @@ -73,7 +65,11 @@ router.get('/', apiAuth, apiAdmin, asyncHandler(async (req, res) => { ipv4Address: a.ipv4Address, services: a.services, lastCheckinAt: a.lastCheckinAt, - online: !!a.lastCheckinAt && now - new Date(a.lastCheckinAt).getTime() <= ONLINE_WINDOW_MS, + // Computed server-side so UI staleness judgments don't depend on the + // client's clock. + secondsSinceCheckin: a.lastCheckinAt + ? Math.max(0, Math.round((now - new Date(a.lastCheckinAt).getTime()) / 1000)) + : null, }))); })); diff --git a/mie-opensource-landing/docs/admins/deploying-agents.md b/mie-opensource-landing/docs/admins/deploying-agents.md index dbc599a6..d16adb39 100644 --- a/mie-opensource-landing/docs/admins/deploying-agents.md +++ b/mie-opensource-landing/docs/admins/deploying-agents.md @@ -92,7 +92,7 @@ cat /etc/nginx/nginx.conf cat /etc/dnsmasq.conf # Run a check-in manually -node /opt/opensource-server/agent/dist/index.js +systemctl start opensource-agent.service ``` The agent also appears on the manager's **Agents** page (`/agents`, admin only) with its last check-in time and service health. diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md index dc495fef..e00e7012 100644 --- a/mie-opensource-landing/docs/developers/agent.md +++ b/mie-opensource-landing/docs/developers/agent.md @@ -9,23 +9,7 @@ Node.js (TypeScript) check-in agent that reports host/service status to the mana A systemd timer launches the oneshot agent every 30 seconds. Each run checks in with the manager (`POST /api/v1/agents`), applies any config changes, and exits. -``` -agent/ -├── src/ # TypeScript sources (compiled to dist/) -│ ├── index.ts # Oneshot entry point: check-in loop -│ ├── config.ts # Environment configuration -│ ├── state.ts # Persisted ETag + last apply results -│ ├── system.ts # Hostname/IP/systemd state collection -│ ├── api.ts # Check-in HTTP client -│ └── apply.ts # Managed services: render/test/apply/reload -├── templates/ # EJS templates rendered locally -│ ├── nginx.conf.ejs -│ └── dnsmasq/ # conf, dhcp-hosts, hosts, dhcp-opts, servers -├── contrib/ -│ ├── systemd/ # opensource-agent.service + .timer (30s) -│ └── postinstall.sh # Enables the timer on package install -└── Makefile # Builds the opensource-agent package (see Release Pipeline) -``` +TypeScript sources live in `agent/src` (compiled to `dist/`), the locally rendered EJS templates in `agent/templates`, and the systemd service/timer units in `agent/contrib/systemd`. The `agent/Makefile` builds the `opensource-agent` package (see Release Pipeline). Service reloads and state queries go through the systemd D-Bus API; config validation uses `nginx -t` and `dnsmasq --test`. ## Check-in Protocol @@ -75,7 +59,7 @@ Read from the process environment (systemd loads `/etc/environment`). Set via co | `SITE_ID` | Yes | Numeric site ID from the manager | | `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | | `API_KEY` | No | Admin API key for remote agents. Not needed on the manager (localhost is trusted). | -| `STATE_DIR` | No | State directory (default `/var/lib/opensource-agent`) | +| `STATE_DIRECTORY` | No | State directory, set by systemd via `StateDirectory=` (default `/var/lib/opensource-agent`) | The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. From dd77c50c87c5d106c7e50b17e3b031e20554ae32 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Wed, 22 Jul 2026 07:44:18 -0700 Subject: [PATCH 3/6] fix(agent): swap @jellybrick/dbus-next for @particle/dbus-next The jellybrick fork's CommonJS bundle has a broken ESM-interop shim: its MessageBus constructor calls 'new fast_xml_builder.default(...)' where __toESM() has set .default to the whole module namespace, so any bus operation fails with 'fast_xml_builder.default is not a constructor'. This broke the agent's initial config apply during docker compose bootstrap. @particle/dbus-next is the original dbus-next codebase (native CJS, no bundler interop) with the vulnerable usocket/node-gyp/request chain removed and xml2js patched; npm audit remains clean. --- agent/package-lock.json | 242 ++++++++++++++++++++++------------------ agent/package.json | 2 +- agent/src/system.ts | 2 +- 3 files changed, 136 insertions(+), 110 deletions(-) diff --git a/agent/package-lock.json b/agent/package-lock.json index 0f8385f6..b1d8fe22 100644 --- a/agent/package-lock.json +++ b/agent/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.0", "license": "Apache-2.0", "dependencies": { - "@jellybrick/dbus-next": "^0.11.1", + "@particle/dbus-next": "^0.11.4", "ejs": "^3.1.10" }, "devDependencies": { @@ -18,30 +18,28 @@ "typescript": "^5.7.0" } }, - "node_modules/@jellybrick/dbus-next": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/@jellybrick/dbus-next/-/dbus-next-0.11.1.tgz", - "integrity": "sha512-MEN8boAd/XN3nTaRULNh/QDWPi1LzDXLLl83EV6AXsdWpZhBzagl8CjBC85g8769JMDhJFytVX8RKyVjUk9nAw==", - "license": "MIT", - "dependencies": { - "fast-xml-builder": "^1.2.0", - "fast-xml-parser": "^5.9.3" - }, + "node_modules/@nornagon/put": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/@nornagon/put/-/put-0.0.8.tgz", + "integrity": "sha512-ugvXJjwF5ldtUpa7D95kruNJ41yFQDEKyF5CW4TgKJnh+W/zmlBzXXeKTyqIgwMFrkePN2JqOBqcF0M0oOunow==", + "license": "MIT/X11", "engines": { - "node": ">=12.20.0" + "node": ">=0.3.0" } }, - "node_modules/@nodable/entities": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", - "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/nodable" - } - ], - "license": "MIT" + "node_modules/@particle/dbus-next": { + "version": "0.11.4", + "resolved": "https://registry.npmjs.org/@particle/dbus-next/-/dbus-next-0.11.4.tgz", + "integrity": "sha512-oJsiPMMYA+DlVbaVV1Ab3xRiurlGtAxdaGvLrIHOjPSyyVvsq3NNMwarF0pvxUnzSwo9t+Hwt69I96ZfRjro0Q==", + "license": "MIT", + "dependencies": { + "@nornagon/put": "0.0.8", + "event-stream": "4.0.1", + "jsbi": "^4.3.0", + "long": "^4.0.0", + "safe-buffer": "^5.2.1", + "xml2js": "^0.6.2" + } }, "node_modules/@types/ejs": { "version": "3.1.5", @@ -60,18 +58,6 @@ "undici-types": "~7.18.0" } }, - "node_modules/anynum": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", - "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT" - }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -93,6 +79,12 @@ "balanced-match": "^1.0.0" } }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "license": "MIT" + }, "node_modules/ejs": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", @@ -108,43 +100,19 @@ "node": ">=0.10.0" } }, - "node_modules/fast-xml-builder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", - "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "license": "MIT", - "dependencies": { - "path-expression-matcher": "^1.6.2", - "xml-naming": "^0.3.0" - } - }, - "node_modules/fast-xml-parser": { - "version": "5.10.1", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", - "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], + "node_modules/event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", "license": "MIT", "dependencies": { - "@nodable/entities": "^3.0.0", - "fast-xml-builder": "^1.2.0", - "is-unsafe": "^2.0.0", - "path-expression-matcher": "^1.6.2", - "strnum": "^2.4.1", - "xml-naming": "^0.3.0" - }, - "bin": { - "fxparser": "src/cli/cli.js" + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" } }, "node_modules/filelist": { @@ -156,16 +124,10 @@ "minimatch": "^5.0.1" } }, - "node_modules/is-unsafe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", - "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], + "node_modules/from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", "license": "MIT" }, "node_modules/jake": { @@ -185,6 +147,24 @@ "node": ">=10" } }, + "node_modules/jsbi": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", + "integrity": "sha512-9fqMSQbhJykSeii05nxKl4m6Eqn2P6rOlYiS+C5Dr/HPIU/7yZxu5qzbs40tgaFORiw2Amd0mirjxatXYMkIew==", + "license": "Apache-2.0" + }, + "node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0" + }, + "node_modules/map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==", + "license": "MIT" + }, "node_modules/minimatch": { "version": "5.1.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", @@ -197,19 +177,16 @@ "node": ">=10" } }, - "node_modules/path-expression-matcher": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", - "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } + "node_modules/pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "license": [ + "MIT", + "Apache2" ], - "license": "MIT", - "engines": { - "node": ">=14.0.0" + "dependencies": { + "through": "~2.3" } }, "node_modules/picocolors": { @@ -218,21 +195,63 @@ "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", "license": "ISC" }, - "node_modules/strnum": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", - "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", "funding": [ { "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" } ], + "license": "MIT" + }, + "node_modules/sax": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "license": "MIT", + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha512-6yHMqgLYDzQDcAkL+tjJDC5nSNuNIx0vZtRZeiPh7Saef7VHX9H5Ijn9l2VIol2zaNYlYEX6KyuT/237A58qEQ==", "license": "MIT", "dependencies": { - "anynum": "^1.0.1" + "duplexer": "~0.1.1", + "through": "~2.3.4" } }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -254,19 +273,26 @@ "dev": true, "license": "MIT" }, - "node_modules/xml-naming": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", - "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", "license": "MIT", "engines": { - "node": ">=16.0.0" + "node": ">=4.0" } } } diff --git a/agent/package.json b/agent/package.json index dfd35eb0..afa3d143 100644 --- a/agent/package.json +++ b/agent/package.json @@ -9,7 +9,7 @@ "start": "node dist/index.js" }, "dependencies": { - "@jellybrick/dbus-next": "^0.11.1", + "@particle/dbus-next": "^0.11.4", "ejs": "^3.1.10" }, "devDependencies": { diff --git a/agent/src/system.ts b/agent/src/system.ts index 5aca9fcf..5a3fda70 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -1,7 +1,7 @@ /** Host system info and systemd unit control via the systemd D-Bus API. */ import os from 'os'; -import dbus, { type MessageBus, type ProxyObject, type Variant } from '@jellybrick/dbus-next'; +import dbus, { type MessageBus, type ProxyObject, type Variant } from '@particle/dbus-next'; /** First non-internal IPv4 address, or null when none is configured. */ export function getPrimaryIpv4(): string | null { From 2bc4be06360229a899db59c699daa1edf277094d Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Wed, 22 Jul 2026 13:57:19 -0700 Subject: [PATCH 4/6] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- create-a-container/routers/api/v1/agents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 98a0bbd9..2076f60c 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -20,7 +20,7 @@ const router = express.Router(); // authenticate with an admin API key. router.post('/', requireLocalhostOrAdmin, asyncHandler(async (req, res) => { const { siteId, hostname, ipv4Address, services } = req.body || {}; - const parsedSiteId = parseInt(siteId, 10); + const parsedSiteId = typeof siteId === 'number' ? siteId : Number(siteId); if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { return fail(res, 422, 'validation_failed', 'siteId and hostname are required'); } From 4c32ec42d071a890eae2132c4c6dba15c6500055 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Thu, 23 Jul 2026 11:14:42 -0400 Subject: [PATCH 5/6] fix: build agent in the docker compose up pipeline --- compose.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/compose.yml b/compose.yml index 045beece..597dae04 100644 --- a/compose.yml +++ b/compose.yml @@ -28,19 +28,20 @@ services: skopeo copy docker://$${IMAGE_REF} oci-archive:$${FILENAME} fi -# This service runs once when bringing the compose stack up to ensure the -# node_modules are populated in the user's create-a-container directory. This -# is usually handled by the manager's Dockerfile, but we're mounting over that -# directory so we need to be sure that the user has them. + # This service runs once when bringing the compose stack up to ensure the + # node_modules are populated in the user's create-a-container directory. This + # is usually handled by the manager's Dockerfile, but we're mounting over that + # directory so we need to be sure that the user has them. node: image: node:24-trixie volumes: - ./:/opt/opensource-server - working_dir: /opt/opensource-server/create-a-container + working_dir: /opt/opensource-server entrypoint: ["/bin/sh", "-c"] command: - | - exec npm ci --no-audit --no-fund + npm -C create-a-container ci --no-audit --no-fund + make -C agent build # Watches the React client and rebuilds the production bundle on change. The # Manager LXC inside Proxmox serves create-a-container/client/dist statically From cfcfda557c9b2c1d5f3070b755658627295b2ce5 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 23 Jul 2026 09:41:39 -0700 Subject: [PATCH 6/6] fix: address remaining Copilot review comments on PR #407 - agent/api.ts: add a 30s AbortSignal timeout to the check-in fetch so the oneshot unit fails fast instead of hanging when the manager stalls - agent/apply.ts: write config files atomically (temp file + rename) so a crash mid-write can't leave truncated configs on disk - agents.js: replace requireLocalhostOrAdmin with a checkinAuth that uses apiAuth/apiAdmin for remote agents, keeping error responses in the v1 JSON envelope; localhost bootstrap bypass unchanged - middlewares: isLocalhostRequest now also checks the first hop of X-Forwarded-For; drop the now-unused requireLocalhostOrAdmin - client: extract OnlineBadge into its own file (one component per file) --- agent/src/api.ts | 5 ++++ agent/src/apply.ts | 19 ++++++++++---- .../src/pages/agents/AgentsListPage.tsx | 15 +---------- .../client/src/pages/agents/OnlineBadge.tsx | 15 +++++++++++ create-a-container/middlewares/index.js | 25 ++++++------------- create-a-container/routers/api/v1/agents.js | 15 ++++++++--- 6 files changed, 54 insertions(+), 40 deletions(-) create mode 100644 create-a-container/client/src/pages/agents/OnlineBadge.tsx diff --git a/agent/src/api.ts b/agent/src/api.ts index a15fc26f..ef1b6b0f 100644 --- a/agent/src/api.ts +++ b/agent/src/api.ts @@ -7,6 +7,10 @@ export type CheckinResult = | { notModified: true } | { notModified: false; etag?: string; config: SiteConfig }; +// Fail fast if the manager is unreachable or the connection stalls, so the +// oneshot unit exits predictably instead of hanging and queueing timer runs. +const CHECKIN_TIMEOUT_MS = 30_000; + export async function checkin( cfg: AgentConfig, body: CheckinRequest, @@ -20,6 +24,7 @@ export async function checkin( method: 'POST', headers, body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), }); if (res.status === 304) return { notModified: true }; diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 9cadf53c..4a518d53 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -1,8 +1,9 @@ /** * Managed services: how to render, test, apply and reload each service's - * configuration. Files are written in place with backup/rollback — if the - * test command rejects the new config, the previous files are restored and - * the apply is reported as a failure at the next check-in. + * configuration. Files are written atomically (temp file + rename) with + * backup/rollback — if the test command rejects the new config, the + * previous files are restored and the apply is reported as a failure at the + * next check-in. */ import fs from 'fs'; @@ -92,6 +93,14 @@ function readIfExists(file: string): string | null { } } +// Write via temp file + rename so a crash mid-write can never leave a +// truncated config on disk. +function writeFileAtomic(dest: string, content: string): void { + const tmp = `${dest}.tmp-${process.pid}`; + fs.writeFileSync(tmp, content); + fs.renameSync(tmp, dest); +} + export async function applyService(svc: ManagedService, config: SiteConfig): Promise { const files = await svc.render(config); if (!files) return 'success'; @@ -103,13 +112,13 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); - fs.writeFileSync(f.dest, f.content); + writeFileAtomic(f.dest, f.content); } const rollback = () => { for (const [dest, prev] of current) { if (prev === null) fs.rmSync(dest, { force: true }); - else fs.writeFileSync(dest, prev); + else writeFileAtomic(dest, prev); } }; diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx index 20056e94..96bbfbb1 100644 --- a/create-a-container/client/src/pages/agents/AgentsListPage.tsx +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -2,7 +2,6 @@ import { useQuery } from '@tanstack/react-query'; import { Alert, AlertDescription, - Badge, PageHeader, Spinner, Table, @@ -18,9 +17,7 @@ import { keys, queries } from '@/lib/queries'; import type { Agent } from '@/lib/types'; import { useDocumentTitle } from '@/lib/useDocumentTitle'; import { AgentServiceBadges } from './AgentServiceBadges'; - -// Agents check in every 30 seconds; three missed intervals means offline. -const OFFLINE_AFTER_SECONDS = 90; +import { OnlineBadge } from './OnlineBadge'; function formatLastCheckin(agent: Agent): string { const seconds = agent.secondsSinceCheckin; @@ -31,16 +28,6 @@ function formatLastCheckin(agent: Agent): string { return new Date(agent.lastCheckinAt).toLocaleString(); } -function OnlineBadge({ agent }: { agent: Agent }) { - const online = - agent.secondsSinceCheckin !== null && agent.secondsSinceCheckin <= OFFLINE_AFTER_SECONDS; - return online ? ( - Online - ) : ( - Offline - ); -} - export function AgentsListPage() { useDocumentTitle('Agents'); const { data, isLoading, error } = useQuery({ diff --git a/create-a-container/client/src/pages/agents/OnlineBadge.tsx b/create-a-container/client/src/pages/agents/OnlineBadge.tsx new file mode 100644 index 00000000..b39c0a22 --- /dev/null +++ b/create-a-container/client/src/pages/agents/OnlineBadge.tsx @@ -0,0 +1,15 @@ +import { Badge } from '@mieweb/ui'; +import type { Agent } from '@/lib/types'; + +// Agents check in every 30 seconds; three missed intervals means offline. +const OFFLINE_AFTER_SECONDS = 90; + +export function OnlineBadge({ agent }: { agent: Agent }) { + const online = + agent.secondsSinceCheckin !== null && agent.secondsSinceCheckin <= OFFLINE_AFTER_SECONDS; + return online ? ( + Online + ) : ( + Offline + ); +} diff --git a/create-a-container/middlewares/index.js b/create-a-container/middlewares/index.js index b1023ac2..b2f8654a 100644 --- a/create-a-container/middlewares/index.js +++ b/create-a-container/middlewares/index.js @@ -74,7 +74,7 @@ function requireAdmin(req, res, next) { } // True when the request comes directly from localhost (and was not proxied -// on behalf of a remote client, per X-Real-IP). +// on behalf of a remote client, per X-Real-IP / X-Forwarded-For). function isLocalhostRequest(req) { const isLocalhost = (ip) => { return ip === '127.0.0.1' || @@ -88,23 +88,13 @@ function isLocalhostRequest(req) { req.ip; const realIp = req.get('X-Real-IP'); + // First hop of X-Forwarded-For is the original client (covers proxies that + // set it without also setting X-Real-IP). + const forwardedFor = (req.get('X-Forwarded-For') || '').split(',')[0].trim(); - return isLocalhost(directIp) && (!realIp || isLocalhost(realIp)); -} - -// Localhost-or-admin middleware -// Allows localhost requests through without auth. Remote requests must authenticate -// as an admin user (via session or API key). -function requireLocalhostOrAdmin(req, res, next) { - if (isLocalhostRequest(req)) { - return next(); - } - - // Not localhost — require auth + admin - requireAuth(req, res, (err) => { - if (err) return next(err); - requireAdmin(req, res, next); - }); + return isLocalhost(directIp) + && (!realIp || isLocalhost(realIp)) + && (!forwardedFor || isLocalhost(forwardedFor)); } const { setCurrentSite, loadSites } = require('./currentSite'); @@ -114,7 +104,6 @@ module.exports = { isLocalhostRequest, requireAuth, requireAdmin, - requireLocalhostOrAdmin, setCurrentSite, loadSites }; diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 2076f60c..84b6b45a 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -9,7 +9,7 @@ const express = require('express'); const { Agent, Site } = require('../../../models'); -const { requireLocalhostOrAdmin } = require('../../../middlewares'); +const { isLocalhostRequest } = require('../../../middlewares'); const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); @@ -17,8 +17,17 @@ const router = express.Router(); // Check-in auth: the manager's own agent checks in over localhost without // credentials (bootstrap: no site, no API key exist yet); remote agents -// authenticate with an admin API key. -router.post('/', requireLocalhostOrAdmin, asyncHandler(async (req, res) => { +// authenticate with an admin API key via apiAuth/apiAdmin so error +// responses follow the v1 JSON envelope. +function checkinAuth(req, res, next) { + if (isLocalhostRequest(req)) return next(); + return apiAuth(req, res, (err) => { + if (err) return next(err); + return apiAdmin(req, res, next); + }); +} + +router.post('/', checkinAuth, asyncHandler(async (req, res) => { const { siteId, hostname, ipv4Address, services } = req.body || {}; const parsedSiteId = typeof siteId === 'number' ? siteId : Number(siteId); if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') {