diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43360594..8b43cbba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release Packages # Triggered by publishing a GitHub release (full or prerelease). Builds the -# three Debian packages and uploads them to the release that triggered the +# four Debian packages and uploads them to the release that triggered the # workflow, together with flat APT repository metadata (Packages, Packages.gz) # so the release can be used directly as an apt source: # @@ -50,7 +50,7 @@ jobs: gzip -k9f Packages # Stable-name aliases for one-off downloads (the _latest URLs resolve # via releases/latest/download for the newest non-prerelease release). - for pkg in opensource-server opensource-docs opensource-agent; do + for pkg in opensource-server opensource-docs opensource-agent opensource-mcp; do f=$(ls ${pkg}_*.deb | head -1) [ -n "$f" ] && cp -f "$f" "${pkg}_latest.deb" done diff --git a/Makefile b/Makefile index 2789cb02..b1da4341 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPONENTS := agent mie-opensource-landing create-a-container +COMPONENTS := agent mie-opensource-landing manager-control-program create-a-container PACKAGER ?= deb # Forwarded to every component Makefile. diff --git a/agent/contrib/systemd/opensource-agent.service b/agent/contrib/systemd/opensource-agent.service index ff2d2814..a7e14eff 100644 --- a/agent/contrib/systemd/opensource-agent.service +++ b/agent/contrib/systemd/opensource-agent.service @@ -8,6 +8,7 @@ After=network-online.target environment.service [Service] Type=oneshot # Runtime config (SITE_ID, MANAGER_URL, API_KEY) lives in /etc/environment. +# Optional LOG_LEVEL (error|warn|info|debug; default info) also read from there. EnvironmentFile=-/etc/environment # systemd creates /var/lib/opensource-agent and passes it as $STATE_DIRECTORY. StateDirectory=opensource-agent diff --git a/agent/src/api.ts b/agent/src/api.ts index ef1b6b0f..1aa0438a 100644 --- a/agent/src/api.ts +++ b/agent/src/api.ts @@ -2,6 +2,7 @@ import type { AgentConfig } from './config'; import type { CheckinRequest, SiteConfig } from './types'; +import { log } from './log'; export type CheckinResult = | { notModified: true } @@ -20,15 +21,34 @@ export async function checkin( if (etag) headers['If-None-Match'] = etag; if (cfg.apiKey) headers['Authorization'] = `Bearer ${cfg.apiKey}`; - const res = await fetch(`${cfg.managerUrl}/api/v1/agents`, { - method: 'POST', - headers, - body: JSON.stringify(body), - signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), - }); + const url = `${cfg.managerUrl}/api/v1/agents`; + log.debug(`check-in POST ${url}${etag ? ` (If-None-Match: ${etag})` : ''}${cfg.apiKey ? ' with API key' : ''}`); + let res: Response; + try { + res = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), + }); + } catch (err) { + // fetch rejects on network errors and on the abort timeout; distinguish + // the timeout so a stalled manager is obvious in the logs. + const reason = + (err as Error)?.name === 'TimeoutError' + ? `no response within ${CHECKIN_TIMEOUT_MS} ms` + : (err as Error)?.message ?? String(err); + log.error(`check-in request to ${url} failed: ${reason}`); + throw err; + } + + log.debug(`check-in response: HTTP ${res.status}`); if (res.status === 304) return { notModified: true }; - if (!res.ok) throw new Error(`Check-in failed: HTTP ${res.status}`); + if (!res.ok) { + log.error(`check-in rejected by manager: HTTP ${res.status}`); + throw new Error(`Check-in failed: HTTP ${res.status}`); + } const { data } = (await res.json()) as { data: SiteConfig }; return { diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 4a518d53..a83c67ef 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -11,6 +11,7 @@ import path from 'path'; import { execFileSync } from 'child_process'; import ejs from 'ejs'; import { reloadOrRestartService, restartService, sighupService } from './system'; +import { log, commandOutput } from './log'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); @@ -36,8 +37,12 @@ function renderTemplate(template: string, data: object): Promise { return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); } -function run(cmd: string[]): void { - execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe' }); +// Run a command, capturing its output. execFileSync with stdio 'pipe' attaches +// the captured stdout/stderr to the thrown error on failure (see commandOutput +// in log.ts), so callers can surface e.g. exactly why `nginx -t` rejected a +// config. Returns the combined stdout for successful runs. +function run(cmd: string[]): string { + return execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe', encoding: 'utf8' }); } export const services: ManagedService[] = [ @@ -102,17 +107,27 @@ function writeFileAtomic(dest: string, content: string): void { } export async function applyService(svc: ManagedService, config: SiteConfig): Promise { + log.debug(`${svc.unit}: rendering config`); const files = await svc.render(config); - if (!files) return 'success'; + if (!files) { + log.debug(`${svc.unit}: nothing to manage yet, skipping`); + return 'success'; + } const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); - if (changed.length === 0) return 'success'; + if (changed.length === 0) { + log.debug(`${svc.unit}: config unchanged, nothing to do`); + return 'success'; + } + + log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`); // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); writeFileAtomic(f.dest, f.content); + log.debug(`${svc.unit}: wrote ${f.dest}`); } const rollback = () => { @@ -120,30 +135,45 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro if (prev === null) fs.rmSync(dest, { force: true }); else writeFileAtomic(dest, prev); } + log.debug(`${svc.unit}: rolled back to previous config`); }; if (svc.test) { + const testCmd = svc.test.join(' '); + log.debug(`${svc.unit}: testing config with \`${testCmd}\``); try { - run(svc.test); + const out = run(svc.test); + const trimmed = out.trim(); + if (trimmed) log.debug(`${svc.unit}: config test output:\n${trimmed}`); + log.info(`${svc.unit}: config test passed`); } catch (err) { rollback(); - console.error(`${svc.unit}: config test failed, rolled back: ${(err as Error).message}`); + // Surface the actual validator output (e.g. the nginx line/column that + // was rejected) — that's the whole point of running the test, and it's + // otherwise lost since the command's stdio is piped. + const output = commandOutput(err); + log.error(`${svc.unit}: config test failed (\`${testCmd}\`), rolled back: ${(err as Error).message}`); + if (output) log.error(`${svc.unit}: config test output:\n${output}`); return 'failure'; } } try { + log.debug(`${svc.unit}: reloading service`); await svc.reload(changed); + log.info(`${svc.unit}: reloaded`); } catch (err) { // The new config passed its test but the service couldn't pick it up: // restore the previous files and reload again (best effort) so the // service keeps running the last known-good config. rollback(); - await svc.reload(changed).catch(() => { /* reported via service state at next check-in */ }); - console.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); + await svc.reload(changed).catch((reErr) => { + log.error(`${svc.unit}: reload after rollback also failed: ${(reErr as Error).message}`); + }); + log.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); return 'failure'; } - console.log(`${svc.unit}: applied ${changed.length} file(s)`); + log.info(`${svc.unit}: applied ${changed.length} file(s)`); return 'success'; } diff --git a/agent/src/config.ts b/agent/src/config.ts index d01c91f3..27b93695 100644 --- a/agent/src/config.ts +++ b/agent/src/config.ts @@ -1,14 +1,19 @@ /** Agent configuration, read from the environment (systemd passes * /etc/environment through EnvironmentFile=). */ +import { setLogLevel, type LogLevel } from './log'; + export interface AgentConfig { siteId: number; managerUrl: string; apiKey?: string; stateDir: string; + logLevel: LogLevel; } export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { + // Set the log level first so any warnings below (and from callers) honor it. + const logLevel = setLogLevel(env.LOG_LEVEL); const siteId = parseInt(env.SITE_ID ?? '', 10); const managerUrl = env.MANAGER_URL; if (!Number.isInteger(siteId) || !managerUrl) { @@ -20,5 +25,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { apiKey: env.API_KEY || undefined, // Set by systemd from StateDirectory=; fallback for manual runs. stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', + logLevel, }; } diff --git a/agent/src/index.ts b/agent/src/index.ts index 5d2a2d9b..071e6faa 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -16,6 +16,7 @@ import { State } from './state'; import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; import { checkin } from './api'; import { services, applyService } from './apply'; +import { log } from './log'; import type { CheckinRequest, ServiceStatus } from './types'; // Safety cap: a flapping server-side config can't keep a single run alive @@ -42,11 +43,18 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const cfg = loadConfig(); const state = State.load(cfg.stateDir); + log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}`); + log.debug(`state dir=${cfg.stateDir}, saved etag=${state.etag ?? '(none)'}`); for (let pass = 0; pass < MAX_PASSES; pass++) { + log.debug(`check-in pass ${pass + 1}/${MAX_PASSES}`); const result = await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); - if (result.notModified) return; + if (result.notModified) { + log.info('check-in: config unchanged (304), nothing to apply'); + return; + } + log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); for (const svc of services) { state.lastApply[svc.unit] = await applyService(svc, result.config); } @@ -61,12 +69,15 @@ async function main(): Promise { // MAX_PASSES exhausted (flapping server-side config): check in once more so // the final pass' apply results reach the manager instead of going stale // until the next timer run. + log.warn(`reached MAX_PASSES (${MAX_PASSES}) without a stable config; reporting final results`); await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); } main() .then(() => disconnectSystemBus()) .catch((err) => { - console.error(err instanceof Error ? err.message : err); + // Log the full error (stack included when present) so a failed run is + // diagnosable from the journal, not just a one-line message. + log.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); process.exit(1); }); diff --git a/agent/src/log.ts b/agent/src/log.ts new file mode 100644 index 00000000..c902e2f7 --- /dev/null +++ b/agent/src/log.ts @@ -0,0 +1,74 @@ +/** + * Minimal leveled logger for the agent. + * + * The agent runs as a systemd oneshot, so everything written to stdout/stderr + * lands in the journal (view with `journalctl -u opensource-agent`). We keep a + * tiny hand-rolled logger rather than pull in a dependency: levels gate what + * gets emitted, error/warn go to stderr and info/debug to stdout, and each + * line is prefixed with its level so journal output is greppable. + * + * The active level comes from LOG_LEVEL (case-insensitive; default "info"). + * An unknown value falls back to "info" with a warning so a typo can't + * silently mute the agent. + */ + +export type LogLevel = 'error' | 'warn' | 'info' | 'debug'; + +const LEVELS: Record = { + error: 0, + warn: 1, + info: 2, + debug: 3, +}; + +let activeLevel: LogLevel = 'info'; + +/** + * Set the active level from a raw env string. Returns the level actually + * applied. Call once at startup from loadConfig(); defaults to "info". + */ +export function setLogLevel(raw: string | undefined): LogLevel { + if (!raw) { + activeLevel = 'info'; + return activeLevel; + } + const normalized = raw.trim().toLowerCase(); + if (normalized in LEVELS) { + activeLevel = normalized as LogLevel; + } else { + activeLevel = 'info'; + log.warn(`Unknown LOG_LEVEL "${raw}", defaulting to "info"`); + } + return activeLevel; +} + +function emit(level: LogLevel, message: string): void { + if (LEVELS[level] > LEVELS[activeLevel]) return; + const line = `[${level}] ${message}`; + if (level === 'error' || level === 'warn') { + console.error(line); + } else { + console.log(line); + } +} + +export const log = { + error: (message: string) => emit('error', message), + warn: (message: string) => emit('warn', message), + info: (message: string) => emit('info', message), + debug: (message: string) => emit('debug', message), +}; + +/** + * Format an error's command output (execFileSync attaches captured stdout / + * stderr to the thrown error when stdio is 'pipe'). Returns a trimmed, + * human-readable blob or null when there is nothing captured — used to surface + * the actual reason a command like `nginx -t` rejected a config. + */ +export function commandOutput(err: unknown): string | null { + const e = err as { stdout?: Buffer | string; stderr?: Buffer | string }; + const parts = [e?.stdout, e?.stderr] + .map((part) => (part == null ? '' : part.toString()).trim()) + .filter((part) => part.length > 0); + return parts.length > 0 ? parts.join('\n') : null; +} diff --git a/agent/src/state.ts b/agent/src/state.ts index 3e3cdbf4..93ae019e 100644 --- a/agent/src/state.ts +++ b/agent/src/state.ts @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; +import { log } from './log'; import type { ApplyResult } from './types'; export class State { @@ -27,7 +28,7 @@ export class State { } catch (err) { if (!(err instanceof SyntaxError)) throw err; // A corrupt state file just means a full re-apply on this run. - console.error(`Ignoring unparsable state file ${state.file}: ${err.message}`); + log.warn(`Ignoring unparsable state file ${state.file}: ${err.message}`); } return state; } diff --git a/compose.yml b/compose.yml index f904fb26..c9e25859 100644 --- a/compose.yml +++ b/compose.yml @@ -77,6 +77,25 @@ services: retries: 60 start_period: 5s + # Assembles the MCP server's runtime tree (the app package plus all vendored + # Python dependencies) into manager-control-program/build via `make build`. + # The opensource-mcp.service inside Proxmox runs `python3 -m + # manager_control_program.server` with PYTHONPATH=/opt/opensource-server/ + # manager-control-program/build, but the repo is mounted read-only there and + # the vendored deps aren't checked in — so, like the `node`/`client` + # services, this host-side step produces the build output first (repo mounted + # read-write); the read-only bind mount inside Proxmox then sees it at the + # same path the package installs to. Runs once and exits. + mcp: + image: astral/uv:0.11.14-trixie + volumes: + - ./:/opt/opensource-server + working_dir: /opt/opensource-server + entrypoint: ["/bin/sh", "-c"] + command: + - | + make -C manager-control-program build + # This service will watch the documentation for any changes and rebuild when # needed. Technically it's listening on port 8000, but we only care about the # HTML being placed in ./mie-opensource-landing/site which is shared with the @@ -136,6 +155,8 @@ services: condition: service_completed_successfully node: condition: service_completed_successfully + mcp: + condition: service_completed_successfully client: condition: service_healthy diff --git a/create-a-container/.fpm b/create-a-container/.fpm index baa9b50a..e4e212e9 100644 --- a/create-a-container/.fpm +++ b/create-a-container/.fpm @@ -8,6 +8,7 @@ --description "MIE opensource-server cluster manager (create-a-container): web UI and REST API for self-service LXC container hosting on Proxmox VE, with automated DNS and reverse-proxy configuration, LDAP authentication and ACME TLS orchestration." --depends opensource-agent --depends opensource-docs +--depends opensource-mcp --depends nodejs --depends sudo --depends libc6 diff --git a/create-a-container/README.md b/create-a-container/README.md index cedcdbb8..c20ae675 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -46,7 +46,10 @@ The Manager is not installed by hand in production. It ships as: - distribution **packages** built from this directory with `make deb`, `make rpm`, or `make apk` (via [fpm](https://fpm.readthedocs.io/)), which install the app under `/opt/opensource-server/create-a-container` and register the - `container-creator` and `job-runner` systemd services. + `container-creator` and `job-runner` systemd services. The package depends + on `opensource-mcp` — the [MCP server](../manager-control-program/) as its + own package — and the Manager reverse-proxies `/mcp` to its service + (`MCP_SERVER_URL`). In both cases the app runs `server.js` (HTTP API + UI) and `job-runner.js` (background worker). Database connection settings come from the environment (see diff --git a/create-a-container/app.js b/create-a-container/app.js index 3f714236..e94018cc 100644 --- a/create-a-container/app.js +++ b/create-a-container/app.js @@ -26,8 +26,16 @@ const { sequelize } = require('./models'); * @param {boolean} [options.rateLimit=true] - disable in tests: assertions on 4xx * responses must not burn the budget. * @param {boolean} [options.accessLog=true] - morgan; disable in tests for quiet output. + * @param {string} [options.mcpServerUrl] - origin of the MCP server to reverse-proxy + * at /mcp (default: $MCP_SERVER_URL). Unset + * disables the proxy. */ -function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { +function buildApp({ + sessionSecrets, + rateLimit = true, + accessLog = true, + mcpServerUrl = process.env.MCP_SERVER_URL, +} = {}) { if (!sessionSecrets || sessionSecrets.length === 0) { throw new Error('buildApp: sessionSecrets is required'); } @@ -46,6 +54,16 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { : process.stdout; app.use(morgan('combined', { stream: accessLogStream })); } + // --- MCP reverse proxy --- + // Mounted before the body parsers so request bodies stream through + // untouched, and before session/rate-limit/CSRF: MCP clients authenticate + // per-request with Bearer API keys (forwarded by the MCP server back to + // /api/v1), not cookies, so none of the cookie-oriented middleware applies. + if (mcpServerUrl) { + const { createMcpProxy } = require('./middlewares/mcp-proxy'); + app.use('/mcp', createMcpProxy(mcpServerUrl)); + } + app.use(express.json()); app.use(express.urlencoded({ extended: true })); diff --git a/create-a-container/contrib/systemd/container-creator.service b/create-a-container/contrib/systemd/container-creator.service index f1ed7eba..710fab6e 100644 --- a/create-a-container/contrib/systemd/container-creator.service +++ b/create-a-container/contrib/systemd/container-creator.service @@ -11,6 +11,8 @@ ExecStart=/usr/bin/node /opt/opensource-server/create-a-container/server.js Restart=on-failure Environment=NODE_ENV=production Environment=ACCESS_LOG=/var/log/opensource-server/access.log +# Runtime configuration (DB connection, MCP_SERVER_URL for the /mcp reverse +# proxy to opensource-mcp.service, etc.) is sourced from this file. EnvironmentFile=/etc/default/container-creator LogsDirectory=opensource-server diff --git a/create-a-container/example.env b/create-a-container/example.env index 93a7a35f..9c6e8ab5 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -28,6 +28,13 @@ POSTGRES_DATABASE= # if set, the file is opened in append mode. ACCESS_LOG= +# --- MCP server reverse proxy (optional) --- +# When set, the Manager reverse-proxies /mcp to this origin — the +# manager-control-program MCP server in HTTP mode. The packaged systemd +# deployment sets this to http://127.0.0.1:8100 (see opensource-mcp.service); +# leave unset to disable the proxy. +MCP_SERVER_URL= + # --- OIDC / single sign-on (optional) --- # SSO is enabled only when OIDC_ISSUER_URL, OIDC_CLIENT_ID, and # OIDC_CLIENT_SECRET are all set. When enabled, the login page redirects to the diff --git a/create-a-container/middlewares/__tests__/mcp-proxy.test.js b/create-a-container/middlewares/__tests__/mcp-proxy.test.js new file mode 100644 index 00000000..177d99d6 --- /dev/null +++ b/create-a-container/middlewares/__tests__/mcp-proxy.test.js @@ -0,0 +1,139 @@ +/** + * Tests for the /mcp reverse proxy: requests stream through to the configured + * MCP server with headers (notably Authorization) and bodies intact, responses + * come back verbatim (including SSE content types), and the proxy degrades to + * a 502 JSON error when the MCP server is down. When no MCP server is + * configured the route is not mounted at all. + */ + +const http = require('http'); +const request = require('supertest'); +const { buildApp } = require('../../app'); +const { resetDb, closeDb } = require('../../tests/helpers/db'); + +/** Stub MCP upstream: echoes requests, and speaks SSE on GET. */ +function startUpstream() { + const server = http.createServer((req, res) => { + if (req.method === 'GET') { + res.writeHead(200, { 'content-type': 'text/event-stream' }); + res.write('event: message\ndata: {"hello":"one"}\n\n'); + res.write('event: message\ndata: {"hello":"two"}\n\n'); + res.end(); + return; + } + let body = ''; + req.on('data', (chunk) => (body += chunk)); + req.on('end', () => { + res.writeHead(201, { + 'content-type': 'application/json', + 'mcp-session-id': 'stub-session-1', + }); + res.end( + JSON.stringify({ + method: req.method, + url: req.url, + authorization: req.headers.authorization || null, + contentType: req.headers['content-type'] || null, + body, + }), + ); + }); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(server)); + }); +} + +function build(mcpServerUrl) { + return buildApp({ + sessionSecrets: ['test-secret'], + rateLimit: false, + accessLog: false, + mcpServerUrl, + }); +} + +describe('/mcp reverse proxy', () => { + let upstream; + let app; + let deadApp; + let bareApp; + + beforeAll(async () => { + await resetDb(); + upstream = await startUpstream(); + const { port } = upstream.address(); + // Build every app variant here, right after resetDb: each buildApp + // constructs a session store that fires an un-awaited CREATE TABLE, and + // constructing one mid-test races that query against closeDb() (see + // tests/helpers/app.js). + app = build(`http://127.0.0.1:${port}`); + const deadPort = await new Promise((resolve) => { + const s = http.createServer(); + s.listen(0, '127.0.0.1', () => { + const p = s.address().port; + s.close(() => resolve(p)); + }); + }); + deadApp = build(`http://127.0.0.1:${deadPort}`); + bareApp = build(undefined); + }); + + afterAll(async () => { + await new Promise((resolve) => upstream.close(resolve)); + await closeDb(); + }); + + test('forwards POST body, path, and Authorization header; returns upstream response', async () => { + const payload = { jsonrpc: '2.0', method: 'initialize', id: 1 }; + const res = await request(app) + .post('/mcp') + .set('Authorization', 'Bearer caller-token') + .set('Content-Type', 'application/json') + .send(payload); + + expect(res.status).toBe(201); + expect(res.headers['mcp-session-id']).toBe('stub-session-1'); + expect(res.body).toEqual({ + method: 'POST', + url: '/mcp', + authorization: 'Bearer caller-token', + contentType: 'application/json', + body: JSON.stringify(payload), + }); + }); + + test('preserves the query string', async () => { + const res = await request(app).post('/mcp?foo=bar').send({}); + expect(res.status).toBe(201); + expect(res.body.url).toBe('/mcp?foo=bar'); + }); + + test('passes SSE responses through with their content type', async () => { + const res = await request(app).get('/mcp').buffer(true).parse( + // supertest has no default parser for text/event-stream; collect raw. + (msg, cb) => { + let data = ''; + msg.on('data', (chunk) => (data += chunk)); + msg.on('end', () => cb(null, data)); + }, + ); + expect(res.status).toBe(200); + expect(res.headers['content-type']).toBe('text/event-stream'); + expect(res.body).toContain('data: {"hello":"one"}'); + expect(res.body).toContain('data: {"hello":"two"}'); + }); + + test('502 with a JSON error when the MCP server is unreachable', async () => { + const res = await request(deadApp).post('/mcp').send({}); + expect(res.status).toBe(502); + expect(res.body.error.code).toBe('mcp_unavailable'); + }); + + test('not mounted when no MCP server is configured', async () => { + // Unproxied POST /mcp falls through to the app-level CSRF guard. + const res = await request(bareApp).post('/mcp').send({}); + expect(res.status).toBe(403); + expect(res.headers['mcp-session-id']).toBeUndefined(); + }); +}); diff --git a/create-a-container/middlewares/api.js b/create-a-container/middlewares/api.js index 6a95efda..ee4e6205 100644 --- a/create-a-container/middlewares/api.js +++ b/create-a-container/middlewares/api.js @@ -36,6 +36,16 @@ function csrfGuard(req, res, next) { if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') { return next(); } + // The manager's own agent checks in over localhost without credentials + // during bootstrap (no site or API key exists yet), so it can carry neither + // a Bearer token nor a CSRF token. The agents router applies the same + // localhost bypass at the route level (checkinAuth); mirror it here so this + // app-level guard doesn't reject the credential-less check-in first. Remote + // clients are never localhost (isLocalhostRequest also rejects proxied + // requests via X-Real-IP / X-Forwarded-For). Required lazily to avoid a + // load-order cycle with middlewares/index. + const { isLocalhostRequest } = require('./index'); + if (isLocalhostRequest(req)) return next(); const auth = req.get('Authorization') || ''; const hasBearer = auth.startsWith('Bearer '); const hasSessionCookie = !!(req.session && req.session.user); diff --git a/create-a-container/middlewares/mcp-proxy.js b/create-a-container/middlewares/mcp-proxy.js new file mode 100644 index 00000000..67ad84a4 --- /dev/null +++ b/create-a-container/middlewares/mcp-proxy.js @@ -0,0 +1,93 @@ +/** + * Streaming reverse proxy for the MCP server (manager-control-program). + * + * The packaged MCP server listens on loopback (see + * opensource-mcp.service); the Manager exposes it at /mcp on its + * public origin so MCP clients get TLS and a stable hostname for free, and + * per-request Authorization headers flow: MCP client -> this proxy -> MCP + * server -> back to this app's /api/v1 as a Bearer API key. + * + * Hand-rolled on node:http instead of a proxy dependency because the needs + * are narrow but strict: + * - bodies must pass through untouched (mount BEFORE express.json, which + * would otherwise consume the stream); + * - responses must stream incrementally (MCP's streamable HTTP transport + * uses long-lived text/event-stream responses); + * - no timeout on the upstream socket (SSE streams idle between events). + */ + +const http = require('http'); +const https = require('https'); + +// Hop-by-hop headers are connection-scoped and must not be forwarded +// (RFC 9110 §7.6.1). `host` is excluded so node sets it from the target. +const HOP_BY_HOP = new Set([ + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'host', +]); + +function filterHeaders(headers) { + const out = {}; + for (const [name, value] of Object.entries(headers)) { + if (!HOP_BY_HOP.has(name.toLowerCase())) out[name] = value; + } + return out; +} + +/** + * @param {string} targetBase Origin of the MCP server, e.g. "http://127.0.0.1:8100". + * The request path is preserved (/mcp -> ${targetBase}/mcp). + * @returns {import('express').RequestHandler} + */ +function createMcpProxy(targetBase) { + const base = new URL(targetBase); + const client = base.protocol === 'https:' ? https : http; + const agent = new client.Agent({ keepAlive: true }); + + return function mcpProxy(req, res) { + // req.originalUrl is the full path+query as received (req.url would have + // the /mcp mount prefix stripped by express). + const target = new URL(req.originalUrl, base); + + const upstream = client.request( + target, + { method: req.method, headers: filterHeaders(req.headers), agent }, + (upstreamRes) => { + res.writeHead(upstreamRes.statusCode, filterHeaders(upstreamRes.headers)); + // flushHeaders so SSE clients see the stream open before any event. + res.flushHeaders(); + upstreamRes.pipe(res); + }, + ); + + // SSE responses idle between events; never time the socket out. + upstream.setTimeout(0); + + upstream.on('error', () => { + if (res.headersSent) { + res.destroy(); + return; + } + res.writeHead(502, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + error: { code: 'mcp_unavailable', message: 'MCP server is unreachable' }, + }), + ); + }); + + // Client gone (or response finished) — release the upstream socket. + res.on('close', () => upstream.destroy()); + + req.pipe(upstream); + }; +} + +module.exports = { createMcpProxy }; diff --git a/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js new file mode 100644 index 00000000..8c95751e --- /dev/null +++ b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js @@ -0,0 +1,56 @@ +/** + * Agent check-in auth — the manager's own agent bootstraps over localhost with + * no site row and no API key, so its POST /api/v1/agents carries neither a + * Bearer token nor a CSRF token. Two guards must let it through: the app-level + * csrfGuard (app.js) and the route-level checkinAuth (agents.js). This pins + * that the credential-less localhost check-in is NOT rejected with 403, while + * a non-localhost (proxied) credential-less check-in still is. + * + * supertest connects from 127.0.0.1, so requests are localhost by default; + * an X-Forwarded-For header with a public client IP makes isLocalhostRequest + * treat the request as proxied-remote (see middlewares/index.js). + */ + +const request = require('supertest'); +const { buildApp } = require('../../../../tests/helpers/app'); +const { resetDb, closeDb } = require('../../../../tests/helpers/db'); + +describe('POST /api/v1/agents check-in auth', () => { + let app; + + beforeEach(async () => { + await resetDb(); + app = buildApp(); + }); + + afterAll(async () => { + await closeDb(); + }); + + test('localhost bootstrap check-in (no Bearer, no CSRF token) is not blocked by CSRF', async () => { + const res = await request(app) + .post('/api/v1/agents') + .send({ siteId: 1, hostname: 'manager.local' }); + + // The site row doesn't exist yet (bootstrap), so the handler skips the + // Agent write and returns the config snapshot. The key assertion is that + // we reached the handler at all — not a 403 from either csrf guard. + expect(res.status).not.toBe(403); + expect(res.status).toBe(200); + expect(res.body.data).toBeDefined(); + }); + + test('non-localhost check-in without credentials is rejected', async () => { + const res = await request(app) + .post('/api/v1/agents') + // A public client IP in X-Forwarded-For marks the request as proxied, + // so the localhost bypass does not apply. + .set('X-Forwarded-For', '203.0.113.7') + .send({ siteId: 1, hostname: 'remote.example.com' }); + + // Either the app-level CSRF guard (403 csrf_invalid) or the route-level + // apiAuth (401 unauthorized) rejects it; the point is it does not succeed. + expect(res.status).not.toBe(200); + expect([401, 403]).toContain(res.status); + }); +}); diff --git a/images/builder/Dockerfile b/images/builder/Dockerfile index a44ce17d..47180ff5 100644 --- a/images/builder/Dockerfile +++ b/images/builder/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Builds the three opensource-server Debian packages with the component +# Builds the four opensource-server Debian packages with the component # Makefiles + fpm. The final stage contains only the built .deb artifacts so # the runtime images can install them with: # @@ -44,7 +44,7 @@ RUN gem install --no-document fpm -v "${FPM_VERSION}" COPY . /usr/src/opensource-server WORKDIR /usr/src/opensource-server -# Build all three packages into ./dist. +# Build all packages into ./dist. RUN make deb # Artifact-only stage. diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index b76bdfc4..d8d7a38a 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -26,13 +26,13 @@ COPY images/manager/wait-proxmox-regenerate-snakeoil.conf /etc/systemd/system/po # First-boot database initialization. COPY images/manager/container-creator-init.service /etc/systemd/system/container-creator-init.service -# Install the manager and docs packages built by the builder image (the agent -# package is already present from the parent image; apt resolves the -# opensource-server -> opensource-docs dependency), then enable the init -# service. +# Install the manager, docs, and MCP packages built by the builder image (the +# agent package is already present from the parent image; apt resolves the +# opensource-server -> opensource-docs/opensource-mcp dependencies), then +# enable the init service. RUN --mount=from=builder,source=/dist,target=/dist \ apt-get update && \ - apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-server_*.deb && \ + apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-mcp_*.deb /dist/opensource-server_*.deb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ systemctl enable container-creator-init.service diff --git a/images/manager/container-creator-init.service b/images/manager/container-creator-init.service index e898e2ee..32082007 100644 --- a/images/manager/container-creator-init.service +++ b/images/manager/container-creator-init.service @@ -22,7 +22,8 @@ ExecStart=/bin/bash -e -c '\ echo "POSTGRES_HOST=$${POSTGRES_HOST}" >> /etc/default/container-creator; \ echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> /etc/default/container-creator; \ echo "POSTGRES_USER=$${POSTGRES_USER}" >> /etc/default/container-creator; \ - echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator;' + echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator; \ + echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator;' [Install] WantedBy=multi-user.target diff --git a/manager-control-program/.fpm b/manager-control-program/.fpm new file mode 100644 index 00000000..fb250ced --- /dev/null +++ b/manager-control-program/.fpm @@ -0,0 +1,14 @@ +--name opensource-mcp +--architecture amd64 +--license Apache-2.0 +--maintainer "Medical Informatics Engineering " +--vendor "Medical Informatics Engineering" +--url "https://github.com/mieweb/opensource-server" +--category admin +--description "MIE opensource-server MCP server (manager-control-program): exposes the create-a-container REST API as Model Context Protocol tools over HTTP, forwarding each caller's Authorization header so every MCP client acts under its own API key. Python dependencies are vendored; runs on the system python3." +--depends "python3 >= 3.13" +--depends "python3 < 3.14" +--config-files /etc/default/opensource-mcp +--deb-no-default-config-files +--after-install contrib/postinstall.sh +--before-remove contrib/preremove.sh diff --git a/manager-control-program/.gitignore b/manager-control-program/.gitignore index 532a9421..cdae812c 100644 --- a/manager-control-program/.gitignore +++ b/manager-control-program/.gitignore @@ -3,3 +3,10 @@ build/ *.egg-info/ __pycache__/ *.pyc + +# packaging build artifacts +/.pkg/ +/*.deb +/*.rpm +/*.apk +.pkg-requirements.txt diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile new file mode 100644 index 00000000..c9c1beaa --- /dev/null +++ b/manager-control-program/Makefile @@ -0,0 +1,114 @@ +.DEFAULT_GOAL := help + +PREFIX ?= /opt/opensource-server +DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/manager-control-program +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system + +# `build` assembles the ready-to-run application tree (the app package plus all +# vendored Python dependencies) into BUILDDIR in the source tree; `install` +# then copies that tree verbatim into DESTBIN. So the dev bind mount and the +# packaged install serve byte-for-byte the same layout, runnable with only the +# system python3 (PYTHONPATH pointed at it — no uv, pip, venv, or network). +BUILDDIR := $(CURDIR)/build + +INSTALL := install +INSTALL_DATA := $(INSTALL) -m 0644 + +# The production target is Debian 13 / amd64 / CPython 3.13. uv selects +# wheels for that platform regardless of the build host's own interpreter, +# and downloads a managed CPython for resolution if the host has none (the +# builder image doesn't). +# +# Compiled wheels are ABI-locked to this exact minor version (cp313), so the +# opensource-mcp package pins its python3 dependency to the matching range +# in .fpm — update both together when bumping. +PYTHON_VERSION := 3.13 +PYTHON_PLATFORM := x86_64-unknown-linux-gnu + +REQUIREMENTS := .pkg-requirements.txt + +# Staging directory for packaging +STAGE := $(CURDIR)/.pkg/buildroot + +.PHONY: help deps build dev test install deb rpm apk package clean + +help: + @echo "manager-control-program — builds the opensource-mcp package." + @echo "" + @echo "Targets:" + @echo " deps install dependencies (uv sync)" + @echo " build assemble the app + vendored deps tree into build/" + @echo " dev run the MCP server locally over stdio (uv)" + @echo " test run tests (none; kept for the repo-wide target set)" + @echo " install copy the build/ tree into DESTDIR" + @echo " deb build the .deb package" + @echo " rpm build the .rpm package" + @echo " apk build the .apk package" + @echo " clean remove the venv, build/, staging dir and built packages" + @echo " help show this message" + @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + +deps: + uv sync --frozen + +# Assemble the ready-to-run application tree into build/: the locked +# requirements are exported from uv.lock and installed as prebuilt wheels for +# the production target (Debian 13 / amd64 / CPython 3.13) directly into +# build/, then the app package is copied alongside them. The result runs with +# `python3 -m manager_control_program.server` and PYTHONPATH=build/, needing +# only the system python3. install/package and the dev bind mount all consume +# this same tree. +build: + rm -rf $(BUILDDIR) + $(INSTALL) -d $(BUILDDIR) + uv export --frozen --no-dev --no-emit-project -o $(REQUIREMENTS) + uv pip install \ + --python $(PYTHON_VERSION) \ + --python-version $(PYTHON_VERSION) \ + --python-platform $(PYTHON_PLATFORM) \ + --only-binary :all: \ + --target $(BUILDDIR) \ + --requirements $(REQUIREMENTS) + rm -f $(REQUIREMENTS) + cp -a manager_control_program $(BUILDDIR)/ + find $(BUILDDIR) -type d -name __pycache__ -prune -exec rm -rf {} + + +dev: + uv run manager-control-program + +# No test suite yet; kept as a no-op so the repo-wide `make test` loop stays +# uniform if this component is ever added to the root COMPONENTS list. +test: + +# Copy the assembled application tree (build/) into DESTBIN/build so the dev +# bind mount and the packaged install expose the identical path +# (.../manager-control-program/build, matching the repo layout), then install +# the systemd unit and its /etc/default config to the system paths. Depends on +# build so `make install` alone produces a complete tree. +install: build + $(INSTALL) -d $(DESTBIN)/build + cp -a $(BUILDDIR)/. $(DESTBIN)/build/ + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/opensource-mcp.service $(UNIT_DIR)/ + $(INSTALL) -d $(DESTDIR)/etc/default + $(INSTALL_DATA) contrib/default/opensource-mcp $(DESTDIR)/etc/default/opensource-mcp + +PACKAGER ?= deb +package: + rm -rf $(STAGE) + $(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX) + fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \ + -C $(STAGE) -p $(CURDIR) -f + +deb: + $(MAKE) package PACKAGER=deb +rpm: + $(MAKE) package PACKAGER=rpm +apk: + $(MAKE) package PACKAGER=apk + +clean: + rm -rf .venv $(REQUIREMENTS) $(BUILDDIR) $(STAGE) + rm -f *.deb *.rpm *.apk diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 6452ade7..a3dec0d6 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -2,72 +2,71 @@ MCP server that exposes the [create-a-container](../create-a-container/) REST API as [Model Context Protocol](https://modelcontextprotocol.io/) tools. It reads the OpenAPI spec at runtime from `create-a-container` and auto-generates MCP tool definitions using [`awslabs-openapi-mcp-server`](https://github.com/awslabs/mcp/tree/main/src/openapi-mcp-server), so it stays in sync with API changes automatically. -## Prerequisites +This README documents the component itself. For setting up an MCP client +(VS Code, Claude Desktop) or hosting a shared server, start with the guide +below. -- Python 3.13+ -- [uv](https://docs.astral.sh/uv/) package manager -- A running `create-a-container` instance with API access -- A bearer token (API key) from `create-a-container` +| If you want to... | Read | +|---|---| +| Connect your editor/assistant, or host a shared HTTP server | [MCP Server](../mie-opensource-landing/docs/users/mcp-server.md) | +| Create the API key the server authenticates with | [API Keys](../mie-opensource-landing/docs/users/creating-containers/api-keys.md) | -## Setup +## Running It + +### Development (local clone) + +Requires Python 3.13+ and [uv](https://docs.astral.sh/uv/): ```bash uv sync +API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key uv run manager-control-program ``` -## Environment Variables +End users don't need a clone — `uvx` runs it straight from git (see the +[MCP Server guide](../mie-opensource-landing/docs/users/mcp-server.md)). -| Variable | Required | Description | -|---|---|---| -| `API_BASE_URL` | **Yes** | Base URL of the `create-a-container` instance (e.g., `https://containers.example.com`) | -| `AUTH_TOKEN` | **Yes** | Bearer token for API authentication (create one at `/apikeys` in `create-a-container`) | - -The server automatically sets `API_SPEC_URL` to `${API_BASE_URL}/api/openapi.json` and `AUTH_TYPE` to `bearer`. +### Production (packaged) -## Usage +This directory builds the **`opensource-mcp`** package (`make deb`, `rpm`, or +`apk` via [fpm](https://fpm.readthedocs.io/)); the `opensource-server` package +depends on it, so it installs automatically with the Manager. The package +stages the app at `/opt/opensource-server/manager-control-program` with **all +Python dependencies vendored** (uv exports the lockfile and installs prebuilt +wheels for Debian 13 / amd64 / CPython 3.13). The target host needs only +`python3` 3.13 — declared as a package dependency — with no uv, pip, venv, or +network access. -### From a Local Clone +It runs as the `opensource-mcp.service` systemd unit: HTTP transport on +`127.0.0.1:8100`, started via `python3 -m manager_control_program.server` +with `PYTHONPATH` pointed at the vendored directory. The Manager +reverse-proxies `/mcp` to it, so MCP clients connect through the Manager's +public origin with TLS. Its runtime configuration (`API_BASE_URL`, +`SERVER_TRANSPORT`/`SERVER_HOST`/`SERVER_PORT`, `LOG_LEVEL`) lives in the +packaged config file `/etc/default/opensource-mcp` — edit it and restart the +service to apply. -```bash -API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key uv run manager-control-program -``` +## Configuration -### From Git (No Clone Required) +Configuration is read from environment variables: -```bash -API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key \ - uvx --from "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program" \ - manager-control-program -``` +| Variable | Required | Description | +|---|---|---| +| `API_BASE_URL` | **Yes** | Base URL of the `create-a-container` instance (e.g., `https://containers.example.com`) | +| `AUTH_TOKEN` | stdio only | Bearer token for API authentication. Unused in HTTP mode, where each caller sends their own token per request | +| `SERVER_TRANSPORT` | No | `stdio` (default), `http` (streamable HTTP), or `sse` (legacy) | +| `SERVER_HOST` | No | Bind address in HTTP mode (default `127.0.0.1`) | +| `SERVER_PORT` | No | Port in HTTP mode (default `8000`) | +| `API_SPEC_URL` | No | OpenAPI spec location; defaults to `${API_BASE_URL}/api/openapi.json` | -### MCP Client Configuration - -Add to your MCP client config (e.g., Claude Desktop, VS Code): - -```json -{ - "mcpServers": { - "container-manager": { - "command": "uvx", - "args": [ - "--from", - "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program", - "manager-control-program" - ], - "env": { - "API_BASE_URL": "https://containers.example.com", - "AUTH_TOKEN": "your-api-key" - } - } - } -} -``` +`AUTH_TYPE` defaults to `bearer` in stdio mode and `none` in HTTP mode. Setting +`AUTH_TYPE=bearer` with `AUTH_TOKEN` alongside an HTTP transport configures a +static fallback used when a request carries no `Authorization` header. ## How It Works ```mermaid graph LR - A[MCP Client
e.g. Claude] -->|MCP protocol
stdio| B[manager-control-program] + A[MCP Client
e.g. Claude] -->|MCP protocol
stdio or HTTP| B[manager-control-program] B -->|GET /api/openapi.json| C[create-a-container] B -->|REST API calls
Bearer auth| C ``` @@ -76,6 +75,11 @@ graph LR 2. Generates an MCP tool for each API operation (list containers, create jobs, etc.) 3. Proxies tool calls as authenticated REST requests to the API +Authentication depends on the transport: + +- **stdio** — one server per user; the static `AUTH_TOKEN` is attached to every API request. +- **HTTP** (`SERVER_TRANSPORT=http`) — one shared server; each incoming MCP request's `Authorization` header is forwarded to the API (see `ForwardAuthorizationHeader` in [`server.py`](manager_control_program/server.py)), so every caller acts under their own identity. No token is needed at startup. + ## Available Tools Tools are generated dynamically from the OpenAPI spec. Typical operations include: diff --git a/manager-control-program/contrib/default/opensource-mcp b/manager-control-program/contrib/default/opensource-mcp new file mode 100644 index 00000000..6c148bdd --- /dev/null +++ b/manager-control-program/contrib/default/opensource-mcp @@ -0,0 +1,18 @@ +# Environment for opensource-mcp.service (see manager_control_program.server). +# This file is sourced by systemd via EnvironmentFile; edit it and restart the +# service to apply changes. Marked as a package config file, so your edits are +# preserved across upgrades. + +# Base URL of the create-a-container REST API the MCP tools call. The packaged +# Manager listens on loopback; the OpenAPI spec is fetched from here at startup. +API_BASE_URL=http://127.0.0.1:3000 + +# HTTP transport on loopback: the Manager reverse-proxies /mcp here, adding TLS +# and a public hostname. Each MCP client authenticates per-request with its own +# Bearer API key, which the server forwards to the API — no AUTH_TOKEN needed. +SERVER_TRANSPORT=http +SERVER_HOST=127.0.0.1 +SERVER_PORT=8100 + +# Other supported overrides (defaults shown in comments): +# LOG_LEVEL=INFO diff --git a/manager-control-program/contrib/postinstall.sh b/manager-control-program/contrib/postinstall.sh new file mode 100644 index 00000000..5bfa9e52 --- /dev/null +++ b/manager-control-program/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +UNITS="opensource-mcp.service" + +# Nothing to do without systemctl (non-systemd container/chroot). +command -v systemctl >/dev/null 2>&1 || exit 0 + +# `systemctl enable` only creates static symlinks, so it works during an image +# build too +systemctl enable $UNITS + +# daemon-reload and restart need a running systemd; skip them at build time. +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + systemctl restart $UNITS +fi + +exit 0 diff --git a/manager-control-program/contrib/preremove.sh b/manager-control-program/contrib/preremove.sh new file mode 100644 index 00000000..9c4f35c5 --- /dev/null +++ b/manager-control-program/contrib/preremove.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +UNITS="opensource-mcp.service" + +# $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final +# removal). Only act on a real removal, not an upgrade. +case "${1:-}" in + upgrade|1) + # rpm upgrade ("1") / deb upgrade: leave units in place. + exit 0 + ;; +esac + +# Nothing to do without systemctl +command -v systemctl >/dev/null 2>&1 || exit 0 + +# Stopping needs a running systemd; disabling (symlink removal) does not. +if [ -d /run/systemd/system ]; then + systemctl stop $UNITS +fi +systemctl disable $UNITS + +exit 0 diff --git a/manager-control-program/contrib/systemd/opensource-mcp.service b/manager-control-program/contrib/systemd/opensource-mcp.service new file mode 100644 index 00000000..67270489 --- /dev/null +++ b/manager-control-program/contrib/systemd/opensource-mcp.service @@ -0,0 +1,23 @@ +[Unit] +Description=MCP server for create-a-container (Model Context Protocol over HTTP) +# The OpenAPI spec is fetched from the Manager at startup, so order after it +# and keep retrying (Restart/RestartSec below) until the Manager is reachable. +After=network.target container-creator.service +Wants=container-creator.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/opensource-server/manager-control-program/build +# The app package and all vendored dependencies are assembled into build/ +# (by `make build`); no uv, pip, or venv exists on the host. +Environment=PYTHONPATH=/opt/opensource-server/manager-control-program/build +# Runtime configuration (API_BASE_URL, SERVER_TRANSPORT/HOST/PORT, LOG_LEVEL) +# lives in the packaged config file below — edit it and restart to apply. +EnvironmentFile=/etc/default/opensource-mcp +ExecStart=/usr/bin/python3 -m manager_control_program.server +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/manager-control-program/manager_control_program/__init__.py b/manager-control-program/manager_control_program/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manager-control-program/manager_control_program/server.py b/manager-control-program/manager_control_program/server.py index 203c32cf..34ddca00 100644 --- a/manager-control-program/manager_control_program/server.py +++ b/manager-control-program/manager_control_program/server.py @@ -1,6 +1,65 @@ import os -import awslabs.openapi_mcp_server +from urllib.parse import urljoin, urlsplit + +import httpx from awslabs.openapi_mcp_server.server import load_config, create_mcp_server, setup_signal_handlers +from fastmcp.server.dependencies import get_http_headers + +# Transports accepted via SERVER_TRANSPORT. "http" is fastmcp's streamable +# HTTP transport ("streamable-http" is an alias); "sse" is the legacy +# HTTP+Server-Sent-Events transport kept for older MCP clients. +HTTP_TRANSPORTS = ("http", "streamable-http", "sse") + + +def _spec_server_path(spec_url: str) -> str: + """Return the path prefix declared by the OpenAPI spec's first server. + + fastmcp builds each tool's request URL from the raw path in the spec + (e.g. "/sites") joined onto the httpx client's base_url, ignoring the + spec's `servers` entry entirely. Our API declares `servers: [{url: + /api/v1}]`, so without this prefix the tools would hit "/sites" instead + of "/api/v1/sites" — which the Manager serves as the SPA's HTML, not JSON. + + We fetch the spec ourselves (the same document awslabs loads) and return + servers[0].url's path. A relative server url ("/api/v1") contributes its + path directly; an absolute one contributes only its path component so the + host stays whatever API_BASE_URL points at. Returns "" when the spec omits + servers or declares the root, leaving base_url unchanged. + """ + spec = httpx.get(spec_url, timeout=30.0).raise_for_status().json() + servers = spec.get("servers") or [] + if not servers: + return "" + url = (servers[0] or {}).get("url", "") or "" + # Keep only the path; a spec-declared host would otherwise override + # API_BASE_URL (and its loopback target). + path = urlsplit(url).path if "://" in url else url + return path.strip("/") + + +class ForwardAuthorizationHeader(httpx.Auth): + """Per-request auth for HTTP mode: forward the MCP caller's credentials. + + When the server runs over an HTTP transport, every tool call arrives as an + HTTP request from the MCP client. fastmcp exposes those request headers + through a context variable, so at the moment the generated tool sends its + API request we copy the caller's Authorization header onto it. Each caller + therefore authenticates to the API as themselves — no shared AUTH_TOKEN + has to exist at startup. + + fastmcp's own header passthrough (OpenAPITool.run) deliberately strips + `authorization`, hence the explicit include here. Outside an HTTP request + context get_http_headers() returns {}, making this a no-op that falls + back to whatever static auth (if any) is configured on the client. + """ + + def auth_flow(self, request): + incoming = get_http_headers(include={"authorization"}) + authorization = incoming.get("authorization") + if authorization: + request.headers["Authorization"] = authorization + yield request + def main(): # We require API_BASE_URL to be set by the user so we know how to route API @@ -19,22 +78,62 @@ def main(): if "API_SPEC_URL" not in os.environ: os.environ["API_SPEC_URL"] = f"{api_base_url}/api/openapi.json" - # We default to Bearer auth with requires the user to have set the - # AUTH_TOKEN environment variable. I'm unsure if any other auth types work, - # but we leave that door open incase it's needed. + # SERVER_TRANSPORT selects how MCP clients connect (awslabs' load_config + # reads the same variable, along with SERVER_HOST/SERVER_PORT): + # - "stdio" (default): single local client, static credentials. + # - "http"/"streamable-http"/"sse": shared network server, per-request + # credentials forwarded from each caller (see ForwardAuthorizationHeader). + transport = os.environ.get("SERVER_TRANSPORT", "stdio").strip().lower() + if transport != "stdio" and transport not in HTTP_TRANSPORTS: + raise RuntimeError( + f"Unsupported SERVER_TRANSPORT '{transport}'. " + f"Expected 'stdio' or one of: {', '.join(HTTP_TRANSPORTS)}." + ) + if "AUTH_TYPE" not in os.environ: - os.environ["AUTH_TYPE"] = "bearer" + if transport == "stdio": + # We default to Bearer auth which requires the user to have set the + # AUTH_TOKEN environment variable. I'm unsure if any other auth + # types work, but we leave that door open incase it's needed. + os.environ["AUTH_TYPE"] = "bearer" + else: + # HTTP mode: callers supply their own Authorization header on each + # request, so don't demand a static AUTH_TOKEN at startup. Setting + # AUTH_TYPE=bearer explicitly (with AUTH_TOKEN) still works and + # acts as a fallback for requests that omit the header. + os.environ["AUTH_TYPE"] = "none" # The rest of this is more-or-less copied from the official - # awslabs.openapi_mpc_server.server:main function with the small exception - # of setting the Accept header to application/json. The official defaults to - # */* which makes our API return HTML instead of the JSON response, breaking - # the API spec. + # awslabs.openapi_mpc_server.server:main function, with two adjustments: + # + # - base_url gets the spec's server path prefix appended (see + # _spec_server_path): fastmcp ignores the spec's `servers` entry, so + # without this the generated tools would request "/sites" rather than + # "/api/v1/sites" and hit the SPA's HTML catch-all instead of the JSON + # API. + # - the Accept header is pinned to application/json; awslabs defaults to + # */*, which lets the API content-negotiate to HTML for some routes. config = load_config() mcp_server = create_mcp_server(config) + + server_path = _spec_server_path(os.environ["API_SPEC_URL"]) + if server_path: + base = str(mcp_server._client.base_url).rstrip("/") + "/" + mcp_server._client.base_url = urljoin(base, server_path + "/") + mcp_server._client.headers['accept'] = 'application/json' setup_signal_handlers() - mcp_server.run() + + if transport == "stdio": + mcp_server.run() + return + + # Forward each caller's Authorization header to the API. Client-level auth + # runs on every request the generated tools send (they use this shared + # httpx client), and takes precedence over any static default header. + mcp_server._client.auth = ForwardAuthorizationHeader() + mcp_server.run(transport=transport, host=config.host, port=config.port) + if __name__=='__main__': main() diff --git a/manager-control-program/pyproject.toml b/manager-control-program/pyproject.toml index 701c01c0..81c20cb1 100644 --- a/manager-control-program/pyproject.toml +++ b/manager-control-program/pyproject.toml @@ -11,3 +11,11 @@ dependencies = [ [project.scripts] "manager-control-program" = "manager_control_program.server:main" + +# Without a build-system, uv treats this as a virtual (non-installable) +# project: `uv sync` never installs the package, so the console script above +# doesn't exist and `uv run manager-control-program` / `uvx --from git+...` +# fail to spawn. +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/manager-control-program/uv.lock b/manager-control-program/uv.lock index 6a5a6ddf..873e499c 100644 --- a/manager-control-program/uv.lock +++ b/manager-control-program/uv.lock @@ -744,7 +744,7 @@ wheels = [ [[package]] name = "manager-control-program" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "awslabs-openapi-mcp-server" }, { name = "fastmcp" }, diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md index df33f01b..a1b68747 100644 --- a/mie-opensource-landing/docs/developers/release-pipeline.md +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -2,7 +2,7 @@ {{ contributor_warning }} -The three deployable components are packaged as Debian packages and published +The four deployable components are packaged as Debian packages and published to GitHub Releases as a flat APT repository. The same component build commands are reused by local development, the container images, and CI. @@ -13,12 +13,14 @@ are reused by local development, the container images, and CI. | [`create-a-container/`](https://github.com/mieweb/opensource-server/tree/main/create-a-container) | `opensource-server` | amd64 | Manager web app, job runner, systemd units | | [`mie-opensource-landing/`](https://github.com/mieweb/opensource-server/tree/main/mie-opensource-landing) | `opensource-docs` | all | Prebuilt documentation site | | [`agent/`](https://github.com/mieweb/opensource-server/tree/main/agent) | `opensource-agent` | all | Check-in agent, config templates, systemd timer, error pages | +| [`manager-control-program/`](https://github.com/mieweb/opensource-server/tree/main/manager-control-program) | `opensource-mcp` | amd64 | MCP server with vendored Python deps, systemd unit | Everything installs under the `/opt/opensource-server` prefix, matching the paths referenced by the systemd units and the agent-rendered nginx configuration. `opensource-server` depends on `opensource-agent` and `opensource-docs` because the manager's nginx config -serves the agent's error pages and the docs site. +serves the agent's error pages and the docs site, and on `opensource-mcp`, +whose MCP service the manager reverse-proxies at `/mcp`. ## The component Makefile contract diff --git a/mie-opensource-landing/docs/users/mcp-server.md b/mie-opensource-landing/docs/users/mcp-server.md index 5cbbaaa7..4701dde9 100644 --- a/mie-opensource-landing/docs/users/mcp-server.md +++ b/mie-opensource-landing/docs/users/mcp-server.md @@ -1,13 +1,18 @@ -# MCP Server for VS Code +# MCP Server Use the MCP (Model Context Protocol) server to manage your containers through AI assistants like GitHub Copilot or Claude directly inside VS Code. +There are two ways to use it: + +- **Run it locally (stdio)** — VS Code launches a private copy of the server for you, authenticated with your API key from the environment. +- **Connect to the built-in server (HTTP)** — packaged deployments already serve MCP at `{{ manager_url }}/mcp`; you just send your API key with each request. Nothing to install, so start here. + **Prerequisites:** - VS Code with an MCP-capable AI extension (e.g., [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot), [Claude for VS Code](https://marketplace.visualstudio.com/items?itemName=AnthropicPublicBeta.claude-for-vscode)) -- [uv](https://docs.astral.sh/uv/getting-started/installation/) installed - An API key from [your server]({{ manager_url }}/apikeys) (see [API Keys](./creating-containers/api-keys.md)) +- [uv](https://docs.astral.sh/uv/getting-started/installation/) installed (local stdio mode only) -## 1. Configure VS Code +## Option A: Run It Locally (stdio) Open your VS Code settings (`Ctrl+Shift+P` → "Preferences: Open User Settings (JSON)") and add the MCP server: @@ -41,11 +46,59 @@ Open your VS Code settings (`Ctrl+Shift+P` → "Preferences: Open User Settings You can also add this to a **workspace-level** `.vscode/settings.json` to share the config with your team (omit `AUTH_TOKEN` and set it as a system environment variable instead). -## 2. Verify the Connection +## Option B: Connect to the Built-in Server (HTTP) + +Every packaged deployment ships the MCP server as a system service and exposes it at `/mcp` on the same domain as the web UI. Nothing to install — point VS Code at that URL and send your API key in the `Authorization` header. Your key is forwarded to the API on every request, so you act under your own account and permissions: + +```json +{ + "mcp": { + "servers": { + "container-manager": { + "url": "{{ manager_url }}/mcp", + "headers": { + "Authorization": "Bearer your-api-key" + } + } + } + } +} +``` + +### How the Built-in Server Is Hosted + +The `opensource-mcp` package (installed automatically as a dependency of `opensource-server`) ships the MCP server with vendored Python dependencies at `/opt/opensource-server/manager-control-program` and runs it as the `opensource-mcp.service` systemd unit, listening on loopback (`127.0.0.1:8100`). The Manager reverse-proxies `/mcp` to it, providing the public hostname and TLS. It is enabled by default; admins can: + +- change settings (e.g. `SERVER_PORT`) in the packaged config file `/etc/default/opensource-mcp`, then restart the service +- disable it entirely with `systemctl disable --now opensource-mcp.service` (and unset `MCP_SERVER_URL` for `container-creator.service` to remove the proxy route) + +### Hosting a Standalone Server + +Outside a packaged deployment, the same HTTP mode runs anywhere `uv` is available: + +```bash +API_BASE_URL=https://your-server-domain SERVER_TRANSPORT=http \ + uvx --from "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program" \ + manager-control-program +``` + +| Variable | Description | +|----------|-------------| +| `SERVER_TRANSPORT` | `http` (streamable HTTP), or `sse` for legacy clients. Defaults to `stdio` | +| `SERVER_HOST` | Bind address (default `127.0.0.1`) | +| `SERVER_PORT` | Port (default `8000`) | + +No `AUTH_TOKEN` is required at startup: each caller authenticates with their own key, and requests without an `Authorization` header are rejected by the API. + +!!! warning + + The MCP server forwards tokens without validating them and serves plain HTTP. If you expose it beyond localhost, put it behind an HTTPS reverse proxy and restrict who can reach it. + +## Verify the Connection After saving the config, restart VS Code. Open the MCP server list (`Ctrl+Shift+P` → "MCP: List Servers") to confirm `container-manager` shows a green status. -## 3. Use It +## Use It Ask your AI assistant to interact with your containers using natural language. Example prompts: