From ea4bef9a7d1d09f893b0075b2498ebce7e28dab2 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Thu, 23 Jul 2026 17:44:28 -0400 Subject: [PATCH 01/12] feat(manager-control-program): HTTP transport with per-request auth forwarding Adds SERVER_TRANSPORT=http|streamable-http|sse (default remains stdio). In HTTP mode the server no longer requires AUTH_TOKEN at startup: a client-level httpx auth hook reads each incoming MCP request's Authorization header from fastmcp's request context and forwards it on the outgoing API call, so every caller authenticates to the API as themselves. fastmcp's built-in header passthrough deliberately strips authorization, hence the explicit include. Setting AUTH_TYPE=bearer with AUTH_TOKEN alongside HTTP mode still works as a static fallback for requests without a header. stdio behavior is unchanged (bearer + AUTH_TOKEN required). Verified end-to-end against a stub API: two HTTP MCP clients with different bearer tokens each hit the API under their own token; a headerless client produced no Authorization header; stdio mode still fails fast without AUTH_TOKEN; invalid transports error clearly. --- manager-control-program/README.md | 37 +++++++++- .../manager_control_program/server.py | 73 +++++++++++++++++-- 2 files changed, 101 insertions(+), 9 deletions(-) diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 6452ade7..7ec565ab 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -20,9 +20,12 @@ uv sync | Variable | Required | Description | |---|---|---| | `API_BASE_URL` | **Yes** | Base URL of the `create-a-container` instance (e.g., `https://containers.example.com`) | -| `AUTH_TOKEN` | **Yes** | Bearer token for API authentication (create one at `/apikeys` in `create-a-container`) | +| `AUTH_TOKEN` | stdio only | Bearer token for API authentication (create one at `/apikeys` in `create-a-container`). Not needed in HTTP mode, where each caller sends their own token | +| `SERVER_TRANSPORT` | No | `stdio` (default), `http` (streamable HTTP), or `sse` (legacy) | +| `SERVER_HOST` | No | Bind address in HTTP mode (default `127.0.0.1`) | +| `SERVER_PORT` | No | Port in HTTP mode (default `8000`) | -The server automatically sets `API_SPEC_URL` to `${API_BASE_URL}/api/openapi.json` and `AUTH_TYPE` to `bearer`. +The server automatically sets `API_SPEC_URL` to `${API_BASE_URL}/api/openapi.json`, and `AUTH_TYPE` to `bearer` (stdio) or `none` (HTTP). ## Usage @@ -63,11 +66,39 @@ Add to your MCP client config (e.g., Claude Desktop, VS Code): } ``` +### HTTP Mode (Shared Server, Per-Request Auth) + +Run one shared MCP server over streamable HTTP instead of one stdio process per client: + +```bash +API_BASE_URL=https://containers.example.com SERVER_TRANSPORT=http SERVER_PORT=8000 \ + uv run manager-control-program +``` + +In HTTP mode no `AUTH_TOKEN` is needed at startup. Instead, each MCP client sends its own API key in the `Authorization` header, and the server forwards that header to the API on every request — so every caller acts under their own identity and permissions: + +```json +{ + "mcpServers": { + "container-manager": { + "url": "http://mcp.example.com:8000/mcp", + "headers": { + "Authorization": "Bearer your-api-key" + } + } + } +} +``` + +Requests without an `Authorization` header are rejected by the API (401) unless you explicitly configure a static fallback by setting `AUTH_TYPE=bearer` and `AUTH_TOKEN` alongside `SERVER_TRANSPORT=http`. + +> **Note:** The MCP server itself does not validate tokens — it forwards them for the API to verify. Bind it to `127.0.0.1` (the default) or otherwise restrict network access, and use HTTPS via a reverse proxy if exposing it beyond localhost. + ## How It Works ```mermaid graph LR - A[MCP Client
e.g. Claude] -->|MCP protocol
stdio| B[manager-control-program] + A[MCP Client
e.g. Claude] -->|MCP protocol
stdio or HTTP| B[manager-control-program] B -->|GET /api/openapi.json| C[create-a-container] B -->|REST API calls
Bearer auth| C ``` diff --git a/manager-control-program/manager_control_program/server.py b/manager-control-program/manager_control_program/server.py index 203c32cf..8f72fbf0 100644 --- a/manager-control-program/manager_control_program/server.py +++ b/manager-control-program/manager_control_program/server.py @@ -1,6 +1,38 @@ import os -import awslabs.openapi_mcp_server + +import httpx from awslabs.openapi_mcp_server.server import load_config, create_mcp_server, setup_signal_handlers +from fastmcp.server.dependencies import get_http_headers + +# Transports accepted via SERVER_TRANSPORT. "http" is fastmcp's streamable +# HTTP transport ("streamable-http" is an alias); "sse" is the legacy +# HTTP+Server-Sent-Events transport kept for older MCP clients. +HTTP_TRANSPORTS = ("http", "streamable-http", "sse") + + +class ForwardAuthorizationHeader(httpx.Auth): + """Per-request auth for HTTP mode: forward the MCP caller's credentials. + + When the server runs over an HTTP transport, every tool call arrives as an + HTTP request from the MCP client. fastmcp exposes those request headers + through a context variable, so at the moment the generated tool sends its + API request we copy the caller's Authorization header onto it. Each caller + therefore authenticates to the API as themselves — no shared AUTH_TOKEN + has to exist at startup. + + fastmcp's own header passthrough (OpenAPITool.run) deliberately strips + `authorization`, hence the explicit include here. Outside an HTTP request + context get_http_headers() returns {}, making this a no-op that falls + back to whatever static auth (if any) is configured on the client. + """ + + def auth_flow(self, request): + incoming = get_http_headers(include={"authorization"}) + authorization = incoming.get("authorization") + if authorization: + request.headers["Authorization"] = authorization + yield request + def main(): # We require API_BASE_URL to be set by the user so we know how to route API @@ -19,11 +51,30 @@ def main(): if "API_SPEC_URL" not in os.environ: os.environ["API_SPEC_URL"] = f"{api_base_url}/api/openapi.json" - # We default to Bearer auth with requires the user to have set the - # AUTH_TOKEN environment variable. I'm unsure if any other auth types work, - # but we leave that door open incase it's needed. + # SERVER_TRANSPORT selects how MCP clients connect (awslabs' load_config + # reads the same variable, along with SERVER_HOST/SERVER_PORT): + # - "stdio" (default): single local client, static credentials. + # - "http"/"streamable-http"/"sse": shared network server, per-request + # credentials forwarded from each caller (see ForwardAuthorizationHeader). + transport = os.environ.get("SERVER_TRANSPORT", "stdio").strip().lower() + if transport != "stdio" and transport not in HTTP_TRANSPORTS: + raise RuntimeError( + f"Unsupported SERVER_TRANSPORT '{transport}'. " + f"Expected 'stdio' or one of: {', '.join(HTTP_TRANSPORTS)}." + ) + if "AUTH_TYPE" not in os.environ: - os.environ["AUTH_TYPE"] = "bearer" + if transport == "stdio": + # We default to Bearer auth which requires the user to have set the + # AUTH_TOKEN environment variable. I'm unsure if any other auth + # types work, but we leave that door open incase it's needed. + os.environ["AUTH_TYPE"] = "bearer" + else: + # HTTP mode: callers supply their own Authorization header on each + # request, so don't demand a static AUTH_TOKEN at startup. Setting + # AUTH_TYPE=bearer explicitly (with AUTH_TOKEN) still works and + # acts as a fallback for requests that omit the header. + os.environ["AUTH_TYPE"] = "none" # The rest of this is more-or-less copied from the official # awslabs.openapi_mpc_server.server:main function with the small exception @@ -34,7 +85,17 @@ def main(): mcp_server = create_mcp_server(config) mcp_server._client.headers['accept'] = 'application/json' setup_signal_handlers() - mcp_server.run() + + if transport == "stdio": + mcp_server.run() + return + + # Forward each caller's Authorization header to the API. Client-level auth + # runs on every request the generated tools send (they use this shared + # httpx client), and takes precedence over any static default header. + mcp_server._client.auth = ForwardAuthorizationHeader() + mcp_server.run(transport=transport, host=config.host, port=config.port) + if __name__=='__main__': main() From 24abfa0339b45d0890d322f9e438d9e73ce5db20 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Thu, 23 Jul 2026 17:48:48 -0400 Subject: [PATCH 02/12] fix(manager-control-program): make the package installable Without a [build-system], uv treats the project as virtual and never installs it into the venv, so the manager-control-program console script doesn't exist: the documented `uv run manager-control-program` and `uvx --from git+...` invocations fail with 'Failed to spawn'. Add a hatchling build-system and the missing package __init__.py. uv.lock: source flips from virtual to editable. --- .../manager_control_program/__init__.py | 0 manager-control-program/pyproject.toml | 8 ++++++++ manager-control-program/uv.lock | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 manager-control-program/manager_control_program/__init__.py diff --git a/manager-control-program/manager_control_program/__init__.py b/manager-control-program/manager_control_program/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/manager-control-program/pyproject.toml b/manager-control-program/pyproject.toml index 701c01c0..81c20cb1 100644 --- a/manager-control-program/pyproject.toml +++ b/manager-control-program/pyproject.toml @@ -11,3 +11,11 @@ dependencies = [ [project.scripts] "manager-control-program" = "manager_control_program.server:main" + +# Without a build-system, uv treats this as a virtual (non-installable) +# project: `uv sync` never installs the package, so the console script above +# doesn't exist and `uv run manager-control-program` / `uvx --from git+...` +# fail to spawn. +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/manager-control-program/uv.lock b/manager-control-program/uv.lock index 6a5a6ddf..873e499c 100644 --- a/manager-control-program/uv.lock +++ b/manager-control-program/uv.lock @@ -744,7 +744,7 @@ wheels = [ [[package]] name = "manager-control-program" version = "0.1.0" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "awslabs-openapi-mcp-server" }, { name = "fastmcp" }, From 548e9035c2cdeb51c197a888040d28aedee3622a Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Thu, 23 Jul 2026 17:50:17 -0400 Subject: [PATCH 03/12] docs(manager-control-program): make the docs site canonical, document HTTP mode The MCP Server page in mie-opensource-landing is the real setup guide: restructure it around the two ways to use the server (Option A: local stdio via uvx, Option B: connecting to a shared HTTP server with your own API key in the Authorization header) and add a hosting section for the new HTTP transport with its env vars and security caveats. Slim the component README to the house style used by create-a-container: component overview, defer-to-docs table, local development, environment variable reference, and how auth works per transport - dropping the MCP client config JSON it previously duplicated from the docs site. --- manager-control-program/README.md | 105 +++++------------- .../docs/users/mcp-server.md | 56 +++++++++- 2 files changed, 80 insertions(+), 81 deletions(-) diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 7ec565ab..07e922d4 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -2,97 +2,45 @@ MCP server that exposes the [create-a-container](../create-a-container/) REST API as [Model Context Protocol](https://modelcontextprotocol.io/) tools. It reads the OpenAPI spec at runtime from `create-a-container` and auto-generates MCP tool definitions using [`awslabs-openapi-mcp-server`](https://github.com/awslabs/mcp/tree/main/src/openapi-mcp-server), so it stays in sync with API changes automatically. -## Prerequisites +This README documents the component itself. For setting up an MCP client +(VS Code, Claude Desktop) or hosting a shared server, start with the guide +below. -- Python 3.13+ -- [uv](https://docs.astral.sh/uv/) package manager -- A running `create-a-container` instance with API access -- A bearer token (API key) from `create-a-container` +| If you want to... | Read | +|---|---| +| Connect your editor/assistant, or host a shared HTTP server | [MCP Server](../mie-opensource-landing/docs/users/mcp-server.md) | +| Create the API key the server authenticates with | [API Keys](../mie-opensource-landing/docs/users/creating-containers/api-keys.md) | -## Setup +## Running It + +### Development (local clone) + +Requires Python 3.13+ and [uv](https://docs.astral.sh/uv/): ```bash uv sync +API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key uv run manager-control-program ``` -## Environment Variables +End users don't need a clone — `uvx` runs it straight from git (see the +[MCP Server guide](../mie-opensource-landing/docs/users/mcp-server.md)). + +## Configuration + +Configuration is read from environment variables: | Variable | Required | Description | |---|---|---| | `API_BASE_URL` | **Yes** | Base URL of the `create-a-container` instance (e.g., `https://containers.example.com`) | -| `AUTH_TOKEN` | stdio only | Bearer token for API authentication (create one at `/apikeys` in `create-a-container`). Not needed in HTTP mode, where each caller sends their own token | +| `AUTH_TOKEN` | stdio only | Bearer token for API authentication. Unused in HTTP mode, where each caller sends their own token per request | | `SERVER_TRANSPORT` | No | `stdio` (default), `http` (streamable HTTP), or `sse` (legacy) | | `SERVER_HOST` | No | Bind address in HTTP mode (default `127.0.0.1`) | | `SERVER_PORT` | No | Port in HTTP mode (default `8000`) | +| `API_SPEC_URL` | No | OpenAPI spec location; defaults to `${API_BASE_URL}/api/openapi.json` | -The server automatically sets `API_SPEC_URL` to `${API_BASE_URL}/api/openapi.json`, and `AUTH_TYPE` to `bearer` (stdio) or `none` (HTTP). - -## Usage - -### From a Local Clone - -```bash -API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key uv run manager-control-program -``` - -### From Git (No Clone Required) - -```bash -API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key \ - uvx --from "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program" \ - manager-control-program -``` - -### MCP Client Configuration - -Add to your MCP client config (e.g., Claude Desktop, VS Code): - -```json -{ - "mcpServers": { - "container-manager": { - "command": "uvx", - "args": [ - "--from", - "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program", - "manager-control-program" - ], - "env": { - "API_BASE_URL": "https://containers.example.com", - "AUTH_TOKEN": "your-api-key" - } - } - } -} -``` - -### HTTP Mode (Shared Server, Per-Request Auth) - -Run one shared MCP server over streamable HTTP instead of one stdio process per client: - -```bash -API_BASE_URL=https://containers.example.com SERVER_TRANSPORT=http SERVER_PORT=8000 \ - uv run manager-control-program -``` - -In HTTP mode no `AUTH_TOKEN` is needed at startup. Instead, each MCP client sends its own API key in the `Authorization` header, and the server forwards that header to the API on every request — so every caller acts under their own identity and permissions: - -```json -{ - "mcpServers": { - "container-manager": { - "url": "http://mcp.example.com:8000/mcp", - "headers": { - "Authorization": "Bearer your-api-key" - } - } - } -} -``` - -Requests without an `Authorization` header are rejected by the API (401) unless you explicitly configure a static fallback by setting `AUTH_TYPE=bearer` and `AUTH_TOKEN` alongside `SERVER_TRANSPORT=http`. - -> **Note:** The MCP server itself does not validate tokens — it forwards them for the API to verify. Bind it to `127.0.0.1` (the default) or otherwise restrict network access, and use HTTPS via a reverse proxy if exposing it beyond localhost. +`AUTH_TYPE` defaults to `bearer` in stdio mode and `none` in HTTP mode. Setting +`AUTH_TYPE=bearer` with `AUTH_TOKEN` alongside an HTTP transport configures a +static fallback used when a request carries no `Authorization` header. ## How It Works @@ -107,6 +55,11 @@ graph LR 2. Generates an MCP tool for each API operation (list containers, create jobs, etc.) 3. Proxies tool calls as authenticated REST requests to the API +Authentication depends on the transport: + +- **stdio** — one server per user; the static `AUTH_TOKEN` is attached to every API request. +- **HTTP** (`SERVER_TRANSPORT=http`) — one shared server; each incoming MCP request's `Authorization` header is forwarded to the API (see `ForwardAuthorizationHeader` in [`server.py`](manager_control_program/server.py)), so every caller acts under their own identity. No token is needed at startup. + ## Available Tools Tools are generated dynamically from the OpenAPI spec. Typical operations include: diff --git a/mie-opensource-landing/docs/users/mcp-server.md b/mie-opensource-landing/docs/users/mcp-server.md index 5cbbaaa7..c4bfc9f5 100644 --- a/mie-opensource-landing/docs/users/mcp-server.md +++ b/mie-opensource-landing/docs/users/mcp-server.md @@ -1,13 +1,18 @@ -# MCP Server for VS Code +# MCP Server Use the MCP (Model Context Protocol) server to manage your containers through AI assistants like GitHub Copilot or Claude directly inside VS Code. +There are two ways to use it: + +- **Run it locally (stdio)** — VS Code launches a private copy of the server for you, authenticated with your API key from the environment. +- **Connect to a shared server (HTTP)** — someone hosts one MCP server for everyone; you connect to its URL and send your own API key with each request. + **Prerequisites:** - VS Code with an MCP-capable AI extension (e.g., [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot), [Claude for VS Code](https://marketplace.visualstudio.com/items?itemName=AnthropicPublicBeta.claude-for-vscode)) -- [uv](https://docs.astral.sh/uv/getting-started/installation/) installed - An API key from [your server]({{ manager_url }}/apikeys) (see [API Keys](./creating-containers/api-keys.md)) +- [uv](https://docs.astral.sh/uv/getting-started/installation/) installed (local stdio mode only) -## 1. Configure VS Code +## Option A: Run It Locally (stdio) Open your VS Code settings (`Ctrl+Shift+P` → "Preferences: Open User Settings (JSON)") and add the MCP server: @@ -41,11 +46,52 @@ Open your VS Code settings (`Ctrl+Shift+P` → "Preferences: Open User Settings You can also add this to a **workspace-level** `.vscode/settings.json` to share the config with your team (omit `AUTH_TOKEN` and set it as a system environment variable instead). -## 2. Verify the Connection +## Option B: Connect to a Shared Server (HTTP) + +If a shared MCP server is available, you don't need `uv` or any local install — point VS Code at its URL and send your API key in the `Authorization` header. The server forwards that header to the API on every request, so you act under your own account and permissions: + +```json +{ + "mcp": { + "servers": { + "container-manager": { + "url": "https://your-mcp-host:8000/mcp", + "headers": { + "Authorization": "Bearer your-api-key" + } + } + } + } +} +``` + +### Hosting a Shared Server + +Start the server with an HTTP transport instead of the default stdio: + +```bash +API_BASE_URL=https://your-server-domain SERVER_TRANSPORT=http \ + uvx --from "manager-control-program @ git+https://github.com/mieweb/opensource-server.git#subdirectory=manager-control-program" \ + manager-control-program +``` + +| Variable | Description | +|----------|-------------| +| `SERVER_TRANSPORT` | `http` (streamable HTTP), or `sse` for legacy clients. Defaults to `stdio` | +| `SERVER_HOST` | Bind address (default `127.0.0.1`) | +| `SERVER_PORT` | Port (default `8000`) | + +No `AUTH_TOKEN` is required at startup: each caller authenticates with their own key, and requests without an `Authorization` header are rejected by the API. + +!!! warning + + The MCP server forwards tokens without validating them and serves plain HTTP. If you expose it beyond localhost, put it behind an HTTPS reverse proxy and restrict who can reach it. + +## Verify the Connection After saving the config, restart VS Code. Open the MCP server list (`Ctrl+Shift+P` → "MCP: List Servers") to confirm `container-manager` shows a green status. -## 3. Use It +## Use It Ask your AI assistant to interact with your containers using natural language. Example prompts: From 06681f8e7ae1d8113521b5de58c999641c67641e Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Thu, 23 Jul 2026 18:13:24 -0400 Subject: [PATCH 04/12] feat(create-a-container): ship the MCP server in the package, proxy /mcp to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Package manager-control-program inside the opensource-server package with vendored Python dependencies so it runs without uv: - manager-control-program gains the standard component Makefile; its install target exports the uv lockfile and installs prebuilt wheels (Debian 13 / amd64 / CPython 3.13) plus the app into /opt/opensource-server/manager-control-program. The target host needs only /usr/bin/python3. - New manager-control-program.service systemd unit runs it via python3 -m with PYTHONPATH at the vendored dir: HTTP transport on 127.0.0.1:8100, per-request Authorization forwarding, no AUTH_TOKEN. Optional overrides via /etc/default/manager-control-program. - create-a-container's install stages it, postinstall/preremove enable/disable it alongside the existing units. - The Manager reverse-proxies /mcp to it (MCP_SERVER_URL, set in container-creator.service): a dependency-free streaming proxy mounted before the body parsers/session/rate-limit/CSRF so MCP request bodies stream through untouched and SSE responses stay open. MCP clients connect at the Manager's public origin with TLS, e.g. https://manager.example.com/mcp. Verified: jest suite covers the proxy (headers/body/query passthrough, SSE content type, 502 when down, unmounted when unconfigured); built the real deb with fpm and checked contents (vendored tree, all three units, postinst); ran the staged tree on a bare CPython 3.13 behind the real proxy middleware with two MCP clients — each caller's token reached the stub API intact through both hops. --- create-a-container/Makefile | 3 + create-a-container/README.md | 5 +- create-a-container/app.js | 20 ++- create-a-container/contrib/postinstall.sh | 2 +- create-a-container/contrib/preremove.sh | 2 +- .../contrib/systemd/container-creator.service | 2 + create-a-container/example.env | 7 + .../middlewares/__tests__/mcp-proxy.test.js | 139 ++++++++++++++++++ create-a-container/middlewares/mcp-proxy.js | 93 ++++++++++++ manager-control-program/Makefile | 79 ++++++++++ manager-control-program/README.md | 17 +++ .../systemd/manager-control-program.service | 29 ++++ .../docs/users/mcp-server.md | 19 ++- 13 files changed, 407 insertions(+), 10 deletions(-) create mode 100644 create-a-container/middlewares/__tests__/mcp-proxy.test.js create mode 100644 create-a-container/middlewares/mcp-proxy.js create mode 100644 manager-control-program/Makefile create mode 100644 manager-control-program/contrib/systemd/manager-control-program.service diff --git a/create-a-container/Makefile b/create-a-container/Makefile index 76334078..b879ddb8 100644 --- a/create-a-container/Makefile +++ b/create-a-container/Makefile @@ -101,6 +101,9 @@ install: build $(INSTALL_DATA) contrib/systemd/job-runner.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)/etc/logrotate.d $(INSTALL_DATA) contrib/opensource-server.logrotate $(DESTDIR)/etc/logrotate.d/opensource-server + # The MCP server ships inside this package (vendored Python deps, no uv + # on the target); the Manager proxies /mcp to it (see MCP_SERVER_URL). + $(MAKE) -C ../manager-control-program install DESTDIR=$(DESTDIR) PREFIX=$(PREFIX) PACKAGER ?= deb package: diff --git a/create-a-container/README.md b/create-a-container/README.md index cedcdbb8..dbd6e038 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -46,7 +46,10 @@ The Manager is not installed by hand in production. It ships as: - distribution **packages** built from this directory with `make deb`, `make rpm`, or `make apk` (via [fpm](https://fpm.readthedocs.io/)), which install the app under `/opt/opensource-server/create-a-container` and register the - `container-creator` and `job-runner` systemd services. + `container-creator`, `job-runner`, and `manager-control-program` systemd + services. The last is the bundled [MCP server](../manager-control-program/) + (Python deps vendored at build time); the Manager reverse-proxies `/mcp` to + it (`MCP_SERVER_URL`). In both cases the app runs `server.js` (HTTP API + UI) and `job-runner.js` (background worker). Database connection settings come from the environment (see diff --git a/create-a-container/app.js b/create-a-container/app.js index 3f714236..e94018cc 100644 --- a/create-a-container/app.js +++ b/create-a-container/app.js @@ -26,8 +26,16 @@ const { sequelize } = require('./models'); * @param {boolean} [options.rateLimit=true] - disable in tests: assertions on 4xx * responses must not burn the budget. * @param {boolean} [options.accessLog=true] - morgan; disable in tests for quiet output. + * @param {string} [options.mcpServerUrl] - origin of the MCP server to reverse-proxy + * at /mcp (default: $MCP_SERVER_URL). Unset + * disables the proxy. */ -function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { +function buildApp({ + sessionSecrets, + rateLimit = true, + accessLog = true, + mcpServerUrl = process.env.MCP_SERVER_URL, +} = {}) { if (!sessionSecrets || sessionSecrets.length === 0) { throw new Error('buildApp: sessionSecrets is required'); } @@ -46,6 +54,16 @@ function buildApp({ sessionSecrets, rateLimit = true, accessLog = true } = {}) { : process.stdout; app.use(morgan('combined', { stream: accessLogStream })); } + // --- MCP reverse proxy --- + // Mounted before the body parsers so request bodies stream through + // untouched, and before session/rate-limit/CSRF: MCP clients authenticate + // per-request with Bearer API keys (forwarded by the MCP server back to + // /api/v1), not cookies, so none of the cookie-oriented middleware applies. + if (mcpServerUrl) { + const { createMcpProxy } = require('./middlewares/mcp-proxy'); + app.use('/mcp', createMcpProxy(mcpServerUrl)); + } + app.use(express.json()); app.use(express.urlencoded({ extended: true })); diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh index e92c7acf..8ef3a2b3 100755 --- a/create-a-container/contrib/postinstall.sh +++ b/create-a-container/contrib/postinstall.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service" +UNITS="container-creator.service job-runner.service manager-control-program.service" # Nothing to do without systemctl (non-systemd container/chroot). command -v systemctl >/dev/null 2>&1 || exit 0 diff --git a/create-a-container/contrib/preremove.sh b/create-a-container/contrib/preremove.sh index a3ebf879..b44cb185 100755 --- a/create-a-container/contrib/preremove.sh +++ b/create-a-container/contrib/preremove.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service" +UNITS="container-creator.service job-runner.service manager-control-program.service" # $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final # removal). Only act on a real removal, not an upgrade. diff --git a/create-a-container/contrib/systemd/container-creator.service b/create-a-container/contrib/systemd/container-creator.service index f1ed7eba..d291ab1c 100644 --- a/create-a-container/contrib/systemd/container-creator.service +++ b/create-a-container/contrib/systemd/container-creator.service @@ -11,6 +11,8 @@ ExecStart=/usr/bin/node /opt/opensource-server/create-a-container/server.js Restart=on-failure Environment=NODE_ENV=production Environment=ACCESS_LOG=/var/log/opensource-server/access.log +# Reverse-proxy /mcp to the packaged MCP server (manager-control-program.service). +Environment=MCP_SERVER_URL=http://127.0.0.1:8100 EnvironmentFile=/etc/default/container-creator LogsDirectory=opensource-server diff --git a/create-a-container/example.env b/create-a-container/example.env index 93a7a35f..e92ba028 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -28,6 +28,13 @@ POSTGRES_DATABASE= # if set, the file is opened in append mode. ACCESS_LOG= +# --- MCP server reverse proxy (optional) --- +# When set, the Manager reverse-proxies /mcp to this origin — the +# manager-control-program MCP server in HTTP mode. The packaged systemd +# deployment sets this to http://127.0.0.1:8100 (see +# manager-control-program.service); leave unset to disable the proxy. +MCP_SERVER_URL= + # --- OIDC / single sign-on (optional) --- # SSO is enabled only when OIDC_ISSUER_URL, OIDC_CLIENT_ID, and # OIDC_CLIENT_SECRET are all set. When enabled, the login page redirects to the diff --git a/create-a-container/middlewares/__tests__/mcp-proxy.test.js b/create-a-container/middlewares/__tests__/mcp-proxy.test.js new file mode 100644 index 00000000..177d99d6 --- /dev/null +++ b/create-a-container/middlewares/__tests__/mcp-proxy.test.js @@ -0,0 +1,139 @@ +/** + * Tests for the /mcp reverse proxy: requests stream through to the configured + * MCP server with headers (notably Authorization) and bodies intact, responses + * come back verbatim (including SSE content types), and the proxy degrades to + * a 502 JSON error when the MCP server is down. When no MCP server is + * configured the route is not mounted at all. + */ + +const http = require('http'); +const request = require('supertest'); +const { buildApp } = require('../../app'); +const { resetDb, closeDb } = require('../../tests/helpers/db'); + +/** Stub MCP upstream: echoes requests, and speaks SSE on GET. */ +function startUpstream() { + const server = http.createServer((req, res) => { + if (req.method === 'GET') { + res.writeHead(200, { 'content-type': 'text/event-stream' }); + res.write('event: message\ndata: {"hello":"one"}\n\n'); + res.write('event: message\ndata: {"hello":"two"}\n\n'); + res.end(); + return; + } + let body = ''; + req.on('data', (chunk) => (body += chunk)); + req.on('end', () => { + res.writeHead(201, { + 'content-type': 'application/json', + 'mcp-session-id': 'stub-session-1', + }); + res.end( + JSON.stringify({ + method: req.method, + url: req.url, + authorization: req.headers.authorization || null, + contentType: req.headers['content-type'] || null, + body, + }), + ); + }); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve(server)); + }); +} + +function build(mcpServerUrl) { + return buildApp({ + sessionSecrets: ['test-secret'], + rateLimit: false, + accessLog: false, + mcpServerUrl, + }); +} + +describe('/mcp reverse proxy', () => { + let upstream; + let app; + let deadApp; + let bareApp; + + beforeAll(async () => { + await resetDb(); + upstream = await startUpstream(); + const { port } = upstream.address(); + // Build every app variant here, right after resetDb: each buildApp + // constructs a session store that fires an un-awaited CREATE TABLE, and + // constructing one mid-test races that query against closeDb() (see + // tests/helpers/app.js). + app = build(`http://127.0.0.1:${port}`); + const deadPort = await new Promise((resolve) => { + const s = http.createServer(); + s.listen(0, '127.0.0.1', () => { + const p = s.address().port; + s.close(() => resolve(p)); + }); + }); + deadApp = build(`http://127.0.0.1:${deadPort}`); + bareApp = build(undefined); + }); + + afterAll(async () => { + await new Promise((resolve) => upstream.close(resolve)); + await closeDb(); + }); + + test('forwards POST body, path, and Authorization header; returns upstream response', async () => { + const payload = { jsonrpc: '2.0', method: 'initialize', id: 1 }; + const res = await request(app) + .post('/mcp') + .set('Authorization', 'Bearer caller-token') + .set('Content-Type', 'application/json') + .send(payload); + + expect(res.status).toBe(201); + expect(res.headers['mcp-session-id']).toBe('stub-session-1'); + expect(res.body).toEqual({ + method: 'POST', + url: '/mcp', + authorization: 'Bearer caller-token', + contentType: 'application/json', + body: JSON.stringify(payload), + }); + }); + + test('preserves the query string', async () => { + const res = await request(app).post('/mcp?foo=bar').send({}); + expect(res.status).toBe(201); + expect(res.body.url).toBe('/mcp?foo=bar'); + }); + + test('passes SSE responses through with their content type', async () => { + const res = await request(app).get('/mcp').buffer(true).parse( + // supertest has no default parser for text/event-stream; collect raw. + (msg, cb) => { + let data = ''; + msg.on('data', (chunk) => (data += chunk)); + msg.on('end', () => cb(null, data)); + }, + ); + expect(res.status).toBe(200); + expect(res.headers['content-type']).toBe('text/event-stream'); + expect(res.body).toContain('data: {"hello":"one"}'); + expect(res.body).toContain('data: {"hello":"two"}'); + }); + + test('502 with a JSON error when the MCP server is unreachable', async () => { + const res = await request(deadApp).post('/mcp').send({}); + expect(res.status).toBe(502); + expect(res.body.error.code).toBe('mcp_unavailable'); + }); + + test('not mounted when no MCP server is configured', async () => { + // Unproxied POST /mcp falls through to the app-level CSRF guard. + const res = await request(bareApp).post('/mcp').send({}); + expect(res.status).toBe(403); + expect(res.headers['mcp-session-id']).toBeUndefined(); + }); +}); diff --git a/create-a-container/middlewares/mcp-proxy.js b/create-a-container/middlewares/mcp-proxy.js new file mode 100644 index 00000000..2430cdf4 --- /dev/null +++ b/create-a-container/middlewares/mcp-proxy.js @@ -0,0 +1,93 @@ +/** + * Streaming reverse proxy for the MCP server (manager-control-program). + * + * The packaged MCP server listens on loopback (see + * manager-control-program.service); the Manager exposes it at /mcp on its + * public origin so MCP clients get TLS and a stable hostname for free, and + * per-request Authorization headers flow: MCP client -> this proxy -> MCP + * server -> back to this app's /api/v1 as a Bearer API key. + * + * Hand-rolled on node:http instead of a proxy dependency because the needs + * are narrow but strict: + * - bodies must pass through untouched (mount BEFORE express.json, which + * would otherwise consume the stream); + * - responses must stream incrementally (MCP's streamable HTTP transport + * uses long-lived text/event-stream responses); + * - no timeout on the upstream socket (SSE streams idle between events). + */ + +const http = require('http'); +const https = require('https'); + +// Hop-by-hop headers are connection-scoped and must not be forwarded +// (RFC 9110 §7.6.1). `host` is excluded so node sets it from the target. +const HOP_BY_HOP = new Set([ + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailer', + 'transfer-encoding', + 'upgrade', + 'host', +]); + +function filterHeaders(headers) { + const out = {}; + for (const [name, value] of Object.entries(headers)) { + if (!HOP_BY_HOP.has(name.toLowerCase())) out[name] = value; + } + return out; +} + +/** + * @param {string} targetBase Origin of the MCP server, e.g. "http://127.0.0.1:8100". + * The request path is preserved (/mcp -> ${targetBase}/mcp). + * @returns {import('express').RequestHandler} + */ +function createMcpProxy(targetBase) { + const base = new URL(targetBase); + const client = base.protocol === 'https:' ? https : http; + const agent = new client.Agent({ keepAlive: true }); + + return function mcpProxy(req, res) { + // req.originalUrl is the full path+query as received (req.url would have + // the /mcp mount prefix stripped by express). + const target = new URL(req.originalUrl, base); + + const upstream = client.request( + target, + { method: req.method, headers: filterHeaders(req.headers), agent }, + (upstreamRes) => { + res.writeHead(upstreamRes.statusCode, filterHeaders(upstreamRes.headers)); + // flushHeaders so SSE clients see the stream open before any event. + res.flushHeaders(); + upstreamRes.pipe(res); + }, + ); + + // SSE responses idle between events; never time the socket out. + upstream.setTimeout(0); + + upstream.on('error', () => { + if (res.headersSent) { + res.destroy(); + return; + } + res.writeHead(502, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + error: { code: 'mcp_unavailable', message: 'MCP server is unreachable' }, + }), + ); + }); + + // Client gone (or response finished) — release the upstream socket. + res.on('close', () => upstream.destroy()); + + req.pipe(upstream); + }; +} + +module.exports = { createMcpProxy }; diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile new file mode 100644 index 00000000..c82497b0 --- /dev/null +++ b/manager-control-program/Makefile @@ -0,0 +1,79 @@ +.DEFAULT_GOAL := help + +PREFIX ?= /opt/opensource-server +DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/manager-control-program +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system + +INSTALL := install +INSTALL_DATA := $(INSTALL) -m 0644 + +# The production target is the manager image: Debian 13 / amd64 / CPython 3.13 +# (python3 comes with the base image). uv selects wheels for that platform +# regardless of the build host's own interpreter, and downloads a managed +# CPython for resolution if the host has none (the builder image doesn't). +PYTHON_VERSION := 3.13 +PYTHON_PLATFORM := x86_64-unknown-linux-gnu + +REQUIREMENTS := .pkg-requirements.txt + +.PHONY: help deps build dev test install deb rpm apk package clean + +help: + @echo "manager-control-program — MCP server for create-a-container." + @echo "" + @echo "Targets:" + @echo " deps install dependencies (uv sync)" + @echo " build nothing to build (kept for the repo-wide target set)" + @echo " dev run the MCP server locally over stdio (uv)" + @echo " test run tests (none; kept for the repo-wide target set)" + @echo " install stage the app with vendored dependencies into DESTDIR" + @echo " clean remove the venv and vendoring artifacts" + @echo " help show this message" + @echo "" + @echo "This component ships inside the opensource-server package: the" + @echo "create-a-container Makefile calls 'install' here while staging." + @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + +deps: + uv sync --frozen + +build: + +dev: + uv run manager-control-program + +# No test suite yet; kept as a no-op so the repo-wide `make test` loop stays +# uniform if this component is ever added to the root COMPONENTS list. +test: + +# Stage the application with vendored dependencies so the target host needs +# only /usr/bin/python3 — no uv, pip, network access, or venv. The locked +# requirements are exported from uv.lock and installed as prebuilt wheels +# directly into the app directory; the systemd unit runs +# `python3 -m manager_control_program.server` with PYTHONPATH pointed there. +install: + $(INSTALL) -d $(DESTBIN) + uv export --frozen --no-dev --no-emit-project -o $(REQUIREMENTS) + uv pip install \ + --python $(PYTHON_VERSION) \ + --python-version $(PYTHON_VERSION) \ + --python-platform $(PYTHON_PLATFORM) \ + --only-binary :all: \ + --target $(DESTBIN) \ + --requirements $(REQUIREMENTS) + rm -f $(REQUIREMENTS) + cp -a manager_control_program $(DESTBIN)/ + find $(DESTBIN) -type d -name __pycache__ -prune -exec rm -rf {} + + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/manager-control-program.service $(UNIT_DIR)/ + +# This component does not build its own package; it ships inside the +# opensource-server package built by create-a-container. +deb rpm apk package: + @echo "manager-control-program ships inside the opensource-server package;" \ + "build it with 'make -C ../create-a-container $@'" + +clean: + rm -rf .venv $(REQUIREMENTS) diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 07e922d4..5545c3c6 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -25,6 +25,23 @@ API_BASE_URL=https://containers.example.com AUTH_TOKEN=your-api-key uv run manag End users don't need a clone — `uvx` runs it straight from git (see the [MCP Server guide](../mie-opensource-landing/docs/users/mcp-server.md)). +### Production (packaged) + +This component ships inside the `opensource-server` package built by +[create-a-container](../create-a-container/): its `make install` calls this +directory's `install` target, which stages the app at +`/opt/opensource-server/manager-control-program` with **all Python +dependencies vendored** (uv exports the lockfile and installs prebuilt wheels +for Debian 13 / amd64 / CPython 3.13). The target host needs only +`/usr/bin/python3` — no uv, pip, venv, or network access. + +It runs as the `manager-control-program.service` systemd unit: HTTP transport +on `127.0.0.1:8100`, started via `python3 -m manager_control_program.server` +with `PYTHONPATH` pointed at the vendored directory. The Manager +reverse-proxies `/mcp` to it, so MCP clients connect through the Manager's +public origin with TLS. Optional overrides (e.g. `SERVER_PORT`) go in +`/etc/default/manager-control-program`. + ## Configuration Configuration is read from environment variables: diff --git a/manager-control-program/contrib/systemd/manager-control-program.service b/manager-control-program/contrib/systemd/manager-control-program.service new file mode 100644 index 00000000..2d3e6252 --- /dev/null +++ b/manager-control-program/contrib/systemd/manager-control-program.service @@ -0,0 +1,29 @@ +[Unit] +Description=MCP server for create-a-container (Model Context Protocol over HTTP) +# The OpenAPI spec is fetched from the Manager at startup, so order after it +# and keep retrying (Restart/RestartSec below) until the Manager is reachable. +After=network.target container-creator.service +Wants=container-creator.service + +[Service] +Type=simple +User=root +WorkingDirectory=/opt/opensource-server/manager-control-program +# Dependencies are vendored into the app directory at package build time; +# no uv, pip, or venv exists on the host. +Environment=PYTHONPATH=/opt/opensource-server/manager-control-program +Environment=API_BASE_URL=http://127.0.0.1:3000 +# HTTP transport on loopback: the Manager proxies /mcp here, adding TLS and a +# public hostname. Each MCP client authenticates per-request with its own +# Bearer API key, which the server forwards to the API — no AUTH_TOKEN needed. +Environment=SERVER_TRANSPORT=http +Environment=SERVER_HOST=127.0.0.1 +Environment=SERVER_PORT=8100 +# Optional overrides (e.g. SERVER_PORT, LOG_LEVEL); absent by default. +EnvironmentFile=-/etc/default/manager-control-program +ExecStart=/usr/bin/python3 -m manager_control_program.server +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/mie-opensource-landing/docs/users/mcp-server.md b/mie-opensource-landing/docs/users/mcp-server.md index c4bfc9f5..9380978f 100644 --- a/mie-opensource-landing/docs/users/mcp-server.md +++ b/mie-opensource-landing/docs/users/mcp-server.md @@ -5,7 +5,7 @@ Use the MCP (Model Context Protocol) server to manage your containers through AI There are two ways to use it: - **Run it locally (stdio)** — VS Code launches a private copy of the server for you, authenticated with your API key from the environment. -- **Connect to a shared server (HTTP)** — someone hosts one MCP server for everyone; you connect to its URL and send your own API key with each request. +- **Connect to the built-in server (HTTP)** — packaged deployments already serve MCP at `{{ manager_url }}/mcp`; you just send your API key with each request. Nothing to install, so start here. **Prerequisites:** - VS Code with an MCP-capable AI extension (e.g., [GitHub Copilot](https://marketplace.visualstudio.com/items?itemName=GitHub.copilot), [Claude for VS Code](https://marketplace.visualstudio.com/items?itemName=AnthropicPublicBeta.claude-for-vscode)) @@ -46,16 +46,16 @@ Open your VS Code settings (`Ctrl+Shift+P` → "Preferences: Open User Settings You can also add this to a **workspace-level** `.vscode/settings.json` to share the config with your team (omit `AUTH_TOKEN` and set it as a system environment variable instead). -## Option B: Connect to a Shared Server (HTTP) +## Option B: Connect to the Built-in Server (HTTP) -If a shared MCP server is available, you don't need `uv` or any local install — point VS Code at its URL and send your API key in the `Authorization` header. The server forwards that header to the API on every request, so you act under your own account and permissions: +Every packaged deployment ships the MCP server as a system service and exposes it at `/mcp` on the same domain as the web UI. Nothing to install — point VS Code at that URL and send your API key in the `Authorization` header. Your key is forwarded to the API on every request, so you act under your own account and permissions: ```json { "mcp": { "servers": { "container-manager": { - "url": "https://your-mcp-host:8000/mcp", + "url": "{{ manager_url }}/mcp", "headers": { "Authorization": "Bearer your-api-key" } @@ -65,9 +65,16 @@ If a shared MCP server is available, you don't need `uv` or any local install } ``` -### Hosting a Shared Server +### How the Built-in Server Is Hosted -Start the server with an HTTP transport instead of the default stdio: +The `opensource-server` package installs the MCP server with vendored Python dependencies at `/opt/opensource-server/manager-control-program` and runs it as the `manager-control-program.service` systemd unit, listening on loopback (`127.0.0.1:8100`). The Manager reverse-proxies `/mcp` to it, providing the public hostname and TLS. It is enabled by default; admins can: + +- override settings (e.g. `SERVER_PORT`) in `/etc/default/manager-control-program`, then restart the service +- disable it entirely with `systemctl disable --now manager-control-program.service` (and unset `MCP_SERVER_URL` for `container-creator.service` to remove the proxy route) + +### Hosting a Standalone Server + +Outside a packaged deployment, the same HTTP mode runs anywhere `uv` is available: ```bash API_BASE_URL=https://your-server-domain SERVER_TRANSPORT=http \ From 63f8130a91a68d2fb23d727a1c3e7af63cb4eb45 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 08:38:17 -0400 Subject: [PATCH 05/12] fix(create-a-container): depend on python3 for the packaged MCP server manager-control-program.service execs /usr/bin/python3, but the package never declared it, so on a fresh install without python3 the service failed to start. Pin the range to the vendored wheels' ABI: compiled wheels are cp313-only, so python3 >= 3.13 and < 3.14 (fpm translates the operators per package format; verified deb and rpm output). The Makefile comment cross-references the pin so a future PYTHON_VERSION bump updates both together. --- create-a-container/.fpm | 2 ++ manager-control-program/Makefile | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/create-a-container/.fpm b/create-a-container/.fpm index baa9b50a..ef39b067 100644 --- a/create-a-container/.fpm +++ b/create-a-container/.fpm @@ -9,6 +9,8 @@ --depends opensource-agent --depends opensource-docs --depends nodejs +--depends "python3 >= 3.13" +--depends "python3 < 3.14" --depends sudo --depends libc6 --depends libgcc-s1 diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile index c82497b0..0719e5c1 100644 --- a/manager-control-program/Makefile +++ b/manager-control-program/Makefile @@ -8,10 +8,14 @@ UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system INSTALL := install INSTALL_DATA := $(INSTALL) -m 0644 -# The production target is the manager image: Debian 13 / amd64 / CPython 3.13 -# (python3 comes with the base image). uv selects wheels for that platform -# regardless of the build host's own interpreter, and downloads a managed -# CPython for resolution if the host has none (the builder image doesn't). +# The production target is Debian 13 / amd64 / CPython 3.13. uv selects +# wheels for that platform regardless of the build host's own interpreter, +# and downloads a managed CPython for resolution if the host has none (the +# builder image doesn't). +# +# Compiled wheels are ABI-locked to this exact minor version (cp313), so the +# opensource-server package pins its python3 dependency to the matching +# range in create-a-container/.fpm — update both together when bumping. PYTHON_VERSION := 3.13 PYTHON_PLATFORM := x86_64-unknown-linux-gnu From 40398411fcd98861d947b22d72e32372d46d49e4 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 09:50:43 -0400 Subject: [PATCH 06/12] refactor(manager-control-program): ship the MCP server as its own opensource-mcp package Split the MCP server out of the opensource-server package into a standalone opensource-mcp package instead of bundling it inside create-a-container's build. - manager-control-program now builds its own deb/rpm/apk: the Makefile gains a package target (stage into .pkg/buildroot, then fpm) and a .fpm declaring the opensource-mcp package, with the python3 >= 3.13, < 3.14 dependency pin moved here from create-a-container/.fpm (it tracks the vendored cp313 wheels' ABI). Own postinstall/preremove enable/disable the unit. - The systemd unit is renamed manager-control-program.service -> opensource-mcp.service, with its EnvironmentFile at /etc/default/opensource-mcp. - create-a-container no longer calls this component's install target or ships its units; it just --depends opensource-mcp. Its postinstall/preremove drop the MCP unit, and MCP_SERVER_URL still points the reverse proxy at the loopback service. - Root Makefile adds manager-control-program to COMPONENTS (ordered before create-a-container so the dependency builds first); the release workflow, builder Dockerfile, manager image, and docs move to a four-package model, and .gitignore covers the new build artifacts (.pkg/, *.deb/rpm/apk). Verified: `make deb` in manager-control-program produces opensource-mcp with the correct python3 dependency range, the opensource-mcp.service unit, and the vendored dependency tree; `make deb` in create-a-container produces opensource-server depending on opensource-mcp and no longer containing the vendored Python code. --- .github/workflows/release.yml | 4 +- Makefile | 2 +- create-a-container/.fpm | 3 +- create-a-container/Makefile | 3 -- create-a-container/README.md | 8 ++-- create-a-container/contrib/postinstall.sh | 2 +- create-a-container/contrib/preremove.sh | 2 +- .../contrib/systemd/container-creator.service | 2 +- create-a-container/example.env | 4 +- create-a-container/middlewares/mcp-proxy.js | 2 +- images/builder/Dockerfile | 4 +- images/manager/Dockerfile | 10 ++--- manager-control-program/.fpm | 13 ++++++ manager-control-program/.gitignore | 7 ++++ manager-control-program/Makefile | 40 ++++++++++++------- manager-control-program/README.md | 23 ++++++----- .../contrib/postinstall.sh | 19 +++++++++ manager-control-program/contrib/preremove.sh | 24 +++++++++++ ...program.service => opensource-mcp.service} | 2 +- .../docs/developers/release-pipeline.md | 6 ++- .../docs/users/mcp-server.md | 6 +-- 21 files changed, 130 insertions(+), 56 deletions(-) create mode 100644 manager-control-program/.fpm create mode 100644 manager-control-program/contrib/postinstall.sh create mode 100644 manager-control-program/contrib/preremove.sh rename manager-control-program/contrib/systemd/{manager-control-program.service => opensource-mcp.service} (95%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43360594..8b43cbba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Release Packages # Triggered by publishing a GitHub release (full or prerelease). Builds the -# three Debian packages and uploads them to the release that triggered the +# four Debian packages and uploads them to the release that triggered the # workflow, together with flat APT repository metadata (Packages, Packages.gz) # so the release can be used directly as an apt source: # @@ -50,7 +50,7 @@ jobs: gzip -k9f Packages # Stable-name aliases for one-off downloads (the _latest URLs resolve # via releases/latest/download for the newest non-prerelease release). - for pkg in opensource-server opensource-docs opensource-agent; do + for pkg in opensource-server opensource-docs opensource-agent opensource-mcp; do f=$(ls ${pkg}_*.deb | head -1) [ -n "$f" ] && cp -f "$f" "${pkg}_latest.deb" done diff --git a/Makefile b/Makefile index 2789cb02..b1da4341 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPONENTS := agent mie-opensource-landing create-a-container +COMPONENTS := agent mie-opensource-landing manager-control-program create-a-container PACKAGER ?= deb # Forwarded to every component Makefile. diff --git a/create-a-container/.fpm b/create-a-container/.fpm index ef39b067..e4e212e9 100644 --- a/create-a-container/.fpm +++ b/create-a-container/.fpm @@ -8,9 +8,8 @@ --description "MIE opensource-server cluster manager (create-a-container): web UI and REST API for self-service LXC container hosting on Proxmox VE, with automated DNS and reverse-proxy configuration, LDAP authentication and ACME TLS orchestration." --depends opensource-agent --depends opensource-docs +--depends opensource-mcp --depends nodejs ---depends "python3 >= 3.13" ---depends "python3 < 3.14" --depends sudo --depends libc6 --depends libgcc-s1 diff --git a/create-a-container/Makefile b/create-a-container/Makefile index b879ddb8..76334078 100644 --- a/create-a-container/Makefile +++ b/create-a-container/Makefile @@ -101,9 +101,6 @@ install: build $(INSTALL_DATA) contrib/systemd/job-runner.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)/etc/logrotate.d $(INSTALL_DATA) contrib/opensource-server.logrotate $(DESTDIR)/etc/logrotate.d/opensource-server - # The MCP server ships inside this package (vendored Python deps, no uv - # on the target); the Manager proxies /mcp to it (see MCP_SERVER_URL). - $(MAKE) -C ../manager-control-program install DESTDIR=$(DESTDIR) PREFIX=$(PREFIX) PACKAGER ?= deb package: diff --git a/create-a-container/README.md b/create-a-container/README.md index dbd6e038..c20ae675 100644 --- a/create-a-container/README.md +++ b/create-a-container/README.md @@ -46,10 +46,10 @@ The Manager is not installed by hand in production. It ships as: - distribution **packages** built from this directory with `make deb`, `make rpm`, or `make apk` (via [fpm](https://fpm.readthedocs.io/)), which install the app under `/opt/opensource-server/create-a-container` and register the - `container-creator`, `job-runner`, and `manager-control-program` systemd - services. The last is the bundled [MCP server](../manager-control-program/) - (Python deps vendored at build time); the Manager reverse-proxies `/mcp` to - it (`MCP_SERVER_URL`). + `container-creator` and `job-runner` systemd services. The package depends + on `opensource-mcp` — the [MCP server](../manager-control-program/) as its + own package — and the Manager reverse-proxies `/mcp` to its service + (`MCP_SERVER_URL`). In both cases the app runs `server.js` (HTTP API + UI) and `job-runner.js` (background worker). Database connection settings come from the environment (see diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh index 8ef3a2b3..e92c7acf 100755 --- a/create-a-container/contrib/postinstall.sh +++ b/create-a-container/contrib/postinstall.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service manager-control-program.service" +UNITS="container-creator.service job-runner.service" # Nothing to do without systemctl (non-systemd container/chroot). command -v systemctl >/dev/null 2>&1 || exit 0 diff --git a/create-a-container/contrib/preremove.sh b/create-a-container/contrib/preremove.sh index b44cb185..a3ebf879 100755 --- a/create-a-container/contrib/preremove.sh +++ b/create-a-container/contrib/preremove.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e -UNITS="container-creator.service job-runner.service manager-control-program.service" +UNITS="container-creator.service job-runner.service" # $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final # removal). Only act on a real removal, not an upgrade. diff --git a/create-a-container/contrib/systemd/container-creator.service b/create-a-container/contrib/systemd/container-creator.service index d291ab1c..80ecae1c 100644 --- a/create-a-container/contrib/systemd/container-creator.service +++ b/create-a-container/contrib/systemd/container-creator.service @@ -11,7 +11,7 @@ ExecStart=/usr/bin/node /opt/opensource-server/create-a-container/server.js Restart=on-failure Environment=NODE_ENV=production Environment=ACCESS_LOG=/var/log/opensource-server/access.log -# Reverse-proxy /mcp to the packaged MCP server (manager-control-program.service). +# Reverse-proxy /mcp to the packaged MCP server (opensource-mcp.service). Environment=MCP_SERVER_URL=http://127.0.0.1:8100 EnvironmentFile=/etc/default/container-creator LogsDirectory=opensource-server diff --git a/create-a-container/example.env b/create-a-container/example.env index e92ba028..9c6e8ab5 100644 --- a/create-a-container/example.env +++ b/create-a-container/example.env @@ -31,8 +31,8 @@ ACCESS_LOG= # --- MCP server reverse proxy (optional) --- # When set, the Manager reverse-proxies /mcp to this origin — the # manager-control-program MCP server in HTTP mode. The packaged systemd -# deployment sets this to http://127.0.0.1:8100 (see -# manager-control-program.service); leave unset to disable the proxy. +# deployment sets this to http://127.0.0.1:8100 (see opensource-mcp.service); +# leave unset to disable the proxy. MCP_SERVER_URL= # --- OIDC / single sign-on (optional) --- diff --git a/create-a-container/middlewares/mcp-proxy.js b/create-a-container/middlewares/mcp-proxy.js index 2430cdf4..67ad84a4 100644 --- a/create-a-container/middlewares/mcp-proxy.js +++ b/create-a-container/middlewares/mcp-proxy.js @@ -2,7 +2,7 @@ * Streaming reverse proxy for the MCP server (manager-control-program). * * The packaged MCP server listens on loopback (see - * manager-control-program.service); the Manager exposes it at /mcp on its + * opensource-mcp.service); the Manager exposes it at /mcp on its * public origin so MCP clients get TLS and a stable hostname for free, and * per-request Authorization headers flow: MCP client -> this proxy -> MCP * server -> back to this app's /api/v1 as a Bearer API key. diff --git a/images/builder/Dockerfile b/images/builder/Dockerfile index a44ce17d..47180ff5 100644 --- a/images/builder/Dockerfile +++ b/images/builder/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Builds the three opensource-server Debian packages with the component +# Builds the four opensource-server Debian packages with the component # Makefiles + fpm. The final stage contains only the built .deb artifacts so # the runtime images can install them with: # @@ -44,7 +44,7 @@ RUN gem install --no-document fpm -v "${FPM_VERSION}" COPY . /usr/src/opensource-server WORKDIR /usr/src/opensource-server -# Build all three packages into ./dist. +# Build all packages into ./dist. RUN make deb # Artifact-only stage. diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index b76bdfc4..d8d7a38a 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -26,13 +26,13 @@ COPY images/manager/wait-proxmox-regenerate-snakeoil.conf /etc/systemd/system/po # First-boot database initialization. COPY images/manager/container-creator-init.service /etc/systemd/system/container-creator-init.service -# Install the manager and docs packages built by the builder image (the agent -# package is already present from the parent image; apt resolves the -# opensource-server -> opensource-docs dependency), then enable the init -# service. +# Install the manager, docs, and MCP packages built by the builder image (the +# agent package is already present from the parent image; apt resolves the +# opensource-server -> opensource-docs/opensource-mcp dependencies), then +# enable the init service. RUN --mount=from=builder,source=/dist,target=/dist \ apt-get update && \ - apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-server_*.deb && \ + apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-mcp_*.deb /dist/opensource-server_*.deb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ systemctl enable container-creator-init.service diff --git a/manager-control-program/.fpm b/manager-control-program/.fpm new file mode 100644 index 00000000..32109c05 --- /dev/null +++ b/manager-control-program/.fpm @@ -0,0 +1,13 @@ +--name opensource-mcp +--architecture amd64 +--license Apache-2.0 +--maintainer "Medical Informatics Engineering " +--vendor "Medical Informatics Engineering" +--url "https://github.com/mieweb/opensource-server" +--category admin +--description "MIE opensource-server MCP server (manager-control-program): exposes the create-a-container REST API as Model Context Protocol tools over HTTP, forwarding each caller's Authorization header so every MCP client acts under its own API key. Python dependencies are vendored; runs on the system python3." +--depends "python3 >= 3.13" +--depends "python3 < 3.14" +--deb-no-default-config-files +--after-install contrib/postinstall.sh +--before-remove contrib/preremove.sh diff --git a/manager-control-program/.gitignore b/manager-control-program/.gitignore index 532a9421..cdae812c 100644 --- a/manager-control-program/.gitignore +++ b/manager-control-program/.gitignore @@ -3,3 +3,10 @@ build/ *.egg-info/ __pycache__/ *.pyc + +# packaging build artifacts +/.pkg/ +/*.deb +/*.rpm +/*.apk +.pkg-requirements.txt diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile index 0719e5c1..23708984 100644 --- a/manager-control-program/Makefile +++ b/manager-control-program/Makefile @@ -14,17 +14,20 @@ INSTALL_DATA := $(INSTALL) -m 0644 # builder image doesn't). # # Compiled wheels are ABI-locked to this exact minor version (cp313), so the -# opensource-server package pins its python3 dependency to the matching -# range in create-a-container/.fpm — update both together when bumping. +# opensource-mcp package pins its python3 dependency to the matching range +# in .fpm — update both together when bumping. PYTHON_VERSION := 3.13 PYTHON_PLATFORM := x86_64-unknown-linux-gnu REQUIREMENTS := .pkg-requirements.txt +# Staging directory for packaging +STAGE := $(CURDIR)/.pkg/buildroot + .PHONY: help deps build dev test install deb rpm apk package clean help: - @echo "manager-control-program — MCP server for create-a-container." + @echo "manager-control-program — builds the opensource-mcp package." @echo "" @echo "Targets:" @echo " deps install dependencies (uv sync)" @@ -32,12 +35,12 @@ help: @echo " dev run the MCP server locally over stdio (uv)" @echo " test run tests (none; kept for the repo-wide target set)" @echo " install stage the app with vendored dependencies into DESTDIR" - @echo " clean remove the venv and vendoring artifacts" + @echo " deb build the .deb package" + @echo " rpm build the .rpm package" + @echo " apk build the .apk package" + @echo " clean remove the venv, staging dir and built packages" @echo " help show this message" @echo "" - @echo "This component ships inside the opensource-server package: the" - @echo "create-a-container Makefile calls 'install' here while staging." - @echo "" @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." deps: @@ -71,13 +74,22 @@ install: cp -a manager_control_program $(DESTBIN)/ find $(DESTBIN) -type d -name __pycache__ -prune -exec rm -rf {} + $(INSTALL) -d $(UNIT_DIR) - $(INSTALL_DATA) contrib/systemd/manager-control-program.service $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/opensource-mcp.service $(UNIT_DIR)/ + +PACKAGER ?= deb +package: + rm -rf $(STAGE) + $(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX) + fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \ + -C $(STAGE) -p $(CURDIR) -f -# This component does not build its own package; it ships inside the -# opensource-server package built by create-a-container. -deb rpm apk package: - @echo "manager-control-program ships inside the opensource-server package;" \ - "build it with 'make -C ../create-a-container $@'" +deb: + $(MAKE) package PACKAGER=deb +rpm: + $(MAKE) package PACKAGER=rpm +apk: + $(MAKE) package PACKAGER=apk clean: - rm -rf .venv $(REQUIREMENTS) + rm -rf .venv $(REQUIREMENTS) $(STAGE) + rm -f *.deb *.rpm *.apk diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 5545c3c6..7d6e03d2 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -27,20 +27,21 @@ End users don't need a clone — `uvx` runs it straight from git (see the ### Production (packaged) -This component ships inside the `opensource-server` package built by -[create-a-container](../create-a-container/): its `make install` calls this -directory's `install` target, which stages the app at -`/opt/opensource-server/manager-control-program` with **all Python -dependencies vendored** (uv exports the lockfile and installs prebuilt wheels -for Debian 13 / amd64 / CPython 3.13). The target host needs only -`/usr/bin/python3` — no uv, pip, venv, or network access. - -It runs as the `manager-control-program.service` systemd unit: HTTP transport -on `127.0.0.1:8100`, started via `python3 -m manager_control_program.server` +This directory builds the **`opensource-mcp`** package (`make deb`, `rpm`, or +`apk` via [fpm](https://fpm.readthedocs.io/)); the `opensource-server` package +depends on it, so it installs automatically with the Manager. The package +stages the app at `/opt/opensource-server/manager-control-program` with **all +Python dependencies vendored** (uv exports the lockfile and installs prebuilt +wheels for Debian 13 / amd64 / CPython 3.13). The target host needs only +`python3` 3.13 — declared as a package dependency — with no uv, pip, venv, or +network access. + +It runs as the `opensource-mcp.service` systemd unit: HTTP transport on +`127.0.0.1:8100`, started via `python3 -m manager_control_program.server` with `PYTHONPATH` pointed at the vendored directory. The Manager reverse-proxies `/mcp` to it, so MCP clients connect through the Manager's public origin with TLS. Optional overrides (e.g. `SERVER_PORT`) go in -`/etc/default/manager-control-program`. +`/etc/default/opensource-mcp`. ## Configuration diff --git a/manager-control-program/contrib/postinstall.sh b/manager-control-program/contrib/postinstall.sh new file mode 100644 index 00000000..5bfa9e52 --- /dev/null +++ b/manager-control-program/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +UNITS="opensource-mcp.service" + +# Nothing to do without systemctl (non-systemd container/chroot). +command -v systemctl >/dev/null 2>&1 || exit 0 + +# `systemctl enable` only creates static symlinks, so it works during an image +# build too +systemctl enable $UNITS + +# daemon-reload and restart need a running systemd; skip them at build time. +if [ -d /run/systemd/system ]; then + systemctl daemon-reload + systemctl restart $UNITS +fi + +exit 0 diff --git a/manager-control-program/contrib/preremove.sh b/manager-control-program/contrib/preremove.sh new file mode 100644 index 00000000..9c4f35c5 --- /dev/null +++ b/manager-control-program/contrib/preremove.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -e + +UNITS="opensource-mcp.service" + +# $1 is "remove"/"purge" (deb) or the remaining-version count (rpm: 0 on final +# removal). Only act on a real removal, not an upgrade. +case "${1:-}" in + upgrade|1) + # rpm upgrade ("1") / deb upgrade: leave units in place. + exit 0 + ;; +esac + +# Nothing to do without systemctl +command -v systemctl >/dev/null 2>&1 || exit 0 + +# Stopping needs a running systemd; disabling (symlink removal) does not. +if [ -d /run/systemd/system ]; then + systemctl stop $UNITS +fi +systemctl disable $UNITS + +exit 0 diff --git a/manager-control-program/contrib/systemd/manager-control-program.service b/manager-control-program/contrib/systemd/opensource-mcp.service similarity index 95% rename from manager-control-program/contrib/systemd/manager-control-program.service rename to manager-control-program/contrib/systemd/opensource-mcp.service index 2d3e6252..39b38be5 100644 --- a/manager-control-program/contrib/systemd/manager-control-program.service +++ b/manager-control-program/contrib/systemd/opensource-mcp.service @@ -20,7 +20,7 @@ Environment=SERVER_TRANSPORT=http Environment=SERVER_HOST=127.0.0.1 Environment=SERVER_PORT=8100 # Optional overrides (e.g. SERVER_PORT, LOG_LEVEL); absent by default. -EnvironmentFile=-/etc/default/manager-control-program +EnvironmentFile=-/etc/default/opensource-mcp ExecStart=/usr/bin/python3 -m manager_control_program.server Restart=on-failure RestartSec=5 diff --git a/mie-opensource-landing/docs/developers/release-pipeline.md b/mie-opensource-landing/docs/developers/release-pipeline.md index df33f01b..a1b68747 100644 --- a/mie-opensource-landing/docs/developers/release-pipeline.md +++ b/mie-opensource-landing/docs/developers/release-pipeline.md @@ -2,7 +2,7 @@ {{ contributor_warning }} -The three deployable components are packaged as Debian packages and published +The four deployable components are packaged as Debian packages and published to GitHub Releases as a flat APT repository. The same component build commands are reused by local development, the container images, and CI. @@ -13,12 +13,14 @@ are reused by local development, the container images, and CI. | [`create-a-container/`](https://github.com/mieweb/opensource-server/tree/main/create-a-container) | `opensource-server` | amd64 | Manager web app, job runner, systemd units | | [`mie-opensource-landing/`](https://github.com/mieweb/opensource-server/tree/main/mie-opensource-landing) | `opensource-docs` | all | Prebuilt documentation site | | [`agent/`](https://github.com/mieweb/opensource-server/tree/main/agent) | `opensource-agent` | all | Check-in agent, config templates, systemd timer, error pages | +| [`manager-control-program/`](https://github.com/mieweb/opensource-server/tree/main/manager-control-program) | `opensource-mcp` | amd64 | MCP server with vendored Python deps, systemd unit | Everything installs under the `/opt/opensource-server` prefix, matching the paths referenced by the systemd units and the agent-rendered nginx configuration. `opensource-server` depends on `opensource-agent` and `opensource-docs` because the manager's nginx config -serves the agent's error pages and the docs site. +serves the agent's error pages and the docs site, and on `opensource-mcp`, +whose MCP service the manager reverse-proxies at `/mcp`. ## The component Makefile contract diff --git a/mie-opensource-landing/docs/users/mcp-server.md b/mie-opensource-landing/docs/users/mcp-server.md index 9380978f..b1bbb98d 100644 --- a/mie-opensource-landing/docs/users/mcp-server.md +++ b/mie-opensource-landing/docs/users/mcp-server.md @@ -67,10 +67,10 @@ Every packaged deployment ships the MCP server as a system service and exposes i ### How the Built-in Server Is Hosted -The `opensource-server` package installs the MCP server with vendored Python dependencies at `/opt/opensource-server/manager-control-program` and runs it as the `manager-control-program.service` systemd unit, listening on loopback (`127.0.0.1:8100`). The Manager reverse-proxies `/mcp` to it, providing the public hostname and TLS. It is enabled by default; admins can: +The `opensource-mcp` package (installed automatically as a dependency of `opensource-server`) ships the MCP server with vendored Python dependencies at `/opt/opensource-server/manager-control-program` and runs it as the `opensource-mcp.service` systemd unit, listening on loopback (`127.0.0.1:8100`). The Manager reverse-proxies `/mcp` to it, providing the public hostname and TLS. It is enabled by default; admins can: -- override settings (e.g. `SERVER_PORT`) in `/etc/default/manager-control-program`, then restart the service -- disable it entirely with `systemctl disable --now manager-control-program.service` (and unset `MCP_SERVER_URL` for `container-creator.service` to remove the proxy route) +- override settings (e.g. `SERVER_PORT`) in `/etc/default/opensource-mcp`, then restart the service +- disable it entirely with `systemctl disable --now opensource-mcp.service` (and unset `MCP_SERVER_URL` for `container-creator.service` to remove the proxy route) ### Hosting a Standalone Server From c6d680d31fdb96aba78331df7fdd413c63891d1c Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 09:56:56 -0400 Subject: [PATCH 07/12] refactor(systemd): move runtime config env vars out of unit files into /etc/default The units this branch added hardcoded configuration via Environment= lines, which admins can't change without editing the shipped unit. Move that config into /etc/default so it can be edited (and survives package upgrades), keeping only fixed runtime constants in the units. - opensource-mcp: ship /etc/default/opensource-mcp (a package config file via fpm --config-files) with API_BASE_URL and SERVER_TRANSPORT/HOST/PORT; the unit now keeps only PYTHONPATH and reads the (now required) EnvironmentFile. Makefile installs it. - container-creator: MCP_SERVER_URL moves from the unit into /etc/default/container-creator, appended by container-creator-init alongside the DB settings it already generates. - README/docs: describe the settings as living in the packaged config file rather than as inline unit overrides. Verified: opensource-mcp.deb registers /etc/default/opensource-mcp as a conffile carrying the four values and its unit body no longer contains them; container-creator's unit no longer sets MCP_SERVER_URL and the init service writes it into /etc/default/container-creator. --- .../contrib/systemd/container-creator.service | 4 ++-- images/manager/container-creator-init.service | 3 ++- manager-control-program/.fpm | 1 + manager-control-program/Makefile | 2 ++ manager-control-program/README.md | 6 ++++-- .../contrib/default/opensource-mcp | 18 ++++++++++++++++++ .../contrib/systemd/opensource-mcp.service | 12 +++--------- .../docs/users/mcp-server.md | 2 +- 8 files changed, 33 insertions(+), 15 deletions(-) create mode 100644 manager-control-program/contrib/default/opensource-mcp diff --git a/create-a-container/contrib/systemd/container-creator.service b/create-a-container/contrib/systemd/container-creator.service index 80ecae1c..710fab6e 100644 --- a/create-a-container/contrib/systemd/container-creator.service +++ b/create-a-container/contrib/systemd/container-creator.service @@ -11,8 +11,8 @@ ExecStart=/usr/bin/node /opt/opensource-server/create-a-container/server.js Restart=on-failure Environment=NODE_ENV=production Environment=ACCESS_LOG=/var/log/opensource-server/access.log -# Reverse-proxy /mcp to the packaged MCP server (opensource-mcp.service). -Environment=MCP_SERVER_URL=http://127.0.0.1:8100 +# Runtime configuration (DB connection, MCP_SERVER_URL for the /mcp reverse +# proxy to opensource-mcp.service, etc.) is sourced from this file. EnvironmentFile=/etc/default/container-creator LogsDirectory=opensource-server diff --git a/images/manager/container-creator-init.service b/images/manager/container-creator-init.service index e898e2ee..32082007 100644 --- a/images/manager/container-creator-init.service +++ b/images/manager/container-creator-init.service @@ -22,7 +22,8 @@ ExecStart=/bin/bash -e -c '\ echo "POSTGRES_HOST=$${POSTGRES_HOST}" >> /etc/default/container-creator; \ echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> /etc/default/container-creator; \ echo "POSTGRES_USER=$${POSTGRES_USER}" >> /etc/default/container-creator; \ - echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator;' + echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator; \ + echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator;' [Install] WantedBy=multi-user.target diff --git a/manager-control-program/.fpm b/manager-control-program/.fpm index 32109c05..fb250ced 100644 --- a/manager-control-program/.fpm +++ b/manager-control-program/.fpm @@ -8,6 +8,7 @@ --description "MIE opensource-server MCP server (manager-control-program): exposes the create-a-container REST API as Model Context Protocol tools over HTTP, forwarding each caller's Authorization header so every MCP client acts under its own API key. Python dependencies are vendored; runs on the system python3." --depends "python3 >= 3.13" --depends "python3 < 3.14" +--config-files /etc/default/opensource-mcp --deb-no-default-config-files --after-install contrib/postinstall.sh --before-remove contrib/preremove.sh diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile index 23708984..89faeb16 100644 --- a/manager-control-program/Makefile +++ b/manager-control-program/Makefile @@ -75,6 +75,8 @@ install: find $(DESTBIN) -type d -name __pycache__ -prune -exec rm -rf {} + $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-mcp.service $(UNIT_DIR)/ + $(INSTALL) -d $(DESTDIR)/etc/default + $(INSTALL_DATA) contrib/default/opensource-mcp $(DESTDIR)/etc/default/opensource-mcp PACKAGER ?= deb package: diff --git a/manager-control-program/README.md b/manager-control-program/README.md index 7d6e03d2..a3dec0d6 100644 --- a/manager-control-program/README.md +++ b/manager-control-program/README.md @@ -40,8 +40,10 @@ It runs as the `opensource-mcp.service` systemd unit: HTTP transport on `127.0.0.1:8100`, started via `python3 -m manager_control_program.server` with `PYTHONPATH` pointed at the vendored directory. The Manager reverse-proxies `/mcp` to it, so MCP clients connect through the Manager's -public origin with TLS. Optional overrides (e.g. `SERVER_PORT`) go in -`/etc/default/opensource-mcp`. +public origin with TLS. Its runtime configuration (`API_BASE_URL`, +`SERVER_TRANSPORT`/`SERVER_HOST`/`SERVER_PORT`, `LOG_LEVEL`) lives in the +packaged config file `/etc/default/opensource-mcp` — edit it and restart the +service to apply. ## Configuration diff --git a/manager-control-program/contrib/default/opensource-mcp b/manager-control-program/contrib/default/opensource-mcp new file mode 100644 index 00000000..6c148bdd --- /dev/null +++ b/manager-control-program/contrib/default/opensource-mcp @@ -0,0 +1,18 @@ +# Environment for opensource-mcp.service (see manager_control_program.server). +# This file is sourced by systemd via EnvironmentFile; edit it and restart the +# service to apply changes. Marked as a package config file, so your edits are +# preserved across upgrades. + +# Base URL of the create-a-container REST API the MCP tools call. The packaged +# Manager listens on loopback; the OpenAPI spec is fetched from here at startup. +API_BASE_URL=http://127.0.0.1:3000 + +# HTTP transport on loopback: the Manager reverse-proxies /mcp here, adding TLS +# and a public hostname. Each MCP client authenticates per-request with its own +# Bearer API key, which the server forwards to the API — no AUTH_TOKEN needed. +SERVER_TRANSPORT=http +SERVER_HOST=127.0.0.1 +SERVER_PORT=8100 + +# Other supported overrides (defaults shown in comments): +# LOG_LEVEL=INFO diff --git a/manager-control-program/contrib/systemd/opensource-mcp.service b/manager-control-program/contrib/systemd/opensource-mcp.service index 39b38be5..5de0c220 100644 --- a/manager-control-program/contrib/systemd/opensource-mcp.service +++ b/manager-control-program/contrib/systemd/opensource-mcp.service @@ -12,15 +12,9 @@ WorkingDirectory=/opt/opensource-server/manager-control-program # Dependencies are vendored into the app directory at package build time; # no uv, pip, or venv exists on the host. Environment=PYTHONPATH=/opt/opensource-server/manager-control-program -Environment=API_BASE_URL=http://127.0.0.1:3000 -# HTTP transport on loopback: the Manager proxies /mcp here, adding TLS and a -# public hostname. Each MCP client authenticates per-request with its own -# Bearer API key, which the server forwards to the API — no AUTH_TOKEN needed. -Environment=SERVER_TRANSPORT=http -Environment=SERVER_HOST=127.0.0.1 -Environment=SERVER_PORT=8100 -# Optional overrides (e.g. SERVER_PORT, LOG_LEVEL); absent by default. -EnvironmentFile=-/etc/default/opensource-mcp +# Runtime configuration (API_BASE_URL, SERVER_TRANSPORT/HOST/PORT, LOG_LEVEL) +# lives in the packaged config file below — edit it and restart to apply. +EnvironmentFile=/etc/default/opensource-mcp ExecStart=/usr/bin/python3 -m manager_control_program.server Restart=on-failure RestartSec=5 diff --git a/mie-opensource-landing/docs/users/mcp-server.md b/mie-opensource-landing/docs/users/mcp-server.md index b1bbb98d..4701dde9 100644 --- a/mie-opensource-landing/docs/users/mcp-server.md +++ b/mie-opensource-landing/docs/users/mcp-server.md @@ -69,7 +69,7 @@ Every packaged deployment ships the MCP server as a system service and exposes i The `opensource-mcp` package (installed automatically as a dependency of `opensource-server`) ships the MCP server with vendored Python dependencies at `/opt/opensource-server/manager-control-program` and runs it as the `opensource-mcp.service` systemd unit, listening on loopback (`127.0.0.1:8100`). The Manager reverse-proxies `/mcp` to it, providing the public hostname and TLS. It is enabled by default; admins can: -- override settings (e.g. `SERVER_PORT`) in `/etc/default/opensource-mcp`, then restart the service +- change settings (e.g. `SERVER_PORT`) in the packaged config file `/etc/default/opensource-mcp`, then restart the service - disable it entirely with `systemctl disable --now opensource-mcp.service` (and unset `MCP_SERVER_URL` for `container-creator.service` to remove the proxy route) ### Hosting a Standalone Server From fa9b6b241117a1be53d9bd957189f3a8a8778933 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 10:44:19 -0400 Subject: [PATCH 08/12] fix(create-a-container): let the localhost agent bootstrap check-in past the app-level CSRF guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manager's own agent checks in over localhost during bootstrap before any site row or API key exists, so its POST /api/v1/agents carries neither a Bearer token nor a CSRF token. The agents router already exempts this via a route-level localhost bypass (checkinAuth), but the app-level csrfGuard in app.js runs first and only exempted Bearer-without-cookie requests — so the credential-less localhost check-in was rejected with 403 csrf_invalid before ever reaching the route. Mirror the route-level bypass in csrfGuard: skip CSRF for localhost requests (isLocalhostRequest also rejects proxied/remote clients via X-Real-IP / X-Forwarded-For, so this doesn't widen the trust surface). isLocalhostRequest is required lazily to avoid a load-order cycle with middlewares/index. Adds a regression test covering the localhost bootstrap check-in (not 403) and a proxied-remote credential-less check-in (still rejected). --- create-a-container/middlewares/api.js | 10 ++++ .../api/v1/__tests__/agents.checkin.test.js | 56 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 create-a-container/routers/api/v1/__tests__/agents.checkin.test.js diff --git a/create-a-container/middlewares/api.js b/create-a-container/middlewares/api.js index 6a95efda..ee4e6205 100644 --- a/create-a-container/middlewares/api.js +++ b/create-a-container/middlewares/api.js @@ -36,6 +36,16 @@ function csrfGuard(req, res, next) { if (req.method === 'GET' || req.method === 'HEAD' || req.method === 'OPTIONS') { return next(); } + // The manager's own agent checks in over localhost without credentials + // during bootstrap (no site or API key exists yet), so it can carry neither + // a Bearer token nor a CSRF token. The agents router applies the same + // localhost bypass at the route level (checkinAuth); mirror it here so this + // app-level guard doesn't reject the credential-less check-in first. Remote + // clients are never localhost (isLocalhostRequest also rejects proxied + // requests via X-Real-IP / X-Forwarded-For). Required lazily to avoid a + // load-order cycle with middlewares/index. + const { isLocalhostRequest } = require('./index'); + if (isLocalhostRequest(req)) return next(); const auth = req.get('Authorization') || ''; const hasBearer = auth.startsWith('Bearer '); const hasSessionCookie = !!(req.session && req.session.user); diff --git a/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js new file mode 100644 index 00000000..8c95751e --- /dev/null +++ b/create-a-container/routers/api/v1/__tests__/agents.checkin.test.js @@ -0,0 +1,56 @@ +/** + * Agent check-in auth — the manager's own agent bootstraps over localhost with + * no site row and no API key, so its POST /api/v1/agents carries neither a + * Bearer token nor a CSRF token. Two guards must let it through: the app-level + * csrfGuard (app.js) and the route-level checkinAuth (agents.js). This pins + * that the credential-less localhost check-in is NOT rejected with 403, while + * a non-localhost (proxied) credential-less check-in still is. + * + * supertest connects from 127.0.0.1, so requests are localhost by default; + * an X-Forwarded-For header with a public client IP makes isLocalhostRequest + * treat the request as proxied-remote (see middlewares/index.js). + */ + +const request = require('supertest'); +const { buildApp } = require('../../../../tests/helpers/app'); +const { resetDb, closeDb } = require('../../../../tests/helpers/db'); + +describe('POST /api/v1/agents check-in auth', () => { + let app; + + beforeEach(async () => { + await resetDb(); + app = buildApp(); + }); + + afterAll(async () => { + await closeDb(); + }); + + test('localhost bootstrap check-in (no Bearer, no CSRF token) is not blocked by CSRF', async () => { + const res = await request(app) + .post('/api/v1/agents') + .send({ siteId: 1, hostname: 'manager.local' }); + + // The site row doesn't exist yet (bootstrap), so the handler skips the + // Agent write and returns the config snapshot. The key assertion is that + // we reached the handler at all — not a 403 from either csrf guard. + expect(res.status).not.toBe(403); + expect(res.status).toBe(200); + expect(res.body.data).toBeDefined(); + }); + + test('non-localhost check-in without credentials is rejected', async () => { + const res = await request(app) + .post('/api/v1/agents') + // A public client IP in X-Forwarded-For marks the request as proxied, + // so the localhost bypass does not apply. + .set('X-Forwarded-For', '203.0.113.7') + .send({ siteId: 1, hostname: 'remote.example.com' }); + + // Either the app-level CSRF guard (403 csrf_invalid) or the route-level + // apiAuth (401 unauthorized) rejects it; the point is it does not succeed. + expect(res.status).not.toBe(200); + expect([401, 403]).toContain(res.status); + }); +}); From fd28a12629b300c8fd19c0e218912b3e204fae84 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 11:01:58 -0400 Subject: [PATCH 09/12] build(manager-control-program): assemble a runnable tree in build/, mount it into the dev compose stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP server runs from /opt/opensource-server/manager-control-program (PYTHONPATH) with only the system python3 — the vendored Python deps must already be present there. In the dev compose stack the repo is mounted read-only into Proxmox and those deps aren't checked in, so nothing populated that path and opensource-mcp.service couldn't start from a fresh checkout. - make build now assembles the ready-to-run tree (app package + vendored wheels for Debian 13 / amd64 / CPython 3.13) into build/ instead of being a no-op. make install just copies build/ into DESTBIN, so the dev bind mount and the packaged install serve byte-for-byte the same layout. clean removes build/. (build/ is already gitignored.) - compose gains an `mcp` service that runs `make -C manager-control-program build` (repo mounted read-write, like the node/client services; the uv image lacks make, so it's apt-installed first). The proxmox service overlays ./manager-control-program/build onto /opt/opensource-server/manager-control-program read-only and depends on `mcp` completing, so the server stands up from scratch. Verified: `make build` then `make install DESTDIR=...` produce the same tree; it imports and runs under CPython 3.13 with only PYTHONPATH set; `make deb` still packages the same contents; and the `mcp` compose service populates build/ from a clean checkout. --- compose.yml | 28 +++++++++++++++ manager-control-program/Makefile | 58 ++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/compose.yml b/compose.yml index f904fb26..e3a3079d 100644 --- a/compose.yml +++ b/compose.yml @@ -77,6 +77,27 @@ services: retries: 60 start_period: 5s + # Assembles the MCP server's runtime tree (the app package plus all vendored + # Python dependencies) into manager-control-program/build via `make build`. + # The opensource-mcp.service inside Proxmox runs `python3 -m + # manager_control_program.server` with PYTHONPATH=/opt/opensource-server/ + # manager-control-program, but the repo is mounted read-only there and the + # vendored deps aren't checked in — so, like the `node`/`client` services, + # this host-side step produces the build output first (repo mounted + # read-write). The proxmox service then overlays build/ onto that runtime + # path via a read-only bind mount (see below). Runs once and exits; the uv + # image lacks make, so install it first. + mcp: + image: astral/uv:0.11.14-trixie-slim + volumes: + - ./:/opt/opensource-server + working_dir: /opt/opensource-server + entrypoint: ["/bin/sh", "-c"] + command: + - | + apt-get update && apt-get install -y --no-install-recommends make + make -C manager-control-program build + # This service will watch the documentation for any changes and rebuild when # needed. Technically it's listening on port 8000, but we only care about the # HTML being placed in ./mie-opensource-landing/site which is shared with the @@ -109,6 +130,11 @@ services: - local:/var/lib/vz - proxmox:/var/lib/pve-cluster - ./:/opt/opensource-server:ro + # The MCP server's runtime tree (app + vendored deps) is built by the + # `mcp` service into manager-control-program/build; overlay it onto the + # path opensource-mcp.service expects (PYTHONPATH), matching where the + # opensource-mcp package installs it in production. + - ./manager-control-program/build:/opt/opensource-server/manager-control-program:ro privileged: true devices: - /dev/loop-control:/dev/loop-control @@ -136,6 +162,8 @@ services: condition: service_completed_successfully node: condition: service_completed_successfully + mcp: + condition: service_completed_successfully client: condition: service_healthy diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile index 89faeb16..5540aee1 100644 --- a/manager-control-program/Makefile +++ b/manager-control-program/Makefile @@ -5,6 +5,13 @@ DESTDIR ?= / DESTBIN := $(DESTDIR)$(PREFIX)/manager-control-program UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system +# `build` assembles the ready-to-run application tree (the app package plus all +# vendored Python dependencies) into BUILDDIR in the source tree; `install` +# then copies that tree verbatim into DESTBIN. So the dev bind mount and the +# packaged install serve byte-for-byte the same layout, runnable with only the +# system python3 (PYTHONPATH pointed at it — no uv, pip, venv, or network). +BUILDDIR := $(CURDIR)/build + INSTALL := install INSTALL_DATA := $(INSTALL) -m 0644 @@ -31,14 +38,14 @@ help: @echo "" @echo "Targets:" @echo " deps install dependencies (uv sync)" - @echo " build nothing to build (kept for the repo-wide target set)" + @echo " build assemble the app + vendored deps tree into build/" @echo " dev run the MCP server locally over stdio (uv)" @echo " test run tests (none; kept for the repo-wide target set)" - @echo " install stage the app with vendored dependencies into DESTDIR" + @echo " install copy the build/ tree into DESTDIR" @echo " deb build the .deb package" @echo " rpm build the .rpm package" @echo " apk build the .apk package" - @echo " clean remove the venv, staging dir and built packages" + @echo " clean remove the venv, build/, staging dir and built packages" @echo " help show this message" @echo "" @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." @@ -46,7 +53,27 @@ help: deps: uv sync --frozen +# Assemble the ready-to-run application tree into build/: the locked +# requirements are exported from uv.lock and installed as prebuilt wheels for +# the production target (Debian 13 / amd64 / CPython 3.13) directly into +# build/, then the app package is copied alongside them. The result runs with +# `python3 -m manager_control_program.server` and PYTHONPATH=build/, needing +# only the system python3. install/package and the dev bind mount all consume +# this same tree. build: + rm -rf $(BUILDDIR) + $(INSTALL) -d $(BUILDDIR) + uv export --frozen --no-dev --no-emit-project -o $(REQUIREMENTS) + uv pip install \ + --python $(PYTHON_VERSION) \ + --python-version $(PYTHON_VERSION) \ + --python-platform $(PYTHON_PLATFORM) \ + --only-binary :all: \ + --target $(BUILDDIR) \ + --requirements $(REQUIREMENTS) + rm -f $(REQUIREMENTS) + cp -a manager_control_program $(BUILDDIR)/ + find $(BUILDDIR) -type d -name __pycache__ -prune -exec rm -rf {} + dev: uv run manager-control-program @@ -55,24 +82,13 @@ dev: # uniform if this component is ever added to the root COMPONENTS list. test: -# Stage the application with vendored dependencies so the target host needs -# only /usr/bin/python3 — no uv, pip, network access, or venv. The locked -# requirements are exported from uv.lock and installed as prebuilt wheels -# directly into the app directory; the systemd unit runs -# `python3 -m manager_control_program.server` with PYTHONPATH pointed there. -install: +# Copy the assembled application tree (build/) into DESTBIN so the dev bind +# mount and the packaged install land the exact same layout, then install the +# systemd unit and its /etc/default config to the system paths. Depends on +# build so `make install` alone produces a complete tree. +install: build $(INSTALL) -d $(DESTBIN) - uv export --frozen --no-dev --no-emit-project -o $(REQUIREMENTS) - uv pip install \ - --python $(PYTHON_VERSION) \ - --python-version $(PYTHON_VERSION) \ - --python-platform $(PYTHON_PLATFORM) \ - --only-binary :all: \ - --target $(DESTBIN) \ - --requirements $(REQUIREMENTS) - rm -f $(REQUIREMENTS) - cp -a manager_control_program $(DESTBIN)/ - find $(DESTBIN) -type d -name __pycache__ -prune -exec rm -rf {} + + cp -a $(BUILDDIR)/. $(DESTBIN)/ $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-mcp.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)/etc/default @@ -93,5 +109,5 @@ apk: $(MAKE) package PACKAGER=apk clean: - rm -rf .venv $(REQUIREMENTS) $(STAGE) + rm -rf .venv $(REQUIREMENTS) $(BUILDDIR) $(STAGE) rm -f *.deb *.rpm *.apk From 9ccaf98843fe2e147045010badca8f1b86ccb2c9 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 11:06:16 -0400 Subject: [PATCH 10/12] build(manager-control-program): serve the build/ tree at the same path in dev and the package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to assembling the runtime tree in build/: make that directory the actual runtime path in every context so it matches the git repo layout (manager-control-program/build). - opensource-mcp.service points WorkingDirectory and PYTHONPATH at /opt/opensource-server/manager-control-program/build. - make install copies build/ into DESTBIN/build (not flattened into DESTBIN), so the package ships the tree at /opt/opensource-server/manager-control-program/build — the same path the read-only repo mount exposes in the dev container. - Drop the proxmox overlay bind mount: the read-only repo mount already exposes manager-control-program/build at that path once the `mcp` service has built it. - Use the non-slim astral/uv image for the `mcp` service, which already ships make, dropping the apt-get step. Verified: the deb installs the tree at .../manager-control-program/build and the unit's PYTHONPATH matches; it imports and runs under CPython 3.13 from build/; the `mcp` compose service builds it from a clean checkout with no apt step. --- compose.yml | 19 ++++++------------- manager-control-program/Makefile | 11 ++++++----- .../contrib/systemd/opensource-mcp.service | 8 ++++---- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/compose.yml b/compose.yml index e3a3079d..c9e25859 100644 --- a/compose.yml +++ b/compose.yml @@ -81,21 +81,19 @@ services: # Python dependencies) into manager-control-program/build via `make build`. # The opensource-mcp.service inside Proxmox runs `python3 -m # manager_control_program.server` with PYTHONPATH=/opt/opensource-server/ - # manager-control-program, but the repo is mounted read-only there and the - # vendored deps aren't checked in — so, like the `node`/`client` services, - # this host-side step produces the build output first (repo mounted - # read-write). The proxmox service then overlays build/ onto that runtime - # path via a read-only bind mount (see below). Runs once and exits; the uv - # image lacks make, so install it first. + # manager-control-program/build, but the repo is mounted read-only there and + # the vendored deps aren't checked in — so, like the `node`/`client` + # services, this host-side step produces the build output first (repo mounted + # read-write); the read-only bind mount inside Proxmox then sees it at the + # same path the package installs to. Runs once and exits. mcp: - image: astral/uv:0.11.14-trixie-slim + image: astral/uv:0.11.14-trixie volumes: - ./:/opt/opensource-server working_dir: /opt/opensource-server entrypoint: ["/bin/sh", "-c"] command: - | - apt-get update && apt-get install -y --no-install-recommends make make -C manager-control-program build # This service will watch the documentation for any changes and rebuild when @@ -130,11 +128,6 @@ services: - local:/var/lib/vz - proxmox:/var/lib/pve-cluster - ./:/opt/opensource-server:ro - # The MCP server's runtime tree (app + vendored deps) is built by the - # `mcp` service into manager-control-program/build; overlay it onto the - # path opensource-mcp.service expects (PYTHONPATH), matching where the - # opensource-mcp package installs it in production. - - ./manager-control-program/build:/opt/opensource-server/manager-control-program:ro privileged: true devices: - /dev/loop-control:/dev/loop-control diff --git a/manager-control-program/Makefile b/manager-control-program/Makefile index 5540aee1..c9c1beaa 100644 --- a/manager-control-program/Makefile +++ b/manager-control-program/Makefile @@ -82,13 +82,14 @@ dev: # uniform if this component is ever added to the root COMPONENTS list. test: -# Copy the assembled application tree (build/) into DESTBIN so the dev bind -# mount and the packaged install land the exact same layout, then install the -# systemd unit and its /etc/default config to the system paths. Depends on +# Copy the assembled application tree (build/) into DESTBIN/build so the dev +# bind mount and the packaged install expose the identical path +# (.../manager-control-program/build, matching the repo layout), then install +# the systemd unit and its /etc/default config to the system paths. Depends on # build so `make install` alone produces a complete tree. install: build - $(INSTALL) -d $(DESTBIN) - cp -a $(BUILDDIR)/. $(DESTBIN)/ + $(INSTALL) -d $(DESTBIN)/build + cp -a $(BUILDDIR)/. $(DESTBIN)/build/ $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-mcp.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)/etc/default diff --git a/manager-control-program/contrib/systemd/opensource-mcp.service b/manager-control-program/contrib/systemd/opensource-mcp.service index 5de0c220..67270489 100644 --- a/manager-control-program/contrib/systemd/opensource-mcp.service +++ b/manager-control-program/contrib/systemd/opensource-mcp.service @@ -8,10 +8,10 @@ Wants=container-creator.service [Service] Type=simple User=root -WorkingDirectory=/opt/opensource-server/manager-control-program -# Dependencies are vendored into the app directory at package build time; -# no uv, pip, or venv exists on the host. -Environment=PYTHONPATH=/opt/opensource-server/manager-control-program +WorkingDirectory=/opt/opensource-server/manager-control-program/build +# The app package and all vendored dependencies are assembled into build/ +# (by `make build`); no uv, pip, or venv exists on the host. +Environment=PYTHONPATH=/opt/opensource-server/manager-control-program/build # Runtime configuration (API_BASE_URL, SERVER_TRANSPORT/HOST/PORT, LOG_LEVEL) # lives in the packaged config file below — edit it and restart to apply. EnvironmentFile=/etc/default/opensource-mcp From 960e7ac3480c93727ae1cec8965acc67a50a2d0e Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 11:23:37 -0400 Subject: [PATCH 11/12] feat(agent): add leveled logging, surfacing nginx config-test output on failure The agent only logged a handful of one-line messages via console.*, and crucially discarded the output of `nginx -t` (run with stdio 'pipe'), so when nginx rejected a rendered config the journal showed "config test failed" with no indication of *why*. - New agent/src/log.ts: a minimal leveled logger (error/warn/info/debug) gated by LOG_LEVEL (default info; unknown values warn and fall back). error/warn go to stderr, info/debug to stdout, each line level-tagged so `journalctl -u opensource-agent` output is greppable. commandOutput() extracts the captured stdout/stderr from a failed execFileSync error. - apply.ts: run() now captures command output; on a failed config test (nginx -t / dnsmasq --test) the actual validator output is logged at error level. Added render/diff/write/test/reload trace logging (debug) plus info-level "applying"/"test passed"/"reloaded" milestones, and the reload-after-rollback failure is now logged instead of swallowed. - index.ts: log startup, per-pass check-in outcome (304 vs new config), and MAX_PASSES exhaustion; the top-level handler now logs the full stack, not just err.message. - api.ts: log the check-in request and distinguish a manager timeout from other network errors and from non-2xx responses. - state.ts: route the corrupt-state warning through the logger. - opensource-agent.service: document the optional LOG_LEVEL env var. Verified: tsc builds clean; default level hides debug and shows info/warn/error; a simulated nginx -t failure surfaces the emerg line; LOG_LEVEL=debug enables debug; unknown levels fall back to info. --- .../contrib/systemd/opensource-agent.service | 1 + agent/src/api.ts | 34 +++++++-- agent/src/apply.ts | 48 +++++++++--- agent/src/config.ts | 6 ++ agent/src/index.ts | 15 +++- agent/src/log.ts | 74 +++++++++++++++++++ agent/src/state.ts | 3 +- 7 files changed, 162 insertions(+), 19 deletions(-) create mode 100644 agent/src/log.ts diff --git a/agent/contrib/systemd/opensource-agent.service b/agent/contrib/systemd/opensource-agent.service index ff2d2814..a7e14eff 100644 --- a/agent/contrib/systemd/opensource-agent.service +++ b/agent/contrib/systemd/opensource-agent.service @@ -8,6 +8,7 @@ After=network-online.target environment.service [Service] Type=oneshot # Runtime config (SITE_ID, MANAGER_URL, API_KEY) lives in /etc/environment. +# Optional LOG_LEVEL (error|warn|info|debug; default info) also read from there. EnvironmentFile=-/etc/environment # systemd creates /var/lib/opensource-agent and passes it as $STATE_DIRECTORY. StateDirectory=opensource-agent diff --git a/agent/src/api.ts b/agent/src/api.ts index ef1b6b0f..1aa0438a 100644 --- a/agent/src/api.ts +++ b/agent/src/api.ts @@ -2,6 +2,7 @@ import type { AgentConfig } from './config'; import type { CheckinRequest, SiteConfig } from './types'; +import { log } from './log'; export type CheckinResult = | { notModified: true } @@ -20,15 +21,34 @@ export async function checkin( if (etag) headers['If-None-Match'] = etag; if (cfg.apiKey) headers['Authorization'] = `Bearer ${cfg.apiKey}`; - const res = await fetch(`${cfg.managerUrl}/api/v1/agents`, { - method: 'POST', - headers, - body: JSON.stringify(body), - signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), - }); + const url = `${cfg.managerUrl}/api/v1/agents`; + log.debug(`check-in POST ${url}${etag ? ` (If-None-Match: ${etag})` : ''}${cfg.apiKey ? ' with API key' : ''}`); + let res: Response; + try { + res = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(CHECKIN_TIMEOUT_MS), + }); + } catch (err) { + // fetch rejects on network errors and on the abort timeout; distinguish + // the timeout so a stalled manager is obvious in the logs. + const reason = + (err as Error)?.name === 'TimeoutError' + ? `no response within ${CHECKIN_TIMEOUT_MS} ms` + : (err as Error)?.message ?? String(err); + log.error(`check-in request to ${url} failed: ${reason}`); + throw err; + } + + log.debug(`check-in response: HTTP ${res.status}`); if (res.status === 304) return { notModified: true }; - if (!res.ok) throw new Error(`Check-in failed: HTTP ${res.status}`); + if (!res.ok) { + log.error(`check-in rejected by manager: HTTP ${res.status}`); + throw new Error(`Check-in failed: HTTP ${res.status}`); + } const { data } = (await res.json()) as { data: SiteConfig }; return { diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 4a518d53..a83c67ef 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -11,6 +11,7 @@ import path from 'path'; import { execFileSync } from 'child_process'; import ejs from 'ejs'; import { reloadOrRestartService, restartService, sighupService } from './system'; +import { log, commandOutput } from './log'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); @@ -36,8 +37,12 @@ function renderTemplate(template: string, data: object): Promise { return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); } -function run(cmd: string[]): void { - execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe' }); +// Run a command, capturing its output. execFileSync with stdio 'pipe' attaches +// the captured stdout/stderr to the thrown error on failure (see commandOutput +// in log.ts), so callers can surface e.g. exactly why `nginx -t` rejected a +// config. Returns the combined stdout for successful runs. +function run(cmd: string[]): string { + return execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe', encoding: 'utf8' }); } export const services: ManagedService[] = [ @@ -102,17 +107,27 @@ function writeFileAtomic(dest: string, content: string): void { } export async function applyService(svc: ManagedService, config: SiteConfig): Promise { + log.debug(`${svc.unit}: rendering config`); const files = await svc.render(config); - if (!files) return 'success'; + if (!files) { + log.debug(`${svc.unit}: nothing to manage yet, skipping`); + return 'success'; + } const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); - if (changed.length === 0) return 'success'; + if (changed.length === 0) { + log.debug(`${svc.unit}: config unchanged, nothing to do`); + return 'success'; + } + + log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`); // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); writeFileAtomic(f.dest, f.content); + log.debug(`${svc.unit}: wrote ${f.dest}`); } const rollback = () => { @@ -120,30 +135,45 @@ export async function applyService(svc: ManagedService, config: SiteConfig): Pro if (prev === null) fs.rmSync(dest, { force: true }); else writeFileAtomic(dest, prev); } + log.debug(`${svc.unit}: rolled back to previous config`); }; if (svc.test) { + const testCmd = svc.test.join(' '); + log.debug(`${svc.unit}: testing config with \`${testCmd}\``); try { - run(svc.test); + const out = run(svc.test); + const trimmed = out.trim(); + if (trimmed) log.debug(`${svc.unit}: config test output:\n${trimmed}`); + log.info(`${svc.unit}: config test passed`); } catch (err) { rollback(); - console.error(`${svc.unit}: config test failed, rolled back: ${(err as Error).message}`); + // Surface the actual validator output (e.g. the nginx line/column that + // was rejected) — that's the whole point of running the test, and it's + // otherwise lost since the command's stdio is piped. + const output = commandOutput(err); + log.error(`${svc.unit}: config test failed (\`${testCmd}\`), rolled back: ${(err as Error).message}`); + if (output) log.error(`${svc.unit}: config test output:\n${output}`); return 'failure'; } } try { + log.debug(`${svc.unit}: reloading service`); await svc.reload(changed); + log.info(`${svc.unit}: reloaded`); } catch (err) { // The new config passed its test but the service couldn't pick it up: // restore the previous files and reload again (best effort) so the // service keeps running the last known-good config. rollback(); - await svc.reload(changed).catch(() => { /* reported via service state at next check-in */ }); - console.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); + await svc.reload(changed).catch((reErr) => { + log.error(`${svc.unit}: reload after rollback also failed: ${(reErr as Error).message}`); + }); + log.error(`${svc.unit}: reload failed, rolled back: ${(err as Error).message}`); return 'failure'; } - console.log(`${svc.unit}: applied ${changed.length} file(s)`); + log.info(`${svc.unit}: applied ${changed.length} file(s)`); return 'success'; } diff --git a/agent/src/config.ts b/agent/src/config.ts index d01c91f3..27b93695 100644 --- a/agent/src/config.ts +++ b/agent/src/config.ts @@ -1,14 +1,19 @@ /** Agent configuration, read from the environment (systemd passes * /etc/environment through EnvironmentFile=). */ +import { setLogLevel, type LogLevel } from './log'; + export interface AgentConfig { siteId: number; managerUrl: string; apiKey?: string; stateDir: string; + logLevel: LogLevel; } export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { + // Set the log level first so any warnings below (and from callers) honor it. + const logLevel = setLogLevel(env.LOG_LEVEL); const siteId = parseInt(env.SITE_ID ?? '', 10); const managerUrl = env.MANAGER_URL; if (!Number.isInteger(siteId) || !managerUrl) { @@ -20,5 +25,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { apiKey: env.API_KEY || undefined, // Set by systemd from StateDirectory=; fallback for manual runs. stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', + logLevel, }; } diff --git a/agent/src/index.ts b/agent/src/index.ts index 5d2a2d9b..071e6faa 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -16,6 +16,7 @@ import { State } from './state'; import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; import { checkin } from './api'; import { services, applyService } from './apply'; +import { log } from './log'; import type { CheckinRequest, ServiceStatus } from './types'; // Safety cap: a flapping server-side config can't keep a single run alive @@ -42,11 +43,18 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const cfg = loadConfig(); const state = State.load(cfg.stateDir); + log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}`); + log.debug(`state dir=${cfg.stateDir}, saved etag=${state.etag ?? '(none)'}`); for (let pass = 0; pass < MAX_PASSES; pass++) { + log.debug(`check-in pass ${pass + 1}/${MAX_PASSES}`); const result = await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); - if (result.notModified) return; + if (result.notModified) { + log.info('check-in: config unchanged (304), nothing to apply'); + return; + } + log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); for (const svc of services) { state.lastApply[svc.unit] = await applyService(svc, result.config); } @@ -61,12 +69,15 @@ async function main(): Promise { // MAX_PASSES exhausted (flapping server-side config): check in once more so // the final pass' apply results reach the manager instead of going stale // until the next timer run. + log.warn(`reached MAX_PASSES (${MAX_PASSES}) without a stable config; reporting final results`); await checkin(cfg, await buildCheckinBody(cfg, state), state.etag); } main() .then(() => disconnectSystemBus()) .catch((err) => { - console.error(err instanceof Error ? err.message : err); + // Log the full error (stack included when present) so a failed run is + // diagnosable from the journal, not just a one-line message. + log.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); process.exit(1); }); diff --git a/agent/src/log.ts b/agent/src/log.ts new file mode 100644 index 00000000..c902e2f7 --- /dev/null +++ b/agent/src/log.ts @@ -0,0 +1,74 @@ +/** + * Minimal leveled logger for the agent. + * + * The agent runs as a systemd oneshot, so everything written to stdout/stderr + * lands in the journal (view with `journalctl -u opensource-agent`). We keep a + * tiny hand-rolled logger rather than pull in a dependency: levels gate what + * gets emitted, error/warn go to stderr and info/debug to stdout, and each + * line is prefixed with its level so journal output is greppable. + * + * The active level comes from LOG_LEVEL (case-insensitive; default "info"). + * An unknown value falls back to "info" with a warning so a typo can't + * silently mute the agent. + */ + +export type LogLevel = 'error' | 'warn' | 'info' | 'debug'; + +const LEVELS: Record = { + error: 0, + warn: 1, + info: 2, + debug: 3, +}; + +let activeLevel: LogLevel = 'info'; + +/** + * Set the active level from a raw env string. Returns the level actually + * applied. Call once at startup from loadConfig(); defaults to "info". + */ +export function setLogLevel(raw: string | undefined): LogLevel { + if (!raw) { + activeLevel = 'info'; + return activeLevel; + } + const normalized = raw.trim().toLowerCase(); + if (normalized in LEVELS) { + activeLevel = normalized as LogLevel; + } else { + activeLevel = 'info'; + log.warn(`Unknown LOG_LEVEL "${raw}", defaulting to "info"`); + } + return activeLevel; +} + +function emit(level: LogLevel, message: string): void { + if (LEVELS[level] > LEVELS[activeLevel]) return; + const line = `[${level}] ${message}`; + if (level === 'error' || level === 'warn') { + console.error(line); + } else { + console.log(line); + } +} + +export const log = { + error: (message: string) => emit('error', message), + warn: (message: string) => emit('warn', message), + info: (message: string) => emit('info', message), + debug: (message: string) => emit('debug', message), +}; + +/** + * Format an error's command output (execFileSync attaches captured stdout / + * stderr to the thrown error when stdio is 'pipe'). Returns a trimmed, + * human-readable blob or null when there is nothing captured — used to surface + * the actual reason a command like `nginx -t` rejected a config. + */ +export function commandOutput(err: unknown): string | null { + const e = err as { stdout?: Buffer | string; stderr?: Buffer | string }; + const parts = [e?.stdout, e?.stderr] + .map((part) => (part == null ? '' : part.toString()).trim()) + .filter((part) => part.length > 0); + return parts.length > 0 ? parts.join('\n') : null; +} diff --git a/agent/src/state.ts b/agent/src/state.ts index 3e3cdbf4..93ae019e 100644 --- a/agent/src/state.ts +++ b/agent/src/state.ts @@ -3,6 +3,7 @@ import fs from 'fs'; import path from 'path'; +import { log } from './log'; import type { ApplyResult } from './types'; export class State { @@ -27,7 +28,7 @@ export class State { } catch (err) { if (!(err instanceof SyntaxError)) throw err; // A corrupt state file just means a full re-apply on this run. - console.error(`Ignoring unparsable state file ${state.file}: ${err.message}`); + log.warn(`Ignoring unparsable state file ${state.file}: ${err.message}`); } return state; } From e229de7b2f8ef4f6722a7d2cc77d74a26c70f2b4 Mon Sep 17 00:00:00 2001 From: Robert Gingras Date: Fri, 24 Jul 2026 11:41:36 -0400 Subject: [PATCH 12/12] fix(manager-control-program): prefix API requests with the spec's server path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MCP tool calls were returning the Manager's SPA HTML instead of JSON. fastmcp builds each generated tool's request URL by joining the raw OpenAPI path (e.g. "/sites") onto the httpx client's base_url and ignores the spec's `servers` entry. Our spec declares `servers: [{url: /api/v1}]`, so with base_url=API_BASE_URL the tools requested http://host:3000/sites — which the Manager serves as the single-page app's HTML catch-all — instead of /api/v1/sites. Read servers[0].url from the fetched spec and append its path to the client's base_url after create_mcp_server, so requests resolve to /api/v1/. A relative server url contributes its path directly; an absolute one contributes only its path so API_BASE_URL's (loopback) host is preserved; an empty/root server leaves base_url unchanged. Verified against a stub API whose spec declares servers:/api/v1: the client base_url becomes http://host/api/v1/ and a tool for "/sites" resolves to /api/v1/sites. --- .../manager_control_program/server.py | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/manager-control-program/manager_control_program/server.py b/manager-control-program/manager_control_program/server.py index 8f72fbf0..34ddca00 100644 --- a/manager-control-program/manager_control_program/server.py +++ b/manager-control-program/manager_control_program/server.py @@ -1,4 +1,5 @@ import os +from urllib.parse import urljoin, urlsplit import httpx from awslabs.openapi_mcp_server.server import load_config, create_mcp_server, setup_signal_handlers @@ -10,6 +11,32 @@ HTTP_TRANSPORTS = ("http", "streamable-http", "sse") +def _spec_server_path(spec_url: str) -> str: + """Return the path prefix declared by the OpenAPI spec's first server. + + fastmcp builds each tool's request URL from the raw path in the spec + (e.g. "/sites") joined onto the httpx client's base_url, ignoring the + spec's `servers` entry entirely. Our API declares `servers: [{url: + /api/v1}]`, so without this prefix the tools would hit "/sites" instead + of "/api/v1/sites" — which the Manager serves as the SPA's HTML, not JSON. + + We fetch the spec ourselves (the same document awslabs loads) and return + servers[0].url's path. A relative server url ("/api/v1") contributes its + path directly; an absolute one contributes only its path component so the + host stays whatever API_BASE_URL points at. Returns "" when the spec omits + servers or declares the root, leaving base_url unchanged. + """ + spec = httpx.get(spec_url, timeout=30.0).raise_for_status().json() + servers = spec.get("servers") or [] + if not servers: + return "" + url = (servers[0] or {}).get("url", "") or "" + # Keep only the path; a spec-declared host would otherwise override + # API_BASE_URL (and its loopback target). + path = urlsplit(url).path if "://" in url else url + return path.strip("/") + + class ForwardAuthorizationHeader(httpx.Auth): """Per-request auth for HTTP mode: forward the MCP caller's credentials. @@ -77,12 +104,23 @@ def main(): os.environ["AUTH_TYPE"] = "none" # The rest of this is more-or-less copied from the official - # awslabs.openapi_mpc_server.server:main function with the small exception - # of setting the Accept header to application/json. The official defaults to - # */* which makes our API return HTML instead of the JSON response, breaking - # the API spec. + # awslabs.openapi_mpc_server.server:main function, with two adjustments: + # + # - base_url gets the spec's server path prefix appended (see + # _spec_server_path): fastmcp ignores the spec's `servers` entry, so + # without this the generated tools would request "/sites" rather than + # "/api/v1/sites" and hit the SPA's HTML catch-all instead of the JSON + # API. + # - the Accept header is pinned to application/json; awslabs defaults to + # */*, which lets the API content-negotiate to HTML for some routes. config = load_config() mcp_server = create_mcp_server(config) + + server_path = _spec_server_path(os.environ["API_SPEC_URL"]) + if server_path: + base = str(mcp_server._client.base_url).rstrip("/") + "/" + mcp_server._client.base_url = urljoin(base, server_path + "/") + mcp_server._client.headers['accept'] = 'application/json' setup_signal_handlers()