diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 887e19e5..942bbde1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -182,6 +182,25 @@ jobs: - name: Test distribution, installer and console packaging run: make check-distribution + compose: + needs: plan + if: needs.plan.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs || '[]'), 'compose') + runs-on: ${{ vars.OAC_USE_GITHUB_RUNNERS == 'true' && 'ubuntu-22.04' || 'blacksmith-2vcpu-ubuntu-2204' }} + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ inputs.ref || github.sha }} + - name: Allocate an isolated Compose project + run: python3 -c 'import uuid; print("COMPOSE_SMOKE_PROJECT=oac-smoke-" + uuid.uuid4().hex)' >> "$GITHUB_ENV" + - name: Start published images and verify the installation + timeout-minutes: 17 + run: python3 scripts/compose-smoke.py + - name: Remove test containers and volumes + if: always() && env.COMPOSE_SMOKE_PROJECT != '' + timeout-minutes: 2 + run: docker compose --env-file /dev/null -p "$COMPOSE_SMOKE_PROJECT" -f deploy/compose/compose.yaml down --volumes --remove-orphans + harness: needs: plan if: needs.plan.result == 'success' && contains(fromJSON(needs.plan.outputs.jobs || '[]'), 'harness') @@ -307,7 +326,7 @@ jobs: # Always report the required check, even when planning or a dependency fails. check: if: always() - needs: [plan, hygiene, distribution, backend, harness, example, web, web-acceptance, website, api, native, lint] + needs: [plan, hygiene, distribution, compose, backend, harness, example, web, web-acceptance, website, api, native, lint] runs-on: ${{ vars.OAC_USE_GITHUB_RUNNERS == 'true' && 'ubuntu-22.04' || 'blacksmith-2vcpu-ubuntu-2204' }} timeout-minutes: 5 steps: diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml new file mode 100644 index 00000000..9f5b92a3 --- /dev/null +++ b/deploy/compose/compose.yaml @@ -0,0 +1,276 @@ +# OpenAgentCore v0.0.3. Images and installer payload are one matched release. +# Set OAC_PUBLIC_URL to your platform's HTTPS origin when ready; startup defaults to localhost. +x-ingress-image: &ingress-image ghcr.io/minimax-ai/openagentcore/ingress:v0.0.3@sha256:e485a8cae0b904389501f00bcee6a2c5b35be83412a9ff81a740f93b8cdbf326 +x-core: &core + image: ghcr.io/minimax-ai/openagentcore/core:v0.0.3@sha256:6934a26d5cb7c878128ea8187ced446336345de7750c415be510740806663575 + platform: linux/amd64 + user: "65532:65532" + read_only: true + init: true + tmpfs: [/tmp:mode=1777] + security_opt: [no-new-privileges:true] + environment: + OAC_PUBLIC_URL: &public-url ${OAC_PUBLIC_URL:-http://localhost:8080} + OAC_DATABASE_URL: postgres://agents_api@database:5432/agents_api?sslmode=disable + OAC_DATABASE_PASSWORD_FILE: /run/database/password + OAC_CREDENTIAL_KEY_FILE: /run/oac/credential.key + OAC_CORE_KEY_DIGESTS_FILE: /run/oac/core-key-digests.json + OAC_PROVIDER_ROOT: /opt/oac + OAC_PROVIDER_STATE_ROOT: /state + OAC_NATIVE_INSTALLER_DIR: /opt/oac/native-installers + OAC_HARNESSES: claude_sdk,codex,mcode + volumes: + - core-config:/run/oac:ro + - database-secret:/run/database:ro + - core-state:/state + +services: + init: + image: *ingress-image + platform: linux/amd64 + restart: "no" + security_opt: [no-new-privileges:true] + command: [python3, /init.py] + configs: + - source: init-script + target: /init.py + volumes: + - core-config:/data/core + - web-secret:/data/web + - database-secret:/data/database + - core-state:/data/state + - node-payload:/data/payload + - database:/data/database-data:ro + + database: + image: postgres:16-alpine@sha256:1a66d744c1b459e13b05a8fca341da84cb63383e99ce262210efee5a319d4551 + platform: linux/amd64 + restart: unless-stopped + depends_on: + init: {condition: service_completed_successfully} + environment: + POSTGRES_USER: agents_api + POSTGRES_DB: agents_api + POSTGRES_PASSWORD_FILE: /run/database/password + volumes: + - database:/var/lib/postgresql/data + - database-secret:/run/database:ro + healthcheck: + test: [CMD-SHELL, "pg_isready -h 127.0.0.1 -U agents_api -d agents_api"] + interval: 2s + timeout: 5s + retries: 30 + + migrate: + <<: *core + restart: "no" + command: [/usr/local/bin/oac-core-migrate] + depends_on: + database: {condition: service_healthy} + + core: + <<: *core + restart: unless-stopped + command: + - /bin/sh + - -ec + - export OAC_INSTALLATION_ID="$$(cat /run/oac/installation.id)"; exec /usr/local/bin/oac-core + depends_on: + migrate: {condition: service_completed_successfully} + + web: + image: ghcr.io/minimax-ai/openagentcore/web:v0.0.3@sha256:d1eb4cc8870aebd080773fd16db92a5a1db205e0aab10066d4db16f1284a88f5 + platform: linux/amd64 + user: "65532:65532" + restart: unless-stopped + read_only: true + security_opt: [no-new-privileges:true] + depends_on: + init: {condition: service_completed_successfully} + environment: + OAC_WEB_ORIGIN: *public-url + OAC_WEB_UPSTREAM: http://core:8091 + OAC_WEB_CORE_KEY_FILE: /run/oac/core.key + OAC_WEB_NODE_PAYLOAD_DIR: /node-payload + volumes: + - web-secret:/run/oac:ro + - node-payload:/node-payload:ro + + gateway: + image: *ingress-image + platform: linux/amd64 + user: "65532:65532" + restart: unless-stopped + security_opt: [no-new-privileges:true] + tmpfs: [/tmp:mode=1777] + environment: + XDG_DATA_HOME: /tmp/data + XDG_CONFIG_HOME: /tmp/config + command: [caddy, run, --config, /etc/caddy/Caddyfile, --adapter, caddyfile] + depends_on: [core, web] + expose: ["8080"] + configs: + - source: gateway-config + target: /etc/caddy/Caddyfile + healthcheck: + test: + - CMD + - python3 + - -c + - | + import urllib.request + client = urllib.request.build_opener(urllib.request.ProxyHandler({})) + for host in ('core:8091', 'web:8080', '127.0.0.1:8080'): + with client.open('http://' + host + '/healthz', timeout=3) as response: + assert response.status == 200 + interval: 5s + timeout: 10s + retries: 30 + + # Explicit operator action; the key is printed only to the attached terminal. + credentials: + image: *ingress-image + platform: linux/amd64 + user: "65532:65532" + profiles: [tools] + read_only: true + logging: {driver: none} + command: [cat, /run/oac/core.key] + volumes: + - web-secret:/run/oac:ro + +volumes: + database: + database-secret: + core-config: + web-secret: + core-state: + node-payload: + +configs: + gateway-config: + content: | + { + admin off + auto_https off + persist_config off + } + http://:8080 { + @core path /v1 /v1/* /api/v1/* + handle @core { + reverse_proxy core:8091 { + flush_interval -1 + } + } + handle { + reverse_proxy web:8080 { + flush_interval -1 + } + } + } + + init-script: + content: | + import base64 + import fcntl + import hashlib + import json + import os + from pathlib import Path + import secrets + import tarfile + import urllib.request + import uuid + + REVISION = 'cc7e1aad3bde47598d161d6372e2e211bb61d9cd' + BASE = 'https://github.com/MiniMax-AI/OpenAgentCore/releases/download/v0.0.3/' + ARCHIVE = 'oac-' + REVISION + '-linux-amd64' + CHECKSUM = '579d2d43accfc45a0a8567db5bc33b407c48b05b0d731bea3fed73e6ade558ae' + MEMBERS = ('manifest.json', 'SHA256SUMS', 'node-install.pyz', 'runtime/seccomp.json') + + def digest(data): + return hashlib.sha256(data).hexdigest() + + def download(): + print('Downloading and verifying the matched node installation metadata', flush=True) + checksum = hashlib.sha256() + with urllib.request.urlopen(BASE + ARCHIVE + '.tar.gz', timeout=60) as response: + class Reader: + def read(self, count=-1): + data = response.read(count) + checksum.update(data) + return data + reader, files = Reader(), {} + with tarfile.open(fileobj=reader, mode='r|gz') as archive: + for member in archive: + name = member.name.removeprefix(ARCHIVE + '/') + if member.name == ARCHIVE + '/' + name and name in MEMBERS: + if name in files or not member.isfile() or member.size > 1024 * 1024: + raise RuntimeError('Invalid release metadata member') + files[name] = archive.extractfile(member).read() + while reader.read(1024 * 1024): + pass + if checksum.hexdigest() != CHECKSUM or set(files) != set(MEMBERS): + raise RuntimeError('Release metadata checksum mismatch') + manifest = json.loads(files['manifest.json']) + if manifest['source_commit'] != REVISION or manifest['platform'] != 'linux/amd64': + raise RuntimeError('Release identity mismatch') + return files + + def write(path, data): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(path.name + '.tmp') + temporary.write_bytes(data) + temporary.chmod(0o600) + os.chown(temporary, 65532, 65532) + os.replace(temporary, path) + + def initialize(root, fetch=download): + for name in ('core', 'web', 'database', 'state', 'payload'): + directory = root / name + directory.mkdir(exist_ok=True) + directory.chmod(0o700) + os.chown(directory, 65532, 65532) + with (root / 'core/.init.lock').open('w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + marker = root / 'core/installation.json' + if marker.exists(): + receipt = json.loads(marker.read_text()) + if receipt['source_commit'] != REVISION: + raise RuntimeError('This volume belongs to another release; create a new installation') + for name, checksum in receipt['files'].items(): + if digest((root / name).read_bytes()) != checksum: + raise RuntimeError('Installation files changed; restore the matching volumes') + print('Existing installation verified', flush=True) + return + if any((root / 'database-data').iterdir()) or any((root / 'state').iterdir()): + raise RuntimeError('Existing data requires its original installation volumes') + files = fetch() + prefix = 'payload/releases/' + REVISION + '/' + for name, data in files.items(): + write(root / (prefix + name), data) + write(root / 'payload/active.json', json.dumps({'source_commit': REVISION}).encode()) + # Parent directories of the public payload must be traversable by Web. + for path in (root / 'payload').rglob('*'): + if path.is_dir(): + path.chmod(0o755) + generators = { + 'web/core.key': lambda: secrets.token_hex(32), + 'database/password': lambda: secrets.token_hex(32), + 'core/credential.key': lambda: base64.b64encode(secrets.token_bytes(32)).decode(), + 'core/installation.id': lambda: str(uuid.uuid4()), + } + for name, generate in generators.items(): + if not (root / name).exists(): + write(root / name, (generate() + '\n').encode()) + key = (root / 'web/core.key').read_text().strip() + write(root / 'core/core-key-digests.json', json.dumps([digest(key.encode())]).encode()) + names = [*generators, 'core/core-key-digests.json', 'payload/active.json', + *(prefix + name for name in MEMBERS)] + receipt = {'source_commit': REVISION, 'files': {name: digest((root / name).read_bytes()) for name in names}} + write(marker, json.dumps(receipt).encode()) + print('Installation initialized; use the credentials service to retrieve the sign-in key', flush=True) + + if __name__ == '__main__': + os.umask(0o077) + initialize(Path('/data')) diff --git a/deploy/compose/dokploy.toml b/deploy/compose/dokploy.toml new file mode 100644 index 00000000..4ba5f428 --- /dev/null +++ b/deploy/compose/dokploy.toml @@ -0,0 +1,11 @@ +[variables] +main_domain = "${domain}" + +# Enable HTTPS and select a certificate provider for this domain in Dokploy before deployment. +[config] +env = ["OAC_PUBLIC_URL=https://${main_domain}"] + +[[config.domains]] +serviceName = "gateway" +port = 8080 +host = "${main_domain}" diff --git a/deploy/compose/local.yaml b/deploy/compose/local.yaml new file mode 100644 index 00000000..339b92b7 --- /dev/null +++ b/deploy/compose/local.yaml @@ -0,0 +1,4 @@ +# Local access; platforms route directly to gateway:8080 without this override. +services: + gateway: + ports: ["127.0.0.1:8080:8080"] diff --git a/deploy/install/test_compose.py b/deploy/install/test_compose.py new file mode 100644 index 00000000..5fded431 --- /dev/null +++ b/deploy/install/test_compose.py @@ -0,0 +1,166 @@ +"""Qualify the self-contained Compose initializer without building images.""" + +import base64 +import copy +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch +import uuid + + +ROOT = Path(__file__).resolve().parents[2] +COMPOSE = ROOT / 'deploy/compose/compose.yaml' + + +class ComposeTests(unittest.TestCase): + @classmethod + def render(cls, public_url=None): + env = dict(os.environ) + env.pop('OAC_PUBLIC_URL', None) + if public_url is not None: + env['OAC_PUBLIC_URL'] = public_url + return json.loads(subprocess.check_output( + ['docker', 'compose', '--env-file', os.devnull, '-f', str(COMPOSE), + '--profile', 'tools', 'config', '--format', 'json'], env=env)) + + @classmethod + def setUpClass(cls): + cls.compose = cls.render() + + def setUp(self): + base = Path.home() / '.oac/tests/compose' + base.mkdir(parents=True, exist_ok=True) + self.temporary = tempfile.TemporaryDirectory(dir=base) + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + (self.root / 'database-data').mkdir() + self.code = {'__name__': 'compose_initializer'} + exec(self.compose['configs']['init-script']['content'], self.code) + self.files = {name: b'fixture' for name in self.code['MEMBERS']} + self.files['manifest.json'] = json.dumps({'source_commit': self.code['REVISION'], 'platform': 'linux/amd64'}).encode() + chown = patch('os.chown') + chown.start() + self.addCleanup(chown.stop) + + def initialize(self, fetch=None): + self.code['initialize'](self.root, fetch or (lambda: self.files)) + + def test_fresh_installation_and_restart_keep_identity_and_keys(self): + self.initialize() + saved = {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob('*') if p.is_file()} + key = saved['web/core.key'].strip() + self.assertEqual(len(key), 64) + self.assertEqual(json.loads(saved['core/core-key-digests.json']), [hashlib.sha256(key).hexdigest()]) + self.assertEqual(len(base64.b64decode(saved['core/credential.key'])), 32) + uuid.UUID(saved['core/installation.id'].decode().strip()) + for name in ('web/core.key', 'database/password', 'core/credential.key'): + self.assertEqual((self.root / name).stat().st_mode & 0o777, 0o600) + self.initialize(lambda: self.fail('A completed installation must not download again')) + self.assertEqual(saved, {str(p.relative_to(self.root)): p.read_bytes() for p in self.root.rglob('*') if p.is_file()}) + + def test_existing_data_without_installation_secrets_is_refused(self): + (self.root / 'database-data/PG_VERSION').write_text('16') + with self.assertRaisesRegex(RuntimeError, 'original installation'): + self.initialize(lambda: self.fail('Must refuse before downloading')) + self.assertFalse((self.root / 'web/core.key').exists()) + + def test_changed_or_missing_keys_and_another_release_are_refused(self): + self.initialize() + path = self.root / 'database/password' + original = path.read_bytes() + path.write_bytes(b'different') + with self.assertRaisesRegex(RuntimeError, 'files changed'): + self.initialize() + path.unlink() + with self.assertRaises(FileNotFoundError): + self.initialize() + path.write_bytes(original) + self.code['REVISION'] = 'f' * 40 + with self.assertRaisesRegex(RuntimeError, 'another release'): + self.initialize() + + def test_interrupted_fresh_initialization_retains_generated_keys(self): + self.initialize() + key = (self.root / 'web/core.key').read_bytes() + (self.root / 'core/installation.json').unlink() + self.initialize() + self.assertEqual(key, (self.root / 'web/core.key').read_bytes()) + + def archive(self): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w:gz') as archive: + for name, data in self.files.items(): + info = tarfile.TarInfo(self.code['ARCHIVE'] + '/' + name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + data = b'ignored image data' * 1000 + info = tarfile.TarInfo(self.code['ARCHIVE'] + '/images/core.tar') + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + return output.getvalue() + + def test_streamed_release_is_fully_verified_and_only_metadata_is_retained(self): + data = self.archive() + self.code['CHECKSUM'] = hashlib.sha256(data).hexdigest() + with patch('urllib.request.urlopen', return_value=io.BytesIO(data)): + self.assertEqual(self.code['download'](), self.files) + self.code['CHECKSUM'] = '0' * 64 + with patch('urllib.request.urlopen', return_value=io.BytesIO(data)): + with self.assertRaisesRegex(RuntimeError, 'checksum mismatch'): + self.code['download']() + del self.files['node-install.pyz'] + data = self.archive() + self.code['CHECKSUM'] = hashlib.sha256(data).hexdigest() + with patch('urllib.request.urlopen', return_value=io.BytesIO(data)): + with self.assertRaisesRegex(RuntimeError, 'checksum mismatch'): + self.code['download']() + + def test_compose_uses_private_services_and_ordered_initialization(self): + services = self.compose['services'] + self.assertEqual(services['database']['depends_on']['init']['condition'], 'service_completed_successfully') + self.assertEqual(services['migrate']['depends_on']['database']['condition'], 'service_healthy') + self.assertIn('pg_isready -h 127.0.0.1', services['database']['healthcheck']['test'][1]) + self.assertEqual(services['core']['depends_on']['migrate']['condition'], 'service_completed_successfully') + for service in services.values(): + self.assertNotIn('build', service) + self.assertNotIn('ports', service) + self.assertIn('@sha256:', service['image']) + for volume in service.get('volumes', []): + self.assertNotIn('docker.sock', volume['source']) + self.assertEqual(volume['type'], 'volume') + self.assertEqual({v['source'] for v in services['web']['volumes']}, {'web-secret', 'node-payload'}) + self.assertEqual(services['credentials']['logging']['driver'], 'none') + schema = json.loads((ROOT / 'deploy/install/config.schema.json').read_text()) + harnesses = schema['properties']['core']['properties']['harnesses']['default'] + self.assertEqual(services['core']['environment']['OAC_HARNESSES'].split(','), harnesses) + + def test_public_url_can_be_configured_after_initial_startup(self): + for value in (None, '', 'https://oac.example.test', 'http://localhost:9080'): + with self.subTest(public_url=value): + configured = self.render(value) + expected = value or 'http://localhost:8080' + for name, setting in (('core', 'OAC_PUBLIC_URL'), ('migrate', 'OAC_PUBLIC_URL'), + ('web', 'OAC_WEB_ORIGIN')): + self.assertEqual(configured['services'][name]['environment'][setting], expected) + self.assertEqual(configured['volumes'], self.compose['volumes']) + + def test_platform_network_injection_keeps_the_credentials_profile_valid(self): + # Dokploy isolated deployments attach a project network to every service. + transformed = copy.deepcopy(self.compose) + transformed['networks']['platform'] = {} + for service in transformed['services'].values(): + service.setdefault('networks', {})['platform'] = None + subprocess.run( + ['docker', 'compose', '-f', '-', '--profile', 'tools', 'config', '--quiet'], + input=json.dumps(transformed), text=True, check=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/configuration.md b/docs/configuration.md index 114e7f66..c629d849 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -110,6 +110,25 @@ Core approves a node's capacity when you generate its Add node command: **Sandbo Set a default in **System** → **Default model configuration**, or use `PUT /core/v1/harnesses/{harness}/model-configuration`. Core encrypts provider keys with `secrets/credential.key` and never returns them. [Model execution](../contracts/agents-api/model-execution.md#deployment-defaults) owns the request fields and replacement rules, and [precedence](../contracts/agents-api/model-execution.md#saved-defaults-and-precedence) says which Sessions use a default. +## Compose installations + +The [standalone Compose template](./getting-started/install-options.md#docker-compose-and-hosting-platforms) uses its Compose definition and the platform's environment as the source of process settings. An unset or empty `OAC_PUBLIC_URL` selects `http://localhost:8080`, allowing startup before a public domain is configured. Core and Web receive that same value. For public access, set `OAC_PUBLIC_URL` to the exact public HTTPS origin without a trailing slash and redeploy Core and Web with the same project and volumes; changing an environment variable requires container recreation, not just a restart. Configure the public origin before adding nodes or executors. The platform owns TLS and routes to `gateway:8080`; Web's installer-managed domain setup is unavailable. + +The initialization service generates secrets and the installation ID once, then verifies them on subsequent deployments. Each secret has one persistent source; Core's key digest is derived from Web's sign-in key. Initialization never replaces missing or changed secrets on an existing installation. Core reads its existing process environment and file settings, so the installer-specific `config.json`, `oac apply` and startup settings snapshot do not apply to this deployment. + +| Compose volume | Content | Readers | +| --- | --- | --- | +| `database` | PostgreSQL data | PostgreSQL; initialization checks whether it is empty | +| `database-secret` | Generated database password | PostgreSQL, migrator and Core | +| `core-config` | Credential encryption key, installation ID, Core key digest and initialization receipt | Migrator and Core | +| `web-secret` | Generated Core sign-in key | Web and the explicit `credentials` tool | +| `core-state` | Private Provider state | Core | +| `node-payload` | Verified node installation metadata | Web | + +Initialization prepares these volumes; application services receive their secret volumes read-only. The `credentials` tool disables container logging. Retrieve its output only in an operator terminal. Database passwords and credential encryption keys are never printed. + +The Compose project name scopes the volumes. Preserve every volume together with that project's definition and public URL. Removing only secret volumes does not reset an installation; initialization refuses to start over an existing database. Core also binds the installation ID to its database. Runtime settings continue to live in [Core's database](#runtime-settings-web). + ## Docker node configuration The node installer writes Docker’s provider configuration into the node’s configuration file; these fields are separate from Core’s `config.json`. Deployment resources, Runtime images and capacity remain in [Core’s database](#runtime-settings-web). diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 11652ce0..287cd0e3 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -14,6 +14,37 @@ With the one-line command, append them after `bash -s --`. The release downloade The installer prints each stage, then a summary of addresses, sign-in details and next steps. Set `NO_COLOR=1` to disable colors. A failed step stops installation without a success message. +## Docker Compose and hosting platforms + +Use the self-contained [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml) with Docker Compose 2.26 or newer on Linux amd64. It pulls the existing, digest-pinned v0.0.3 images and starts PostgreSQL, Core, Web and an HTTP gateway. The one-time initialization service generates random secrets in persistent volumes and prepares the node installer; the migration service initializes the database before Core starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and volumes. + +For a local trial, download `compose.yaml` and the [local port override](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/local.yaml) into one directory, then run: + +```sh +docker compose -f compose.yaml -f local.yaml up -d --wait --wait-timeout 900 +docker compose -f compose.yaml run --rm credentials +``` + +The `credentials` command prints the generated Core key to your terminal without storing it in container logs. Open `http://localhost:8080` and use that key to sign in. All installation secrets are generated automatically; keep the same Compose project and its volumes when restarting. + +The first initialization downloads and verifies the release's approximately 385 MB control archive, retaining only the small node installation metadata. Later starts verify the saved files without downloading again. Image downloads are additional. An interrupted first initialization can be rerun; an existing database with missing installation secrets is refused. + +You can deploy before choosing a domain: leave `OAC_PUBLIC_URL` unset or empty, then follow [Compose configuration](../configuration.md#compose-installations) to set it and redeploy once the platform's domain is ready. The initial localhost origin allows services to start; Web accepts the configured host only, so platform-domain access becomes available after that redeployment. + +### Dokploy + +Create a Docker Compose application and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin, enable isolated deployment, and add a domain for service `gateway`, port `8080`. Enable **HTTPS** and select a certificate provider such as **Let's Encrypt** for that domain before deploying. Deploy without `local.yaml`; internal services publish no host ports. The [template metadata](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/dokploy.toml) supplies the generated domain and environment when packaging this Compose file for Dokploy's template catalog; HTTPS and its certificate provider still need to be enabled after import. + +### Coolify + +Create a **Docker Compose Empty** service and paste `compose.yaml`. Set `OAC_PUBLIC_URL` to the public HTTPS origin and assign that domain to `gateway` on port `8080`. Add Coolify's `exclude_from_hc: true` to the `init`, `migrate` and `credentials` service definitions so completed initialization and optional tooling do not affect its overall health. Save and deploy without `local.yaml`; Coolify supplies HTTPS. + +On either platform, open its server terminal and run `docker compose ls` to find the deployed project name and Compose file. Using those exact values and the deployment's `OAC_PUBLIC_URL`, run `docker compose -p -f run --rm credentials`, then sign in at the configured origin. The [Dokploy domain guide](https://docs.dokploy.com/docs/core/docker-compose/domains) and [Coolify Compose guide](https://coolify.io/docs/services/configuration/docker-compose) describe their domain and service controls. These are importable deployment files; no hosted marketplace listing is published by this repository. + +After signing in, choose the sandbox backend and add nodes using [Nodes](./nodes.md). The Compose stack deploys the control plane; execution machines remain separate. + +Stop with `docker compose stop` using the same files and environment. Back up all [installation volumes](../configuration.md#compose-installations) together while the services are stopped. Follow the [installation version policy](./operations.md#installation-version-policy): a different release needs a new Compose project and fresh volumes. + ## Process settings These flags seed the installation's `config.json` once. Their defaults, valid values and restart behavior are defined in the [configuration reference](../configuration.md#settings). After installation, edit that file and run `oac apply`; rerunning the installer only [repairs](./operations.md#installation-version-policy) the installation. diff --git a/docs/maintainers.md b/docs/maintainers.md index 9434b170..722f5e1a 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -163,7 +163,8 @@ The planner compares the PR event's tested merge commit with its verified first | Group | Checks and consumers | | --- | --- | | `hygiene` | Names, repository links, bundled documentation integrity, and CI planner/gate tests; runs for every change | -| `distribution` | Harness catalog and installer schema, install/apply/recovery/cleanup tests, release/download and bundle contracts, Go console tests and build; needs no pnpm install or browser | +| `distribution` | Harness catalog and installer schema, install/apply/recovery/cleanup tests, Compose parsing and initialization fixtures, release/download and bundle contracts, Go console tests and build; needs Docker Compose for template parsing, no pnpm install or browser | +| `compose` | Real startup from empty volumes using the template's published images, sign-in and API access, file upload and node installer download, then URL reconfiguration and container recreation with preserved credentials and data; needs Docker and network access, no image build or model credentials | | `backend` | Parallel parts, each with a dedicated PostgreSQL guard: `runtime` (sqlc freshness, Runtime/shared Go tests, Linux microsandbox helper, daemon build), `core` (standalone Core build, Core service and client tests) and three `store` shards of the serial Core persistence integration package | | `harness` | Claude SDK tests and packaging, MiniMax companion scripts | | `example` | Optional application typecheck, tests, build and isolated browser acceptance | @@ -175,6 +176,8 @@ The planner compares the PR event's tested merge commit with its verified first Known workflow changes select their consumers: the CI review and actionlint workflows run hygiene and lint; native workflow changes add native checks; API acceptance workflow changes add API checks with container acceptance enabled. The shared Node action selects every job that uses it plus lint. A new or unclassified workflow/action selects the full gate until its consumers are declared in the planner. Planner tests and CI measurement scripts run hygiene; changing the planner itself runs the full gate. +Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. This checks the declared release images and generic Compose behavior; it does not run a Dokploy/Coolify instance or execute a model. + Go module and workspace inputs select backend, API (including the container), native and distribution checks. Node manifests, lockfiles and package-manager configuration select Harness, example, Web, Web acceptance and native checks. The root TypeScript configuration selects Web and example checks; the adapter TypeScript configuration retains the Node consumer group. Each selected set includes hygiene. Mixed changes accumulate their consumers, and every job reads the same plan instead of maintaining its own path list. For example, a notification-only PR skips database, browser and native jobs, while a notification plus Core change adds backend and API checks. Ordinary Markdown and documentation-site configuration run hygiene only, including documentation inside source directories. Generated catalog files and configuration reference sections retain their distribution freshness checks. Core `.go`, `.sql`, helper scripts and configuration inputs select backend/API checks; Web source, styles and assets select Web checks. Embedded native assets and declared test fixture directories select their consumers regardless of suffix, including Markdown prompts and extensionless data. Installer changes add distribution checks. Web changes add Web checks and all browser shards; Core/DB changes add backend and official-client acceptance. Shared contracts, SDKs, Runtime inputs and dependencies propagate to their consumers according to the planner. Generated catalog and protocol inputs include the installer, client and UI consumers. Do not duplicate path lists in reusable workflows or put a `paths` filter on the required workflow. diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 34a9224f..e7e6bd86 100644 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -8,7 +8,7 @@ from pathlib import Path, PurePosixPath import subprocess -JOBS = ("hygiene", "distribution", "backend", "harness", "example", "web", "web-acceptance", "website", "api", "native", "lint") +JOBS = ("hygiene", "distribution", "compose", "backend", "harness", "example", "web", "web-acceptance", "website", "api", "native", "lint") NODE_JOBS = ("harness", "example", "web", "web-acceptance", "website", "native") GO_JOBS = ("distribution", "backend", "api", "native") # Exact file matches keep new workflows/actions conservative until classified. @@ -53,6 +53,8 @@ (("packages/claude-sdk-adapter/", "packages/mcode-harness/"), WEB, ("harness", "native", "backend", "distribution")), (("packages/tsconfig/",), (".json",), NODE_JOBS), (("deploy/install/",), (".py", ".json", ".sh"), ("distribution",)), + (("deploy/compose/",), (".yaml", ".toml"), ("distribution", "compose")), + (("scripts/compose-smoke.py", "deploy/install/test_compose.py"), (".py",), ("distribution", "compose")), (("deploy/install-release.sh", "scripts/install-release.", "scripts/publish-core-release.", "scripts/core-distribution-manifest.", "scripts/build-core-distribution.sh", "scripts/config-reference.py", "scripts/build-web.sh"), SCRIPTS, ("distribution",)), diff --git a/scripts/ci_plan_test.py b/scripts/ci_plan_test.py index 5e0be83f..0c7d93bf 100644 --- a/scripts/ci_plan_test.py +++ b/scripts/ci_plan_test.py @@ -21,6 +21,12 @@ def test_documents_only_need_repository_integrity(self): def test_installer_does_not_download_a_browser_or_run_database_tests(self): self.assertEqual(self.jobs("deploy/install/install.py", "scripts/install-release.test.py"), {"hygiene", "distribution"}) + def test_compose_inputs_select_live_and_fixture_checks_without_image_builds(self): + for path in ("deploy/compose/compose.yaml", "deploy/compose/local.yaml", "deploy/compose/dokploy.toml", + "scripts/compose-smoke.py", "deploy/install/test_compose.py"): + self.assertEqual(self.jobs(path), {"hygiene", "distribution", "compose"}) + self.assertFalse(ci.select([path])["image"]) + def test_web_and_core_have_different_consumers(self): self.assertEqual(self.jobs("apps/web/src/app.tsx"), {"hygiene", "web", "web-acceptance"}) plan = ci.select(["services/core/internal/store/sessions.go"]) @@ -235,6 +241,15 @@ def test_only_deliberately_unselected_jobs_may_skip(self): with self.subTest(state=state), self.assertRaises(ValueError): ci.check_results(plan, needs | {"web": {"result": state}}) + def test_compose_smoke_must_succeed_when_selected(self): + plan = ci.select(["deploy/compose/compose.yaml"]) + needs = {job: {"result": "success" if job in plan["jobs"] else "skipped"} for job in ci.JOBS} + needs["plan"] = {"result": "success"} + ci.check_results(plan, needs) + for state in ("failure", "cancelled", "skipped"): + with self.subTest(state=state), self.assertRaises(ValueError): + ci.check_results(plan, needs | {"compose": {"result": state}}) + def test_matrix_result_failure_is_not_hidden_by_other_jobs(self): plan = ci.full("test") needs = {job: {"result": "success"} for job in (*ci.JOBS, "plan")} diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py new file mode 100644 index 00000000..e27830c3 --- /dev/null +++ b/scripts/compose-smoke.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Exercise the published Compose installation in an isolated Docker project.""" + +import hashlib +import http.cookiejar +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import tempfile +import urllib.error +import urllib.request +import uuid + + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + os.umask(0o077) + project = os.environ.get('COMPOSE_SMOKE_PROJECT', 'oac-smoke-' + uuid.uuid4().hex) + if not re.fullmatch(r'oac-smoke-[a-f0-9]{32}', project): + raise ValueError('COMPOSE_SMOKE_PROJECT must contain a unique oac-smoke- UUID hex value') + artifacts = Path.home() / '.oac/tests' + artifacts.mkdir(parents=True, exist_ok=True) + directory = Path(tempfile.mkdtemp(prefix=project + '-', dir=artifacts)) + override = directory / 'ports.json' + + def publish(port): + override.write_text(json.dumps({'services': {'gateway': {'ports': [ + {'target': 8080, 'published': str(port), 'host_ip': '127.0.0.1'}, + ]}}})) + + publish(0) + env = {**os.environ, 'COMPOSE_PROGRESS': 'plain'} + env.pop('OAC_PUBLIC_URL', None) + command = ['docker', 'compose', '--env-file', os.devnull, '-p', project, + '-f', str(ROOT / 'deploy/compose/compose.yaml'), '-f', str(override)] + + def compose(*args, timeout=120): + result = subprocess.run(command + list(args), env=env, cwd=ROOT, capture_output=True, timeout=timeout) + if result.returncode: + # Startup/configuration stderr helps diagnose failures before containers exist. + # Tool output and application logs may contain credentials and stay private. + detail = '' if args[0] in ('run', 'logs') else result.stderr.decode(errors='replace')[-4096:] + raise RuntimeError(f'Compose {args[0]} failed with exit code {result.returncode}\n{detail}') + return result.stdout + + def client(): + return urllib.request.build_opener(urllib.request.ProxyHandler({}), + urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())) + + browser = client() + origin = 'http://localhost:8080' + address = '' + + def request(path, body=None, headers=None, status=200): + if body is not None and not isinstance(body, bytes): + body = json.dumps(body).encode() + req = urllib.request.Request(address + path, data=body, headers={ + 'Host': origin.split('://', 1)[1], 'Origin': origin, + 'Content-Type': 'application/json', 'OpenAI-Beta': 'agents=v1', **(headers or {}), + }) + try: + response = browser.open(req, timeout=30) + except urllib.error.HTTPError as error: + response = error + with response: + if response.status != status: + raise RuntimeError(f'{req.get_method()} {path}: expected {status}, got {response.status}') + return response.read() + + def get(path, **kwargs): + return json.loads(request(path, **kwargs)) + + def private_logs(*keys): + logs = compose('logs', '--no-color').decode() + assert all(key not in logs for key in keys), 'Credentials appeared in container logs' + return logs + + def terminate(_signum, _frame): + raise SystemExit(1) + + signal.signal(signal.SIGTERM, terminate) + try: + print('Starting published images with an unset public URL and empty volumes', flush=True) + compose('up', '-d', '--wait', '--wait-timeout', '600', timeout=900) + address = 'http://' + compose('port', 'gateway', '8080').decode().strip() + key = compose('run', '--rm', '-T', '--no-deps', 'credentials').decode().strip() + assert len(key) == 64, 'Missing generated sign-in key' + request('/healthz') + assert b'