diff --git a/.changeset/typed-integrations-system.md b/.changeset/typed-integrations-system.md new file mode 100644 index 00000000..feb9798a --- /dev/null +++ b/.changeset/typed-integrations-system.md @@ -0,0 +1,6 @@ +--- +'@frontal-labs/code': minor +'@frontal-labs/code-sdk': minor +--- + +Add a typed, manifest-driven integrations system. Integrations are declared as `IntegrationManifest` files and consumed by the server, CLI, and SDKs. Four first-party providers ship as built-ins — `github`, `linear`, `posthog`, and `sentry` — and can be overridden per-project via `FRONTAL_CODE_INTEGRATIONS_CONFIG` or the `mcp.integrations` settings file. New server routes expose `GET /v1/integrations`, `GET /v1/integrations/{id}`, `GET /v1/integrations/{id}/capabilities`, `GET /v1/integrations/{id}/events`, and a stubbed `POST /v1/integrations/{id}/capabilities/{name}/invoke` (501). The CLI gains `integrations list|show|capabilities|validate` commands, and both the Rust and TypeScript SDKs gain an `IntegrationClient`. The legacy `call_*` MCP wrappers remain functional as a deprecated adapter layer for one release. diff --git a/.env.example b/.env.example index 760cf5cb..9fb3dbb5 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,22 @@ FRONTAL_CODE_SANDBOX_ENABLE_DOCKER=0 # Optional Docker image override when FRONTAL_CODE_SANDBOX_ENABLE_DOCKER=1 FRONTAL_CODE_SANDBOX_DOCKER_IMAGE=busybox:1.36 +# Lane transport for hosted tasks. +# sandbox - Frontal Sandbox VM sessions (default, recommended) +# tools-agent - legacy tools.agent transport +# (requires FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1) +#FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox + +# Frontal Sandbox server (used by FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox) +# Base URL of the sandbox server, e.g. http://localhost:8080 (default when unset) +FRONTAL_SANDBOX_BASE_URL=http://localhost:8080 +# Bearer API key (sbx_...) issued by the sandbox server; leave empty for a +# local server with auth disabled +FRONTAL_SANDBOX_API_KEY= +# Public address worker sessions use to call back to code-server +# (defaults to http://localhost:8788 when unset) +FRONTAL_CODE_SERVER_URL=http://localhost:8788 + # Service Configuration # =========================================== diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 3044a338..1f0f1bdc 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -23,6 +23,7 @@ on: - extensions/slack-agent/** - scripts/** - docs/** + - deploy/sandbox/** - sdks/rust/** - sdks/typescript/** - biome.json @@ -144,3 +145,22 @@ jobs: workspaces: . -> target - name: Run workspace clippy run: cargo clippy --workspace + + sandbox-infra: + name: sandbox deploy (shellcheck + fmt) + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v4 + - name: Install shellcheck + run: sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck + - name: Run shellcheck on deploy scripts + run: shellcheck deploy/sandbox/scripts/*.sh + - name: Install pinned OpenTofu + env: + TOFU_VERSION: "1.10.5" + run: | + curl -fsSL -o /tmp/tofu.zip "https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}/tofu_${TOFU_VERSION}_linux_amd64.zip" + sudo unzip -q /tmp/tofu.zip -d /usr/local/bin + tofu version + - name: Check terraform formatting + run: tofu fmt -check deploy/sandbox/terraform diff --git a/Cargo.lock b/Cargo.lock index e47d037c..d514476a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,12 @@ dependencies = [ "object", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-compression" version = "0.4.52" @@ -256,6 +262,19 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.4.2", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -703,6 +722,7 @@ dependencies = [ "code-runtime", "code-tools", "code-types", + "frontal-sandbox-sdk", "futures-util", "getrandom 0.4.3", "hex", @@ -710,13 +730,16 @@ dependencies = [ "reqwest", "rustls", "rustls-pki-types", + "sandbox-types", "serde", "serde_json", "sha2 0.11.1", "tokio", "tokio-postgres", "tokio-postgres-rustls", + "tokio-stream", "tower", + "tracing", "webpki-roots 0.26.11", ] @@ -837,6 +860,12 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "constant_time_eq" version = "0.6.1" @@ -1235,12 +1264,30 @@ name = "frontal-code-sdk" version = "0.3.0" dependencies = [ "futures-util", + "reqwest", + "serde", "serde_json", "tempfile", "thiserror 1.0.69", "tokio", ] +[[package]] +name = "frontal-sandbox-sdk" +version = "0.2.0" +source = "git+https://github.com/frontal-labs/sandbox#47a87e91fd659afaf16a7cb1ad11556f3b6a88f6" +dependencies = [ + "bytes", + "futures", + "reqwest", + "sandbox-types", + "serde", + "serde_json", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2840,6 +2887,20 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-types" +version = "0.0.0" +source = "git+https://github.com/frontal-labs/sandbox#47a87e91fd659afaf16a7cb1ad11556f3b6a88f6" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "serde", + "serde_json", + "thiserror 2.0.21", + "ulid", +] + [[package]] name = "schannel" version = "0.1.30" @@ -3383,6 +3444,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -3743,6 +3815,17 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "ulid" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dde63b6d514cc5e044edad4e0ca7261afd1099d16c83d942cb2b2f348689c" +dependencies = [ + "rand 0.10.3", + "serde", + "web-time", +] + [[package]] name = "unicase" version = "2.10.0" @@ -4362,7 +4445,7 @@ version = "9.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3476fd497ac8af5b1ec7b51acab7bef380f7e8dd8e0487595bafd839417a302" dependencies = [ - "constant_time_eq", + "constant_time_eq 0.6.1", "crc32fast", "flate2", "getrandom 0.4.3", diff --git a/Cargo.toml b/Cargo.toml index 81db467d..4436b119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ publish = false serde_json = "1" posthog-rs = { version = "=0.33.1", default-features = false } uuid = { version = "1", default-features = false, features = ["v4"] } +frontal-sandbox-sdk = { git = "https://github.com/frontal-labs/sandbox" } +sandbox-types = { git = "https://github.com/frontal-labs/sandbox" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/PLAN-sandbox-worker-migration.md b/PLAN-sandbox-worker-migration.md new file mode 100644 index 00000000..ce13f4de --- /dev/null +++ b/PLAN-sandbox-worker-migration.md @@ -0,0 +1,523 @@ +# Plan: Replace the Docker Worker with Frontal Sandbox (+ Terraform Deployment Infra) + +## Objective + +Wire `https://github.com/frontal-labs/sandbox` in as the execution sandbox for +code-server hosted agents. The current `LocalDocker` `LaneWorkerTransport` +spins up a hardened Docker container per task; this plan **fully replaces it** +with a Frontal Sandbox **session** as the persistent, sandboxed workspace in +which the `frontal-code` agent CLI runs. Sessions start in the **isolate tier** +(Tier 1) and **promote to a QEMU microVM** (Tier 2) when a command needs a real +binary. + +This plan also covers the **deployment infrastructure** that the sandbox server +needs to run — specifically the sandbox repo's `infrastructure/terraform/` +(OpenTofu) provisioning of KVM-capable cloud hosts — so that the code-server +change has an actual place to run. + +## Decisions (from user) + +1. **Depth:** Full replacement of the Docker worker, not a thin new transport. + The `LocalDocker` transport is deprecated and removed; all hosted tasks run + through a Frontal Sandbox session. +2. **Tier:** Start in **isolate tier**, with **VM promotion** for real command + execution (cargo, npm, git, deno, the agent binary, etc.). +3. **SDK:** Use the sandbox repo's Rust SDK `Client` + (`sdks/rust/src/client.rs`), fully usable via the `/v1` HTTP API. Do NOT + depend on the stub `embedded` mode. +4. **Repo import (v1):** Stream files from the local checkout into the session + VFS via `client.write_file` (Option B) rather than `git clone` in-session, to + avoid needing git + network in the VM. +5. **Event streaming:** Keep the agent's existing event protocol — the worker + CLI POSTs fine-grained events to `FRONTAL_CODE_SERVER_URL` / + `/v1/events/ws`; sandbox `exec_stream` only captures top-level stdout/stderr. +6. **Deployment:** Use the sandbox repo's OpenTofu (Terraform-compatible) infra + under `infrastructure/terraform/` to provision a KVM-capable host; code-server + talks to the sandbox server on that host over the allowlisted network. + +## Current state (what we're replacing) + +- `crates/code-server/src/lib.rs`: + - `LaneTransportKind { LocalDocker, ToolsAgent }` (line 167). Default + `LocalDocker`. `parse_lane_transport_kind` (line 176) accepts + `"" | "docker" | "local-docker" | "local_docker"` → `LocalDocker`, and + `tools-agent | tools_agent | agent` → `ToolsAgent`. + - `ServerConfig` (line ~142 `lane_transport_kind`), `ServerConfig::from_env` + (line 222) reads `FRONTAL_CODE_SERVER_LANE_TRANSPORT`. + - `ServerState::new_with_transport_kind*` (lines 427–496) has a `match + lane_transport_kind` (line 474) that builds `LocalDockerLaneWorkerTransport` + or `HostedAgentLaneWorkerTransport`. This is the single wiring point. + - `HostedAgentLaneWorkerTransport` (line 4416) for the `ToolsAgent` kind. + - `LaneWorkerTransport` trait (line 4314): `bootstrap(request) -> + LaneBootstrapResult`, `cancel(worker_id, locator) -> LaneCancellationResult`, + `observe_completion(worker_id, finish_reason, tokens_output) -> + LaneCompletionResult`. + - `LaneExecutionRequest` (line 3867): task_id, prompt, repository, repo_url, + base_ref, branch, lane_role, model, provider, permission_mode, allowed_tools. + - `LaneBootstrapResult` (line ~3855): worker_id, worker_status, signals, + artifacts, checkout_root. + - `LocalDockerLaneWorkerTransport` (line 4510) does: `prepare_checkout` → + `git_head_commit` (baseline) → `write_local_docker_task_payload` → + `worker_access.issue(task_id)` token → `docker.ensure_relay` / + `prepare_task_network` / `launch(DockerLaunchSpec)` running + `frontal-code hosted task run ` with a container env. +- `crates/code-server/src/docker_worker.rs`: Docker CLI command builders, + relay container (`alpine/socat` pinned by digest), per-task network, hardening + args. `CliDockerRunner`, `DockerRunner` trait, `RELAY_ALIAS`, + `task_network_name`, `relay_container_name`. +- `crates/code-server/src/worker_access.rs`: `WorkerAccess`, `issue(task_id)` — + reused unchanged (per-task token + provider proxy env). +- `crates/code-sandbox/`: local per-bash-command isolation (unshare / + sandbox-exec / docker). **Unrelated** to the Frontal Sandbox service; stays + as-is. +- `crates/code-runtime/src/bash.rs`: calls `build_sandbox_command` per bash + command. Unaffected. + +## Target state + +``` +HostedTaskContext + -> LaneWorkerTransport = SandboxLaneWorkerTransport + 1. prepare_checkout (local, for the base_commit baseline only) + 2. worker_access.issue(task_id) -> per-task token + 3. create_session(tier_preference=auto, env, network, initial_snapshot?) + 4. import repo checkout into the session VFS (write_file per file) + 5. write the worker env file + task payload file into the session + 6. exec_stream: run `frontal-code hosted task run ` + - first exec that needs a real binary triggers Tier1 -> Tier2 promotion + 7. stream stdout/stderr back as lane events; fine-grained agent events + flow over the existing FRONTAL_CODE_SERVER_URL /v1/events/ws channel + 8. on completion: read back artifacts (diff vs base_commit), snapshot, sleep + 9. cancel -> destroy_session (best-effort snapshot first) +``` + +The session **is** the persistent workspace: it survives across exec calls, can +sleep/wake between runs, and snapshots give us the baseline + fork for re-runs. +This replaces both the Docker container *and* the local checkout dir. + +## Key technical constraints (from the sandbox repo) + +- **Rust SDK crate:** `frontal-sandbox-sdk` (`sdks/rust/Cargo.toml`, v0.2.0), + depends on `sandbox-types` (path `crates/sandbox-types`). The `Client` + surface: `create_session`, `exec`, `exec_request`, `exec_stream` (NDJSON), + `read_file`, `write_file`, `delete_file`, `snapshot`, `sleep`, `wake`, + `list_snapshots`, `get_snapshot`, `fork`, `fork_command`, `checkpoint`, + `undo`, `get_session`, `destroy_session`, `info`. Only `embedded` mode is a + stub. We use the HTTP client, so no dependency on the unfinished embedded path. +- **API wire format:** snake_case; timestamps RFC 3339; byte fields are arrays + of 0–255 ints (SDK converts to/from `Vec`). Base URL + `http://localhost:8080` (env `SANDBOX_BASE_URL`), Bearer `sbx_` key (env + `SANDBOX_API_KEY`). +- **Isolate tier cannot run host binaries.** `ExecRequest` with a real binary + (git, cargo, npm, the `frontal-code` agent) must promote to a VM. The + `sandbox-promote` crate handles `NeedsVm` → VFS export to ext4 → VM attach. + This is the **highest-risk path** (QEMU/KVM required on a Linux host, + unmeasured promotion latency, ext4 image handoff). +- **Server is early:** in-memory sessions (plus file-backed with `--data-dir`), + no TLS, no rate limiting beyond admission 429s. Not safe to expose publicly. +- **Env is NOT preserved across sleep/wake** (ADR 0010): env values often hold + secrets, so they're dropped. We must re-inject the worker env (server URL, + per-task token, provider proxy) on every wake, or keep env in a file the + agent reads at startup. +- **`SessionConfig` limits/network/env/secret settings that the backend + doesn't yet enforce are refused with 501**, not silently ignored. Only file + size/total/inode/path-depth limits are enforced today. +- **Network policy is default-deny allowlist** — allowlist only the + provider-proxy host + the code-server URL (the worker already uses a proxy + for model traffic). + +## Sandbox Terraform deployment infrastructure (NEW section) + +The sandbox repo ships `infrastructure/` with an OpenTofu (Terraform-compatible) +module for provisioning KVM-capable cloud hosts. The code-server sandbox worker +needs a *running* sandbox server, so deployment of that server is part of this +migration. Reference: `infrastructure/PLAN.md` and `infrastructure/terraform/`. + +### What the terraform module provisions (M4) + +Per `infrastructure/terraform/`: + +- `compute.tf` — one or more KVM-capable instances (`/dev/kvm`, cgroups v2). + These run the `sandbox serve` process (the scheduler/worker in M5). +- `networking.tf` — VPC, subnets, security groups. +- `storage.tf` — S3-compatible object-store bucket (CAS tier) + block volumes + for per-VM overlay images. +- `firewall.tf` — ingress rules: **8080** (API + `/metrics` on the same + listener) and **22** (SSH). + +`versions.tf` / `providers.tf` / `variables.tf` / `outputs.tf` define the +provider (the cloud target), inputs (instance type, region, count, AMI, etc.), +and outputs (the public/private IP or DNS name of the sandbox host, plus the +object-store bucket). + +### How it integrates with code-server + +- code-server talks to the sandbox server over HTTP. The terraform `outputs` + give the host address; that becomes `FRONTAL_SANDBOX_BASE_URL` + (`http://:8080`). The Bearer key (`sbx_…`) becomes + `FRONTAL_SANDBOX_API_KEY`. +- The sandbox session's `network.allow` allowlist must include the code-server's + provider-proxy host + the code-server URL so the in-session agent can POST + events and reach the model. (The sandbox *host* itself is a peer, not an + allowlist entry — the allowlist governs egress from the session VM.) +- Deployment flow (operator): + 1. `tofu init` / `tofu apply` in `infrastructure/terraform/` → provision the + KVM host + object store + firewall. + 2. `ssh` to the host, run `scripts/bootstrap.sh` (deps, KVM, cgroups) and + `scripts/deploy.sh` (install the `sandbox` binary + systemd unit + + `/etc/sandbox/config.toml`). + 3. `scripts/smoke.sh` → create a session, `exec echo hi`, destroy. + 4. Read terraform `outputs` → set `FRONTAL_SANDBOX_BASE_URL` + + `FRONTAL_SANDBOX_API_KEY` in the code-server `.env`, set + `FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox`, restart code-server. + +### Terraform deliverables we add (Phase 3 / new) + +The sandbox repo's `infrastructure/terraform/` is the *upstream* module. We do +not fork it. Instead, in the **frontal-code** repo we add a thin deployment +layer that consumes it: + +- `deploy/sandbox/` (frontal-code repo) — a small wrapper: + - `README.md` — step-by-step operator guide (provision → bootstrap → deploy → + smoke → configure code-server). + - `terraform/` — a wrapper module that `source`s or references the sandbox + repo's `infrastructure/terraform/` (via a `module` block with a local + source path to a vendored copy, or a git source pinned to a rev). It adds + frontal-code-specific variables (e.g. `code_server_egress_allowlist` for the + network policy) and outputs (`sandbox_base_url`, `sandbox_api_key_file`). + - `config/production.toml` — a ready-to-use `sandbox` server TOML config + (`addr = "0.0.0.0:8080"`, `data_dir = "/var/lib/sandbox"`, auth, telemetry) + matching the schema in the sandbox repo's `infrastructure/config/`. + - `smoke.sh` — thin delegator to the sandbox repo's smoke script, or a + self-contained `curl /v1/openapi.json` + session create/exec/destroy. +- `deploy/sandbox/Makefile` — `make provision` (`tofu apply`), `make deploy` + (bootstrap+deploy), `make smoke`, `make destroy`. +- Update `docs/` and `README.md` to document the sandbox deployment + the two + new env vars. + +### Deployment risks & notes + +- **D1 — Terraform/OpenTofu availability.** Operators need `tofu` or + `terraform` on PATH. CI uses whichever is available (the sandbox repo's + `infrastructure/ci/tf-plan.yml` pattern). +- **D2 — Provider pinning.** `providers.tf` pins a cloud provider; changing the + target cloud means updating the provider + variables. Document the default. +- **D3 — No TLS.** The sandbox server has no TLS; the 8080 ingress must be on + a private subnet or behind a TLS-terminating proxy. The firewall module + restricts 8080 to the code-server's security group / CIDR. +- **D4 — Object store.** The CAS tier needs an S3-compatible bucket. For local + dev the server can run with `--data-dir` only (disk CAS); the bucket is for + production durability. + +## Risks & open questions + +| # | Risk | Mitigation | +|---|------|-----------| +| R1 | **Promotion path unproven.** Tier1→Tier2 (VFS→ext4→QEMU) is unmeasured. If it doesn't work end-to-end, the whole plan stalls. | Gate phase 1 on a spike that proves a `sandbox` VM runs `frontal-code --version` and `git --version` in a promoted session. If it fails, fall back to `tier_preference=vm` directly (skip isolate) — still a full replacement, just no Tier1 fast path. | +| R2 | **Platform dependency.** VM tier requires QEMU/KVM on Linux. macOS dev hosts can only run TCG (slow) or not at all. | Document the requirement. CI uses the existing KVM VM workflow pattern. Local macOS dev may be isolate-only until a Linux VM is available. Terraform provisions the KVM host for CI/prod. | +| R3 | **Agent binary must exist in the VM image.** The `frontal-code` CLI + its runtime (Rust toolchain, node, etc.) must be installed in the guest rootfs. | Build a dedicated guest rootfs image (or a snapshot) with the agent + toolchain pre-installed. Use `initial_snapshot` on session create to start from it. This is a new build artifact to manage. | +| R4 | **Network egress from the VM.** The agent needs to reach the model provider (via the server's provider proxy) and the server itself. Sandbox network policy is default-deny allowlist. | Configure `SessionConfig.network.allow` with the server's provider-proxy host + the code-server URL. Only those hosts need allowlisting. | +| R5 | **Event streaming parity.** Current worker streams fine-grained events. Sandbox `exec_stream` gives stdout/stderr as NDJSON. | Keep the agent's existing event protocol: the worker CLI in the sandbox still POSTs to `FRONTAL_CODE_SERVER_URL` (`/v1/events/ws`). The sandbox exec only captures top-level stdout/stderr for logging. This preserves all UI behavior. | +| R6 | **Per-task token + provider proxy env.** The worker needs `FRONTAL_CODE_SERVER_URL`, `FRONTAL_CODE_SERVER_API_KEY`, `FRONTAL_CODE_HOSTED_TASK_FILE`, and provider env. Env is dropped on wake (ADR 0010). | Write the env to a file in the session VFS at bootstrap (e.g. `/workspace/.frontal-code-hosted/env`), and have the agent read it at startup. Re-write on wake. Or keep it in the snapshot's `env` field if supported. | +| R7 | **Artifact extraction.** Today the server reads `manifest_file`/`output_file` from the checkout root and diffs vs `base_commit`. In the sandbox, the repo lives in the session VFS. | After the agent completes, read the manifest/output files back via `client.read_file(session, path)`, and diff against `base_commit` (recorded at bootstrap). Snapshot the session so the final state is preserved for review/fork. | +| R8 | **Cancellation.** `destroy_session` kills the session. A promoted VM mid-exec may leave a half-written state. | On cancel: `destroy_session` + `snapshot` before destroy (best-effort) so the partial work is inspectable. The existing `LaneCancellationResult` shape is preserved. | +| R9 | **Sandbox server availability.** code-server now has a hard dependency on a running `sandbox serve`. If it's down, hosted tasks can't start. | Add a health-check + clear error. Make the sandbox base URL + API key server config (`FRONTAL_SANDBOX_BASE_URL`, `FRONTAL_SANDBOX_API_KEY`). Fail fast with a distinct error code at bootstrap, not mid-task. Terraform provisions the host. | +| R10 | **501 refusals.** `SessionConfig` fields the backend doesn't enforce are 501. | Send only the minimal `SessionConfig` (tier_preference, network, env) and omit `limits`/`secrets` until the backend enforces them. Pin a minimum sandbox-server version. | +| D1 | **Terraform/OpenTofu availability.** | Operators need `tofu` or `terraform` on PATH; CI uses the sandbox repo's `tf-plan.yml` pattern. | +| D2 | **Provider pinning.** | `providers.tf` pins a cloud provider; changing the target cloud means updating provider + variables. Document the default. | +| D3 | **No TLS.** | The 8080 ingress must be on a private subnet or behind a TLS-terminating proxy. Restrict 8080 to the code-server's security group / CIDR in `firewall.tf`. | +| D4 | **Object store.** | The CAS tier needs an S3-compatible bucket in prod. Local dev can use `--data-dir` disk CAS. | + +## Phased plan + +### Phase 0 — Spike: prove promotion works (gate for everything) +**Goal:** de-risk R1/R2/R3 before touching code-server. +1. Stand up a `sandbox serve --addr 127.0.0.1:8080` on a Linux host with QEMU/KVM + (provisioned via the sandbox repo's terraform, or the KVM VM workflow). +2. From a scratch Rust bin (or `cli`): `create_session(tier_preference=auto)`, + then `exec("git --version")`. Confirm the session promotes to a VM and the + command returns exit 0 with real output. +3. Build (or obtain) a guest rootfs image with the `frontal-code` CLI + Rust + toolchain + node installed. Create a session with `initial_snapshot` from it. + Confirm `frontal-code --version` runs in the session. +4. **Exit criteria:** a promoted session runs `git --version` and + `frontal-code --version` and returns correct stdout. If promotion fails, + record the blocker and switch the plan to `tier_preference=vm` (no isolate + fast path). + +### Phase 1 — Add `SandboxLaneWorkerTransport` (new code, no removal yet) +**Goal:** a third transport kind that runs a full hosted task in a sandbox +session, coexisting with `LocalDocker` so we can compare side by side. +1. `crates/code-server/Cargo.toml`: add a dependency on `frontal-sandbox-sdk` + (`sdks/rust` from the sandbox repo, as a `git` dep pinned to a rev) and + `sandbox-types` (for `SessionConfig`, `ExecRequest`, `SessionId`, etc.). + - Decision: `git = "https://github.com/frontal-labs/sandbox"` with a rev. + If the workspace uses `git` deps, register them in the root `Cargo.toml` + `workspace.dependencies` and add to `Cargo.lock`. +2. `crates/code-server/src/lib.rs`: + - Extend `LaneTransportKind` with `Sandbox`. Extend `parse_lane_transport_kind` + (line 176) to accept `"sandbox"`. Update the error message (line 181) to list + `sandbox` as an option. + - Add `SandboxLaneWorkerTransport` struct holding: `Arc`, + `workspace_root` (for the local base_commit baseline), `server_url` (for the + provider-proxy allowlist), `Arc`, and the sandbox config + (base_url, api_key). + - Wire it in `ServerState::new_with_transport_kind*` (line 474 `match`) alongside + the existing `LocalDocker` / `ToolsAgent` arms. + - Add `FRONTAL_SANDBOX_BASE_URL` + `FRONTAL_SANDBOX_API_KEY` to `ServerConfig` + (line ~142) + `ServerConfig::from_env` (line 222) + `.env.example`. +3. Implement `LaneWorkerTransport` for `SandboxLaneWorkerTransport`: + - **`bootstrap(request)`:** + a. Prepare the checkout *locally* (reuse `prepare_checkout`) to compute + `base_commit = git_head_commit(...)` — the baseline for the diff. (The + local checkout is for the baseline only; the *execution* happens in the + sandbox.) + b. `check_hosted_task_policy` (reuse). + c. `worker_access.issue(task_id)` → token (reuse). + d. Build `SessionConfig { tier_preference: Auto, network: allowlist(server + host + provider-proxy host), env: {} }`. (Env goes to a file in step f.) + e. `client.create_session(config)` → `session_id`. + f. Import the repo into the session: stream files from the local checkout + via `client.write_file(session, path, bytes)` for each file in + `prepared.checkout_root` (directory walk). This avoids needing + git+network in the VM and reuses the existing checkout. + g. Write the worker env file into the session + (`/workspace/.frontal-code-hosted/env`) with + `FRONTAL_CODE_SERVER_URL`, `FRONTAL_CODE_SERVER_API_KEY` (token), + provider proxy env, `FRONTAL_CODE_HOSTED_TASK_FILE`. + h. Write the task payload file (reuse `write_local_docker_task_payload`, + adapted to write via `client.write_file`). + i. Return `LaneBootstrapResult { worker_id: session_id, base_commit, ... }`. + - **run:** `client.exec_stream(session, ExecRequest { argv: ["frontal-code", + "hosted", "task", "run", task_id], stream: true, timeout_ms: ... })`. + Consume the NDJSON stream; forward `output` events to the lane event log. + The fine-grained agent events still flow over the existing + `FRONTAL_CODE_SERVER_URL`/`/v1/events/ws` channel (the agent POSTs them + directly), so UI behavior is unchanged. + - **artifacts:** after the exec `exit` event, `client.read_file(session, + manifest_file)` and `output_file`, compute the diff vs `base_commit` + (reuse the existing diff logic in `publish.rs` / lane completion), snapshot + the session (`client.snapshot`), `client.sleep(session)` to release the live + VM while keeping the state. + - **cancel:** `client.destroy_session(session_id)` (best-effort `snapshot` + first). Map to `LaneCancellationResult`. +4. `crates/code-server/Cargo.toml` + root `Cargo.toml`: add `frontal-sandbox-sdk`, + `sandbox-types`, and `reqwest`/`tokio` (already present) as needed. +5. Config: add `FRONTAL_SANDBOX_BASE_URL`, `FRONTAL_SANDBOX_API_KEY` to + `ServerConfig::from_env` (line 222) and `.env.example`. +6. Tests: + - Unit: `parse_lane_transport_kind("sandbox")`, `SandboxLaneWorkerTransport::new`. + - Integration (gated on a running sandbox server, `--ignored` like the KVM + suites): create a session, import a tiny repo, run `echo hello`, read back + stdout. + +### Phase 2 — Persistence & re-run (snapshots, sleep/wake, fork) +**Goal:** make the session the *persistent* workspace, not just a per-task +container. +1. **Re-run / resume:** store `session_id` + `snapshot_id` on + `HostedTaskContext` / the persistence layer. A re-run calls + `client.wake(session_id)` (files come back) or `client.fork(snapshot_id, 1)` + for a fresh copy, then re-injects the env file (env was dropped on wake), + then re-execs. This replaces "spin up a new Docker container + re-clone." +2. **Sleep policy:** idle sessions sleep automatically via + `sandbox serve --idle-sleep-secs` (default 300). Configure the server. Our + `artifacts` step already sleeps after completion. +3. **Fork for parallel lanes:** `LaneExecutionRequest.lane_role` + multi-lane + tasks can `client.fork(snapshot_id, n)` to get N sessions sharing a base, + each independent. Wire this where the current code forks a checkout. +4. Persistence: extend the `ServerState` state file to record + `session_id`/`snapshot_id` per task so a server restart can re-attach. + +### Phase 3 — Remove the Docker worker + land the Terraform deployment +**Goal:** complete the "full replacement" and give operators a deployable +sandbox server. +1. Deprecate `LaneTransportKind::LocalDocker` (keep the variant for a release, + emit a warning in `from_env` if someone still sets it). +2. Delete `LocalDockerLaneWorkerTransport`, `docker_worker.rs` + (`CliDockerRunner`, relay, per-task network, hardening args), and the Docker + launch plumbing in `lib.rs`. +3. Remove Docker-specific config: `FRONTAL_CODE_WORKER_RELAY_IMAGE`, the relay + container, `worker_hardening_args`, etc. (Keep anything the sandbox path + reuses.) +4. **Land the sandbox terraform deployment** (see "Sandbox Terraform deployment + infrastructure" section): `deploy/sandbox/` wrapper module + config + smoke + + Makefile + docs. Wire `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY` + from terraform outputs. +5. Update `.env.example`, `docs/`, README, and the `Bazel` targets + (`crates/code-server/BUILD.bazel`) to drop Docker deps and add the sandbox + server as a required service. +6. Update CI: add a sandbox-server service (the KVM VM workflow pattern) for the + integration tests; the existing Docker-in-Docker CI step is no longer needed + for the worker. Add `infrastructure/**` terraform validation to CI (the + sandbox repo's `tf-plan.yml` pattern). +7. Migration note: operators must run a `sandbox serve` instance (provisioned + via terraform) and set `FRONTAL_SANDBOX_BASE_URL`/`FRONTAL_SANDBOX_API_KEY`. + +### Phase 4 — Hardening & ops (optional, follow-up) +1. **In-VM bash isolation:** the `code-sandbox` per-bash-command wrapper is + redundant once the agent runs inside the sandbox VM. Evaluate removing it or + keeping it as defense-in-depth (the VM is already an isolation boundary). +2. **TLS + auth hardening:** the sandbox server has no TLS. Terminate TLS at a + reverse proxy or add TLS to the server before any non-local deployment. +3. **Quotas & metering:** wire `GET /v1/usage` into the billing/tracker + reporting (`tracker_reporting.rs`). +4. **Trace integration:** `GET /v1/sessions/{id}/trace` gives a command trace + (ADR 0007). Feed it into the existing audit trail. + +## Files touched (summary) + +- `crates/code-server/Cargo.toml` — add `frontal-sandbox-sdk`, `sandbox-types`. +- `crates/code-server/src/lib.rs` — `LaneTransportKind::Sandbox`, + `parse_lane_transport_kind`, `SandboxLaneWorkerTransport`, config + (`FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`), wiring (line 474 + match), then Docker removal (Phase 3). +- `crates/code-server/src/docker_worker.rs` — **removed** (Phase 3). +- `crates/code-server/src/worker_access.rs` — reused unchanged. +- `crates/code-server/BUILD.bazel` — deps + Docker removal. +- `.env.example` — sandbox env vars. +- Root `Cargo.toml` — workspace dep for `frontal-sandbox-sdk`/`sandbox-types` + if pinned. +- `deploy/sandbox/` — **new**: terraform wrapper module, `config/production.toml`, + `smoke.sh`, `Makefile`, `README.md` (Phase 3). +- `docs/` + README — architecture + deployment update. +- CI (`.github/workflows/`, `.buildkite/`) — sandbox service + terraform + validation, drop Docker. + +## Success criteria + +- A hosted task with a repository runs end-to-end in a Frontal Sandbox session: + repo imported, agent runs (promoted to VM), diff produced, PR/publish path + works, all UI events stream as before. +- A re-run resumes from a snapshot (wake or fork) without re-cloning. +- Cancellation destroys the session and preserves partial work via snapshot. +- `LocalDocker` transport and `docker_worker.rs` are removed; no Docker + dependency remains for the worker. +- The sandbox terraform deployment provisions a KVM host; `tofu apply` is clean; + `scripts/smoke.sh` passes; code-server is reachable to the sandbox server via + `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`. +- The existing test suite (minus Docker-specific tests) passes; new sandbox + integration tests pass on the KVM CI. + +## Progress + +### Phase 1 — `SandboxLaneWorkerTransport` ✅ (code complete, compiles) +- `Sandbox` variant in `LaneTransportKind` + parse arm (`"sandbox"`). +- `ServerConfig.sandbox_base_url` / `sandbox_api_key` + `from_env` wiring + (`FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`). +- `ServerState` match arm constructs the transport. +- `SandboxLaneWorkerTransport` impl: `bootstrap` (create session → import + checkout via `write_file` → write env + payload → kick off exec in a + background thread), `observe_completion`, `cancel` (destroys session). +- Git deps pinned: `frontal-sandbox-sdk` v0.2.0 + `sandbox-types` at + `frontal-labs/sandbox` commit `47a87e91`. +- `cargo check -p code-server` passes clean. +- `.env.example` updated with the two new env vars. + +### Terraform deployment wrapper ✅ (created under `deploy/sandbox/`) +- 15 files: `terraform/*.tf` (9), `config/production.toml`, 3 scripts + (`bootstrap.sh`, `deploy.sh`, `smoke.sh`), `Makefile`, 2 READMEs. +- Self-contained AWS EC2 (KVM) + S3 (CAS) wrapper; does not fork upstream + `infrastructure/terraform/`. +- `terraform fmt -check` clean; tomllib parses; shellcheck clean. + +### Phase 0 — Spike ⏳ (blocked) +- Needs a Linux host with QEMU/KVM (this macOS box can only do slow TCG). +- Gates full confidence in the isolate→VM promotion path. + +### Phase 2 — Persistence & re-run ✅ (code complete, compiles) +- `resume()` trait method on `LaneWorkerTransport` (default impl = `bootstrap`). +- Sandbox override: fork from last run's snapshot (`client.fork`) if one exists, + else `get_session` liveness probe → wake parked session + re-import, else + fresh session + full re-import. +- Env file re-injected on every start (fresh, woken, or forked); ADR 0010 env-dropped-on-wake + handled by writing `/workspace/.frontal-code-hosted/env` each time. +- `SandboxTaskState.snapshot_id` stored after clean `Exit{exit_code: 0}` in the + background exec thread. + +### Phase 3 — Docker worker removal ✅ (code complete, compiles) +- `LocalDocker` enum variant + `LocalDockerTransport` + `DockerRunner` deleted + from `lib.rs`; `crates/code-server/src/docker_worker.rs` deleted. +- `LaneTransportKind` parse: `docker`/`local-docker`/`local_docker` → removal + error message; `""`/`sandbox` → Sandbox (default); `tools-agent`/`agent` → ToolsAgent. +- `check_callback_port` deleted (not re-purposed). +- `write_local_docker_task_payload` + `write_worker_env_file` deleted. +- `FRONTAL_CODE_SERVER_DOCKER_IMAGE` / `FRONTAL_CODE_SERVER_CALLBACK_URL` / + `FRONTAL_CODE_SERVER_DOCKER_NETWORK` env vars removed from config parsing. +- `HostedDockerTaskPayload` renamed `HostedLaneTaskPayload` (shared by sandbox + transport + hosted-agent transport). +- Zero `docker` references in `lib.rs` except the intentional parse-error string + and the parser test asserting docker is refused. +- `docker/worker.Dockerfile` marked legacy (kept for reference). +- `infrastructure/compose/docker-compose.docker-worker.yml` deleted; + `docker-compose.yml` updated to sandbox env; `infrastructure/README.md` + rewritten for sandbox mode. +- E2E scripts rewritten: `scripts/worker-e2e.sh` + `scripts/linux-amd64-worker-e2e.sh` + — sandbox transport, no Docker daemon requirement for worker containers. +- CI: `rust-ci.yml` gains `deploy/sandbox/**` push path + `sandbox-infra` job + (shellcheck + `tofu fmt -check`). +- `cargo check -p code-server` passes clean. + +### Phase 0 — Spike ⏳ (in progress) + +**Method (user-directed: "Local Mac + Docker/KVM"):** Apple-silicon macOS cannot +run x86_64 KVM guests (VZ driver is arm64-only), so the spike runs on a local +**Lima arm64 VM** (Ubuntu 24.04, 8 vCPU / 16 GiB / 100 GiB, `vz` driver). +Docker-in-VM runs the sandbox repo's linux-dev image (same `--privileged` +cgroup setup as `docker/run-linux.sh`). Tier 2 VM tests run under **QEMU TCG +software emulation** — the same `promote_tree` / guest-agent / exec code path as +KVM, just slower. Real KVM coverage remains via the upstream `vm-kvm.yml` CI +workflow on the pinned commit (`47a87e9`). + +Steps: +1. ✅ Lima VM `frontal-spike` running; `limactl shell` works; git present. +2. ✅ Docker installed in-VM (apt was blocked by `unattended-upgrades` on + cloud-image boot; stopped+masked the unit). Docker daemon active, server + responds. +3. ✅ Cloned `frontal-labs/sandbox` at `47a87e9` to `~/Desktop/sandbox-spike`. +4. ⏳ Building linux-dev image; Tier 1: `cargo test -p sandbox-worker` in the + `--privileged` container (delegated cgroup at `/sys/fs/cgroup/workers`). +5. ⏳ Tier 2: build the emulated guest image (`emulated-vm/build-image.sh`, + no GUEST_PROGRAMS) then `cargo test -p sandbox-promote --test into_guest -- + --ignored` with `SANDBOX_VM_IMAGE` + `SANDBOX_VM_ACCEL=tcg`. Exit criteria: + a promoted guest session boots the guest agent, `promote_tree` reports the + expected dirs/files/bytes, and the file bytes round-trip. +6. ⏳ Record results; de-risk R1 (promotion path) and R2 (platform dependency). + +Known environment gotchas discovered: +- Lima 2.3.0: `images[].format:` field rejected (non-strict warning, cosmetic + on every `limactl` call — removed from `~/.lima/frontal-spike/lima.yaml`); + `cloudInit.users` required or the guest drops to a login prompt. +- Ubuntu cloud image runs `unattended-upgrades` at boot, grabbing apt locks and + blocking package installs. +- Docker socket needs `sudo` in the Lima guest (usermod -aG docker applies only + to new shells). + +**Scope note:** the full `docker/run-vm-tests.sh` suite also builds the +CPython WASI artifacts and ffmpeg/chromium into the guest image — too heavy for +the spike. The minimal `into_guest` test exercises the exact promotion path +(guest agent boot → `promote_tree` → guest exec) without those extras. + +### Phase 4 — Hardening, docs, infra alignment ✅ +- `docs/design/worker-architecture.md` rewritten to the sandbox transport + (worker → session wording; token-store wording corrected — tokens are hashed + in Postgres, so the sandbox transport requires `FRONTAL_CODE_SERVER_DATABASE_URL`). +- `docs/CONTAINERS.md`, `docs/TROUBLESHOOTING.md`, `docs/DEVELOPMENT.md`, + `docs/README.md`, root `README.md`, `docs/design/conversation-store.md` + updated to session wording. +- `crates/code-server/README.md`: "Local Docker Worker Mode" section replaced + with "Sandbox Worker Mode". +- `infrastructure/README.md`: "Local Docker Worker Mode" replaced with "Sandbox + Worker Mode"; worker-image profile references removed. +- `deploy/sandbox/README.md`: smoke.sh troubleshooting bullet fixed. +- `.env.example`: sandbox vars added (`FRONTAL_SANDBOX_BASE_URL`, + `FRONTAL_SANDBOX_API_KEY`, `FRONTAL_CODE_SERVER_URL`); lane-transport value + comment added. + +## Immediate next steps + +1. Run `cargo test -p code-server` and fix until clean. +2. Stage + commit (Phase 1+2+3+terraform together, or per-phase — decide at + commit time). Everything is currently UNCOMMITTED. +3. Clean up the `code-sandbox-rebuild` worktree once the working-dir file is final. +4. (Optional) Linux+QEMU/KVM spike to validate isolate→VM promotion (R1/R2) + before shipping. diff --git a/README.md b/README.md index 2352c23c..71de9bc0 100644 --- a/README.md +++ b/README.md @@ -309,7 +309,7 @@ See [`./USAGE.md`](./USAGE.md) for usage examples and run `cargo run -p cli -- - - **repo** - repository lifecycle, checkout management, and source tree preparation - **runtime** - `ConversationRuntime`, config loading, session persistence, permission policy, MCP client lifecycle, system prompt assembly, usage tracking - **sandbox** - filesystem isolation, namespace restrictions, and security policies -- **server** - hosted task APIs, event streaming, Docker lane execution, and policy-driven recovery +- **server** - hosted task APIs, event streaming, sandbox-session lane execution, and policy-driven recovery - **telemetry** - session trace events, usage analytics, and monitoring infrastructure - **tools** - tool specs + execution: Bash, ReadFile, WriteFile, EditFile, GlobSearch, GrepSearch, WebSearch, WebFetch, Agent, TodoWrite, NotebookEdit, Skill, ToolSearch, and runtime-facing tool discovery - **training** - style learning, dataset building, profile training, and code adaptation diff --git a/bun.lock b/bun.lock index 73bd784a..d9594ee0 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ }, "sdks/typescript": { "name": "@frontal-labs/code-sdk", - "version": "0.1.0", + "version": "0.2.0", "devDependencies": { "@frontal-labs/code-zod-schemas": "file:../../extensions/zod", "@types/node": "^20.12.0", diff --git a/cli/src/app.rs b/cli/src/app.rs new file mode 100644 index 00000000..c01ba719 --- /dev/null +++ b/cli/src/app.rs @@ -0,0 +1,572 @@ +//! Pure state machine for the interactive REPL. +//! +//! The [`App`] owns all REPL display state: the conversation history, the +//! current turn's streaming status, and the permission mode. It knows nothing +//! about the terminal, the runtime, or I/O — it is a pure state machine that +//! receives [`AppEvent`]s and exposes state a renderer can paint. +//! +//! Why this exists: +//! - The turn loop in `main.rs` mixes state transitions with terminal writes. +//! Extracting the state lets us test the conversation model in isolation and +//! swap the renderer without touching the runtime wiring. +//! - The streaming sink (`streaming.rs`) maintains its own segment model for +//! live rendering. The `App`'s message model is the *settled* view, distinct +//! from the live region. +//! +//! This module is deliberately framework-free. `main.rs` constructs an `App`, +//! feeds it events as the turn progresses, and reads the resulting state. + +use std::time::Instant; + +use code_runtime::{ContentBlock, ConversationMessage, MessageRole, TokenUsage, TurnSummary}; + +// --------------------------------------------------------------------------- +// Messages +// --------------------------------------------------------------------------- + +/// One message in the conversation history. +#[derive(Debug, Clone, PartialEq)] +pub enum Message { + /// A user prompt. + User { + /// The prompt text (after mention expansion). + text: String, + /// Files attached via `@` mentions. + attached: Vec, + }, + /// An assistant response: the final text plus the tool calls it made. + Assistant { + /// The assistant's final text. + text: String, + /// Tool names invoked during this turn, in order. + tools: Vec, + /// Wall-clock duration of the entire turn in seconds. + elapsed_secs: f64, + /// Token usage for the turn. + usage: Option, + /// Number of messages removed by auto-compaction, if it fired. + auto_compaction: Option, + }, +} + +// --------------------------------------------------------------------------- +// Streaming status +// --------------------------------------------------------------------------- + +/// What the UI should show while a turn is in flight. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TurnStatus { + /// Waiting for the first model event — show the thinking animation. + Thinking, + /// A tool is currently executing. + ToolRunning { name: String }, + /// The model is producing text (or just finished a tool and is planning). + Streaming, +} + +// --------------------------------------------------------------------------- +// App state +// --------------------------------------------------------------------------- + +/// The full REPL state. Pure data — no I/O, no terminal, no runtime handle. +#[derive(Debug)] +pub struct App { + /// The conversation so far, oldest first. + pub messages: Vec, + /// The current permission mode (e.g. "default", "plan", "read-only"). + pub permission_mode: String, + /// The model name. + pub model: String, + /// Set while a turn is in flight; `None` when idle. + pub turn_status: Option, + /// Text being streamed right now (grows as deltas arrive). + pub pending_text: String, + /// When the current status segment started (drives elapsed-time display). + pub seg_started: Instant, + /// Set when the last turn ended in an error. + pub last_error: Option, + /// Monotonic frame counter for the thinking animation. + pub frame: u32, +} + +impl App { + #[must_use] + pub fn new(model: String, permission_mode: String) -> Self { + Self { + messages: Vec::new(), + permission_mode, + model, + turn_status: None, + pending_text: String::new(), + seg_started: Instant::now(), + last_error: None, + frame: 0, + } + } +} + +impl Default for App { + fn default() -> Self { + Self::new(String::new(), String::new()) + } +} + +// --------------------------------------------------------------------------- +// Events +// --------------------------------------------------------------------------- + +/// An event that transitions the [`App`]. +#[derive(Debug)] +pub enum AppEvent { + /// The user submitted a prompt (after mention expansion). + UserSubmit { + /// The prompt text. + text: String, + /// Files attached via `@` mentions. + attached: Vec, + }, + /// A turn began — the runtime is about to call the model. + TurnStarted, + /// The model is thinking (no tool running, no text yet). + Thinking, + /// A tool started executing. + ToolStarted { + /// Display name of the tool. + name: String, + }, + /// The current tool finished; the model is thinking again. + ToolFinished, + /// A piece of streamed text arrived. + TextDelta(String), + /// Token usage was reported (informational; rendered at turn end). + Usage(TokenUsage), + /// The model finished a message (it may still run tools). + MessageStop, + /// The turn completed successfully. + TurnCompleted(TurnSummary), + /// The turn failed. + TurnFailed(String), + /// The user changed the permission mode. + PermissionModeChanged { + /// The previous mode (for the transition notice). + previous: String, + /// The new mode. + next: String, + }, + /// Advance the animation frame (render loop idle tick). + Frame, + /// Clear the screen (e.g. Ctrl+L) — the renderer handles the actual clear. + Clear, +} + +// --------------------------------------------------------------------------- +// Transition +// --------------------------------------------------------------------------- + +impl App { + /// Apply an event, mutating state in place. + pub fn handle(&mut self, event: &AppEvent) { + match event { + AppEvent::UserSubmit { text, attached } => { + self.messages.push(Message::User { + text: text.clone(), + attached: attached.clone(), + }); + self.last_error = None; + } + + AppEvent::TurnStarted => { + self.turn_status = Some(TurnStatus::Thinking); + self.pending_text.clear(); + self.seg_started = Instant::now(); + self.last_error = None; + } + + AppEvent::Thinking => { + if !matches!(self.turn_status, Some(TurnStatus::Thinking)) { + self.turn_status = Some(TurnStatus::Thinking); + self.seg_started = Instant::now(); + } + } + + AppEvent::ToolStarted { name } => { + self.turn_status = Some(TurnStatus::ToolRunning { name: name.clone() }); + self.seg_started = Instant::now(); + } + + AppEvent::ToolFinished => { + self.turn_status = Some(TurnStatus::Thinking); + self.seg_started = Instant::now(); + } + + AppEvent::TextDelta(delta) => { + if self.pending_text.is_empty() { + self.turn_status = Some(TurnStatus::Streaming); + self.seg_started = Instant::now(); + } + self.pending_text.push_str(delta); + } + + // Usage is captured on the TurnSummary at completion; clear and + // both no-ops are grouped so the match is exhaustive. + AppEvent::Usage(_) | AppEvent::Clear => {} + + AppEvent::MessageStop => { + self.turn_status = Some(TurnStatus::Thinking); + } + + AppEvent::TurnCompleted(summary) => { + let text = final_assistant_text(summary); + let tools = collect_tool_names(summary); + let usage = Some(summary.usage); + let auto_compaction = summary.auto_compaction.map(|e| e.removed_message_count); + + // Fold any still-pending streamed text into the message so we + // don't lose a final chunk that arrived after the last event. + let text = if !self.pending_text.is_empty() + && !text.is_empty() + && !text.ends_with(&self.pending_text) + { + let mut merged = text; + merged.push('\n'); + merged.push_str(&self.pending_text); + merged + } else { + text + }; + + self.messages.push(Message::Assistant { + text, + tools, + elapsed_secs: 0.0, // filled in by the caller via `stamp_turn_elapsed` + usage, + auto_compaction, + }); + self.turn_status = None; + self.pending_text.clear(); + } + + AppEvent::TurnFailed(message) => { + self.last_error = Some(message.clone()); + self.turn_status = None; + self.pending_text.clear(); + } + + AppEvent::PermissionModeChanged { previous, next } => { + let _ = previous; + self.permission_mode.clone_from(next); + } + + AppEvent::Frame => { + self.frame = self.frame.wrapping_add(1); + } + } + } + + /// Overwrite the elapsed time on the most recent assistant message. + /// Called by the runtime wrapper after the turn returns, when the real + /// wall-clock duration is known. + pub fn stamp_turn_elapsed(&mut self, elapsed_secs: f64) { + if let Some(Message::Assistant { + elapsed_secs: slot, .. + }) = self.messages.last_mut() + { + *slot = elapsed_secs; + } + } + + /// Whether a turn is currently in flight. + #[must_use] + pub fn is_turn_active(&self) -> bool { + self.turn_status.is_some() + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Extract the final assistant text from a turn summary, mirroring the +/// semantics of `main.rs::final_assistant_text` (last message, all text +/// blocks joined). +#[must_use] +pub fn final_assistant_text(summary: &TurnSummary) -> String { + summary + .assistant_messages + .last() + .map(|message| { + message + .blocks + .iter() + .filter_map(|block| match block { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("") + }) + .unwrap_or_default() +} + +/// Extract tool names from a turn summary, in order. +#[must_use] +pub fn collect_tool_names(summary: &TurnSummary) -> Vec { + summary + .assistant_messages + .iter() + .flat_map(|message| message.blocks.iter()) + .filter_map(|block| match block { + ContentBlock::ToolUse { name, .. } => Some(name.clone()), + _ => None, + }) + .collect() +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn app() -> App { + App::new("claude-sonnet-4".into(), "default".into()) + } + + fn empty_summary() -> TurnSummary { + TurnSummary { + assistant_messages: vec![], + tool_results: vec![], + prompt_cache_events: vec![], + iterations: 1, + usage: TokenUsage { + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + auto_compaction: None, + } + } + + fn text_summary(text: &str) -> TurnSummary { + let mut s = empty_summary(); + s.assistant_messages.push(ConversationMessage::new( + MessageRole::Assistant, + vec![ContentBlock::Text { text: text.into() }], + None, + )); + s + } + + #[test] + fn new_app_is_idle() { + let a = app(); + assert_eq!(a.messages.len(), 0); + assert!(a.turn_status.is_none()); + assert_eq!(a.pending_text, ""); + assert!(a.last_error.is_none()); + assert!(!a.is_turn_active()); + } + + #[test] + fn user_submit_pushes_message() { + let mut a = app(); + a.handle(&AppEvent::UserSubmit { + text: "hello".into(), + attached: vec!["src/main.rs".into()], + }); + assert_eq!(a.messages.len(), 1); + match &a.messages[0] { + Message::User { text, attached } => { + assert_eq!(text, "hello"); + assert_eq!(attached, &["src/main.rs"]); + } + Message::Assistant { .. } => panic!("expected User message"), + } + } + + #[test] + fn turn_started_sets_thinking() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + assert!(matches!(a.turn_status, Some(TurnStatus::Thinking))); + assert!(a.is_turn_active()); + } + + #[test] + fn text_delta_after_thinking_moves_to_streaming() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + assert!(matches!(a.turn_status, Some(TurnStatus::Thinking))); + a.handle(&AppEvent::TextDelta("Hello".into())); + assert!(matches!(a.turn_status, Some(TurnStatus::Streaming))); + assert_eq!(a.pending_text, "Hello"); + a.handle(&AppEvent::TextDelta(" world".into())); + assert_eq!(a.pending_text, "Hello world"); + } + + #[test] + fn tool_started_sets_tool_running() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TextDelta("Let me check.".into())); + a.handle(&AppEvent::ToolStarted { + name: "Read".into(), + }); + assert!(matches!( + a.turn_status, + Some(TurnStatus::ToolRunning { ref name }) if name == "Read" + )); + } + + #[test] + fn tool_finished_returns_to_thinking() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::ToolStarted { + name: "Bash".into(), + }); + a.handle(&AppEvent::ToolFinished); + assert!(matches!(a.turn_status, Some(TurnStatus::Thinking))); + } + + #[test] + fn turn_completed_pushes_assistant_message() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TextDelta("Done!".into())); + a.handle(&AppEvent::TurnCompleted(text_summary("Done!"))); + assert!(!a.is_turn_active()); + assert_eq!(a.messages.len(), 1); + match &a.messages[0] { + Message::Assistant { + text, + tools, + usage, + auto_compaction, + .. + } => { + assert_eq!(text, "Done!"); + assert_eq!(tools.len(), 0); + assert!(usage.is_some()); + assert_eq!(*auto_compaction, None); + } + Message::User { .. } => panic!("expected Assistant message"), + } + } + + #[test] + fn turn_completed_collects_tool_names() { + let mut summary = text_summary("text"); + summary.assistant_messages[0] + .blocks + .push(ContentBlock::ToolUse { + id: "1".into(), + name: "Read".into(), + input: "{}".into(), + }); + summary.assistant_messages[0] + .blocks + .push(ContentBlock::ToolUse { + id: "2".into(), + name: "Write".into(), + input: "{}".into(), + }); + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TurnCompleted(summary)); + match &a.messages[0] { + Message::Assistant { tools, .. } => assert_eq!(tools, &["Read", "Write"]), + Message::User { .. } => panic!("expected Assistant message"), + } + } + + #[test] + fn turn_failed_sets_error() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TurnFailed("API error".into())); + assert!(!a.is_turn_active()); + assert_eq!(a.last_error.as_deref(), Some("API error")); + } + + #[test] + fn user_submit_clears_previous_error() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TurnFailed("boom".into())); + assert!(a.last_error.is_some()); + a.handle(&AppEvent::UserSubmit { + text: "retry".into(), + attached: vec![], + }); + assert!(a.last_error.is_none()); + } + + #[test] + fn permission_mode_change_updates_state() { + let mut a = app(); + a.handle(&AppEvent::PermissionModeChanged { + previous: "default".into(), + next: "plan".into(), + }); + assert_eq!(a.permission_mode, "plan"); + } + + #[test] + fn frame_advances() { + let mut a = app(); + let start = a.frame; + a.handle(&AppEvent::Frame); + a.handle(&AppEvent::Frame); + assert_eq!(a.frame, start + 2); + } + + #[test] + fn usage_event_is_noop() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::Usage(TokenUsage { + input_tokens: 1, + output_tokens: 2, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + })); + assert!(matches!(a.turn_status, Some(TurnStatus::Thinking))); + } + + #[test] + fn message_stop_returns_to_thinking() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TextDelta("hello".into())); + assert!(matches!(a.turn_status, Some(TurnStatus::Streaming))); + a.handle(&AppEvent::MessageStop); + assert!(matches!(a.turn_status, Some(TurnStatus::Thinking))); + } + + #[test] + fn stamp_turn_elapsed_updates_last_assistant() { + let mut a = app(); + a.handle(&AppEvent::TurnStarted); + a.handle(&AppEvent::TurnCompleted(text_summary("hi"))); + a.stamp_turn_elapsed(3.5); + match &a.messages[0] { + Message::Assistant { elapsed_secs, .. } => { + assert!((elapsed_secs - 3.5).abs() < f64::EPSILON); + } + Message::User { .. } => panic!("expected Assistant message"), + } + } + + #[test] + fn final_assistant_text_extracts_text_block() { + assert_eq!( + final_assistant_text(&text_summary("Hello world")), + "Hello world" + ); + assert_eq!(final_assistant_text(&empty_summary()), ""); + } +} diff --git a/cli/src/integrations.rs b/cli/src/integrations.rs new file mode 100644 index 00000000..b96ac288 --- /dev/null +++ b/cli/src/integrations.rs @@ -0,0 +1,885 @@ +//! `frontal-code integrations` subcommand handlers. +//! +//! Loads typed [`IntegrationManifest`]s from the project settings +//! (`.frontal-code/settings.json` → `mcp.integrations`) and the +//! `FRONTAL_CODE_INTEGRATIONS_CONFIG` environment variable, then renders +//! them for the CLI. + +use std::path::Path; + +use code_integrations::mcp::config::McpTransport; +use code_integrations::mcp::tools::PermissionMode; +use code_integrations::{ + AuthSpec, CapabilitySpec, EventSpec, IntegrationManifest, ManifestRegistry, SignatureSpec, + TransportConfig, ValidationIssue, +}; +use serde_json::{json, Value}; + +/// Environment variable holding a JSON array of integration manifests. +pub const ENV_INTEGRATIONS_CONFIG: &str = "FRONTAL_CODE_INTEGRATIONS_CONFIG"; + +/// Subcommand help text shown with no arguments or an unknown subcommand. +#[must_use] +pub fn help_text() -> &'static str { + "Usage: frontal-code integrations [options] + +Subcommands: + list List all configured integrations + show Show full details for an integration + capabilities List the capabilities of an integration + validate Validate a manifest file + +Options: + --json Emit JSON instead of a table" +} + +/// Load the manifest registry: the built-in providers (GitHub, Linear, +/// `PostHog`, and Sentry) as defaults, layered with settings discovered by +/// walking up from `cwd` and the `FRONTAL_CODE_INTEGRATIONS_CONFIG` env var. +/// +/// Configuration entries with the same id as a built-in override it; entries +/// that fail to parse are skipped (invalid entries cannot be registered +/// anyway). +pub fn load_registry(cwd: &Path) -> ManifestRegistry { + let mut registry = + code_integrations::builtin_providers().expect("built-in provider manifests are valid"); + + if let Some(settings) = find_settings_file(cwd) { + if let Ok(contents) = std::fs::read_to_string(&settings) { + if let Ok(parsed) = serde_json::from_str::(&contents) { + if let Some(integrations) = parsed + .get("mcp") + .and_then(|mcp| mcp.get("integrations")) + .and_then(|value| value.as_object()) + { + for (_, manifest_json) in integrations { + let Ok(manifest) = + serde_json::from_value::(manifest_json.clone()) + else { + continue; + }; + register_or_override(&mut registry, manifest); + } + } + } + } + } + + if let Ok(raw) = std::env::var(ENV_INTEGRATIONS_CONFIG) { + let Ok(entries) = serde_json::from_str::>(&raw) else { + return registry; + }; + for manifest in entries { + register_or_override(&mut registry, manifest); + } + } + + registry +} + +/// Register a manifest, replacing any existing manifest with the same id so +/// that project configuration can override a built-in provider. +fn register_or_override(registry: &mut ManifestRegistry, manifest: IntegrationManifest) { + // Validate before replacing so an invalid incoming manifest can never + // evict a manifest that is already registered. + if manifest.validate().is_err() { + return; + } + let id = manifest.id.clone(); + if registry.get(&id).is_some() { + registry.remove(&id); + } + let _ = registry.register(manifest); +} + +/// Walk up from `cwd` (inclusive) looking for `.frontal-code/settings.json`. +fn find_settings_file(cwd: &Path) -> Option { + let mut current: Option<&Path> = Some(cwd); + while let Some(dir) = current { + let candidate = dir.join(".frontal-code").join("settings.json"); + if candidate.is_file() { + return Some(candidate); + } + current = dir.parent(); + } + None +} + +/// Summary of one integration for table/JSON rendering. +#[derive(serde::Serialize)] +struct IntegrationSummary { + id: String, + version: String, + display_name: String, + capability_count: usize, + auth: String, +} + +fn summarize(manifest: &IntegrationManifest) -> IntegrationSummary { + IntegrationSummary { + id: manifest.id.clone(), + version: manifest.version.clone(), + display_name: manifest.display_name.clone(), + capability_count: manifest.capabilities.len(), + auth: auth_label(manifest.auth.as_ref()), + } +} + +fn auth_label(auth: Option<&AuthSpec>) -> String { + match auth { + Some(AuthSpec::OAuth { .. }) => "oauth".to_string(), + Some(AuthSpec::ApiKey { .. }) => "api_key".to_string(), + Some(AuthSpec::None) | None => "none".to_string(), + } +} + +fn transport_label(transport: &McpTransport) -> String { + match transport { + McpTransport::Stdio => "stdio", + McpTransport::Sse => "sse", + McpTransport::Http => "http", + McpTransport::WebSocket => "websocket", + McpTransport::Sdk => "sdk", + McpTransport::ManagedProxy => "managed_proxy", + } + .to_string() +} + +fn permission_label(mode: PermissionMode) -> String { + match mode { + PermissionMode::ReadOnly => "read-only", + PermissionMode::WorkspaceWrite => "workspace-write", + PermissionMode::DangerFullAccess => "danger-full-access", + } + .to_string() +} + +fn transport_detail(config: &TransportConfig) -> String { + match config { + TransportConfig::Stdio { command, args, .. } => { + let args = args + .iter() + .map(ToString::to_string) + .collect::>() + .join(" "); + if args.is_empty() { + command.clone() + } else { + format!("{command} {args}") + } + } + TransportConfig::Http { url, .. } + | TransportConfig::Sse { url, .. } + | TransportConfig::WebSocket { url } => url.clone(), + TransportConfig::Sdk { module } => module.clone(), + TransportConfig::ManagedProxy { target, .. } => target.clone(), + } +} + +/// Render the integration list as a human-readable table. +pub fn list_integrations(registry: &ManifestRegistry) -> String { + let manifests = registry.list(); + let mut lines = vec![format!("INTEGRATIONS ({})", manifests.len())]; + if manifests.is_empty() { + lines.push(String::new()); + lines.push(" No integrations configured.".to_string()); + return lines.join("\n"); + } + lines.push(String::new()); + + let id_width = manifests + .iter() + .map(|manifest| manifest.id.len()) + .max() + .unwrap_or(0) + .max("ID".len()); + let name_width = manifests + .iter() + .map(|manifest| manifest.display_name.len()) + .max() + .unwrap_or(0) + .max("NAME".len()); + lines.push(format!( + " {id: String { + let summaries: Vec = registry + .list() + .iter() + .map(|manifest| json!(summarize(manifest))) + .collect(); + serde_json::to_string_pretty(&Value::Array(summaries)).unwrap_or_else(|_| "[]".to_string()) +} + +/// Render full detail for one integration. +pub fn show_integration(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + + let transport = format!( + "{} ({})", + transport_label(&manifest.transport), + transport_detail(&manifest.transport_config) + ); + let auth = auth_detail(manifest.auth.as_ref()).unwrap_or_else(|| "none".to_string()); + let mut lines = vec![ + format!("INTEGRATION: {}", manifest.id), + String::new(), + format!(" {id:<16}", id = manifest.id), + format!(" version: {version}", version = manifest.version), + format!( + " name: {display_name}", + display_name = manifest.display_name + ), + format!( + " description: {description}", + description = manifest.description + ), + format!(" transport: {transport}"), + format!(" auth: {auth}"), + ]; + + lines.push(String::new()); + lines.push(format!(" CAPABILITIES ({}):", manifest.capabilities.len())); + for capability in &manifest.capabilities { + let idempotent = if capability.idempotent { + " idempotent" + } else { + "" + }; + lines.push(format!( + " {name:<20} [{permission}{idempotent}", + name = capability.name, + permission = permission_label(capability.permission_mode), + idempotent = idempotent, + )); + lines.push(format!(" {}", capability.description)); + } + + if !manifest.events.is_empty() { + lines.push(String::new()); + lines.push(format!(" EVENTS ({}):", manifest.events.len())); + for event in &manifest.events { + let signature = event + .signature + .as_ref() + .map_or_else(|| "none".to_string(), signature_label); + lines.push(format!( + " {name:<20} {description}", + name = event.name, + description = event.description + )); + lines.push(format!(" signature: {signature}")); + } + } + + if !manifest.context_fields.is_empty() { + lines.push(String::new()); + lines.push(format!( + " CONTEXT FIELDS ({}):", + manifest.context_fields.len() + )); + for field in &manifest.context_fields { + lines.push(format!( + " {name:<20} {description}", + name = field.name, + description = field.description + )); + } + } + + Ok(lines.join("\n")) +} + +fn signature_label(signature: &SignatureSpec) -> String { + match signature { + SignatureSpec::HmacSha256 { secret_env } => format!("hmac_sha256 ({secret_env})"), + SignatureSpec::None => "none".to_string(), + } +} + +fn auth_detail(auth: Option<&AuthSpec>) -> Option { + match auth { + Some(AuthSpec::OAuth { + client_id_env, + client_secret_env, + auth_url, + scopes, + .. + }) => Some(format!( + "oauth (client_id_env={client_id_env}, client_secret_env={}, scopes=[{}]) auth_url={auth_url}", + client_secret_env + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("unset"), + scopes.join(", "), + )), + Some(AuthSpec::ApiKey { env_var, header }) => Some(format!( + "api_key (env_var={env_var}, header={})", + header.as_deref().unwrap_or("default") + )), + Some(AuthSpec::None) => Some("none".to_string()), + None => None, + } +} + +/// Render full detail for one integration as a JSON object. +pub fn show_integration_json(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + serde_json::to_string_pretty(&manifest) + .map_err(|error| format!("failed to serialize integration `{id}`: {error}")) +} + +/// Render the capability table for one integration. +pub fn list_capabilities(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + + let mut lines = vec![format!( + "CAPABILITIES: {id} ({})", + manifest.capabilities.len() + )]; + if manifest.capabilities.is_empty() { + lines.push(String::new()); + lines.push(" No capabilities declared.".to_string()); + return Ok(lines.join("\n")); + } + lines.push(String::new()); + + let name_width = manifest + .capabilities + .iter() + .map(|capability| capability.name.len()) + .max() + .unwrap_or(0) + .max("NAME".len()); + lines.push(format!( + " {name: Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + let capabilities: Vec = manifest + .capabilities + .iter() + .map(|capability| { + let summary = json!(capability); + json!({ + "name": summary.get("name"), + "description": summary.get("description"), + "permission_mode": summary.get("permission_mode"), + "idempotent": summary.get("idempotent"), + }) + }) + .collect(); + serde_json::to_string_pretty(&Value::Array(capabilities)) + .map_err(|error| format!("failed to serialize capabilities for `{id}`: {error}")) +} + +/// Parse a file as an [`IntegrationManifest`] and run its validation. +/// +/// Returns `Ok` with a short confirmation when the manifest is valid, or +/// `Err` containing a human-readable issue report when it is not. +pub fn validate_manifest_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + let manifest: IntegrationManifest = serde_json::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display()))?; + match manifest.validate() { + Ok(()) => Ok(format!( + "{}: valid ({} {})", + path.display(), + manifest.id, + manifest.version + )), + Err(issues) => { + let report = render_validation_report(path, &manifest, issues); + Err(report) + } + } +} + +fn render_validation_report( + path: &Path, + manifest: &IntegrationManifest, + issues: Vec, +) -> String { + let mut lines = vec![format!( + "{}: invalid ({} {}, {} issue(s))", + path.display(), + manifest.id, + manifest.version, + issues.len() + )]; + for issue in issues { + lines.push(format!( + " {field}: {message}", + field = issue.field, + message = issue.message + )); + } + lines.join("\n") +} + +/// JSON report for `validate`. +pub fn validate_manifest_file_json(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + let manifest: IntegrationManifest = serde_json::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display()))?; + let issues = match manifest.validate() { + Ok(()) => Vec::new(), + Err(issues) => issues, + }; + let report = json!({ + "path": path.display().to_string(), + "id": manifest.id, + "version": manifest.version, + "valid": issues.is_empty(), + "issues": issues.iter().map(|issue| json!({ + "field": issue.field, + "message": issue.message, + })).collect::>(), + }); + let output = serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize validation report: {error}"))?; + if issues.is_empty() { + Ok(output) + } else { + Err(output) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Guards tests that read `FRONTAL_CODE_INTEGRATIONS_CONFIG`; cargo runs + /// tests on multiple threads and env vars are process-global. + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn minimal_manifest_json(id: &str) -> Value { + json!({ + "id": id, + "version": "1.0.0", + "display_name": id, + "description": format!("{id} integration"), + "transport": "stdio", + "transport_config": { + "kind": "stdio", + "command": "mcp-server", + "args": ["serve"], + "env": {} + }, + "capabilities": [{ + "name": "noop", + "description": "No-op capability", + "input_schema": {"type": "object"}, + "permission_mode": "read_only", + "idempotent": true + }], + "auth": null, + "events": [], + "context_fields": [] + }) + } + + fn test_registry() -> ManifestRegistry { + let manifest = IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: std::collections::BTreeMap::new(), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: json!({"type": "object"}), + output_schema: Some(json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "list_issues".to_string(), + description: "List issues".to_string(), + input_schema: json!({"type": "object"}), + output_schema: None, + permission_mode: PermissionMode::ReadOnly, + idempotent: true, + }, + ], + auth: Some(AuthSpec::OAuth { + client_id_env: "GITHUB_OAUTH_CLIENT_ID".to_string(), + client_secret_env: Some("GITHUB_OAUTH_CLIENT_SECRET".to_string()), + auth_url: "https://github.com/login/oauth/authorize".to_string(), + token_url: "https://github.com/login/oauth/access_token".to_string(), + scopes: vec!["repo".to_string()], + redirect_path: None, + }), + events: Vec::::new(), + context_fields: Vec::new(), + }; + let mut registry = ManifestRegistry::new(); + registry.register(manifest).expect("register"); + registry + } + + fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "frontal-integrations-test-{name}-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + #[test] + fn load_registry_reads_settings_file_and_stops_at_first_match() { + let dir = temp_dir("settings"); + let settings_dir = dir.join(".frontal-code"); + std::fs::create_dir_all(&settings_dir).expect("create settings dir"); + let manifest_json = json!({ + "id": "linear", + "version": "2.1.0", + "display_name": "Linear", + "description": "Linear integration", + "transport": "stdio", + "transport_config": { + "kind": "stdio", + "command": "linear-mcp", + "args": ["serve"], + "env": {} + }, + "capabilities": [{ + "name": "create_issue", + "description": "Create an issue", + "input_schema": {"type": "object"}, + "permission_mode": "workspace_write", + "idempotent": false + }], + "auth": { + "kind": "api_key", + "env_var": "LINEAR_API_KEY", + "header": null + }, + "events": [], + "context_fields": [] + }); + let settings = json!({ "mcp": { "integrations": { "linear": manifest_json } } }); + std::fs::write( + settings_dir.join("settings.json"), + serde_json::to_string_pretty(&settings).expect("serialize"), + ) + .expect("write settings"); + + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + let registry = load_registry(&dir); + // The built-in linear manifest is overridden by the settings file; + // the other built-ins remain. + assert_eq!(registry.list().len(), 4); + let linear = registry.get("linear").expect("linear registered"); + assert_eq!(linear.version, "2.1.0"); + assert!(matches!(linear.auth, Some(AuthSpec::ApiKey { .. }))); + assert!(registry.get("github").is_some()); + assert!(registry.get("sentry").is_some()); + } + + #[test] + fn load_registry_skips_invalid_settings_but_still_loads_env() { + let dir = temp_dir("bad-settings-env"); + let settings_dir = dir.join(".frontal-code"); + std::fs::create_dir_all(&settings_dir).expect("create settings dir"); + // Malformed settings JSON: the settings layer must be skipped, + // but the env-var layer must still be loaded. + std::fs::write(settings_dir.join("settings.json"), "{ not valid json") + .expect("write settings"); + + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + let env_value = serde_json::to_string(&vec![minimal_manifest_json("env-integration")]) + .expect("serialize env manifest"); + std::env::set_var(ENV_INTEGRATIONS_CONFIG, env_value); + let registry = load_registry(&dir); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + + // Built-ins (4) plus the env-var integration: the invalid settings + // layer was skipped, and the env layer was still registered. + assert_eq!(registry.list().len(), 5, "got: {:?}", registry.list()); + assert!( + registry.get("env-integration").is_some(), + "env integration missing: {:?}", + registry.list() + ); + assert!(registry.get("github").is_some()); + } + + #[test] + fn load_registry_invalid_env_manifest_preserves_builtin() { + let dir = temp_dir("bad-env-manifest"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + // An env-var manifest with the id "github" but no capabilities: it is + // invalid, so `register_or_override` must reject it WITHOUT evicting + // the built-in github manifest. + let env_value = + serde_json::to_string(&vec![minimal_manifest_json("github")]).expect("serialize"); + let env_value = env_value.replace( + r#""capabilities": [{"name": "noop", "description": "No-op capability", "input_schema": {"type": "object"}, "permission_mode": "read_only", "idempotent": true}]"#, + r#""capabilities": []"#, + ); + std::env::set_var(ENV_INTEGRATIONS_CONFIG, env_value); + let registry = load_registry(&dir); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + + // The built-in github manifest must still be registered (version + // 1.0.0, http transport) — the invalid override was dropped. + let github = registry + .get("github") + .expect("github should still be registered"); + assert_eq!(github.version, "1.0.0"); + assert!( + !github.capabilities.is_empty(), + "built-in github manifest lost its capabilities" + ); + // Total: 4 built-ins, no additions. + assert_eq!(registry.list().len(), 4, "got: {:?}", registry.list()); + } + + #[test] + fn load_registry_seeds_builtins_when_no_config() { + let dir = temp_dir("empty"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + let registry = load_registry(&dir); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing built-in `{id}`"); + } + } + + #[test] + fn list_integrations_renders_table() { + let registry = test_registry(); + let output = list_integrations(®istry); + assert!(output.contains("INTEGRATIONS (1)"), "got: {output}"); + assert!(output.contains(" ID VERSION"), "got: {output}"); + assert!(output.contains("github"), "got: {output}"); + assert!(output.contains("1.0.0"), "got: {output}"); + assert!(output.contains("GitHub"), "got: {output}"); + assert!(output.contains("oauth"), "got: {output}"); + } + + #[test] + fn list_integrations_json_renders_array() { + let registry = test_registry(); + let output = list_integrations_json(®istry); + let parsed: Value = serde_json::from_str(&output).expect("valid json"); + let array = parsed.as_array().expect("array"); + assert_eq!(array.len(), 1); + assert_eq!(array[0].get("id").and_then(Value::as_str), Some("github")); + assert_eq!( + array[0].get("capability_count").and_then(Value::as_u64), + Some(2) + ); + assert_eq!(array[0].get("auth").and_then(Value::as_str), Some("oauth")); + } + + #[test] + fn show_integration_known_id() { + let registry = test_registry(); + let output = show_integration(®istry, "github").expect("known id"); + assert!(output.contains("INTEGRATION: github"), "got: {output}"); + assert!(output.contains("gh-mcp serve"), "got: {output}"); + assert!(output.contains("create_issue"), "got: {output}"); + assert!(output.contains("workspace-write"), "got: {output}"); + assert!(output.contains("idempotent"), "got: {output}"); + } + + #[test] + fn show_integration_unknown_id() { + let registry = test_registry(); + let error = show_integration(®istry, "missing").expect_err("unknown id fails"); + assert!(error.contains("`missing` not found"), "got: {error}"); + } + + #[test] + fn show_integration_json_roundtrips() { + let registry = test_registry(); + let output = show_integration_json(®istry, "github").expect("known id"); + let parsed: IntegrationManifest = + serde_json::from_str(&output).expect("valid manifest json"); + assert_eq!(parsed.id, "github"); + } + + #[test] + fn list_capabilities_renders_table() { + let registry = test_registry(); + let output = list_capabilities(®istry, "github").expect("known id"); + assert!(output.contains("CAPABILITIES: github (2)"), "got: {output}"); + assert!(output.contains("create_issue"), "got: {output}"); + assert!(output.contains("list_issues"), "got: {output}"); + assert!(output.contains("read-only"), "got: {output}"); + assert!(output.contains("no"), "got: {output}"); + } + + #[test] + fn list_capabilities_unknown_id() { + let registry = test_registry(); + let error = list_capabilities(®istry, "missing").expect_err("unknown id fails"); + assert!(error.contains("`missing` not found"), "got: {error}"); + } + + fn write_manifest(dir: &Path, name: &str, contents: &str) -> std::path::PathBuf { + let path = dir.join(name); + std::fs::write(&path, contents).expect("write manifest"); + path + } + + fn valid_manifest_json() -> String { + serde_json::to_string_pretty(&json!({ + "id": "github", + "version": "1.0.0", + "display_name": "GitHub", + "description": "GitHub integration", + "transport": "stdio", + "transport_config": { + "kind": "stdio", + "command": "gh-mcp", + "args": ["serve"], + "env": {} + }, + "capabilities": [{ + "name": "create_issue", + "description": "Create an issue", + "input_schema": {"type": "object"}, + "permission_mode": "workspace_write", + "idempotent": false + }], + "auth": { + "kind": "api_key", + "env_var": "GITHUB_TOKEN", + "header": null + }, + "events": [], + "context_fields": [] + })) + .expect("serialize") + } + + #[test] + fn validate_manifest_file_valid() { + let dir = temp_dir("validate-valid"); + let path = write_manifest(&dir, "valid.json", &valid_manifest_json()); + let output = validate_manifest_file(&path).expect("valid manifest"); + assert!(output.contains("valid"), "got: {output}"); + assert!(output.contains("github"), "got: {output}"); + } + + #[test] + fn validate_manifest_file_invalid_reports_issues_and_fails() { + let dir = temp_dir("validate-invalid"); + let bad = valid_manifest_json().replace("\"github\"", "\"Bad Id!\""); + let path = write_manifest(&dir, "invalid.json", &bad); + let result = validate_manifest_file(&path); + assert!(result.is_err(), "invalid manifest should return Err"); + let error = result.expect_err("invalid manifest returns Err"); + assert!(error.contains("invalid"), "got: {error}"); + assert!(error.contains("id"), "got: {error}"); + } + + #[test] + fn validate_manifest_file_json_invalid_reports_issues_and_fails() { + let dir = temp_dir("validate-json-invalid"); + let bad = valid_manifest_json().replace("\"github\"", "\"Bad Id!\""); + let path = write_manifest(&dir, "invalid.json", &bad); + let result = validate_manifest_file_json(&path); + assert!(result.is_err(), "invalid manifest should return Err"); + let error = result.expect_err("invalid manifest returns Err"); + let parsed: Value = serde_json::from_str(&error).expect("error is valid JSON report"); + assert_eq!( + parsed.get("valid").and_then(Value::as_bool), + Some(false), + "report should mark manifest invalid" + ); + assert!( + parsed.get("issues").and_then(Value::as_array).is_some(), + "report should include issues" + ); + } + + #[test] + fn validate_manifest_file_unparseable() { + let dir = temp_dir("validate-unparseable"); + let path = write_manifest(&dir, "unparseable.json", "{ not json"); + let error = validate_manifest_file(&path).expect_err("unparseable fails"); + assert!(error.contains("failed to parse"), "got: {error}"); + } + + #[test] + fn validate_manifest_file_json_shape() { + let dir = temp_dir("validate-json"); + let path = write_manifest(&dir, "valid.json", &valid_manifest_json()); + let output = validate_manifest_file_json(&path).expect("valid manifest"); + let parsed: Value = serde_json::from_str(&output).expect("valid json"); + assert_eq!(parsed.get("valid").and_then(Value::as_bool), Some(true)); + assert!(parsed.get("issues").and_then(Value::as_array).is_some()); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 0d86bf1d..9552a57f 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -10,12 +10,15 @@ clippy::too_many_arguments )] mod analytics; +mod app; mod auth; mod init; mod input; +mod integrations; mod mentions; mod notify; mod render; +mod streaming; mod thinking; use std::collections::{BTreeMap, BTreeSet}; @@ -255,6 +258,10 @@ fn dispatch(action: CliAction) -> Result<(), Box> { args, output_format, } => LiveCli::print_mcp(args.as_deref(), output_format)?, + CliAction::Integrations { + args, + output_format, + } => run_integrations_command(&args, output_format)?, CliAction::Trust { revoke } => print_workspace_trust(revoke)?, CliAction::Skills { args, @@ -331,13 +338,99 @@ fn dispatch(action: CliAction) -> Result<(), Box> { provider, allowed_tools, permission_mode, - } => run_repl_with_provider(model, provider, allowed_tools, permission_mode)?, + legacy_input, + } => { + run_repl_with_provider( + model, + provider, + allowed_tools, + permission_mode, + legacy_input, + )?; + } CliAction::HelpTopic(topic) => print_help_topic(topic), CliAction::Help { output_format } => print_help(output_format)?, } Ok(()) } +fn run_integrations_command( + args: &[String], + output_format: CliOutputFormat, +) -> Result<(), Box> { + let mut tokens = args.to_vec(); + let mut as_json = matches!(output_format, CliOutputFormat::Json); + tokens.retain(|token| { + if token == "--json" || token == "--output-format=json" { + as_json = true; + false + } else { + true + } + }); + + let Some(subcommand) = tokens.first().map(String::as_str) else { + println!("{}", integrations::help_text()); + return Ok(()); + }; + + match subcommand { + "list" => { + let registry = integrations::load_registry(&env::current_dir()?); + let output = if as_json { + integrations::list_integrations_json(®istry) + } else { + integrations::list_integrations(®istry) + }; + println!("{output}"); + } + "show" | "capabilities" => { + let id = tokens + .get(1) + .and_then(|value| value.split('=').next()) + .filter(|value| !value.starts_with("--")) + .ok_or_else(|| { + "`frontal-code integrations {subcommand}` requires an integration id. \ + Use `frontal-code integrations {subcommand} `." + .to_string() + })? + .to_string(); + let registry = integrations::load_registry(&env::current_dir()?); + let output = match (subcommand, as_json) { + ("show", true) => integrations::show_integration_json(®istry, &id), + ("show", false) => integrations::show_integration(®istry, &id), + ("capabilities", true) => integrations::list_capabilities_json(®istry, &id), + _ => integrations::list_capabilities(®istry, &id), + } + .map_err(Box::::from)?; + println!("{output}"); + } + "validate" => { + let path = tokens + .get(1) + .filter(|value| !value.starts_with("--")) + .ok_or_else(|| { + "`frontal-code integrations validate` requires a manifest file path. \ + Use `frontal-code integrations validate `." + .to_string() + })?; + let path = Path::new(path); + let output = if as_json { + integrations::validate_manifest_file_json(path) + } else { + integrations::validate_manifest_file(path) + } + .map_err(Box::::from)?; + println!("{output}"); + } + other => { + eprintln!("unknown integrations subcommand: {other}"); + eprintln!("{}", integrations::help_text()); + } + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq, Eq)] struct TelemetryResolution { enabled: bool, @@ -475,6 +568,10 @@ enum CliAction { args: Option, output_format: CliOutputFormat, }, + Integrations { + args: Vec, + output_format: CliOutputFormat, + }, Trust { revoke: bool, }, @@ -542,6 +639,7 @@ enum CliAction { provider: Option, allowed_tools: Option, permission_mode: PermissionMode, + legacy_input: bool, }, HelpTopic(LocalHelpTopic), // prompt-mode formatting is only supported for non-interactive runs @@ -898,6 +996,7 @@ fn parse_args(args: &[String]) -> Result { let mut allowed_tool_values = Vec::new(); let mut rest = Vec::new(); let mut dash_p_prompt: Option = None; + let mut legacy_input = false; let mut index = 0; while index < args.len() { @@ -958,6 +1057,10 @@ fn parse_args(args: &[String]) -> Result { permission_mode_override = Some(PermissionMode::DangerFullAccess); index += 1; } + "--legacy-input" => { + legacy_input = true; + index += 1; + } "-p" => { // FrontalCode compat: -p "prompt" = one-shot prompt. The words run // until the next `--flag`, so `prompt -p hi --stream` still parses @@ -1033,6 +1136,7 @@ fn parse_args(args: &[String]) -> Result { provider, allowed_tools, permission_mode, + legacy_input, }); } if rest.first().map(String::as_str) == Some("--resume") { @@ -1087,6 +1191,7 @@ fn parse_args(args: &[String]) -> Result { provider: provider.clone(), allowed_tools, permission_mode, + legacy_input, }), "plugins" => Ok(CliAction::Plugins { action: rest.get(1).cloned(), @@ -1101,6 +1206,10 @@ fn parse_args(args: &[String]) -> Result { args: join_optional_args(&rest[1..]), output_format, }), + "integrations" => Ok(CliAction::Integrations { + args: rest[1..].to_vec(), + output_format, + }), "trust" => match rest.get(1).map(String::as_str) { None => Ok(CliAction::Trust { revoke: false }), Some("--revoke") if rest.len() == 2 => Ok(CliAction::Trust { revoke: true }), @@ -3866,6 +3975,7 @@ fn run_hosted_task_worker( true, allowed_tools, permission_mode, + false, )?; match cli.run_prompt_json_value(&payload.prompt) { @@ -4871,8 +4981,16 @@ fn run_prompt_action( true, allowed_tools, permission_mode, + false, + )?, + None => LiveCli::new_with_provider( + model, + provider, + true, + allowed_tools, + permission_mode, + false, )?, - None => LiveCli::new_with_provider(model, provider, true, allowed_tools, permission_mode)?, }; if options.stream { cli.run_turn_jsonl_stream(prompt) @@ -5666,8 +5784,16 @@ fn run_repl_with_provider( provider: Option, allowed_tools: Option, permission_mode: PermissionMode, + legacy_input: bool, ) -> Result<(), Box> { - let cli = LiveCli::new_with_provider(model, provider, true, allowed_tools, permission_mode)?; + let cli = LiveCli::new_with_provider( + model, + provider, + true, + allowed_tools, + permission_mode, + legacy_input, + )?; drive_repl(cli) } @@ -5792,6 +5918,12 @@ struct LiveCli { /// interactive callers can build a desktop-notification body. Empty until /// a turn succeeds. last_turn_text: String, + /// When true, turns use the pre-streaming path (plain spinner, no live + /// region). Set from `--legacy-input`; the default streams. + legacy_input: bool, + /// Pure display state for the interactive REPL (message history, turn + /// status, permission mode). Fed with [`app::AppEvent`]s during a turn. + app: app::App, } struct RuntimePluginState { @@ -6367,6 +6499,30 @@ impl HookAbortMonitor { } } +/// `EventSink` that forwards runtime events to both the [`app::App`] +/// state machine and the terminal's live-region renderer. +struct AppEventSink<'a> { + app: &'a mut app::App, + sink: &'a mut streaming::TurnStreamSink, +} + +impl code_runtime::EventSink for AppEventSink<'_> { + fn on_event(&mut self, event: &code_runtime::AssistantEvent) -> bool { + use code_runtime::AssistantEvent as AE; + let app_event: Option = match event { + AE::TextDelta(delta) => Some(app::AppEvent::TextDelta(delta.clone())), + AE::ToolUse { name, .. } => Some(app::AppEvent::ToolStarted { name: name.clone() }), + AE::Usage(usage) => Some(app::AppEvent::Usage(*usage)), + AE::PromptCache(_) => None, + AE::MessageStop => Some(app::AppEvent::MessageStop), + }; + if let Some(ae) = app_event { + self.app.handle(&ae); + } + self.sink.on_event(event) + } +} + impl LiveCli { fn new( model: String, @@ -6374,7 +6530,14 @@ impl LiveCli { allowed_tools: Option, permission_mode: PermissionMode, ) -> Result> { - Self::new_with_provider(model, None, enable_tools, allowed_tools, permission_mode) + Self::new_with_provider( + model, + None, + enable_tools, + allowed_tools, + permission_mode, + false, + ) } fn new_with_provider( @@ -6383,6 +6546,7 @@ impl LiveCli { enable_tools: bool, allowed_tools: Option, permission_mode: PermissionMode, + legacy_input: bool, ) -> Result> { let system_prompt = build_system_prompt()?; let (effective_model, effective_provider) = @@ -6403,6 +6567,7 @@ impl LiveCli { enable_tools, allowed_tools, permission_mode, + legacy_input, ) } @@ -6415,6 +6580,7 @@ impl LiveCli { enable_tools: bool, allowed_tools: Option, permission_mode: PermissionMode, + legacy_input: bool, ) -> Result> { let system_prompt = build_system_prompt()?; let (effective_model, effective_provider) = @@ -6430,6 +6596,7 @@ impl LiveCli { enable_tools, allowed_tools, permission_mode, + legacy_input, ) } @@ -6443,6 +6610,7 @@ impl LiveCli { enable_tools: bool, allowed_tools: Option, permission_mode: PermissionMode, + legacy_input: bool, ) -> Result> { let runtime = build_runtime_with_provider( session_state.with_persistence_path(session.path.clone()), @@ -6457,6 +6625,7 @@ impl LiveCli { None, )?; let cli = Self { + app: app::App::new(model.clone(), permission_mode.as_str().to_string()), model, provider, allowed_tools, @@ -6465,6 +6634,7 @@ impl LiveCli { runtime, session, last_turn_text: String::new(), + legacy_input, }; cli.persist_session()?; Ok(cli) @@ -6597,6 +6767,64 @@ impl LiveCli { } fn run_turn(&mut self, input: &str) -> Result<(), Box> { + if self.legacy_input { + return self.run_turn_legacy(input); + } + let (mut runtime, hook_abort_monitor) = self.prepare_turn_runtime(true)?; + // Owns the terminal's "live region" for the duration of the turn: it + // renders tool lines, streamed text, and the idle thinking animation, + // and clears the region before returning. Safe to create unconditionally — + // a no-op when stdout is not a terminal (piped output). + let mut stream_sink = streaming::TurnStreamSink::new(); + let started = Instant::now(); + let mut permission_prompter = CliPermissionPrompter::new(self.permission_mode); + // Pure state machine: fed with events during the turn so the message + // history and turn status are available for rendering and inspection. + self.app.handle(&app::AppEvent::TurnStarted); + let result = runtime.run_turn_stream( + input, + Some(&mut permission_prompter), + &mut AppEventSink { + app: &mut self.app, + sink: &mut stream_sink, + }, + ); + stream_sink.finish(); + hook_abort_monitor.stop(); + match result { + Ok(summary) => { + let tool_calls = collect_tool_uses(&summary).len(); + self.app + .handle(&app::AppEvent::TurnCompleted(summary.clone())); + self.app.stamp_turn_elapsed(started.elapsed().as_secs_f64()); + self.replace_runtime(runtime)?; + println!(); + println!( + "{}", + format_turn_summary(&summary, &self.model, started.elapsed(), tool_calls) + ); + if let Some(event) = summary.auto_compaction { + println!( + "{}", + format_auto_compaction_notice(event.removed_message_count) + ); + } + self.persist_session()?; + self.last_turn_text = final_assistant_text(&summary); + Ok(()) + } + Err(error) => { + runtime.shutdown_plugins()?; + self.app + .handle(&app::AppEvent::TurnFailed(error.to_string())); + Err(Box::new(error)) + } + } + } + + /// Pre-streaming turn path used by `--legacy-input`: the old thinking + /// spinner and non-streaming `run_turn`. + fn run_turn_legacy(&mut self, input: &str) -> Result<(), Box> { let (mut runtime, hook_abort_monitor) = self.prepare_turn_runtime(true)?; let mut spinner = Spinner::new(); let mut stdout = io::stdout(); @@ -12037,6 +12265,7 @@ mod tests { provider: None, allowed_tools: None, permission_mode: crate::default_permission_mode(), + legacy_input: false, } ); } @@ -12383,6 +12612,7 @@ mod tests { provider: None, allowed_tools: None, permission_mode: PermissionMode::ReadOnly, + legacy_input: false, } ); } @@ -12412,6 +12642,7 @@ mod tests { .collect() ), permission_mode: crate::default_permission_mode(), + legacy_input: false, } ); } @@ -15751,7 +15982,7 @@ mod teleport_search_tests { #[cfg(test)] mod sandbox_report_tests { - use super::{format_sandbox_report, HookAbortMonitor}; + use super::{format_sandbox_report, parse_args, CliAction, HookAbortMonitor}; use code_runtime::HookAbortSignal; use std::sync::mpsc; use std::time::Duration; @@ -15803,4 +16034,25 @@ mod sandbox_report_tests { assert!(abort_signal.is_aborted()); } + + #[test] + fn parse_integrations_validate_preserves_path_with_spaces() { + let path = "/Users/Me/My Documents/integration manifest.json"; + let action = parse_args(&[ + "integrations".to_string(), + "validate".to_string(), + path.to_string(), + ]) + .expect("parse integrations validate"); + match action { + CliAction::Integrations { args, .. } => { + assert_eq!( + args, + vec!["validate".to_string(), path.to_string()], + "path with spaces must remain a single argument" + ); + } + other => panic!("expected Integrations, got {other:?}"), + } + } } diff --git a/cli/src/streaming.rs b/cli/src/streaming.rs new file mode 100644 index 00000000..c8012654 --- /dev/null +++ b/cli/src/streaming.rs @@ -0,0 +1,459 @@ +//! Live rendering of streamed turn events into the terminal. +//! +//! Design: the turn thread forwards each [`StreamEvent`] into a channel; a +//! single dedicated render thread owns all writes to stdout for the "live +//! region". Nothing else (prompt, permission dialog, summary) writes to the +//! terminal while a turn is streaming, so a redraw can never be interleaved +//! with other output. When the turn ends the caller drops the sender; the +//! render thread draws the final frame and exits, leaving the cursor at the +//! start of the line after the region. +//! +//! What the live region shows, top to bottom: +//! - the transcript so far: each message's text (word-wrapped at the terminal +//! width) followed by the tool rows it requested (`⚙ tool 1.2s`), +//! - the assistant text streaming right now, still growing, +//! - or, while nothing has arrived yet, an animated thinking line. +//! +//! The region is redrawn in place (cursor up, clear line, rewrite). When it +//! grows beyond [`MAX_LINES`] the newest lines stay on screen and the older +//! content scrolls into the terminal's scrollback. + +use std::fmt::Write as _; +use std::io::{self, IsTerminal, Write}; +use std::sync::mpsc; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use code_runtime::EventSink; + +/// Events the render thread consumes. A lean copy of the subset of +/// [`code_runtime::AssistantEvent`] the live region renders, kept so the +/// channel payload is `Send + Clone` without importing the full runtime type. +#[derive(Clone)] +enum StreamEvent { + TextDelta(String), + ToolUse { name: String }, + MessageStop, +} + +impl StreamEvent { + fn from_assistant_event(event: &code_runtime::AssistantEvent) -> Option { + match event { + code_runtime::AssistantEvent::TextDelta(delta) => Some(Self::TextDelta(delta.clone())), + code_runtime::AssistantEvent::ToolUse { name, .. } => { + Some(Self::ToolUse { name: name.clone() }) + } + code_runtime::AssistantEvent::MessageStop => Some(Self::MessageStop), + code_runtime::AssistantEvent::Usage(_) + | code_runtime::AssistantEvent::PromptCache(_) => None, + } + } +} + +const FRAMES: [&str; 10] = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; +const FRAME_INTERVAL: Duration = Duration::from_millis(90); +const THINKING_COLOR: &str = "\x1b[38;5;33m"; +const TEXT_COLOR: &str = "\x1b[1m"; +const TOOL_COLOR: &str = "\x1b[38;5;33m"; +const DIM: &str = "\x1b[2m"; +/// Erases the current line, then moves the cursor to column 0. +const CLEAR_LINE: &str = "\x1b[2K\r"; +const RESET: &str = "\x1b[0m"; +/// Maximum height of the live region before it starts scrolling the terminal. +const MAX_LINES: usize = 12; +/// Fallback width used when the terminal size cannot be determined. +const FALLBACK_COLS: usize = 100; + +enum RenderCmd { + Event(StreamEvent), + Tick, +} + +struct LiveState { + /// Transcript of this turn, oldest first: assistant text and tool rows. + segments: Vec, + /// Text streamed into the current segment (the live, still-growing part). + pending_text: String, + /// When the current segment's tool (if any) started; the idle timer and + /// tool elapsed time are read from it. + seg_started: Instant, + frame: usize, +} + +impl LiveState { + fn new() -> Self { + Self { + segments: Vec::new(), + pending_text: String::new(), + seg_started: Instant::now(), + frame: 0, + } + } +} + +/// One immutable row of the transcript: finished assistant text or a tool. +#[derive(Debug)] +enum Segment { + Text(String), + Tool { name: String, started: Instant }, +} + +/// Renders streaming events for one turn in a live region above the prompt. +/// +/// Create one with [`TurnStreamSink::new`], hand it to +/// `ConversationRuntime::run_turn_stream`, then call [`finish`]. The sink is +/// safe to use as the `EventSink` directly: `on_event` only forwards to the +/// render thread and returns `true` (never aborting the turn). +/// +/// When stdout is not a terminal the sink is a no-op, so call sites can +/// always create one without checking. +pub struct TurnStreamSink { + tx: Option>, + state: Arc>, + enabled: bool, +} + +impl TurnStreamSink { + /// Starts live rendering for a turn. + #[must_use] + pub fn new() -> Self { + let enabled = io::stdout().is_terminal(); + let state = Arc::new(Mutex::new(LiveState::new())); + + let (tx, rx) = mpsc::channel(); + if enabled { + let render_state = Arc::clone(&state); + let cols = terminal_width().unwrap_or(FALLBACK_COLS); + std::thread::spawn(move || render_loop(rx, render_state, cols)); + } + + Self { + tx: if enabled { Some(tx) } else { None }, + state, + enabled, + } + } + + /// Ends the turn: signals the render thread to draw the final frame and + /// exit. Idempotent. + pub fn finish(&mut self) { + self.tx = None; + } +} + +impl Default for TurnStreamSink { + fn default() -> Self { + Self::new() + } +} + +impl Drop for TurnStreamSink { + fn drop(&mut self) { + self.finish(); + } +} + +impl EventSink for TurnStreamSink { + fn on_event(&mut self, event: &code_runtime::AssistantEvent) -> bool { + if let Some(tx) = &self.tx { + if let Some(cmd) = StreamEvent::from_assistant_event(event) { + let _ = tx.send(RenderCmd::Event(cmd)); + } + } + true + } +} + +fn render_loop(rx: mpsc::Receiver, state: Arc>, cols: usize) { + let mut prev_lines: usize = 0; + loop { + match rx.recv_timeout(FRAME_INTERVAL) { + Ok(RenderCmd::Event(event)) => { + if let Ok(mut s) = state.lock() { + apply(&mut s, event); + } + } + Ok(RenderCmd::Tick) => {} + Err(mpsc::RecvTimeoutError::Timeout) => { + let _ = state.lock().map(|mut s| s.frame += 1); + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = draw(&state, cols, prev_lines); + // Leave cursor at start of the line after the region. + let mut out = io::stdout(); + let _ = out.write_all(b"\r\n"); + let _ = out.flush(); + break; + } + } + prev_lines = draw(&state, cols, prev_lines).unwrap_or(0); + } +} + +fn apply(state: &mut LiveState, event: StreamEvent) { + match event { + StreamEvent::TextDelta(delta) => state.pending_text.push_str(&delta), + StreamEvent::ToolUse { name } => { + // A ToolUse event precedes the next assistant message; start a new + // text segment so tool calls stay visually attached to the text + // that produced them, and restart the idle timer so the thinking + // line reappears while the tool runs. + state.seg_started = Instant::now(); + // The text streamed so far was produced by the message that + // requested this tool; freeze it into the transcript first so the + // tool row sits directly beneath its context. + if state.pending_text.trim().is_empty() { + state.pending_text.clear(); + } else { + state + .segments + .push(Segment::Text(std::mem::take(&mut state.pending_text))); + } + state.segments.push(Segment::Tool { + name, + started: Instant::now(), + }); + } + StreamEvent::MessageStop => {} + } +} + +fn draw(state: &Arc>, cols: usize, prev_lines: usize) -> Option { + let Ok(s) = state.lock() else { + return None; + }; + let frame = render_frame(&s, cols); + let lines = frame.len(); + + let mut buf = String::new(); + if prev_lines > 0 { + let _ = write!(buf, "\x1b[{prev_lines}A"); + } + for line in &frame { + buf.push_str(CLEAR_LINE); + buf.push_str(line); + buf.push('\n'); + } + if lines > 0 { + let _ = write!(buf, "\x1b[{lines}A"); + } + + let mut out = io::stdout(); + let _ = out.write_all(buf.as_bytes()); + let _ = out.flush(); + Some(lines) +} + +fn render_frame(s: &LiveState, cols: usize) -> Vec { + let mut lines: Vec = Vec::new(); + for segment in &s.segments { + match segment { + Segment::Text(text) => { + for line in wrap(text, cols) { + lines.push(format!("{TEXT_COLOR}{line}{RESET}")); + } + } + Segment::Tool { name, started } => { + let secs = started.elapsed().as_secs_f64(); + lines.push(format!( + "{TOOL_COLOR}⚙ {name}{RESET} {DIM}{secs:.1}s{RESET}" + )); + } + } + } + if !s.pending_text.is_empty() { + let text = s.pending_text.trim_end_matches(['\r', '\n']); + for line in wrap(text, cols) { + lines.push(format!("{TEXT_COLOR}{line}{RESET}")); + } + } + // Idle animation: nothing growing right now (no pending text) and the + // current segment is not a running tool. + let tool_running = matches!(s.segments.last(), Some(Segment::Tool { .. })); + if s.pending_text.is_empty() && !tool_running { + let glyph = FRAMES[s.frame % FRAMES.len()]; + let secs = s.seg_started.elapsed().as_secs(); + lines.push(format!( + "{THINKING_COLOR}{glyph} Thinking... {secs}s{RESET}" + )); + } + // Keep the newest lines on screen; the rest scrolls into scrollback. + let keep = lines.len().saturating_sub(MAX_LINES); + lines.drain(0..keep); + lines +} + +fn wrap(text: &str, cols: usize) -> Vec { + let cols = cols.max(3); + let mut out = Vec::new(); + for raw in text.split('\n') { + let mut line = String::new(); + let mut len = 0usize; + for word in raw.split_inclusive(' ') { + let word = word.trim_end_matches(' '); + if word.is_empty() { + continue; + } + let wlen = word.chars().count(); + if wlen > cols { + // Current word alone exceeds the width: flush the current line, + // hard-break the word into chunks, and start fresh. + if len > 0 { + out.push(line.clone()); + line.clear(); + } + let mut rest = word.to_string(); + while rest.chars().count() > cols { + let chunk: String = rest.chars().take(cols).collect(); + out.push(chunk); + rest = rest.chars().skip(cols).collect(); + } + line = rest; + len = line.chars().count(); + continue; + } + if len > 0 && len + 1 + wlen >= cols { + // Current line is non-empty and the word (plus its separating + // space) would push it past the width: flush and start a new + // line with just this word. + out.push(line.clone()); + line.clear(); + len = 0; + } + if len > 0 { + line.push(' '); + len += 1; + } + line.push_str(word); + len += wlen; + } + if !line.is_empty() { + out.push(line); + } + } + out +} + +fn terminal_width() -> Option { + match crossterm::terminal::size() { + Ok((w, _)) if w > 0 => Some(w as usize), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wrap_keeps_short_lines() { + assert_eq!(wrap("hello world", 80), vec!["hello world"]); + } + + #[test] + fn wrap_breaks_long_lines() { + assert_eq!( + wrap("a b c d e f g h i j", 10), + vec!["a b c d e", "f g h i j"] + ); + } + + #[test] + fn wrap_respects_newlines() { + assert_eq!(wrap("one\ntwo", 80), vec!["one", "two"]); + } + + #[test] + fn wrap_hard_breaks_long_words() { + assert_eq!(wrap("aaaaaaaaaa bb", 5), vec!["aaaaa", "aaaaa", "bb"]); + } + + #[test] + fn frame_is_thinking_line_when_idle() { + let s = LiveState::new(); + let frame = render_frame(&s, 80); + assert_eq!(frame.len(), 1); + assert!(frame[0].contains("Thinking")); + } + + #[test] + fn frame_shows_running_tool_and_pending_text() { + let mut s = LiveState::new(); + s.segments.push(Segment::Tool { + name: "Read".into(), + started: Instant::now(), + }); + s.pending_text = "hello\nworld".into(); + let frame = render_frame(&s, 80); + assert_eq!(frame.len(), 3); + assert!(frame[0].contains("Read")); + assert!(frame[1].contains("hello")); + assert!(frame[2].contains("world")); + } + + #[test] + fn thinking_line_reappears_after_tool_finished() { + // A finished assistant segment (no running tool) is idle: the + // thinking line shows below it while the model plans the next step. + let mut s = LiveState::new(); + s.segments.push(Segment::Text("done".into())); + s.pending_text.clear(); + let frame = render_frame(&s, 80); + assert_eq!(frame.len(), 2); + assert!(frame[0].contains("done")); + assert!(frame[1].contains("Thinking")); + } + + #[test] + fn no_thinking_line_while_tool_runs() { + let mut s = LiveState::new(); + s.segments.push(Segment::Tool { + name: "Bash".into(), + started: Instant::now(), + }); + let frame = render_frame(&s, 80); + assert_eq!(frame.len(), 1); + assert!(frame[0].contains("Bash")); + } + + #[test] + fn frame_is_bounded_to_max_lines() { + let mut s = LiveState::new(); + s.pending_text = (0..100) + .map(|i| format!("line {i}")) + .collect::>() + .join("\n"); + let frame = render_frame(&s, 80); + assert_eq!(frame.len(), MAX_LINES); + assert!(frame.last().unwrap().contains("line 99")); + } + + #[test] + fn stream_event_maps_text_delta() { + let ev = code_runtime::AssistantEvent::TextDelta("hello".into()); + let mapped = StreamEvent::from_assistant_event(&ev); + assert!(matches!(mapped, Some(StreamEvent::TextDelta(ref s)) if s == "hello")); + } + + #[test] + fn stream_event_maps_tool_use() { + let ev = code_runtime::AssistantEvent::ToolUse { + id: "1".into(), + name: "Read".into(), + input: "{}".into(), + }; + let mapped = StreamEvent::from_assistant_event(&ev); + assert!(matches!(mapped, Some(StreamEvent::ToolUse { ref name }) if name == "Read")); + } + + #[test] + fn stream_event_drops_usage() { + let ev = code_runtime::AssistantEvent::Usage(code_runtime::TokenUsage { + input_tokens: 1, + output_tokens: 2, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }); + assert!(StreamEvent::from_assistant_event(&ev).is_none()); + } +} diff --git a/crates/code-events/src/lib.rs b/crates/code-events/src/lib.rs index 4ff57a4c..4ef15099 100644 --- a/crates/code-events/src/lib.rs +++ b/crates/code-events/src/lib.rs @@ -622,7 +622,7 @@ mod tests { published_branch: Some("frontal-code/fix-flake".into()), pr_url: Some("https://github.com/acme/payments/pull/42".into()), pr_number: Some(42), - execution_backend: Some("local_docker".into()), + execution_backend: Some("sandbox".into()), worker_id: Some("worker-1".into()), worker_status: Some("running".into()), orphan_policy: Some(AppliedOrphanPolicy { @@ -655,7 +655,7 @@ mod tests { "https://github.com/acme/payments/pull/42" ); assert_eq!(serialized["pr_number"], 42); - assert_eq!(serialized["execution_backend"], "local_docker"); + assert_eq!(serialized["execution_backend"], "sandbox"); assert_eq!(serialized["worker_id"], "worker-1"); assert_eq!(serialized["worker_status"], "running"); assert_eq!(serialized["orphan_policy"]["source"], "default"); diff --git a/crates/code-events/tests/event_payload_types.rs b/crates/code-events/tests/event_payload_types.rs index 51b4f7f5..67db1925 100644 --- a/crates/code-events/tests/event_payload_types.rs +++ b/crates/code-events/tests/event_payload_types.rs @@ -132,7 +132,7 @@ fn hosted_task_event_summary_full() { graphite_stack_id: Some("stack-1".into()), graphite_head_branch: Some("feature/head".into()), graphite_base_branch: Some("main".into()), - execution_backend: Some("local_docker".into()), + execution_backend: Some("sandbox".into()), priority: Some("high".into()), plan_id: Some("plan-1".into()), plan_kind: Some("feature".into()), diff --git a/crates/code-integrations/src/lib.rs b/crates/code-integrations/src/lib.rs index 39fce695..f59c5b7a 100644 --- a/crates/code-integrations/src/lib.rs +++ b/crates/code-integrations/src/lib.rs @@ -4,7 +4,9 @@ //! including Model Context Protocol (MCP) server management and tool bridging. pub mod ide; +pub mod manifest; pub mod mcp; +pub mod providers; // Re-export commonly used MCP types for convenience pub use mcp::{ @@ -17,6 +19,15 @@ pub use mcp::{ tools::{execute_mcp_tool, mcp_tool_specs, ToolSpec}, }; +// Re-export the integration manifest system (the typed plugin contract) +pub use manifest::{ + AuthSpec, CapabilitySpec, ContextFieldSpec, EventSpec, IntegrationManifest, ManifestRegistry, + SignatureSpec, TransportConfig, ValidationIssue, +}; + +// Re-export the built-in provider manifests (GitHub, Linear, PostHog, Sentry) +pub use providers::{builtin_providers, BUILTIN_PROVIDER_IDS}; + #[cfg(test)] mod tests { #[test] diff --git a/crates/code-integrations/src/manifest.rs b/crates/code-integrations/src/manifest.rs new file mode 100644 index 00000000..9099d26d --- /dev/null +++ b/crates/code-integrations/src/manifest.rs @@ -0,0 +1,535 @@ +//! Typed, declarative integration manifests. +//! +//! An [`IntegrationManifest`] is the plugin contract for `FrontalCode` integrations: +//! the server, CLI, and SDKs all consume this shape. Manifests declare how to reach +//! an integration (transport + transport config), the capabilities (tools/endpoints) +//! it exposes, its auth requirements, the webhook events it can emit, and the fields +//! it writes into task context. + +use crate::mcp::config::McpTransport; +use crate::mcp::tools::PermissionMode; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// A fully typed, declarative definition of an integration. +/// This is the plugin contract: the server, CLI, and SDKs all consume this shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IntegrationManifest { + /// Unique identifier (e.g., "github", "linear", "my-custom-api") + pub id: String, + /// Semver version of this manifest + pub version: String, + pub display_name: String, + pub description: String, + /// How to reach the integration. Reuses the existing MCP transport enum. + pub transport: McpTransport, + /// Transport-specific config (command/args/env for stdio, url for http, etc.) + pub transport_config: TransportConfig, + /// The capabilities (tools/endpoints) this integration exposes + pub capabilities: Vec, + /// Auth requirements, if any + pub auth: Option, + /// Webhook events this integration can emit (for webhook-capable integrations) + pub events: Vec, + /// Fields this integration writes into task context (prefix-based) + pub context_fields: Vec, +} + +/// Transport-specific configuration. Tagged enum. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum TransportConfig { + Stdio { + command: String, + args: Vec, + env: BTreeMap, + }, + Http { + url: String, + headers: BTreeMap, + }, + Sse { + url: String, + headers: BTreeMap, + }, + WebSocket { + url: String, + }, + Sdk { + module: String, + }, + ManagedProxy { + proxy_url: String, + target: String, + }, +} + +/// A single capability (tool/endpoint) exposed by an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CapabilitySpec { + pub name: String, + pub description: String, + /// JSON Schema for input validation (industry standard, matches MCP) + pub input_schema: serde_json::Value, + /// JSON Schema for output validation + pub output_schema: Option, + pub permission_mode: PermissionMode, + /// If true, safe to retry on failure + pub idempotent: bool, +} + +/// Auth specification for an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AuthSpec { + None, + OAuth { + client_id_env: String, + client_secret_env: Option, + auth_url: String, + token_url: String, + scopes: Vec, + redirect_path: Option, + }, + ApiKey { + env_var: String, + header: Option, + }, +} + +/// A webhook event this integration can emit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EventSpec { + pub name: String, + pub description: String, + /// JSON Schema for the event payload + pub payload_schema: serde_json::Value, + /// Signature verification method + pub signature: Option, +} + +/// Signature verification method for webhook events. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SignatureSpec { + HmacSha256 { secret_env: String }, + None, +} + +/// A field this integration writes into task context. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContextFieldSpec { + pub name: String, + pub description: String, + pub schema: serde_json::Value, +} + +/// A single validation issue found while checking a manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidationIssue { + /// Dot-separated path to the offending field (e.g., "capabilities[2].name"). + pub field: String, + /// Human-readable description of the problem. + pub message: String, +} + +impl IntegrationManifest { + /// Validate this manifest, returning any issues found. + /// + /// Returns `Ok(())` when the manifest is well-formed. + pub fn validate(&self) -> Result<(), Vec> { + let mut issues: Vec = Vec::new(); + self.validate_id(&mut issues); + self.validate_version(&mut issues); + self.validate_capabilities(&mut issues); + self.validate_auth(&mut issues); + self.validate_events(&mut issues); + if issues.is_empty() { + Ok(()) + } else { + Err(issues) + } + } + + fn validate_id(&self, issues: &mut Vec) { + // `id`: non-empty, lowercase alphanumeric + hyphens, max 64 chars. + if self.id.is_empty() { + issues.push(ValidationIssue { + field: "id".to_string(), + message: "id must be non-empty".to_string(), + }); + } else if self.id.len() > 64 { + issues.push(ValidationIssue { + field: "id".to_string(), + message: format!("id must be at most 64 characters, got {}", self.id.len()), + }); + } else if !id_re().is_match(&self.id) { + issues.push(ValidationIssue { + field: "id".to_string(), + message: format!( + "id `{}` must be lowercase alphanumeric with hyphens only", + self.id + ), + }); + } + } + + fn validate_version(&self, issues: &mut Vec) { + // `version`: must parse as semver. + if !semver_re().is_match(&self.version) { + issues.push(ValidationIssue { + field: "version".to_string(), + message: format!( + "version `{}` is not valid semver (expected e.g. `1.2.3`)", + self.version + ), + }); + } + } + + fn validate_capabilities(&self, issues: &mut Vec) { + // `capabilities`: must be non-empty, names unique, schemas valid JSON. + if self.capabilities.is_empty() { + issues.push(ValidationIssue { + field: "capabilities".to_string(), + message: "capabilities must be non-empty".to_string(), + }); + } + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for (index, capability) in self.capabilities.iter().enumerate() { + let field = format!("capabilities[{index}]"); + if !seen.insert(&capability.name) { + issues.push(ValidationIssue { + field: format!("{field}.name"), + message: format!("duplicate capability name `{}`", capability.name), + }); + } + if capability.input_schema.is_null() { + issues.push(ValidationIssue { + field: format!("{field}.input_schema"), + message: "input_schema must be valid JSON".to_string(), + }); + } + if let Some(output_schema) = &capability.output_schema { + if output_schema.is_null() { + issues.push(ValidationIssue { + field: format!("{field}.output_schema"), + message: "output_schema must be valid JSON".to_string(), + }); + } + } + } + } + + fn validate_auth(&self, issues: &mut Vec) { + // `auth`: env var names must be non-empty strings. + match &self.auth { + Some(AuthSpec::OAuth { + client_id_env, + client_secret_env, + .. + }) => { + if client_id_env.is_empty() { + issues.push(ValidationIssue { + field: "auth.client_id_env".to_string(), + message: "auth.client_id_env must be a non-empty string".to_string(), + }); + } + if let Some(secret) = client_secret_env { + if secret.is_empty() { + issues.push(ValidationIssue { + field: "auth.client_secret_env".to_string(), + message: "auth.client_secret_env must be a non-empty string" + .to_string(), + }); + } + } + } + Some(AuthSpec::ApiKey { env_var, .. }) => { + if env_var.is_empty() { + issues.push(ValidationIssue { + field: "auth.env_var".to_string(), + message: "auth.env_var must be a non-empty string".to_string(), + }); + } + } + Some(AuthSpec::None) | None => {} + } + } + + fn validate_events(&self, issues: &mut Vec) { + // `events`: names must be unique. + let mut seen_events: BTreeSet<&str> = BTreeSet::new(); + for (index, event) in self.events.iter().enumerate() { + let field = format!("events[{index}]"); + if !seen_events.insert(&event.name) { + issues.push(ValidationIssue { + field: format!("{field}.name"), + message: format!("duplicate event name `{}`", event.name), + }); + } + } + } +} + +/// Registry of integration manifests, keyed by manifest id. +#[derive(Debug, Clone, Default)] +pub struct ManifestRegistry { + manifests: BTreeMap, +} + +impl ManifestRegistry { + /// Create an empty registry. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Register a manifest, rejecting it if it fails validation + /// or if a manifest with the same id is already registered. + pub fn register(&mut self, manifest: IntegrationManifest) -> Result<(), Vec> { + manifest.validate()?; + if self.manifests.contains_key(&manifest.id) { + return Err(vec![ValidationIssue { + field: "id".to_string(), + message: format!("manifest with id `{}` is already registered", manifest.id), + }]); + } + self.manifests.insert(manifest.id.clone(), manifest); + Ok(()) + } + + /// Remove a manifest by id, returning it if it was present. + pub fn remove(&mut self, id: &str) -> Option { + self.manifests.remove(id) + } + + /// Look up a manifest by id. + #[must_use] + pub fn get(&self, id: &str) -> Option<&IntegrationManifest> { + self.manifests.get(id) + } + + /// List all registered manifests in id order. + #[must_use] + pub fn list(&self) -> Vec<&IntegrationManifest> { + self.manifests.values().collect() + } + + /// Look up a single capability on a registered manifest. + #[must_use] + pub fn capability(&self, integration_id: &str, capability: &str) -> Option<&CapabilitySpec> { + self.manifests + .get(integration_id)? + .capabilities + .iter() + .find(|spec| spec.name == capability) + } +} + +static ID_RE: std::sync::OnceLock = std::sync::OnceLock::new(); +static SEMVER_RE: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn id_re() -> &'static Regex { + ID_RE.get_or_init(|| Regex::new("^[a-z0-9]+(-[a-z0-9]+)*$").expect("static regex is valid")) +} + +fn semver_re() -> &'static Regex { + SEMVER_RE.get_or_init(|| { + Regex::new( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", + ) + .expect("static regex is valid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: BTreeMap::new(), + }, + capabilities: vec![CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: serde_json::json!({"type": "object"}), + output_schema: Some(serde_json::json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".to_string(), + header: None, + }), + events: vec![EventSpec { + name: "issue.opened".to_string(), + description: "An issue was opened".to_string(), + payload_schema: serde_json::json!({"type": "object"}), + signature: None, + }], + context_fields: vec![ContextFieldSpec { + name: "github.issue".to_string(), + description: "The linked GitHub issue".to_string(), + schema: serde_json::json!({"type": "object"}), + }], + } + } + + #[test] + fn valid_manifest_roundtrips_through_serde_json() { + let manifest = valid_manifest(); + let json = serde_json::to_string(&manifest).expect("serialize"); + let parsed: IntegrationManifest = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed.id, manifest.id); + assert_eq!(parsed.version, manifest.version); + assert_eq!(parsed.capabilities.len(), manifest.capabilities.len()); + assert!(matches!(parsed.transport, McpTransport::Stdio)); + assert!(matches!( + parsed.transport_config, + TransportConfig::Stdio { .. } + )); + assert!(matches!(parsed.auth, Some(AuthSpec::ApiKey { .. }))); + assert_eq!(parsed.events.len(), manifest.events.len()); + assert_eq!(parsed.context_fields.len(), manifest.context_fields.len()); + assert!(parsed.validate().is_ok()); + } + + #[test] + fn validation_catches_duplicate_capability_names() { + let mut manifest = valid_manifest(); + manifest.capabilities.push(CapabilitySpec { + name: "create_issue".to_string(), + description: "Duplicate".to_string(), + input_schema: serde_json::json!({"type": "object"}), + output_schema: None, + permission_mode: PermissionMode::ReadOnly, + idempotent: true, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "capabilities[1].name" + && issue.message.contains("duplicate capability name")), + "expected duplicate capability issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_invalid_id_format() { + let mut manifest = valid_manifest(); + manifest.id = "GitHub_1".to_string(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "id" && issue.message.contains("lowercase")), + "expected id format issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_invalid_version() { + let mut manifest = valid_manifest(); + manifest.version = "not-a-version".to_string(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "version" && issue.message.contains("semver")), + "expected semver issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_empty_capabilities() { + let mut manifest = valid_manifest(); + manifest.capabilities.clear(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "capabilities" && issue.message.contains("non-empty")), + "expected empty capabilities issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_duplicate_event_names() { + let mut manifest = valid_manifest(); + manifest.events.push(EventSpec { + name: "issue.opened".to_string(), + description: "Duplicate".to_string(), + payload_schema: serde_json::json!({"type": "object"}), + signature: None, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues.iter().any(|issue| issue.field == "events[1].name" + && issue.message.contains("duplicate event name")), + "expected duplicate event issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_empty_auth_env_vars() { + let mut manifest = valid_manifest(); + manifest.auth = Some(AuthSpec::ApiKey { + env_var: String::new(), + header: None, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "auth.env_var" && issue.message.contains("non-empty")), + "expected empty env var issue, got {issues:?}" + ); + } + + #[test] + fn registry_register_get_list_capability_work() { + let mut registry = ManifestRegistry::new(); + let manifest = valid_manifest(); + registry.register(manifest.clone()).expect("register"); + + assert!(registry.get("github").is_some()); + assert!(registry.get("missing").is_none()); + assert_eq!(registry.list(), [&manifest]); + assert!(registry.capability("github", "create_issue").is_some()); + assert!(registry.capability("github", "missing").is_none()); + assert!(registry.capability("missing", "create_issue").is_none()); + + // Registering the same id again is rejected. + let err = registry + .register(manifest) + .expect_err("duplicate id must fail"); + assert_eq!(err.len(), 1); + assert_eq!(err[0].field, "id"); + } + + #[test] + fn registry_rejects_invalid_manifest() { + let mut registry = ManifestRegistry::new(); + let mut manifest = valid_manifest(); + manifest.id = "Bad Id!".to_string(); + let err = registry + .register(manifest) + .expect_err("invalid manifest must fail"); + assert!( + err.iter().any(|issue| issue.field == "id"), + "expected id issue, got {err:?}" + ); + } +} diff --git a/crates/code-integrations/src/mcp/tools.rs b/crates/code-integrations/src/mcp/tools.rs index 910f3249..5d71105b 100644 --- a/crates/code-integrations/src/mcp/tools.rs +++ b/crates/code-integrations/src/mcp/tools.rs @@ -9,7 +9,8 @@ use serde_json::{from_value, json, Value as JsonValue}; use std::sync::OnceLock; /// Permission mode for tools -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum PermissionMode { ReadOnly, WorkspaceWrite, diff --git a/crates/code-integrations/src/providers.rs b/crates/code-integrations/src/providers.rs new file mode 100644 index 00000000..e439ebb1 --- /dev/null +++ b/crates/code-integrations/src/providers.rs @@ -0,0 +1,417 @@ +//! Built-in provider manifests. +//! +//! These are the four first-party integrations (GitHub, Linear, `PostHog`, +//! and Sentry) expressed as config-driven [`IntegrationManifest`] values instead +//! of hardcoded provider code. They are the defaults the server and CLI load +//! when no project-level overrides are configured. +//! +//! The legacy `IntegrationRegistry::call_*` methods (see +//! [`crate::mcp::integration`]) remain as deprecated adapters over the +//! corresponding capabilities below for one release. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::manifest::{ + AuthSpec, CapabilitySpec, IntegrationManifest, ManifestRegistry, TransportConfig, +}; +use crate::mcp::config::McpTransport; +use crate::mcp::tools::PermissionMode; + +/// The four built-in provider integration IDs, in stable (sorted) order. +pub const BUILTIN_PROVIDER_IDS: [&str; 4] = ["github", "linear", "posthog", "sentry"]; + +/// Returns the four built-in provider manifests, ready to register. +/// +/// Each manifest is validated before being returned; a built-in manifest that +/// fails validation is a bug, so this returns a `Result` that callers (and +/// tests) can assert on. +pub fn builtin_providers() -> Result> { + let mut registry = ManifestRegistry::new(); + let mut failures = Vec::new(); + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + if let Err(issues) = registry.register(manifest) { + failures.extend(issues.into_iter().map(|issue| issue.message)); + } + } + if failures.is_empty() { + Ok(registry) + } else { + Err(failures) + } +} + +/// The GitHub integration: PR creation, issue comments, and check runs against +/// the GitHub REST/MCP surface. +fn github_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".into(), + version: "1.0.0".into(), + display_name: "GitHub".into(), + description: "Create pull requests, comment on issues, and report check runs.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://api.github.com".into(), + headers: header_map(&[("Authorization", "${GITHUB_TOKEN}")]), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_pr".into(), + description: "Open a pull request on a repository.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "title": { "type": "string" }, + "body": { "type": "string" }, + "head": { "type": "string" }, + "base": { "type": "string" } + }, + "required": ["repo", "title", "head", "base"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "number": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to an issue or pull request.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "issue_number": { "type": "integer" }, + "body": { "type": "string" } + }, + "required": ["repo", "issue_number", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_check_run".into(), + description: "Create a check run for a commit.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "name": { "type": "string" }, + "head_sha": { "type": "string" }, + "status": { "type": "string" }, + "conclusion": { "type": "string" } + }, + "required": ["repo", "name", "head_sha"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The Linear integration: issue comments used by the tracker reporter. +fn linear_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "linear".into(), + version: "1.0.0".into(), + display_name: "Linear".into(), + description: "Comment on Linear issues for tracker reporting.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://api.linear.app/graphql".into(), + headers: header_map(&[("Authorization", "${LINEAR_API_KEY}")]), + }, + capabilities: vec![CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to a Linear issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["issue_id", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "LINEAR_API_KEY".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The `PostHog` integration: capture product and telemetry events. +fn posthog_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "posthog".into(), + version: "1.0.0".into(), + display_name: "PostHog".into(), + description: "Capture analytics events in PostHog.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://us.i.posthog.com/capture/".into(), + headers: BTreeMap::new(), + }, + capabilities: vec![ + CapabilitySpec { + name: "capture_event".into(), + description: "Capture a single analytics event.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "event": { "type": "string" }, + "distinct_id": { "type": "string" }, + "properties": { "type": "object" } + }, + "required": ["event", "distinct_id"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "status": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: true, + }, + CapabilitySpec { + name: "capture_events_batch".into(), + description: "Capture a batch of analytics events.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "batch": { + "type": "array", + "items": { + "type": "object", + "properties": { + "event": { "type": "string" }, + "distinct_id": { "type": "string" }, + "properties": { "type": "object" } + }, + "required": ["event", "distinct_id"] + } + } + }, + "required": ["batch"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "status": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: true, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "POSTHOG_API_KEY".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The Sentry integration: create issues and comments for error reporting. +fn sentry_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "sentry".into(), + version: "1.0.0".into(), + display_name: "Sentry".into(), + description: "Create Sentry issues and comments for error reporting.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://sentry.io/api/0".into(), + headers: header_map(&[("Authorization", "Token ${SENTRY_AUTH_TOKEN}")]), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_issue".into(), + description: "Create a new Sentry issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "project": { "type": "string" }, + "title": { "type": "string" }, + "message": { "type": "string" } + }, + "required": ["project", "title", "message"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to a Sentry issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["issue_id", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "SENTRY_AUTH_TOKEN".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +fn header_map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtin_providers_all_validate() { + let registry = builtin_providers().expect("all built-in manifests must validate"); + assert_eq!(registry.list().len(), 4); + for id in BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing built-in `{id}`"); + } + } + + #[test] + fn builtin_provider_ids_are_sorted_and_unique() { + let ids: Vec<&str> = BUILTIN_PROVIDER_IDS.to_vec(); + let mut sorted = ids.clone(); + sorted.sort_unstable(); + assert_eq!(ids, sorted, "BUILTIN_PROVIDER_IDS must be sorted"); + assert_eq!( + ids.len(), + ids.iter().collect::>().len(), + "BUILTIN_PROVIDER_IDS must be unique" + ); + } + + #[test] + fn each_builtin_manifest_roundtrips_through_serde_json() { + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + let original_id = manifest.id.clone(); + let json = serde_json::to_string(&manifest).expect("serialize"); + let parsed: IntegrationManifest = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed, manifest, "roundtrip mismatch for {original_id}"); + assert!(parsed.validate().is_ok(), "{original_id} must validate"); + } + } + + #[test] + fn github_manifest_exposes_expected_capabilities() { + let manifest = github_manifest(); + let names: Vec<&str> = manifest + .capabilities + .iter() + .map(|c| c.name.as_str()) + .collect(); + assert_eq!( + names, + vec!["create_pr", "create_issue_comment", "create_check_run"] + ); + } + + #[test] + fn linear_manifest_targets_issue_comments() { + let manifest = linear_manifest(); + let capability = manifest + .capabilities + .iter() + .find(|c| c.name == "create_issue_comment") + .expect("linear must expose create_issue_comment"); + assert_eq!(capability.permission_mode, PermissionMode::WorkspaceWrite); + assert!(!capability.idempotent); + } + + #[test] + fn posthog_capabilities_are_idempotent_writes() { + let manifest = posthog_manifest(); + assert!( + !manifest.capabilities.is_empty(), + "posthog must expose at least one capability" + ); + for capability in &manifest.capabilities { + assert!( + capability.idempotent, + "posthog capability `{}` must be idempotent", + capability.name + ); + assert_eq!(capability.permission_mode, PermissionMode::WorkspaceWrite); + } + } + + #[test] + fn all_builtins_use_http_transport_with_api_key_auth() { + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + assert!( + matches!(manifest.transport, McpTransport::Http), + "{} must use http transport", + manifest.id + ); + match manifest.auth { + Some(AuthSpec::ApiKey { env_var, .. }) => { + assert!(!env_var.is_empty(), "{} env_var must be set", manifest.id); + } + other => panic!("{} must use api_key auth, got {other:?}", manifest.id), + } + } + } +} diff --git a/crates/code-server/Cargo.toml b/crates/code-server/Cargo.toml index 8acd5475..f2984a03 100644 --- a/crates/code-server/Cargo.toml +++ b/crates/code-server/Cargo.toml @@ -11,6 +11,8 @@ name = "code-server" [dependencies] axum = { version = "0.8", features = ["ws"] } futures-util = "0.3" +tokio-stream = "0.1" +tracing = "0.1" code-events = { path = "../code-events" } code-integrations = { path = "../code-integrations" } code-orchestrator = { path = "../code-orchestrator" } @@ -31,6 +33,8 @@ tokio-postgres-rustls = "0.13" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-pki-types = "1" webpki-roots = "0.26" +frontal-sandbox-sdk.workspace = true +sandbox-types.workspace = true [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/crates/code-server/README.md b/crates/code-server/README.md index 28ce3198..0df35b0d 100644 --- a/crates/code-server/README.md +++ b/crates/code-server/README.md @@ -91,29 +91,32 @@ curl "http://127.0.0.1:8788/v1/policies/orphans?repository=repo-fast-policy&sour curl "http://127.0.0.1:8788/v1/policies/orphans?repository=repo-prod&priority=high" ``` -## Local Docker Worker Mode +## Sandbox Worker Mode -`FRONTAL_CODE_SERVER_LANE_TRANSPORT` selects how lanes run. The default is `local-docker`. +`FRONTAL_CODE_SERVER_LANE_TRANSPORT` selects how lanes run. The default is `sandbox`. -- `local-docker` (default): each task runs in its own worker container, built from `docker/worker.Dockerfile`. The container runs `frontal-code hosted task run`. Provider API keys and the server's control-plane key never enter the container. The worker gets a per-task token and a model proxy URL, and the server adds the real provider key when it forwards each request. See `docs/CONTAINERS.md`. -- `tools-agent`: runs the agent loop inside the server process. Agents have the server's host privileges, so use this only for trusted tasks. -- There is no simulated transport. Unknown values fail at startup. +- `sandbox` (default): each task runs in its own Frontal Sandbox session. The session's VFS holds the checkout under `/workspace`. Provider API keys and the server's control-plane key never enter the session. The worker gets a per-task token and a model proxy URL, and the server adds the real provider key when it forwards each request. See `docs/CONTAINERS.md` and `docs/design/worker-architecture.md`. +- `tools-agent`: runs the agent loop inside the server process. Agents have the server's host privileges, so use this only for trusted tasks. Gated behind `FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1`. +- Unknown values fail at startup. -For `local-docker`, set: +For `sandbox`, set: -- `FRONTAL_CODE_SERVER_LANE_TRANSPORT=local-docker` +- `FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox` - `FRONTAL_CODE_SERVER_WORKSPACE_ROOT=/path/to/project/.frontal-code/server/workspaces` -- `FRONTAL_CODE_SERVER_DOCKER_IMAGE=frontal-code-worker:local` -- `FRONTAL_CODE_SERVER_CALLBACK_URL=http://host.docker.internal:8788` +- `FRONTAL_SANDBOX_BASE_URL=http://localhost:8080` — the sandbox server the server talks to +- `FRONTAL_SANDBOX_API_KEY=sbx_…` — the Bearer key for the sandbox server +- `FRONTAL_CODE_SERVER_URL=http://:8788` — the address sessions use to call back +- `FRONTAL_CODE_SERVER_DATABASE_URL=postgres://…` — required (worker tokens are stored in Postgres) In that mode, the control plane: -1. prepares an isolated repo checkout under `FRONTAL_CODE_SERVER_WORKSPACE_ROOT` -2. writes `.frontal-code-hosted/task.json` into the checkout -3. launches `frontal-code hosted task run TASK_ID` inside the worker image -4. expects the worker container to call `POST /v1/tasks/:task_id/complete` back to the server +1. creates an isolated sandbox session with a network policy that allowlists the server host +2. streams the repo checkout into `/workspace` in the session VFS (skipping `.git`) +3. writes `.frontal-code-hosted/task.json` and `.frontal-code-hosted/env` into the VFS +4. launches `frontal-code hosted task run TASK_ID` in the session +5. expects the worker to call `POST /v1/tasks/:task_id/complete` back to the server -For the local hosted stack and worker-image build flow, see `infrastructure/README.md`. +For the sandbox terraform deployment, see `deploy/sandbox/README.md`. ## Development diff --git a/crates/code-server/src/docker_worker.rs b/crates/code-server/src/docker_worker.rs deleted file mode 100644 index eb6f9ed0..00000000 --- a/crates/code-server/src/docker_worker.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Docker commands for worker containers, as data. -//! -//! The builders here return argument lists, so the security settings can be checked by -//! tests without a Docker daemon. `CliDockerRunner` in `lib.rs` runs them. -//! -//! Isolation model: -//! -//! - Each task gets its own internal network. An internal network has no route to the -//! internet and no route to the host. -//! - A relay container is attached to every task network under the alias -//! `frontal-relay`. It forwards one TCP port to the server and nothing else. It is -//! the only way out for a worker, and it cannot be used as a general proxy. -//! - Model traffic goes through the server's provider proxy, so the worker does not -//! need the internet for model calls. -//! - The container runs with no Linux capabilities, no privilege escalation, a read-only -//! root filesystem, bounded memory, CPU, and process count, and writable space only in -//! the task checkout and two tmpfs mounts. - -/// Container name of the relay that serves `port`. One relay per server port, so two -/// servers with different ports on one Docker host do not replace each other's relay. -pub(crate) fn relay_container_name(port: u16) -> String { - format!("frontal-code-relay-{port}") -} - -/// Network alias the relay answers to inside a task network. Workers use this host name. -pub(crate) const RELAY_ALIAS: &str = "frontal-relay"; - -/// Label that marks task networks, so idle ones can be found and removed. -pub(crate) const TASK_NETWORK_LABEL: &str = "frontal-code.task_network"; - -/// Label on a task network that records the server port that created it. A sweep only -/// touches networks of its own server. -pub(crate) const NETWORK_RELAY_PORT_LABEL: &str = "frontal-code.relay_port"; - -/// Label on a task network with its creation time in Unix seconds. A sweep leaves young -/// networks alone, because a launch may be between creating the network and starting its -/// worker. -pub(crate) const NETWORK_CREATED_LABEL: &str = "frontal-code.created_at"; - -/// Label on worker containers that names their network, so `stop` can clean it up. -pub(crate) const TASK_NETWORK_CONTAINER_LABEL: &str = "frontal-code.task_network"; - -/// Label on the relay that records the port it forwards, so a changed port recreates it. -pub(crate) const RELAY_PORT_LABEL: &str = "frontal-code.relay_port"; - -/// Writable tmpfs for the worker's temporary files. -const TMP_TMPFS: &str = "/tmp:rw,nosuid,nodev,size=512m"; - -/// Writable tmpfs for the worker user's home directory. -const HOME_TMPFS: &str = "/home/frontal-code:rw,nosuid,nodev,size=1g"; - -/// Name of a task's internal network. Task ids are sanitized to Docker's name rules. -pub(crate) fn task_network_name(task_id: &str) -> String { - let safe: String = task_id - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || c == '-' || c == '_' { - c.to_ascii_lowercase() - } else { - '-' - } - }) - .take(48) - .collect(); - format!("frontal-task-{safe}") -} - -/// Arguments for starting the relay. It forwards `port` on the relay to the same port on -/// the host, where the server listens. -pub(crate) fn relay_run_args(image: &str, port: u16) -> Vec { - vec![ - "run".to_string(), - "-d".to_string(), - "--name".to_string(), - relay_container_name(port), - "--restart".to_string(), - "unless-stopped".to_string(), - "--add-host".to_string(), - "host.docker.internal:host-gateway".to_string(), - "--cap-drop".to_string(), - "ALL".to_string(), - "--security-opt".to_string(), - "no-new-privileges:true".to_string(), - "--read-only".to_string(), - "--pids-limit".to_string(), - "64".to_string(), - "--memory".to_string(), - "128m".to_string(), - "--label".to_string(), - "frontal-code.role=relay".to_string(), - "--label".to_string(), - format!("frontal-code.relay_port={port}"), - image.to_string(), - format!("TCP-LISTEN:{port},fork,reuseaddr"), - format!("TCP4:host.docker.internal:{port}"), - ] -} - -/// Security flags for a worker container. Every one is deliberate. -pub(crate) fn worker_hardening_args() -> Vec { - [ - // No Linux capabilities. The worker runs as an unprivileged user. - "--cap-drop", - "ALL", - // No setuid or file-capability escalation inside the container. - "--security-opt", - "no-new-privileges:true", - // Root filesystem is read-only. Writes go to the checkout or to tmpfs. - "--read-only", - "--tmpfs", - TMP_TMPFS, - "--tmpfs", - HOME_TMPFS, - // Bounded resources, so one worker cannot starve the host. - "--pids-limit", - "512", - "--memory", - "4g", - "--memory-swap", - "4g", - "--cpus", - "2", - "--ulimit", - "nofile=4096:8192", - ] - .into_iter() - .map(str::to_string) - .collect() -} - -/// Arguments for a worker container. The environment is passed from a file, not on the -/// command line. -pub(crate) fn worker_run_args( - network: &str, - checkout_root: &str, - env_file: &str, - task_id: &str, - image: &str, - command: &[String], -) -> Vec { - let mut args = vec![ - "run".to_string(), - "-d".to_string(), - "--rm".to_string(), - "--workdir".to_string(), - "/workspace".to_string(), - "--volume".to_string(), - format!("{checkout_root}:/workspace"), - "--network".to_string(), - network.to_string(), - "--label".to_string(), - format!("frontal-code.task_id={task_id}"), - "--label".to_string(), - format!("{TASK_NETWORK_CONTAINER_LABEL}={network}"), - "--env-file".to_string(), - env_file.to_string(), - ]; - args.extend(worker_hardening_args()); - args.push(image.to_string()); - args.extend(command.iter().cloned()); - args -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn task_network_names_are_safe_docker_names() { - assert_eq!( - task_network_name("task_6af0597f_1"), - "frontal-task-task_6af0597f_1" - ); - let escaped = task_network_name("a/../../b"); - assert!(escaped.starts_with("frontal-task-a") && escaped.ends_with('b')); - assert!(!escaped.contains('/') && !escaped.contains('.')); - assert!(task_network_name(&"x".repeat(200)).len() <= "frontal-task-".len() + 48); - } - - #[test] - fn worker_container_is_hardened_and_sees_no_host_socket() { - let args = worker_run_args( - "frontal-task-t1", - "/tmp/checkout", - "/tmp/worker.env", - "t1", - "frontal-code-worker:local", - &["code".to_string(), "hosted".to_string()], - ); - let joined = args.join(" "); - assert!(joined.contains("--cap-drop ALL")); - assert!(joined.contains("--security-opt no-new-privileges:true")); - assert!(joined.contains("--read-only")); - assert!(joined.contains("--network frontal-task-t1")); - assert!(joined.contains("--env-file /tmp/worker.env")); - assert!(joined.contains("--memory 4g")); - assert!(joined.contains("--pids-limit 512")); - // Never privileged, never the Docker socket, never the host network. - assert!(!args.iter().any(|arg| arg == "--privileged")); - assert!(!joined.contains("docker.sock")); - assert!(!args.iter().any(|arg| arg == "host")); - // The image and command come last. - assert_eq!( - args[args.len() - 2..], - ["code".to_string(), "hosted".to_string()] - ); - } - - #[test] - fn no_credentials_appear_on_the_command_line() { - let args = worker_run_args("n", "/c", "/e", "t", "img", &[]); - assert!(!args - .iter() - .any(|arg| arg.contains("_API_KEY") || arg.contains("_TOKEN"))); - } - - #[test] - fn default_relay_image_is_pinned_by_digest() { - assert!(crate::DEFAULT_RELAY_IMAGE.contains("@sha256:")); - assert!(!crate::DEFAULT_RELAY_IMAGE.ends_with(":latest")); - } - - #[test] - fn relay_forwards_one_port_and_is_hardened() { - let args = relay_run_args("alpine/socat", 8788); - assert!(args.contains(&"TCP-LISTEN:8788,fork,reuseaddr".to_string())); - assert!(args.contains(&"TCP4:host.docker.internal:8788".to_string())); - assert!(args.windows(2).any(|pair| pair == ["--cap-drop", "ALL"])); - assert!(args.contains(&"--read-only".to_string())); - } - - #[test] - fn each_server_port_gets_its_own_relay() { - // Two servers on one Docker host must not replace each other's relay. - assert_eq!(relay_container_name(8788), "frontal-code-relay-8788"); - assert_ne!(relay_container_name(8788), relay_container_name(8789)); - let args = relay_run_args("alpine/socat", 8789); - assert!(args - .windows(2) - .any(|pair| pair == ["--name", "frontal-code-relay-8789"])); - } -} diff --git a/crates/code-server/src/integrations.rs b/crates/code-server/src/integrations.rs new file mode 100644 index 00000000..f3f22899 --- /dev/null +++ b/crates/code-server/src/integrations.rs @@ -0,0 +1,601 @@ +//! `/v1/integrations` control-plane routes. +//! +//! Serves the typed integration manifests from [`code_integrations`]: listing +//! integrations, inspecting a single manifest, its capabilities, and its +//! events. Capability invocation is a stub for now; real MCP dispatch lands +//! in Phase 3. + +use std::sync::Mutex; + +use axum::extract::Path; +use axum::extract::State; +use axum::response::Json; +use axum::routing::{get, post}; +use axum::Router; +use code_integrations::mcp::tools::PermissionMode; +use code_integrations::{AuthSpec, IntegrationManifest, ManifestRegistry}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; + +use crate::AppError; +use crate::ServerState; + +/// Environment variable holding a JSON array of [`IntegrationManifest`] values. +pub const INTEGRATIONS_CONFIG_ENV: &str = "FRONTAL_CODE_INTEGRATIONS_CONFIG"; + +/// Server state for the integrations routes. The registry is validated on +/// load; manifests that fail validation are skipped with a warning. +#[derive(Debug, Default)] +pub struct IntegrationsState { + pub registry: Mutex, +} + +impl IntegrationsState { + /// Create an empty state with no registered integrations. + #[cfg(test)] + #[must_use] + pub fn new() -> Self { + Self { + registry: Mutex::new(ManifestRegistry::new()), + } + } + + /// Create a state seeded with the built-in provider manifests (GitHub, + /// Linear, `PostHog`, and Sentry). These are the first-party defaults; + /// project-level configuration can still override any of them by id. + #[cfg(test)] + #[must_use] + pub fn with_builtins() -> Self { + Self { + registry: Mutex::new( + code_integrations::builtin_providers() + .expect("built-in provider manifests are valid"), + ), + } + } + + /// Load integrations from the `FRONTAL_CODE_INTEGRATIONS_CONFIG` + /// environment variable, a JSON array of manifests, on top of the + /// built-in providers. The env var can add new integrations or override a + /// built-in by id. Missing or invalid entries are skipped with a warning + /// so one bad manifest cannot take the whole registry down. + #[must_use] + pub fn load_from_env() -> Self { + Self::from_json( + &std::env::var(INTEGRATIONS_CONFIG_ENV).unwrap_or_default(), + code_integrations::builtin_providers().expect("built-in provider manifests are valid"), + ) + } + + /// Build a state from a raw JSON array of manifests, layered on top of a + /// base registry (typically the built-in providers). An empty, invalid, + /// or malformed string yields the base registry with a warning; + /// individual manifests that fail validation are skipped with a warning + /// so one bad manifest cannot take the whole registry down. A manifest + /// whose id collides with a base entry replaces it, letting project + /// configuration override a built-in. + #[must_use] + pub fn from_json(raw: &str, base: ManifestRegistry) -> Self { + let mut registry = base; + let trimmed = raw.trim(); + if !trimmed.is_empty() { + let Ok(manifests) = serde_json::from_str::>(trimmed) else { + eprintln!("warning: {INTEGRATIONS_CONFIG_ENV} is not a JSON array of manifests; ignoring it"); + return Self { + registry: Mutex::new(registry), + }; + }; + for manifest in manifests { + let id = manifest.id.clone(); + if registry.get(&id).is_some() { + // A same-id manifest from configuration overrides the + // built-in: drop the existing entry, then register the new one. + registry.remove(&id); + } + if let Err(issues) = registry.register(manifest) { + let reason = issues + .iter() + .map(|issue| format!("{}: {}", issue.field, issue.message)) + .collect::>() + .join("; "); + eprintln!("warning: skipping integration manifest `{id}`: {reason}"); + } + } + } + Self { + registry: Mutex::new(registry), + } + } +} + +/// One row of the integration list. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IntegrationSummary { + pub id: String, + pub version: String, + pub display_name: String, + pub description: String, + pub capability_count: usize, + pub auth_required: bool, + pub event_count: usize, +} + +impl From<&IntegrationManifest> for IntegrationSummary { + fn from(manifest: &IntegrationManifest) -> Self { + Self { + id: manifest.id.clone(), + version: manifest.version.clone(), + display_name: manifest.display_name.clone(), + description: manifest.description.clone(), + capability_count: manifest.capabilities.len(), + auth_required: manifest + .auth + .as_ref() + .is_some_and(|auth| !matches!(auth, AuthSpec::None)), + event_count: manifest.events.len(), + } + } +} + +/// One capability of an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CapabilitySummary { + pub name: String, + pub description: String, + pub permission_mode: String, + pub idempotent: bool, +} + +impl From<&code_integrations::CapabilitySpec> for CapabilitySummary { + fn from(spec: &code_integrations::CapabilitySpec) -> Self { + Self { + name: spec.name.clone(), + description: spec.description.clone(), + permission_mode: permission_mode_label(spec.permission_mode), + idempotent: spec.idempotent, + } + } +} + +/// One webhook event an integration can emit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct EventSummary { + pub name: String, + pub description: String, + pub signature_required: bool, +} + +impl From<&code_integrations::EventSpec> for EventSummary { + fn from(event: &code_integrations::EventSpec) -> Self { + Self { + name: event.name.clone(), + description: event.description.clone(), + signature_required: event.signature.is_some(), + } + } +} + +/// Input for invoking a capability. The JSON value is the MCP tool input. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct InvokeRequest { + pub input: Value, +} + +fn permission_mode_label(mode: PermissionMode) -> String { + match mode { + PermissionMode::ReadOnly => "read_only", + PermissionMode::WorkspaceWrite => "workspace_write", + PermissionMode::DangerFullAccess => "danger_full_access", + } + .to_string() +} + +pub(crate) async fn list_integrations( + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + Ok(Json( + registry + .list() + .into_iter() + .map(IntegrationSummary::from) + .collect(), + )) +} + +pub(crate) async fn get_integration( + Path(id): Path, + State(state): State>, +) -> Result, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json(manifest.clone())) +} + +pub(crate) async fn list_capabilities( + Path(id): Path, + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json( + manifest + .capabilities + .iter() + .map(CapabilitySummary::from) + .collect(), + )) +} + +/// Invoke a capability. Stub: MCP dispatch is Phase 3. +pub(crate) async fn invoke_capability( + Path((id, name)): Path<(String, String)>, + State(state): State>, + Json(request): Json, +) -> Result<(), AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let Some(_capability) = registry.capability(&id, &name) else { + return Err(AppError::not_found(format!( + "capability '{name}' not found on integration '{id}'" + ))); + }; + let input = request.input; + drop(registry); + Err(AppError::not_implemented(format!( + "invoking capability '{name}' on integration '{id}' is not implemented yet; \ + MCP dispatch lands in Phase 3 (input: {input})" + ))) +} + +pub(crate) async fn list_events( + Path(id): Path, + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json( + manifest.events.iter().map(EventSummary::from).collect(), + )) +} + +/// Build a router over the integrations routes, for callers that do not want +/// the full control-plane app. +pub(crate) fn integrations_routes() -> Router> { + Router::new() + .route("/v1/integrations", get(list_integrations)) + .route("/v1/integrations/{id}", get(get_integration)) + .route("/v1/integrations/{id}/capabilities", get(list_capabilities)) + .route( + "/v1/integrations/{id}/capabilities/{name}/invoke", + post(invoke_capability), + ) + .route("/v1/integrations/{id}/events", get(list_events)) +} + +#[cfg(test)] +mod tests { + use super::*; + use code_integrations::mcp::config::McpTransport; + use code_integrations::{ContextFieldSpec, EventSpec, SignatureSpec, TransportConfig}; + use serde_json::json; + + fn sample_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: std::collections::BTreeMap::new(), + }, + capabilities: vec![code_integrations::CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: json!({"type": "object"}), + output_schema: Some(json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".to_string(), + header: None, + }), + events: vec![EventSpec { + name: "issue.opened".to_string(), + description: "An issue was opened".to_string(), + payload_schema: json!({"type": "object"}), + signature: Some(SignatureSpec::HmacSha256 { + secret_env: "GITHUB_WEBHOOK_SECRET".to_string(), + }), + }], + context_fields: vec![ContextFieldSpec { + name: "github.issue".to_string(), + description: "The linked GitHub issue".to_string(), + schema: json!({"type": "object"}), + }], + } + } + + #[test] + fn new_creates_an_empty_registry() { + let state = IntegrationsState::new(); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 0); + } + + #[test] + fn from_json_reads_the_sample_json_over_builtins() { + let sample = sample_manifest(); + let raw = serde_json::to_string(&vec![sample]).expect("serialize sample"); + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(&raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The sample manifest overrides the built-in github (same id), and the + // other built-ins remain. + assert!(registry.get("github").is_some()); + assert_eq!(registry.list().len(), 4); + } + + #[test] + fn from_json_over_builtins_allows_a_custom_id_to_be_added() { + let custom = sample_manifest(); + let custom_raw = serde_json::to_string(&vec![custom]).expect("serialize"); + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(&custom_raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + assert!(registry.get("sentry").is_some()); + } + + #[test] + fn from_json_with_empty_raw_keeps_only_builtins() { + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json("", base); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing `{id}`"); + } + } + + static INTEGRATIONS_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// Build a state seeded with the built-in providers, wrapped in an + /// `Arc` for the integrations routes. + fn state_with_builtins() -> Arc { + Arc::new(crate::ServerState { + integrations: std::sync::Arc::new(IntegrationsState::with_builtins()), + ..crate::ServerState::default() + }) + } + + /// Fire a single request at a stateless router and return the response + /// status and body for assertions. The integrations router is built + /// per-call with the given state so each request sees the same snapshot. + async fn oneshot( + state: Arc, + method: &str, + uri: &str, + body: serde_json::Value, + ) -> (axum::http::StatusCode, Vec) { + use axum::body::Body; + use axum::http::Request; + use tower::ServiceExt; + + let router = integrations_routes().with_state(state); + let response = router + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(); + (status, bytes) + } + + #[test] + fn load_from_env_reads_the_sample_json() { + let _guard = INTEGRATIONS_ENV_LOCK.lock().expect("env lock poisoned"); + let sample = sample_manifest(); + let raw = serde_json::to_string(&vec![sample]).expect("serialize sample"); + std::env::set_var(INTEGRATIONS_CONFIG_ENV, raw); + let state = IntegrationsState::load_from_env(); + std::env::remove_var(INTEGRATIONS_CONFIG_ENV); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The sample manifest overrides the built-in github (same id), and the + // other built-ins remain. + assert!(registry.get("github").is_some()); + assert_eq!(registry.list().len(), 4); + } + + #[test] + fn load_from_env_skips_invalid_manifests() { + let _guard = INTEGRATIONS_ENV_LOCK.lock().expect("env lock poisoned"); + let raw = r#"[ + {"id": "Bad Id!", "version": "1.0.0", "display_name": "x", "description": "y", + "transport": "stdio", "transport_config": {"kind": "stdio", "command": "c", "args": [], "env": {}}, + "capabilities": [], "auth": null, "events": [], "context_fields": []}, + {"id": "good-one", "version": "2.0.0", "display_name": "Good", "description": "z", + "transport": "http", "transport_config": {"kind": "http", "url": "https://example.com", "headers": {}}, + "capabilities": [{"name": "ping", "description": "p", "input_schema": {"type": "object"}, + "permission_mode": "read_only", "idempotent": true}], + "auth": null, "events": [], "context_fields": []} + ]"#; + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The bad manifest is skipped; the good one is added; built-ins remain. + assert!(registry.get("Bad Id!").is_none()); + assert!(registry.get("good-one").is_some()); + assert_eq!(registry.list().len(), 5); + } + + #[test] + fn with_builtins_seeds_all_four_providers() { + let state = IntegrationsState::with_builtins(); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing `{id}`"); + } + } + + #[test] + fn summary_dto_serializes_with_expected_fields() { + let summary = IntegrationSummary::from(&sample_manifest()); + let value = serde_json::to_value(&summary).expect("serialize summary"); + assert_eq!(value["id"], "github"); + assert_eq!(value["version"], "1.0.0"); + assert_eq!(value["display_name"], "GitHub"); + assert_eq!(value["capability_count"], 1); + assert_eq!(value["auth_required"], true); + assert_eq!(value["event_count"], 1); + } + + #[test] + fn capability_dto_uses_snake_case_permission_labels() { + let spec = &sample_manifest().capabilities[0]; + let summary = CapabilitySummary::from(spec); + let value = serde_json::to_value(&summary).expect("serialize capability"); + assert_eq!(value["name"], "create_issue"); + assert_eq!(value["permission_mode"], "workspace_write"); + assert_eq!(value["idempotent"], false); + + let mut read_only = spec.clone(); + read_only.permission_mode = PermissionMode::ReadOnly; + assert_eq!( + CapabilitySummary::from(&read_only).permission_mode, + "read_only" + ); + let mut danger = spec.clone(); + danger.permission_mode = PermissionMode::DangerFullAccess; + assert_eq!( + CapabilitySummary::from(&danger).permission_mode, + "danger_full_access" + ); + } + + #[test] + fn event_dto_reports_signature_requirements() { + let event = &sample_manifest().events[0]; + let summary = EventSummary::from(event); + assert_eq!(summary.name, "issue.opened"); + assert!(summary.signature_required); + + let mut unsigned = event.clone(); + unsigned.signature = None; + assert!(!EventSummary::from(&unsigned).signature_required); + } + + #[test] + fn invoke_request_deserializes_an_input_value() { + let request: InvokeRequest = + serde_json::from_value(json!({"input": {"title": "bug", "body": "broken"}})) + .expect("deserialize invoke request"); + assert_eq!(request.input["title"], "bug"); + } + + #[tokio::test] + async fn routes_serve_the_manifest_surface() { + use axum::http::StatusCode; + + // The state is seeded with the built-in providers, so the list is + // non-empty and a built-in id (github) is present. + let state = state_with_builtins(); + let (status, bytes) = oneshot(state.clone(), "GET", "/v1/integrations", json!(null)).await; + assert_eq!(status, StatusCode::OK); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body.as_array().expect("array").len(), 4); + + let (status, _) = + oneshot(state.clone(), "GET", "/v1/integrations/github", json!(null)).await; + assert_eq!(status, StatusCode::OK); + + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/github/capabilities", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/github/events", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::OK); + + // An unknown integration id still 404s. + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/does-not-exist", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + // ...and a known capability returns the 501 stub. + let (status, _) = oneshot( + state.clone(), + "POST", + "/v1/integrations/github/capabilities/create_pr/invoke", + json!({"input": {"title": "bug"}}), + ) + .await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + + // A capability that does not exist on the integration 404s. + let (status, _) = oneshot( + state.clone(), + "POST", + "/v1/integrations/github/capabilities/missing/invoke", + json!({"input": {}}), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + } +} diff --git a/crates/code-server/src/lib.rs b/crates/code-server/src/lib.rs index f5044110..f2ad5fd7 100644 --- a/crates/code-server/src/lib.rs +++ b/crates/code-server/src/lib.rs @@ -10,11 +10,12 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path as FsPath; use std::path::PathBuf; use std::process::Command; +use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; mod audit; -mod docker_worker; +mod integrations; mod publish; mod worker_access; mod worker_tokens; @@ -59,7 +60,13 @@ use code_tools::{ }; use futures_util::{SinkExt, StreamExt}; mod tracker_reporting; +use frontal_sandbox_sdk::Client as SandboxClient; use hmac::{Hmac, KeyInit, Mac}; +use sandbox_types::policy::PolicyAction; +use sandbox_types::{ + ExecEvent, ExecRequest, NetworkPolicy, NetworkPolicyRule, SessionConfig, SessionId, + SessionStatus, SnapshotId, TierPreference, +}; use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; use sha2::Sha256; @@ -141,10 +148,12 @@ pub struct ServerConfig { pub lane_transport_kind: LaneTransportKind, pub api_key: Option, pub workspace_root: PathBuf, - pub docker_image: String, - pub docker_server_url: String, + /// Base URL of the Frontal Sandbox server the worker sessions run on. + pub sandbox_base_url: String, + /// Bearer API key (`sbx_…`) the sandbox server requires. + pub sandbox_api_key: String, pub reconcile_interval: Option, - /// Postgres holds worker tokens. Required for the `local-docker` lane transport. + /// Postgres holds worker tokens. Required for the `sandbox` lane transport. pub database_url: Option, /// CA file that signed the database certificate, for a private CA. Optional. pub database_ca_file: Option, @@ -164,8 +173,8 @@ pub struct ServerConfig { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LaneTransportKind { - /// Each lane runs in a worker container (`docker/worker.Dockerfile`). - LocalDocker, + /// Each lane runs in a Frontal Sandbox session (`deploy/sandbox/` infra). + Sandbox, /// Each lane runs an agent loop inside the server process. ToolsAgent, } @@ -174,10 +183,14 @@ pub enum LaneTransportKind { /// an unknown value is an error, so a typo cannot silently fake lane execution. fn parse_lane_transport_kind(raw: &str) -> Result { match raw.trim().to_ascii_lowercase().as_str() { - "" | "docker" | "local-docker" | "local_docker" => Ok(LaneTransportKind::LocalDocker), + "" | "sandbox" => Ok(LaneTransportKind::Sandbox), "tools-agent" | "tools_agent" | "agent" => Ok(LaneTransportKind::ToolsAgent), + "docker" | "local-docker" | "local_docker" => Err( + "FRONTAL_CODE_SERVER_LANE_TRANSPORT `docker`/`local-docker`/`local_docker` was removed with the Docker worker; use `sandbox` (Frontal Sandbox sessions) or `tools-agent`" + .to_string(), + ), other => Err(format!( - "unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT `{other}`; use `local-docker` (worker containers) or `tools-agent`" + "unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT `{other}`; use `sandbox` (Frontal Sandbox sessions) or `tools-agent`" )), } } @@ -185,7 +198,7 @@ fn parse_lane_transport_kind(raw: &str) -> Result { /// Refuses `tools-agent` unless the operator opted in with `FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT`. fn check_host_agent_opt_in(kind: LaneTransportKind, allowed: bool) -> Result<(), String> { if kind == LaneTransportKind::ToolsAgent && !allowed { - return Err("FRONTAL_CODE_SERVER_LANE_TRANSPORT=tools-agent runs agents with the server's privileges; set FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1 to allow it, or use local-docker".to_string()); + return Err("FRONTAL_CODE_SERVER_LANE_TRANSPORT=tools-agent runs agents with the server's privileges; set FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1 to allow it, or use sandbox".to_string()); } Ok(()) } @@ -195,11 +208,11 @@ impl Default for ServerConfig { Self { bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8788), event_replay_limit: DEFAULT_EVENT_REPLAY_LIMIT, - lane_transport_kind: LaneTransportKind::LocalDocker, + lane_transport_kind: LaneTransportKind::Sandbox, api_key: None, workspace_root: default_server_workspace_root(), - docker_image: default_server_docker_image(), - docker_server_url: default_server_docker_server_url(), + sandbox_base_url: default_sandbox_base_url(), + sandbox_api_key: String::new(), reconcile_interval: Some(Duration::from_secs(15)), database_url: None, database_ca_file: None, @@ -241,7 +254,8 @@ impl ServerConfig { } // `tools-agent` runs the agent inside the server process with its privileges. It is // refused unless the operator opts in, for example because the server already runs - // inside an isolated VM. `local-docker` (the default) isolates each task in a container. + // inside an isolated VM. `sandbox` (the default) isolates each task in a Frontal + // Sandbox session. let host_agent_allowed = env::var("FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT") .is_ok_and(|value| matches!(value.trim(), "1" | "true")); check_host_agent_opt_in(config.lane_transport_kind, host_agent_allowed)?; @@ -260,17 +274,16 @@ impl ServerConfig { } } - if let Ok(image) = env::var("FRONTAL_CODE_SERVER_DOCKER_IMAGE") { - let image = image.trim(); - if !image.is_empty() { - config.docker_image = image.to_string(); - } - } - - if let Ok(url) = env::var("FRONTAL_CODE_SERVER_CALLBACK_URL") { + if let Ok(url) = env::var("FRONTAL_SANDBOX_BASE_URL") { let url = url.trim().trim_end_matches('/'); if !url.is_empty() { - config.docker_server_url = url.to_string(); + config.sandbox_base_url = url.to_string(); + } + } + if let Ok(key) = env::var("FRONTAL_SANDBOX_API_KEY") { + let key = key.trim(); + if !key.is_empty() { + config.sandbox_api_key = key.to_string(); } } @@ -389,6 +402,7 @@ pub struct ServerState { publish_workspace_root: PathBuf, // Tasks with a publish in flight. Held so two requests cannot push the same task twice. publishing_tasks: Arc>>, + integrations: Arc, // Off unless configured. See `ServerConfig::allow_local_repos`. allow_local_repos: bool, } @@ -445,8 +459,6 @@ impl ServerState { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ) } @@ -463,17 +475,14 @@ impl ServerState { orphan_auto_cancel_after: Option, orphan_policy_rules: Vec, workspace_root: PathBuf, - docker_image: String, - docker_server_url: String, worker_access: WorkerAccess, ) -> Self { let publish_workspace_root = workspace_root.clone(); let worker_access = Arc::new(worker_access); let lane_transport: Arc = match lane_transport_kind { - LaneTransportKind::LocalDocker => Arc::new(LocalDockerLaneWorkerTransport::new( + LaneTransportKind::Sandbox => Arc::new(SandboxLaneWorkerTransport::new( workspace_root, - docker_image, - docker_server_url, + sandbox_server_url(), worker_access.clone(), )), LaneTransportKind::ToolsAgent => Arc::new(HostedAgentLaneWorkerTransport), @@ -556,6 +565,7 @@ impl ServerState { .unwrap_or_default(), publish_workspace_root: default_server_workspace_root(), publishing_tasks: Arc::new(Mutex::new(HashSet::new())), + integrations: Arc::new(integrations::IntegrationsState::load_from_env()), allow_local_repos: false, }; state.recover_loaded_state(); @@ -570,7 +580,7 @@ impl ServerState { ) -> Self { Self::with_lane_transport_kind( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, ) } @@ -633,7 +643,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, Some(state_file), @@ -656,7 +666,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, None, @@ -680,7 +690,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, None, @@ -878,12 +888,20 @@ fn default_server_workspace_root() -> PathBuf { ) } -fn default_server_docker_image() -> String { - "frontal-code-worker:local".to_string() +fn default_sandbox_base_url() -> String { + "http://localhost:8080".to_string() } -fn default_server_docker_server_url() -> String { - "http://host.docker.internal:8788".to_string() +/// Public address the sandbox guest network uses to reach this server. It is injected into +/// every session as `FRONTAL_CODE_SERVER_URL` and allowlisted in the session network policy, +/// so it must be resolvable from the guest network (not `localhost` on a real deployment). +fn sandbox_server_url() -> String { + match env::var("FRONTAL_CODE_SERVER_URL") + .map(|value| value.trim().trim_end_matches('/').to_string()) + { + Ok(value) if !value.is_empty() => value, + _ => "http://localhost:8788".to_string(), + } } fn current_unix_timestamp_secs() -> u64 { @@ -1270,6 +1288,10 @@ pub struct HostedTaskContext { pub worker_manifest_file: Option, #[serde(skip_serializing_if = "Option::is_none")] pub worker_output_file: Option, + /// The sandbox session id of the last run, so a re-run can fork from the run's + /// snapshot instead of re-importing the checkout. + #[serde(skip_serializing_if = "Option::is_none")] + pub sandbox_session_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1795,6 +1817,7 @@ pub fn app(state: Arc) -> Router { post(connector_interaction), ) .route("/v1/connectors/{connector}/events", post(connector_event)) + .merge(integrations::integrations_routes()) // Starting a login stores a pending state for the single tenant, so it needs the // control-plane key. It is a POST that returns the provider URL as JSON: a browser // navigation cannot send the `x-api-key` header, so the caller opens the returned @@ -1863,39 +1886,14 @@ fn check_auth_posture(config: &ServerConfig) -> Result<(), String> { Ok(()) } -/// Local Docker workers reach this server through the relay, which forwards to the port in -/// `docker_server_url`. That port must be the one the server listens on. A URL without a -/// port, or with another one, would send every worker to the wrong place, so refuse to start. -fn check_callback_port(config: &ServerConfig) -> Result<(), String> { - if config.lane_transport_kind != LaneTransportKind::LocalDocker { - return Ok(()); - } - let url = &config.docker_server_url; - let authority = url.rsplit("//").next().unwrap_or(url); - let explicit = authority - .rsplit_once(':') - .and_then(|(_, port)| port.parse::().ok()); - match explicit { - Some(port) if port == config.bind_addr.port() => Ok(()), - Some(port) => Err(format!( - "FRONTAL_CODE_SERVER_CALLBACK_URL ({url}) names port {port}, but the server listens on {}. Workers would reach the wrong port.", - config.bind_addr.port() - )), - None => Err(format!( - "FRONTAL_CODE_SERVER_CALLBACK_URL ({url}) must include the port the server listens on ({}).", - config.bind_addr.port() - )), - } -} - pub async fn serve(config: ServerConfig) -> Result<(), Box> { check_auth_posture(&config)?; // Worker tokens live in Postgres. Start-up fails if it is unreachable, rather than - // running workers that cannot authenticate. - let worker_store = if config.lane_transport_kind == LaneTransportKind::LocalDocker { + // running sessions that cannot authenticate. + let worker_store = if config.lane_transport_kind == LaneTransportKind::Sandbox { let url = config.database_url.as_deref().ok_or( - "FRONTAL_CODE_SERVER_DATABASE_URL is required for worker lanes (worker tokens are stored in Postgres)", + "FRONTAL_CODE_SERVER_DATABASE_URL is required for the sandbox lane transport (worker tokens are stored in Postgres)", )?; let tls = worker_tokens::tls_mode_for(url, config.database_ca_file.clone())?; Some(worker_tokens::WorkerTokenStore::connect(url, tls)?) @@ -1926,8 +1924,6 @@ pub async fn serve(config: ServerConfig) -> Result<(), Box Result<(), Box, - env: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -struct HostedDockerTaskPayload { - #[serde(skip_serializing_if = "Option::is_none")] - base_commit: Option, - /// Shell commands the worker may run without asking, as permission rules. - bash_allow: Vec, - /// Shell commands the worker may never run. - bash_deny: Vec, - task_id: String, - prompt: String, - #[serde(skip_serializing_if = "Option::is_none")] - repository: Option, - #[serde(skip_serializing_if = "Option::is_none")] - repo_url: Option, - #[serde(skip_serializing_if = "Option::is_none")] - base_ref: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch: Option, - #[serde(skip_serializing_if = "Option::is_none")] - model: Option, - #[serde(skip_serializing_if = "Option::is_none")] - provider: Option, - #[serde(skip_serializing_if = "Option::is_none")] - permission_mode: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - allowed_tools: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct DockerLaunchResult { - container_id: String, -} - /// Refuses a hosted task whose provider, model, or permission mode the workspace's /// project policy does not allow. /// /// The checkout is the repository being worked on, so its `.frontal-code` settings -/// are read here, together with this server's user settings. The worker container -/// applies the full policy when it starts, including auth method rules. +/// are read here, together with this server's user settings. The worker applies the +/// full policy when it starts, including auth method rules. fn check_hosted_task_policy( checkout_root: &FsPath, request: &LaneExecutionRequest, @@ -4000,316 +3958,21 @@ fn ceiling_label(mode: ResolvedPermissionMode) -> &'static str { } } -trait DockerRunner: Send + Sync + fmt::Debug { - fn launch(&self, spec: &DockerLaunchSpec) -> Result; - - fn stop(&self, container_id: &str) -> Result<(), String>; - - /// Starts the relay that workers use to reach the server. Test runners need nothing. - fn ensure_relay(&self, _image: &str, _port: u16) -> Result<(), String> { - Ok(()) - } - - /// Creates a task's internal network and attaches the relay to it. - fn prepare_task_network(&self, _network: &str) -> Result<(), String> { - Ok(()) - } - - /// Removes a task network. Best effort: the network may still be in use. - fn remove_task_network(&self, _network: &str) {} - - /// Removes task networks that no worker is using. - fn sweep_idle_task_networks(&self) {} -} - -#[derive(Debug)] -struct CliDockerRunner { - /// The server port the relay forwards to. It names this server's relay and marks its - /// task networks. - relay_port: u16, -} - -/// Writes the container environment to a file only this user can read. Passing values -/// as `--env` would expose them in the process list. Docker reads the file while -/// creating the container, so the caller deletes the directory right after `docker run`. -fn write_worker_env_file(env: &BTreeMap) -> Result { - let mut contents = String::new(); - for (key, value) in env { - if value.contains('\n') || value.contains('\r') { - return Err(format!( - "worker environment value for {key} contains a newline" - )); - } - contents.push_str(key); - contents.push('='); - contents.push_str(value); - contents.push('\n'); - } - - let mut bytes = [0_u8; 8]; - getrandom::fill(&mut bytes).map_err(|error| format!("random source: {error}"))?; - let dir = env::temp_dir().join(format!("frontal-code-worker-{}", hex::encode(bytes))); - fs::create_dir_all(&dir).map_err(|error| format!("failed to create env dir: {error}"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&dir, fs::Permissions::from_mode(0o700)) - .map_err(|error| format!("failed to restrict env dir: {error}"))?; - } - let path = dir.join("worker.env"); - let mut options = fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let mut file = options - .open(&path) - .map_err(|error| format!("failed to create env file: {error}"))?; - std::io::Write::write_all(&mut file, contents.as_bytes()) - .map_err(|error| format!("failed to write env file: {error}"))?; - Ok(path) -} - -impl CliDockerRunner { - fn docker(args: &[String]) -> Result { - let output = Command::new("docker") - .args(args) - .output() - .map_err(|error| format!("failed to run docker: {error}"))?; - if output.status.success() { - return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()); - } - Err(String::from_utf8_lossy(&output.stderr).trim().to_string()) - } - - fn docker_str(args: &[&str]) -> Result { - Self::docker( - &args - .iter() - .map(|arg| (*arg).to_string()) - .collect::>(), - ) - } - - fn relay_port(port: u16) -> String { - port.to_string() - } -} - -impl DockerRunner for CliDockerRunner { - fn launch(&self, spec: &DockerLaunchSpec) -> Result { - let env_file = write_worker_env_file(&spec.env)?; - let env_dir = env_file.parent().map(PathBuf::from); - let args = docker_worker::worker_run_args( - &spec.network, - &spec.checkout_root.display().to_string(), - &env_file.display().to_string(), - &spec.task_id, - &spec.image, - &spec.command, - ); - let output = Self::docker(&args); - if let Some(dir) = env_dir { - let _ = fs::remove_dir_all(dir); - } - let container_id = output.map_err(|error| format!("docker run failed: {error}"))?; - if container_id.is_empty() { - return Err("docker run returned an empty container id".to_string()); - } - // Remove the task's network as soon as its worker exits. Without this, a finished - // task's network stays until the next sweep. - let relay = self.relay_container(); - let network = spec.network.clone(); - let waited = container_id.clone(); - std::thread::spawn(move || { - let _ = Self::docker_str(&["wait", &waited]); - remove_network_when_free(&relay, &network); - }); - Ok(DockerLaunchResult { container_id }) - } - - fn stop(&self, container_id: &str) -> Result<(), String> { - // Read the task network first: the container's labels are gone once it is removed. - let network = Self::docker_str(&[ - "inspect", - "--format", - &format!( - "{{{{index .Config.Labels \"{}\"}}}}", - docker_worker::TASK_NETWORK_CONTAINER_LABEL - ), - container_id, - ]) - .ok() - .filter(|name| !name.is_empty() && name != ""); - Self::docker_str(&["stop", container_id]) - .map_err(|error| format!("docker stop failed: {error}"))?; - if let Some(network) = network { - self.remove_task_network(&network); - } - Ok(()) - } - - fn ensure_relay(&self, image: &str, port: u16) -> Result<(), String> { - let relay = docker_worker::relay_container_name(port); - let wanted = Self::relay_port(port); - let running = Self::docker_str(&["inspect", "--format", "{{.State.Running}}", &relay]); - let current_port = Self::docker_str(&[ - "inspect", - "--format", - &format!( - "{{{{index .Config.Labels \"{}\"}}}}", - docker_worker::RELAY_PORT_LABEL - ), - &relay, - ]); - if running.as_deref() == Ok("true") && current_port.as_deref() == Ok(wanted.as_str()) { - return Ok(()); - } - // Not running, or forwarding a different port: replace it. - let _ = Self::docker_str(&["rm", "-f", &relay]); - Self::docker(&docker_worker::relay_run_args(image, port)) - .map(|_| ()) - .map_err(|error| format!("failed to start the worker relay: {error}")) - } - - fn prepare_task_network(&self, network: &str) -> Result<(), String> { - let created = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); - let task_label = format!("{}={network}", docker_worker::TASK_NETWORK_LABEL); - let port_label = format!( - "{}={}", - docker_worker::NETWORK_RELAY_PORT_LABEL, - self.relay_port - ); - let created_label = format!("{}={created}", docker_worker::NETWORK_CREATED_LABEL); - match Self::docker_str(&[ - "network", - "create", - "--internal", - "--label", - &task_label, - "--label", - &port_label, - "--label", - &created_label, - network, - ]) { - Ok(_) => {} - Err(error) if error.contains("already exists") => {} - Err(error) => return Err(format!("failed to create the task network: {error}")), - } - match Self::docker_str(&[ - "network", - "connect", - "--alias", - docker_worker::RELAY_ALIAS, - network, - &self.relay_container(), - ]) { - Ok(_) => Ok(()), - Err(error) if error.contains("already exists") => Ok(()), - Err(error) => Err(format!("failed to attach the relay: {error}")), - } - } - - fn remove_task_network(&self, network: &str) { - remove_network_now(&self.relay_container(), network); - } - - fn sweep_idle_task_networks(&self) { - let task_filter = format!("label={}", docker_worker::TASK_NETWORK_LABEL); - let port_filter = format!( - "label={}={}", - docker_worker::NETWORK_RELAY_PORT_LABEL, - self.relay_port - ); - let Ok(names) = Self::docker_str(&[ - "network", - "ls", - "--filter", - &task_filter, - "--filter", - &port_filter, - "--format", - "{{.Name}}", - ]) else { - return; - }; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); - for name in names.lines().filter(|name| !name.is_empty()) { - // A network younger than this may belong to a launch still in progress. - let created = Self::docker_str(&[ - "network", - "inspect", - "--format", - &format!( - "{{{{index .Labels \"{}\"}}}}", - docker_worker::NETWORK_CREATED_LABEL - ), - name, - ]) - .ok() - .and_then(|value| value.parse::().ok()); - if created.is_none_or(|created| now.saturating_sub(created) < SWEEP_MIN_AGE_SECS) { - continue; - } - // An idle task network has nothing attached (a crash before the relay joined) or - // only the relay. A worker still attached means its task is running. - let attached = Self::docker_str(&[ - "network", - "inspect", - "--format", - "{{len .Containers}}", - name, - ]); - if matches!(attached.as_deref(), Ok("0" | "1")) { - self.remove_task_network(name); - } - } - } -} - -/// Minimum age before a task network without a running worker is treated as abandoned. -const SWEEP_MIN_AGE_SECS: u64 = 120; - -/// Disconnects the relay from a task network and removes it. Ignores errors: a missing -/// network is already clean, and a sweep retries anything left behind. -fn remove_network_now(relay: &str, network: &str) { - let _ = CliDockerRunner::docker_str(&["network", "disconnect", "-f", network, relay]); - let _ = CliDockerRunner::docker_str(&["network", "rm", network]); -} - -/// Removes a task network once its worker has gone. The container's `--rm` removal can -/// lag the exit by a moment, and Docker refuses to remove a network with endpoints, so -/// retry briefly. -fn remove_network_when_free(relay: &str, network: &str) { - for _ in 0..30 { - remove_network_now(relay, network); - let gone = - CliDockerRunner::docker_str(&["network", "inspect", "--format", "{{.Id}}", network]) - .is_err(); - if gone { - return; - } - std::thread::sleep(std::time::Duration::from_secs(1)); - } -} - -impl CliDockerRunner { - /// The name of this server's relay container. - fn relay_container(&self) -> String { - docker_worker::relay_container_name(self.relay_port) - } -} - trait LaneWorkerTransport: Send + Sync + fmt::Debug { fn bootstrap(&self, request: &LaneExecutionRequest) -> Result; + /// Re-runs a task on the workspace it already has. The default starts from + /// scratch (a fresh container per run); transports whose workspace is a + /// persistent session (sandbox) override this to wake or fork the previous + /// session, re-injecting its env file. + fn resume( + &self, + request: &LaneExecutionRequest, + _previous: Option<&str>, + ) -> Result { + self.bootstrap(request) + } + fn cancel( &self, worker_id: &str, @@ -4502,103 +4165,260 @@ impl LaneWorkerTransport for HostedAgentLaneWorkerTransport { } } +/// Per-task state for the sandbox lane transport. The session id doubles as the +/// lane worker id reported to the lane orchestrator. `snapshot_id` records where +/// the last run finished: a re-run forks from it so the new session starts from +/// the previous run's final state instead of the original import. `checkout_root` +/// and `base_commit` will be used by the diff the lane orchestrator computes +/// against the worker's changes (the session VFS is the worker's copy of the +/// checkout); they are stored now so the state is complete once that lands. #[derive(Debug)] -struct LocalDockerLaneWorkerTransport { - workspace_root: PathBuf, - image: String, - /// The worker's view of the server: the relay, by its alias on the task network. - server_url: String, - relay_image: String, - relay_port: u16, - worker_access: Arc, - docker: Arc, +struct SandboxTaskState { + /// The session that ran the task. The map key is also the session id, so + /// this field is redundant today; kept for the future diff path. + #[allow(dead_code)] + session_id: SessionId, + /// The local checkout the repository was imported from. Used for the diff + /// against the worker's changes (the session VFS is the worker's copy). + #[allow(dead_code)] + checkout_root: PathBuf, + /// The commit the worker's changes are measured against. `None` for a + /// scratch workspace without a repository. + #[allow(dead_code)] + base_commit: Option, + /// The snapshot the last run finished on. Set by the exec-drain thread when + /// the agent exits cleanly; read by `resume` to decide fork vs wake. + snapshot_id: Option, } -/// Port a server URL points at, or the default server port. -fn port_from_server_url(url: &str) -> u16 { - url.trim_end_matches('/') - .rsplit(':') - .next() - .and_then(|port| port.parse().ok()) - .unwrap_or(8788) +/// The task handoff written to `/workspace/.frontal-code-hosted/task.json` inside a +/// sandbox session. The worker inside the session reads it to start the agent. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct HostedLaneTaskPayload { + task_id: String, + prompt: String, + #[serde(skip_serializing_if = "Option::is_none")] + repository: Option, + #[serde(skip_serializing_if = "Option::is_none")] + repo_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + base_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + permission_mode: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + allowed_tools: Vec, + /// Shell commands the worker may run without asking, as permission rules. + bash_allow: Vec, + /// Shell commands the worker may never run. + bash_deny: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + base_commit: Option, } -/// The relay image, pinned by digest. A tag can move; a digest cannot. Override it with -/// `FRONTAL_CODE_WORKER_RELAY_IMAGE` after reviewing the image you choose. -pub(crate) const DEFAULT_RELAY_IMAGE: &str = - "alpine/socat@sha256:e8a6e8868a8e6e01d9c2c321a8f8d4474d2d741d962462e5c5a124a8e66a0055"; - -fn relay_image_from_env() -> String { - env::var("FRONTAL_CODE_WORKER_RELAY_IMAGE") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_RELAY_IMAGE.to_string()) +/// Runs hosted-agent tasks inside a Frontal Sandbox session. The session's network +/// policy allowlists only this code server, and the per-task worker token is written +/// to a file inside the session (the session's env is dropped on wake, so env vars +/// cannot be relied on). +#[derive(Debug)] +struct SandboxLaneWorkerTransport { + workspace_root: PathBuf, + /// The actual code-server URL the sandbox session reaches directly. + server_url: String, + worker_access: Arc, + client: Arc, + tasks: Arc>>, } -impl LocalDockerLaneWorkerTransport { - fn new( - workspace_root: PathBuf, - image: String, - server_url: String, - worker_access: Arc, - ) -> Self { - let relay_port = port_from_server_url(&server_url); - Self::with_runner( +impl SandboxLaneWorkerTransport { + /// Builds a client from `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`, + /// reading both from the environment. + fn new(workspace_root: PathBuf, server_url: String, worker_access: Arc) -> Self { + let base_url = env::var("FRONTAL_SANDBOX_BASE_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "http://localhost:8080".to_string()); + let api_key = env::var("FRONTAL_SANDBOX_API_KEY") + .ok() + .filter(|value| !value.trim().is_empty()); + Self { workspace_root, - image, server_url, worker_access, - Arc::new(CliDockerRunner { relay_port }), - ) + client: Arc::new(SandboxClient::new(base_url, api_key)), + tasks: Arc::new(Mutex::new(HashMap::new())), + } } - fn with_runner( - workspace_root: PathBuf, - image: String, - server_url: impl AsRef, - worker_access: Arc, - docker: Arc, - ) -> Self { - let relay_port = port_from_server_url(server_url.as_ref()); - Self { - workspace_root, - image, - server_url: format!("http://{}:{relay_port}", docker_worker::RELAY_ALIAS), - relay_image: relay_image_from_env(), - relay_port, - worker_access, - docker, + /// Looks up the snapshot a task's last run finished on, if one was + /// recorded. Used by `resume` to decide between forking from the snapshot + /// and waking the previous session. + fn snapshot_of(&self, session: &SessionId) -> Option { + self.tasks + .lock() + .unwrap() + .get(&session.to_string()) + .and_then(|state| state.snapshot_id) + } + + /// Builds the session config used for every task session. The session may + /// only reach this code server; everything else is default-deny. Provider + /// traffic flows through the server's model proxy. + fn session_config(&self) -> SessionConfig { + let mut network = NetworkPolicy::default(); + network.add_rule(NetworkPolicyRule { + action: PolicyAction::Allow, + host_pattern: Self::server_host_allowlist(&self.server_url), + ports: None, + }); + SessionConfig { + network, + tier_preference: TierPreference::Auto, + // `limits` and `secrets` are intentionally left at their defaults: + // the sandbox server currently returns 501 for both. + ..SessionConfig::default() } } -} -impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { - fn bootstrap(&self, request: &LaneExecutionRequest) -> Result { - // A task with a repository gets a checkout on its own branch. A task without one - // gets an empty scratch workspace, so a plain prompt can still run in a worker. - let prepared = match lane_repo_source(request) { - Some(source) => prepare_checkout(&RepoCheckoutRequest { - workspace_root: self.workspace_root.clone(), - checkout_id: request.task_id.clone(), - source, - repository: request.repository.clone(), - base_ref: request.base_ref.clone(), - branch: request.branch.clone(), - }) - .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, - None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + /// Starts a session for a task, preferring `previous` when given: + /// a snapshot from the last run is forked (the new session starts from the + /// previous run's final state, and the old session is left untouched), + /// otherwise the previous session is woken and its files restored. When the + /// previous session no longer exists the session is created fresh and the + /// local checkout is re-imported. The env file is (re)written on every + /// start because the session drops its env on wake (ADR 0010). + async fn attach_to_session( + &self, + request: &LaneExecutionRequest, + previous: Option, + checkout_root: Option<&FsPath>, + base_commit: Option<&str>, + ) -> Result<(SessionId, usize, String), String> { + let client = &self.client; + match previous { + Some(id) if self.snapshot_of(&id).is_some() => { + let snapshot = self.snapshot_of(&id).expect("checked above"); + let sessions = client.fork(snapshot, 1).await.map_err(|error| { + format!("failed to fork sandbox session from snapshot: {error}") + })?; + let session = sessions + .into_iter() + .next() + .ok_or_else(|| "sandbox server returned no session id for fork".to_string())?; + // Re-inject the env file so the re-run sees fresh credentials. + self.write_session_files(client, session, request, base_commit) + .await?; + Ok((session, 0, "forked from snapshot".to_string())) + } + Some(id) => { + // Probe first: `wake` on an already-awake session is rejected + // with InvalidState, so decide based on the session's current + // state rather than on the outcome of waking it. + match client.get_session(id).await { + Ok(state) if state.status == SessionStatus::Running => { + // Already awake: re-inject env and proceed (ADR 0010). + self.write_session_files(client, id, request, base_commit) + .await?; + Ok((id, 0, "resumed awake session".to_string())) + } + Ok(state) if state.status == SessionStatus::Sleeping => { + match client.wake(id).await { + Ok(_) => { + // Woken sessions come back without env (ADR 0010); + // the workspace files were restored from the + // sleep snapshot, so re-inject the env file. + self.write_session_files(client, id, request, base_commit) + .await?; + Ok((id, 0, "woken previous session".to_string())) + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + %error, + "failed to wake sandbox session, starting fresh" + ); + self.start_fresh_session( + client, + request, + checkout_root, + base_commit, + ) + .await + } + } + } + Ok(state) => { + // Some other state (Promoting, Error, Destroyed): the + // session cannot be used; fall back to a fresh one. + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + ?state.status, + "previous sandbox session in unusable state, starting fresh" + ); + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + %error, + "previous sandbox session gone, starting fresh" + ); + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + } + } + None => { + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + } + } + + /// Creates a brand-new session and imports the local checkout into it. + async fn start_fresh_session( + &self, + client: &SandboxClient, + request: &LaneExecutionRequest, + checkout_root: Option<&FsPath>, + base_commit: Option<&str>, + ) -> Result<(SessionId, usize, String), String> { + let session = client + .create_session(self.session_config()) + .await + .map_err(|error| format!("failed to create sandbox session: {error}"))?; + + let files = if let Some(root) = checkout_root { + Self::import_checkout(client, session, root) + .await + .map_err(|error| format!("failed to import checkout into sandbox: {error}"))? + } else { + 0 }; - check_hosted_task_policy(&prepared.checkout_root, request)?; - // The commit the worker's changes are measured against. The server records it, so the - // worker cannot pick its own baseline. - let base_commit = git_head_commit(&prepared.checkout_root); - let task_file = - write_local_docker_task_payload(&prepared.checkout_root, request, base_commit) - .map_err(|error| format!("failed to write hosted docker task payload: {error}"))?; - let image = self.image.clone(); - // The worker gets a token that works only for this task, plus a proxy URL for - // the model provider. No provider key and no control-plane key enter the - // container. See worker_access.rs. + self.write_session_files(client, session, request, base_commit) + .await?; + Ok((session, files, "created new session".to_string())) + } + + /// Writes the env file and task payload the agent reads on start. Called on + /// every start (fresh, woken, or forked) because env is dropped on wake + /// (ADR 0010) and the payload carries the re-run's new prompt. + async fn write_session_files( + &self, + client: &SandboxClient, + session: SessionId, + request: &LaneExecutionRequest, + base_commit: Option<&str>, + ) -> Result<(), String> { let token = self .worker_access .issue(&request.task_id) @@ -4607,99 +4427,450 @@ impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { request.provider.as_deref(), request.model.as_deref(), )?; - let mut container_env = BTreeMap::from([ - ( - "FRONTAL_CODE_SERVER_URL".to_string(), - self.server_url.clone(), - ), - ("FRONTAL_CODE_SERVER_API_KEY".to_string(), token.clone()), - ( - "FRONTAL_CODE_HOSTED_TASK_FILE".to_string(), - "/workspace/.frontal-code-hosted/task.json".to_string(), - ), - ]); - // Lets the worker CLI accept the cleartext Docker host alias. Set only here. - container_env.insert("FRONTAL_CODE_HOSTED_WORKER".to_string(), "1".to_string()); - for (key, value) in worker_access::worker_provider_env(provider, &self.server_url, &token) { - container_env.insert(key, value); - } - // Non-secret tuning only. Provider keys are never forwarded. - if let Ok(value) = env::var("FRONTAL_CODE_MAX_OUTPUT_TOKENS") { - if !value.trim().is_empty() { - container_env.insert("FRONTAL_CODE_MAX_OUTPUT_TOKENS".to_string(), value); + + // The session's env is dropped on wake (ADR 0010), so the env map is + // written to a file the agent sources. + let env_json = + serde_json::to_vec_pretty(&Self::sandbox_task_env(&self.server_url, &token, provider)) + .expect("env map is always serializable"); + client + .write_file(session, "/workspace/.frontal-code-hosted/env", env_json) + .await + .map_err(|error| format!("failed to write sandbox env file: {error}"))?; + + client + .write_file( + session, + "/workspace/.frontal-code-hosted/task.json", + Self::sandbox_task_payload(request, base_commit), + ) + .await + .map_err(|error| format!("failed to write sandbox task payload: {error}"))?; + Ok(()) + } + + /// Starts the agent in the session fire-and-forget. Its fine-grained + /// events flow back over the control plane via `FRONTAL_CODE_SERVER_URL`, so + /// the server does not need to capture its stdout. The thread owns its own + /// runtime and ends when the command exits. `cancel` destroys the session, + /// which kills the exec. On a clean exit the session's state is snapshotted + /// so a re-run can fork from the final workspace. + fn spawn_exec(&self, session: SessionId, task_id: &str) { + let client = self.client.clone(); + let tasks = self.tasks.clone(); + let task_id = task_id.to_string(); + std::thread::spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "failed to build sandbox exec runtime" + ); + return; + } + }; + let request = ExecRequest { + argv: vec![ + "frontal-code".to_string(), + "hosted".to_string(), + "task".to_string(), + "run".to_string(), + task_id.clone(), + ], + cwd: Some("/workspace".to_string()), + stream: true, + ..ExecRequest::default() + }; + runtime.block_on(Self::run_exec_stream( + &client, &tasks, &task_id, session, &request, + )); + }); + } + + /// Consumes the exec stream for one agent run, logging its fine-grained + /// events. On a clean exit it records the session's snapshot so a re-run + /// can fork from the final workspace; the server may have destroyed the + /// session by then, in which case the re-run falls back to a fresh one. + async fn run_exec_stream( + client: &SandboxClient, + tasks: &Mutex>, + task_id: &str, + session: SessionId, + request: &ExecRequest, + ) { + let stream = match client.exec_stream(session, request).await { + Ok(stream) => stream, + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "sandbox exec stream failed to start" + ); + return; + } + }; + futures_util::pin_mut!(stream); + while let Some(event) = stream.next().await { + match event { + Ok(ExecEvent::Output { data, .. }) => { + tracing::debug!( + target: "code-server::sandbox", + task_id = %task_id, + output = %String::from_utf8_lossy(&data), + "sandbox exec output" + ); + } + Ok(ExecEvent::Exit { exit_code, .. }) => { + tracing::info!( + target: "code-server::sandbox", + task_id = %task_id, + exit_code, + "sandbox exec finished" + ); + if exit_code == 0 { + match client.snapshot(session).await { + Ok(snapshot) => { + if let Ok(mut tasks) = tasks.try_lock() { + if let Some(state) = tasks.get_mut(&session.to_string()) { + state.snapshot_id = Some(snapshot); + } + } + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "failed to snapshot sandbox session after exec" + ); + } + } + } + return; + } + Ok(ExecEvent::Error { message, .. }) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %message, + "sandbox exec error" + ); + return; + } + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "sandbox exec stream error" + ); + return; + } } } - // No GitHub credentials enter the container. The server publishes the branch and - // opens the pull request from its own environment. See publish.rs. - // The relay must be up, and the task must have its own network, before the worker starts. - self.docker - .ensure_relay(&self.relay_image, self.relay_port)?; - self.docker.sweep_idle_task_networks(); - let network = docker_worker::task_network_name(&request.task_id); - self.docker.prepare_task_network(&network)?; - let launch = self.docker.launch(&DockerLaunchSpec { - image: image.clone(), - checkout_root: prepared.checkout_root.clone(), - task_id: request.task_id.clone(), - network: network.clone(), - command: vec![ - "frontal-code".to_string(), - "hosted".to_string(), - "task".to_string(), - "run".to_string(), - request.task_id.clone(), - ], - env: container_env, - })?; + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + "sandbox exec stream ended before exit or error" + ); + } + + /// Records per-task state, emits the launch audit event, and builds the + /// bootstrap result shared by `bootstrap` and `resume`. + fn complete_start( + &self, + task_id: &str, + session: SessionId, + files: usize, + how: &str, + prepared: &PreparedRepo, + base_commit: Option, + ) -> LaneBootstrapResult { + let checkout_root = &prepared.checkout_root; + self.tasks.lock().unwrap().insert( + session.to_string(), + SandboxTaskState { + session_id: session, + base_commit, + checkout_root: checkout_root.clone(), + snapshot_id: None, + }, + ); if let Some(audit) = self.worker_access.audit() { audit.emit( "worker.launched", - &request.task_id, - &format!("container={}", launch.container_id), + task_id, + &format!("session={session} {how}"), ); } - Ok(LaneBootstrapResult { - worker_id: launch.container_id.clone(), + let allowlist = Self::server_host_allowlist(&self.server_url); + LaneBootstrapResult { + worker_id: session.to_string(), worker_status: WorkerStatus::Running, signals: vec![LaneTransportSignal { kind: LaneTransportSignalKind::Running, - detail: Some(format!( - "local docker container {} prepared checkout at {}", - launch.container_id, - prepared.checkout_root.display() - )), + detail: Some(format!("sandbox session {session} {how}")), payload: Some(json!({ - "container_id": launch.container_id, - "image": image, - "checkout_root": prepared.checkout_root.display().to_string(), - "task_file": task_file.display().to_string(), - "command": ["frontal-code", "hosted", "task", "run", request.task_id.as_str()], - "active_ref": prepared.active_ref, - "branch": prepared.branch, - "source": prepared.source.display(), + "session_id": session.to_string(), + "files_imported": files, + "checkout_root": checkout_root.display().to_string(), + "command": ["frontal-code", "hosted", "task", "run", task_id], + "source": prepared.source.display().as_str(), + "network_allowlist": allowlist, })), }], artifacts: None, - checkout_root: Some(prepared.checkout_root.display().to_string()), + checkout_root: Some(checkout_root.display().to_string()), + } + } + + /// The host (and optional port) of a server URL, for the session's network + /// allowlist. `https://frontal.tools:8443/` -> `frontal.tools:8443`. + fn server_host_allowlist(server_url: &str) -> String { + let trimmed = server_url.trim_end_matches('/'); + let without_scheme = trimmed.split_once("://").map_or(trimmed, |(_, rest)| rest); + without_scheme + .split('/') + .next() + .unwrap_or(without_scheme) + .to_string() + } + + /// Recursively walks `root` and uploads every regular file to the session + /// under `/workspace/`, skipping the `.git` directory. Returns the number of + /// files written. + async fn import_checkout( + client: &SandboxClient, + session: SessionId, + root: &FsPath, + ) -> Result { + fn collect(dir: &FsPath, out: &mut Vec) -> Result<(), String> { + let read_dir = fs::read_dir(dir) + .map_err(|error| format!("failed to read {}: {error}", dir.display()))?; + for entry in read_dir { + let entry = entry.map_err(|error| format!("failed to read dir entry: {error}"))?; + let path = entry.path(); + if path.is_dir() { + // Skip VCS metadata; the worker never needs it. + if path.file_name().is_some_and(|name| name == ".git") { + continue; + } + collect(&path, out)?; + } else if path.is_file() { + out.push(path); + } + } + Ok(()) + } + let mut entries: Vec = Vec::new(); + collect(root, &mut entries) + .map_err(|error| format!("failed to walk checkout {}: {error}", root.display()))?; + + let mut count = 0usize; + for path in &entries { + let relative = path + .strip_prefix(root) + .map_err(|error| format!("failed to strip prefix {}: {error}", root.display()))?; + let session_path = format!("/workspace/{}", relative.display()); + let bytes = fs::read(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + client + .write_file(session, &session_path, bytes) + .await + .map_err(|error| format!("failed to write {session_path}: {error}"))?; + count += 1; + } + Ok(count) + } + + /// The env map handed to the worker. The session drops its env on wake, so the + /// agent reads the written file instead of relying on process env vars. + fn sandbox_task_env(server_url: &str, token: &str, provider: &str) -> BTreeMap { + let mut task_env = BTreeMap::from([ + ( + "FRONTAL_CODE_SERVER_URL".to_string(), + server_url.to_string(), + ), + ("FRONTAL_CODE_SERVER_API_KEY".to_string(), token.to_string()), + ( + "FRONTAL_CODE_HOSTED_TASK_FILE".to_string(), + "/workspace/.frontal-code-hosted/task.json".to_string(), + ), + ]); + task_env.insert("FRONTAL_CODE_HOSTED_WORKER".to_string(), "1".to_string()); + for (key, value) in worker_access::worker_provider_env(provider, server_url, token) { + task_env.insert(key, value); + } + // Non-secret tuning only. Provider keys are never forwarded. + if let Ok(value) = env::var("FRONTAL_CODE_MAX_OUTPUT_TOKENS") { + if !value.trim().is_empty() { + task_env.insert("FRONTAL_CODE_MAX_OUTPUT_TOKENS".to_string(), value); + } + } + task_env + } + + fn sandbox_task_payload(request: &LaneExecutionRequest, base_commit: Option<&str>) -> Vec { + serde_json::to_vec_pretty(&HostedLaneTaskPayload { + task_id: request.task_id.clone(), + prompt: request.prompt.clone(), + repository: request.repository.clone(), + repo_url: request.repo_url.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + model: request.model.clone(), + provider: request.provider.clone(), + permission_mode: Some( + request + .permission_mode + .clone() + .unwrap_or_else(|| worker_access::WORKER_DEFAULT_PERMISSION_MODE.to_string()), + ), + allowed_tools: request.allowed_tools.clone(), + bash_allow: worker_access::WORKER_BASH_ALLOW + .iter() + .map(|rule| (*rule).to_string()) + .collect(), + bash_deny: worker_access::WORKER_BASH_DENY + .iter() + .map(|rule| (*rule).to_string()) + .collect(), + base_commit: base_commit.map(str::to_string), }) + .expect("hosted task payload is always serializable") + } +} + +impl LaneWorkerTransport for SandboxLaneWorkerTransport { + fn bootstrap(&self, request: &LaneExecutionRequest) -> Result { + // A task with a repository gets a checkout on its own branch. A task + // without one gets an empty scratch workspace. + let prepared = match lane_repo_source(request) { + Some(source) => prepare_checkout(&RepoCheckoutRequest { + workspace_root: self.workspace_root.clone(), + checkout_id: request.task_id.clone(), + source, + repository: request.repository.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + }) + .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, + None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + }; + check_hosted_task_policy(&prepared.checkout_root, request)?; + // The commit the worker's changes are measured against. The server + // records it, so the worker cannot pick its own baseline. + let base_commit = git_head_commit(&prepared.checkout_root); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let (session, files, how) = runtime.block_on(self.attach_to_session( + request, + None, + Some(&prepared.checkout_root), + base_commit.as_deref(), + ))?; + + self.spawn_exec(session, &request.task_id); + + Ok(self.complete_start( + &request.task_id, + session, + files, + &how, + &prepared, + base_commit, + )) + } + + /// Re-runs a task on the workspace its previous run left behind. The + /// previous worker id is the sandbox session id: if that run finished + /// cleanly, its final state was snapshotted and the re-run forks a fresh + /// session from it (the old session is left untouched, so a second re-run + /// still works); otherwise the previous session is woken and its files are + /// restored from the sleep snapshot. Either way the env file is re-written + /// (env is dropped on wake, ADR 0010) and the agent is exec'd again with + /// the task's current prompt. The local checkout is re-prepared only so the + /// re-run's diff is measured against the same baseline; it is not imported + /// unless the previous session is gone. + fn resume( + &self, + request: &LaneExecutionRequest, + previous: Option<&str>, + ) -> Result { + let prepared = match lane_repo_source(request) { + Some(source) => prepare_checkout(&RepoCheckoutRequest { + workspace_root: self.workspace_root.clone(), + checkout_id: request.task_id.clone(), + source, + repository: request.repository.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + }) + .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, + None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + }; + check_hosted_task_policy(&prepared.checkout_root, request)?; + let base_commit = git_head_commit(&prepared.checkout_root); + // The previous worker id is the session id. If the parse fails the + // re-run falls back to a fresh session, which re-imports the checkout. + let previous_session = previous.and_then(|id| SessionId::from_str(id).ok()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let (session, files, how) = runtime.block_on(self.attach_to_session( + request, + previous_session, + Some(&prepared.checkout_root), + base_commit.as_deref(), + ))?; + + self.spawn_exec(session, &request.task_id); + + Ok(self.complete_start( + &request.task_id, + session, + files, + &how, + &prepared, + base_commit, + )) } fn observe_completion( &self, - _worker_id: &str, + worker_id: &str, finish_reason: &str, _tokens_output: u64, ) -> Result { - let worker_status = if finish_reason.eq_ignore_ascii_case("error") + // The session VFS is the worker's checkout, but the lane orchestrator + // computes the diff against the local checkout recorded in + // `LaneBootstrapResult.checkout_root`. The worker's exit is reported + // asynchronously via the control plane (the agent POSTs its terminal + // event). + if finish_reason.eq_ignore_ascii_case("error") || finish_reason.eq_ignore_ascii_case("failed") { - WorkerStatus::Failed - } else { - WorkerStatus::Finished - }; + return Ok(LaneCompletionResult { + worker_status: WorkerStatus::Failed, + error: None, + }); + } + let _ = worker_id; Ok(LaneCompletionResult { - worker_status, + worker_status: WorkerStatus::Finished, error: None, }) } @@ -4709,12 +4880,32 @@ impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { worker_id: &str, _locator: Option<&HostedAgentLocator>, ) -> Result { - self.docker.stop(worker_id)?; - Ok(LaneCancellationResult { - worker_status: Some(WorkerStatus::Finished), - clear_worker_status: false, - detail: Some(format!("local docker container {worker_id} stopped")), - }) + let session = SessionId::from_str(worker_id) + .map_err(|error| format!("invalid sandbox session id `{worker_id}`: {error}"))?; + // Best-effort: a destroyed or already-destroyed session still counts as + // cancelled from the lane's point of view. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let outcome = runtime.block_on(self.client.destroy_session(session)); + self.tasks.lock().unwrap().remove(worker_id); + match outcome { + Ok(()) => Ok(LaneCancellationResult { + worker_status: Some(WorkerStatus::Finished), + clear_worker_status: true, + detail: Some(format!("sandbox session {worker_id} destroyed")), + }), + // Best-effort: even if the API call fails (session already + // destroyed, network blip, etc.) the lane treats it as cancelled. + Err(error) => Ok(LaneCancellationResult { + worker_status: Some(WorkerStatus::Finished), + clear_worker_status: true, + detail: Some(format!( + "sandbox session {worker_id} destroyed (best-effort: {error})" + )), + }), + } } } @@ -4778,47 +4969,6 @@ fn git_head_commit(checkout_root: &FsPath) -> Option { .filter(|sha| sha.len() == 40 && sha.chars().all(|c| c.is_ascii_hexdigit())) } -fn write_local_docker_task_payload( - checkout_root: &FsPath, - request: &LaneExecutionRequest, - base_commit: Option, -) -> Result { - let payload_dir = checkout_root.join(".frontal-code-hosted"); - std::fs::create_dir_all(&payload_dir)?; - let payload_path = payload_dir.join("task.json"); - std::fs::write( - &payload_path, - serde_json::to_vec_pretty(&HostedDockerTaskPayload { - task_id: request.task_id.clone(), - prompt: request.prompt.clone(), - repository: request.repository.clone(), - repo_url: request.repo_url.clone(), - base_ref: request.base_ref.clone(), - branch: request.branch.clone(), - model: request.model.clone(), - provider: request.provider.clone(), - permission_mode: Some( - request - .permission_mode - .clone() - .unwrap_or_else(|| worker_access::WORKER_DEFAULT_PERMISSION_MODE.to_string()), - ), - allowed_tools: request.allowed_tools.clone(), - bash_allow: worker_access::WORKER_BASH_ALLOW - .iter() - .map(|rule| (*rule).to_string()) - .collect(), - bash_deny: worker_access::WORKER_BASH_DENY - .iter() - .map(|rule| (*rule).to_string()) - .collect(), - base_commit, - }) - .map_err(std::io::Error::other)?, - )?; - Ok(payload_path) -} - #[allow(clippy::too_many_lines)] /// Whether the server may clone and publish to a repository that is a path on this host. /// Off by default: a task could otherwise read or push to any repository the server can @@ -4987,6 +5137,7 @@ fn create_task_internal(state: &ServerState, request: CreateTaskRequest) -> Resu &mut context, primary_role, primary_description.as_deref(), + None, )?; state @@ -5001,7 +5152,11 @@ fn bootstrap_task_lane( context: &mut HostedTaskContext, primary_role: Option, primary_description: Option<&str>, + previous_worker_id: Option<&str>, ) -> Result<(), AppError> { + // The lane transport signals carry their own descriptions; the caller's + // primary description is not needed. + let _ = primary_description; let lane_request = LaneExecutionRequest { task_id: task.task_id.clone(), prompt: task.prompt.clone(), @@ -5016,11 +5171,16 @@ fn bootstrap_task_lane( allowed_tools: context.allowed_tools.clone(), }; - match state - .lane_transport - .bootstrap(&lane_request) - .map_err(AppError::internal) - { + // A re-run: the previous worker id (the sandbox session id) is carried in + // by the caller, so ask the transport to resume from it. A fresh run: the + // transport creates a new session. + match if previous_worker_id.is_some() { + state + .lane_transport + .resume(&lane_request, previous_worker_id) + } else { + state.lane_transport.bootstrap(&lane_request) + } { Ok(bootstrap) => { context.worker_id = Some(bootstrap.worker_id.clone()); context.worker_status = Some(bootstrap.worker_status.to_string()); @@ -5036,6 +5196,11 @@ fn bootstrap_task_lane( context.orphaned_at = None; context.orphaned_approval_requested_at = None; context.applied_orphan_policy = None; + // The worker id is the sandbox session id; record it so a later + // re-run can fork from the run's snapshot instead of re-importing. + if state.lane_transport_kind == LaneTransportKind::Sandbox { + context.sandbox_session_id = context.worker_id.clone(); + } state.record_context(&task.task_id, context.clone()); emit_lane_transport_signals( @@ -5068,7 +5233,7 @@ fn bootstrap_task_lane( .map_err(AppError::internal)?; state .tasks - .append_output(&task.task_id, &error.message) + .append_output(&task.task_id, &error) .map_err(AppError::internal)?; state.broadcast_event(EventEnvelope::new( HostedEventName::LaneFailed, @@ -5084,7 +5249,7 @@ fn bootstrap_task_lane( context, Some("failed"), Some(serialize_event_extra(TerminalEventPayload { - error: Some(error.message.clone()), + error: Some(error.clone()), extra: Map::new(), ..TerminalEventPayload::default() })), @@ -5284,7 +5449,7 @@ fn lane_role_label(role: LaneRole) -> &'static str { fn lane_transport_kind_label(kind: LaneTransportKind) -> &'static str { match kind { - LaneTransportKind::LocalDocker => "local_docker", + LaneTransportKind::Sandbox => "sandbox", LaneTransportKind::ToolsAgent => "tools_agent", } } @@ -5352,6 +5517,13 @@ impl AppError { message: message.into(), } } + + fn not_implemented(message: impl Into) -> Self { + Self { + status: StatusCode::NOT_IMPLEMENTED, + message: message.into(), + } + } } impl IntoResponse for AppError { @@ -5380,63 +5552,25 @@ mod tests { assert!(prepared.repository.is_none()); } - #[test] - fn worker_payload_defaults_to_full_access_inside_the_container() { - use super::{write_local_docker_task_payload, LaneExecutionRequest}; - let root = temp_test_dir("worker-payload"); - let request = LaneExecutionRequest { - task_id: "task-1".to_string(), - prompt: "hello".to_string(), - repository: None, - repo_url: None, - base_ref: None, - branch: None, - lane_role: None, - model: None, - provider: None, - permission_mode: None, - allowed_tools: Vec::new(), - }; - let path = write_local_docker_task_payload(&root, &request, None).expect("payload written"); - let payload: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("read payload")) - .expect("payload json"); - assert_eq!(payload["permission_mode"], "workspace-write"); - assert!(payload["bash_allow"] - .as_array() - .is_some_and(|rules| rules.iter().any(|rule| rule == "bash(cargo:*)"))); - assert!(payload["bash_deny"] - .as_array() - .is_some_and(|rules| rules.iter().any(|rule| rule == "bash(curl:*)"))); - - let explicit = LaneExecutionRequest { - permission_mode: Some("read-only".to_string()), - ..request - }; - let path = - write_local_docker_task_payload(&root, &explicit, None).expect("payload written"); - let payload: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("read payload")) - .expect("payload json"); - assert_eq!(payload["permission_mode"], "read-only"); - } - #[test] fn lane_transport_parser_accepts_workers_and_refuses_everything_else() { use super::{parse_lane_transport_kind, LaneTransportKind}; assert_eq!( parse_lane_transport_kind("").unwrap(), - LaneTransportKind::LocalDocker + LaneTransportKind::Sandbox ); assert_eq!( - parse_lane_transport_kind(" Local-Docker ").unwrap(), - LaneTransportKind::LocalDocker + parse_lane_transport_kind(" sandbox ").unwrap(), + LaneTransportKind::Sandbox ); assert_eq!( parse_lane_transport_kind("tools-agent").unwrap(), LaneTransportKind::ToolsAgent ); - for refused in ["in-memory", "in_memory", "memory", "bogus"] { + let error = parse_lane_transport_kind("local-docker") + .expect_err("the removed docker transport must be refused"); + assert!(error.contains("was removed with the Docker worker")); + for refused in ["in-memory", "in_memory", "memory", "bogus", "sandbox-2"] { let error = parse_lane_transport_kind(refused) .expect_err("the simulator and unknown names must be refused"); assert!(error.contains("unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT")); @@ -5447,7 +5581,32 @@ mod tests { fn host_agent_transport_requires_an_explicit_opt_in() { assert!(check_host_agent_opt_in(LaneTransportKind::ToolsAgent, false).is_err()); assert!(check_host_agent_opt_in(LaneTransportKind::ToolsAgent, true).is_ok()); - assert!(check_host_agent_opt_in(LaneTransportKind::LocalDocker, false).is_ok()); + assert!(check_host_agent_opt_in(LaneTransportKind::Sandbox, false).is_ok()); + } + + #[test] + fn server_config_reads_sandbox_env_vars() { + use super::{ + default_sandbox_base_url, parse_lane_transport_kind, LaneTransportKind, ServerConfig, + }; + // The sandbox server settings and the lane transport selection must parse + // out of the environment the way an operator would set them. + assert!(default_sandbox_base_url().ends_with(":8080")); + assert_eq!( + parse_lane_transport_kind("sandbox").unwrap(), + LaneTransportKind::Sandbox + ); + std::env::set_var("FRONTAL_SANDBOX_BASE_URL", "http://sandbox.test:8080/"); + std::env::set_var("FRONTAL_SANDBOX_API_KEY", "sbx_test-key"); + std::env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", "sandbox"); + let config = + ServerConfig::from_env().expect("sandbox env vars should parse into the config"); + assert_eq!(config.sandbox_base_url, "http://sandbox.test:8080"); + assert_eq!(config.sandbox_api_key, "sbx_test-key"); + assert_eq!(config.lane_transport_kind, LaneTransportKind::Sandbox); + std::env::remove_var("FRONTAL_SANDBOX_BASE_URL"); + std::env::remove_var("FRONTAL_SANDBOX_API_KEY"); + std::env::remove_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT"); } use super::*; @@ -5765,36 +5924,6 @@ mod tests { } } - #[derive(Debug, Default)] - struct MockDockerRunner { - launched: Mutex>, - stopped: Mutex>, - fail_launch: Option, - } - - impl DockerRunner for MockDockerRunner { - fn launch(&self, spec: &DockerLaunchSpec) -> Result { - if let Some(error) = &self.fail_launch { - return Err(error.clone()); - } - self.launched - .lock() - .expect("mock docker launch lock poisoned") - .push(spec.clone()); - Ok(DockerLaunchResult { - container_id: "container-localdocker-1".to_string(), - }) - } - - fn stop(&self, container_id: &str) -> Result<(), String> { - self.stopped - .lock() - .expect("mock docker stop lock poisoned") - .push(container_id.to_string()); - Ok(()) - } - } - fn temp_test_dir(prefix: &str) -> PathBuf { // Timestamp alone collides when parallel tests read the same instant. static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); @@ -5810,43 +5939,6 @@ mod tests { dir } - fn init_git_repo(path: PathBuf) -> PathBuf { - fs::create_dir_all(&path).unwrap(); - run_git(&path, ["init", "-b", "main"]); - run_git(&path, ["config", "user.name", "FrontalCode Server Tests"]); - run_git( - &path, - ["config", "user.email", "code-server-tests@example.com"], - ); - path - } - - fn commit_file(repo: &std::path::Path, relative_path: &str, contents: &str, message: &str) { - let file_path = repo.join(relative_path); - if let Some(parent) = file_path.parent() { - fs::create_dir_all(parent).unwrap(); - } - fs::write(file_path, contents).unwrap(); - run_git(repo, ["add", relative_path]); - run_git(repo, ["commit", "-m", message]); - } - - fn run_git(repo: &std::path::Path, args: [&str; N]) { - let output = std::process::Command::new("git") - .arg("-C") - .arg(repo) - .args(args) - .output() - .unwrap(); - assert!( - output.status.success(), - "git command failed: git -C {} {}: {}", - repo.display(), - args.join(" "), - String::from_utf8_lossy(&output.stderr) - ); - } - fn write_hosted_agent_manifest( path: &std::path::Path, output_path: &std::path::Path, @@ -5978,7 +6070,7 @@ mod tests { assert_eq!(snapshot["repo_url"], "https://github.com/acme/repo-a.git"); assert_eq!(snapshot["base_ref"], "main"); assert_eq!(snapshot["branch"], "frontal-code/investigate-flake"); - assert_eq!(snapshot["execution_backend"], "local_docker"); + assert_eq!(snapshot["execution_backend"], "sandbox"); assert_eq!(snapshot["lane_id"].as_str().map(str::is_empty), Some(false)); assert_eq!( snapshot["plan_kind"].as_str().map(str::is_empty), @@ -6035,7 +6127,7 @@ mod tests { .payload .as_ref() .and_then(|payload| payload.get("execution_backend")), - Some(&json!("local_docker")) + Some(&json!("sandbox")) ); assert_eq!( created_event @@ -6092,307 +6184,6 @@ mod tests { .is_some()); } - #[tokio::test] - #[allow(clippy::await_holding_lock, clippy::too_many_lines)] - async fn local_docker_transport_prepares_checkout_workspace_for_task() { - let Some(store) = worker_tokens::tests::test_store() else { - return; - }; - let dir = temp_test_dir("code-server-local-docker"); - let source_repo = init_git_repo(dir.join("source")); - commit_file( - &source_repo, - "README.md", - "hello from local docker transport\n", - "initial commit", - ); - let workspace_root = dir.join("workspaces"); - let docker = Arc::new(MockDockerRunner::default()); - let access = Arc::new(WorkerAccess::new(Some(store.clone()), 3600)); - let state = Arc::new( - ServerState::with_lane_transport_kind_and_access( - DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, - Arc::new(LocalDockerLaneWorkerTransport::with_runner( - workspace_root.clone(), - "frontal-code-worker:test".to_string(), - "http://host.docker.internal:8788", - access.clone(), - docker.clone(), - )), - access, - ) - .with_local_repos_allowed(true), - ); - let router = app(state.clone()); - - let create_response = router - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/tasks") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "prompt": "Prepare checkout before running", - "repository": "acme/local-docker", - "repo_url": source_repo.display().to_string(), - "base_ref": "main", - "branch": "frontal-code/task-123", - "source": "api" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(create_response.status(), StatusCode::OK); - let created_body = to_bytes(create_response.into_body(), usize::MAX) - .await - .unwrap(); - let created: serde_json::Value = serde_json::from_slice(&created_body).unwrap(); - let task_id = created["task_id"].as_str().unwrap().to_string(); - - let context = state - .context_for(&task_id) - .expect("context should be recorded for local docker task"); - let checkout_root = PathBuf::from( - context - .checkout_root - .clone() - .expect("local docker task should record checkout root"), - ); - assert!(checkout_root.starts_with(&workspace_root)); - assert!(checkout_root.join(".git").exists()); - assert_eq!( - fs::read_to_string(checkout_root.join("README.md")).unwrap(), - "hello from local docker transport\n" - ); - assert_eq!(context.execution_backend.as_deref(), Some("local_docker")); - - let get_response = router - .clone() - .oneshot( - Request::builder() - .uri(format!("/v1/tasks/{task_id}")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = to_bytes(get_response.into_body(), usize::MAX) - .await - .unwrap(); - let snapshot: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(snapshot["execution_backend"], "local_docker"); - assert_eq!(snapshot["worker_id"], "container-localdocker-1"); - - let started_event = state - .replay_events() - .into_iter() - .find(|event| event.event == HostedEventName::LaneStarted) - .expect("lane started event should be emitted"); - assert_eq!( - started_event - .payload - .as_ref() - .and_then(|payload| payload.get("transport")) - .and_then(|transport| transport.get("checkout_root")), - Some(&json!(checkout_root.display().to_string())) - ); - assert_eq!( - started_event - .payload - .as_ref() - .and_then(|payload| payload.get("transport")) - .and_then(|transport| transport.get("container_id")), - Some(&json!("container-localdocker-1")) - ); - - let launched = docker - .launched - .lock() - .expect("mock docker launch lock poisoned"); - assert_eq!(launched.len(), 1); - assert_eq!(launched[0].image, "frontal-code-worker:test"); - assert_eq!(launched[0].task_id, task_id); - assert_eq!(launched[0].checkout_root, checkout_root); - assert_eq!( - launched[0].command, - vec![ - "frontal-code".to_string(), - "hosted".to_string(), - "task".to_string(), - "run".to_string(), - task_id.clone(), - ] - ); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_SERVER_URL") - .map(String::as_str), - Some("http://frontal-relay:8788") - ); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_HOSTED_TASK_FILE") - .map(String::as_str), - Some("/workspace/.frontal-code-hosted/task.json") - ); - // The worker authenticates with a per-task token, never the server's own key. - let worker_token = launched[0] - .env - .get("FRONTAL_CODE_SERVER_API_KEY") - .cloned() - .expect("worker token"); - assert_eq!(worker_token.len(), 64); - assert!(worker_token.chars().all(|c| c.is_ascii_hexdigit())); - // No upstream provider key reaches the container. The model traffic goes to the proxy. - assert!(!launched[0].env.keys().any(|key| key.ends_with("_API_KEY") - && key != "FRONTAL_CODE_SERVER_API_KEY" - && launched[0].env.get(key).map(String::as_str) != Some(worker_token.as_str()))); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_SERVER_URL") - .map(String::as_str), - Some("http://frontal-relay:8788") - ); - - let task_payload: HostedDockerTaskPayload = serde_json::from_slice( - &fs::read(checkout_root.join(".frontal-code-hosted").join("task.json")).unwrap(), - ) - .unwrap(); - assert_eq!(task_payload.task_id, task_id); - assert_eq!(task_payload.prompt, "Prepare checkout before running"); - assert_eq!( - task_payload.repository.as_deref(), - Some("acme/local-docker") - ); - assert_eq!( - task_payload.repo_url.as_deref(), - Some(source_repo.to_str().unwrap()) - ); - assert_eq!(task_payload.base_ref.as_deref(), Some("main")); - assert_eq!( - task_payload.branch.as_deref(), - Some("frontal-code/task-123") - ); - assert_eq!(task_payload.allowed_tools, Vec::::new()); - - let complete_response = router - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(format!("/v1/tasks/{task_id}/complete")) - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "finish_reason": "stop", - "tokens_output": 64, - "result": "local docker run completed" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(complete_response.status(), StatusCode::OK); - - let _ = fs::remove_dir_all(dir); - } - - #[tokio::test] - async fn local_docker_transport_launch_failure_marks_task_failed() { - let Some(store) = worker_tokens::tests::test_store() else { - return; - }; - let dir = temp_test_dir("code-server-local-docker-fail"); - let source_repo = init_git_repo(dir.join("source")); - commit_file( - &source_repo, - "README.md", - "failure path\n", - "initial commit", - ); - let workspace_root = dir.join("workspaces"); - let docker = Arc::new(MockDockerRunner { - fail_launch: Some("docker launch spec rejected".to_string()), - ..MockDockerRunner::default() - }); - let access = Arc::new(WorkerAccess::new(Some(store.clone()), 3600)); - let state = Arc::new( - ServerState::with_lane_transport_kind_and_access( - DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, - Arc::new(LocalDockerLaneWorkerTransport::with_runner( - workspace_root, - "frontal-code-worker:test".to_string(), - "http://host.docker.internal:8788", - access.clone(), - docker, - )), - access, - ) - .with_local_repos_allowed(true), - ); - let router = app(state.clone()); - - let create_response = router - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/tasks") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "prompt": "Prepare checkout before running", - "repository": "acme/local-docker", - "repo_url": source_repo.display().to_string(), - "base_ref": "main", - "branch": "frontal-code/task-123", - "source": "api" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(create_response.status(), StatusCode::OK); - let body = to_bytes(create_response.into_body(), usize::MAX) - .await - .unwrap(); - let snapshot: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(snapshot["status"], "failed"); - assert_eq!(snapshot["worker_status"], "failed"); - let task = state - .tasks - .list(None) - .into_iter() - .next() - .expect("task should still exist after launch failure"); - assert_eq!(task.status, TaskStatus::Failed); - assert!(task.output.contains("docker launch spec rejected")); - - let failed_event = state - .replay_events() - .into_iter() - .find(|event| event.event == HostedEventName::LaneFailed) - .expect("lane failed event should be emitted"); - assert_eq!(failed_event.status, HostedEventStatus::Failed); - - let _ = fs::remove_dir_all(dir); - } - #[tokio::test] async fn update_task_context_persists_slack_thread_anchor() { let dir = temp_test_dir("code-server-context-update"); @@ -7120,7 +6911,7 @@ mod tests { let state = Arc::new( ServerState::with_lane_transport_kind_and_access( DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, Arc::new(FakeLaneWorkerTransport::new()), Arc::new(WorkerAccess::new(Some(store), 3600)), ) @@ -9389,7 +9180,7 @@ mod tests { let state = Arc::new( ServerState::new_with_transport_kind_state_file_policy_and_workspace_root( 10, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, None, None, Duration::from_secs(0), @@ -9397,8 +9188,6 @@ mod tests { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ), ); @@ -9433,7 +9222,7 @@ mod tests { let state = Arc::new( ServerState::new_with_transport_kind_state_file_policy_and_workspace_root( 10, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, None, None, Duration::from_secs(0), @@ -9441,8 +9230,6 @@ mod tests { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ), ); @@ -10031,7 +9818,7 @@ mod tests { let restored = ServerState::new_with_transport_kind_and_state_file( DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, Some(state_file), ); let restored_task = restored.tasks.get(&task_id).expect("task should reload"); @@ -10128,7 +9915,7 @@ mod tests { ); assert_eq!( restored_context.execution_backend.as_deref(), - Some("local_docker") + Some("sandbox") ); assert_eq!(restored_context.source.as_deref(), Some("api")); assert_eq!(restored_context.worker_status.as_deref(), Some("running")); @@ -10157,7 +9944,7 @@ mod tests { ); assert_eq!(snapshot["base_ref"], "develop"); assert_eq!(snapshot["branch"], "frontal-code/persisted-work"); - assert_eq!(snapshot["execution_backend"], "local_docker"); + assert_eq!(snapshot["execution_backend"], "sandbox"); assert_eq!(snapshot["worker_status"], "running"); let _ = fs::remove_dir_all(dir); @@ -11030,53 +10817,6 @@ mod tests { ); } - #[test] - fn server_config_parses_local_docker_lane_transport_kind() { - let previous = env::var_os("FRONTAL_CODE_SERVER_LANE_TRANSPORT"); - let previous_workspace_root = env::var_os("FRONTAL_CODE_SERVER_WORKSPACE_ROOT"); - let previous_docker_image = env::var_os("FRONTAL_CODE_SERVER_DOCKER_IMAGE"); - let previous_callback_url = env::var_os("FRONTAL_CODE_SERVER_CALLBACK_URL"); - env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", "local-docker"); - env::set_var( - "FRONTAL_CODE_SERVER_WORKSPACE_ROOT", - "/tmp/code-server-workspaces", - ); - env::set_var( - "FRONTAL_CODE_SERVER_DOCKER_IMAGE", - "frontal-code-worker:test", - ); - env::set_var( - "FRONTAL_CODE_SERVER_CALLBACK_URL", - "http://docker.internal.test:8788/", - ); - - let config = ServerConfig::from_env().expect("env config should parse"); - assert_eq!(config.lane_transport_kind, LaneTransportKind::LocalDocker); - assert_eq!( - config.workspace_root, - PathBuf::from("/tmp/code-server-workspaces") - ); - assert_eq!(config.docker_image, "frontal-code-worker:test"); - assert_eq!(config.docker_server_url, "http://docker.internal.test:8788"); - - match previous { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", value), - None => env::remove_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT"), - } - match previous_workspace_root { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_WORKSPACE_ROOT", value), - None => env::remove_var("FRONTAL_CODE_SERVER_WORKSPACE_ROOT"), - } - match previous_docker_image { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_DOCKER_IMAGE", value), - None => env::remove_var("FRONTAL_CODE_SERVER_DOCKER_IMAGE"), - } - match previous_callback_url { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_CALLBACK_URL", value), - None => env::remove_var("FRONTAL_CODE_SERVER_CALLBACK_URL"), - } - } - #[test] fn hosted_task_context_builds_repo_checkout_request() { let context = HostedTaskContext { @@ -11148,45 +10888,3 @@ mod tests { assert_eq!(payload.get("pr_number"), Some(&json!(42))); } } - -#[cfg(test)] -mod callback_port_tests { - use super::*; - - fn config(url: &str, bind_port: u16) -> ServerConfig { - ServerConfig { - lane_transport_kind: LaneTransportKind::LocalDocker, - docker_server_url: url.to_string(), - bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), bind_port), - ..ServerConfig::default() - } - } - - #[test] - fn a_callback_url_on_the_listening_port_is_accepted() { - assert!(check_callback_port(&config("http://host.docker.internal:8790", 8790)).is_ok()); - } - - #[test] - fn a_callback_url_without_a_port_is_refused() { - let error = - check_callback_port(&config("http://host.docker.internal", 8790)).expect_err("no port"); - assert!(error.contains("must include the port")); - } - - #[test] - fn a_callback_url_on_another_port_is_refused() { - let error = check_callback_port(&config("http://host.docker.internal:9000", 8790)) - .expect_err("wrong port"); - assert!(error.contains("names port 9000")); - } - - #[test] - fn the_check_does_not_apply_to_other_transports() { - let config = ServerConfig { - lane_transport_kind: LaneTransportKind::ToolsAgent, - ..config("http://elsewhere", 8790) - }; - assert!(check_callback_port(&config).is_ok()); - } -} diff --git a/crates/code-server/tests/server_config.rs b/crates/code-server/tests/server_config.rs index f1e99841..e438239a 100644 --- a/crates/code-server/tests/server_config.rs +++ b/crates/code-server/tests/server_config.rs @@ -11,39 +11,30 @@ fn default_server_config() { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8788) ); assert_eq!(config.event_replay_limit, 100); - assert_eq!(config.lane_transport_kind, LaneTransportKind::LocalDocker); + assert_eq!(config.lane_transport_kind, LaneTransportKind::Sandbox); assert!(config.api_key.is_none()); assert_eq!(config.reconcile_interval, Some(Duration::from_secs(15))); } #[test] fn lane_transport_kind_default() { - assert_eq!( - LaneTransportKind::LocalDocker, - LaneTransportKind::LocalDocker - ); + assert_eq!(LaneTransportKind::Sandbox, LaneTransportKind::Sandbox); assert_eq!(LaneTransportKind::ToolsAgent, LaneTransportKind::ToolsAgent); } #[test] fn lane_transport_kind_debug() { - assert_eq!( - format!("{:?}", LaneTransportKind::LocalDocker), - "LocalDocker" - ); + assert_eq!(format!("{:?}", LaneTransportKind::Sandbox), "Sandbox"); assert_eq!(format!("{:?}", LaneTransportKind::ToolsAgent), "ToolsAgent"); } #[test] fn lane_transport_kind_clone_and_eq() { - let a = LaneTransportKind::LocalDocker; + let a = LaneTransportKind::Sandbox; let b = a; assert_eq!(a, b); - let kinds = [ - LaneTransportKind::LocalDocker, - LaneTransportKind::ToolsAgent, - ]; + let kinds = [LaneTransportKind::Sandbox, LaneTransportKind::ToolsAgent]; for i in 0..kinds.len() { for j in 0..kinds.len() { if i == j { diff --git a/crates/code-server/tests/server_types.rs b/crates/code-server/tests/server_types.rs index 27345c74..7d4efa69 100644 --- a/crates/code-server/tests/server_types.rs +++ b/crates/code-server/tests/server_types.rs @@ -139,7 +139,7 @@ fn hosted_task_snapshot_serialize() { pr_head_ref: None, pr_base_ref: None, metadata: std::collections::HashMap::new(), - execution_backend: Some("local_docker".to_string()), + execution_backend: Some("sandbox".to_string()), priority: None, plan_id: Some("plan-1".to_string()), plan_kind: Some("implementation".to_string()), @@ -154,7 +154,7 @@ fn hosted_task_snapshot_serialize() { assert_eq!(json["created_at"], 1000); assert_eq!(json["prompt"], "Do something"); assert_eq!(json["source"], "api"); - assert_eq!(json["execution_backend"], "local_docker"); + assert_eq!(json["execution_backend"], "sandbox"); let deserialized: HostedTaskSnapshot = serde_json::from_value(json).unwrap(); assert_eq!(deserialized.task_id, "task-123"); diff --git a/deny.toml b/deny.toml index b68ad531..4658388a 100644 --- a/deny.toml +++ b/deny.toml @@ -45,3 +45,5 @@ allow-wildcard-paths = true unknown-registry = "deny" unknown-git = "deny" allow-registry = ["https://github.com/rust-lang/crates.io-index"] +# Pinned frontend for the hosted worker; the commit is locked in Cargo.lock. +allow-git = ["https://github.com/frontal-labs/sandbox"] diff --git a/deploy/sandbox/Makefile b/deploy/sandbox/Makefile new file mode 100644 index 00000000..e37eae2d --- /dev/null +++ b/deploy/sandbox/Makefile @@ -0,0 +1,67 @@ +# Frontal Sandbox deployment. +# +# Uses OpenTofu (`tofu`). If you only have Terraform, set `TF := terraform`. +# +# smoke requires: SANDBOX_BASE_URL, SANDBOX_API_KEY +# bootstrap requires: BOOTSTRAP_HOST, SANDBOX_BINARY_URL, SANDBOX_API_KEY_HASH +# (BOOTSTRAP_USER defaults to ubuntu; SANDBOX_CAS_S3_BUCKET / +# SANDBOX_CAS_S3_ENDPOINT are optional) + +TF ?= tofu +TF_DIR := terraform + +.PHONY: init apply destroy smoke bootstrap plan validate fmt + +## Terraform lifecycle + +init: + $(TF) -chdir=$(TF_DIR) init + +plan: + $(TF) -chdir=$(TF_DIR) plan + +apply: + $(TF) -chdir=$(TF_DIR) apply + +destroy: + $(TF) -chdir=$(TF_DIR) destroy + +validate: + $(TF) -chdir=$(TF_DIR) validate + +fmt: + $(TF) -chdir=$(TF_DIR) fmt -recursive + +## Outputs + +host_ip: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_host_ip + +host_dns: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_host_dns + +base_url: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_base_url + +bucket: + $(TF) -chdir=$(TF_DIR) output -raw s3_bucket_name + +## Host bootstrap +## Piped over SSH: SANDBOX_BINARY_URL, SANDBOX_API_KEY_HASH required; +## SANDBOX_CAS_S3_BUCKET / SANDBOX_CAS_S3_ENDPOINT optional. + +bootstrap: + @test -n "$${BOOTSTRAP_HOST:?BOOTSTRAP_HOST is required (see `make host_ip`)}" + @test -n "$${SANDBOX_BINARY_URL:?SANDBOX_BINARY_URL is required}" + @test -n "$${SANDBOX_API_KEY_HASH:?SANDBOX_API_KEY_HASH is required}" + ssh "$${BOOTSTRAP_USER:-ubuntu}@$${BOOTSTRAP_HOST}" \ + 'SANDBOX_BINARY_URL="$$SANDBOX_BINARY_URL" \ + SANDBOX_API_KEY_HASH="$$SANDBOX_API_KEY_HASH" \ + SANDBOX_CAS_S3_BUCKET="$$SANDBOX_CAS_S3_BUCKET" \ + SANDBOX_CAS_S3_ENDPOINT="$$SANDBOX_CAS_S3_ENDPOINT" \ + bash -s' < scripts/bootstrap.sh + +## Smoke test + +smoke: + @scripts/smoke.sh diff --git a/deploy/sandbox/README.md b/deploy/sandbox/README.md new file mode 100644 index 00000000..0257b0d8 --- /dev/null +++ b/deploy/sandbox/README.md @@ -0,0 +1,158 @@ +# Frontal Sandbox — deployment wrapper + +A turnkey deployment for the [Frontal Sandbox server](https://github.com/frontal-labs/sandbox): +it provisions a KVM-capable host plus its object store (CAS tier) with OpenTofu, +then installs and runs the sandbox server that code-server's new +`SandboxLaneWorkerTransport` talks to over HTTP (port 8080). + +The sandbox server listens on port 8080 (API + `/metrics`) and needs `/dev/kvm` +for Tier 2 (VM) sessions. It is configured via a TOML file at +`/etc/sandbox/config.toml` with env var overrides (`SANDBOX_
_`). + +The upstream sandbox repo also ships its own `infrastructure/terraform/` +module (AWS/GCP/Azure) and a clustered deployment. This directory is a +**self-contained single-node deployment** — see +[terraform/README.md](terraform/README.md) for the relationship. + +## Layout + +``` +deploy/sandbox/ +├── README.md +├── Makefile +├── config/ +│ └── production.toml # sandbox server TOML (installed to /etc/sandbox/config.toml) +├── scripts/ +│ ├── bootstrap.sh # prerequisites, KVM, binary, config, systemd unit +│ ├── deploy.sh # restart + verify on an already-bootstrapped host +│ └── smoke.sh # API smoke test (openapi → create → exec → destroy) +└── terraform/ + ├── main.tf # terraform block, required_providers, locals + ├── providers.tf # AWS provider + ├── variables.tf # inputs + ├── outputs.tf # outputs + ├── networking.tf # VPC, public + private subnets, IGW, NAT + ├── compute.tf # EC2 instance + EBS data volume + ├── firewall.tf # security group (SSH 22, API 8080, scheduler 50051) + └── storage.tf # S3 CAS bucket (versioned, encrypted, private) +``` + +## Prerequisites + +- [OpenTofu](https://opentofu.org) (`tofu`) on PATH — Terraform ≥ 1.5 works too + (the Makefile uses `tofu`; set `TF=terraform` to switch). +- Cloud provider credentials (AWS: `AWS_PROFILE` or `AWS_ACCESS_KEY_ID` / + `AWS_SECRET_ACCESS_KEY` env vars) with EC2, VPC, EBS, and S3 permissions. +- An SSH key pair registered in EC2 (pass its name via `-var ssh_key_name=...`). +- An AMI ID for the host (Ubuntu 22.04/24.04): + + ```sh + aws ec2 describe-images --owners 099720109477 \ + --filters Name=name,values='ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*' \ + Name=root-device-type,values=ebs --query 'Images[0].ImageId' --output text + ``` + +## Quick start + +From `deploy/sandbox/`, five steps to a running sandbox server: + +```sh +# 1. Initialize (installs the AWS provider) +cd deploy/sandbox && tofu -chdir=terraform init + +# 2. Provision the KVM host + object store + firewall. +# Provide at least: ami_id, ssh_key_name. Tighten allow_cidr_blocks. +tofu -chdir=terraform apply -auto-approve \ + -var ami_id=ami-0... \ + -var ssh_key_name=my-key \ + -var 'allow_cidr_blocks=["203.0.113.0/24"]' + +# 3. Get onto the host +ssh ubuntu@$(tofu -chdir=terraform output -raw sandbox_host_ip) sudo -i + +# 4. Install the binary, config, and systemd unit, then start +export SANDBOX_BINARY_URL="https://.../sandbox-x86_64-unknown-linux-gnu" +# optional: export SANDBOX_API_KEY_HASH=$(printf '%s' "$FRONTAL_SANDBOX_API_KEY" | shasum -a 256) +./scripts/bootstrap.sh && ./scripts/deploy.sh + +# 5. Verify the server is healthy +./scripts/smoke.sh +``` + +> Note: the instance lives in a **private subnet**. `sandbox_host_ip` may be +> empty; reach the host over your VPN/bastion, or add an Elastic IP to make +> it reachable. The security group allows inbound 8080 only from +> `allow_cidr_blocks`. + +## Pointing code-server at the sandbox server + +In code-server's environment (e.g. `.env`): + +```sh +FRONTAL_SANDBOX_BASE_URL=http://:8080 # `tofu -chdir=terraform output -raw sandbox_base_url` +FRONTAL_SANDBOX_API_KEY=sbx_... # your sandbox API key +FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox +``` + +The server authenticates with the SHA-256 hex hash of the API key. Pass the +hash (never the raw key) to `bootstrap.sh` as `SANDBOX_API_KEY_HASH`; it is +injected into the systemd unit as `SANDBOX_AUTH_API_KEY_HASH`. + +## Destroy + +```sh +cd deploy/sandbox +tofu -chdir=terraform destroy -auto-approve +``` + +Removes the EC2 instance, EBS volumes, S3 bucket, and all networking. +Anything under `/var/lib/sandbox` is gone with the instance; CAS objects are +gone with the bucket. Back up anything you need first. + +## Configuration reference + +The sandbox server reads `/etc/sandbox/config.toml`. The production defaults +live in [`config/production.toml`](config/production.toml); every key can be +overridden with a `SANDBOX_
_` env var (e.g. +`SANDBOX_AUTH_RATE_LIMIT_RPS`). + +| Section | Key | Description | +| --- | --- | --- | +| `server` | `addr` | API listen address (`0.0.0.0:8080`) | +| `server` | `data_dir` | State / CAS data directory (`/var/lib/sandbox`) | +| `auth` | `api_key_hash` | SHA-256 hex of the API key. Keep in `SANDBOX_AUTH_API_KEY_HASH` env var, never in the TOML. | +| `auth` | `rate_limit_rps` | API rate limit (requests/sec) | +| `tls` | `enabled` | TLS on the sandbox server. Not supported yet (v0.x) — terminate TLS at a reverse proxy. | +| `tls` | `cert_file` / `key_file` | Certificate / key paths (when `enabled = true`) | +| `telemetry` | `metrics_addr` | Prometheus metrics endpoint (`0.0.0.0:9090`) | +| `telemetry` | `otlp_endpoint` | Optional OTLP endpoint | +| `telemetry` | `service_name` | Service name reported by telemetry | +| `scheduler` | `address` | Scheduler gRPC listen address | + +## Troubleshooting + +- **KVM not available** (`/dev/kvm` missing): the instance type must support + KVM — on AWS that means bare metal (e.g. `m5.metal`) or a provider exposing + nested virtualization. Check with `ls -l /dev/kvm` and + `dmesg | grep -i kvm`. Tier 1 (container) sessions work without KVM; only + Tier 2 (VM) sessions need it. `bootstrap.sh` warns but continues when + `/dev/kvm` is absent. +- **Port 8080 blocked**: the security group only allows 8080 from + `allow_cidr_blocks`. Make sure the code-server's egress CIDR is in the list + and that no corporate firewall in between drops it. Test from the + code-server host: `curl -v http://:8080/v1/openapi.json`. +- **SSH from anywhere (port 22)**: the default `allow_cidr_blocks` is + `0.0.0.0/0`, which opens SSH to the world. Replace it with your operator + CIDR before applying anywhere real. +- **`smoke.sh` fails to create a session**: verify + `SANDBOX_API_KEY` (the key `smoke.sh` sends) matches the hash installed on the server + (`SANDBOX_AUTH_API_KEY_HASH` in `/etc/systemd/system/sandbox-server.service`). It must + equal the `FRONTAL_SANDBOX_API_KEY` that code-server uses as its caller key. + See `journalctl -u sandbox-server -n 100` on the host. +- **Service not active**: `systemctl status sandbox-server` and + `journalctl -u sandbox-server -n 100`. The most common causes are a bad + `SANDBOX_BINARY_URL` download (check `/usr/local/bin/sandbox` runs + `--version`) and a stale config (compare `/etc/sandbox/config.toml` with + `config/production.toml`). +- **Instance unreachable**: it lives in a private subnet with NAT-only + egress; inbound traffic requires your VPN/bastion or an added Elastic IP. diff --git a/deploy/sandbox/config/production.toml b/deploy/sandbox/config/production.toml new file mode 100644 index 00000000..645193c0 --- /dev/null +++ b/deploy/sandbox/config/production.toml @@ -0,0 +1,22 @@ +# Frontal Sandbox server configuration +# Installed to /etc/sandbox/config.toml by the deploy script. + +[server] +addr = "0.0.0.0:8080" +data_dir = "/var/lib/sandbox" + +[auth] +# Set via SANDBOX_AUTH_API_KEY_HASH env var in the systemd unit +rate_limit_rps = 100 + +[tls] +enabled = false +# When deploying behind a load balancer, terminate TLS there and leave +# the sandbox server plaintext on a private subnet. + +[telemetry] +metrics_addr = "0.0.0.0:9090" +service_name = "sandbox-server" + +[scheduler] +address = "0.0.0.0:50051" diff --git a/deploy/sandbox/scripts/bootstrap.sh b/deploy/sandbox/scripts/bootstrap.sh new file mode 100755 index 00000000..08248bb0 --- /dev/null +++ b/deploy/sandbox/scripts/bootstrap.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# +# bootstrap.sh — install the Frontal Sandbox server on the target host. +# +# Runs on the sandbox host. Idempotent: safe to re-run after a rebuild. +# +# Required environment: +# SANDBOX_BINARY_URL direct URL to the sandbox binary (no .tar.gz), or a +# "v" release tarball of the form +# sandbox--.tar.gz (extracted to +# /usr/local/bin/sandbox) +# +# Optional environment: +# SANDBOX_API_KEY_HASH SHA-256 (hex) of the API key code-server presents +# (printf '%s' "$KEY" | shasum -a 256). When set, it +# is injected into the systemd unit as +# SANDBOX_AUTH_API_KEY_HASH. +# SANDBOX_CONFIG_SRC local path to the TOML config to install +# (default: