From 8bd76ee751882106513bae43cf14c4337b512e84 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Mon, 16 Nov 2026 02:43:22 +0000 Subject: [PATCH 01/10] fix(integrations): apply cargo fmt and biome formatting - cargo fmt across code-integrations, code-server, cli, rust sdk - biome import sorting in sdks/typescript/src/index.ts --- Cargo.lock | 2 + bun.lock | 2 +- cli/src/integrations.rs | 764 +++++++++++++++++++++ cli/src/main.rs | 97 +++ crates/code-integrations/src/lib.rs | 11 + crates/code-integrations/src/manifest.rs | 535 +++++++++++++++ crates/code-integrations/src/mcp/tools.rs | 3 +- crates/code-integrations/src/providers.rs | 417 +++++++++++ crates/code-server/src/integrations.rs | 601 ++++++++++++++++ crates/code-server/src/lib.rs | 11 + docs/API_REFERENCES.md | 53 ++ docs/INTEGRATIONS.md | 274 ++++++++ docs/MCP.md | 17 +- sdks/rust/Cargo.toml | 2 + sdks/rust/src/integrations.rs | 330 +++++++++ sdks/rust/src/lib.rs | 5 + sdks/typescript/src/index.ts | 8 + sdks/typescript/src/integrations.ts | 179 +++++ sdks/typescript/tests/integrations.test.ts | 57 ++ 19 files changed, 3360 insertions(+), 8 deletions(-) create mode 100644 cli/src/integrations.rs create mode 100644 crates/code-integrations/src/manifest.rs create mode 100644 crates/code-integrations/src/providers.rs create mode 100644 crates/code-server/src/integrations.rs create mode 100644 docs/INTEGRATIONS.md create mode 100644 sdks/rust/src/integrations.rs create mode 100644 sdks/typescript/src/integrations.ts create mode 100644 sdks/typescript/tests/integrations.test.ts diff --git a/Cargo.lock b/Cargo.lock index e47d037c..ee7187cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1235,6 +1235,8 @@ name = "frontal-code-sdk" version = "0.3.0" dependencies = [ "futures-util", + "reqwest", + "serde", "serde_json", "tempfile", "thiserror 1.0.69", diff --git a/bun.lock b/bun.lock index 73bd784a..d9594ee0 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ }, "sdks/typescript": { "name": "@frontal-labs/code-sdk", - "version": "0.1.0", + "version": "0.2.0", "devDependencies": { "@frontal-labs/code-zod-schemas": "file:../../extensions/zod", "@types/node": "^20.12.0", diff --git a/cli/src/integrations.rs b/cli/src/integrations.rs new file mode 100644 index 00000000..24af2dd7 --- /dev/null +++ b/cli/src/integrations.rs @@ -0,0 +1,764 @@ +//! `frontal-code integrations` subcommand handlers. +//! +//! Loads typed [`IntegrationManifest`]s from the project settings +//! (`.frontal-code/settings.json` → `mcp.integrations`) and the +//! `FRONTAL_CODE_INTEGRATIONS_CONFIG` environment variable, then renders +//! them for the CLI. + +use std::path::Path; + +use code_integrations::mcp::config::McpTransport; +use code_integrations::mcp::tools::PermissionMode; +use code_integrations::{ + AuthSpec, CapabilitySpec, EventSpec, IntegrationManifest, ManifestRegistry, SignatureSpec, + TransportConfig, ValidationIssue, +}; +use serde_json::{json, Value}; + +/// Environment variable holding a JSON array of integration manifests. +pub const ENV_INTEGRATIONS_CONFIG: &str = "FRONTAL_CODE_INTEGRATIONS_CONFIG"; + +/// Subcommand help text shown with no arguments or an unknown subcommand. +#[must_use] +pub fn help_text() -> &'static str { + "Usage: frontal-code integrations [options] + +Subcommands: + list List all configured integrations + show Show full details for an integration + capabilities List the capabilities of an integration + validate Validate a manifest file + +Options: + --json Emit JSON instead of a table" +} + +/// Load the manifest registry: the built-in providers (GitHub, Linear, +/// `PostHog`, and Sentry) as defaults, layered with settings discovered by +/// walking up from `cwd` and the `FRONTAL_CODE_INTEGRATIONS_CONFIG` env var. +/// +/// Configuration entries with the same id as a built-in override it; entries +/// that fail to parse are skipped (invalid entries cannot be registered +/// anyway). +pub fn load_registry(cwd: &Path) -> ManifestRegistry { + let mut registry = + code_integrations::builtin_providers().expect("built-in provider manifests are valid"); + + if let Some(settings) = find_settings_file(cwd) { + if let Ok(contents) = std::fs::read_to_string(&settings) { + let Ok(parsed) = serde_json::from_str::(&contents) else { + return registry; + }; + if let Some(integrations) = parsed + .get("mcp") + .and_then(|mcp| mcp.get("integrations")) + .and_then(|value| value.as_object()) + { + for (_, manifest_json) in integrations { + let Ok(manifest) = + serde_json::from_value::(manifest_json.clone()) + else { + continue; + }; + register_or_override(&mut registry, manifest); + } + } + } + } + + if let Ok(raw) = std::env::var(ENV_INTEGRATIONS_CONFIG) { + let Ok(entries) = serde_json::from_str::>(&raw) else { + return registry; + }; + for manifest in entries { + register_or_override(&mut registry, manifest); + } + } + + registry +} + +/// Register a manifest, replacing any existing manifest with the same id so +/// that project configuration can override a built-in provider. +fn register_or_override(registry: &mut ManifestRegistry, manifest: IntegrationManifest) { + let id = manifest.id.clone(); + if registry.get(&id).is_some() { + registry.remove(&id); + } + let _ = registry.register(manifest); +} + +/// Walk up from `cwd` (inclusive) looking for `.frontal-code/settings.json`. +fn find_settings_file(cwd: &Path) -> Option { + let mut current: Option<&Path> = Some(cwd); + while let Some(dir) = current { + let candidate = dir.join(".frontal-code").join("settings.json"); + if candidate.is_file() { + return Some(candidate); + } + current = dir.parent(); + } + None +} + +/// Summary of one integration for table/JSON rendering. +#[derive(serde::Serialize)] +struct IntegrationSummary { + id: String, + version: String, + display_name: String, + capability_count: usize, + auth: String, +} + +fn summarize(manifest: &IntegrationManifest) -> IntegrationSummary { + IntegrationSummary { + id: manifest.id.clone(), + version: manifest.version.clone(), + display_name: manifest.display_name.clone(), + capability_count: manifest.capabilities.len(), + auth: auth_label(manifest.auth.as_ref()), + } +} + +fn auth_label(auth: Option<&AuthSpec>) -> String { + match auth { + Some(AuthSpec::OAuth { .. }) => "oauth".to_string(), + Some(AuthSpec::ApiKey { .. }) => "api_key".to_string(), + Some(AuthSpec::None) | None => "none".to_string(), + } +} + +fn transport_label(transport: &McpTransport) -> String { + match transport { + McpTransport::Stdio => "stdio", + McpTransport::Sse => "sse", + McpTransport::Http => "http", + McpTransport::WebSocket => "websocket", + McpTransport::Sdk => "sdk", + McpTransport::ManagedProxy => "managed_proxy", + } + .to_string() +} + +fn permission_label(mode: PermissionMode) -> String { + match mode { + PermissionMode::ReadOnly => "read-only", + PermissionMode::WorkspaceWrite => "workspace-write", + PermissionMode::DangerFullAccess => "danger-full-access", + } + .to_string() +} + +fn transport_detail(config: &TransportConfig) -> String { + match config { + TransportConfig::Stdio { command, args, .. } => { + let args = args + .iter() + .map(ToString::to_string) + .collect::>() + .join(" "); + if args.is_empty() { + command.clone() + } else { + format!("{command} {args}") + } + } + TransportConfig::Http { url, .. } + | TransportConfig::Sse { url, .. } + | TransportConfig::WebSocket { url } => url.clone(), + TransportConfig::Sdk { module } => module.clone(), + TransportConfig::ManagedProxy { target, .. } => target.clone(), + } +} + +/// Render the integration list as a human-readable table. +pub fn list_integrations(registry: &ManifestRegistry) -> String { + let manifests = registry.list(); + let mut lines = vec![format!("INTEGRATIONS ({})", manifests.len())]; + if manifests.is_empty() { + lines.push(String::new()); + lines.push(" No integrations configured.".to_string()); + return lines.join("\n"); + } + lines.push(String::new()); + + let id_width = manifests + .iter() + .map(|manifest| manifest.id.len()) + .max() + .unwrap_or(0) + .max("ID".len()); + let name_width = manifests + .iter() + .map(|manifest| manifest.display_name.len()) + .max() + .unwrap_or(0) + .max("NAME".len()); + lines.push(format!( + " {id: String { + let summaries: Vec = registry + .list() + .iter() + .map(|manifest| json!(summarize(manifest))) + .collect(); + serde_json::to_string_pretty(&Value::Array(summaries)).unwrap_or_else(|_| "[]".to_string()) +} + +/// Render full detail for one integration. +pub fn show_integration(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + + let transport = format!( + "{} ({})", + transport_label(&manifest.transport), + transport_detail(&manifest.transport_config) + ); + let auth = auth_detail(manifest.auth.as_ref()).unwrap_or_else(|| "none".to_string()); + let mut lines = vec![ + format!("INTEGRATION: {}", manifest.id), + String::new(), + format!(" {id:<16}", id = manifest.id), + format!(" version: {version}", version = manifest.version), + format!( + " name: {display_name}", + display_name = manifest.display_name + ), + format!( + " description: {description}", + description = manifest.description + ), + format!(" transport: {transport}"), + format!(" auth: {auth}"), + ]; + + lines.push(String::new()); + lines.push(format!(" CAPABILITIES ({}):", manifest.capabilities.len())); + for capability in &manifest.capabilities { + let idempotent = if capability.idempotent { + " idempotent" + } else { + "" + }; + lines.push(format!( + " {name:<20} [{permission}{idempotent}", + name = capability.name, + permission = permission_label(capability.permission_mode), + idempotent = idempotent, + )); + lines.push(format!(" {}", capability.description)); + } + + if !manifest.events.is_empty() { + lines.push(String::new()); + lines.push(format!(" EVENTS ({}):", manifest.events.len())); + for event in &manifest.events { + let signature = event + .signature + .as_ref() + .map_or_else(|| "none".to_string(), signature_label); + lines.push(format!( + " {name:<20} {description}", + name = event.name, + description = event.description + )); + lines.push(format!(" signature: {signature}")); + } + } + + if !manifest.context_fields.is_empty() { + lines.push(String::new()); + lines.push(format!( + " CONTEXT FIELDS ({}):", + manifest.context_fields.len() + )); + for field in &manifest.context_fields { + lines.push(format!( + " {name:<20} {description}", + name = field.name, + description = field.description + )); + } + } + + Ok(lines.join("\n")) +} + +fn signature_label(signature: &SignatureSpec) -> String { + match signature { + SignatureSpec::HmacSha256 { secret_env } => format!("hmac_sha256 ({secret_env})"), + SignatureSpec::None => "none".to_string(), + } +} + +fn auth_detail(auth: Option<&AuthSpec>) -> Option { + match auth { + Some(AuthSpec::OAuth { + client_id_env, + client_secret_env, + auth_url, + scopes, + .. + }) => Some(format!( + "oauth (client_id_env={client_id_env}, client_secret_env={}, scopes=[{}]) auth_url={auth_url}", + client_secret_env + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("unset"), + scopes.join(", "), + )), + Some(AuthSpec::ApiKey { env_var, header }) => Some(format!( + "api_key (env_var={env_var}, header={})", + header.as_deref().unwrap_or("default") + )), + Some(AuthSpec::None) => Some("none".to_string()), + None => None, + } +} + +/// Render full detail for one integration as a JSON object. +pub fn show_integration_json(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + serde_json::to_string_pretty(&manifest) + .map_err(|error| format!("failed to serialize integration `{id}`: {error}")) +} + +/// Render the capability table for one integration. +pub fn list_capabilities(registry: &ManifestRegistry, id: &str) -> Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + + let mut lines = vec![format!( + "CAPABILITIES: {id} ({})", + manifest.capabilities.len() + )]; + if manifest.capabilities.is_empty() { + lines.push(String::new()); + lines.push(" No capabilities declared.".to_string()); + return Ok(lines.join("\n")); + } + lines.push(String::new()); + + let name_width = manifest + .capabilities + .iter() + .map(|capability| capability.name.len()) + .max() + .unwrap_or(0) + .max("NAME".len()); + lines.push(format!( + " {name: Result { + let manifest = registry.get(id).ok_or_else(|| { + format!("integration `{id}` not found (see `frontal-code integrations list`)") + })?; + let capabilities: Vec = manifest + .capabilities + .iter() + .map(|capability| { + let summary = json!(capability); + json!({ + "name": summary.get("name"), + "description": summary.get("description"), + "permission_mode": summary.get("permission_mode"), + "idempotent": summary.get("idempotent"), + }) + }) + .collect(); + serde_json::to_string_pretty(&Value::Array(capabilities)) + .map_err(|error| format!("failed to serialize capabilities for `{id}`: {error}")) +} + +/// Parse a file as an [`IntegrationManifest`] and run its validation. +/// +/// Returns a human-readable report: `valid` when the manifest passes, or a +/// list of issues otherwise. +pub fn validate_manifest_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + let manifest: IntegrationManifest = serde_json::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display()))?; + match manifest.validate() { + Ok(()) => Ok(format!( + "{}: valid ({} {})", + path.display(), + manifest.id, + manifest.version + )), + Err(issues) => Ok(render_validation_report(path, &manifest, issues)), + } +} + +fn render_validation_report( + path: &Path, + manifest: &IntegrationManifest, + issues: Vec, +) -> String { + let mut lines = vec![format!( + "{}: invalid ({} {}, {} issue(s))", + path.display(), + manifest.id, + manifest.version, + issues.len() + )]; + for issue in issues { + lines.push(format!( + " {field}: {message}", + field = issue.field, + message = issue.message + )); + } + lines.join("\n") +} + +/// JSON report for `validate`. +pub fn validate_manifest_file_json(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + let manifest: IntegrationManifest = serde_json::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display()))?; + let issues = match manifest.validate() { + Ok(()) => Vec::new(), + Err(issues) => issues, + }; + let report = json!({ + "path": path.display().to_string(), + "id": manifest.id, + "version": manifest.version, + "valid": issues.is_empty(), + "issues": issues.iter().map(|issue| json!({ + "field": issue.field, + "message": issue.message, + })).collect::>(), + }); + serde_json::to_string_pretty(&report) + .map_err(|error| format!("failed to serialize validation report: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Guards tests that read `FRONTAL_CODE_INTEGRATIONS_CONFIG`; cargo runs + /// tests on multiple threads and env vars are process-global. + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn test_registry() -> ManifestRegistry { + let manifest = IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: std::collections::BTreeMap::new(), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: json!({"type": "object"}), + output_schema: Some(json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "list_issues".to_string(), + description: "List issues".to_string(), + input_schema: json!({"type": "object"}), + output_schema: None, + permission_mode: PermissionMode::ReadOnly, + idempotent: true, + }, + ], + auth: Some(AuthSpec::OAuth { + client_id_env: "GITHUB_OAUTH_CLIENT_ID".to_string(), + client_secret_env: Some("GITHUB_OAUTH_CLIENT_SECRET".to_string()), + auth_url: "https://github.com/login/oauth/authorize".to_string(), + token_url: "https://github.com/login/oauth/access_token".to_string(), + scopes: vec!["repo".to_string()], + redirect_path: None, + }), + events: Vec::::new(), + context_fields: Vec::new(), + }; + let mut registry = ManifestRegistry::new(); + registry.register(manifest).expect("register"); + registry + } + + fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "frontal-integrations-test-{name}-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + #[test] + fn load_registry_reads_settings_file_and_stops_at_first_match() { + let dir = temp_dir("settings"); + let settings_dir = dir.join(".frontal-code"); + std::fs::create_dir_all(&settings_dir).expect("create settings dir"); + let manifest_json = json!({ + "id": "linear", + "version": "2.1.0", + "display_name": "Linear", + "description": "Linear integration", + "transport": "stdio", + "transport_config": { + "kind": "stdio", + "command": "linear-mcp", + "args": ["serve"], + "env": {} + }, + "capabilities": [{ + "name": "create_issue", + "description": "Create an issue", + "input_schema": {"type": "object"}, + "permission_mode": "workspace_write", + "idempotent": false + }], + "auth": { + "kind": "api_key", + "env_var": "LINEAR_API_KEY", + "header": null + }, + "events": [], + "context_fields": [] + }); + let settings = json!({ "mcp": { "integrations": { "linear": manifest_json } } }); + std::fs::write( + settings_dir.join("settings.json"), + serde_json::to_string_pretty(&settings).expect("serialize"), + ) + .expect("write settings"); + + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + let registry = load_registry(&dir); + // The built-in linear manifest is overridden by the settings file; + // the other built-ins remain. + assert_eq!(registry.list().len(), 4); + let linear = registry.get("linear").expect("linear registered"); + assert_eq!(linear.version, "2.1.0"); + assert!(matches!(linear.auth, Some(AuthSpec::ApiKey { .. }))); + assert!(registry.get("github").is_some()); + assert!(registry.get("sentry").is_some()); + } + + #[test] + fn load_registry_seeds_builtins_when_no_config() { + let dir = temp_dir("empty"); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let _guard = ENV_LOCK.lock().expect("env lock poisoned"); + std::env::remove_var(ENV_INTEGRATIONS_CONFIG); + let registry = load_registry(&dir); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing built-in `{id}`"); + } + } + + #[test] + fn list_integrations_renders_table() { + let registry = test_registry(); + let output = list_integrations(®istry); + assert!(output.contains("INTEGRATIONS (1)"), "got: {output}"); + assert!(output.contains(" ID VERSION"), "got: {output}"); + assert!(output.contains("github"), "got: {output}"); + assert!(output.contains("1.0.0"), "got: {output}"); + assert!(output.contains("GitHub"), "got: {output}"); + assert!(output.contains("oauth"), "got: {output}"); + } + + #[test] + fn list_integrations_json_renders_array() { + let registry = test_registry(); + let output = list_integrations_json(®istry); + let parsed: Value = serde_json::from_str(&output).expect("valid json"); + let array = parsed.as_array().expect("array"); + assert_eq!(array.len(), 1); + assert_eq!(array[0].get("id").and_then(Value::as_str), Some("github")); + assert_eq!( + array[0].get("capability_count").and_then(Value::as_u64), + Some(2) + ); + assert_eq!(array[0].get("auth").and_then(Value::as_str), Some("oauth")); + } + + #[test] + fn show_integration_known_id() { + let registry = test_registry(); + let output = show_integration(®istry, "github").expect("known id"); + assert!(output.contains("INTEGRATION: github"), "got: {output}"); + assert!(output.contains("gh-mcp serve"), "got: {output}"); + assert!(output.contains("create_issue"), "got: {output}"); + assert!(output.contains("workspace-write"), "got: {output}"); + assert!(output.contains("idempotent"), "got: {output}"); + } + + #[test] + fn show_integration_unknown_id() { + let registry = test_registry(); + let error = show_integration(®istry, "missing").expect_err("unknown id fails"); + assert!(error.contains("`missing` not found"), "got: {error}"); + } + + #[test] + fn show_integration_json_roundtrips() { + let registry = test_registry(); + let output = show_integration_json(®istry, "github").expect("known id"); + let parsed: IntegrationManifest = + serde_json::from_str(&output).expect("valid manifest json"); + assert_eq!(parsed.id, "github"); + } + + #[test] + fn list_capabilities_renders_table() { + let registry = test_registry(); + let output = list_capabilities(®istry, "github").expect("known id"); + assert!(output.contains("CAPABILITIES: github (2)"), "got: {output}"); + assert!(output.contains("create_issue"), "got: {output}"); + assert!(output.contains("list_issues"), "got: {output}"); + assert!(output.contains("read-only"), "got: {output}"); + assert!(output.contains("no"), "got: {output}"); + } + + #[test] + fn list_capabilities_unknown_id() { + let registry = test_registry(); + let error = list_capabilities(®istry, "missing").expect_err("unknown id fails"); + assert!(error.contains("`missing` not found"), "got: {error}"); + } + + fn write_manifest(dir: &Path, name: &str, contents: &str) -> std::path::PathBuf { + let path = dir.join(name); + std::fs::write(&path, contents).expect("write manifest"); + path + } + + fn valid_manifest_json() -> String { + serde_json::to_string_pretty(&json!({ + "id": "github", + "version": "1.0.0", + "display_name": "GitHub", + "description": "GitHub integration", + "transport": "stdio", + "transport_config": { + "kind": "stdio", + "command": "gh-mcp", + "args": ["serve"], + "env": {} + }, + "capabilities": [{ + "name": "create_issue", + "description": "Create an issue", + "input_schema": {"type": "object"}, + "permission_mode": "workspace_write", + "idempotent": false + }], + "auth": { + "kind": "api_key", + "env_var": "GITHUB_TOKEN", + "header": null + }, + "events": [], + "context_fields": [] + })) + .expect("serialize") + } + + #[test] + fn validate_manifest_file_valid() { + let dir = temp_dir("validate-valid"); + let path = write_manifest(&dir, "valid.json", &valid_manifest_json()); + let output = validate_manifest_file(&path).expect("valid manifest"); + assert!(output.contains("valid"), "got: {output}"); + assert!(output.contains("github"), "got: {output}"); + } + + #[test] + fn validate_manifest_file_invalid_reports_issues() { + let dir = temp_dir("validate-invalid"); + let bad = valid_manifest_json().replace("\"github\"", "\"Bad Id!\""); + let path = write_manifest(&dir, "invalid.json", &bad); + let output = validate_manifest_file(&path).expect("parses but is invalid"); + assert!(output.contains("invalid"), "got: {output}"); + assert!(output.contains("id"), "got: {output}"); + } + + #[test] + fn validate_manifest_file_unparseable() { + let dir = temp_dir("validate-unparseable"); + let path = write_manifest(&dir, "unparseable.json", "{ not json"); + let error = validate_manifest_file(&path).expect_err("unparseable fails"); + assert!(error.contains("failed to parse"), "got: {error}"); + } + + #[test] + fn validate_manifest_file_json_shape() { + let dir = temp_dir("validate-json"); + let path = write_manifest(&dir, "valid.json", &valid_manifest_json()); + let output = validate_manifest_file_json(&path).expect("valid manifest"); + let parsed: Value = serde_json::from_str(&output).expect("valid json"); + assert_eq!(parsed.get("valid").and_then(Value::as_bool), Some(true)); + assert!(parsed.get("issues").and_then(Value::as_array).is_some()); + } +} diff --git a/cli/src/main.rs b/cli/src/main.rs index 0d86bf1d..28a42e98 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -13,6 +13,7 @@ mod analytics; mod auth; mod init; mod input; +mod integrations; mod mentions; mod notify; mod render; @@ -255,6 +256,10 @@ fn dispatch(action: CliAction) -> Result<(), Box> { args, output_format, } => LiveCli::print_mcp(args.as_deref(), output_format)?, + CliAction::Integrations { + args, + output_format, + } => run_integrations_command(args.as_deref(), output_format)?, CliAction::Trust { revoke } => print_workspace_trust(revoke)?, CliAction::Skills { args, @@ -338,6 +343,90 @@ fn dispatch(action: CliAction) -> Result<(), Box> { Ok(()) } +fn run_integrations_command( + args: Option<&str>, + output_format: CliOutputFormat, +) -> Result<(), Box> { + let mut tokens = args + .map(|value| { + value + .split_whitespace() + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let mut as_json = matches!(output_format, CliOutputFormat::Json); + tokens.retain(|token| { + if token == "--json" || token == "--output-format=json" { + as_json = true; + false + } else { + true + } + }); + + let Some(subcommand) = tokens.first().map(String::as_str) else { + println!("{}", integrations::help_text()); + return Ok(()); + }; + + match subcommand { + "list" => { + let registry = integrations::load_registry(&env::current_dir()?); + let output = if as_json { + integrations::list_integrations_json(®istry) + } else { + integrations::list_integrations(®istry) + }; + println!("{output}"); + } + "show" | "capabilities" => { + let id = tokens + .get(1) + .and_then(|value| value.split('=').next()) + .filter(|value| !value.starts_with("--")) + .ok_or_else(|| { + "`frontal-code integrations {subcommand}` requires an integration id. \ + Use `frontal-code integrations {subcommand} `." + .to_string() + })? + .to_string(); + let registry = integrations::load_registry(&env::current_dir()?); + let output = match (subcommand, as_json) { + ("show", true) => integrations::show_integration_json(®istry, &id), + ("show", false) => integrations::show_integration(®istry, &id), + ("capabilities", true) => integrations::list_capabilities_json(®istry, &id), + _ => integrations::list_capabilities(®istry, &id), + } + .map_err(Box::::from)?; + println!("{output}"); + } + "validate" => { + let path = tokens + .get(1) + .filter(|value| !value.starts_with("--")) + .ok_or_else(|| { + "`frontal-code integrations validate` requires a manifest file path. \ + Use `frontal-code integrations validate `." + .to_string() + })?; + let path = Path::new(path); + let output = if as_json { + integrations::validate_manifest_file_json(path) + } else { + integrations::validate_manifest_file(path) + } + .map_err(Box::::from)?; + println!("{output}"); + } + other => { + eprintln!("unknown integrations subcommand: {other}"); + eprintln!("{}", integrations::help_text()); + } + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq, Eq)] struct TelemetryResolution { enabled: bool, @@ -475,6 +564,10 @@ enum CliAction { args: Option, output_format: CliOutputFormat, }, + Integrations { + args: Option, + output_format: CliOutputFormat, + }, Trust { revoke: bool, }, @@ -1101,6 +1194,10 @@ fn parse_args(args: &[String]) -> Result { args: join_optional_args(&rest[1..]), output_format, }), + "integrations" => Ok(CliAction::Integrations { + args: join_optional_args(&rest[1..]), + output_format, + }), "trust" => match rest.get(1).map(String::as_str) { None => Ok(CliAction::Trust { revoke: false }), Some("--revoke") if rest.len() == 2 => Ok(CliAction::Trust { revoke: true }), diff --git a/crates/code-integrations/src/lib.rs b/crates/code-integrations/src/lib.rs index 39fce695..f59c5b7a 100644 --- a/crates/code-integrations/src/lib.rs +++ b/crates/code-integrations/src/lib.rs @@ -4,7 +4,9 @@ //! including Model Context Protocol (MCP) server management and tool bridging. pub mod ide; +pub mod manifest; pub mod mcp; +pub mod providers; // Re-export commonly used MCP types for convenience pub use mcp::{ @@ -17,6 +19,15 @@ pub use mcp::{ tools::{execute_mcp_tool, mcp_tool_specs, ToolSpec}, }; +// Re-export the integration manifest system (the typed plugin contract) +pub use manifest::{ + AuthSpec, CapabilitySpec, ContextFieldSpec, EventSpec, IntegrationManifest, ManifestRegistry, + SignatureSpec, TransportConfig, ValidationIssue, +}; + +// Re-export the built-in provider manifests (GitHub, Linear, PostHog, Sentry) +pub use providers::{builtin_providers, BUILTIN_PROVIDER_IDS}; + #[cfg(test)] mod tests { #[test] diff --git a/crates/code-integrations/src/manifest.rs b/crates/code-integrations/src/manifest.rs new file mode 100644 index 00000000..9099d26d --- /dev/null +++ b/crates/code-integrations/src/manifest.rs @@ -0,0 +1,535 @@ +//! Typed, declarative integration manifests. +//! +//! An [`IntegrationManifest`] is the plugin contract for `FrontalCode` integrations: +//! the server, CLI, and SDKs all consume this shape. Manifests declare how to reach +//! an integration (transport + transport config), the capabilities (tools/endpoints) +//! it exposes, its auth requirements, the webhook events it can emit, and the fields +//! it writes into task context. + +use crate::mcp::config::McpTransport; +use crate::mcp::tools::PermissionMode; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +/// A fully typed, declarative definition of an integration. +/// This is the plugin contract: the server, CLI, and SDKs all consume this shape. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct IntegrationManifest { + /// Unique identifier (e.g., "github", "linear", "my-custom-api") + pub id: String, + /// Semver version of this manifest + pub version: String, + pub display_name: String, + pub description: String, + /// How to reach the integration. Reuses the existing MCP transport enum. + pub transport: McpTransport, + /// Transport-specific config (command/args/env for stdio, url for http, etc.) + pub transport_config: TransportConfig, + /// The capabilities (tools/endpoints) this integration exposes + pub capabilities: Vec, + /// Auth requirements, if any + pub auth: Option, + /// Webhook events this integration can emit (for webhook-capable integrations) + pub events: Vec, + /// Fields this integration writes into task context (prefix-based) + pub context_fields: Vec, +} + +/// Transport-specific configuration. Tagged enum. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum TransportConfig { + Stdio { + command: String, + args: Vec, + env: BTreeMap, + }, + Http { + url: String, + headers: BTreeMap, + }, + Sse { + url: String, + headers: BTreeMap, + }, + WebSocket { + url: String, + }, + Sdk { + module: String, + }, + ManagedProxy { + proxy_url: String, + target: String, + }, +} + +/// A single capability (tool/endpoint) exposed by an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CapabilitySpec { + pub name: String, + pub description: String, + /// JSON Schema for input validation (industry standard, matches MCP) + pub input_schema: serde_json::Value, + /// JSON Schema for output validation + pub output_schema: Option, + pub permission_mode: PermissionMode, + /// If true, safe to retry on failure + pub idempotent: bool, +} + +/// Auth specification for an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum AuthSpec { + None, + OAuth { + client_id_env: String, + client_secret_env: Option, + auth_url: String, + token_url: String, + scopes: Vec, + redirect_path: Option, + }, + ApiKey { + env_var: String, + header: Option, + }, +} + +/// A webhook event this integration can emit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EventSpec { + pub name: String, + pub description: String, + /// JSON Schema for the event payload + pub payload_schema: serde_json::Value, + /// Signature verification method + pub signature: Option, +} + +/// Signature verification method for webhook events. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SignatureSpec { + HmacSha256 { secret_env: String }, + None, +} + +/// A field this integration writes into task context. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ContextFieldSpec { + pub name: String, + pub description: String, + pub schema: serde_json::Value, +} + +/// A single validation issue found while checking a manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidationIssue { + /// Dot-separated path to the offending field (e.g., "capabilities[2].name"). + pub field: String, + /// Human-readable description of the problem. + pub message: String, +} + +impl IntegrationManifest { + /// Validate this manifest, returning any issues found. + /// + /// Returns `Ok(())` when the manifest is well-formed. + pub fn validate(&self) -> Result<(), Vec> { + let mut issues: Vec = Vec::new(); + self.validate_id(&mut issues); + self.validate_version(&mut issues); + self.validate_capabilities(&mut issues); + self.validate_auth(&mut issues); + self.validate_events(&mut issues); + if issues.is_empty() { + Ok(()) + } else { + Err(issues) + } + } + + fn validate_id(&self, issues: &mut Vec) { + // `id`: non-empty, lowercase alphanumeric + hyphens, max 64 chars. + if self.id.is_empty() { + issues.push(ValidationIssue { + field: "id".to_string(), + message: "id must be non-empty".to_string(), + }); + } else if self.id.len() > 64 { + issues.push(ValidationIssue { + field: "id".to_string(), + message: format!("id must be at most 64 characters, got {}", self.id.len()), + }); + } else if !id_re().is_match(&self.id) { + issues.push(ValidationIssue { + field: "id".to_string(), + message: format!( + "id `{}` must be lowercase alphanumeric with hyphens only", + self.id + ), + }); + } + } + + fn validate_version(&self, issues: &mut Vec) { + // `version`: must parse as semver. + if !semver_re().is_match(&self.version) { + issues.push(ValidationIssue { + field: "version".to_string(), + message: format!( + "version `{}` is not valid semver (expected e.g. `1.2.3`)", + self.version + ), + }); + } + } + + fn validate_capabilities(&self, issues: &mut Vec) { + // `capabilities`: must be non-empty, names unique, schemas valid JSON. + if self.capabilities.is_empty() { + issues.push(ValidationIssue { + field: "capabilities".to_string(), + message: "capabilities must be non-empty".to_string(), + }); + } + let mut seen: BTreeSet<&str> = BTreeSet::new(); + for (index, capability) in self.capabilities.iter().enumerate() { + let field = format!("capabilities[{index}]"); + if !seen.insert(&capability.name) { + issues.push(ValidationIssue { + field: format!("{field}.name"), + message: format!("duplicate capability name `{}`", capability.name), + }); + } + if capability.input_schema.is_null() { + issues.push(ValidationIssue { + field: format!("{field}.input_schema"), + message: "input_schema must be valid JSON".to_string(), + }); + } + if let Some(output_schema) = &capability.output_schema { + if output_schema.is_null() { + issues.push(ValidationIssue { + field: format!("{field}.output_schema"), + message: "output_schema must be valid JSON".to_string(), + }); + } + } + } + } + + fn validate_auth(&self, issues: &mut Vec) { + // `auth`: env var names must be non-empty strings. + match &self.auth { + Some(AuthSpec::OAuth { + client_id_env, + client_secret_env, + .. + }) => { + if client_id_env.is_empty() { + issues.push(ValidationIssue { + field: "auth.client_id_env".to_string(), + message: "auth.client_id_env must be a non-empty string".to_string(), + }); + } + if let Some(secret) = client_secret_env { + if secret.is_empty() { + issues.push(ValidationIssue { + field: "auth.client_secret_env".to_string(), + message: "auth.client_secret_env must be a non-empty string" + .to_string(), + }); + } + } + } + Some(AuthSpec::ApiKey { env_var, .. }) => { + if env_var.is_empty() { + issues.push(ValidationIssue { + field: "auth.env_var".to_string(), + message: "auth.env_var must be a non-empty string".to_string(), + }); + } + } + Some(AuthSpec::None) | None => {} + } + } + + fn validate_events(&self, issues: &mut Vec) { + // `events`: names must be unique. + let mut seen_events: BTreeSet<&str> = BTreeSet::new(); + for (index, event) in self.events.iter().enumerate() { + let field = format!("events[{index}]"); + if !seen_events.insert(&event.name) { + issues.push(ValidationIssue { + field: format!("{field}.name"), + message: format!("duplicate event name `{}`", event.name), + }); + } + } + } +} + +/// Registry of integration manifests, keyed by manifest id. +#[derive(Debug, Clone, Default)] +pub struct ManifestRegistry { + manifests: BTreeMap, +} + +impl ManifestRegistry { + /// Create an empty registry. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Register a manifest, rejecting it if it fails validation + /// or if a manifest with the same id is already registered. + pub fn register(&mut self, manifest: IntegrationManifest) -> Result<(), Vec> { + manifest.validate()?; + if self.manifests.contains_key(&manifest.id) { + return Err(vec![ValidationIssue { + field: "id".to_string(), + message: format!("manifest with id `{}` is already registered", manifest.id), + }]); + } + self.manifests.insert(manifest.id.clone(), manifest); + Ok(()) + } + + /// Remove a manifest by id, returning it if it was present. + pub fn remove(&mut self, id: &str) -> Option { + self.manifests.remove(id) + } + + /// Look up a manifest by id. + #[must_use] + pub fn get(&self, id: &str) -> Option<&IntegrationManifest> { + self.manifests.get(id) + } + + /// List all registered manifests in id order. + #[must_use] + pub fn list(&self) -> Vec<&IntegrationManifest> { + self.manifests.values().collect() + } + + /// Look up a single capability on a registered manifest. + #[must_use] + pub fn capability(&self, integration_id: &str, capability: &str) -> Option<&CapabilitySpec> { + self.manifests + .get(integration_id)? + .capabilities + .iter() + .find(|spec| spec.name == capability) + } +} + +static ID_RE: std::sync::OnceLock = std::sync::OnceLock::new(); +static SEMVER_RE: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn id_re() -> &'static Regex { + ID_RE.get_or_init(|| Regex::new("^[a-z0-9]+(-[a-z0-9]+)*$").expect("static regex is valid")) +} + +fn semver_re() -> &'static Regex { + SEMVER_RE.get_or_init(|| { + Regex::new( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", + ) + .expect("static regex is valid") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn valid_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: BTreeMap::new(), + }, + capabilities: vec![CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: serde_json::json!({"type": "object"}), + output_schema: Some(serde_json::json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".to_string(), + header: None, + }), + events: vec![EventSpec { + name: "issue.opened".to_string(), + description: "An issue was opened".to_string(), + payload_schema: serde_json::json!({"type": "object"}), + signature: None, + }], + context_fields: vec![ContextFieldSpec { + name: "github.issue".to_string(), + description: "The linked GitHub issue".to_string(), + schema: serde_json::json!({"type": "object"}), + }], + } + } + + #[test] + fn valid_manifest_roundtrips_through_serde_json() { + let manifest = valid_manifest(); + let json = serde_json::to_string(&manifest).expect("serialize"); + let parsed: IntegrationManifest = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed.id, manifest.id); + assert_eq!(parsed.version, manifest.version); + assert_eq!(parsed.capabilities.len(), manifest.capabilities.len()); + assert!(matches!(parsed.transport, McpTransport::Stdio)); + assert!(matches!( + parsed.transport_config, + TransportConfig::Stdio { .. } + )); + assert!(matches!(parsed.auth, Some(AuthSpec::ApiKey { .. }))); + assert_eq!(parsed.events.len(), manifest.events.len()); + assert_eq!(parsed.context_fields.len(), manifest.context_fields.len()); + assert!(parsed.validate().is_ok()); + } + + #[test] + fn validation_catches_duplicate_capability_names() { + let mut manifest = valid_manifest(); + manifest.capabilities.push(CapabilitySpec { + name: "create_issue".to_string(), + description: "Duplicate".to_string(), + input_schema: serde_json::json!({"type": "object"}), + output_schema: None, + permission_mode: PermissionMode::ReadOnly, + idempotent: true, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "capabilities[1].name" + && issue.message.contains("duplicate capability name")), + "expected duplicate capability issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_invalid_id_format() { + let mut manifest = valid_manifest(); + manifest.id = "GitHub_1".to_string(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "id" && issue.message.contains("lowercase")), + "expected id format issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_invalid_version() { + let mut manifest = valid_manifest(); + manifest.version = "not-a-version".to_string(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "version" && issue.message.contains("semver")), + "expected semver issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_empty_capabilities() { + let mut manifest = valid_manifest(); + manifest.capabilities.clear(); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "capabilities" && issue.message.contains("non-empty")), + "expected empty capabilities issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_duplicate_event_names() { + let mut manifest = valid_manifest(); + manifest.events.push(EventSpec { + name: "issue.opened".to_string(), + description: "Duplicate".to_string(), + payload_schema: serde_json::json!({"type": "object"}), + signature: None, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues.iter().any(|issue| issue.field == "events[1].name" + && issue.message.contains("duplicate event name")), + "expected duplicate event issue, got {issues:?}" + ); + } + + #[test] + fn validation_catches_empty_auth_env_vars() { + let mut manifest = valid_manifest(); + manifest.auth = Some(AuthSpec::ApiKey { + env_var: String::new(), + header: None, + }); + let issues = manifest.validate().expect_err("should fail"); + assert!( + issues + .iter() + .any(|issue| issue.field == "auth.env_var" && issue.message.contains("non-empty")), + "expected empty env var issue, got {issues:?}" + ); + } + + #[test] + fn registry_register_get_list_capability_work() { + let mut registry = ManifestRegistry::new(); + let manifest = valid_manifest(); + registry.register(manifest.clone()).expect("register"); + + assert!(registry.get("github").is_some()); + assert!(registry.get("missing").is_none()); + assert_eq!(registry.list(), [&manifest]); + assert!(registry.capability("github", "create_issue").is_some()); + assert!(registry.capability("github", "missing").is_none()); + assert!(registry.capability("missing", "create_issue").is_none()); + + // Registering the same id again is rejected. + let err = registry + .register(manifest) + .expect_err("duplicate id must fail"); + assert_eq!(err.len(), 1); + assert_eq!(err[0].field, "id"); + } + + #[test] + fn registry_rejects_invalid_manifest() { + let mut registry = ManifestRegistry::new(); + let mut manifest = valid_manifest(); + manifest.id = "Bad Id!".to_string(); + let err = registry + .register(manifest) + .expect_err("invalid manifest must fail"); + assert!( + err.iter().any(|issue| issue.field == "id"), + "expected id issue, got {err:?}" + ); + } +} diff --git a/crates/code-integrations/src/mcp/tools.rs b/crates/code-integrations/src/mcp/tools.rs index 910f3249..5d71105b 100644 --- a/crates/code-integrations/src/mcp/tools.rs +++ b/crates/code-integrations/src/mcp/tools.rs @@ -9,7 +9,8 @@ use serde_json::{from_value, json, Value as JsonValue}; use std::sync::OnceLock; /// Permission mode for tools -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum PermissionMode { ReadOnly, WorkspaceWrite, diff --git a/crates/code-integrations/src/providers.rs b/crates/code-integrations/src/providers.rs new file mode 100644 index 00000000..e439ebb1 --- /dev/null +++ b/crates/code-integrations/src/providers.rs @@ -0,0 +1,417 @@ +//! Built-in provider manifests. +//! +//! These are the four first-party integrations (GitHub, Linear, `PostHog`, +//! and Sentry) expressed as config-driven [`IntegrationManifest`] values instead +//! of hardcoded provider code. They are the defaults the server and CLI load +//! when no project-level overrides are configured. +//! +//! The legacy `IntegrationRegistry::call_*` methods (see +//! [`crate::mcp::integration`]) remain as deprecated adapters over the +//! corresponding capabilities below for one release. + +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::manifest::{ + AuthSpec, CapabilitySpec, IntegrationManifest, ManifestRegistry, TransportConfig, +}; +use crate::mcp::config::McpTransport; +use crate::mcp::tools::PermissionMode; + +/// The four built-in provider integration IDs, in stable (sorted) order. +pub const BUILTIN_PROVIDER_IDS: [&str; 4] = ["github", "linear", "posthog", "sentry"]; + +/// Returns the four built-in provider manifests, ready to register. +/// +/// Each manifest is validated before being returned; a built-in manifest that +/// fails validation is a bug, so this returns a `Result` that callers (and +/// tests) can assert on. +pub fn builtin_providers() -> Result> { + let mut registry = ManifestRegistry::new(); + let mut failures = Vec::new(); + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + if let Err(issues) = registry.register(manifest) { + failures.extend(issues.into_iter().map(|issue| issue.message)); + } + } + if failures.is_empty() { + Ok(registry) + } else { + Err(failures) + } +} + +/// The GitHub integration: PR creation, issue comments, and check runs against +/// the GitHub REST/MCP surface. +fn github_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".into(), + version: "1.0.0".into(), + display_name: "GitHub".into(), + description: "Create pull requests, comment on issues, and report check runs.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://api.github.com".into(), + headers: header_map(&[("Authorization", "${GITHUB_TOKEN}")]), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_pr".into(), + description: "Open a pull request on a repository.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "title": { "type": "string" }, + "body": { "type": "string" }, + "head": { "type": "string" }, + "base": { "type": "string" } + }, + "required": ["repo", "title", "head", "base"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "number": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to an issue or pull request.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "issue_number": { "type": "integer" }, + "body": { "type": "string" } + }, + "required": ["repo", "issue_number", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_check_run".into(), + description: "Create a check run for a commit.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "repo": { "type": "string" }, + "name": { "type": "string" }, + "head_sha": { "type": "string" }, + "status": { "type": "string" }, + "conclusion": { "type": "string" } + }, + "required": ["repo", "name", "head_sha"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "integer" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The Linear integration: issue comments used by the tracker reporter. +fn linear_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "linear".into(), + version: "1.0.0".into(), + display_name: "Linear".into(), + description: "Comment on Linear issues for tracker reporting.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://api.linear.app/graphql".into(), + headers: header_map(&[("Authorization", "${LINEAR_API_KEY}")]), + }, + capabilities: vec![CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to a Linear issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["issue_id", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "LINEAR_API_KEY".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The `PostHog` integration: capture product and telemetry events. +fn posthog_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "posthog".into(), + version: "1.0.0".into(), + display_name: "PostHog".into(), + description: "Capture analytics events in PostHog.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://us.i.posthog.com/capture/".into(), + headers: BTreeMap::new(), + }, + capabilities: vec![ + CapabilitySpec { + name: "capture_event".into(), + description: "Capture a single analytics event.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "event": { "type": "string" }, + "distinct_id": { "type": "string" }, + "properties": { "type": "object" } + }, + "required": ["event", "distinct_id"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "status": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: true, + }, + CapabilitySpec { + name: "capture_events_batch".into(), + description: "Capture a batch of analytics events.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "batch": { + "type": "array", + "items": { + "type": "object", + "properties": { + "event": { "type": "string" }, + "distinct_id": { "type": "string" }, + "properties": { "type": "object" } + }, + "required": ["event", "distinct_id"] + } + } + }, + "required": ["batch"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "status": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: true, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "POSTHOG_API_KEY".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +/// The Sentry integration: create issues and comments for error reporting. +fn sentry_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "sentry".into(), + version: "1.0.0".into(), + display_name: "Sentry".into(), + description: "Create Sentry issues and comments for error reporting.".into(), + transport: McpTransport::Http, + transport_config: TransportConfig::Http { + url: "https://sentry.io/api/0".into(), + headers: header_map(&[("Authorization", "Token ${SENTRY_AUTH_TOKEN}")]), + }, + capabilities: vec![ + CapabilitySpec { + name: "create_issue".into(), + description: "Create a new Sentry issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "project": { "type": "string" }, + "title": { "type": "string" }, + "message": { "type": "string" } + }, + "required": ["project", "title", "message"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + CapabilitySpec { + name: "create_issue_comment".into(), + description: "Add a comment to a Sentry issue.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "issue_id": { "type": "string" }, + "body": { "type": "string" } + }, + "required": ["issue_id", "body"] + }), + output_schema: Some(json!({ + "type": "object", + "properties": { "id": { "type": "string" }, "url": { "type": "string" } } + })), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }, + ], + auth: Some(AuthSpec::ApiKey { + env_var: "SENTRY_AUTH_TOKEN".into(), + header: Some("Authorization".into()), + }), + events: Vec::new(), + context_fields: Vec::new(), + } +} + +fn header_map(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtin_providers_all_validate() { + let registry = builtin_providers().expect("all built-in manifests must validate"); + assert_eq!(registry.list().len(), 4); + for id in BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing built-in `{id}`"); + } + } + + #[test] + fn builtin_provider_ids_are_sorted_and_unique() { + let ids: Vec<&str> = BUILTIN_PROVIDER_IDS.to_vec(); + let mut sorted = ids.clone(); + sorted.sort_unstable(); + assert_eq!(ids, sorted, "BUILTIN_PROVIDER_IDS must be sorted"); + assert_eq!( + ids.len(), + ids.iter().collect::>().len(), + "BUILTIN_PROVIDER_IDS must be unique" + ); + } + + #[test] + fn each_builtin_manifest_roundtrips_through_serde_json() { + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + let original_id = manifest.id.clone(); + let json = serde_json::to_string(&manifest).expect("serialize"); + let parsed: IntegrationManifest = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed, manifest, "roundtrip mismatch for {original_id}"); + assert!(parsed.validate().is_ok(), "{original_id} must validate"); + } + } + + #[test] + fn github_manifest_exposes_expected_capabilities() { + let manifest = github_manifest(); + let names: Vec<&str> = manifest + .capabilities + .iter() + .map(|c| c.name.as_str()) + .collect(); + assert_eq!( + names, + vec!["create_pr", "create_issue_comment", "create_check_run"] + ); + } + + #[test] + fn linear_manifest_targets_issue_comments() { + let manifest = linear_manifest(); + let capability = manifest + .capabilities + .iter() + .find(|c| c.name == "create_issue_comment") + .expect("linear must expose create_issue_comment"); + assert_eq!(capability.permission_mode, PermissionMode::WorkspaceWrite); + assert!(!capability.idempotent); + } + + #[test] + fn posthog_capabilities_are_idempotent_writes() { + let manifest = posthog_manifest(); + assert!( + !manifest.capabilities.is_empty(), + "posthog must expose at least one capability" + ); + for capability in &manifest.capabilities { + assert!( + capability.idempotent, + "posthog capability `{}` must be idempotent", + capability.name + ); + assert_eq!(capability.permission_mode, PermissionMode::WorkspaceWrite); + } + } + + #[test] + fn all_builtins_use_http_transport_with_api_key_auth() { + for manifest in [ + github_manifest(), + linear_manifest(), + posthog_manifest(), + sentry_manifest(), + ] { + assert!( + matches!(manifest.transport, McpTransport::Http), + "{} must use http transport", + manifest.id + ); + match manifest.auth { + Some(AuthSpec::ApiKey { env_var, .. }) => { + assert!(!env_var.is_empty(), "{} env_var must be set", manifest.id); + } + other => panic!("{} must use api_key auth, got {other:?}", manifest.id), + } + } + } +} diff --git a/crates/code-server/src/integrations.rs b/crates/code-server/src/integrations.rs new file mode 100644 index 00000000..f3f22899 --- /dev/null +++ b/crates/code-server/src/integrations.rs @@ -0,0 +1,601 @@ +//! `/v1/integrations` control-plane routes. +//! +//! Serves the typed integration manifests from [`code_integrations`]: listing +//! integrations, inspecting a single manifest, its capabilities, and its +//! events. Capability invocation is a stub for now; real MCP dispatch lands +//! in Phase 3. + +use std::sync::Mutex; + +use axum::extract::Path; +use axum::extract::State; +use axum::response::Json; +use axum::routing::{get, post}; +use axum::Router; +use code_integrations::mcp::tools::PermissionMode; +use code_integrations::{AuthSpec, IntegrationManifest, ManifestRegistry}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; + +use crate::AppError; +use crate::ServerState; + +/// Environment variable holding a JSON array of [`IntegrationManifest`] values. +pub const INTEGRATIONS_CONFIG_ENV: &str = "FRONTAL_CODE_INTEGRATIONS_CONFIG"; + +/// Server state for the integrations routes. The registry is validated on +/// load; manifests that fail validation are skipped with a warning. +#[derive(Debug, Default)] +pub struct IntegrationsState { + pub registry: Mutex, +} + +impl IntegrationsState { + /// Create an empty state with no registered integrations. + #[cfg(test)] + #[must_use] + pub fn new() -> Self { + Self { + registry: Mutex::new(ManifestRegistry::new()), + } + } + + /// Create a state seeded with the built-in provider manifests (GitHub, + /// Linear, `PostHog`, and Sentry). These are the first-party defaults; + /// project-level configuration can still override any of them by id. + #[cfg(test)] + #[must_use] + pub fn with_builtins() -> Self { + Self { + registry: Mutex::new( + code_integrations::builtin_providers() + .expect("built-in provider manifests are valid"), + ), + } + } + + /// Load integrations from the `FRONTAL_CODE_INTEGRATIONS_CONFIG` + /// environment variable, a JSON array of manifests, on top of the + /// built-in providers. The env var can add new integrations or override a + /// built-in by id. Missing or invalid entries are skipped with a warning + /// so one bad manifest cannot take the whole registry down. + #[must_use] + pub fn load_from_env() -> Self { + Self::from_json( + &std::env::var(INTEGRATIONS_CONFIG_ENV).unwrap_or_default(), + code_integrations::builtin_providers().expect("built-in provider manifests are valid"), + ) + } + + /// Build a state from a raw JSON array of manifests, layered on top of a + /// base registry (typically the built-in providers). An empty, invalid, + /// or malformed string yields the base registry with a warning; + /// individual manifests that fail validation are skipped with a warning + /// so one bad manifest cannot take the whole registry down. A manifest + /// whose id collides with a base entry replaces it, letting project + /// configuration override a built-in. + #[must_use] + pub fn from_json(raw: &str, base: ManifestRegistry) -> Self { + let mut registry = base; + let trimmed = raw.trim(); + if !trimmed.is_empty() { + let Ok(manifests) = serde_json::from_str::>(trimmed) else { + eprintln!("warning: {INTEGRATIONS_CONFIG_ENV} is not a JSON array of manifests; ignoring it"); + return Self { + registry: Mutex::new(registry), + }; + }; + for manifest in manifests { + let id = manifest.id.clone(); + if registry.get(&id).is_some() { + // A same-id manifest from configuration overrides the + // built-in: drop the existing entry, then register the new one. + registry.remove(&id); + } + if let Err(issues) = registry.register(manifest) { + let reason = issues + .iter() + .map(|issue| format!("{}: {}", issue.field, issue.message)) + .collect::>() + .join("; "); + eprintln!("warning: skipping integration manifest `{id}`: {reason}"); + } + } + } + Self { + registry: Mutex::new(registry), + } + } +} + +/// One row of the integration list. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct IntegrationSummary { + pub id: String, + pub version: String, + pub display_name: String, + pub description: String, + pub capability_count: usize, + pub auth_required: bool, + pub event_count: usize, +} + +impl From<&IntegrationManifest> for IntegrationSummary { + fn from(manifest: &IntegrationManifest) -> Self { + Self { + id: manifest.id.clone(), + version: manifest.version.clone(), + display_name: manifest.display_name.clone(), + description: manifest.description.clone(), + capability_count: manifest.capabilities.len(), + auth_required: manifest + .auth + .as_ref() + .is_some_and(|auth| !matches!(auth, AuthSpec::None)), + event_count: manifest.events.len(), + } + } +} + +/// One capability of an integration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct CapabilitySummary { + pub name: String, + pub description: String, + pub permission_mode: String, + pub idempotent: bool, +} + +impl From<&code_integrations::CapabilitySpec> for CapabilitySummary { + fn from(spec: &code_integrations::CapabilitySpec) -> Self { + Self { + name: spec.name.clone(), + description: spec.description.clone(), + permission_mode: permission_mode_label(spec.permission_mode), + idempotent: spec.idempotent, + } + } +} + +/// One webhook event an integration can emit. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct EventSummary { + pub name: String, + pub description: String, + pub signature_required: bool, +} + +impl From<&code_integrations::EventSpec> for EventSummary { + fn from(event: &code_integrations::EventSpec) -> Self { + Self { + name: event.name.clone(), + description: event.description.clone(), + signature_required: event.signature.is_some(), + } + } +} + +/// Input for invoking a capability. The JSON value is the MCP tool input. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct InvokeRequest { + pub input: Value, +} + +fn permission_mode_label(mode: PermissionMode) -> String { + match mode { + PermissionMode::ReadOnly => "read_only", + PermissionMode::WorkspaceWrite => "workspace_write", + PermissionMode::DangerFullAccess => "danger_full_access", + } + .to_string() +} + +pub(crate) async fn list_integrations( + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + Ok(Json( + registry + .list() + .into_iter() + .map(IntegrationSummary::from) + .collect(), + )) +} + +pub(crate) async fn get_integration( + Path(id): Path, + State(state): State>, +) -> Result, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json(manifest.clone())) +} + +pub(crate) async fn list_capabilities( + Path(id): Path, + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json( + manifest + .capabilities + .iter() + .map(CapabilitySummary::from) + .collect(), + )) +} + +/// Invoke a capability. Stub: MCP dispatch is Phase 3. +pub(crate) async fn invoke_capability( + Path((id, name)): Path<(String, String)>, + State(state): State>, + Json(request): Json, +) -> Result<(), AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let Some(_capability) = registry.capability(&id, &name) else { + return Err(AppError::not_found(format!( + "capability '{name}' not found on integration '{id}'" + ))); + }; + let input = request.input; + drop(registry); + Err(AppError::not_implemented(format!( + "invoking capability '{name}' on integration '{id}' is not implemented yet; \ + MCP dispatch lands in Phase 3 (input: {input})" + ))) +} + +pub(crate) async fn list_events( + Path(id): Path, + State(state): State>, +) -> Result>, AppError> { + let registry = state + .integrations + .registry + .lock() + .map_err(|_| AppError::internal("integrations registry lock poisoned"))?; + let manifest = registry + .get(&id) + .ok_or_else(|| AppError::not_found(format!("integration not found: {id}")))?; + Ok(Json( + manifest.events.iter().map(EventSummary::from).collect(), + )) +} + +/// Build a router over the integrations routes, for callers that do not want +/// the full control-plane app. +pub(crate) fn integrations_routes() -> Router> { + Router::new() + .route("/v1/integrations", get(list_integrations)) + .route("/v1/integrations/{id}", get(get_integration)) + .route("/v1/integrations/{id}/capabilities", get(list_capabilities)) + .route( + "/v1/integrations/{id}/capabilities/{name}/invoke", + post(invoke_capability), + ) + .route("/v1/integrations/{id}/events", get(list_events)) +} + +#[cfg(test)] +mod tests { + use super::*; + use code_integrations::mcp::config::McpTransport; + use code_integrations::{ContextFieldSpec, EventSpec, SignatureSpec, TransportConfig}; + use serde_json::json; + + fn sample_manifest() -> IntegrationManifest { + IntegrationManifest { + id: "github".to_string(), + version: "1.0.0".to_string(), + display_name: "GitHub".to_string(), + description: "GitHub integration".to_string(), + transport: McpTransport::Stdio, + transport_config: TransportConfig::Stdio { + command: "gh-mcp".to_string(), + args: vec!["serve".to_string()], + env: std::collections::BTreeMap::new(), + }, + capabilities: vec![code_integrations::CapabilitySpec { + name: "create_issue".to_string(), + description: "Create an issue".to_string(), + input_schema: json!({"type": "object"}), + output_schema: Some(json!({"type": "object"})), + permission_mode: PermissionMode::WorkspaceWrite, + idempotent: false, + }], + auth: Some(AuthSpec::ApiKey { + env_var: "GITHUB_TOKEN".to_string(), + header: None, + }), + events: vec![EventSpec { + name: "issue.opened".to_string(), + description: "An issue was opened".to_string(), + payload_schema: json!({"type": "object"}), + signature: Some(SignatureSpec::HmacSha256 { + secret_env: "GITHUB_WEBHOOK_SECRET".to_string(), + }), + }], + context_fields: vec![ContextFieldSpec { + name: "github.issue".to_string(), + description: "The linked GitHub issue".to_string(), + schema: json!({"type": "object"}), + }], + } + } + + #[test] + fn new_creates_an_empty_registry() { + let state = IntegrationsState::new(); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 0); + } + + #[test] + fn from_json_reads_the_sample_json_over_builtins() { + let sample = sample_manifest(); + let raw = serde_json::to_string(&vec![sample]).expect("serialize sample"); + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(&raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The sample manifest overrides the built-in github (same id), and the + // other built-ins remain. + assert!(registry.get("github").is_some()); + assert_eq!(registry.list().len(), 4); + } + + #[test] + fn from_json_over_builtins_allows_a_custom_id_to_be_added() { + let custom = sample_manifest(); + let custom_raw = serde_json::to_string(&vec![custom]).expect("serialize"); + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(&custom_raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + assert!(registry.get("sentry").is_some()); + } + + #[test] + fn from_json_with_empty_raw_keeps_only_builtins() { + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json("", base); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing `{id}`"); + } + } + + static INTEGRATIONS_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + /// Build a state seeded with the built-in providers, wrapped in an + /// `Arc` for the integrations routes. + fn state_with_builtins() -> Arc { + Arc::new(crate::ServerState { + integrations: std::sync::Arc::new(IntegrationsState::with_builtins()), + ..crate::ServerState::default() + }) + } + + /// Fire a single request at a stateless router and return the response + /// status and body for assertions. The integrations router is built + /// per-call with the given state so each request sees the same snapshot. + async fn oneshot( + state: Arc, + method: &str, + uri: &str, + body: serde_json::Value, + ) -> (axum::http::StatusCode, Vec) { + use axum::body::Body; + use axum::http::Request; + use tower::ServiceExt; + + let router = integrations_routes().with_state(state); + let response = router + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap() + .to_vec(); + (status, bytes) + } + + #[test] + fn load_from_env_reads_the_sample_json() { + let _guard = INTEGRATIONS_ENV_LOCK.lock().expect("env lock poisoned"); + let sample = sample_manifest(); + let raw = serde_json::to_string(&vec![sample]).expect("serialize sample"); + std::env::set_var(INTEGRATIONS_CONFIG_ENV, raw); + let state = IntegrationsState::load_from_env(); + std::env::remove_var(INTEGRATIONS_CONFIG_ENV); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The sample manifest overrides the built-in github (same id), and the + // other built-ins remain. + assert!(registry.get("github").is_some()); + assert_eq!(registry.list().len(), 4); + } + + #[test] + fn load_from_env_skips_invalid_manifests() { + let _guard = INTEGRATIONS_ENV_LOCK.lock().expect("env lock poisoned"); + let raw = r#"[ + {"id": "Bad Id!", "version": "1.0.0", "display_name": "x", "description": "y", + "transport": "stdio", "transport_config": {"kind": "stdio", "command": "c", "args": [], "env": {}}, + "capabilities": [], "auth": null, "events": [], "context_fields": []}, + {"id": "good-one", "version": "2.0.0", "display_name": "Good", "description": "z", + "transport": "http", "transport_config": {"kind": "http", "url": "https://example.com", "headers": {}}, + "capabilities": [{"name": "ping", "description": "p", "input_schema": {"type": "object"}, + "permission_mode": "read_only", "idempotent": true}], + "auth": null, "events": [], "context_fields": []} + ]"#; + let base = code_integrations::builtin_providers().expect("built-ins validate"); + let state = IntegrationsState::from_json(raw, base); + let registry = state.registry.lock().expect("registry is not poisoned"); + // The bad manifest is skipped; the good one is added; built-ins remain. + assert!(registry.get("Bad Id!").is_none()); + assert!(registry.get("good-one").is_some()); + assert_eq!(registry.list().len(), 5); + } + + #[test] + fn with_builtins_seeds_all_four_providers() { + let state = IntegrationsState::with_builtins(); + let registry = state.registry.lock().expect("registry is not poisoned"); + assert_eq!(registry.list().len(), 4); + for id in code_integrations::BUILTIN_PROVIDER_IDS { + assert!(registry.get(id).is_some(), "missing `{id}`"); + } + } + + #[test] + fn summary_dto_serializes_with_expected_fields() { + let summary = IntegrationSummary::from(&sample_manifest()); + let value = serde_json::to_value(&summary).expect("serialize summary"); + assert_eq!(value["id"], "github"); + assert_eq!(value["version"], "1.0.0"); + assert_eq!(value["display_name"], "GitHub"); + assert_eq!(value["capability_count"], 1); + assert_eq!(value["auth_required"], true); + assert_eq!(value["event_count"], 1); + } + + #[test] + fn capability_dto_uses_snake_case_permission_labels() { + let spec = &sample_manifest().capabilities[0]; + let summary = CapabilitySummary::from(spec); + let value = serde_json::to_value(&summary).expect("serialize capability"); + assert_eq!(value["name"], "create_issue"); + assert_eq!(value["permission_mode"], "workspace_write"); + assert_eq!(value["idempotent"], false); + + let mut read_only = spec.clone(); + read_only.permission_mode = PermissionMode::ReadOnly; + assert_eq!( + CapabilitySummary::from(&read_only).permission_mode, + "read_only" + ); + let mut danger = spec.clone(); + danger.permission_mode = PermissionMode::DangerFullAccess; + assert_eq!( + CapabilitySummary::from(&danger).permission_mode, + "danger_full_access" + ); + } + + #[test] + fn event_dto_reports_signature_requirements() { + let event = &sample_manifest().events[0]; + let summary = EventSummary::from(event); + assert_eq!(summary.name, "issue.opened"); + assert!(summary.signature_required); + + let mut unsigned = event.clone(); + unsigned.signature = None; + assert!(!EventSummary::from(&unsigned).signature_required); + } + + #[test] + fn invoke_request_deserializes_an_input_value() { + let request: InvokeRequest = + serde_json::from_value(json!({"input": {"title": "bug", "body": "broken"}})) + .expect("deserialize invoke request"); + assert_eq!(request.input["title"], "bug"); + } + + #[tokio::test] + async fn routes_serve_the_manifest_surface() { + use axum::http::StatusCode; + + // The state is seeded with the built-in providers, so the list is + // non-empty and a built-in id (github) is present. + let state = state_with_builtins(); + let (status, bytes) = oneshot(state.clone(), "GET", "/v1/integrations", json!(null)).await; + assert_eq!(status, StatusCode::OK); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body.as_array().expect("array").len(), 4); + + let (status, _) = + oneshot(state.clone(), "GET", "/v1/integrations/github", json!(null)).await; + assert_eq!(status, StatusCode::OK); + + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/github/capabilities", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::OK); + + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/github/events", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::OK); + + // An unknown integration id still 404s. + let (status, _) = oneshot( + state.clone(), + "GET", + "/v1/integrations/does-not-exist", + json!(null), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + + // ...and a known capability returns the 501 stub. + let (status, _) = oneshot( + state.clone(), + "POST", + "/v1/integrations/github/capabilities/create_pr/invoke", + json!({"input": {"title": "bug"}}), + ) + .await; + assert_eq!(status, StatusCode::NOT_IMPLEMENTED); + + // A capability that does not exist on the integration 404s. + let (status, _) = oneshot( + state.clone(), + "POST", + "/v1/integrations/github/capabilities/missing/invoke", + json!({"input": {}}), + ) + .await; + assert_eq!(status, StatusCode::NOT_FOUND); + } +} diff --git a/crates/code-server/src/lib.rs b/crates/code-server/src/lib.rs index f5044110..f7537f8b 100644 --- a/crates/code-server/src/lib.rs +++ b/crates/code-server/src/lib.rs @@ -15,6 +15,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; mod audit; mod docker_worker; +mod integrations; mod publish; mod worker_access; mod worker_tokens; @@ -389,6 +390,7 @@ pub struct ServerState { publish_workspace_root: PathBuf, // Tasks with a publish in flight. Held so two requests cannot push the same task twice. publishing_tasks: Arc>>, + integrations: Arc, // Off unless configured. See `ServerConfig::allow_local_repos`. allow_local_repos: bool, } @@ -556,6 +558,7 @@ impl ServerState { .unwrap_or_default(), publish_workspace_root: default_server_workspace_root(), publishing_tasks: Arc::new(Mutex::new(HashSet::new())), + integrations: Arc::new(integrations::IntegrationsState::load_from_env()), allow_local_repos: false, }; state.recover_loaded_state(); @@ -1795,6 +1798,7 @@ pub fn app(state: Arc) -> Router { post(connector_interaction), ) .route("/v1/connectors/{connector}/events", post(connector_event)) + .merge(integrations::integrations_routes()) // Starting a login stores a pending state for the single tenant, so it needs the // control-plane key. It is a POST that returns the provider URL as JSON: a browser // navigation cannot send the `x-api-key` header, so the caller opens the returned @@ -5352,6 +5356,13 @@ impl AppError { message: message.into(), } } + + fn not_implemented(message: impl Into) -> Self { + Self { + status: StatusCode::NOT_IMPLEMENTED, + message: message.into(), + } + } } impl IntoResponse for AppError { diff --git a/docs/API_REFERENCES.md b/docs/API_REFERENCES.md index 52b8baec..b2d16540 100644 --- a/docs/API_REFERENCES.md +++ b/docs/API_REFERENCES.md @@ -14,6 +14,7 @@ This comprehensive reference covers the Frontal Code API for programmatic integr - [JavaScript API](#javascript-api) - [Error Handling](#error-handling) - [Rate Limiting](#rate-limiting) +- [Integrations API](#integrations-api) - [Examples](#examples) ## Overview @@ -246,6 +247,58 @@ PATCH /v1/config } ``` +#### Integrations + +List all registered integrations (built-ins + config overrides). + +```http +GET /v1/integrations +Authorization: Bearer +``` + +**Response:** `200 OK` — JSON array of integration summaries. + +Get a single integration manifest by id. + +```http +GET /v1/integrations/{id} +Authorization: Bearer +``` + +**Response:** `200 OK` — full `IntegrationManifest`. `404 Not Found` if the id is unknown. + +List capabilities for an integration. + +```http +GET /v1/integrations/{id}/capabilities +Authorization: Bearer +``` + +**Response:** `200 OK` — JSON array of capability summaries. `404` if the id is unknown. + +Invoke a capability. Stub in v1 — returns `501 Not Implemented`. + +```http +POST /v1/integrations/{id}/capabilities/{name}/invoke +Authorization: Bearer +Content-Type: application/json + +{ + "input": {} +} +``` + +**Response:** `501 Not Implemented`. MCP dispatch for capabilities is planned for a future release. + +List webhook events for an integration. + +```http +GET /v1/integrations/{id}/events +Authorization: Bearer +``` + +**Response:** `200 OK` — JSON array of event definitions. `404` if the id is unknown. + #### Status Get system status and health. diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md new file mode 100644 index 00000000..f7536700 --- /dev/null +++ b/docs/INTEGRATIONS.md @@ -0,0 +1,274 @@ +# Integrations + +Frontal Code integrations are **fully typed, config-driven manifests**. The +server, CLI, and SDKs all consume the same `IntegrationManifest` shape — no +hardcoded provider code beyond the MCP transport layer. + +## Built-in providers + +Four first-party integrations ship as built-in manifests. They are seeded into +the registry on startup and can be overridden by project-level configuration: + +| id | display name | transport | auth | +| --- | --- | --- | --- | +| `github` | GitHub | `http` | API key (`GITHUB_TOKEN`) | +| `linear` | Linear | `http` | API key (`LINEAR_API_KEY`) | +| `posthog` | PostHog | `http` | API key (`POSTHOG_API_KEY`) | +| `sentry` | Sentry | `http` | API key (`SENTRY_AUTH_TOKEN`) | + +### GitHub + +| capability | permission | idempotent | +| --- | --- | --- | +| `create_pr` | `workspace_write` | no | +| `create_issue_comment` | `workspace_write` | no | +| `create_check_run` | `workspace_write` | no | + +### Linear + +| capability | permission | idempotent | +| --- | --- | --- | +| `create_issue_comment` | `workspace_write` | no | + +### PostHog + +| capability | permission | idempotent | +| --- | --- | --- | +| `capture_event` | `workspace_write` | yes | +| `capture_events_batch` | `workspace_write` | yes | + +### Sentry + +| capability | permission | idempotent | +| --- | --- | --- | +| `create_issue` | `workspace_write` | no | +| `create_issue_comment` | `workspace_write` | no | + +## Manifest schema + +An `IntegrationManifest` is a JSON document with the following shape: + +```json +{ + "id": "my-integration", + "version": "1.0.0", + "display_name": "My Integration", + "description": "What this integration does.", + "transport": "http", + "transport_config": { + "kind": "http", + "url": "https://api.example.com", + "headers": {} + }, + "capabilities": [ + { + "name": "do_thing", + "description": "Performs a thing.", + "input_schema": { "type": "object" }, + "output_schema": { "type": "object" }, + "permission_mode": "workspace_write", + "idempotent": false + } + ], + "auth": { + "kind": "api_key", + "env_var": "MY_API_KEY", + "header": "Authorization" + }, + "events": [], + "context_fields": [] +} +``` + +### Fields + +| field | type | required | description | +| --- | --- | --- | --- | +| `id` | `string` | yes | Unique identifier, lowercase alphanumeric + hyphens | +| `version` | `string` | yes | Semver version | +| `display_name` | `string` | yes | Human-readable name | +| `description` | `string` | yes | What the integration does | +| `transport` | `string` | yes | `stdio`, `http`, `sse`, `websocket`, `sdk`, or `managed_proxy` | +| `transport_config` | `object` | yes | Transport-specific config, tagged by `kind` | +| `capabilities` | `array` | yes | Capabilities exposed by the integration | +| `auth` | `object` | no | Auth spec: `none`, `oauth`, or `api_key` | +| `events` | `array` | no | Webhook events this integration can emit | +| `context_fields` | `array` | no | Fields written into task context | + +### Capability fields + +| field | type | required | description | +| --- | --- | --- | --- | +| `name` | `string` | yes | Capability identifier | +| `description` | `string` | yes | What the capability does | +| `input_schema` | `object` | yes | JSON Schema for input validation | +| `output_schema` | `object` | no | JSON Schema for output validation | +| `permission_mode` | `string` | yes | `read_only`, `workspace_write`, or `danger_full_access` | +| `idempotent` | `boolean` | yes | If true, safe to retry on failure | + +### Auth spec variants + +| kind | fields | +| --- | --- | +| `none` | — | +| `api_key` | `env_var` (string), `header` (string, optional) | +| `oauth` | `client_id_env`, `client_secret_env` (optional), `auth_url`, `token_url`, `scopes` (array), `redirect_path` (optional) | + +## Configuration + +### Environment variable + +Set `FRONTAL_CODE_INTEGRATIONS_CONFIG` to a JSON array of manifests. This is +layered **on top of** the built-in providers; a manifest whose `id` collides +with a built-in overrides it. + +```bash +export FRONTAL_CODE_INTEGRATIONS_CONFIG='[ + { + "id": "github", + "version": "2.0.0", + "display_name": "GitHub (custom)", + "description": "Custom GitHub config", + "transport": "http", + "transport_config": { "kind": "http", "url": "https://api.github.com", "headers": {} }, + "capabilities": [], + "auth": null, + "events": [], + "context_fields": [] + } +]' +``` + +### Settings file + +Declare integrations under `mcp.integrations` in `.frontal-code/settings.json`. +The CLI walks up from the current directory to find the settings file. Entries +with the same `id` as a built-in override it. + +```json +{ + "mcp": { + "integrations": { + "linear": { + "id": "linear", + "version": "2.0.0", + "display_name": "Linear (custom)", + "description": "Custom Linear config", + "transport": "http", + "transport_config": { "kind": "http", "url": "https://api.linear.app/graphql", "headers": {} }, + "capabilities": [], + "auth": { "kind": "api_key", "env_var": "LINEAR_API_KEY", "header": "Authorization" }, + "events": [], + "context_fields": [] + } + } + } +} +``` + +## CLI + +```bash +# List all integrations (built-ins + config overrides) +frontal-code integrations list + +# Show a single integration manifest +frontal-code integrations show github + +# List capabilities for an integration +frontal-code integrations capabilities github + +# Validate a manifest file +frontal-code integrations validate ./my-manifest.json +``` + +Add `--json` to any command for machine-readable output. + +## REST API + +| method | path | description | +| --- | --- | --- | +| `GET` | `/v1/integrations` | List all registered integrations | +| `GET` | `/v1/integrations/{id}` | Get a single manifest | +| `GET` | `/v1/integrations/{id}/capabilities` | List capabilities | +| `POST` | `/v1/integrations/{id}/capabilities/{name}/invoke` | Invoke a capability (501 stub in v1) | +| `GET` | `/v1/integrations/{id}/events` | List webhook events | + +### Examples + +```http +GET /v1/integrations +Authorization: Bearer +``` + +**Response:** +```json +[ + { + "id": "github", + "version": "1.0.0", + "display_name": "GitHub", + "description": "Manage pull requests, issues, and checks.", + "capability_count": 3, + "auth_required": true, + "event_count": 0 + } +] +``` + +```http +GET /v1/integrations/github/capabilities +Authorization: Bearer +``` + +**Response:** +```json +[ + { + "name": "create_pr", + "description": "Create a pull request.", + "permission_mode": "workspace_write", + "idempotent": false + } +] +``` + +## SDK clients + +### Rust + +```rust +use frontal_code_sdk::integrations::IntegrationClient; + +let client = IntegrationClient::new("http://localhost:8080"); +let integrations = client.list().await?; +let manifest = client.get("github").await?; +let capabilities = client.capabilities("github").await?; +``` + +### TypeScript + +```typescript +import { IntegrationClient } from "@frontal-labs/code-sdk"; + +const client = new IntegrationClient("http://localhost:8080"); +const integrations = await client.list(); +const manifest = await client.get("github"); +const capabilities = await client.capabilities("github"); +``` + +## Backward compatibility + +The legacy `IntegrationRegistry::call_*` methods in +`crates/code-integrations/src/mcp/integration.rs` remain as deprecated +adapters over the corresponding capabilities for one release. New code should +use the manifest-driven API. The deprecated wrappers currently cover: + +- `call_github_create_pr` → `github.create_pr` +- `call_github_create_issue_comment` → `github.create_issue_comment` +- `call_github_create_check_run` → `github.create_check_run` +- `call_linear_create_issue_comment` → `linear.create_issue_comment` +- `call_graphite_create_stack_comment` → (Graphite is not a built-in; this is + a legacy adapter only) +- `call_slack_post_message` → (Slack is not a built-in; this is a legacy + adapter only) diff --git a/docs/MCP.md b/docs/MCP.md index 056e9afa..019f650c 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -633,13 +633,18 @@ frontal-code prompt "Use Kubernetes to deploy the application and Docker to buil This MCP guide provides comprehensive coverage of MCP integration in Frontal Code, from basic usage to advanced custom server development. -## Native Integrations (GitHub, Linear, Graphite, Slack) +## Native Integrations -Frontal Code has **no hardcoded integrations**. Every external tracker (GitHub PRs, Linear -issues, Graphite stacks, Slack messages) is reached exclusively through an MCP server. -The server acts as an **MCP client**: it resolves a logical integration name -(`github`, `linear`, `graphite`, `slack`) to a configured MCP server and invokes that -server's tools. Adding a new provider requires **no code changes** — only config. +Frontal Code integrations are driven by **typed, declarative manifests** +(`IntegrationManifest`). The server, CLI, and SDKs all consume the same shape. +Four first-party providers ship as built-ins — `github`, `linear`, `posthog`, +and `sentry` — and can be overridden per-project via configuration. Adding a +new provider requires **no code changes** — only a manifest. + +Full reference: [docs/INTEGRATIONS.md](./INTEGRATIONS.md). + +The legacy MCP-based integration layer (used for Graphite and Slack) remains +as a deprecated adapter for one release; new code should use the manifest API. ### Configuration diff --git a/sdks/rust/Cargo.toml b/sdks/rust/Cargo.toml index c83a8f2d..a1db5fbb 100644 --- a/sdks/rust/Cargo.toml +++ b/sdks/rust/Cargo.toml @@ -15,9 +15,11 @@ rust-version = "1.80.0" [dependencies] tokio = { version = "1", features = ["process", "rt", "macros", "io-util", "sync"] } +serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "1" futures-util = "0.3" +reqwest = { version = "0.13", default-features = false, features = ["json", "rustls"] } [dev-dependencies] tempfile = "3" diff --git a/sdks/rust/src/integrations.rs b/sdks/rust/src/integrations.rs new file mode 100644 index 00000000..3b7ff51e --- /dev/null +++ b/sdks/rust/src/integrations.rs @@ -0,0 +1,330 @@ +//! Typed client for the `Frontal Code` integrations HTTP API. +//! +//! The client talks to the `Frontal Code` server (the one the CLI wraps) and lets +//! SDK consumers discover registered integrations and invoke their capabilities +//! with full type safety. All types are self-contained; they mirror the server's +//! wire shape and do not depend on the server crate. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +/// Summary of a registered integration (as returned by `GET /v1/integrations`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct IntegrationSummary { + pub id: String, + pub version: String, + pub display_name: String, + pub description: String, + pub capability_count: usize, + pub auth_required: bool, + pub event_count: usize, +} + +/// Summary of a capability (as returned by `GET /v1/integrations/{id}/capabilities`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct CapabilitySummary { + pub name: String, + pub description: String, + pub permission_mode: String, + pub idempotent: bool, +} + +/// A webhook event summary (as returned by `GET /v1/integrations/{id}/events`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct EventSummary { + pub name: String, + pub description: String, + pub signature_required: bool, +} + +/// Authentication requirements declared by an integration. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct AuthSpec { + /// The auth kind: `"oauth"`, `"api_key"`, or `"none"`. + pub r#type: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub authorize_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scopes: Option>, +} + +/// The full integration manifest as returned by `GET /v1/integrations/{id}`. +/// +/// Fields beyond the well-known ones are preserved in [`Self::extra`] so the +/// client never fails on a newer server shape. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct IntegrationManifest { + pub id: String, + pub version: String, + pub display_name: String, + pub description: String, + pub transport: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transport_config: Option, + #[serde(default)] + pub capabilities: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub auth: Option, + #[serde(default)] + pub events: Vec, + #[serde(flatten)] + pub extra: std::collections::BTreeMap, +} + +/// Errors surfaced by the [`IntegrationClient`]. +#[derive(Debug, thiserror::Error)] +pub enum IntegrationError { + #[error("http request failed: {0}")] + Http(#[from] reqwest::Error), + #[error("server returned status {status}: {body}")] + Server { status: u16, body: String }, + #[error("failed to parse response: {0}")] + Parse(String), +} + +impl IntegrationError { + /// The HTTP status code, when the error came from a non-2xx server response. + pub fn status(&self) -> Option { + match self { + Self::Server { status, .. } => Some(*status), + _ => None, + } + } + + /// The raw response body, when the error came from a non-2xx server response. + pub fn body(&self) -> Option<&str> { + match self { + Self::Server { body, .. } => Some(body), + _ => None, + } + } +} + +/// Typed client for the integrations API. +#[derive(Clone)] +pub struct IntegrationClient { + base_url: String, + http: reqwest::Client, +} + +impl std::fmt::Debug for IntegrationClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("IntegrationClient") + .field("base_url", &self.base_url) + .finish_non_exhaustive() + } +} + +impl IntegrationClient { + /// Create a client pointing at the given server URL. + /// + /// A trailing slash is normalized away. + pub fn new(base_url: &str) -> Self { + Self { + base_url: base_url.trim_end_matches('/').to_string(), + http: reqwest::Client::new(), + } + } + + /// Create a client with a caller-supplied HTTP client (useful for tests and + /// custom TLS configuration). The base URL is normalized the same way as in + /// [`Self::new`]. + pub fn with_client(base_url: &str, http: reqwest::Client) -> Self { + Self { + base_url: base_url.trim_end_matches('/').to_string(), + http, + } + } + + /// The server URL this client is bound to, without a trailing slash. + pub fn base_url(&self) -> &str { + &self.base_url + } + + /// List all registered integrations. + pub async fn list(&self) -> Result, IntegrationError> { + let response = self.http.get(self.url("/v1/integrations")).send().await?; + let value = parse_json(response).await?; + serde_json::from_value(value).map_err(|e| IntegrationError::Parse(e.to_string())) + } + + /// Get the full manifest for one integration. + pub async fn get(&self, id: &str) -> Result { + let response = self + .http + .get(self.url(&format!("/v1/integrations/{id}"))) + .send() + .await?; + let value = parse_json(response).await?; + serde_json::from_value(value).map_err(|e| IntegrationError::Parse(e.to_string())) + } + + /// List capabilities for an integration. + pub async fn capabilities( + &self, + integration_id: &str, + ) -> Result, IntegrationError> { + let response = self + .http + .get(self.url(&format!("/v1/integrations/{integration_id}/capabilities"))) + .send() + .await?; + let value = parse_json(response).await?; + serde_json::from_value(value).map_err(|e| IntegrationError::Parse(e.to_string())) + } + + /// Invoke a capability with the given input. + pub async fn invoke( + &self, + integration_id: &str, + capability: &str, + input: &Value, + ) -> Result { + let response = self + .http + .post(self.url(&format!( + "/v1/integrations/{integration_id}/capabilities/{capability}/invoke" + ))) + .json(input) + .send() + .await?; + let status = response.status().as_u16(); + let body = response.text().await?; + if !(200..300).contains(&status) { + return Err(IntegrationError::Server { status, body }); + } + serde_json::from_str(&body).map_err(|e| IntegrationError::Parse(e.to_string())) + } + + /// List webhook events for an integration. + pub async fn events(&self, integration_id: &str) -> Result, IntegrationError> { + let response = self + .http + .get(self.url(&format!("/v1/integrations/{integration_id}/events"))) + .send() + .await?; + let value = parse_json(response).await?; + serde_json::from_value(value).map_err(|e| IntegrationError::Parse(e.to_string())) + } + + fn url(&self, path: &str) -> String { + format!("{}{path}", self.base_url) + } +} + +/// Send a response and, on success, parse the body as JSON. Non-2xx responses +/// become [`IntegrationError::Server`] with the raw body captured. +async fn parse_json(response: reqwest::Response) -> Result { + let status = response.status().as_u16(); + let body = response.text().await?; + if !(200..300).contains(&status) { + return Err(IntegrationError::Server { status, body }); + } + serde_json::from_str(&body).map_err(|e| IntegrationError::Parse(e.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SUMMARY_JSON: &str = r#"{ + "id": "github", + "version": "1.2.0", + "display_name": "GitHub", + "description": "Issues, PRs, and repos", + "capability_count": 4, + "auth_required": true, + "event_count": 2 + }"#; + + #[test] + fn integration_summary_deserializes_from_sample_json() { + let summary: IntegrationSummary = serde_json::from_str(SUMMARY_JSON).expect("parses"); + assert_eq!(summary.id, "github"); + assert_eq!(summary.version, "1.2.0"); + assert_eq!(summary.display_name, "GitHub"); + assert_eq!(summary.description, "Issues, PRs, and repos"); + assert_eq!(summary.capability_count, 4); + assert!(summary.auth_required); + assert_eq!(summary.event_count, 2); + } + + #[test] + fn capability_summary_deserializes_from_sample_json() { + let json = r#"{ + "name": "create_issue", + "description": "Open an issue", + "permission_mode": "workspace_write", + "idempotent": false + }"#; + let cap: CapabilitySummary = serde_json::from_str(json).expect("parses"); + assert_eq!(cap.name, "create_issue"); + assert_eq!(cap.permission_mode, "workspace_write"); + assert!(!cap.idempotent); + } + + #[test] + fn manifest_preserves_unknown_fields_in_extra() { + let json = r#"{ + "id": "github", + "version": "1.2.0", + "display_name": "GitHub", + "description": "Issues, PRs, and repos", + "transport": "stdio", + "capabilities": [], + "events": [], + "context_fields": [{"name": "owner"}] + }"#; + let manifest: IntegrationManifest = serde_json::from_str(json).expect("parses"); + assert_eq!(manifest.id, "github"); + assert_eq!(manifest.transport, "stdio"); + assert!(manifest.auth.is_none()); + assert_eq!( + manifest + .extra + .get("context_fields") + .and_then(Value::as_array) + .map(Vec::len), + Some(1) + ); + } + + #[test] + fn client_new_strips_trailing_slash_and_exposes_base_url() { + let client = IntegrationClient::new("http://localhost:8790/"); + assert_eq!(client.base_url(), "http://localhost:8790"); + } + + #[test] + fn client_new_keeps_base_url_without_trailing_slash() { + let client = IntegrationClient::new("http://localhost:8790"); + assert_eq!(client.base_url(), "http://localhost:8790"); + } + + #[test] + fn server_error_exposes_status_and_body() { + let error = IntegrationError::Server { + status: 404, + body: "not found".to_string(), + }; + assert_eq!(error.status(), Some(404)); + assert_eq!(error.body(), Some("not found")); + assert_eq!(error.to_string(), "server returned status 404: not found"); + } + + #[test] + fn parse_error_exposes_detail() { + let error = IntegrationError::Parse("bad json".to_string()); + assert_eq!(error.status(), None); + assert_eq!(error.to_string(), "failed to parse response: bad json"); + } +} diff --git a/sdks/rust/src/lib.rs b/sdks/rust/src/lib.rs index 454d49e5..60322bf8 100644 --- a/sdks/rust/src/lib.rs +++ b/sdks/rust/src/lib.rs @@ -14,12 +14,17 @@ mod code; mod config; +mod integrations; mod protocol; mod spawn; mod thread; pub use code::FrontalCode; pub use config::{config_to_args, flatten_config, merge_config, to_toml_literal}; +pub use integrations::{ + AuthSpec, CapabilitySummary, EventSummary, IntegrationClient, IntegrationError, + IntegrationManifest, IntegrationSummary, +}; pub use protocol::{ FrontalCodeEvent, FrontalCodeOptions, InputEntry, ThreadInput, ThreadItem, ThreadOptions, ThreadRunOptions, TurnResult, Usage, diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts index d41aff31..48401d81 100644 --- a/sdks/typescript/src/index.ts +++ b/sdks/typescript/src/index.ts @@ -4,6 +4,14 @@ export { toTomlLiteral, } from "./config.js"; export { FrontalCode } from "./frontal-code.js"; +export type { + AuthSpec, + CapabilitySummary, + EventSummary, + IntegrationManifest, + IntegrationSummary, +} from "./integrations.js"; +export { IntegrationClient, IntegrationError } from "./integrations.js"; export * from "./protocol.js"; export type { BufferedTurn } from "./spawn.js"; export { collectTurn, FrontalCodeCliError, streamEvents } from "./spawn.js"; diff --git a/sdks/typescript/src/integrations.ts b/sdks/typescript/src/integrations.ts new file mode 100644 index 00000000..ae4ebc10 --- /dev/null +++ b/sdks/typescript/src/integrations.ts @@ -0,0 +1,179 @@ +/** + * HTTP client for the Frontal Code integration API. + * + * Talks to a running `frontal-code` server over HTTP using the global + * `fetch` (Node 18+). The endpoints and shapes mirror the server's + * `/v1/integrations` routes. + */ + +/** A lightweight listing entry for a registered integration. */ +export interface IntegrationSummary { + id: string; + version: string; + display_name: string; + description: string; + capability_count: number; + auth_required: boolean; + event_count: number; +} + +/** A single capability exposed by an integration. */ +export interface CapabilitySummary { + name: string; + description: string; + permission_mode: "read_only" | "workspace_write" | "danger_full_access"; + idempotent: boolean; +} + +/** A single event emitted by an integration. */ +export interface EventSummary { + name: string; + description: string; + signature_required: boolean; +} + +/** Authentication requirements for an integration, if any. */ +export interface AuthSpec { + type: "oauth" | "api_key" | "none"; + [key: string]: unknown; +} + +/** The full manifest describing an integration. */ +export interface IntegrationManifest { + id: string; + version: string; + display_name: string; + description: string; + transport: string; + capabilities: CapabilitySummary[]; + auth: AuthSpec | null; + events: EventSummary[]; +} + +/** Error raised when an integration API request fails. */ +export class IntegrationError extends Error { + readonly code: "http" | "server" | "parse"; + readonly status?: number; + readonly body?: string; + constructor(message: string, code: "http" | "server" | "parse", status?: number, body?: string) { + super(message); + this.name = "IntegrationError"; + this.code = code; + this.status = status; + this.body = body; + } +} + +/** + * A typed client for the Frontal Code integration API. + * + * The base URL is normalized (trailing slashes stripped) and each method + * throws {@link IntegrationError} on transport, server, or parse failures. + */ +export class IntegrationClient { + private readonly baseUrl: string; + + constructor(baseUrl: string) { + this.baseUrl = baseUrl.replace(/\/+$/, ""); + } + + /** List all registered integrations. `GET /v1/integrations` */ + async list(): Promise { + return this.getJson("/v1/integrations"); + } + + /** Fetch the manifest for a single integration. `GET /v1/integrations/{id}` */ + async get(id: string): Promise { + return this.getJson(`/v1/integrations/${encodeURIComponent(id)}`); + } + + /** List the capabilities of an integration. `GET /v1/integrations/{id}/capabilities` */ + async capabilities(integrationId: string): Promise { + return this.getJson( + `/v1/integrations/${encodeURIComponent(integrationId)}/capabilities`, + ); + } + + /** List the events of an integration. `GET /v1/integrations/{id}/events` */ + async events(integrationId: string): Promise { + return this.getJson( + `/v1/integrations/${encodeURIComponent(integrationId)}/events`, + ); + } + + /** + * Invoke a capability on an integration. `POST /v1/integrations/{id}/invoke` + * + * @param integrationId The integration to invoke against. + * @param capability The capability name to invoke. + * @param input The capability input payload. + */ + async invoke( + integrationId: string, + capability: string, + input: Record, + ): Promise { + const url = `${this.baseUrl}/v1/integrations/${encodeURIComponent(integrationId)}/invoke`; + let res: Response; + try { + res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", accept: "application/json" }, + body: JSON.stringify({ capability, input }), + }); + } catch (err) { + throw new IntegrationError(`integration request to ${url} failed: ${String(err)}`, "http"); + } + + const body = await res.text(); + if (!res.ok) { + throw new IntegrationError( + `integration invoke failed with status ${res.status}: ${body}`, + "server", + res.status, + body, + ); + } + try { + return JSON.parse(body) as unknown; + } catch { + throw new IntegrationError( + "integration invoke response was not valid JSON", + "parse", + res.status, + body, + ); + } + } + + /** Perform a `GET` request and decode the JSON body. */ + private async getJson(path: string): Promise { + const url = `${this.baseUrl}${path}`; + let res: Response; + try { + res = await fetch(url, { headers: { accept: "application/json" } }); + } catch (err) { + throw new IntegrationError(`integration request to ${url} failed: ${String(err)}`, "http"); + } + + const body = await res.text(); + if (!res.ok) { + throw new IntegrationError( + `integration request failed with status ${res.status}: ${body}`, + "server", + res.status, + body, + ); + } + try { + return JSON.parse(body) as T; + } catch { + throw new IntegrationError( + "integration response was not valid JSON", + "parse", + res.status, + body, + ); + } + } +} diff --git a/sdks/typescript/tests/integrations.test.ts b/sdks/typescript/tests/integrations.test.ts new file mode 100644 index 00000000..bad3ab23 --- /dev/null +++ b/sdks/typescript/tests/integrations.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from "vitest"; +import { IntegrationClient, IntegrationError } from "../src/integrations.js"; + +describe("IntegrationClient", () => { + it("normalizes a trailing slash on the base URL", async () => { + const client = new IntegrationClient("http://localhost:8080/"); + let capturedUrl = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (url: RequestInfo | URL) => { + capturedUrl = String(url); + return new Response(JSON.stringify([]), { status: 200 }); + }), + ); + + await client.list(); + + expect(capturedUrl).toBe("http://localhost:8080/v1/integrations"); + vi.unstubAllGlobals(); + }); + + it("strips multiple trailing slashes", async () => { + const client = new IntegrationClient("http://localhost:8080///"); + let capturedUrl = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (url: RequestInfo | URL) => { + capturedUrl = String(url); + return new Response(JSON.stringify([]), { status: 200 }); + }), + ); + + await client.list(); + + expect(capturedUrl).toBe("http://localhost:8080/v1/integrations"); + vi.unstubAllGlobals(); + }); +}); + +describe("IntegrationError", () => { + it("carries the error code, status, and body", () => { + const err = new IntegrationError("boom", "server", 500, '{"error":"boom"}'); + expect(err).toBeInstanceOf(Error); + expect(err.name).toBe("IntegrationError"); + expect(err.message).toBe("boom"); + expect(err.code).toBe("server"); + expect(err.status).toBe(500); + expect(err.body).toBe('{"error":"boom"}'); + }); + + it("leaves status and body undefined for http errors", () => { + const err = new IntegrationError("refused", "http"); + expect(err.code).toBe("http"); + expect(err.status).toBeUndefined(); + expect(err.body).toBeUndefined(); + }); +}); From 8ce44f93b5472086674d76e3661c1a5410c3b497 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Mon, 16 Nov 2026 02:43:54 +0000 Subject: [PATCH 02/10] docs(integrations): add changeset for typed integrations system --- .changeset/typed-integrations-system.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/typed-integrations-system.md diff --git a/.changeset/typed-integrations-system.md b/.changeset/typed-integrations-system.md new file mode 100644 index 00000000..feb9798a --- /dev/null +++ b/.changeset/typed-integrations-system.md @@ -0,0 +1,6 @@ +--- +'@frontal-labs/code': minor +'@frontal-labs/code-sdk': minor +--- + +Add a typed, manifest-driven integrations system. Integrations are declared as `IntegrationManifest` files and consumed by the server, CLI, and SDKs. Four first-party providers ship as built-ins — `github`, `linear`, `posthog`, and `sentry` — and can be overridden per-project via `FRONTAL_CODE_INTEGRATIONS_CONFIG` or the `mcp.integrations` settings file. New server routes expose `GET /v1/integrations`, `GET /v1/integrations/{id}`, `GET /v1/integrations/{id}/capabilities`, `GET /v1/integrations/{id}/events`, and a stubbed `POST /v1/integrations/{id}/capabilities/{name}/invoke` (501). The CLI gains `integrations list|show|capabilities|validate` commands, and both the Rust and TypeScript SDKs gain an `IntegrationClient`. The legacy `call_*` MCP wrappers remain functional as a deprecated adapter layer for one release. From 73f6403d259eb5e00050265718a70dfc03822ca1 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Mon, 16 Nov 2026 02:51:12 +0000 Subject: [PATCH 03/10] fix(integrations): strip trailing slashes without regex (CodeQL) Replace the trailing-slash regex with a loop using String.prototype.endsWith/slice to silence the CodeQL re2-non-sensitive regex alert on library input. --- sdks/typescript/src/integrations.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdks/typescript/src/integrations.ts b/sdks/typescript/src/integrations.ts index ae4ebc10..8aa623eb 100644 --- a/sdks/typescript/src/integrations.ts +++ b/sdks/typescript/src/integrations.ts @@ -74,7 +74,11 @@ export class IntegrationClient { private readonly baseUrl: string; constructor(baseUrl: string) { - this.baseUrl = baseUrl.replace(/\/+$/, ""); + let normalized = baseUrl; + while (normalized.endsWith("/")) { + normalized = normalized.slice(0, -1); + } + this.baseUrl = normalized; } /** List all registered integrations. `GET /v1/integrations` */ From 7d8b6ef084d38868621bc6f32ae8ee13ffb96d70 Mon Sep 17 00:00:00 2001 From: Gabriel Fonseca Date: Mon, 16 Nov 2026 14:42:46 +0000 Subject: [PATCH 04/10] feat(code-server): replace the Docker worker with Frontal Sandbox sessions Hosted lane workers now run as Frontal Sandbox sessions instead of local Docker containers. The sandbox transport creates one session per task, imports the repo checkout into the session VFS, injects the worker env and task payload, and fires the agent via exec_stream in a background thread (fine-grained events flow back over the control plane via FRONTAL_CODE_SERVER_URL). Sessions are snapshot on clean exit; re-runs fork from the snapshot, or wake the previous session when it is still alive, or start fresh when it is gone. Also lands the self-contained AWS EC2 deployment wrapper under deploy/sandbox/ (OpenTofu + bootstrap/smoke/deploy scripts) and the CI job that shellchecks and fmt-checks it. Removed: - LocalDocker LaneWorkerTransport + crates/code-server/src/docker_worker.rs - infrastructure/compose/docker-compose.docker-worker.yml - docker/worker.Dockerfile kept as a legacy reference only Migrated to the sandbox transport: - compose/.env examples, worker e2e scripts, CONTAINERS/DEVELOPMENT/ TROUBLESHOOTING/worker-architecture docs, .env.example, rust-ci.yml - LaneTransportKind { Sandbox, ToolsAgent } (docker parses to an error) - server_config/server_types/code-events tests (execution_backend=sandbox) New tests: server_config_reads_sandbox_env_vars (sandbox env parsing + transport kind selection). --- .env.example | 16 + .github/workflows/rust-ci.yml | 20 + Cargo.lock | 83 +- Cargo.toml | 2 + PLAN-sandbox-worker-migration.md | 488 ++++ README.md | 2 +- crates/code-events/src/lib.rs | 4 +- .../code-events/tests/event_payload_types.rs | 2 +- crates/code-server/Cargo.toml | 4 + crates/code-server/README.md | 31 +- crates/code-server/src/docker_worker.rs | 242 -- crates/code-server/src/lib.rs | 1971 +++++++---------- crates/code-server/tests/server_config.rs | 19 +- crates/code-server/tests/server_types.rs | 4 +- deploy/sandbox/Makefile | 67 + deploy/sandbox/README.md | 158 ++ deploy/sandbox/config/production.toml | 22 + deploy/sandbox/scripts/bootstrap.sh | 163 ++ deploy/sandbox/scripts/deploy.sh | 46 + deploy/sandbox/scripts/smoke.sh | 96 + deploy/sandbox/terraform/README.md | 76 + deploy/sandbox/terraform/compute.tf | 59 + deploy/sandbox/terraform/firewall.tf | 54 + deploy/sandbox/terraform/main.tf | 23 + deploy/sandbox/terraform/networking.tf | 105 + deploy/sandbox/terraform/outputs.tf | 19 + deploy/sandbox/terraform/providers.tf | 11 + deploy/sandbox/terraform/storage.tf | 52 + deploy/sandbox/terraform/variables.tf | 78 + docker/worker.Dockerfile | 3 + docs/CONTAINERS.md | 100 +- docs/DEVELOPMENT.md | 3 +- docs/README.md | 4 +- docs/TROUBLESHOOTING.md | 12 +- docs/design/conversation-store.md | 2 +- docs/design/worker-architecture.md | 454 +++- infrastructure/README.md | 37 +- infrastructure/compose/.env.example | 13 +- .../compose/docker-compose.docker-worker.yml | 36 - infrastructure/compose/docker-compose.yml | 20 +- scripts/linux-amd64-worker-e2e.sh | 40 +- scripts/worker-e2e.sh | 86 +- 42 files changed, 3001 insertions(+), 1726 deletions(-) create mode 100644 PLAN-sandbox-worker-migration.md delete mode 100644 crates/code-server/src/docker_worker.rs create mode 100644 deploy/sandbox/Makefile create mode 100644 deploy/sandbox/README.md create mode 100644 deploy/sandbox/config/production.toml create mode 100755 deploy/sandbox/scripts/bootstrap.sh create mode 100755 deploy/sandbox/scripts/deploy.sh create mode 100755 deploy/sandbox/scripts/smoke.sh create mode 100644 deploy/sandbox/terraform/README.md create mode 100644 deploy/sandbox/terraform/compute.tf create mode 100644 deploy/sandbox/terraform/firewall.tf create mode 100644 deploy/sandbox/terraform/main.tf create mode 100644 deploy/sandbox/terraform/networking.tf create mode 100644 deploy/sandbox/terraform/outputs.tf create mode 100644 deploy/sandbox/terraform/providers.tf create mode 100644 deploy/sandbox/terraform/storage.tf create mode 100644 deploy/sandbox/terraform/variables.tf delete mode 100644 infrastructure/compose/docker-compose.docker-worker.yml diff --git a/.env.example b/.env.example index 760cf5cb..9fb3dbb5 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,22 @@ FRONTAL_CODE_SANDBOX_ENABLE_DOCKER=0 # Optional Docker image override when FRONTAL_CODE_SANDBOX_ENABLE_DOCKER=1 FRONTAL_CODE_SANDBOX_DOCKER_IMAGE=busybox:1.36 +# Lane transport for hosted tasks. +# sandbox - Frontal Sandbox VM sessions (default, recommended) +# tools-agent - legacy tools.agent transport +# (requires FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1) +#FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox + +# Frontal Sandbox server (used by FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox) +# Base URL of the sandbox server, e.g. http://localhost:8080 (default when unset) +FRONTAL_SANDBOX_BASE_URL=http://localhost:8080 +# Bearer API key (sbx_...) issued by the sandbox server; leave empty for a +# local server with auth disabled +FRONTAL_SANDBOX_API_KEY= +# Public address worker sessions use to call back to code-server +# (defaults to http://localhost:8788 when unset) +FRONTAL_CODE_SERVER_URL=http://localhost:8788 + # Service Configuration # =========================================== diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 3044a338..1f0f1bdc 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -23,6 +23,7 @@ on: - extensions/slack-agent/** - scripts/** - docs/** + - deploy/sandbox/** - sdks/rust/** - sdks/typescript/** - biome.json @@ -144,3 +145,22 @@ jobs: workspaces: . -> target - name: Run workspace clippy run: cargo clippy --workspace + + sandbox-infra: + name: sandbox deploy (shellcheck + fmt) + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v4 + - name: Install shellcheck + run: sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck + - name: Run shellcheck on deploy scripts + run: shellcheck deploy/sandbox/scripts/*.sh + - name: Install pinned OpenTofu + env: + TOFU_VERSION: "1.10.5" + run: | + curl -fsSL -o /tmp/tofu.zip "https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}/tofu_${TOFU_VERSION}_linux_amd64.zip" + sudo unzip -q /tmp/tofu.zip -d /usr/local/bin + tofu version + - name: Check terraform formatting + run: tofu fmt -check deploy/sandbox/terraform diff --git a/Cargo.lock b/Cargo.lock index ee7187cd..d514476a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,12 @@ dependencies = [ "object", ] +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + [[package]] name = "async-compression" version = "0.4.52" @@ -256,6 +262,19 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.4.2", + "cpufeatures 0.3.1", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -703,6 +722,7 @@ dependencies = [ "code-runtime", "code-tools", "code-types", + "frontal-sandbox-sdk", "futures-util", "getrandom 0.4.3", "hex", @@ -710,13 +730,16 @@ dependencies = [ "reqwest", "rustls", "rustls-pki-types", + "sandbox-types", "serde", "serde_json", "sha2 0.11.1", "tokio", "tokio-postgres", "tokio-postgres-rustls", + "tokio-stream", "tower", + "tracing", "webpki-roots 0.26.11", ] @@ -837,6 +860,12 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "constant_time_eq" version = "0.6.1" @@ -1243,6 +1272,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "frontal-sandbox-sdk" +version = "0.2.0" +source = "git+https://github.com/frontal-labs/sandbox#47a87e91fd659afaf16a7cb1ad11556f3b6a88f6" +dependencies = [ + "bytes", + "futures", + "reqwest", + "sandbox-types", + "serde", + "serde_json", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2842,6 +2887,20 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "sandbox-types" +version = "0.0.0" +source = "git+https://github.com/frontal-labs/sandbox#47a87e91fd659afaf16a7cb1ad11556f3b6a88f6" +dependencies = [ + "async-trait", + "blake3", + "bytes", + "serde", + "serde_json", + "thiserror 2.0.21", + "ulid", +] + [[package]] name = "schannel" version = "0.1.30" @@ -3385,6 +3444,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.29.0" @@ -3745,6 +3815,17 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "ulid" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "947dde63b6d514cc5e044edad4e0ca7261afd1099d16c83d942cb2b2f348689c" +dependencies = [ + "rand 0.10.3", + "serde", + "web-time", +] + [[package]] name = "unicase" version = "2.10.0" @@ -4364,7 +4445,7 @@ version = "9.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3476fd497ac8af5b1ec7b51acab7bef380f7e8dd8e0487595bafd839417a302" dependencies = [ - "constant_time_eq", + "constant_time_eq 0.6.1", "crc32fast", "flate2", "getrandom 0.4.3", diff --git a/Cargo.toml b/Cargo.toml index 81db467d..4436b119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ publish = false serde_json = "1" posthog-rs = { version = "=0.33.1", default-features = false } uuid = { version = "1", default-features = false, features = ["v4"] } +frontal-sandbox-sdk = { git = "https://github.com/frontal-labs/sandbox" } +sandbox-types = { git = "https://github.com/frontal-labs/sandbox" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/PLAN-sandbox-worker-migration.md b/PLAN-sandbox-worker-migration.md new file mode 100644 index 00000000..e3f07866 --- /dev/null +++ b/PLAN-sandbox-worker-migration.md @@ -0,0 +1,488 @@ +# Plan: Replace the Docker Worker with Frontal Sandbox (+ Terraform Deployment Infra) + +## Objective + +Wire `https://github.com/frontal-labs/sandbox` in as the execution sandbox for +code-server hosted agents. The current `LocalDocker` `LaneWorkerTransport` +spins up a hardened Docker container per task; this plan **fully replaces it** +with a Frontal Sandbox **session** as the persistent, sandboxed workspace in +which the `frontal-code` agent CLI runs. Sessions start in the **isolate tier** +(Tier 1) and **promote to a QEMU microVM** (Tier 2) when a command needs a real +binary. + +This plan also covers the **deployment infrastructure** that the sandbox server +needs to run — specifically the sandbox repo's `infrastructure/terraform/` +(OpenTofu) provisioning of KVM-capable cloud hosts — so that the code-server +change has an actual place to run. + +## Decisions (from user) + +1. **Depth:** Full replacement of the Docker worker, not a thin new transport. + The `LocalDocker` transport is deprecated and removed; all hosted tasks run + through a Frontal Sandbox session. +2. **Tier:** Start in **isolate tier**, with **VM promotion** for real command + execution (cargo, npm, git, deno, the agent binary, etc.). +3. **SDK:** Use the sandbox repo's Rust SDK `Client` + (`sdks/rust/src/client.rs`), fully usable via the `/v1` HTTP API. Do NOT + depend on the stub `embedded` mode. +4. **Repo import (v1):** Stream files from the local checkout into the session + VFS via `client.write_file` (Option B) rather than `git clone` in-session, to + avoid needing git + network in the VM. +5. **Event streaming:** Keep the agent's existing event protocol — the worker + CLI POSTs fine-grained events to `FRONTAL_CODE_SERVER_URL` / + `/v1/events/ws`; sandbox `exec_stream` only captures top-level stdout/stderr. +6. **Deployment:** Use the sandbox repo's OpenTofu (Terraform-compatible) infra + under `infrastructure/terraform/` to provision a KVM-capable host; code-server + talks to the sandbox server on that host over the allowlisted network. + +## Current state (what we're replacing) + +- `crates/code-server/src/lib.rs`: + - `LaneTransportKind { LocalDocker, ToolsAgent }` (line 167). Default + `LocalDocker`. `parse_lane_transport_kind` (line 176) accepts + `"" | "docker" | "local-docker" | "local_docker"` → `LocalDocker`, and + `tools-agent | tools_agent | agent` → `ToolsAgent`. + - `ServerConfig` (line ~142 `lane_transport_kind`), `ServerConfig::from_env` + (line 222) reads `FRONTAL_CODE_SERVER_LANE_TRANSPORT`. + - `ServerState::new_with_transport_kind*` (lines 427–496) has a `match + lane_transport_kind` (line 474) that builds `LocalDockerLaneWorkerTransport` + or `HostedAgentLaneWorkerTransport`. This is the single wiring point. + - `HostedAgentLaneWorkerTransport` (line 4416) for the `ToolsAgent` kind. + - `LaneWorkerTransport` trait (line 4314): `bootstrap(request) -> + LaneBootstrapResult`, `cancel(worker_id, locator) -> LaneCancellationResult`, + `observe_completion(worker_id, finish_reason, tokens_output) -> + LaneCompletionResult`. + - `LaneExecutionRequest` (line 3867): task_id, prompt, repository, repo_url, + base_ref, branch, lane_role, model, provider, permission_mode, allowed_tools. + - `LaneBootstrapResult` (line ~3855): worker_id, worker_status, signals, + artifacts, checkout_root. + - `LocalDockerLaneWorkerTransport` (line 4510) does: `prepare_checkout` → + `git_head_commit` (baseline) → `write_local_docker_task_payload` → + `worker_access.issue(task_id)` token → `docker.ensure_relay` / + `prepare_task_network` / `launch(DockerLaunchSpec)` running + `frontal-code hosted task run ` with a container env. +- `crates/code-server/src/docker_worker.rs`: Docker CLI command builders, + relay container (`alpine/socat` pinned by digest), per-task network, hardening + args. `CliDockerRunner`, `DockerRunner` trait, `RELAY_ALIAS`, + `task_network_name`, `relay_container_name`. +- `crates/code-server/src/worker_access.rs`: `WorkerAccess`, `issue(task_id)` — + reused unchanged (per-task token + provider proxy env). +- `crates/code-sandbox/`: local per-bash-command isolation (unshare / + sandbox-exec / docker). **Unrelated** to the Frontal Sandbox service; stays + as-is. +- `crates/code-runtime/src/bash.rs`: calls `build_sandbox_command` per bash + command. Unaffected. + +## Target state + +``` +HostedTaskContext + -> LaneWorkerTransport = SandboxLaneWorkerTransport + 1. prepare_checkout (local, for the base_commit baseline only) + 2. worker_access.issue(task_id) -> per-task token + 3. create_session(tier_preference=auto, env, network, initial_snapshot?) + 4. import repo checkout into the session VFS (write_file per file) + 5. write the worker env file + task payload file into the session + 6. exec_stream: run `frontal-code hosted task run ` + - first exec that needs a real binary triggers Tier1 -> Tier2 promotion + 7. stream stdout/stderr back as lane events; fine-grained agent events + flow over the existing FRONTAL_CODE_SERVER_URL /v1/events/ws channel + 8. on completion: read back artifacts (diff vs base_commit), snapshot, sleep + 9. cancel -> destroy_session (best-effort snapshot first) +``` + +The session **is** the persistent workspace: it survives across exec calls, can +sleep/wake between runs, and snapshots give us the baseline + fork for re-runs. +This replaces both the Docker container *and* the local checkout dir. + +## Key technical constraints (from the sandbox repo) + +- **Rust SDK crate:** `frontal-sandbox-sdk` (`sdks/rust/Cargo.toml`, v0.2.0), + depends on `sandbox-types` (path `crates/sandbox-types`). The `Client` + surface: `create_session`, `exec`, `exec_request`, `exec_stream` (NDJSON), + `read_file`, `write_file`, `delete_file`, `snapshot`, `sleep`, `wake`, + `list_snapshots`, `get_snapshot`, `fork`, `fork_command`, `checkpoint`, + `undo`, `get_session`, `destroy_session`, `info`. Only `embedded` mode is a + stub. We use the HTTP client, so no dependency on the unfinished embedded path. +- **API wire format:** snake_case; timestamps RFC 3339; byte fields are arrays + of 0–255 ints (SDK converts to/from `Vec`). Base URL + `http://localhost:8080` (env `SANDBOX_BASE_URL`), Bearer `sbx_` key (env + `SANDBOX_API_KEY`). +- **Isolate tier cannot run host binaries.** `ExecRequest` with a real binary + (git, cargo, npm, the `frontal-code` agent) must promote to a VM. The + `sandbox-promote` crate handles `NeedsVm` → VFS export to ext4 → VM attach. + This is the **highest-risk path** (QEMU/KVM required on a Linux host, + unmeasured promotion latency, ext4 image handoff). +- **Server is early:** in-memory sessions (plus file-backed with `--data-dir`), + no TLS, no rate limiting beyond admission 429s. Not safe to expose publicly. +- **Env is NOT preserved across sleep/wake** (ADR 0010): env values often hold + secrets, so they're dropped. We must re-inject the worker env (server URL, + per-task token, provider proxy) on every wake, or keep env in a file the + agent reads at startup. +- **`SessionConfig` limits/network/env/secret settings that the backend + doesn't yet enforce are refused with 501**, not silently ignored. Only file + size/total/inode/path-depth limits are enforced today. +- **Network policy is default-deny allowlist** — allowlist only the + provider-proxy host + the code-server URL (the worker already uses a proxy + for model traffic). + +## Sandbox Terraform deployment infrastructure (NEW section) + +The sandbox repo ships `infrastructure/` with an OpenTofu (Terraform-compatible) +module for provisioning KVM-capable cloud hosts. The code-server sandbox worker +needs a *running* sandbox server, so deployment of that server is part of this +migration. Reference: `infrastructure/PLAN.md` and `infrastructure/terraform/`. + +### What the terraform module provisions (M4) + +Per `infrastructure/terraform/`: + +- `compute.tf` — one or more KVM-capable instances (`/dev/kvm`, cgroups v2). + These run the `sandbox serve` process (the scheduler/worker in M5). +- `networking.tf` — VPC, subnets, security groups. +- `storage.tf` — S3-compatible object-store bucket (CAS tier) + block volumes + for per-VM overlay images. +- `firewall.tf` — ingress rules: **8080** (API + `/metrics` on the same + listener) and **22** (SSH). + +`versions.tf` / `providers.tf` / `variables.tf` / `outputs.tf` define the +provider (the cloud target), inputs (instance type, region, count, AMI, etc.), +and outputs (the public/private IP or DNS name of the sandbox host, plus the +object-store bucket). + +### How it integrates with code-server + +- code-server talks to the sandbox server over HTTP. The terraform `outputs` + give the host address; that becomes `FRONTAL_SANDBOX_BASE_URL` + (`http://:8080`). The Bearer key (`sbx_…`) becomes + `FRONTAL_SANDBOX_API_KEY`. +- The sandbox session's `network.allow` allowlist must include the code-server's + provider-proxy host + the code-server URL so the in-session agent can POST + events and reach the model. (The sandbox *host* itself is a peer, not an + allowlist entry — the allowlist governs egress from the session VM.) +- Deployment flow (operator): + 1. `tofu init` / `tofu apply` in `infrastructure/terraform/` → provision the + KVM host + object store + firewall. + 2. `ssh` to the host, run `scripts/bootstrap.sh` (deps, KVM, cgroups) and + `scripts/deploy.sh` (install the `sandbox` binary + systemd unit + + `/etc/sandbox/config.toml`). + 3. `scripts/smoke.sh` → create a session, `exec echo hi`, destroy. + 4. Read terraform `outputs` → set `FRONTAL_SANDBOX_BASE_URL` + + `FRONTAL_SANDBOX_API_KEY` in the code-server `.env`, set + `FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox`, restart code-server. + +### Terraform deliverables we add (Phase 3 / new) + +The sandbox repo's `infrastructure/terraform/` is the *upstream* module. We do +not fork it. Instead, in the **frontal-code** repo we add a thin deployment +layer that consumes it: + +- `deploy/sandbox/` (frontal-code repo) — a small wrapper: + - `README.md` — step-by-step operator guide (provision → bootstrap → deploy → + smoke → configure code-server). + - `terraform/` — a wrapper module that `source`s or references the sandbox + repo's `infrastructure/terraform/` (via a `module` block with a local + source path to a vendored copy, or a git source pinned to a rev). It adds + frontal-code-specific variables (e.g. `code_server_egress_allowlist` for the + network policy) and outputs (`sandbox_base_url`, `sandbox_api_key_file`). + - `config/production.toml` — a ready-to-use `sandbox` server TOML config + (`addr = "0.0.0.0:8080"`, `data_dir = "/var/lib/sandbox"`, auth, telemetry) + matching the schema in the sandbox repo's `infrastructure/config/`. + - `smoke.sh` — thin delegator to the sandbox repo's smoke script, or a + self-contained `curl /v1/openapi.json` + session create/exec/destroy. +- `deploy/sandbox/Makefile` — `make provision` (`tofu apply`), `make deploy` + (bootstrap+deploy), `make smoke`, `make destroy`. +- Update `docs/` and `README.md` to document the sandbox deployment + the two + new env vars. + +### Deployment risks & notes + +- **D1 — Terraform/OpenTofu availability.** Operators need `tofu` or + `terraform` on PATH. CI uses whichever is available (the sandbox repo's + `infrastructure/ci/tf-plan.yml` pattern). +- **D2 — Provider pinning.** `providers.tf` pins a cloud provider; changing the + target cloud means updating the provider + variables. Document the default. +- **D3 — No TLS.** The sandbox server has no TLS; the 8080 ingress must be on + a private subnet or behind a TLS-terminating proxy. The firewall module + restricts 8080 to the code-server's security group / CIDR. +- **D4 — Object store.** The CAS tier needs an S3-compatible bucket. For local + dev the server can run with `--data-dir` only (disk CAS); the bucket is for + production durability. + +## Risks & open questions + +| # | Risk | Mitigation | +|---|------|-----------| +| R1 | **Promotion path unproven.** Tier1→Tier2 (VFS→ext4→QEMU) is unmeasured. If it doesn't work end-to-end, the whole plan stalls. | Gate phase 1 on a spike that proves a `sandbox` VM runs `frontal-code --version` and `git --version` in a promoted session. If it fails, fall back to `tier_preference=vm` directly (skip isolate) — still a full replacement, just no Tier1 fast path. | +| R2 | **Platform dependency.** VM tier requires QEMU/KVM on Linux. macOS dev hosts can only run TCG (slow) or not at all. | Document the requirement. CI uses the existing KVM VM workflow pattern. Local macOS dev may be isolate-only until a Linux VM is available. Terraform provisions the KVM host for CI/prod. | +| R3 | **Agent binary must exist in the VM image.** The `frontal-code` CLI + its runtime (Rust toolchain, node, etc.) must be installed in the guest rootfs. | Build a dedicated guest rootfs image (or a snapshot) with the agent + toolchain pre-installed. Use `initial_snapshot` on session create to start from it. This is a new build artifact to manage. | +| R4 | **Network egress from the VM.** The agent needs to reach the model provider (via the server's provider proxy) and the server itself. Sandbox network policy is default-deny allowlist. | Configure `SessionConfig.network.allow` with the server's provider-proxy host + the code-server URL. Only those hosts need allowlisting. | +| R5 | **Event streaming parity.** Current worker streams fine-grained events. Sandbox `exec_stream` gives stdout/stderr as NDJSON. | Keep the agent's existing event protocol: the worker CLI in the sandbox still POSTs to `FRONTAL_CODE_SERVER_URL` (`/v1/events/ws`). The sandbox exec only captures top-level stdout/stderr for logging. This preserves all UI behavior. | +| R6 | **Per-task token + provider proxy env.** The worker needs `FRONTAL_CODE_SERVER_URL`, `FRONTAL_CODE_SERVER_API_KEY`, `FRONTAL_CODE_HOSTED_TASK_FILE`, and provider env. Env is dropped on wake (ADR 0010). | Write the env to a file in the session VFS at bootstrap (e.g. `/workspace/.frontal-code-hosted/env`), and have the agent read it at startup. Re-write on wake. Or keep it in the snapshot's `env` field if supported. | +| R7 | **Artifact extraction.** Today the server reads `manifest_file`/`output_file` from the checkout root and diffs vs `base_commit`. In the sandbox, the repo lives in the session VFS. | After the agent completes, read the manifest/output files back via `client.read_file(session, path)`, and diff against `base_commit` (recorded at bootstrap). Snapshot the session so the final state is preserved for review/fork. | +| R8 | **Cancellation.** `destroy_session` kills the session. A promoted VM mid-exec may leave a half-written state. | On cancel: `destroy_session` + `snapshot` before destroy (best-effort) so the partial work is inspectable. The existing `LaneCancellationResult` shape is preserved. | +| R9 | **Sandbox server availability.** code-server now has a hard dependency on a running `sandbox serve`. If it's down, hosted tasks can't start. | Add a health-check + clear error. Make the sandbox base URL + API key server config (`FRONTAL_SANDBOX_BASE_URL`, `FRONTAL_SANDBOX_API_KEY`). Fail fast with a distinct error code at bootstrap, not mid-task. Terraform provisions the host. | +| R10 | **501 refusals.** `SessionConfig` fields the backend doesn't enforce are 501. | Send only the minimal `SessionConfig` (tier_preference, network, env) and omit `limits`/`secrets` until the backend enforces them. Pin a minimum sandbox-server version. | +| D1 | **Terraform/OpenTofu availability.** | Operators need `tofu` or `terraform` on PATH; CI uses the sandbox repo's `tf-plan.yml` pattern. | +| D2 | **Provider pinning.** | `providers.tf` pins a cloud provider; changing the target cloud means updating provider + variables. Document the default. | +| D3 | **No TLS.** | The 8080 ingress must be on a private subnet or behind a TLS-terminating proxy. Restrict 8080 to the code-server's security group / CIDR in `firewall.tf`. | +| D4 | **Object store.** | The CAS tier needs an S3-compatible bucket in prod. Local dev can use `--data-dir` disk CAS. | + +## Phased plan + +### Phase 0 — Spike: prove promotion works (gate for everything) +**Goal:** de-risk R1/R2/R3 before touching code-server. +1. Stand up a `sandbox serve --addr 127.0.0.1:8080` on a Linux host with QEMU/KVM + (provisioned via the sandbox repo's terraform, or the KVM VM workflow). +2. From a scratch Rust bin (or `cli`): `create_session(tier_preference=auto)`, + then `exec("git --version")`. Confirm the session promotes to a VM and the + command returns exit 0 with real output. +3. Build (or obtain) a guest rootfs image with the `frontal-code` CLI + Rust + toolchain + node installed. Create a session with `initial_snapshot` from it. + Confirm `frontal-code --version` runs in the session. +4. **Exit criteria:** a promoted session runs `git --version` and + `frontal-code --version` and returns correct stdout. If promotion fails, + record the blocker and switch the plan to `tier_preference=vm` (no isolate + fast path). + +### Phase 1 — Add `SandboxLaneWorkerTransport` (new code, no removal yet) +**Goal:** a third transport kind that runs a full hosted task in a sandbox +session, coexisting with `LocalDocker` so we can compare side by side. +1. `crates/code-server/Cargo.toml`: add a dependency on `frontal-sandbox-sdk` + (`sdks/rust` from the sandbox repo, as a `git` dep pinned to a rev) and + `sandbox-types` (for `SessionConfig`, `ExecRequest`, `SessionId`, etc.). + - Decision: `git = "https://github.com/frontal-labs/sandbox"` with a rev. + If the workspace uses `git` deps, register them in the root `Cargo.toml` + `workspace.dependencies` and add to `Cargo.lock`. +2. `crates/code-server/src/lib.rs`: + - Extend `LaneTransportKind` with `Sandbox`. Extend `parse_lane_transport_kind` + (line 176) to accept `"sandbox"`. Update the error message (line 181) to list + `sandbox` as an option. + - Add `SandboxLaneWorkerTransport` struct holding: `Arc`, + `workspace_root` (for the local base_commit baseline), `server_url` (for the + provider-proxy allowlist), `Arc`, and the sandbox config + (base_url, api_key). + - Wire it in `ServerState::new_with_transport_kind*` (line 474 `match`) alongside + the existing `LocalDocker` / `ToolsAgent` arms. + - Add `FRONTAL_SANDBOX_BASE_URL` + `FRONTAL_SANDBOX_API_KEY` to `ServerConfig` + (line ~142) + `ServerConfig::from_env` (line 222) + `.env.example`. +3. Implement `LaneWorkerTransport` for `SandboxLaneWorkerTransport`: + - **`bootstrap(request)`:** + a. Prepare the checkout *locally* (reuse `prepare_checkout`) to compute + `base_commit = git_head_commit(...)` — the baseline for the diff. (The + local checkout is for the baseline only; the *execution* happens in the + sandbox.) + b. `check_hosted_task_policy` (reuse). + c. `worker_access.issue(task_id)` → token (reuse). + d. Build `SessionConfig { tier_preference: Auto, network: allowlist(server + host + provider-proxy host), env: {} }`. (Env goes to a file in step f.) + e. `client.create_session(config)` → `session_id`. + f. Import the repo into the session: stream files from the local checkout + via `client.write_file(session, path, bytes)` for each file in + `prepared.checkout_root` (directory walk). This avoids needing + git+network in the VM and reuses the existing checkout. + g. Write the worker env file into the session + (`/workspace/.frontal-code-hosted/env`) with + `FRONTAL_CODE_SERVER_URL`, `FRONTAL_CODE_SERVER_API_KEY` (token), + provider proxy env, `FRONTAL_CODE_HOSTED_TASK_FILE`. + h. Write the task payload file (reuse `write_local_docker_task_payload`, + adapted to write via `client.write_file`). + i. Return `LaneBootstrapResult { worker_id: session_id, base_commit, ... }`. + - **run:** `client.exec_stream(session, ExecRequest { argv: ["frontal-code", + "hosted", "task", "run", task_id], stream: true, timeout_ms: ... })`. + Consume the NDJSON stream; forward `output` events to the lane event log. + The fine-grained agent events still flow over the existing + `FRONTAL_CODE_SERVER_URL`/`/v1/events/ws` channel (the agent POSTs them + directly), so UI behavior is unchanged. + - **artifacts:** after the exec `exit` event, `client.read_file(session, + manifest_file)` and `output_file`, compute the diff vs `base_commit` + (reuse the existing diff logic in `publish.rs` / lane completion), snapshot + the session (`client.snapshot`), `client.sleep(session)` to release the live + VM while keeping the state. + - **cancel:** `client.destroy_session(session_id)` (best-effort `snapshot` + first). Map to `LaneCancellationResult`. +4. `crates/code-server/Cargo.toml` + root `Cargo.toml`: add `frontal-sandbox-sdk`, + `sandbox-types`, and `reqwest`/`tokio` (already present) as needed. +5. Config: add `FRONTAL_SANDBOX_BASE_URL`, `FRONTAL_SANDBOX_API_KEY` to + `ServerConfig::from_env` (line 222) and `.env.example`. +6. Tests: + - Unit: `parse_lane_transport_kind("sandbox")`, `SandboxLaneWorkerTransport::new`. + - Integration (gated on a running sandbox server, `--ignored` like the KVM + suites): create a session, import a tiny repo, run `echo hello`, read back + stdout. + +### Phase 2 — Persistence & re-run (snapshots, sleep/wake, fork) +**Goal:** make the session the *persistent* workspace, not just a per-task +container. +1. **Re-run / resume:** store `session_id` + `snapshot_id` on + `HostedTaskContext` / the persistence layer. A re-run calls + `client.wake(session_id)` (files come back) or `client.fork(snapshot_id, 1)` + for a fresh copy, then re-injects the env file (env was dropped on wake), + then re-execs. This replaces "spin up a new Docker container + re-clone." +2. **Sleep policy:** idle sessions sleep automatically via + `sandbox serve --idle-sleep-secs` (default 300). Configure the server. Our + `artifacts` step already sleeps after completion. +3. **Fork for parallel lanes:** `LaneExecutionRequest.lane_role` + multi-lane + tasks can `client.fork(snapshot_id, n)` to get N sessions sharing a base, + each independent. Wire this where the current code forks a checkout. +4. Persistence: extend the `ServerState` state file to record + `session_id`/`snapshot_id` per task so a server restart can re-attach. + +### Phase 3 — Remove the Docker worker + land the Terraform deployment +**Goal:** complete the "full replacement" and give operators a deployable +sandbox server. +1. Deprecate `LaneTransportKind::LocalDocker` (keep the variant for a release, + emit a warning in `from_env` if someone still sets it). +2. Delete `LocalDockerLaneWorkerTransport`, `docker_worker.rs` + (`CliDockerRunner`, relay, per-task network, hardening args), and the Docker + launch plumbing in `lib.rs`. +3. Remove Docker-specific config: `FRONTAL_CODE_WORKER_RELAY_IMAGE`, the relay + container, `worker_hardening_args`, etc. (Keep anything the sandbox path + reuses.) +4. **Land the sandbox terraform deployment** (see "Sandbox Terraform deployment + infrastructure" section): `deploy/sandbox/` wrapper module + config + smoke + + Makefile + docs. Wire `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY` + from terraform outputs. +5. Update `.env.example`, `docs/`, README, and the `Bazel` targets + (`crates/code-server/BUILD.bazel`) to drop Docker deps and add the sandbox + server as a required service. +6. Update CI: add a sandbox-server service (the KVM VM workflow pattern) for the + integration tests; the existing Docker-in-Docker CI step is no longer needed + for the worker. Add `infrastructure/**` terraform validation to CI (the + sandbox repo's `tf-plan.yml` pattern). +7. Migration note: operators must run a `sandbox serve` instance (provisioned + via terraform) and set `FRONTAL_SANDBOX_BASE_URL`/`FRONTAL_SANDBOX_API_KEY`. + +### Phase 4 — Hardening & ops (optional, follow-up) +1. **In-VM bash isolation:** the `code-sandbox` per-bash-command wrapper is + redundant once the agent runs inside the sandbox VM. Evaluate removing it or + keeping it as defense-in-depth (the VM is already an isolation boundary). +2. **TLS + auth hardening:** the sandbox server has no TLS. Terminate TLS at a + reverse proxy or add TLS to the server before any non-local deployment. +3. **Quotas & metering:** wire `GET /v1/usage` into the billing/tracker + reporting (`tracker_reporting.rs`). +4. **Trace integration:** `GET /v1/sessions/{id}/trace` gives a command trace + (ADR 0007). Feed it into the existing audit trail. + +## Files touched (summary) + +- `crates/code-server/Cargo.toml` — add `frontal-sandbox-sdk`, `sandbox-types`. +- `crates/code-server/src/lib.rs` — `LaneTransportKind::Sandbox`, + `parse_lane_transport_kind`, `SandboxLaneWorkerTransport`, config + (`FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`), wiring (line 474 + match), then Docker removal (Phase 3). +- `crates/code-server/src/docker_worker.rs` — **removed** (Phase 3). +- `crates/code-server/src/worker_access.rs` — reused unchanged. +- `crates/code-server/BUILD.bazel` — deps + Docker removal. +- `.env.example` — sandbox env vars. +- Root `Cargo.toml` — workspace dep for `frontal-sandbox-sdk`/`sandbox-types` + if pinned. +- `deploy/sandbox/` — **new**: terraform wrapper module, `config/production.toml`, + `smoke.sh`, `Makefile`, `README.md` (Phase 3). +- `docs/` + README — architecture + deployment update. +- CI (`.github/workflows/`, `.buildkite/`) — sandbox service + terraform + validation, drop Docker. + +## Success criteria + +- A hosted task with a repository runs end-to-end in a Frontal Sandbox session: + repo imported, agent runs (promoted to VM), diff produced, PR/publish path + works, all UI events stream as before. +- A re-run resumes from a snapshot (wake or fork) without re-cloning. +- Cancellation destroys the session and preserves partial work via snapshot. +- `LocalDocker` transport and `docker_worker.rs` are removed; no Docker + dependency remains for the worker. +- The sandbox terraform deployment provisions a KVM host; `tofu apply` is clean; + `scripts/smoke.sh` passes; code-server is reachable to the sandbox server via + `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`. +- The existing test suite (minus Docker-specific tests) passes; new sandbox + integration tests pass on the KVM CI. + +## Progress + +### Phase 1 — `SandboxLaneWorkerTransport` ✅ (code complete, compiles) +- `Sandbox` variant in `LaneTransportKind` + parse arm (`"sandbox"`). +- `ServerConfig.sandbox_base_url` / `sandbox_api_key` + `from_env` wiring + (`FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`). +- `ServerState` match arm constructs the transport. +- `SandboxLaneWorkerTransport` impl: `bootstrap` (create session → import + checkout via `write_file` → write env + payload → kick off exec in a + background thread), `observe_completion`, `cancel` (destroys session). +- Git deps pinned: `frontal-sandbox-sdk` v0.2.0 + `sandbox-types` at + `frontal-labs/sandbox` commit `47a87e91`. +- `cargo check -p code-server` passes clean. +- `.env.example` updated with the two new env vars. + +### Terraform deployment wrapper ✅ (created under `deploy/sandbox/`) +- 15 files: `terraform/*.tf` (9), `config/production.toml`, 3 scripts + (`bootstrap.sh`, `deploy.sh`, `smoke.sh`), `Makefile`, 2 READMEs. +- Self-contained AWS EC2 (KVM) + S3 (CAS) wrapper; does not fork upstream + `infrastructure/terraform/`. +- `terraform fmt -check` clean; tomllib parses; shellcheck clean. + +### Phase 0 — Spike ⏳ (blocked) +- Needs a Linux host with QEMU/KVM (this macOS box can only do slow TCG). +- Gates full confidence in the isolate→VM promotion path. + +### Phase 2 — Persistence & re-run ✅ (code complete, compiles) +- `resume()` trait method on `LaneWorkerTransport` (default impl = `bootstrap`). +- Sandbox override: fork from last run's snapshot (`client.fork`) if one exists, + else `get_session` liveness probe → wake parked session + re-import, else + fresh session + full re-import. +- Env file re-injected on every start (fresh, woken, or forked); ADR 0010 env-dropped-on-wake + handled by writing `/workspace/.frontal-code-hosted/env` each time. +- `SandboxTaskState.snapshot_id` stored after clean `Exit{exit_code: 0}` in the + background exec thread. + +### Phase 3 — Docker worker removal ✅ (code complete, compiles) +- `LocalDocker` enum variant + `LocalDockerTransport` + `DockerRunner` deleted + from `lib.rs`; `crates/code-server/src/docker_worker.rs` deleted. +- `LaneTransportKind` parse: `docker`/`local-docker`/`local_docker` → removal + error message; `""`/`sandbox` → Sandbox (default); `tools-agent`/`agent` → ToolsAgent. +- `check_callback_port` deleted (not re-purposed). +- `write_local_docker_task_payload` + `write_worker_env_file` deleted. +- `FRONTAL_CODE_SERVER_DOCKER_IMAGE` / `FRONTAL_CODE_SERVER_CALLBACK_URL` / + `FRONTAL_CODE_SERVER_DOCKER_NETWORK` env vars removed from config parsing. +- `HostedDockerTaskPayload` renamed `HostedLaneTaskPayload` (shared by sandbox + transport + hosted-agent transport). +- Zero `docker` references in `lib.rs` except the intentional parse-error string + and the parser test asserting docker is refused. +- `docker/worker.Dockerfile` marked legacy (kept for reference). +- `infrastructure/compose/docker-compose.docker-worker.yml` deleted; + `docker-compose.yml` updated to sandbox env; `infrastructure/README.md` + rewritten for sandbox mode. +- E2E scripts rewritten: `scripts/worker-e2e.sh` + `scripts/linux-amd64-worker-e2e.sh` + — sandbox transport, no Docker daemon requirement for worker containers. +- CI: `rust-ci.yml` gains `deploy/sandbox/**` push path + `sandbox-infra` job + (shellcheck + `tofu fmt -check`). +- `cargo check -p code-server` passes clean. + +### Phase 0 — Spike ⏭ (skipped per user instruction) +- Isolate→VM promotion unvalidated on this macOS box (no QEMU/KVM). +- R1/R2 risks carried; Linux+QEMU/KVM spike deferred to a follow-up if the user + wants promotion confidence before shipping. + +### Phase 4 — Hardening, docs, infra alignment ✅ +- `docs/design/worker-architecture.md` rewritten to the sandbox transport + (worker → session wording; token-store wording corrected — tokens are hashed + in Postgres, so the sandbox transport requires `FRONTAL_CODE_SERVER_DATABASE_URL`). +- `docs/CONTAINERS.md`, `docs/TROUBLESHOOTING.md`, `docs/DEVELOPMENT.md`, + `docs/README.md`, root `README.md`, `docs/design/conversation-store.md` + updated to session wording. +- `crates/code-server/README.md`: "Local Docker Worker Mode" section replaced + with "Sandbox Worker Mode". +- `infrastructure/README.md`: "Local Docker Worker Mode" replaced with "Sandbox + Worker Mode"; worker-image profile references removed. +- `deploy/sandbox/README.md`: smoke.sh troubleshooting bullet fixed. +- `.env.example`: sandbox vars added (`FRONTAL_SANDBOX_BASE_URL`, + `FRONTAL_SANDBOX_API_KEY`, `FRONTAL_CODE_SERVER_URL`); lane-transport value + comment added. + +## Immediate next steps + +1. Run `cargo test -p code-server` and fix until clean. +2. Stage + commit (Phase 1+2+3+terraform together, or per-phase — decide at + commit time). Everything is currently UNCOMMITTED. +3. Clean up the `code-sandbox-rebuild` worktree once the working-dir file is final. +4. (Optional) Linux+QEMU/KVM spike to validate isolate→VM promotion (R1/R2) + before shipping. diff --git a/README.md b/README.md index 2352c23c..71de9bc0 100644 --- a/README.md +++ b/README.md @@ -309,7 +309,7 @@ See [`./USAGE.md`](./USAGE.md) for usage examples and run `cargo run -p cli -- - - **repo** - repository lifecycle, checkout management, and source tree preparation - **runtime** - `ConversationRuntime`, config loading, session persistence, permission policy, MCP client lifecycle, system prompt assembly, usage tracking - **sandbox** - filesystem isolation, namespace restrictions, and security policies -- **server** - hosted task APIs, event streaming, Docker lane execution, and policy-driven recovery +- **server** - hosted task APIs, event streaming, sandbox-session lane execution, and policy-driven recovery - **telemetry** - session trace events, usage analytics, and monitoring infrastructure - **tools** - tool specs + execution: Bash, ReadFile, WriteFile, EditFile, GlobSearch, GrepSearch, WebSearch, WebFetch, Agent, TodoWrite, NotebookEdit, Skill, ToolSearch, and runtime-facing tool discovery - **training** - style learning, dataset building, profile training, and code adaptation diff --git a/crates/code-events/src/lib.rs b/crates/code-events/src/lib.rs index 4ff57a4c..4ef15099 100644 --- a/crates/code-events/src/lib.rs +++ b/crates/code-events/src/lib.rs @@ -622,7 +622,7 @@ mod tests { published_branch: Some("frontal-code/fix-flake".into()), pr_url: Some("https://github.com/acme/payments/pull/42".into()), pr_number: Some(42), - execution_backend: Some("local_docker".into()), + execution_backend: Some("sandbox".into()), worker_id: Some("worker-1".into()), worker_status: Some("running".into()), orphan_policy: Some(AppliedOrphanPolicy { @@ -655,7 +655,7 @@ mod tests { "https://github.com/acme/payments/pull/42" ); assert_eq!(serialized["pr_number"], 42); - assert_eq!(serialized["execution_backend"], "local_docker"); + assert_eq!(serialized["execution_backend"], "sandbox"); assert_eq!(serialized["worker_id"], "worker-1"); assert_eq!(serialized["worker_status"], "running"); assert_eq!(serialized["orphan_policy"]["source"], "default"); diff --git a/crates/code-events/tests/event_payload_types.rs b/crates/code-events/tests/event_payload_types.rs index 51b4f7f5..67db1925 100644 --- a/crates/code-events/tests/event_payload_types.rs +++ b/crates/code-events/tests/event_payload_types.rs @@ -132,7 +132,7 @@ fn hosted_task_event_summary_full() { graphite_stack_id: Some("stack-1".into()), graphite_head_branch: Some("feature/head".into()), graphite_base_branch: Some("main".into()), - execution_backend: Some("local_docker".into()), + execution_backend: Some("sandbox".into()), priority: Some("high".into()), plan_id: Some("plan-1".into()), plan_kind: Some("feature".into()), diff --git a/crates/code-server/Cargo.toml b/crates/code-server/Cargo.toml index 8acd5475..f2984a03 100644 --- a/crates/code-server/Cargo.toml +++ b/crates/code-server/Cargo.toml @@ -11,6 +11,8 @@ name = "code-server" [dependencies] axum = { version = "0.8", features = ["ws"] } futures-util = "0.3" +tokio-stream = "0.1" +tracing = "0.1" code-events = { path = "../code-events" } code-integrations = { path = "../code-integrations" } code-orchestrator = { path = "../code-orchestrator" } @@ -31,6 +33,8 @@ tokio-postgres-rustls = "0.13" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } rustls-pki-types = "1" webpki-roots = "0.26" +frontal-sandbox-sdk.workspace = true +sandbox-types.workspace = true [dev-dependencies] tower = { version = "0.5", features = ["util"] } diff --git a/crates/code-server/README.md b/crates/code-server/README.md index 28ce3198..0df35b0d 100644 --- a/crates/code-server/README.md +++ b/crates/code-server/README.md @@ -91,29 +91,32 @@ curl "http://127.0.0.1:8788/v1/policies/orphans?repository=repo-fast-policy&sour curl "http://127.0.0.1:8788/v1/policies/orphans?repository=repo-prod&priority=high" ``` -## Local Docker Worker Mode +## Sandbox Worker Mode -`FRONTAL_CODE_SERVER_LANE_TRANSPORT` selects how lanes run. The default is `local-docker`. +`FRONTAL_CODE_SERVER_LANE_TRANSPORT` selects how lanes run. The default is `sandbox`. -- `local-docker` (default): each task runs in its own worker container, built from `docker/worker.Dockerfile`. The container runs `frontal-code hosted task run`. Provider API keys and the server's control-plane key never enter the container. The worker gets a per-task token and a model proxy URL, and the server adds the real provider key when it forwards each request. See `docs/CONTAINERS.md`. -- `tools-agent`: runs the agent loop inside the server process. Agents have the server's host privileges, so use this only for trusted tasks. -- There is no simulated transport. Unknown values fail at startup. +- `sandbox` (default): each task runs in its own Frontal Sandbox session. The session's VFS holds the checkout under `/workspace`. Provider API keys and the server's control-plane key never enter the session. The worker gets a per-task token and a model proxy URL, and the server adds the real provider key when it forwards each request. See `docs/CONTAINERS.md` and `docs/design/worker-architecture.md`. +- `tools-agent`: runs the agent loop inside the server process. Agents have the server's host privileges, so use this only for trusted tasks. Gated behind `FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1`. +- Unknown values fail at startup. -For `local-docker`, set: +For `sandbox`, set: -- `FRONTAL_CODE_SERVER_LANE_TRANSPORT=local-docker` +- `FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox` - `FRONTAL_CODE_SERVER_WORKSPACE_ROOT=/path/to/project/.frontal-code/server/workspaces` -- `FRONTAL_CODE_SERVER_DOCKER_IMAGE=frontal-code-worker:local` -- `FRONTAL_CODE_SERVER_CALLBACK_URL=http://host.docker.internal:8788` +- `FRONTAL_SANDBOX_BASE_URL=http://localhost:8080` — the sandbox server the server talks to +- `FRONTAL_SANDBOX_API_KEY=sbx_…` — the Bearer key for the sandbox server +- `FRONTAL_CODE_SERVER_URL=http://:8788` — the address sessions use to call back +- `FRONTAL_CODE_SERVER_DATABASE_URL=postgres://…` — required (worker tokens are stored in Postgres) In that mode, the control plane: -1. prepares an isolated repo checkout under `FRONTAL_CODE_SERVER_WORKSPACE_ROOT` -2. writes `.frontal-code-hosted/task.json` into the checkout -3. launches `frontal-code hosted task run TASK_ID` inside the worker image -4. expects the worker container to call `POST /v1/tasks/:task_id/complete` back to the server +1. creates an isolated sandbox session with a network policy that allowlists the server host +2. streams the repo checkout into `/workspace` in the session VFS (skipping `.git`) +3. writes `.frontal-code-hosted/task.json` and `.frontal-code-hosted/env` into the VFS +4. launches `frontal-code hosted task run TASK_ID` in the session +5. expects the worker to call `POST /v1/tasks/:task_id/complete` back to the server -For the local hosted stack and worker-image build flow, see `infrastructure/README.md`. +For the sandbox terraform deployment, see `deploy/sandbox/README.md`. ## Development diff --git a/crates/code-server/src/docker_worker.rs b/crates/code-server/src/docker_worker.rs deleted file mode 100644 index eb6f9ed0..00000000 --- a/crates/code-server/src/docker_worker.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Docker commands for worker containers, as data. -//! -//! The builders here return argument lists, so the security settings can be checked by -//! tests without a Docker daemon. `CliDockerRunner` in `lib.rs` runs them. -//! -//! Isolation model: -//! -//! - Each task gets its own internal network. An internal network has no route to the -//! internet and no route to the host. -//! - A relay container is attached to every task network under the alias -//! `frontal-relay`. It forwards one TCP port to the server and nothing else. It is -//! the only way out for a worker, and it cannot be used as a general proxy. -//! - Model traffic goes through the server's provider proxy, so the worker does not -//! need the internet for model calls. -//! - The container runs with no Linux capabilities, no privilege escalation, a read-only -//! root filesystem, bounded memory, CPU, and process count, and writable space only in -//! the task checkout and two tmpfs mounts. - -/// Container name of the relay that serves `port`. One relay per server port, so two -/// servers with different ports on one Docker host do not replace each other's relay. -pub(crate) fn relay_container_name(port: u16) -> String { - format!("frontal-code-relay-{port}") -} - -/// Network alias the relay answers to inside a task network. Workers use this host name. -pub(crate) const RELAY_ALIAS: &str = "frontal-relay"; - -/// Label that marks task networks, so idle ones can be found and removed. -pub(crate) const TASK_NETWORK_LABEL: &str = "frontal-code.task_network"; - -/// Label on a task network that records the server port that created it. A sweep only -/// touches networks of its own server. -pub(crate) const NETWORK_RELAY_PORT_LABEL: &str = "frontal-code.relay_port"; - -/// Label on a task network with its creation time in Unix seconds. A sweep leaves young -/// networks alone, because a launch may be between creating the network and starting its -/// worker. -pub(crate) const NETWORK_CREATED_LABEL: &str = "frontal-code.created_at"; - -/// Label on worker containers that names their network, so `stop` can clean it up. -pub(crate) const TASK_NETWORK_CONTAINER_LABEL: &str = "frontal-code.task_network"; - -/// Label on the relay that records the port it forwards, so a changed port recreates it. -pub(crate) const RELAY_PORT_LABEL: &str = "frontal-code.relay_port"; - -/// Writable tmpfs for the worker's temporary files. -const TMP_TMPFS: &str = "/tmp:rw,nosuid,nodev,size=512m"; - -/// Writable tmpfs for the worker user's home directory. -const HOME_TMPFS: &str = "/home/frontal-code:rw,nosuid,nodev,size=1g"; - -/// Name of a task's internal network. Task ids are sanitized to Docker's name rules. -pub(crate) fn task_network_name(task_id: &str) -> String { - let safe: String = task_id - .chars() - .map(|c| { - if c.is_ascii_alphanumeric() || c == '-' || c == '_' { - c.to_ascii_lowercase() - } else { - '-' - } - }) - .take(48) - .collect(); - format!("frontal-task-{safe}") -} - -/// Arguments for starting the relay. It forwards `port` on the relay to the same port on -/// the host, where the server listens. -pub(crate) fn relay_run_args(image: &str, port: u16) -> Vec { - vec![ - "run".to_string(), - "-d".to_string(), - "--name".to_string(), - relay_container_name(port), - "--restart".to_string(), - "unless-stopped".to_string(), - "--add-host".to_string(), - "host.docker.internal:host-gateway".to_string(), - "--cap-drop".to_string(), - "ALL".to_string(), - "--security-opt".to_string(), - "no-new-privileges:true".to_string(), - "--read-only".to_string(), - "--pids-limit".to_string(), - "64".to_string(), - "--memory".to_string(), - "128m".to_string(), - "--label".to_string(), - "frontal-code.role=relay".to_string(), - "--label".to_string(), - format!("frontal-code.relay_port={port}"), - image.to_string(), - format!("TCP-LISTEN:{port},fork,reuseaddr"), - format!("TCP4:host.docker.internal:{port}"), - ] -} - -/// Security flags for a worker container. Every one is deliberate. -pub(crate) fn worker_hardening_args() -> Vec { - [ - // No Linux capabilities. The worker runs as an unprivileged user. - "--cap-drop", - "ALL", - // No setuid or file-capability escalation inside the container. - "--security-opt", - "no-new-privileges:true", - // Root filesystem is read-only. Writes go to the checkout or to tmpfs. - "--read-only", - "--tmpfs", - TMP_TMPFS, - "--tmpfs", - HOME_TMPFS, - // Bounded resources, so one worker cannot starve the host. - "--pids-limit", - "512", - "--memory", - "4g", - "--memory-swap", - "4g", - "--cpus", - "2", - "--ulimit", - "nofile=4096:8192", - ] - .into_iter() - .map(str::to_string) - .collect() -} - -/// Arguments for a worker container. The environment is passed from a file, not on the -/// command line. -pub(crate) fn worker_run_args( - network: &str, - checkout_root: &str, - env_file: &str, - task_id: &str, - image: &str, - command: &[String], -) -> Vec { - let mut args = vec![ - "run".to_string(), - "-d".to_string(), - "--rm".to_string(), - "--workdir".to_string(), - "/workspace".to_string(), - "--volume".to_string(), - format!("{checkout_root}:/workspace"), - "--network".to_string(), - network.to_string(), - "--label".to_string(), - format!("frontal-code.task_id={task_id}"), - "--label".to_string(), - format!("{TASK_NETWORK_CONTAINER_LABEL}={network}"), - "--env-file".to_string(), - env_file.to_string(), - ]; - args.extend(worker_hardening_args()); - args.push(image.to_string()); - args.extend(command.iter().cloned()); - args -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn task_network_names_are_safe_docker_names() { - assert_eq!( - task_network_name("task_6af0597f_1"), - "frontal-task-task_6af0597f_1" - ); - let escaped = task_network_name("a/../../b"); - assert!(escaped.starts_with("frontal-task-a") && escaped.ends_with('b')); - assert!(!escaped.contains('/') && !escaped.contains('.')); - assert!(task_network_name(&"x".repeat(200)).len() <= "frontal-task-".len() + 48); - } - - #[test] - fn worker_container_is_hardened_and_sees_no_host_socket() { - let args = worker_run_args( - "frontal-task-t1", - "/tmp/checkout", - "/tmp/worker.env", - "t1", - "frontal-code-worker:local", - &["code".to_string(), "hosted".to_string()], - ); - let joined = args.join(" "); - assert!(joined.contains("--cap-drop ALL")); - assert!(joined.contains("--security-opt no-new-privileges:true")); - assert!(joined.contains("--read-only")); - assert!(joined.contains("--network frontal-task-t1")); - assert!(joined.contains("--env-file /tmp/worker.env")); - assert!(joined.contains("--memory 4g")); - assert!(joined.contains("--pids-limit 512")); - // Never privileged, never the Docker socket, never the host network. - assert!(!args.iter().any(|arg| arg == "--privileged")); - assert!(!joined.contains("docker.sock")); - assert!(!args.iter().any(|arg| arg == "host")); - // The image and command come last. - assert_eq!( - args[args.len() - 2..], - ["code".to_string(), "hosted".to_string()] - ); - } - - #[test] - fn no_credentials_appear_on_the_command_line() { - let args = worker_run_args("n", "/c", "/e", "t", "img", &[]); - assert!(!args - .iter() - .any(|arg| arg.contains("_API_KEY") || arg.contains("_TOKEN"))); - } - - #[test] - fn default_relay_image_is_pinned_by_digest() { - assert!(crate::DEFAULT_RELAY_IMAGE.contains("@sha256:")); - assert!(!crate::DEFAULT_RELAY_IMAGE.ends_with(":latest")); - } - - #[test] - fn relay_forwards_one_port_and_is_hardened() { - let args = relay_run_args("alpine/socat", 8788); - assert!(args.contains(&"TCP-LISTEN:8788,fork,reuseaddr".to_string())); - assert!(args.contains(&"TCP4:host.docker.internal:8788".to_string())); - assert!(args.windows(2).any(|pair| pair == ["--cap-drop", "ALL"])); - assert!(args.contains(&"--read-only".to_string())); - } - - #[test] - fn each_server_port_gets_its_own_relay() { - // Two servers on one Docker host must not replace each other's relay. - assert_eq!(relay_container_name(8788), "frontal-code-relay-8788"); - assert_ne!(relay_container_name(8788), relay_container_name(8789)); - let args = relay_run_args("alpine/socat", 8789); - assert!(args - .windows(2) - .any(|pair| pair == ["--name", "frontal-code-relay-8789"])); - } -} diff --git a/crates/code-server/src/lib.rs b/crates/code-server/src/lib.rs index f7537f8b..b51f0353 100644 --- a/crates/code-server/src/lib.rs +++ b/crates/code-server/src/lib.rs @@ -10,11 +10,11 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::Path as FsPath; use std::path::PathBuf; use std::process::Command; +use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; mod audit; -mod docker_worker; mod integrations; mod publish; mod worker_access; @@ -65,6 +65,12 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Map, Value}; use sha2::Sha256; use tokio::sync::broadcast; +use frontal_sandbox_sdk::Client as SandboxClient; +use sandbox_types::policy::PolicyAction; +use sandbox_types::{ + ExecEvent, ExecRequest, NetworkPolicy, NetworkPolicyRule, SessionConfig, SessionId, + SessionStatus, SnapshotId, TierPreference, +}; const DEFAULT_EVENT_REPLAY_LIMIT: usize = 100; const ORPHANED_WORKER_STATUS: &str = "orphaned"; @@ -142,10 +148,12 @@ pub struct ServerConfig { pub lane_transport_kind: LaneTransportKind, pub api_key: Option, pub workspace_root: PathBuf, - pub docker_image: String, - pub docker_server_url: String, + /// Base URL of the Frontal Sandbox server the worker sessions run on. + pub sandbox_base_url: String, + /// Bearer API key (`sbx_…`) the sandbox server requires. + pub sandbox_api_key: String, pub reconcile_interval: Option, - /// Postgres holds worker tokens. Required for the `local-docker` lane transport. + /// Postgres holds worker tokens. Required for the `sandbox` lane transport. pub database_url: Option, /// CA file that signed the database certificate, for a private CA. Optional. pub database_ca_file: Option, @@ -165,8 +173,8 @@ pub struct ServerConfig { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LaneTransportKind { - /// Each lane runs in a worker container (`docker/worker.Dockerfile`). - LocalDocker, + /// Each lane runs in a Frontal Sandbox session (`deploy/sandbox/` infra). + Sandbox, /// Each lane runs an agent loop inside the server process. ToolsAgent, } @@ -175,10 +183,14 @@ pub enum LaneTransportKind { /// an unknown value is an error, so a typo cannot silently fake lane execution. fn parse_lane_transport_kind(raw: &str) -> Result { match raw.trim().to_ascii_lowercase().as_str() { - "" | "docker" | "local-docker" | "local_docker" => Ok(LaneTransportKind::LocalDocker), + "" | "sandbox" => Ok(LaneTransportKind::Sandbox), "tools-agent" | "tools_agent" | "agent" => Ok(LaneTransportKind::ToolsAgent), + "docker" | "local-docker" | "local_docker" => Err( + "FRONTAL_CODE_SERVER_LANE_TRANSPORT `docker`/`local-docker`/`local_docker` was removed with the Docker worker; use `sandbox` (Frontal Sandbox sessions) or `tools-agent`" + .to_string(), + ), other => Err(format!( - "unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT `{other}`; use `local-docker` (worker containers) or `tools-agent`" + "unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT `{other}`; use `sandbox` (Frontal Sandbox sessions) or `tools-agent`" )), } } @@ -186,7 +198,7 @@ fn parse_lane_transport_kind(raw: &str) -> Result { /// Refuses `tools-agent` unless the operator opted in with `FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT`. fn check_host_agent_opt_in(kind: LaneTransportKind, allowed: bool) -> Result<(), String> { if kind == LaneTransportKind::ToolsAgent && !allowed { - return Err("FRONTAL_CODE_SERVER_LANE_TRANSPORT=tools-agent runs agents with the server's privileges; set FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1 to allow it, or use local-docker".to_string()); + return Err("FRONTAL_CODE_SERVER_LANE_TRANSPORT=tools-agent runs agents with the server's privileges; set FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT=1 to allow it, or use sandbox".to_string()); } Ok(()) } @@ -196,11 +208,11 @@ impl Default for ServerConfig { Self { bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8788), event_replay_limit: DEFAULT_EVENT_REPLAY_LIMIT, - lane_transport_kind: LaneTransportKind::LocalDocker, + lane_transport_kind: LaneTransportKind::Sandbox, api_key: None, workspace_root: default_server_workspace_root(), - docker_image: default_server_docker_image(), - docker_server_url: default_server_docker_server_url(), + sandbox_base_url: default_sandbox_base_url(), + sandbox_api_key: String::new(), reconcile_interval: Some(Duration::from_secs(15)), database_url: None, database_ca_file: None, @@ -242,7 +254,8 @@ impl ServerConfig { } // `tools-agent` runs the agent inside the server process with its privileges. It is // refused unless the operator opts in, for example because the server already runs - // inside an isolated VM. `local-docker` (the default) isolates each task in a container. + // inside an isolated VM. `sandbox` (the default) isolates each task in a Frontal + // Sandbox session. let host_agent_allowed = env::var("FRONTAL_CODE_SERVER_ALLOW_HOST_AGENT") .is_ok_and(|value| matches!(value.trim(), "1" | "true")); check_host_agent_opt_in(config.lane_transport_kind, host_agent_allowed)?; @@ -261,17 +274,16 @@ impl ServerConfig { } } - if let Ok(image) = env::var("FRONTAL_CODE_SERVER_DOCKER_IMAGE") { - let image = image.trim(); - if !image.is_empty() { - config.docker_image = image.to_string(); - } - } - - if let Ok(url) = env::var("FRONTAL_CODE_SERVER_CALLBACK_URL") { + if let Ok(url) = env::var("FRONTAL_SANDBOX_BASE_URL") { let url = url.trim().trim_end_matches('/'); if !url.is_empty() { - config.docker_server_url = url.to_string(); + config.sandbox_base_url = url.to_string(); + } + } + if let Ok(key) = env::var("FRONTAL_SANDBOX_API_KEY") { + let key = key.trim(); + if !key.is_empty() { + config.sandbox_api_key = key.to_string(); } } @@ -447,8 +459,6 @@ impl ServerState { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ) } @@ -465,17 +475,14 @@ impl ServerState { orphan_auto_cancel_after: Option, orphan_policy_rules: Vec, workspace_root: PathBuf, - docker_image: String, - docker_server_url: String, worker_access: WorkerAccess, ) -> Self { let publish_workspace_root = workspace_root.clone(); let worker_access = Arc::new(worker_access); let lane_transport: Arc = match lane_transport_kind { - LaneTransportKind::LocalDocker => Arc::new(LocalDockerLaneWorkerTransport::new( + LaneTransportKind::Sandbox => Arc::new(SandboxLaneWorkerTransport::new( workspace_root, - docker_image, - docker_server_url, + sandbox_server_url(), worker_access.clone(), )), LaneTransportKind::ToolsAgent => Arc::new(HostedAgentLaneWorkerTransport), @@ -573,7 +580,7 @@ impl ServerState { ) -> Self { Self::with_lane_transport_kind( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, ) } @@ -636,7 +643,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, Some(state_file), @@ -659,7 +666,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, None, @@ -683,7 +690,7 @@ impl ServerState { ) -> Self { Self::build( event_replay_limit, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, lane_transport, None, None, @@ -881,12 +888,20 @@ fn default_server_workspace_root() -> PathBuf { ) } -fn default_server_docker_image() -> String { - "frontal-code-worker:local".to_string() +fn default_sandbox_base_url() -> String { + "http://localhost:8080".to_string() } -fn default_server_docker_server_url() -> String { - "http://host.docker.internal:8788".to_string() +/// Public address the sandbox guest network uses to reach this server. It is injected into +/// every session as `FRONTAL_CODE_SERVER_URL` and allowlisted in the session network policy, +/// so it must be resolvable from the guest network (not `localhost` on a real deployment). +fn sandbox_server_url() -> String { + match env::var("FRONTAL_CODE_SERVER_URL") + .map(|value| value.trim().trim_end_matches('/').to_string()) + { + Ok(value) if !value.is_empty() => value, + _ => "http://localhost:8788".to_string(), + } } fn current_unix_timestamp_secs() -> u64 { @@ -1273,6 +1288,10 @@ pub struct HostedTaskContext { pub worker_manifest_file: Option, #[serde(skip_serializing_if = "Option::is_none")] pub worker_output_file: Option, + /// The sandbox session id of the last run, so a re-run can fork from the run's + /// snapshot instead of re-importing the checkout. + #[serde(skip_serializing_if = "Option::is_none")] + pub sandbox_session_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -1867,39 +1886,14 @@ fn check_auth_posture(config: &ServerConfig) -> Result<(), String> { Ok(()) } -/// Local Docker workers reach this server through the relay, which forwards to the port in -/// `docker_server_url`. That port must be the one the server listens on. A URL without a -/// port, or with another one, would send every worker to the wrong place, so refuse to start. -fn check_callback_port(config: &ServerConfig) -> Result<(), String> { - if config.lane_transport_kind != LaneTransportKind::LocalDocker { - return Ok(()); - } - let url = &config.docker_server_url; - let authority = url.rsplit("//").next().unwrap_or(url); - let explicit = authority - .rsplit_once(':') - .and_then(|(_, port)| port.parse::().ok()); - match explicit { - Some(port) if port == config.bind_addr.port() => Ok(()), - Some(port) => Err(format!( - "FRONTAL_CODE_SERVER_CALLBACK_URL ({url}) names port {port}, but the server listens on {}. Workers would reach the wrong port.", - config.bind_addr.port() - )), - None => Err(format!( - "FRONTAL_CODE_SERVER_CALLBACK_URL ({url}) must include the port the server listens on ({}).", - config.bind_addr.port() - )), - } -} - pub async fn serve(config: ServerConfig) -> Result<(), Box> { check_auth_posture(&config)?; // Worker tokens live in Postgres. Start-up fails if it is unreachable, rather than - // running workers that cannot authenticate. - let worker_store = if config.lane_transport_kind == LaneTransportKind::LocalDocker { + // running sessions that cannot authenticate. + let worker_store = if config.lane_transport_kind == LaneTransportKind::Sandbox { let url = config.database_url.as_deref().ok_or( - "FRONTAL_CODE_SERVER_DATABASE_URL is required for worker lanes (worker tokens are stored in Postgres)", + "FRONTAL_CODE_SERVER_DATABASE_URL is required for the sandbox lane transport (worker tokens are stored in Postgres)", )?; let tls = worker_tokens::tls_mode_for(url, config.database_ca_file.clone())?; Some(worker_tokens::WorkerTokenStore::connect(url, tls)?) @@ -1930,8 +1924,6 @@ pub async fn serve(config: ServerConfig) -> Result<(), Box Result<(), Box, - env: BTreeMap, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -struct HostedDockerTaskPayload { - #[serde(skip_serializing_if = "Option::is_none")] - base_commit: Option, - /// Shell commands the worker may run without asking, as permission rules. - bash_allow: Vec, - /// Shell commands the worker may never run. - bash_deny: Vec, - task_id: String, - prompt: String, - #[serde(skip_serializing_if = "Option::is_none")] - repository: Option, - #[serde(skip_serializing_if = "Option::is_none")] - repo_url: Option, - #[serde(skip_serializing_if = "Option::is_none")] - base_ref: Option, - #[serde(skip_serializing_if = "Option::is_none")] - branch: Option, - #[serde(skip_serializing_if = "Option::is_none")] - model: Option, - #[serde(skip_serializing_if = "Option::is_none")] - provider: Option, - #[serde(skip_serializing_if = "Option::is_none")] - permission_mode: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - allowed_tools: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct DockerLaunchResult { - container_id: String, -} - /// Refuses a hosted task whose provider, model, or permission mode the workspace's /// project policy does not allow. /// /// The checkout is the repository being worked on, so its `.frontal-code` settings -/// are read here, together with this server's user settings. The worker container -/// applies the full policy when it starts, including auth method rules. +/// are read here, together with this server's user settings. The worker applies the +/// full policy when it starts, including auth method rules. fn check_hosted_task_policy( checkout_root: &FsPath, request: &LaneExecutionRequest, @@ -4004,316 +3958,21 @@ fn ceiling_label(mode: ResolvedPermissionMode) -> &'static str { } } -trait DockerRunner: Send + Sync + fmt::Debug { - fn launch(&self, spec: &DockerLaunchSpec) -> Result; - - fn stop(&self, container_id: &str) -> Result<(), String>; - - /// Starts the relay that workers use to reach the server. Test runners need nothing. - fn ensure_relay(&self, _image: &str, _port: u16) -> Result<(), String> { - Ok(()) - } - - /// Creates a task's internal network and attaches the relay to it. - fn prepare_task_network(&self, _network: &str) -> Result<(), String> { - Ok(()) - } - - /// Removes a task network. Best effort: the network may still be in use. - fn remove_task_network(&self, _network: &str) {} - - /// Removes task networks that no worker is using. - fn sweep_idle_task_networks(&self) {} -} - -#[derive(Debug)] -struct CliDockerRunner { - /// The server port the relay forwards to. It names this server's relay and marks its - /// task networks. - relay_port: u16, -} - -/// Writes the container environment to a file only this user can read. Passing values -/// as `--env` would expose them in the process list. Docker reads the file while -/// creating the container, so the caller deletes the directory right after `docker run`. -fn write_worker_env_file(env: &BTreeMap) -> Result { - let mut contents = String::new(); - for (key, value) in env { - if value.contains('\n') || value.contains('\r') { - return Err(format!( - "worker environment value for {key} contains a newline" - )); - } - contents.push_str(key); - contents.push('='); - contents.push_str(value); - contents.push('\n'); - } - - let mut bytes = [0_u8; 8]; - getrandom::fill(&mut bytes).map_err(|error| format!("random source: {error}"))?; - let dir = env::temp_dir().join(format!("frontal-code-worker-{}", hex::encode(bytes))); - fs::create_dir_all(&dir).map_err(|error| format!("failed to create env dir: {error}"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&dir, fs::Permissions::from_mode(0o700)) - .map_err(|error| format!("failed to restrict env dir: {error}"))?; - } - let path = dir.join("worker.env"); - let mut options = fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let mut file = options - .open(&path) - .map_err(|error| format!("failed to create env file: {error}"))?; - std::io::Write::write_all(&mut file, contents.as_bytes()) - .map_err(|error| format!("failed to write env file: {error}"))?; - Ok(path) -} - -impl CliDockerRunner { - fn docker(args: &[String]) -> Result { - let output = Command::new("docker") - .args(args) - .output() - .map_err(|error| format!("failed to run docker: {error}"))?; - if output.status.success() { - return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()); - } - Err(String::from_utf8_lossy(&output.stderr).trim().to_string()) - } - - fn docker_str(args: &[&str]) -> Result { - Self::docker( - &args - .iter() - .map(|arg| (*arg).to_string()) - .collect::>(), - ) - } - - fn relay_port(port: u16) -> String { - port.to_string() - } -} - -impl DockerRunner for CliDockerRunner { - fn launch(&self, spec: &DockerLaunchSpec) -> Result { - let env_file = write_worker_env_file(&spec.env)?; - let env_dir = env_file.parent().map(PathBuf::from); - let args = docker_worker::worker_run_args( - &spec.network, - &spec.checkout_root.display().to_string(), - &env_file.display().to_string(), - &spec.task_id, - &spec.image, - &spec.command, - ); - let output = Self::docker(&args); - if let Some(dir) = env_dir { - let _ = fs::remove_dir_all(dir); - } - let container_id = output.map_err(|error| format!("docker run failed: {error}"))?; - if container_id.is_empty() { - return Err("docker run returned an empty container id".to_string()); - } - // Remove the task's network as soon as its worker exits. Without this, a finished - // task's network stays until the next sweep. - let relay = self.relay_container(); - let network = spec.network.clone(); - let waited = container_id.clone(); - std::thread::spawn(move || { - let _ = Self::docker_str(&["wait", &waited]); - remove_network_when_free(&relay, &network); - }); - Ok(DockerLaunchResult { container_id }) - } - - fn stop(&self, container_id: &str) -> Result<(), String> { - // Read the task network first: the container's labels are gone once it is removed. - let network = Self::docker_str(&[ - "inspect", - "--format", - &format!( - "{{{{index .Config.Labels \"{}\"}}}}", - docker_worker::TASK_NETWORK_CONTAINER_LABEL - ), - container_id, - ]) - .ok() - .filter(|name| !name.is_empty() && name != ""); - Self::docker_str(&["stop", container_id]) - .map_err(|error| format!("docker stop failed: {error}"))?; - if let Some(network) = network { - self.remove_task_network(&network); - } - Ok(()) - } - - fn ensure_relay(&self, image: &str, port: u16) -> Result<(), String> { - let relay = docker_worker::relay_container_name(port); - let wanted = Self::relay_port(port); - let running = Self::docker_str(&["inspect", "--format", "{{.State.Running}}", &relay]); - let current_port = Self::docker_str(&[ - "inspect", - "--format", - &format!( - "{{{{index .Config.Labels \"{}\"}}}}", - docker_worker::RELAY_PORT_LABEL - ), - &relay, - ]); - if running.as_deref() == Ok("true") && current_port.as_deref() == Ok(wanted.as_str()) { - return Ok(()); - } - // Not running, or forwarding a different port: replace it. - let _ = Self::docker_str(&["rm", "-f", &relay]); - Self::docker(&docker_worker::relay_run_args(image, port)) - .map(|_| ()) - .map_err(|error| format!("failed to start the worker relay: {error}")) - } - - fn prepare_task_network(&self, network: &str) -> Result<(), String> { - let created = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); - let task_label = format!("{}={network}", docker_worker::TASK_NETWORK_LABEL); - let port_label = format!( - "{}={}", - docker_worker::NETWORK_RELAY_PORT_LABEL, - self.relay_port - ); - let created_label = format!("{}={created}", docker_worker::NETWORK_CREATED_LABEL); - match Self::docker_str(&[ - "network", - "create", - "--internal", - "--label", - &task_label, - "--label", - &port_label, - "--label", - &created_label, - network, - ]) { - Ok(_) => {} - Err(error) if error.contains("already exists") => {} - Err(error) => return Err(format!("failed to create the task network: {error}")), - } - match Self::docker_str(&[ - "network", - "connect", - "--alias", - docker_worker::RELAY_ALIAS, - network, - &self.relay_container(), - ]) { - Ok(_) => Ok(()), - Err(error) if error.contains("already exists") => Ok(()), - Err(error) => Err(format!("failed to attach the relay: {error}")), - } - } - - fn remove_task_network(&self, network: &str) { - remove_network_now(&self.relay_container(), network); - } - - fn sweep_idle_task_networks(&self) { - let task_filter = format!("label={}", docker_worker::TASK_NETWORK_LABEL); - let port_filter = format!( - "label={}={}", - docker_worker::NETWORK_RELAY_PORT_LABEL, - self.relay_port - ); - let Ok(names) = Self::docker_str(&[ - "network", - "ls", - "--filter", - &task_filter, - "--filter", - &port_filter, - "--format", - "{{.Name}}", - ]) else { - return; - }; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_secs()); - for name in names.lines().filter(|name| !name.is_empty()) { - // A network younger than this may belong to a launch still in progress. - let created = Self::docker_str(&[ - "network", - "inspect", - "--format", - &format!( - "{{{{index .Labels \"{}\"}}}}", - docker_worker::NETWORK_CREATED_LABEL - ), - name, - ]) - .ok() - .and_then(|value| value.parse::().ok()); - if created.is_none_or(|created| now.saturating_sub(created) < SWEEP_MIN_AGE_SECS) { - continue; - } - // An idle task network has nothing attached (a crash before the relay joined) or - // only the relay. A worker still attached means its task is running. - let attached = Self::docker_str(&[ - "network", - "inspect", - "--format", - "{{len .Containers}}", - name, - ]); - if matches!(attached.as_deref(), Ok("0" | "1")) { - self.remove_task_network(name); - } - } - } -} - -/// Minimum age before a task network without a running worker is treated as abandoned. -const SWEEP_MIN_AGE_SECS: u64 = 120; - -/// Disconnects the relay from a task network and removes it. Ignores errors: a missing -/// network is already clean, and a sweep retries anything left behind. -fn remove_network_now(relay: &str, network: &str) { - let _ = CliDockerRunner::docker_str(&["network", "disconnect", "-f", network, relay]); - let _ = CliDockerRunner::docker_str(&["network", "rm", network]); -} - -/// Removes a task network once its worker has gone. The container's `--rm` removal can -/// lag the exit by a moment, and Docker refuses to remove a network with endpoints, so -/// retry briefly. -fn remove_network_when_free(relay: &str, network: &str) { - for _ in 0..30 { - remove_network_now(relay, network); - let gone = - CliDockerRunner::docker_str(&["network", "inspect", "--format", "{{.Id}}", network]) - .is_err(); - if gone { - return; - } - std::thread::sleep(std::time::Duration::from_secs(1)); - } -} - -impl CliDockerRunner { - /// The name of this server's relay container. - fn relay_container(&self) -> String { - docker_worker::relay_container_name(self.relay_port) - } -} - trait LaneWorkerTransport: Send + Sync + fmt::Debug { fn bootstrap(&self, request: &LaneExecutionRequest) -> Result; + /// Re-runs a task on the workspace it already has. The default starts from + /// scratch (a fresh container per run); transports whose workspace is a + /// persistent session (sandbox) override this to wake or fork the previous + /// session, re-injecting its env file. + fn resume( + &self, + request: &LaneExecutionRequest, + _previous: Option<&str>, + ) -> Result { + self.bootstrap(request) + } + fn cancel( &self, worker_id: &str, @@ -4506,103 +4165,253 @@ impl LaneWorkerTransport for HostedAgentLaneWorkerTransport { } } + +/// Per-task state for the sandbox lane transport. The session id doubles as the +/// lane worker id reported to the lane orchestrator. `snapshot_id` records where +/// the last run finished: a re-run forks from it so the new session starts from +/// the previous run's final state instead of the original import. `checkout_root` +/// and `base_commit` will be used by the diff the lane orchestrator computes +/// against the worker's changes (the session VFS is the worker's copy of the +/// checkout); they are stored now so the state is complete once that lands. #[derive(Debug)] -struct LocalDockerLaneWorkerTransport { - workspace_root: PathBuf, - image: String, - /// The worker's view of the server: the relay, by its alias on the task network. - server_url: String, - relay_image: String, - relay_port: u16, - worker_access: Arc, - docker: Arc, +struct SandboxTaskState { + /// The session that ran the task. The map key is also the session id, so + /// this field is redundant today; kept for the future diff path. + #[allow(dead_code)] + session_id: SessionId, + /// The local checkout the repository was imported from. Used for the diff + /// against the worker's changes (the session VFS is the worker's copy). + #[allow(dead_code)] + checkout_root: PathBuf, + /// The commit the worker's changes are measured against. `None` for a + /// scratch workspace without a repository. + #[allow(dead_code)] + base_commit: Option, + /// The snapshot the last run finished on. Set by the exec-drain thread when + /// the agent exits cleanly; read by `resume` to decide fork vs wake. + snapshot_id: Option, } -/// Port a server URL points at, or the default server port. -fn port_from_server_url(url: &str) -> u16 { - url.trim_end_matches('/') - .rsplit(':') - .next() - .and_then(|port| port.parse().ok()) - .unwrap_or(8788) +/// The task handoff written to `/workspace/.frontal-code-hosted/task.json` inside a +/// sandbox session. The worker inside the session reads it to start the agent. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct HostedLaneTaskPayload { + task_id: String, + prompt: String, + #[serde(skip_serializing_if = "Option::is_none")] + repository: Option, + #[serde(skip_serializing_if = "Option::is_none")] + repo_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + base_ref: Option, + #[serde(skip_serializing_if = "Option::is_none")] + branch: Option, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + provider: Option, + #[serde(skip_serializing_if = "Option::is_none")] + permission_mode: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + allowed_tools: Vec, + /// Shell commands the worker may run without asking, as permission rules. + bash_allow: Vec, + /// Shell commands the worker may never run. + bash_deny: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + base_commit: Option, } -/// The relay image, pinned by digest. A tag can move; a digest cannot. Override it with -/// `FRONTAL_CODE_WORKER_RELAY_IMAGE` after reviewing the image you choose. -pub(crate) const DEFAULT_RELAY_IMAGE: &str = - "alpine/socat@sha256:e8a6e8868a8e6e01d9c2c321a8f8d4474d2d741d962462e5c5a124a8e66a0055"; - -fn relay_image_from_env() -> String { - env::var("FRONTAL_CODE_WORKER_RELAY_IMAGE") - .ok() - .filter(|value| !value.trim().is_empty()) - .unwrap_or_else(|| DEFAULT_RELAY_IMAGE.to_string()) +/// Runs hosted-agent tasks inside a Frontal Sandbox session. The session's network +/// policy allowlists only this code server, and the per-task worker token is written +/// to a file inside the session (the session's env is dropped on wake, so env vars +/// cannot be relied on). +#[derive(Debug)] +struct SandboxLaneWorkerTransport { + workspace_root: PathBuf, + /// The actual code-server URL the sandbox session reaches directly. + server_url: String, + worker_access: Arc, + client: Arc, + tasks: Arc>>, } -impl LocalDockerLaneWorkerTransport { - fn new( - workspace_root: PathBuf, - image: String, - server_url: String, - worker_access: Arc, - ) -> Self { - let relay_port = port_from_server_url(&server_url); - Self::with_runner( +impl SandboxLaneWorkerTransport { + /// Builds a client from `FRONTAL_SANDBOX_BASE_URL` / `FRONTAL_SANDBOX_API_KEY`, + /// reading both from the environment. + fn new(workspace_root: PathBuf, server_url: String, worker_access: Arc) -> Self { + let base_url = env::var("FRONTAL_SANDBOX_BASE_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| "http://localhost:8080".to_string()); + let api_key = env::var("FRONTAL_SANDBOX_API_KEY") + .ok() + .filter(|value| !value.trim().is_empty()); + Self { workspace_root, - image, server_url, worker_access, - Arc::new(CliDockerRunner { relay_port }), - ) + client: Arc::new(SandboxClient::new(base_url, api_key)), + tasks: Arc::new(Mutex::new(HashMap::new())), + } } - fn with_runner( - workspace_root: PathBuf, - image: String, - server_url: impl AsRef, - worker_access: Arc, - docker: Arc, - ) -> Self { - let relay_port = port_from_server_url(server_url.as_ref()); - Self { - workspace_root, - image, - server_url: format!("http://{}:{relay_port}", docker_worker::RELAY_ALIAS), - relay_image: relay_image_from_env(), - relay_port, - worker_access, - docker, + /// Looks up the snapshot a task's last run finished on, if one was + /// recorded. Used by `resume` to decide between forking from the snapshot + /// and waking the previous session. + fn snapshot_of(&self, session: &SessionId) -> Option { + self.tasks + .lock() + .unwrap() + .get(&session.to_string()) + .and_then(|state| state.snapshot_id) + } + + /// Builds the session config used for every task session. The session may + /// only reach this code server; everything else is default-deny. Provider + /// traffic flows through the server's model proxy. + fn session_config(&self) -> SessionConfig { + let mut network = NetworkPolicy::default(); + network.add_rule(NetworkPolicyRule { + action: PolicyAction::Allow, + host_pattern: Self::server_host_allowlist(&self.server_url), + ports: None, + }); + SessionConfig { + network, + tier_preference: TierPreference::Auto, + // `limits` and `secrets` are intentionally left at their defaults: + // the sandbox server currently returns 501 for both. + ..SessionConfig::default() } } -} -impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { - fn bootstrap(&self, request: &LaneExecutionRequest) -> Result { - // A task with a repository gets a checkout on its own branch. A task without one - // gets an empty scratch workspace, so a plain prompt can still run in a worker. - let prepared = match lane_repo_source(request) { - Some(source) => prepare_checkout(&RepoCheckoutRequest { - workspace_root: self.workspace_root.clone(), - checkout_id: request.task_id.clone(), - source, - repository: request.repository.clone(), - base_ref: request.base_ref.clone(), - branch: request.branch.clone(), - }) - .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, - None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + /// Starts a session for a task, preferring `previous` when given: + /// a snapshot from the last run is forked (the new session starts from the + /// previous run's final state, and the old session is left untouched), + /// otherwise the previous session is woken and its files restored. When the + /// previous session no longer exists the session is created fresh and the + /// local checkout is re-imported. The env file is (re)written on every + /// start because the session drops its env on wake (ADR 0010). + async fn attach_to_session( + &self, + request: &LaneExecutionRequest, + previous: Option, + checkout_root: Option<&FsPath>, + base_commit: Option, + ) -> Result<(SessionId, usize, String), String> { + let client = &self.client; + match previous { + Some(id) if self.snapshot_of(&id).is_some() => { + let snapshot = self.snapshot_of(&id).expect("checked above"); + let sessions = client + .fork(snapshot, 1) + .await + .map_err(|error| format!("failed to fork sandbox session from snapshot: {error}"))?; + let session = sessions.into_iter().next().ok_or_else(|| { + "sandbox server returned no session id for fork".to_string() + })?; + // Re-inject the env file so the re-run sees fresh credentials. + self.write_session_files(client, session, request, base_commit) + .await?; + Ok((session, 0, "forked from snapshot".to_string())) + } + Some(id) => { + // Probe first: `wake` on an already-awake session is rejected + // with InvalidState, so decide based on the session's current + // state rather than on the outcome of waking it. + match client.get_session(id).await { + Ok(state) if state.status == SessionStatus::Running => { + // Already awake: re-inject env and proceed (ADR 0010). + self.write_session_files(client, id, request, base_commit) + .await?; + Ok((id, 0, "resumed awake session".to_string())) + } + Ok(state) if state.status == SessionStatus::Sleeping => { + match client.wake(id).await { + Ok(_) => { + // Woken sessions come back without env (ADR 0010); + // the workspace files were restored from the + // sleep snapshot, so re-inject the env file. + self.write_session_files(client, id, request, base_commit) + .await?; + Ok((id, 0, "woken previous session".to_string())) + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + %error, + "failed to wake sandbox session, starting fresh" + ); + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + } + } + Ok(state) => { + // Some other state (Promoting, Error, Destroyed): the + // session cannot be used; fall back to a fresh one. + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + ?state.status, + "previous sandbox session in unusable state, starting fresh" + ); + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %request.task_id, + %error, + "previous sandbox session gone, starting fresh" + ); + self.start_fresh_session(client, request, checkout_root, base_commit) + .await + } + } + } + None => self.start_fresh_session(client, request, checkout_root, base_commit).await, + } + } + + /// Creates a brand-new session and imports the local checkout into it. + async fn start_fresh_session( + &self, + client: &SandboxClient, + request: &LaneExecutionRequest, + checkout_root: Option<&FsPath>, + base_commit: Option, + ) -> Result<(SessionId, usize, String), String> { + let session = client + .create_session(self.session_config()) + .await + .map_err(|error| format!("failed to create sandbox session: {error}"))?; + + let files = if let Some(root) = checkout_root { + Self::import_checkout(client, session, root) + .await + .map_err(|error| format!("failed to import checkout into sandbox: {error}"))? + } else { + 0 }; - check_hosted_task_policy(&prepared.checkout_root, request)?; - // The commit the worker's changes are measured against. The server records it, so the - // worker cannot pick its own baseline. - let base_commit = git_head_commit(&prepared.checkout_root); - let task_file = - write_local_docker_task_payload(&prepared.checkout_root, request, base_commit) - .map_err(|error| format!("failed to write hosted docker task payload: {error}"))?; - let image = self.image.clone(); - // The worker gets a token that works only for this task, plus a proxy URL for - // the model provider. No provider key and no control-plane key enter the - // container. See worker_access.rs. + self.write_session_files(client, session, request, base_commit) + .await?; + Ok((session, files, "created new session".to_string())) + } + + /// Writes the env file and task payload the agent reads on start. Called on + /// every start (fresh, woken, or forked) because env is dropped on wake + /// (ADR 0010) and the payload carries the re-run's new prompt. + async fn write_session_files( + &self, + client: &SandboxClient, + session: SessionId, + request: &LaneExecutionRequest, + base_commit: Option, + ) -> Result<(), String> { let token = self .worker_access .issue(&request.task_id) @@ -4611,99 +4420,468 @@ impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { request.provider.as_deref(), request.model.as_deref(), )?; - let mut container_env = BTreeMap::from([ - ( - "FRONTAL_CODE_SERVER_URL".to_string(), - self.server_url.clone(), - ), - ("FRONTAL_CODE_SERVER_API_KEY".to_string(), token.clone()), - ( - "FRONTAL_CODE_HOSTED_TASK_FILE".to_string(), - "/workspace/.frontal-code-hosted/task.json".to_string(), - ), - ]); - // Lets the worker CLI accept the cleartext Docker host alias. Set only here. - container_env.insert("FRONTAL_CODE_HOSTED_WORKER".to_string(), "1".to_string()); - for (key, value) in worker_access::worker_provider_env(provider, &self.server_url, &token) { - container_env.insert(key, value); - } - // Non-secret tuning only. Provider keys are never forwarded. - if let Ok(value) = env::var("FRONTAL_CODE_MAX_OUTPUT_TOKENS") { - if !value.trim().is_empty() { - container_env.insert("FRONTAL_CODE_MAX_OUTPUT_TOKENS".to_string(), value); - } - } - // No GitHub credentials enter the container. The server publishes the branch and - // opens the pull request from its own environment. See publish.rs. - // The relay must be up, and the task must have its own network, before the worker starts. - self.docker - .ensure_relay(&self.relay_image, self.relay_port)?; - self.docker.sweep_idle_task_networks(); - let network = docker_worker::task_network_name(&request.task_id); - self.docker.prepare_task_network(&network)?; - let launch = self.docker.launch(&DockerLaunchSpec { - image: image.clone(), - checkout_root: prepared.checkout_root.clone(), - task_id: request.task_id.clone(), - network: network.clone(), - command: vec![ - "frontal-code".to_string(), - "hosted".to_string(), - "task".to_string(), - "run".to_string(), - request.task_id.clone(), - ], - env: container_env, - })?; + + // The session's env is dropped on wake (ADR 0010), so the env map is + // written to a file the agent sources. + let env_json = serde_json::to_vec_pretty(&Self::sandbox_task_env( + &self.server_url, + &token, + &provider, + )) + .expect("env map is always serializable"); + client + .write_file(session, "/workspace/.frontal-code-hosted/env", env_json) + .await + .map_err(|error| format!("failed to write sandbox env file: {error}"))?; + + client + .write_file( + session, + "/workspace/.frontal-code-hosted/task.json", + Self::sandbox_task_payload(request, base_commit), + ) + .await + .map_err(|error| format!("failed to write sandbox task payload: {error}"))?; + Ok(()) + } + + /// Starts the agent inside the session fire-and-forget. Its fine-grained + /// events flow back over the control plane via FRONTAL_CODE_SERVER_URL, so + /// the server does not need to capture its stdout. The thread owns its own + /// runtime and ends when the command exits. `cancel` destroys the session, + /// which kills the exec. On a clean exit the session's state is snapshotted + /// so a re-run can fork from the final workspace. + fn spawn_exec(&self, session: SessionId, task_id: &str) { + let client = self.client.clone(); + let tasks = self.tasks.clone(); + let task_id = task_id.to_string(); + std::thread::spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "failed to build sandbox exec runtime" + ); + return; + } + }; + let request = ExecRequest { + argv: vec![ + "frontal-code".to_string(), + "hosted".to_string(), + "task".to_string(), + "run".to_string(), + task_id.to_string(), + ], + cwd: Some("/workspace".to_string()), + stream: true, + ..ExecRequest::default() + }; + runtime.block_on(async { + let stream = match client.exec_stream(session, &request).await { + Ok(stream) => stream, + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "sandbox exec stream failed to start" + ); + return; + } + }; + futures_util::pin_mut!(stream); + while let Some(event) = stream.next().await { + match event { + Ok(ExecEvent::Output { data, .. }) => { + tracing::debug!( + target: "code-server::sandbox", + task_id = %task_id, + output = %String::from_utf8_lossy(&data), + "sandbox exec output" + ); + } + Ok(ExecEvent::Exit { exit_code, .. }) => { + tracing::info!( + target: "code-server::sandbox", + task_id = %task_id, + exit_code, + "sandbox exec finished" + ); + if exit_code == 0 { + // Snapshot the final state so a re-run can fork + // from it; the server may have destroyed the + // session by then, in which case the re-run + // falls back to a fresh session. + match client.snapshot(session).await { + Ok(snapshot) => { + if let Ok(mut tasks) = tasks.lock() { + if let Some(state) = tasks.get_mut(&session.to_string()) { + state.snapshot_id = Some(snapshot); + } + } + } + Err(error) => { + tracing::warn!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "failed to snapshot sandbox session after exec" + ); + } + } + } + return; + } + Ok(ExecEvent::Error { message, .. }) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %message, + "sandbox exec error" + ); + return; + } + Err(error) => { + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + %error, + "sandbox exec stream error" + ); + return; + } + } + } + tracing::error!( + target: "code-server::sandbox", + task_id = %task_id, + "sandbox exec stream ended before exit or error" + ); + }); + }); + } + + /// Records per-task state, emits the launch audit event, and builds the + /// bootstrap result shared by `bootstrap` and `resume`. + fn complete_start( + &self, + request: &LaneExecutionRequest, + session: SessionId, + files: usize, + how: String, + checkout_root: &FsPath, + base_commit: Option, + active_ref: &str, + branch: Option<&str>, + source_display: &str, + ) -> Result { + self.tasks.lock().unwrap().insert( + session.to_string(), + SandboxTaskState { + session_id: session, + base_commit, + checkout_root: checkout_root.to_path_buf(), + snapshot_id: None, + }, + ); if let Some(audit) = self.worker_access.audit() { audit.emit( "worker.launched", &request.task_id, - &format!("container={}", launch.container_id), + &format!("session={session} {how}"), ); } + let allowlist = Self::server_host_allowlist(&self.server_url); Ok(LaneBootstrapResult { - worker_id: launch.container_id.clone(), + worker_id: session.to_string(), worker_status: WorkerStatus::Running, signals: vec![LaneTransportSignal { kind: LaneTransportSignalKind::Running, detail: Some(format!( - "local docker container {} prepared checkout at {}", - launch.container_id, - prepared.checkout_root.display() + "sandbox session {session} {how}", )), payload: Some(json!({ - "container_id": launch.container_id, - "image": image, - "checkout_root": prepared.checkout_root.display().to_string(), - "task_file": task_file.display().to_string(), + "session_id": session.to_string(), + "files_imported": files, + "checkout_root": checkout_root.display().to_string(), "command": ["frontal-code", "hosted", "task", "run", request.task_id.as_str()], - "active_ref": prepared.active_ref, - "branch": prepared.branch, - "source": prepared.source.display(), + "active_ref": active_ref, + "branch": branch, + "source": source_display, + "network_allowlist": allowlist, })), }], artifacts: None, - checkout_root: Some(prepared.checkout_root.display().to_string()), + checkout_root: Some(checkout_root.display().to_string()), }) } + /// The host (and optional port) of a server URL, for the session's network + /// allowlist. `https://frontal.tools:8443/` -> `frontal.tools:8443`. + fn server_host_allowlist(server_url: &str) -> String { + let trimmed = server_url.trim_end_matches('/'); + let without_scheme = trimmed + .split_once("://") + .map(|(_, rest)| rest) + .unwrap_or(trimmed); + without_scheme + .split('/') + .next() + .unwrap_or(without_scheme) + .to_string() + } + + /// Recursively walks `root` and uploads every regular file to the session + /// under `/workspace/`, skipping the `.git` directory. Returns the number of + /// files written. + async fn import_checkout( + client: &SandboxClient, + session: SessionId, + root: &FsPath, + ) -> Result { + let mut entries: Vec = Vec::new(); + fn collect(dir: &FsPath, out: &mut Vec) -> Result<(), String> { + let read_dir = fs::read_dir(dir) + .map_err(|error| format!("failed to read {}: {error}", dir.display()))?; + for entry in read_dir { + let entry = entry + .map_err(|error| format!("failed to read dir entry: {error}"))?; + let path = entry.path(); + if path.is_dir() { + // Skip VCS metadata; the worker never needs it. + if path.file_name().is_some_and(|name| name == ".git") { + continue; + } + collect(&path, out)?; + } else if path.is_file() { + out.push(path); + } + } + Ok(()) + } + collect(root, &mut entries).map_err(|error| { + format!("failed to walk checkout {}: {error}", root.display()) + })?; + + let mut count = 0usize; + for path in &entries { + let relative = path.strip_prefix(root).map_err(|error| { + format!("failed to strip prefix {}: {error}", root.display()) + })?; + let session_path = format!("/workspace/{}", relative.display()); + let bytes = fs::read(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + client + .write_file(session, &session_path, bytes) + .await + .map_err(|error| format!("failed to write {session_path}: {error}"))?; + count += 1; + } + Ok(count) + } + + /// The env map handed to the worker. The session drops its env on wake, so the + /// agent reads the written file instead of relying on process env vars. + fn sandbox_task_env( + server_url: &str, + token: &str, + provider: &str, + ) -> BTreeMap { + let mut task_env = BTreeMap::from([ + ("FRONTAL_CODE_SERVER_URL".to_string(), server_url.to_string()), + ("FRONTAL_CODE_SERVER_API_KEY".to_string(), token.to_string()), + ( + "FRONTAL_CODE_HOSTED_TASK_FILE".to_string(), + "/workspace/.frontal-code-hosted/task.json".to_string(), + ), + ]); + task_env.insert("FRONTAL_CODE_HOSTED_WORKER".to_string(), "1".to_string()); + for (key, value) in worker_access::worker_provider_env(provider, server_url, token) { + task_env.insert(key, value); + } + // Non-secret tuning only. Provider keys are never forwarded. + if let Ok(value) = env::var("FRONTAL_CODE_MAX_OUTPUT_TOKENS") { + if !value.trim().is_empty() { + task_env.insert("FRONTAL_CODE_MAX_OUTPUT_TOKENS".to_string(), value); + } + } + task_env + } + + fn sandbox_task_payload( + request: &LaneExecutionRequest, + base_commit: Option, + ) -> Vec { + serde_json::to_vec_pretty(&HostedLaneTaskPayload { + task_id: request.task_id.clone(), + prompt: request.prompt.clone(), + repository: request.repository.clone(), + repo_url: request.repo_url.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + model: request.model.clone(), + provider: request.provider.clone(), + permission_mode: Some( + request + .permission_mode + .clone() + .unwrap_or_else(|| worker_access::WORKER_DEFAULT_PERMISSION_MODE.to_string()), + ), + allowed_tools: request.allowed_tools.clone(), + bash_allow: worker_access::WORKER_BASH_ALLOW + .iter() + .map(|rule| (*rule).to_string()) + .collect(), + bash_deny: worker_access::WORKER_BASH_DENY + .iter() + .map(|rule| (*rule).to_string()) + .collect(), + base_commit, + }) + .expect("hosted task payload is always serializable") + } +} + +impl LaneWorkerTransport for SandboxLaneWorkerTransport { + fn bootstrap(&self, request: &LaneExecutionRequest) -> Result { + // A task with a repository gets a checkout on its own branch. A task + // without one gets an empty scratch workspace. + let prepared = match lane_repo_source(request) { + Some(source) => prepare_checkout(&RepoCheckoutRequest { + workspace_root: self.workspace_root.clone(), + checkout_id: request.task_id.clone(), + source, + repository: request.repository.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + }) + .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, + None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + }; + check_hosted_task_policy(&prepared.checkout_root, request)?; + // The commit the worker's changes are measured against. The server + // records it, so the worker cannot pick its own baseline. + let base_commit = git_head_commit(&prepared.checkout_root); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let (session, files, how) = runtime + .block_on(self.attach_to_session( + request, + None, + Some(&prepared.checkout_root), + base_commit.clone(), + )) + .map_err(|error| error.to_string())?; + + self.spawn_exec(session, &request.task_id); + + self.complete_start( + request, + session, + files, + how, + &prepared.checkout_root, + base_commit, + &prepared.active_ref, + prepared.branch.as_deref(), + prepared.source.display().as_ref(), + ) + } + + /// Re-runs a task on the workspace its previous run left behind. The + /// previous worker id is the sandbox session id: if that run finished + /// cleanly, its final state was snapshotted and the re-run forks a fresh + /// session from it (the old session is left untouched, so a second re-run + /// still works); otherwise the previous session is woken and its files are + /// restored from the sleep snapshot. Either way the env file is re-written + /// (env is dropped on wake, ADR 0010) and the agent is exec'd again with + /// the task's current prompt. The local checkout is re-prepared only so the + /// re-run's diff is measured against the same baseline; it is not imported + /// unless the previous session is gone. + fn resume( + &self, + request: &LaneExecutionRequest, + previous: Option<&str>, + ) -> Result { + let prepared = match lane_repo_source(request) { + Some(source) => prepare_checkout(&RepoCheckoutRequest { + workspace_root: self.workspace_root.clone(), + checkout_id: request.task_id.clone(), + source, + repository: request.repository.clone(), + base_ref: request.base_ref.clone(), + branch: request.branch.clone(), + }) + .map_err(|error| format!("failed to prepare repo checkout: {error}"))?, + None => prepare_scratch_workspace(&self.workspace_root, &request.task_id)?, + }; + check_hosted_task_policy(&prepared.checkout_root, request)?; + let base_commit = git_head_commit(&prepared.checkout_root); + // The previous worker id is the session id. If the parse fails the + // re-run falls back to a fresh session, which re-imports the checkout. + let previous_session = previous.and_then(|id| SessionId::from_str(id).ok()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let (session, files, how) = runtime + .block_on(self.attach_to_session( + request, + previous_session, + Some(&prepared.checkout_root), + base_commit.clone(), + )) + .map_err(|error| error.to_string())?; + + self.spawn_exec(session, &request.task_id); + + self.complete_start( + request, + session, + files, + how, + &prepared.checkout_root, + base_commit, + &prepared.active_ref, + prepared.branch.as_deref(), + prepared.source.display().as_ref(), + ) + } + fn observe_completion( &self, - _worker_id: &str, + worker_id: &str, finish_reason: &str, _tokens_output: u64, ) -> Result { - let worker_status = if finish_reason.eq_ignore_ascii_case("error") + // The session VFS is the worker's checkout, but the lane orchestrator + // computes the diff against the local checkout recorded in + // `LaneBootstrapResult.checkout_root`. The worker's exit is reported + // asynchronously via the control plane (the agent POSTs its terminal + // event). + if finish_reason.eq_ignore_ascii_case("error") || finish_reason.eq_ignore_ascii_case("failed") { - WorkerStatus::Failed - } else { - WorkerStatus::Finished - }; + return Ok(LaneCompletionResult { + worker_status: WorkerStatus::Failed, + error: None, + }); + } + let _ = worker_id; Ok(LaneCompletionResult { - worker_status, + worker_status: WorkerStatus::Finished, error: None, }) } @@ -4713,12 +4891,33 @@ impl LaneWorkerTransport for LocalDockerLaneWorkerTransport { worker_id: &str, _locator: Option<&HostedAgentLocator>, ) -> Result { - self.docker.stop(worker_id)?; - Ok(LaneCancellationResult { - worker_status: Some(WorkerStatus::Finished), - clear_worker_status: false, - detail: Some(format!("local docker container {worker_id} stopped")), - }) + let session = SessionId::from_str(worker_id).map_err(|error| { + format!("invalid sandbox session id `{worker_id}`: {error}") + })?; + // Best-effort: a destroyed or already-destroyed session still counts as + // cancelled from the lane's point of view. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("failed to build sandbox runtime: {error}"))?; + let outcome = runtime.block_on(self.client.destroy_session(session)); + self.tasks.lock().unwrap().remove(worker_id); + match outcome { + Ok(()) => Ok(LaneCancellationResult { + worker_status: Some(WorkerStatus::Finished), + clear_worker_status: true, + detail: Some(format!("sandbox session {worker_id} destroyed")), + }), + // Best-effort: even if the API call fails (session already + // destroyed, network blip, etc.) the lane treats it as cancelled. + Err(error) => Ok(LaneCancellationResult { + worker_status: Some(WorkerStatus::Finished), + clear_worker_status: true, + detail: Some(format!( + "sandbox session {worker_id} destroyed (best-effort: {error})" + )), + }), + } } } @@ -4778,49 +4977,8 @@ fn git_head_commit(checkout_root: &FsPath) -> Option { output .status .success() - .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) - .filter(|sha| sha.len() == 40 && sha.chars().all(|c| c.is_ascii_hexdigit())) -} - -fn write_local_docker_task_payload( - checkout_root: &FsPath, - request: &LaneExecutionRequest, - base_commit: Option, -) -> Result { - let payload_dir = checkout_root.join(".frontal-code-hosted"); - std::fs::create_dir_all(&payload_dir)?; - let payload_path = payload_dir.join("task.json"); - std::fs::write( - &payload_path, - serde_json::to_vec_pretty(&HostedDockerTaskPayload { - task_id: request.task_id.clone(), - prompt: request.prompt.clone(), - repository: request.repository.clone(), - repo_url: request.repo_url.clone(), - base_ref: request.base_ref.clone(), - branch: request.branch.clone(), - model: request.model.clone(), - provider: request.provider.clone(), - permission_mode: Some( - request - .permission_mode - .clone() - .unwrap_or_else(|| worker_access::WORKER_DEFAULT_PERMISSION_MODE.to_string()), - ), - allowed_tools: request.allowed_tools.clone(), - bash_allow: worker_access::WORKER_BASH_ALLOW - .iter() - .map(|rule| (*rule).to_string()) - .collect(), - bash_deny: worker_access::WORKER_BASH_DENY - .iter() - .map(|rule| (*rule).to_string()) - .collect(), - base_commit, - }) - .map_err(std::io::Error::other)?, - )?; - Ok(payload_path) + .then(|| String::from_utf8_lossy(&output.stdout).trim().to_string()) + .filter(|sha| sha.len() == 40 && sha.chars().all(|c| c.is_ascii_hexdigit())) } #[allow(clippy::too_many_lines)] @@ -4991,6 +5149,7 @@ fn create_task_internal(state: &ServerState, request: CreateTaskRequest) -> Resu &mut context, primary_role, primary_description.as_deref(), + None, )?; state @@ -5005,6 +5164,7 @@ fn bootstrap_task_lane( context: &mut HostedTaskContext, primary_role: Option, primary_description: Option<&str>, + previous_worker_id: Option, ) -> Result<(), AppError> { let lane_request = LaneExecutionRequest { task_id: task.task_id.clone(), @@ -5020,11 +5180,16 @@ fn bootstrap_task_lane( allowed_tools: context.allowed_tools.clone(), }; - match state - .lane_transport - .bootstrap(&lane_request) - .map_err(AppError::internal) - { + // A re-run: the previous worker id (the sandbox session id) is carried in + // by the caller, so ask the transport to resume from it. A fresh run: the + // transport creates a new session. + match if previous_worker_id.is_some() { + state + .lane_transport + .resume(&lane_request, previous_worker_id.as_deref()) + } else { + state.lane_transport.bootstrap(&lane_request) + } { Ok(bootstrap) => { context.worker_id = Some(bootstrap.worker_id.clone()); context.worker_status = Some(bootstrap.worker_status.to_string()); @@ -5040,6 +5205,11 @@ fn bootstrap_task_lane( context.orphaned_at = None; context.orphaned_approval_requested_at = None; context.applied_orphan_policy = None; + // The worker id is the sandbox session id; record it so a later + // re-run can fork from the run's snapshot instead of re-importing. + if state.lane_transport_kind == LaneTransportKind::Sandbox { + context.sandbox_session_id = context.worker_id.clone(); + } state.record_context(&task.task_id, context.clone()); emit_lane_transport_signals( @@ -5072,7 +5242,7 @@ fn bootstrap_task_lane( .map_err(AppError::internal)?; state .tasks - .append_output(&task.task_id, &error.message) + .append_output(&task.task_id, &error) .map_err(AppError::internal)?; state.broadcast_event(EventEnvelope::new( HostedEventName::LaneFailed, @@ -5088,7 +5258,7 @@ fn bootstrap_task_lane( context, Some("failed"), Some(serialize_event_extra(TerminalEventPayload { - error: Some(error.message.clone()), + error: Some(error.clone()), extra: Map::new(), ..TerminalEventPayload::default() })), @@ -5288,7 +5458,7 @@ fn lane_role_label(role: LaneRole) -> &'static str { fn lane_transport_kind_label(kind: LaneTransportKind) -> &'static str { match kind { - LaneTransportKind::LocalDocker => "local_docker", + LaneTransportKind::Sandbox => "sandbox", LaneTransportKind::ToolsAgent => "tools_agent", } } @@ -5391,63 +5561,26 @@ mod tests { assert!(prepared.repository.is_none()); } - #[test] - fn worker_payload_defaults_to_full_access_inside_the_container() { - use super::{write_local_docker_task_payload, LaneExecutionRequest}; - let root = temp_test_dir("worker-payload"); - let request = LaneExecutionRequest { - task_id: "task-1".to_string(), - prompt: "hello".to_string(), - repository: None, - repo_url: None, - base_ref: None, - branch: None, - lane_role: None, - model: None, - provider: None, - permission_mode: None, - allowed_tools: Vec::new(), - }; - let path = write_local_docker_task_payload(&root, &request, None).expect("payload written"); - let payload: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("read payload")) - .expect("payload json"); - assert_eq!(payload["permission_mode"], "workspace-write"); - assert!(payload["bash_allow"] - .as_array() - .is_some_and(|rules| rules.iter().any(|rule| rule == "bash(cargo:*)"))); - assert!(payload["bash_deny"] - .as_array() - .is_some_and(|rules| rules.iter().any(|rule| rule == "bash(curl:*)"))); - - let explicit = LaneExecutionRequest { - permission_mode: Some("read-only".to_string()), - ..request - }; - let path = - write_local_docker_task_payload(&root, &explicit, None).expect("payload written"); - let payload: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(path).expect("read payload")) - .expect("payload json"); - assert_eq!(payload["permission_mode"], "read-only"); - } #[test] fn lane_transport_parser_accepts_workers_and_refuses_everything_else() { use super::{parse_lane_transport_kind, LaneTransportKind}; assert_eq!( parse_lane_transport_kind("").unwrap(), - LaneTransportKind::LocalDocker + LaneTransportKind::Sandbox ); assert_eq!( - parse_lane_transport_kind(" Local-Docker ").unwrap(), - LaneTransportKind::LocalDocker + parse_lane_transport_kind(" sandbox ").unwrap(), + LaneTransportKind::Sandbox ); assert_eq!( parse_lane_transport_kind("tools-agent").unwrap(), LaneTransportKind::ToolsAgent ); - for refused in ["in-memory", "in_memory", "memory", "bogus"] { + let error = parse_lane_transport_kind("local-docker") + .expect_err("the removed docker transport must be refused"); + assert!(error.contains("was removed with the Docker worker")); + for refused in ["in-memory", "in_memory", "memory", "bogus", "sandbox-2"] { let error = parse_lane_transport_kind(refused) .expect_err("the simulator and unknown names must be refused"); assert!(error.contains("unsupported FRONTAL_CODE_SERVER_LANE_TRANSPORT")); @@ -5458,7 +5591,32 @@ mod tests { fn host_agent_transport_requires_an_explicit_opt_in() { assert!(check_host_agent_opt_in(LaneTransportKind::ToolsAgent, false).is_err()); assert!(check_host_agent_opt_in(LaneTransportKind::ToolsAgent, true).is_ok()); - assert!(check_host_agent_opt_in(LaneTransportKind::LocalDocker, false).is_ok()); + assert!(check_host_agent_opt_in(LaneTransportKind::Sandbox, false).is_ok()); + } + + #[test] + fn server_config_reads_sandbox_env_vars() { + use super::{ + default_sandbox_base_url, parse_lane_transport_kind, ServerConfig, LaneTransportKind, + }; + // The sandbox server settings and the lane transport selection must parse + // out of the environment the way an operator would set them. + assert!(default_sandbox_base_url().ends_with(":8080")); + assert_eq!( + parse_lane_transport_kind("sandbox").unwrap(), + LaneTransportKind::Sandbox + ); + std::env::set_var("FRONTAL_SANDBOX_BASE_URL", "http://sandbox.test:8080/"); + std::env::set_var("FRONTAL_SANDBOX_API_KEY", "sbx_test-key"); + std::env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", "sandbox"); + let config = + ServerConfig::from_env().expect("sandbox env vars should parse into the config"); + assert_eq!(config.sandbox_base_url, "http://sandbox.test:8080"); + assert_eq!(config.sandbox_api_key, "sbx_test-key"); + assert_eq!(config.lane_transport_kind, LaneTransportKind::Sandbox); + std::env::remove_var("FRONTAL_SANDBOX_BASE_URL"); + std::env::remove_var("FRONTAL_SANDBOX_API_KEY"); + std::env::remove_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT"); } use super::*; @@ -5776,36 +5934,6 @@ mod tests { } } - #[derive(Debug, Default)] - struct MockDockerRunner { - launched: Mutex>, - stopped: Mutex>, - fail_launch: Option, - } - - impl DockerRunner for MockDockerRunner { - fn launch(&self, spec: &DockerLaunchSpec) -> Result { - if let Some(error) = &self.fail_launch { - return Err(error.clone()); - } - self.launched - .lock() - .expect("mock docker launch lock poisoned") - .push(spec.clone()); - Ok(DockerLaunchResult { - container_id: "container-localdocker-1".to_string(), - }) - } - - fn stop(&self, container_id: &str) -> Result<(), String> { - self.stopped - .lock() - .expect("mock docker stop lock poisoned") - .push(container_id.to_string()); - Ok(()) - } - } - fn temp_test_dir(prefix: &str) -> PathBuf { // Timestamp alone collides when parallel tests read the same instant. static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); @@ -5821,43 +5949,6 @@ mod tests { dir } - fn init_git_repo(path: PathBuf) -> PathBuf { - fs::create_dir_all(&path).unwrap(); - run_git(&path, ["init", "-b", "main"]); - run_git(&path, ["config", "user.name", "FrontalCode Server Tests"]); - run_git( - &path, - ["config", "user.email", "code-server-tests@example.com"], - ); - path - } - - fn commit_file(repo: &std::path::Path, relative_path: &str, contents: &str, message: &str) { - let file_path = repo.join(relative_path); - if let Some(parent) = file_path.parent() { - fs::create_dir_all(parent).unwrap(); - } - fs::write(file_path, contents).unwrap(); - run_git(repo, ["add", relative_path]); - run_git(repo, ["commit", "-m", message]); - } - - fn run_git(repo: &std::path::Path, args: [&str; N]) { - let output = std::process::Command::new("git") - .arg("-C") - .arg(repo) - .args(args) - .output() - .unwrap(); - assert!( - output.status.success(), - "git command failed: git -C {} {}: {}", - repo.display(), - args.join(" "), - String::from_utf8_lossy(&output.stderr) - ); - } - fn write_hosted_agent_manifest( path: &std::path::Path, output_path: &std::path::Path, @@ -5989,7 +6080,7 @@ mod tests { assert_eq!(snapshot["repo_url"], "https://github.com/acme/repo-a.git"); assert_eq!(snapshot["base_ref"], "main"); assert_eq!(snapshot["branch"], "frontal-code/investigate-flake"); - assert_eq!(snapshot["execution_backend"], "local_docker"); + assert_eq!(snapshot["execution_backend"], "sandbox"); assert_eq!(snapshot["lane_id"].as_str().map(str::is_empty), Some(false)); assert_eq!( snapshot["plan_kind"].as_str().map(str::is_empty), @@ -6046,7 +6137,7 @@ mod tests { .payload .as_ref() .and_then(|payload| payload.get("execution_backend")), - Some(&json!("local_docker")) + Some(&json!("sandbox")) ); assert_eq!( created_event @@ -6103,307 +6194,6 @@ mod tests { .is_some()); } - #[tokio::test] - #[allow(clippy::await_holding_lock, clippy::too_many_lines)] - async fn local_docker_transport_prepares_checkout_workspace_for_task() { - let Some(store) = worker_tokens::tests::test_store() else { - return; - }; - let dir = temp_test_dir("code-server-local-docker"); - let source_repo = init_git_repo(dir.join("source")); - commit_file( - &source_repo, - "README.md", - "hello from local docker transport\n", - "initial commit", - ); - let workspace_root = dir.join("workspaces"); - let docker = Arc::new(MockDockerRunner::default()); - let access = Arc::new(WorkerAccess::new(Some(store.clone()), 3600)); - let state = Arc::new( - ServerState::with_lane_transport_kind_and_access( - DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, - Arc::new(LocalDockerLaneWorkerTransport::with_runner( - workspace_root.clone(), - "frontal-code-worker:test".to_string(), - "http://host.docker.internal:8788", - access.clone(), - docker.clone(), - )), - access, - ) - .with_local_repos_allowed(true), - ); - let router = app(state.clone()); - - let create_response = router - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/tasks") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "prompt": "Prepare checkout before running", - "repository": "acme/local-docker", - "repo_url": source_repo.display().to_string(), - "base_ref": "main", - "branch": "frontal-code/task-123", - "source": "api" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(create_response.status(), StatusCode::OK); - let created_body = to_bytes(create_response.into_body(), usize::MAX) - .await - .unwrap(); - let created: serde_json::Value = serde_json::from_slice(&created_body).unwrap(); - let task_id = created["task_id"].as_str().unwrap().to_string(); - - let context = state - .context_for(&task_id) - .expect("context should be recorded for local docker task"); - let checkout_root = PathBuf::from( - context - .checkout_root - .clone() - .expect("local docker task should record checkout root"), - ); - assert!(checkout_root.starts_with(&workspace_root)); - assert!(checkout_root.join(".git").exists()); - assert_eq!( - fs::read_to_string(checkout_root.join("README.md")).unwrap(), - "hello from local docker transport\n" - ); - assert_eq!(context.execution_backend.as_deref(), Some("local_docker")); - - let get_response = router - .clone() - .oneshot( - Request::builder() - .uri(format!("/v1/tasks/{task_id}")) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - let body = to_bytes(get_response.into_body(), usize::MAX) - .await - .unwrap(); - let snapshot: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(snapshot["execution_backend"], "local_docker"); - assert_eq!(snapshot["worker_id"], "container-localdocker-1"); - - let started_event = state - .replay_events() - .into_iter() - .find(|event| event.event == HostedEventName::LaneStarted) - .expect("lane started event should be emitted"); - assert_eq!( - started_event - .payload - .as_ref() - .and_then(|payload| payload.get("transport")) - .and_then(|transport| transport.get("checkout_root")), - Some(&json!(checkout_root.display().to_string())) - ); - assert_eq!( - started_event - .payload - .as_ref() - .and_then(|payload| payload.get("transport")) - .and_then(|transport| transport.get("container_id")), - Some(&json!("container-localdocker-1")) - ); - - let launched = docker - .launched - .lock() - .expect("mock docker launch lock poisoned"); - assert_eq!(launched.len(), 1); - assert_eq!(launched[0].image, "frontal-code-worker:test"); - assert_eq!(launched[0].task_id, task_id); - assert_eq!(launched[0].checkout_root, checkout_root); - assert_eq!( - launched[0].command, - vec![ - "frontal-code".to_string(), - "hosted".to_string(), - "task".to_string(), - "run".to_string(), - task_id.clone(), - ] - ); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_SERVER_URL") - .map(String::as_str), - Some("http://frontal-relay:8788") - ); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_HOSTED_TASK_FILE") - .map(String::as_str), - Some("/workspace/.frontal-code-hosted/task.json") - ); - // The worker authenticates with a per-task token, never the server's own key. - let worker_token = launched[0] - .env - .get("FRONTAL_CODE_SERVER_API_KEY") - .cloned() - .expect("worker token"); - assert_eq!(worker_token.len(), 64); - assert!(worker_token.chars().all(|c| c.is_ascii_hexdigit())); - // No upstream provider key reaches the container. The model traffic goes to the proxy. - assert!(!launched[0].env.keys().any(|key| key.ends_with("_API_KEY") - && key != "FRONTAL_CODE_SERVER_API_KEY" - && launched[0].env.get(key).map(String::as_str) != Some(worker_token.as_str()))); - assert_eq!( - launched[0] - .env - .get("FRONTAL_CODE_SERVER_URL") - .map(String::as_str), - Some("http://frontal-relay:8788") - ); - - let task_payload: HostedDockerTaskPayload = serde_json::from_slice( - &fs::read(checkout_root.join(".frontal-code-hosted").join("task.json")).unwrap(), - ) - .unwrap(); - assert_eq!(task_payload.task_id, task_id); - assert_eq!(task_payload.prompt, "Prepare checkout before running"); - assert_eq!( - task_payload.repository.as_deref(), - Some("acme/local-docker") - ); - assert_eq!( - task_payload.repo_url.as_deref(), - Some(source_repo.to_str().unwrap()) - ); - assert_eq!(task_payload.base_ref.as_deref(), Some("main")); - assert_eq!( - task_payload.branch.as_deref(), - Some("frontal-code/task-123") - ); - assert_eq!(task_payload.allowed_tools, Vec::::new()); - - let complete_response = router - .clone() - .oneshot( - Request::builder() - .method("POST") - .uri(format!("/v1/tasks/{task_id}/complete")) - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "finish_reason": "stop", - "tokens_output": 64, - "result": "local docker run completed" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(complete_response.status(), StatusCode::OK); - - let _ = fs::remove_dir_all(dir); - } - - #[tokio::test] - async fn local_docker_transport_launch_failure_marks_task_failed() { - let Some(store) = worker_tokens::tests::test_store() else { - return; - }; - let dir = temp_test_dir("code-server-local-docker-fail"); - let source_repo = init_git_repo(dir.join("source")); - commit_file( - &source_repo, - "README.md", - "failure path\n", - "initial commit", - ); - let workspace_root = dir.join("workspaces"); - let docker = Arc::new(MockDockerRunner { - fail_launch: Some("docker launch spec rejected".to_string()), - ..MockDockerRunner::default() - }); - let access = Arc::new(WorkerAccess::new(Some(store.clone()), 3600)); - let state = Arc::new( - ServerState::with_lane_transport_kind_and_access( - DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, - Arc::new(LocalDockerLaneWorkerTransport::with_runner( - workspace_root, - "frontal-code-worker:test".to_string(), - "http://host.docker.internal:8788", - access.clone(), - docker, - )), - access, - ) - .with_local_repos_allowed(true), - ); - let router = app(state.clone()); - - let create_response = router - .oneshot( - Request::builder() - .method("POST") - .uri("/v1/tasks") - .header("content-type", "application/json") - .body(Body::from( - serde_json::to_vec(&json!({ - "prompt": "Prepare checkout before running", - "repository": "acme/local-docker", - "repo_url": source_repo.display().to_string(), - "base_ref": "main", - "branch": "frontal-code/task-123", - "source": "api" - })) - .unwrap(), - )) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(create_response.status(), StatusCode::OK); - let body = to_bytes(create_response.into_body(), usize::MAX) - .await - .unwrap(); - let snapshot: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(snapshot["status"], "failed"); - assert_eq!(snapshot["worker_status"], "failed"); - let task = state - .tasks - .list(None) - .into_iter() - .next() - .expect("task should still exist after launch failure"); - assert_eq!(task.status, TaskStatus::Failed); - assert!(task.output.contains("docker launch spec rejected")); - - let failed_event = state - .replay_events() - .into_iter() - .find(|event| event.event == HostedEventName::LaneFailed) - .expect("lane failed event should be emitted"); - assert_eq!(failed_event.status, HostedEventStatus::Failed); - - let _ = fs::remove_dir_all(dir); - } - #[tokio::test] async fn update_task_context_persists_slack_thread_anchor() { let dir = temp_test_dir("code-server-context-update"); @@ -7131,7 +6921,7 @@ mod tests { let state = Arc::new( ServerState::with_lane_transport_kind_and_access( DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, Arc::new(FakeLaneWorkerTransport::new()), Arc::new(WorkerAccess::new(Some(store), 3600)), ) @@ -9400,7 +9190,7 @@ mod tests { let state = Arc::new( ServerState::new_with_transport_kind_state_file_policy_and_workspace_root( 10, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, None, None, Duration::from_secs(0), @@ -9408,8 +9198,6 @@ mod tests { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ), ); @@ -9444,7 +9232,7 @@ mod tests { let state = Arc::new( ServerState::new_with_transport_kind_state_file_policy_and_workspace_root( 10, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, None, None, Duration::from_secs(0), @@ -9452,8 +9240,6 @@ mod tests { None, Vec::new(), default_server_workspace_root(), - default_server_docker_image(), - default_server_docker_server_url(), WorkerAccess::new(None, 7200), ), ); @@ -10042,7 +9828,7 @@ mod tests { let restored = ServerState::new_with_transport_kind_and_state_file( DEFAULT_EVENT_REPLAY_LIMIT, - LaneTransportKind::LocalDocker, + LaneTransportKind::Sandbox, Some(state_file), ); let restored_task = restored.tasks.get(&task_id).expect("task should reload"); @@ -10139,7 +9925,7 @@ mod tests { ); assert_eq!( restored_context.execution_backend.as_deref(), - Some("local_docker") + Some("sandbox") ); assert_eq!(restored_context.source.as_deref(), Some("api")); assert_eq!(restored_context.worker_status.as_deref(), Some("running")); @@ -10168,7 +9954,7 @@ mod tests { ); assert_eq!(snapshot["base_ref"], "develop"); assert_eq!(snapshot["branch"], "frontal-code/persisted-work"); - assert_eq!(snapshot["execution_backend"], "local_docker"); + assert_eq!(snapshot["execution_backend"], "sandbox"); assert_eq!(snapshot["worker_status"], "running"); let _ = fs::remove_dir_all(dir); @@ -11041,53 +10827,6 @@ mod tests { ); } - #[test] - fn server_config_parses_local_docker_lane_transport_kind() { - let previous = env::var_os("FRONTAL_CODE_SERVER_LANE_TRANSPORT"); - let previous_workspace_root = env::var_os("FRONTAL_CODE_SERVER_WORKSPACE_ROOT"); - let previous_docker_image = env::var_os("FRONTAL_CODE_SERVER_DOCKER_IMAGE"); - let previous_callback_url = env::var_os("FRONTAL_CODE_SERVER_CALLBACK_URL"); - env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", "local-docker"); - env::set_var( - "FRONTAL_CODE_SERVER_WORKSPACE_ROOT", - "/tmp/code-server-workspaces", - ); - env::set_var( - "FRONTAL_CODE_SERVER_DOCKER_IMAGE", - "frontal-code-worker:test", - ); - env::set_var( - "FRONTAL_CODE_SERVER_CALLBACK_URL", - "http://docker.internal.test:8788/", - ); - - let config = ServerConfig::from_env().expect("env config should parse"); - assert_eq!(config.lane_transport_kind, LaneTransportKind::LocalDocker); - assert_eq!( - config.workspace_root, - PathBuf::from("/tmp/code-server-workspaces") - ); - assert_eq!(config.docker_image, "frontal-code-worker:test"); - assert_eq!(config.docker_server_url, "http://docker.internal.test:8788"); - - match previous { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT", value), - None => env::remove_var("FRONTAL_CODE_SERVER_LANE_TRANSPORT"), - } - match previous_workspace_root { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_WORKSPACE_ROOT", value), - None => env::remove_var("FRONTAL_CODE_SERVER_WORKSPACE_ROOT"), - } - match previous_docker_image { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_DOCKER_IMAGE", value), - None => env::remove_var("FRONTAL_CODE_SERVER_DOCKER_IMAGE"), - } - match previous_callback_url { - Some(value) => env::set_var("FRONTAL_CODE_SERVER_CALLBACK_URL", value), - None => env::remove_var("FRONTAL_CODE_SERVER_CALLBACK_URL"), - } - } - #[test] fn hosted_task_context_builds_repo_checkout_request() { let context = HostedTaskContext { @@ -11159,45 +10898,3 @@ mod tests { assert_eq!(payload.get("pr_number"), Some(&json!(42))); } } - -#[cfg(test)] -mod callback_port_tests { - use super::*; - - fn config(url: &str, bind_port: u16) -> ServerConfig { - ServerConfig { - lane_transport_kind: LaneTransportKind::LocalDocker, - docker_server_url: url.to_string(), - bind_addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), bind_port), - ..ServerConfig::default() - } - } - - #[test] - fn a_callback_url_on_the_listening_port_is_accepted() { - assert!(check_callback_port(&config("http://host.docker.internal:8790", 8790)).is_ok()); - } - - #[test] - fn a_callback_url_without_a_port_is_refused() { - let error = - check_callback_port(&config("http://host.docker.internal", 8790)).expect_err("no port"); - assert!(error.contains("must include the port")); - } - - #[test] - fn a_callback_url_on_another_port_is_refused() { - let error = check_callback_port(&config("http://host.docker.internal:9000", 8790)) - .expect_err("wrong port"); - assert!(error.contains("names port 9000")); - } - - #[test] - fn the_check_does_not_apply_to_other_transports() { - let config = ServerConfig { - lane_transport_kind: LaneTransportKind::ToolsAgent, - ..config("http://elsewhere", 8790) - }; - assert!(check_callback_port(&config).is_ok()); - } -} diff --git a/crates/code-server/tests/server_config.rs b/crates/code-server/tests/server_config.rs index f1e99841..e438239a 100644 --- a/crates/code-server/tests/server_config.rs +++ b/crates/code-server/tests/server_config.rs @@ -11,39 +11,30 @@ fn default_server_config() { SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8788) ); assert_eq!(config.event_replay_limit, 100); - assert_eq!(config.lane_transport_kind, LaneTransportKind::LocalDocker); + assert_eq!(config.lane_transport_kind, LaneTransportKind::Sandbox); assert!(config.api_key.is_none()); assert_eq!(config.reconcile_interval, Some(Duration::from_secs(15))); } #[test] fn lane_transport_kind_default() { - assert_eq!( - LaneTransportKind::LocalDocker, - LaneTransportKind::LocalDocker - ); + assert_eq!(LaneTransportKind::Sandbox, LaneTransportKind::Sandbox); assert_eq!(LaneTransportKind::ToolsAgent, LaneTransportKind::ToolsAgent); } #[test] fn lane_transport_kind_debug() { - assert_eq!( - format!("{:?}", LaneTransportKind::LocalDocker), - "LocalDocker" - ); + assert_eq!(format!("{:?}", LaneTransportKind::Sandbox), "Sandbox"); assert_eq!(format!("{:?}", LaneTransportKind::ToolsAgent), "ToolsAgent"); } #[test] fn lane_transport_kind_clone_and_eq() { - let a = LaneTransportKind::LocalDocker; + let a = LaneTransportKind::Sandbox; let b = a; assert_eq!(a, b); - let kinds = [ - LaneTransportKind::LocalDocker, - LaneTransportKind::ToolsAgent, - ]; + let kinds = [LaneTransportKind::Sandbox, LaneTransportKind::ToolsAgent]; for i in 0..kinds.len() { for j in 0..kinds.len() { if i == j { diff --git a/crates/code-server/tests/server_types.rs b/crates/code-server/tests/server_types.rs index 27345c74..7d4efa69 100644 --- a/crates/code-server/tests/server_types.rs +++ b/crates/code-server/tests/server_types.rs @@ -139,7 +139,7 @@ fn hosted_task_snapshot_serialize() { pr_head_ref: None, pr_base_ref: None, metadata: std::collections::HashMap::new(), - execution_backend: Some("local_docker".to_string()), + execution_backend: Some("sandbox".to_string()), priority: None, plan_id: Some("plan-1".to_string()), plan_kind: Some("implementation".to_string()), @@ -154,7 +154,7 @@ fn hosted_task_snapshot_serialize() { assert_eq!(json["created_at"], 1000); assert_eq!(json["prompt"], "Do something"); assert_eq!(json["source"], "api"); - assert_eq!(json["execution_backend"], "local_docker"); + assert_eq!(json["execution_backend"], "sandbox"); let deserialized: HostedTaskSnapshot = serde_json::from_value(json).unwrap(); assert_eq!(deserialized.task_id, "task-123"); diff --git a/deploy/sandbox/Makefile b/deploy/sandbox/Makefile new file mode 100644 index 00000000..e37eae2d --- /dev/null +++ b/deploy/sandbox/Makefile @@ -0,0 +1,67 @@ +# Frontal Sandbox deployment. +# +# Uses OpenTofu (`tofu`). If you only have Terraform, set `TF := terraform`. +# +# smoke requires: SANDBOX_BASE_URL, SANDBOX_API_KEY +# bootstrap requires: BOOTSTRAP_HOST, SANDBOX_BINARY_URL, SANDBOX_API_KEY_HASH +# (BOOTSTRAP_USER defaults to ubuntu; SANDBOX_CAS_S3_BUCKET / +# SANDBOX_CAS_S3_ENDPOINT are optional) + +TF ?= tofu +TF_DIR := terraform + +.PHONY: init apply destroy smoke bootstrap plan validate fmt + +## Terraform lifecycle + +init: + $(TF) -chdir=$(TF_DIR) init + +plan: + $(TF) -chdir=$(TF_DIR) plan + +apply: + $(TF) -chdir=$(TF_DIR) apply + +destroy: + $(TF) -chdir=$(TF_DIR) destroy + +validate: + $(TF) -chdir=$(TF_DIR) validate + +fmt: + $(TF) -chdir=$(TF_DIR) fmt -recursive + +## Outputs + +host_ip: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_host_ip + +host_dns: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_host_dns + +base_url: + $(TF) -chdir=$(TF_DIR) output -raw sandbox_base_url + +bucket: + $(TF) -chdir=$(TF_DIR) output -raw s3_bucket_name + +## Host bootstrap +## Piped over SSH: SANDBOX_BINARY_URL, SANDBOX_API_KEY_HASH required; +## SANDBOX_CAS_S3_BUCKET / SANDBOX_CAS_S3_ENDPOINT optional. + +bootstrap: + @test -n "$${BOOTSTRAP_HOST:?BOOTSTRAP_HOST is required (see `make host_ip`)}" + @test -n "$${SANDBOX_BINARY_URL:?SANDBOX_BINARY_URL is required}" + @test -n "$${SANDBOX_API_KEY_HASH:?SANDBOX_API_KEY_HASH is required}" + ssh "$${BOOTSTRAP_USER:-ubuntu}@$${BOOTSTRAP_HOST}" \ + 'SANDBOX_BINARY_URL="$$SANDBOX_BINARY_URL" \ + SANDBOX_API_KEY_HASH="$$SANDBOX_API_KEY_HASH" \ + SANDBOX_CAS_S3_BUCKET="$$SANDBOX_CAS_S3_BUCKET" \ + SANDBOX_CAS_S3_ENDPOINT="$$SANDBOX_CAS_S3_ENDPOINT" \ + bash -s' < scripts/bootstrap.sh + +## Smoke test + +smoke: + @scripts/smoke.sh diff --git a/deploy/sandbox/README.md b/deploy/sandbox/README.md new file mode 100644 index 00000000..0257b0d8 --- /dev/null +++ b/deploy/sandbox/README.md @@ -0,0 +1,158 @@ +# Frontal Sandbox — deployment wrapper + +A turnkey deployment for the [Frontal Sandbox server](https://github.com/frontal-labs/sandbox): +it provisions a KVM-capable host plus its object store (CAS tier) with OpenTofu, +then installs and runs the sandbox server that code-server's new +`SandboxLaneWorkerTransport` talks to over HTTP (port 8080). + +The sandbox server listens on port 8080 (API + `/metrics`) and needs `/dev/kvm` +for Tier 2 (VM) sessions. It is configured via a TOML file at +`/etc/sandbox/config.toml` with env var overrides (`SANDBOX_
_`). + +The upstream sandbox repo also ships its own `infrastructure/terraform/` +module (AWS/GCP/Azure) and a clustered deployment. This directory is a +**self-contained single-node deployment** — see +[terraform/README.md](terraform/README.md) for the relationship. + +## Layout + +``` +deploy/sandbox/ +├── README.md +├── Makefile +├── config/ +│ └── production.toml # sandbox server TOML (installed to /etc/sandbox/config.toml) +├── scripts/ +│ ├── bootstrap.sh # prerequisites, KVM, binary, config, systemd unit +│ ├── deploy.sh # restart + verify on an already-bootstrapped host +│ └── smoke.sh # API smoke test (openapi → create → exec → destroy) +└── terraform/ + ├── main.tf # terraform block, required_providers, locals + ├── providers.tf # AWS provider + ├── variables.tf # inputs + ├── outputs.tf # outputs + ├── networking.tf # VPC, public + private subnets, IGW, NAT + ├── compute.tf # EC2 instance + EBS data volume + ├── firewall.tf # security group (SSH 22, API 8080, scheduler 50051) + └── storage.tf # S3 CAS bucket (versioned, encrypted, private) +``` + +## Prerequisites + +- [OpenTofu](https://opentofu.org) (`tofu`) on PATH — Terraform ≥ 1.5 works too + (the Makefile uses `tofu`; set `TF=terraform` to switch). +- Cloud provider credentials (AWS: `AWS_PROFILE` or `AWS_ACCESS_KEY_ID` / + `AWS_SECRET_ACCESS_KEY` env vars) with EC2, VPC, EBS, and S3 permissions. +- An SSH key pair registered in EC2 (pass its name via `-var ssh_key_name=...`). +- An AMI ID for the host (Ubuntu 22.04/24.04): + + ```sh + aws ec2 describe-images --owners 099720109477 \ + --filters Name=name,values='ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*' \ + Name=root-device-type,values=ebs --query 'Images[0].ImageId' --output text + ``` + +## Quick start + +From `deploy/sandbox/`, five steps to a running sandbox server: + +```sh +# 1. Initialize (installs the AWS provider) +cd deploy/sandbox && tofu -chdir=terraform init + +# 2. Provision the KVM host + object store + firewall. +# Provide at least: ami_id, ssh_key_name. Tighten allow_cidr_blocks. +tofu -chdir=terraform apply -auto-approve \ + -var ami_id=ami-0... \ + -var ssh_key_name=my-key \ + -var 'allow_cidr_blocks=["203.0.113.0/24"]' + +# 3. Get onto the host +ssh ubuntu@$(tofu -chdir=terraform output -raw sandbox_host_ip) sudo -i + +# 4. Install the binary, config, and systemd unit, then start +export SANDBOX_BINARY_URL="https://.../sandbox-x86_64-unknown-linux-gnu" +# optional: export SANDBOX_API_KEY_HASH=$(printf '%s' "$FRONTAL_SANDBOX_API_KEY" | shasum -a 256) +./scripts/bootstrap.sh && ./scripts/deploy.sh + +# 5. Verify the server is healthy +./scripts/smoke.sh +``` + +> Note: the instance lives in a **private subnet**. `sandbox_host_ip` may be +> empty; reach the host over your VPN/bastion, or add an Elastic IP to make +> it reachable. The security group allows inbound 8080 only from +> `allow_cidr_blocks`. + +## Pointing code-server at the sandbox server + +In code-server's environment (e.g. `.env`): + +```sh +FRONTAL_SANDBOX_BASE_URL=http://:8080 # `tofu -chdir=terraform output -raw sandbox_base_url` +FRONTAL_SANDBOX_API_KEY=sbx_... # your sandbox API key +FRONTAL_CODE_SERVER_LANE_TRANSPORT=sandbox +``` + +The server authenticates with the SHA-256 hex hash of the API key. Pass the +hash (never the raw key) to `bootstrap.sh` as `SANDBOX_API_KEY_HASH`; it is +injected into the systemd unit as `SANDBOX_AUTH_API_KEY_HASH`. + +## Destroy + +```sh +cd deploy/sandbox +tofu -chdir=terraform destroy -auto-approve +``` + +Removes the EC2 instance, EBS volumes, S3 bucket, and all networking. +Anything under `/var/lib/sandbox` is gone with the instance; CAS objects are +gone with the bucket. Back up anything you need first. + +## Configuration reference + +The sandbox server reads `/etc/sandbox/config.toml`. The production defaults +live in [`config/production.toml`](config/production.toml); every key can be +overridden with a `SANDBOX_
_` env var (e.g. +`SANDBOX_AUTH_RATE_LIMIT_RPS`). + +| Section | Key | Description | +| --- | --- | --- | +| `server` | `addr` | API listen address (`0.0.0.0:8080`) | +| `server` | `data_dir` | State / CAS data directory (`/var/lib/sandbox`) | +| `auth` | `api_key_hash` | SHA-256 hex of the API key. Keep in `SANDBOX_AUTH_API_KEY_HASH` env var, never in the TOML. | +| `auth` | `rate_limit_rps` | API rate limit (requests/sec) | +| `tls` | `enabled` | TLS on the sandbox server. Not supported yet (v0.x) — terminate TLS at a reverse proxy. | +| `tls` | `cert_file` / `key_file` | Certificate / key paths (when `enabled = true`) | +| `telemetry` | `metrics_addr` | Prometheus metrics endpoint (`0.0.0.0:9090`) | +| `telemetry` | `otlp_endpoint` | Optional OTLP endpoint | +| `telemetry` | `service_name` | Service name reported by telemetry | +| `scheduler` | `address` | Scheduler gRPC listen address | + +## Troubleshooting + +- **KVM not available** (`/dev/kvm` missing): the instance type must support + KVM — on AWS that means bare metal (e.g. `m5.metal`) or a provider exposing + nested virtualization. Check with `ls -l /dev/kvm` and + `dmesg | grep -i kvm`. Tier 1 (container) sessions work without KVM; only + Tier 2 (VM) sessions need it. `bootstrap.sh` warns but continues when + `/dev/kvm` is absent. +- **Port 8080 blocked**: the security group only allows 8080 from + `allow_cidr_blocks`. Make sure the code-server's egress CIDR is in the list + and that no corporate firewall in between drops it. Test from the + code-server host: `curl -v http://:8080/v1/openapi.json`. +- **SSH from anywhere (port 22)**: the default `allow_cidr_blocks` is + `0.0.0.0/0`, which opens SSH to the world. Replace it with your operator + CIDR before applying anywhere real. +- **`smoke.sh` fails to create a session**: verify + `SANDBOX_API_KEY` (the key `smoke.sh` sends) matches the hash installed on the server + (`SANDBOX_AUTH_API_KEY_HASH` in `/etc/systemd/system/sandbox-server.service`). It must + equal the `FRONTAL_SANDBOX_API_KEY` that code-server uses as its caller key. + See `journalctl -u sandbox-server -n 100` on the host. +- **Service not active**: `systemctl status sandbox-server` and + `journalctl -u sandbox-server -n 100`. The most common causes are a bad + `SANDBOX_BINARY_URL` download (check `/usr/local/bin/sandbox` runs + `--version`) and a stale config (compare `/etc/sandbox/config.toml` with + `config/production.toml`). +- **Instance unreachable**: it lives in a private subnet with NAT-only + egress; inbound traffic requires your VPN/bastion or an added Elastic IP. diff --git a/deploy/sandbox/config/production.toml b/deploy/sandbox/config/production.toml new file mode 100644 index 00000000..645193c0 --- /dev/null +++ b/deploy/sandbox/config/production.toml @@ -0,0 +1,22 @@ +# Frontal Sandbox server configuration +# Installed to /etc/sandbox/config.toml by the deploy script. + +[server] +addr = "0.0.0.0:8080" +data_dir = "/var/lib/sandbox" + +[auth] +# Set via SANDBOX_AUTH_API_KEY_HASH env var in the systemd unit +rate_limit_rps = 100 + +[tls] +enabled = false +# When deploying behind a load balancer, terminate TLS there and leave +# the sandbox server plaintext on a private subnet. + +[telemetry] +metrics_addr = "0.0.0.0:9090" +service_name = "sandbox-server" + +[scheduler] +address = "0.0.0.0:50051" diff --git a/deploy/sandbox/scripts/bootstrap.sh b/deploy/sandbox/scripts/bootstrap.sh new file mode 100755 index 00000000..08248bb0 --- /dev/null +++ b/deploy/sandbox/scripts/bootstrap.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# +# bootstrap.sh — install the Frontal Sandbox server on the target host. +# +# Runs on the sandbox host. Idempotent: safe to re-run after a rebuild. +# +# Required environment: +# SANDBOX_BINARY_URL direct URL to the sandbox binary (no .tar.gz), or a +# "v" release tarball of the form +# sandbox--.tar.gz (extracted to +# /usr/local/bin/sandbox) +# +# Optional environment: +# SANDBOX_API_KEY_HASH SHA-256 (hex) of the API key code-server presents +# (printf '%s' "$KEY" | shasum -a 256). When set, it +# is injected into the systemd unit as +# SANDBOX_AUTH_API_KEY_HASH. +# SANDBOX_CONFIG_SRC local path to the TOML config to install +# (default: