From f71a50a1c0ef3f4edb88315373bbb783ffa6239d Mon Sep 17 00:00:00 2001 From: Volker Mauel Date: Fri, 20 Nov 2026 12:38:11 +0100 Subject: [PATCH] feat: OpenShell fleet - spawn, delegate, drain-then-delete workers --- .github/workflows/openshell-smoke.yml | 74 ++ .../SettingsScheduledTasksRouteScreen.tsx | 40 +- .../settings/scheduledTaskDraft.test.ts | 53 +- .../features/settings/scheduledTaskDraft.ts | 14 +- apps/mobile/src/i18n/translations/de.json | 2 + apps/mobile/src/i18n/translations/en.json | 2 + apps/mobile/src/i18n/translations/es.json | 2 + apps/mobile/src/i18n/translations/fr.json | 2 + apps/mobile/src/i18n/translations/ja.json | 2 + apps/mobile/src/i18n/translations/ko.json | 2 + apps/mobile/src/i18n/translations/pt-BR.json | 2 + apps/mobile/src/i18n/translations/ru.json | 2 + apps/mobile/src/i18n/translations/sv.json | 2 + apps/mobile/src/i18n/translations/zh.json | 2 + .../transferBudgetV2.integration.test.ts | 8 +- apps/server/package.json | 1 + .../OpenShellFleetBundleImport.test.ts | 665 ++++++++++ .../environment/OpenShellFleetBundleImport.ts | 598 +++++++++ .../environment/OpenShellFleetDrain.test.ts | 632 ++++++++++ .../src/environment/OpenShellFleetDrain.ts | 448 +++++++ .../OpenShellFleetDrainAdapters.test.ts | 803 ++++++++++++ .../OpenShellFleetDrainAdapters.ts | 614 +++++++++ .../environment/OpenShellFleetService.test.ts | 220 ++++ .../src/environment/OpenShellFleetService.ts | 562 +++++++++ .../environment/OpenShellFleetStore.test.ts | 133 ++ .../src/environment/OpenShellFleetStore.ts | 286 +++++ .../src/environment/OpenShellGatewayAuth.ts | 260 ++++ apps/server/src/mcp/McpHttpServer.test.ts | 2 + apps/server/src/mcp/McpHttpServer.ts | 19 +- .../src/mcp/McpInvocationContext.test.ts | 45 + apps/server/src/mcp/McpInvocationContext.ts | 12 + ...OrchestratorMcpToolkit.integration.test.ts | 2 + apps/server/src/mcp/linkOrigin.test.ts | 3 +- apps/server/src/mcp/toolkits/core.test.ts | 2 + .../toolkits/orchestrator/handlers.test.ts | 174 +++ .../src/mcp/toolkits/orchestrator/handlers.ts | 123 +- .../src/mcp/toolkits/orchestrator/tools.ts | 66 +- .../src/mcp/toolkits/project/handlers.test.ts | 26 + .../src/mcp/toolkits/project/handlers.ts | 8 + .../Adapters/ClaudeAdapterV2.ts | 1 + .../src/orchestration-v2/Orchestrator.ts | 37 +- .../OrchestratorSandboxRefusal.test.ts | 108 ++ .../src/orchestration-v2/ProjectionStore.ts | 2 +- .../src/orchestration-v2/ShellStream.ts | 17 +- .../ThreadLaunchService.test.ts | 49 + .../orchestration-v2/ThreadLaunchService.ts | 65 +- apps/server/src/orchestration-v2/http.ts | 11 +- .../src/peer/PeerEnvironmentRegistry.test.ts | 497 ++++++++ .../src/peer/PeerEnvironmentRegistry.ts | 588 +++++++++ apps/server/src/peer/PeerForwarding.test.ts | 7 +- apps/server/src/peer/PeerLinkRequests.test.ts | 2 + apps/server/src/peer/PeerLinks.ts | 2 +- apps/server/src/peer/PeerMcpClient.test.ts | 103 +- apps/server/src/peer/PeerMcpClient.ts | 99 +- .../src/peer/PeerSessionResolve.test.ts | 87 ++ apps/server/src/peer/PeerSessionResolve.ts | 49 + apps/server/src/peer/RemoteDelegation.test.ts | 4 +- apps/server/src/persistence/Migrations.ts | 6 + .../Migrations/109_FleetPeerEnvironments.ts | 24 + .../Migrations/110_OpenShellFleetState.ts | 25 + .../Migrations/111_FleetDrainImports.ts | 21 + .../reconcileV2PreviewMigration.test.ts | 6 + .../src/provider/AgentContextBudget.test.ts | 2 + .../src/provider/agentContextBudget.json | 2 +- .../ScheduledTaskService.test.ts | 44 + .../scheduledTasks/ScheduledTaskService.ts | 12 + apps/server/src/server.ts | 28 +- apps/server/src/serverSettings.test.ts | 3 + apps/server/src/settingsSecrets.ts | 33 + apps/server/src/ws.ts | 14 + .../components/settings/LabsSettingsPanel.tsx | 157 ++- .../settings/ScheduledTasksSettings.tsx | 52 +- .../scheduledTasksSettings.logic.test.ts | 27 + .../settings/scheduledTasksSettings.logic.ts | 14 +- .../src/components/settings/settingsSearch.ts | 40 +- apps/web/src/i18n/translations/de.json | 20 + apps/web/src/i18n/translations/en.json | 20 + apps/web/src/i18n/translations/es.json | 20 + apps/web/src/i18n/translations/fr.json | 20 + apps/web/src/i18n/translations/ja.json | 20 + apps/web/src/i18n/translations/ko.json | 20 + apps/web/src/i18n/translations/pt-BR.json | 20 + apps/web/src/i18n/translations/ru.json | 20 + apps/web/src/i18n/translations/sv.json | 20 + apps/web/src/i18n/translations/zh.json | 20 + evals/mcp-tool-use/run.test.ts | 21 +- evals/mcp-tool-use/run.ts | 8 +- knip.jsonc | 1 + packages/contracts/src/environmentHttp.ts | 2 + packages/contracts/src/index.ts | 1 + .../contracts/src/openshellGateway.test.ts | 306 +++++ packages/contracts/src/openshellGateway.ts | 339 +++++ packages/contracts/src/orchestrationV2.ts | 14 + packages/contracts/src/orchestratorMcp.ts | 81 +- packages/contracts/src/settings.ts | 6 + packages/openshell/README.md | 55 + packages/openshell/package.json | 33 + .../src/OpenShellGatewayDriver.test.ts | 1113 +++++++++++++++++ .../openshell/src/OpenShellGatewayDriver.ts | 954 ++++++++++++++ packages/openshell/src/gatewayClient.test.ts | 100 ++ packages/openshell/src/gatewayClient.ts | 485 +++++++ packages/openshell/src/sandboxPolicy.test.ts | 57 + packages/openshell/src/sandboxPolicy.ts | 77 ++ packages/openshell/src/workerBaggage.test.ts | 110 ++ packages/openshell/src/workerBaggage.ts | 234 ++++ packages/openshell/tsconfig.json | 7 + .../vendor/nvidia-openshell-sdk-0.1.7.tgz | Bin 0 -> 189989 bytes .../src/orchestrationV2ThreadError.test.ts | 60 + .../shared/src/orchestrationV2ThreadError.ts | 16 +- packages/shared/src/serverSettings.test.ts | 65 + pnpm-lock.yaml | 114 +- scripts/lib/tagged-source-version.ts | 1 + scripts/openshell-smoke.test.ts | 137 ++ scripts/openshell-smoke.ts | 430 +++++++ scripts/package.json | 1 + scripts/release-smoke.ts | 10 + 116 files changed, 13043 insertions(+), 146 deletions(-) create mode 100644 .github/workflows/openshell-smoke.yml create mode 100644 apps/server/src/environment/OpenShellFleetBundleImport.test.ts create mode 100644 apps/server/src/environment/OpenShellFleetBundleImport.ts create mode 100644 apps/server/src/environment/OpenShellFleetDrain.test.ts create mode 100644 apps/server/src/environment/OpenShellFleetDrain.ts create mode 100644 apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts create mode 100644 apps/server/src/environment/OpenShellFleetDrainAdapters.ts create mode 100644 apps/server/src/environment/OpenShellFleetService.test.ts create mode 100644 apps/server/src/environment/OpenShellFleetService.ts create mode 100644 apps/server/src/environment/OpenShellFleetStore.test.ts create mode 100644 apps/server/src/environment/OpenShellFleetStore.ts create mode 100644 apps/server/src/environment/OpenShellGatewayAuth.ts create mode 100644 apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts create mode 100644 apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts create mode 100644 apps/server/src/peer/PeerEnvironmentRegistry.test.ts create mode 100644 apps/server/src/peer/PeerEnvironmentRegistry.ts create mode 100644 apps/server/src/peer/PeerSessionResolve.test.ts create mode 100644 apps/server/src/peer/PeerSessionResolve.ts create mode 100644 apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts create mode 100644 apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts create mode 100644 apps/server/src/persistence/Migrations/111_FleetDrainImports.ts create mode 100644 packages/contracts/src/openshellGateway.test.ts create mode 100644 packages/contracts/src/openshellGateway.ts create mode 100644 packages/openshell/README.md create mode 100644 packages/openshell/package.json create mode 100644 packages/openshell/src/OpenShellGatewayDriver.test.ts create mode 100644 packages/openshell/src/OpenShellGatewayDriver.ts create mode 100644 packages/openshell/src/gatewayClient.test.ts create mode 100644 packages/openshell/src/gatewayClient.ts create mode 100644 packages/openshell/src/sandboxPolicy.test.ts create mode 100644 packages/openshell/src/sandboxPolicy.ts create mode 100644 packages/openshell/src/workerBaggage.test.ts create mode 100644 packages/openshell/src/workerBaggage.ts create mode 100644 packages/openshell/tsconfig.json create mode 100644 packages/openshell/vendor/nvidia-openshell-sdk-0.1.7.tgz create mode 100644 packages/shared/src/orchestrationV2ThreadError.test.ts create mode 100644 scripts/openshell-smoke.test.ts create mode 100644 scripts/openshell-smoke.ts diff --git a/.github/workflows/openshell-smoke.yml b/.github/workflows/openshell-smoke.yml new file mode 100644 index 0000000000000..5d52145f2fcba --- /dev/null +++ b/.github/workflows/openshell-smoke.yml @@ -0,0 +1,74 @@ +name: OpenShell gateway smoke + +# Opt-in live check of the OpenShell fleet integration against a REAL gateway. +# The job runs only when a repository administrator configures gateway access; +# otherwise every step skips and pull requests stay green. +# +# How to enable (repository secrets, set once by an administrator): +# OPENSHELL_SMOKE_GATEWAY_URL the gateway control-plane base URL (https) +# OPENSHELL_SMOKE_GATEWAY_CREDENTIAL the bearer credential the gateway's OIDC +# token provider returns to the OpenShell SDK +# Optional repository variable: +# OPENSHELL_SMOKE_TEMPLATE sandbox template to spawn; unset falls +# back to a worker-labeled template listed +# by the gateway. +# What the gateway must expose: health, sandbox template listing, sandbox +# create/wait/get/delete, and label filtering on `t3-managed-by`. +# Budget: the script defaults to 900 seconds (OPENSHELL_SMOKE_TIMEOUT_SECS). + +on: + workflow_dispatch: + schedule: + # Avoid minute zero, when GitHub scheduled jobs are busiest. + - cron: "17 4 * * 1" + pull_request: + paths: + - "packages/openshell/**" + - "apps/server/src/environment/**" + - "scripts/openshell-smoke.ts" + - ".github/workflows/openshell-smoke.yml" + +permissions: + contents: read + +concurrency: + group: openshell-smoke-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + smoke: + name: Gateway smoke + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + # Same presence pattern as the EXPO_TOKEN check in mobile-eas-production.yml. + - id: smoke-secrets + name: Check for gateway secrets + env: + SMOKE_URL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_URL }} + SMOKE_CREDENTIAL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_CREDENTIAL }} + run: | + if [ -n "$SMOKE_URL" ] && [ -n "$SMOKE_CREDENTIAL" ]; then + echo "present=true" >> "$GITHUB_OUTPUT" + else + echo "present=false" >> "$GITHUB_OUTPUT" + echo "OPENSHELL_SMOKE_GATEWAY_URL and OPENSHELL_SMOKE_GATEWAY_CREDENTIAL are not configured; skipping the live gateway smoke." + fi + + - name: Checkout + if: steps.smoke-secrets.outputs.present == 'true' + uses: actions/checkout@v6 + + - name: Setup Vite+ + if: steps.smoke-secrets.outputs.present == 'true' + uses: ./.github/actions/setup-ci-deps + with: + task-cache-read-token: ${{ secrets.T3_TASK_CACHE_READ_TOKEN }} + + - name: Run the gateway smoke + if: steps.smoke-secrets.outputs.present == 'true' + env: + OPENSHELL_SMOKE_GATEWAY_URL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_URL }} + OPENSHELL_SMOKE_GATEWAY_CREDENTIAL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_CREDENTIAL }} + OPENSHELL_SMOKE_TEMPLATE: ${{ vars.OPENSHELL_SMOKE_TEMPLATE }} + run: node scripts/openshell-smoke.ts diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx index 8e15dad08ac23..66bddc72673c4 100644 --- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx +++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx @@ -80,6 +80,7 @@ import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-en import { changeDraftProject, editDraft, + leavesSandboxWorkspace, parseIntervalMinutes, scheduledTaskDefaultModel, scheduleFromDraft, @@ -891,11 +892,13 @@ function TaskForm({ ? { type: "root" } : draft.workspace === "existing_worktree" ? { type: "existing_worktree", worktreePath: draft.checkoutPath.trim() } - : { - type: "worktree", - baseRef: worktreeBaseRef, - startFromOrigin: draft.startFromOrigin, - }, + : draft.workspace === "sandbox" + ? { type: "sandbox" } + : { + type: "worktree", + baseRef: worktreeBaseRef, + startFromOrigin: draft.startFromOrigin, + }, // Tasks have no mode picker; a mode above the administrator's ceiling is refused. runtimeMode: clampManagedRuntimeMode(config?.managedPolicy, draft.runtimeMode) ?? draft.runtimeMode, @@ -1091,9 +1094,20 @@ function TaskForm({ ? gt("New worktree") : draft.workspace === "root" ? gt("Project checkout") - : gt("Specific checkout") + : draft.workspace === "sandbox" + ? gt("Fleet worker sandbox") + : gt("Specific checkout") } actions={[ + ...(draft.storedSandbox + ? [ + { + id: "sandbox", + title: gt("Fleet worker sandbox"), + state: draft.workspace === "sandbox" ? ("on" as const) : undefined, + }, + ] + : []), { id: "worktree", title: gt("New worktree"), @@ -1111,10 +1125,22 @@ function TaskForm({ }, ]} onSelect={(id) => { - if (id === "worktree" || id === "root" || id === "existing_worktree") + if ( + id === "worktree" || + id === "root" || + id === "existing_worktree" || + id === "sandbox" + ) setDraft((current) => ({ ...current, workspace: id })); }} /> + {leavesSandboxWorkspace(draft) ? ( + + + Future runs will no longer use the fleet worker sandbox this task is paired with. + + + ) : null} {draft.workspace === "worktree" ? ( { ).toBe("release"); }); }); + +describe("sandbox workspace retention", () => { + const task = (workspaceStrategy: ScheduledTask["workspaceStrategy"]): ScheduledTask => ({ + id: ScheduledTaskId.make("task"), + title: "Nightly review", + prompt: "Review open issues", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project"), + threadId: null, + workspaceStrategy, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + createdBy: "user", + creationSource: "mobile", + createdAt: "2026-09-17T00:00:00.000Z", + updatedAt: "2026-09-17T00:00:00.000Z", + nextRunAt: null, + lastRunAt: null, + lastRunStatus: "never", + lastRunError: null, + runCount: 0, + }); + + it("keeps a stored sandbox strategy when editing an unrelated field", () => { + const initial = editDraft(task({ type: "sandbox" })); + expect(initial.workspace).toBe("sandbox"); + expect(initial.storedSandbox).toBe(true); + expect(leavesSandboxWorkspace(initial)).toBe(false); + // Renaming the task does not move it off its fleet worker. + const renamed = { ...initial, title: "Renamed" }; + expect(hasScheduledTaskDraftChanges(initial, renamed)).toBe(true); + expect(leavesSandboxWorkspace(renamed)).toBe(false); + }); + + it("surfaces a warning only when the workspace control leaves the sandbox", () => { + const initial = editDraft(task({ type: "sandbox" })); + const moved = { ...initial, workspace: "worktree" as const }; + expect(leavesSandboxWorkspace(moved)).toBe(true); + expect(hasScheduledTaskDraftChanges(initial, moved)).toBe(true); + // A task that never ran in a sandbox is never flagged. + const worktreeDraft = editDraft( + task({ type: "worktree", baseRef: "release", startFromOrigin: false }), + ); + expect(worktreeDraft.storedSandbox).toBe(false); + expect(leavesSandboxWorkspace({ ...worktreeDraft, workspace: "root" as const })).toBe(false); + }); +}); diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts index 5bacd4be83c81..2b3f01282934a 100644 --- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts +++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts @@ -197,7 +197,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche }; } -type Workspace = "worktree" | "root" | "existing_worktree"; +type Workspace = "worktree" | "root" | "existing_worktree" | "sandbox"; export type ScheduledTaskDraft = { readonly task: ScheduledTask | null; readonly title: string; @@ -209,6 +209,8 @@ export type ScheduledTaskDraft = { readonly schedule: ScheduleDraft; readonly threadMode: ScheduledTaskThreadMode; readonly workspace: Workspace; + /** The stored task runs in a fleet worker sandbox; leaving it is surfaced. */ + readonly storedSandbox: boolean; readonly baseRef: string; readonly checkoutPath: string; readonly enabled: boolean; @@ -301,6 +303,7 @@ export function createDraft( schedule: DEFAULT_SCHEDULE, threadMode: "new_thread", workspace: "worktree", + storedSandbox: false, baseRef: "", checkoutPath: "", enabled: true, @@ -309,6 +312,11 @@ export function createDraft( }; } +/** An edit that moves a stored sandbox task off its fleet worker; the editor surfaces it. */ +export function leavesSandboxWorkspace(draft: ScheduledTaskDraft): boolean { + return draft.storedSandbox && draft.workspace !== "sandbox"; +} + export function editDraft(task: ScheduledTask): ScheduledTaskDraft { return { task, @@ -320,7 +328,11 @@ export function editDraft(task: ScheduledTask): ScheduledTaskDraft { modelSelectionIsExplicit: true, schedule: scheduleDraftForTask(task), threadMode: scheduledTaskThreadMode(task), + // A sandbox workspace stays: editing an unrelated field must not silently + // move the task off its fleet worker. workspace: task.workspaceStrategy.type, + storedSandbox: task.workspaceStrategy.type === "sandbox", + // Main's newer editor keeps the base automatic unless a worktree pinned one. baseRef: task.workspaceStrategy.type === "worktree" ? task.workspaceStrategy.baseRef : "", checkoutPath: task.workspaceStrategy.type === "existing_worktree" diff --git a/apps/mobile/src/i18n/translations/de.json b/apps/mobile/src/i18n/translations/de.json index 8777f08e663ce..1ebd3a20d6d72 100644 --- a/apps/mobile/src/i18n/translations/de.json +++ b/apps/mobile/src/i18n/translations/de.json @@ -3595,6 +3595,7 @@ "85728e592594c929": "Ausführen in", "c7fe4313def0ef2f": "Neuer Worktree", "e91a39ca1b5d9bbd": "Projekt-Checkout", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Bestimmter Checkout", "8b724804e55e48e6": "Checkout-Pfad", "b325f97ceab8bc64": "Zeitplan", @@ -3653,6 +3654,7 @@ "3c5185df378edef9": "Wird nur ausgeführt, wenn alle Filter zutreffen. Andere Anfragen werden als gefiltert protokolliert. Trenne „ist eines von“-Werte mit Kommas; ein Wert darf kein Komma enthalten.", "d1a74724f70edb0b": "Diese Aufgabe existiert nicht mehr.", "c191069dded46bcc": "Diese Umgebung ist getrennt. Verbinde sie erneut, bevor du speicherst.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Anfragen mit demselben Schlüssel werden innerhalb von 7 Tagen nur einmal ausgeführt.", "a5bbe09b0c89d86c": "Beim Zusammenfassen wird die neueste Payload ausgeführt, wenn das Zeitfenster ab der ersten Zustellung endet. Leere Schlüssel werden normal ausgeführt.", "0e2233fb297f7c07": "Intervalle müssen mindestens 1 Minute betragen. Passe dieses Intervall vor dem Speichern an.", diff --git a/apps/mobile/src/i18n/translations/en.json b/apps/mobile/src/i18n/translations/en.json index dfe3a473fe248..4d0f4917942e4 100644 --- a/apps/mobile/src/i18n/translations/en.json +++ b/apps/mobile/src/i18n/translations/en.json @@ -3597,6 +3597,7 @@ "85728e592594c929": "Run in", "c7fe4313def0ef2f": "New worktree", "e91a39ca1b5d9bbd": "Project checkout", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Specific checkout", "8b724804e55e48e6": "Checkout path", "b325f97ceab8bc64": "Schedule", @@ -3655,6 +3656,7 @@ "3c5185df378edef9": "Runs only when every filter matches. Other requests are logged as filtered. Separate “is one of” values with commas; a value can't contain a comma.", "d1a74724f70edb0b": "This task no longer exists.", "c191069dded46bcc": "This environment is disconnected. Reconnect before saving.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Requests with the same key run only once within 7 days.", "a5bbe09b0c89d86c": "Coalescing runs the latest payload when the window from the first delivery closes. Empty keys run normally.", "0e2233fb297f7c07": "Intervals must be at least 1 minute. Update this interval before saving.", diff --git a/apps/mobile/src/i18n/translations/es.json b/apps/mobile/src/i18n/translations/es.json index 05f58a63227d3..001c976cdf1cf 100644 --- a/apps/mobile/src/i18n/translations/es.json +++ b/apps/mobile/src/i18n/translations/es.json @@ -3590,6 +3590,7 @@ "85728e592594c929": "Ejecutar en", "c7fe4313def0ef2f": "Nuevo worktree", "e91a39ca1b5d9bbd": "Checkout del proyecto", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Checkout concreto", "8b724804e55e48e6": "Ruta del checkout", "b325f97ceab8bc64": "Programación", @@ -3648,6 +3649,7 @@ "3c5185df378edef9": "Se ejecuta solo cuando coinciden todos los filtros. Las demás solicitudes se registran como filtradas. Separa los valores de «es uno de» con comas; un valor no puede contener una coma.", "d1a74724f70edb0b": "Esta tarea ya no existe.", "c191069dded46bcc": "Este entorno está desconectado. Vuelve a conectarlo antes de guardar.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Las solicitudes con la misma clave solo se ejecutan una vez en 7 días.", "a5bbe09b0c89d86c": "La agrupación ejecuta la carga más reciente cuando se cierra la ventana abierta por la primera entrega. Las claves vacías se ejecutan con normalidad.", "0e2233fb297f7c07": "Los intervalos deben ser de al menos 1 minuto. Actualiza este intervalo antes de guardar.", diff --git a/apps/mobile/src/i18n/translations/fr.json b/apps/mobile/src/i18n/translations/fr.json index f661c315215c0..7d1497e707dec 100644 --- a/apps/mobile/src/i18n/translations/fr.json +++ b/apps/mobile/src/i18n/translations/fr.json @@ -3590,6 +3590,7 @@ "85728e592594c929": "Exécuter dans", "c7fe4313def0ef2f": "Nouveau worktree", "e91a39ca1b5d9bbd": "Checkout du projet", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Checkout précis", "8b724804e55e48e6": "Chemin du checkout", "b325f97ceab8bc64": "Planification", @@ -3648,6 +3649,7 @@ "3c5185df378edef9": "S’exécute uniquement si tous les filtres correspondent. Les autres requêtes sont journalisées comme filtrées. Séparez les valeurs « fait partie de » par des virgules ; une valeur ne peut pas contenir de virgule.", "d1a74724f70edb0b": "Cette tâche n’existe plus.", "c191069dded46bcc": "Cet environnement est déconnecté. Reconnectez-le avant d’enregistrer.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Les requêtes avec la même clé ne s’exécutent qu’une fois en 7 jours.", "a5bbe09b0c89d86c": "Le regroupement exécute la dernière charge utile quand la fenêtre ouverte par la première livraison se ferme. Les clés vides s’exécutent normalement.", "0e2233fb297f7c07": "Les intervalles doivent être d’au moins 1 minute. Modifiez cet intervalle avant d’enregistrer.", diff --git a/apps/mobile/src/i18n/translations/ja.json b/apps/mobile/src/i18n/translations/ja.json index 56616809052d3..69e48be8e8448 100644 --- a/apps/mobile/src/i18n/translations/ja.json +++ b/apps/mobile/src/i18n/translations/ja.json @@ -3586,6 +3586,7 @@ "85728e592594c929": "実行場所", "c7fe4313def0ef2f": "新しいワークツリー", "e91a39ca1b5d9bbd": "プロジェクトのチェックアウト", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "特定のチェックアウト", "8b724804e55e48e6": "チェックアウトのパス", "b325f97ceab8bc64": "スケジュール", @@ -3644,6 +3645,7 @@ "3c5185df378edef9": "すべてのフィルターに一致した場合のみ実行します。それ以外のリクエストはフィルター済みとして記録されます。「いずれかに一致」の値はカンマで区切ってください。値にカンマを含めることはできません。", "d1a74724f70edb0b": "このタスクは存在しません。", "c191069dded46bcc": "この環境は切断されています。保存する前に再接続してください。", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "同じキーのリクエストは 7 日以内に 1 回だけ実行されます。", "a5bbe09b0c89d86c": "集約すると、最初の配信からの期間が終了したときに最新のペイロードで実行します。キーが空の場合は通常どおり実行します。", "0e2233fb297f7c07": "間隔は 1 分以上にする必要があります。保存する前にこの間隔を更新してください。", diff --git a/apps/mobile/src/i18n/translations/ko.json b/apps/mobile/src/i18n/translations/ko.json index 443285f8664c4..bddfcba0eeb05 100644 --- a/apps/mobile/src/i18n/translations/ko.json +++ b/apps/mobile/src/i18n/translations/ko.json @@ -3590,6 +3590,7 @@ "85728e592594c929": "실행 위치", "c7fe4313def0ef2f": "새 워크트리", "e91a39ca1b5d9bbd": "프로젝트 체크아웃", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "특정 체크아웃", "8b724804e55e48e6": "체크아웃 경로", "b325f97ceab8bc64": "일정", @@ -3648,6 +3649,7 @@ "3c5185df378edef9": "모든 필터가 일치할 때만 실행합니다. 다른 요청은 필터링됨으로 기록됩니다. “다음 중 하나” 값은 쉼표로 구분하며, 값에는 쉼표를 넣을 수 없습니다.", "d1a74724f70edb0b": "이 작업은 더 이상 존재하지 않습니다.", "c191069dded46bcc": "이 환경의 연결이 끊어졌습니다. 저장하기 전에 다시 연결하세요.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "키가 같은 요청은 7일 이내에 한 번만 실행됩니다.", "a5bbe09b0c89d86c": "병합하면 첫 전송부터 시작된 기간이 끝날 때 최신 페이로드로 실행합니다. 키가 비어 있으면 정상적으로 실행됩니다.", "0e2233fb297f7c07": "간격은 1분 이상이어야 합니다. 저장하기 전에 간격을 업데이트하세요.", diff --git a/apps/mobile/src/i18n/translations/pt-BR.json b/apps/mobile/src/i18n/translations/pt-BR.json index 33793cc553d34..bf75b05b8fcc9 100644 --- a/apps/mobile/src/i18n/translations/pt-BR.json +++ b/apps/mobile/src/i18n/translations/pt-BR.json @@ -3590,6 +3590,7 @@ "85728e592594c929": "Executar em", "c7fe4313def0ef2f": "Novo worktree", "e91a39ca1b5d9bbd": "Checkout do projeto", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Checkout específico", "8b724804e55e48e6": "Caminho do checkout", "b325f97ceab8bc64": "Agendamento", @@ -3648,6 +3649,7 @@ "3c5185df378edef9": "Executa só quando todos os filtros correspondem. As outras requisições são registradas como filtradas. Separe os valores de “é um de” com vírgulas; um valor não pode conter vírgula.", "d1a74724f70edb0b": "Esta tarefa não existe mais.", "c191069dded46bcc": "Este ambiente está desconectado. Reconecte antes de salvar.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Requisições com a mesma chave executam só uma vez em 7 dias.", "a5bbe09b0c89d86c": "O agrupamento executa o payload mais recente quando a janela iniciada pela primeira entrega fecha. Chaves vazias executam normalmente.", "0e2233fb297f7c07": "Os intervalos devem ser de pelo menos 1 minuto. Atualize este intervalo antes de salvar.", diff --git a/apps/mobile/src/i18n/translations/ru.json b/apps/mobile/src/i18n/translations/ru.json index d0e2d9577a9ab..fc450831f12a4 100644 --- a/apps/mobile/src/i18n/translations/ru.json +++ b/apps/mobile/src/i18n/translations/ru.json @@ -3644,6 +3644,7 @@ "85728e592594c929": "Выполнять в", "c7fe4313def0ef2f": "Новое рабочее дерево", "e91a39ca1b5d9bbd": "Копия проекта", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Конкретная копия", "8b724804e55e48e6": "Путь к копии", "b325f97ceab8bc64": "Расписание", @@ -3702,6 +3703,7 @@ "3c5185df378edef9": "Выполняется, только если совпадают все фильтры. Остальные запросы записываются как отфильтрованные. Значения для «одно из» разделяйте запятыми; значение не может содержать запятую.", "d1a74724f70edb0b": "Эта задача больше не существует.", "c191069dded46bcc": "Это окружение отключено. Переподключитесь перед сохранением.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Запросы с одинаковым ключом выполняются только один раз в течение 7 дней.", "a5bbe09b0c89d86c": "При объединении последний payload выполняется, когда закрывается окно, отсчитанное от первой доставки. Пустые ключи выполняются как обычно.", "0e2233fb297f7c07": "Интервал должен быть не меньше 1 минуты. Измените интервал перед сохранением.", diff --git a/apps/mobile/src/i18n/translations/sv.json b/apps/mobile/src/i18n/translations/sv.json index 7fd2f945cc033..fbe2eeba444dc 100644 --- a/apps/mobile/src/i18n/translations/sv.json +++ b/apps/mobile/src/i18n/translations/sv.json @@ -3596,6 +3596,7 @@ "85728e592594c929": "Kör i", "c7fe4313def0ef2f": "Ny worktree", "e91a39ca1b5d9bbd": "Projektets utcheckning", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "Specifik utcheckning", "8b724804e55e48e6": "Sökväg till utcheckning", "b325f97ceab8bc64": "Schema", @@ -3654,6 +3655,7 @@ "3c5185df378edef9": "Körs bara när alla filter matchar. Andra begäranden loggas som filtrerade. Separera värden för ”är en av” med kommatecken; ett värde får inte innehålla kommatecken.", "d1a74724f70edb0b": "Uppgiften finns inte längre.", "c191069dded46bcc": "Miljön är frånkopplad. Återanslut innan du sparar.", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "Begäranden med samma nyckel körs bara en gång inom 7 dagar.", "a5bbe09b0c89d86c": "Sammanslagning kör den senaste nyttolasten när fönstret från den första leveransen stängs. Tomma nycklar körs som vanligt.", "0e2233fb297f7c07": "Intervall måste vara minst 1 minut. Uppdatera intervallet innan du sparar.", diff --git a/apps/mobile/src/i18n/translations/zh.json b/apps/mobile/src/i18n/translations/zh.json index 54c72efe3b4fb..bff9aea153025 100644 --- a/apps/mobile/src/i18n/translations/zh.json +++ b/apps/mobile/src/i18n/translations/zh.json @@ -3592,6 +3592,7 @@ "85728e592594c929": "运行位置", "c7fe4313def0ef2f": "新工作树", "e91a39ca1b5d9bbd": "项目工作副本", + "c1b7be381b2a2df1": "Fleet worker sandbox", "27a6259d0e7c054e": "指定工作副本", "8b724804e55e48e6": "工作副本路径", "b325f97ceab8bc64": "定时规则", @@ -3650,6 +3651,7 @@ "3c5185df378edef9": "仅在所有筛选条件都匹配时运行。其他请求会记录为已筛除。“属于其中之一”的多个值用逗号分隔;单个值不能包含逗号。", "d1a74724f70edb0b": "此任务已不存在。", "c191069dded46bcc": "此环境已断开连接。保存前请重新连接。", + "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.", "0d15db4c81851479": "键相同的请求在 7 天内只运行一次。", "a5bbe09b0c89d86c": "合并会在自首次投递起的窗口结束时,用最新的负载运行。空键正常运行。", "0e2233fb297f7c07": "间隔必须至少为 1 分钟。保存前请更新此间隔。", diff --git a/apps/server/integration/transferBudgetV2.integration.test.ts b/apps/server/integration/transferBudgetV2.integration.test.ts index 4aaac3b9e30e9..e4df9fd314ca2 100644 --- a/apps/server/integration/transferBudgetV2.integration.test.ts +++ b/apps/server/integration/transferBudgetV2.integration.test.ts @@ -109,7 +109,9 @@ const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Req const CLIENT_SHELL_OPT_INS = { omitIdleSubagents: "1", omitOlderSettled: "1", -} satisfies Required>; +} satisfies Required< + Omit +>; const CLIENT_THREAD_SUBSCRIPTION = { requestCompletionMarker: true, acceptBoundedSnapshot: true, @@ -133,7 +135,9 @@ const clientShellResumeQuery = (resumeFrom: number) => query({ ...CLIENT_SHELL_OPT_INS, resumeFrom: String(resumeFrom), - } satisfies Required); + } satisfies Required< + Omit + >); const decodeThreadSnapshot = Schema.decodeUnknownEffect( Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)), diff --git a/apps/server/package.json b/apps/server/package.json index 9b9b3c022cacb..0942d991fcd5c 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -86,6 +86,7 @@ "@handy-computer/models": "0.2.3", "@opencode-ai/sdk": "^1.3.15", "@t3tools/contracts": "workspace:*", + "@t3tools/openshell": "workspace:*", "@t3tools/provider-testing": "workspace:*", "@t3tools/secret-provider-core": "workspace:*", "@t3tools/secret-provider-definitions": "workspace:*", diff --git a/apps/server/src/environment/OpenShellFleetBundleImport.test.ts b/apps/server/src/environment/OpenShellFleetBundleImport.test.ts new file mode 100644 index 0000000000000..12503179b8bf3 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetBundleImport.test.ts @@ -0,0 +1,665 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as SqlClient from "effect/sql/SqlClient"; + +import { EnvironmentId } from "@t3tools/contracts"; + +import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts"; +import * as Sqlite from "../persistence/Sqlite.ts"; +import Migration0111 from "../persistence/Migrations/111_FleetDrainImports.ts"; +import * as OpenShellFleetDrainAdapters from "./OpenShellFleetDrainAdapters.ts"; +import * as OpenShellFleetBundleImport from "./OpenShellFleetBundleImport.ts"; + +const ENVIRONMENT = "environment:worker"; +const DRAIN_ID = "drain:environment:worker:1"; + +const base64 = (text: string) => Buffer.from(text, "utf8").toString("base64"); + +/** A minimal wire-format bounded snapshot for one archived-candidate thread. */ +const snapshotJson = (input: { + readonly threadId: string; + readonly snapshotSequence: number; + readonly title?: string; + readonly itemText?: string; + readonly compacted?: boolean; +}) => ({ + snapshotSequence: input.snapshotSequence, + ...(input.compacted === true ? { turnItemsOmitLocalVisible: true } : {}), + projection: { + thread: { + createdBy: "system", + creationSource: "server", + id: input.threadId, + projectId: "project:drain-test", + title: input.title ?? "Worker thread", + providerInstanceId: "codex", + modelSelection: { instanceId: "codex", model: "gpt-5.4" }, + runtimeMode: "auto", + interactionMode: "default", + branch: null, + worktreePath: null, + activeProviderThreadId: null, + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: input.threadId }, + forkedFrom: null, + createdAt: "2026-11-06T10:00:00.000Z", + updatedAt: "2026-11-06T11:00:00.000Z", + archivedAt: null, + deletedAt: null, + }, + runs: [], + attempts: [], + nodes: [], + subagents: [], + providerSessions: [], + providerThreads: [ + { + id: "provider-thread:1", + driver: "codex", + providerInstanceId: "codex", + providerSessionId: null, + appThreadId: input.threadId, + ownerNodeId: null, + nativeThreadRef: { driver: "codex", nativeId: "native-1", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + pendingBackgroundTasks: [], + createdAt: "2026-11-06T10:00:00.000Z", + updatedAt: "2026-11-06T11:00:00.000Z", + }, + ], + providerTurns: [], + runtimeRequests: [], + messages: [ + { + createdBy: "user", + creationSource: "server", + id: `${input.threadId}:msg:1`, + threadId: input.threadId, + runId: null, + nodeId: null, + role: "user", + text: "hello from the worker", + attachments: [], + streaming: false, + createdAt: "2026-11-06T10:00:00.000Z", + updatedAt: "2026-11-06T10:00:00.000Z", + }, + ], + plans: [], + turnItems: [ + { + id: `${input.threadId}:item:1`, + threadId: input.threadId, + runId: null, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "completed", + title: null, + startedAt: "2026-11-06T10:00:00.000Z", + completedAt: "2026-11-06T10:00:00.000Z", + updatedAt: "2026-11-06T10:00:00.000Z", + createdBy: "user", + creationSource: "server", + type: "user_message", + messageId: `${input.threadId}:msg:1`, + inputIntent: "turn_start", + text: input.itemText ?? "hello from the worker", + attachments: [], + }, + ], + checkpointScopes: [], + checkpoints: [], + contextHandoffs: [], + contextTransfers: [], + visibleTurnItems: [], + updatedAt: "2026-11-06T11:00:00.000Z", + }, + historyCursor: null, + hasMoreHistory: false, + latestLocalTurnOrdinal: 1, +}); + +interface WrittenEvents { + readonly writes: Array>; +} + +interface FakeStoreState { + readonly rows: Map; + readonly exists: Set; +} + +const harness = Effect.acquireUseRelease( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectory(); + const attachmentsDir = yield* fs.makeTempDirectory(); + return { fs, path, stateDir, attachmentsDir }; + }), + (resources) => + Effect.gen(function* () { + const { fs, path, stateDir, attachmentsDir } = resources; + const written: WrittenEvents["writes"] = []; + const store: FakeStoreState = { rows: new Map(), exists: new Set() }; + let writesUntilFailure: number | undefined; + const eventSink = { + write: (input: { readonly events: ReadonlyArray<{ id: string; type: string }> }) => + Effect.suspend(() => { + if (writesUntilFailure !== undefined) { + if (writesUntilFailure === 0) { + return Effect.fail( + new OpenShellFleetBundleImport.FleetDrainImportError({ + operation: "record", + message: "the event sink is unavailable", + }), + ); + } + writesUntilFailure -= 1; + } + return Effect.sync(() => { + written.push(input.events); + // The fake projection learns about the thread from its creation event, + // as the real event sink does. + for (const event of input.events as ReadonlyArray<{ + readonly type?: string; + readonly threadId?: unknown; + }>) + if (event.type === "thread.created") store.exists.add(String(event.threadId)); + return input.events; + }); + }), + }; + const importer = OpenShellFleetBundleImport.makeFleetBundleImporter({ + stateDir, + attachmentsDir, + fs, + path, + now: Effect.succeed(1_700_000_000_000), + eventSink: eventSink as never, + store: { + find: (threadId) => Effect.succeed(store.rows.get(threadId) ?? null), + threadExists: (threadId) => Effect.succeed(store.exists.has(threadId)), + record: (row) => + Effect.sync(() => { + store.rows.set(row.threadId, row); + }), + }, + }); + const writeBundle = (input: { + readonly threadId: string; + readonly snapshotSequence: number; + readonly title?: string; + readonly compacted?: boolean; + readonly attachments?: ReadonlyArray<{ name: string; bytes: string }>; + readonly attachmentsSkipped?: boolean; + }) => + Effect.gen(function* () { + const snapshot = snapshotJson(input); + const sink = yield* OpenShellFleetDrainAdapters.fileDrainSink({ stateDir, fs, path }); + yield* sink.write({ + environmentId: EnvironmentId.make(ENVIRONMENT), + drainId: DRAIN_ID, + threadId: input.threadId, + body: JSON.stringify({ threadId: input.threadId, snapshot, pages: [] }), + snapshotSequence: input.snapshotSequence, + ...(input.attachments === undefined && !input.attachmentsSkipped + ? {} + : { + attachments: { + files: input.attachments ?? [], + oversizeSkipped: input.attachmentsSkipped === true, + }, + }), + }); + }); + return { + importer, + written, + store, + stateDir, + attachmentsDir, + fs, + path, + writeBundle, + failWrites: (count?: number) => { + writesUntilFailure = count; + }, + }; + }), + ({ fs, stateDir, attachmentsDir }) => + Effect.all([ + fs.remove(stateDir, { recursive: true }), + fs.remove(attachmentsDir, { recursive: true }), + ]).pipe(Effect.ignore), +); + +const eventTypes = (written: WrittenEvents["writes"]) => + written.flatMap((batch) => batch.map((event) => event.type)); + +const importedThreadIds = (written: WrittenEvents["writes"]) => + written.flatMap((batch) => + batch.filter((event) => event.type === "thread.created").map((event) => event.id), + ); + +describe("fleet bundle import", () => { + it.effect("imports a drained thread as an archived thread with a visible marker", () => + Effect.gen(function* () { + const { importer, written, store, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + expect(result.skipped).toBe(0); + expect(eventTypes(written)).toContain("thread.created"); + expect(eventTypes(written)).toContain("turn-item.updated"); + expect(eventTypes(written)).toContain("message.updated"); + expect(eventTypes(written)).toContain("provider-thread.updated"); + const head = written + .flatMap((batch) => batch) + .find((event) => event.type === "thread.created") as unknown as { + payload: { archivedAt: string | null; title: string; tags?: ReadonlyArray }; + }; + expect(head.payload.archivedAt).not.toBeNull(); + expect(head.payload.title).toContain("(drained from environment:worker)"); + expect(head.payload.tags).toContain("fleet-drain:environment:worker"); + expect(store.rows.get("thread:a")?.snapshotSequence).toBe(3); + expect(store.rows.get("thread:a")?.drainId).toBe(DRAIN_ID); + expect(importedThreadIds(written)).toHaveLength(1); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a checkout artifact is not imported as a thread", () => + Effect.gen(function* () { + const { importer, writeBundle, stateDir, fs, path } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + // A git bundle lands beside the thread copies under the worktree: key. + yield* fs.writeFileString( + path.join( + stateDir, + "fleet-drain", + encodeURIComponent(ENVIRONMENT), + encodeURIComponent(DRAIN_ID), + `thread-checkout.json`, + ), + JSON.stringify({ threadId: "worktree:/work/a", snapshotSequence: null, body: "QlVORExF" }), + ); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + expect(result.skipped).toBe(0); + expect(result.warnings).toHaveLength(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("the same threadId and snapshotSequence imports only once", () => + Effect.gen(function* () { + const { importer, written, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID }); + const firstCount = written.flat().length; + + const again = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(again.imported).toBe(0); + expect(again.skipped).toBe(1); + // No new events and no duplicate thread creation. + expect(written.flat().length).toBe(firstCount); + expect(importedThreadIds(written)).toHaveLength(1); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a changed snapshotSequence supersedes the previous import", () => + Effect.gen(function* () { + const { importer, written, store, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3, title: "Before" }); + yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID }); + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9, title: "After" }); + written.length = 0; + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + expect(result.skipped).toBe(0); + // The thread shell is refreshed, not re-created. + expect(eventTypes(written)).not.toContain("thread.created"); + expect(eventTypes(written)).toContain("thread.metadata-updated"); + expect(eventTypes(written)).toContain("turn-item.updated"); + const head = written + .flatMap((batch) => batch) + .find((event) => event.type === "thread.metadata-updated") as unknown as { + payload: { title: string }; + }; + expect(head.payload.title).toContain("After"); + expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("does not let another worker's sequence supersede the prior owner", () => + Effect.gen(function* () { + const { importer, store, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9 }); + yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID }); + const owned = store.rows.get("thread:a"); + if (owned === undefined) throw new Error("expected the first import record"); + store.rows.set("thread:a", { ...owned, environmentId: "environment:worker-a" }); + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 10 }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(1); + expect(result.warnings.join(" ")).toContain("drained from environment:worker-a"); + expect(store.rows.get("thread:a")?.environmentId).toBe("environment:worker-a"); + expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("never overwrites a local thread that was not drain-imported", () => + Effect.gen(function* () { + const { importer, written, store, writeBundle } = yield* harness; + store.exists.add("thread:local"); + yield* writeBundle({ threadId: "thread:local", snapshotSequence: 1 }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(1); + expect(result.warnings).toHaveLength(1); + expect(written).toHaveLength(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a broken bundle warns and never fails the whole import", () => + Effect.gen(function* () { + const { importer, written, stateDir, fs, path, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 1 }); + yield* fs.writeFileString( + path.join( + stateDir, + "fleet-drain", + encodeURIComponent(ENVIRONMENT), + encodeURIComponent(DRAIN_ID), + "thread-broken.json", + ), + '{"threadId":"thread:broken"}', + ); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + expect(result.warnings).toHaveLength(1); + expect(written.flat().length).toBeGreaterThan(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("a compacted snapshot is refused instead of silently losing its omitted items", () => + Effect.gen(function* () { + const { importer, written, store, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 1, compacted: true }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(1); + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain("compacted snapshot"); + expect(written).toHaveLength(0); + expect(store.rows.get("thread:a")).toBeUndefined(); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("import resumability", () => { + it.effect("resumes an import that failed after claiming the thread", () => + Effect.gen(function* () { + const { importer, store, writeBundle, failWrites, written } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + // The head lands, the first transcript batch fails. + failWrites(1); + const first = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + expect(first.imported).toBe(0); + expect(first.warnings).toHaveLength(1); + expect(store.rows.get("thread:a")?.importedAt).toBe(""); + + failWrites(undefined); + written.length = 0; + const second = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(second.imported).toBe(1); + expect(store.rows.get("thread:a")?.importedAt).not.toBe(""); + // The retry replays the thread instead of treating its own head as a collision. + expect(eventTypes(written)).toContain("turn-item.updated"); + expect(eventTypes(written)).toContain("message.updated"); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("refuses a bundle whose records belong to another thread", () => + Effect.gen(function* () { + const { importer, store, writeBundle, written, stateDir, fs, path } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + const file = path.join( + stateDir, + "fleet-drain", + encodeURIComponent(ENVIRONMENT), + encodeURIComponent(DRAIN_ID), + `thread-${encodeURIComponent("thread:a")}.json`, + ); + const envelope = JSON.parse(yield* fs.readFileString(file)) as { body: string }; + envelope.body = envelope.body.replaceAll("thread:a", "thread:victim"); + yield* fs.writeFileString(file, JSON.stringify(envelope)); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(1); + expect(result.warnings).toHaveLength(1); + expect(written).toEqual([]); + expect(store.rows.size).toBe(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); + it.effect("refuses a bundle older than the imported snapshot", () => + Effect.gen(function* () { + const { importer, store, writeBundle } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9 }); + yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID }); + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + + const older = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(older.imported).toBe(0); + expect(older.skipped).toBe(1); + expect(older.warnings.join(" ")).toContain("newer snapshot"); + expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("refuses an unreadable history page before writing anything", () => + Effect.gen(function* () { + const { importer, store, writeBundle, written, stateDir, fs, path } = yield* harness; + yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 }); + const file = path.join( + stateDir, + "fleet-drain", + encodeURIComponent(ENVIRONMENT), + encodeURIComponent(DRAIN_ID), + `thread-${encodeURIComponent("thread:a")}.json`, + ); + const envelope = JSON.parse(yield* fs.readFileString(file)) as { body: string }; + const body = JSON.parse(envelope.body) as { pages: unknown }; + body.pages = [{ items: [{ item: { threadId: "thread:victim" } }] }]; + envelope.body = JSON.stringify(body); + yield* fs.writeFileString(file, JSON.stringify(envelope)); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(0); + expect(result.warnings).toHaveLength(1); + expect(written).toEqual([]); + expect(store.rows.size).toBe(0); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("attachment restore", () => { + it.effect("restores manifest attachments flat, keyed by the attachment id", () => + Effect.gen(function* () { + const { importer, store, attachmentsDir, fs, writeBundle } = yield* harness; + const segment = toSafeThreadAttachmentSegment("thread:a"); + const name = `${segment}-1111aaaa.png`; + yield* writeBundle({ + threadId: "thread:a", + snapshotSequence: 2, + attachments: [{ name, bytes: base64("png-bytes") }], + }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + const restored = yield* fs.readFile(`${attachmentsDir}/${name}`); + expect(new TextDecoder().decode(restored)).toBe("png-bytes"); + expect(store.rows.get("thread:a")?.oversizeAttachmentsSkipped).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("refuses an attachment named for another thread", () => + Effect.gen(function* () { + const { importer, store, attachmentsDir, fs, writeBundle } = yield* harness; + yield* writeBundle({ + threadId: "thread:a", + snapshotSequence: 2, + attachments: [{ name: "other-thread-1111aaaa.png", bytes: base64("png-bytes") }], + }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + const entries = yield* fs.readDirectory(attachmentsDir); + expect(entries).toEqual([]); + expect(store.rows.get("thread:a")?.snapshotSequence).toBe(2); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("records an oversize skip in the import marker", () => + Effect.gen(function* () { + const { importer, store, writeBundle } = yield* harness; + yield* writeBundle({ + threadId: "thread:big", + snapshotSequence: 2, + attachmentsSkipped: true, + }); + + const result = yield* importer.importBundle({ + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + }); + + expect(result.imported).toBe(1); + expect(store.rows.get("thread:big")?.oversizeAttachmentsSkipped).toBe(true); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("sqlite import store", () => { + it.effect("round-trips rows and supersedes on conflict", () => + Effect.gen(function* () { + const context = yield* Layer.build(Sqlite.layerMemory); + const sql = Context.get(context, SqlClient.SqlClient); + yield* Migration0111.pipe(Effect.provideService(SqlClient.SqlClient, sql)); + const store = OpenShellFleetBundleImport.sqliteFleetDrainImportStore(sql); + + expect(yield* store.find("thread:x")).toBeNull(); + yield* store.record({ + threadId: "thread:x", + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + snapshotSequence: 4, + importedAt: "2026-11-06T10:00:00.000Z", + oversizeAttachmentsSkipped: true, + }); + const row = yield* store.find("thread:x"); + expect(row).toMatchObject({ + threadId: "thread:x", + environmentId: ENVIRONMENT, + drainId: DRAIN_ID, + snapshotSequence: 4, + oversizeAttachmentsSkipped: true, + }); + expect(yield* store.threadExists("thread:x")).toBe(false); + + yield* store.record({ + threadId: "thread:x", + environmentId: ENVIRONMENT, + drainId: "drain:environment:worker:2", + snapshotSequence: 5, + importedAt: "2026-11-06T12:00:00.000Z", + oversizeAttachmentsSkipped: false, + }); + expect(yield* store.find("thread:x")).toMatchObject({ + snapshotSequence: 5, + drainId: "drain:environment:worker:2", + oversizeAttachmentsSkipped: false, + }); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/environment/OpenShellFleetBundleImport.ts b/apps/server/src/environment/OpenShellFleetBundleImport.ts new file mode 100644 index 0000000000000..d2a5d524b7f9c --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetBundleImport.ts @@ -0,0 +1,598 @@ +import { + EventId, + type OrchestrationV2AppThread, + type OrchestrationV2ConversationMessage, + type OrchestrationV2DomainEvent, + OrchestrationV2ThreadBoundedSnapshot, + OrchestrationV2TurnItem, + ThreadId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import { normalizeAttachmentRelativePath } from "../attachmentPaths.ts"; +import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts"; +import * as EventSink from "../orchestration-v2/EventSink.ts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as SqlClient from "effect/sql/SqlClient"; + +/** + * Drains a worker bundle into an ARCHIVED read-only thread (spec 7.4): the snapshot's projection + * replays through the EventSink (lossless; no lossy AgentSessionImporter fallback). Idempotency: + * migration-091 rows mark {environmentId, drainId, snapshotSequence} — same sequence skips, + * changed supersedes via projection upserts; crashes before the marker only warn (content stays). + */ /** One attachment file recorded in a bundle envelope's manifest. */ +export interface BundleAttachmentManifestEntry { + readonly name: string; + readonly hash: string; + readonly size: number; +} + +/** Transcription of one imports-table row; `snapshotSequence` decides currency. */ +export interface FleetDrainImportRow { + readonly threadId: string; + readonly environmentId: string; + readonly drainId: string; + readonly snapshotSequence: number | null; + readonly importedAt: string; + readonly oversizeAttachmentsSkipped: boolean; +} + +export class FleetDrainImportError extends Schema.TaggedError< + FleetDrainImportError, + { readonly brand: unique symbol } +>()("FleetDrainImportError", { + operation: Schema.Literals(["read-bundle", "decode", "record", "restore-attachments"]), + threadId: Schema.optional(ThreadId), + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +/** Where import state lives; sqlite-backed in production, faked in tests. */ +export interface FleetDrainImportStore { + readonly find: ( + threadId: string, + ) => Effect.Effect; + readonly threadExists: (threadId: string) => Effect.Effect; + readonly record: (row: FleetDrainImportRow) => Effect.Effect; +} + +const EVENT_PREFIX = "fleet-drain"; +/** `importedAt` value of a row claimed before its events were written. */ +const PENDING_IMPORTED_AT = ""; +const TRANSCRIPT_EVENT_BATCH_SIZE = 100; +const TITLE_SUFFIX_OF = (environmentId: string) => ` (drained from ${environmentId})`; + +const decodeSnapshot = Schema.decodeUnknownEffect( + Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot), +); +const decodeTurnItem = Schema.decodeUnknownEffect(Schema.toCodecJson(OrchestrationV2TurnItem)); + +const decodeEnvelopeJson = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ + threadId: Schema.optional(Schema.String), + /** The copied thread as JSON: `{ threadId, snapshot, pages }`. */ + body: Schema.optional(Schema.String), + attachments: Schema.optional( + Schema.Array( + Schema.Struct({ + name: Schema.String, + hash: Schema.String, + size: Schema.Number, + }), + ), + ), + attachmentsSkipped: Schema.optional(Schema.Boolean), + }), + ), +); + +const decodeBodyJson = Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ + snapshot: Schema.optional(Schema.Unknown), + pages: Schema.optional( + Schema.Array( + Schema.Struct({ + items: Schema.optional( + Schema.Array(Schema.Struct({ item: Schema.optional(Schema.Unknown) })), + ), + }), + ), + ), + }), + ), +); + +const archivedThreadPayload = (input: { + readonly thread: OrchestrationV2AppThread; + readonly environmentId: string; + readonly archivedAt: DateTime.Utc; +}): OrchestrationV2AppThread => ({ + ...input.thread, + title: input.thread.title.endsWith(TITLE_SUFFIX_OF(input.environmentId)) + ? input.thread.title + : `${input.thread.title}${TITLE_SUFFIX_OF(input.environmentId)}`, + tags: [...new Set([...(input.thread.tags ?? []), `${EVENT_PREFIX}:${input.environmentId}`])], + archivedAt: input.archivedAt, + updatedAt: input.archivedAt, +}); + +const importEventId = (parts: { + readonly threadId: string; + readonly snapshotSequence: number | null; + readonly kind: string; + readonly ref: string; +}): EventId => + EventId.make( + `${EVENT_PREFIX}:${parts.threadId}:s${parts.snapshotSequence ?? "none"}:${parts.kind}:${parts.ref}`, + ); + +const transcriptEvents = (input: { + readonly threadId: ThreadId; + readonly occurredAt: DateTime.Utc; + readonly snapshotSequence: number | null; + readonly turnItems: ReadonlyArray; + readonly messages: ReadonlyArray; +}): Array => [ + ...input.turnItems.map((item): OrchestrationV2DomainEvent => ({ + id: importEventId({ + threadId: input.threadId, + snapshotSequence: input.snapshotSequence, + kind: "turn-item", + ref: item.id, + }), + type: "turn-item.updated", + threadId: input.threadId, + occurredAt: input.occurredAt, + payload: item, + })), + ...input.messages.map((message): OrchestrationV2DomainEvent => ({ + id: importEventId({ + threadId: input.threadId, + snapshotSequence: input.snapshotSequence, + kind: "message", + ref: message.id, + }), + type: "message.updated", + threadId: input.threadId, + occurredAt: input.occurredAt, + payload: message, + })), +]; + +const inBatches = (items: ReadonlyArray, size: number): Array> => { + const batches: Array> = []; + for (let index = 0; index < items.length; index += size) { + batches.push(items.slice(index, index + size)); + } + return batches; +}; + +const restoreAttachments = Effect.fn("FleetBundleImport.restoreAttachments")(function* (input: { + readonly manifest: ReadonlyArray; + readonly bundleDir: string; + readonly attachmentsDir: string; + readonly threadId: string; + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; +}) { + const segment = toSafeThreadAttachmentSegment(input.threadId); + if (segment === null) return; + yield* input.fs.makeDirectory(input.attachmentsDir, { recursive: true }).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "restore-attachments", + threadId: ThreadId.make(input.threadId), + message: `Attachment directory for ${input.threadId} could not be created.`, + cause, + }), + ), + ); + for (const entry of input.manifest) { + // The store keeps attachments flat as `-`; only + // such a name may land in the attachments root. + const name = normalizeAttachmentRelativePath(entry.name); + if (name === null || name.includes("/") || name !== entry.name) continue; + if (!name.startsWith(`${segment}-`)) continue; + const source = input.path.join( + input.bundleDir, + `thread-${encodeURIComponent(input.threadId)}-att-${entry.hash.slice(0, 16)}.bin`, + ); + const bytes = yield* input.fs.readFile(source).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "restore-attachments", + threadId: ThreadId.make(input.threadId), + message: `Attachment ${entry.name} could not be read back from the bundle.`, + cause, + }), + ), + ); + const target = input.path.join(input.attachmentsDir, name); + yield* input.fs.writeFile(target, bytes).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "restore-attachments", + threadId: ThreadId.make(input.threadId), + message: `Attachment ${entry.name} could not be restored.`, + cause, + }), + ), + ); + } +}); + +export interface FleetBundleImporterShape { + /** Imports every bundle in a drain directory; per-thread problems warn, never throw. */ + readonly importBundle: (input: { + readonly environmentId: string; + readonly drainId: string; + }) => Effect.Effect< + { + readonly imported: number; + readonly skipped: number; + readonly warnings: ReadonlyArray; + }, + FleetDrainImportError + >; +} + +interface BundleEnvelope { + readonly threadId?: string | undefined; + /** The copied thread as JSON: `{ threadId, snapshot, pages }`. */ + readonly body?: string | undefined; + readonly attachments?: ReadonlyArray | undefined; + readonly attachmentsSkipped?: boolean | undefined; +} + +export const makeFleetBundleImporter = (input: { + readonly stateDir: string; + readonly attachmentsDir: string; + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly now: Effect.Effect; + readonly eventSink: Pick; + readonly store: FleetDrainImportStore; +}): FleetBundleImporterShape => { + const { fs, path, stateDir } = input; + return { + importBundle: Effect.fn("FleetBundleImport.importBundle")(function* (request) { + const bundleDir = path.join( + stateDir, + "fleet-drain", + encodeURIComponent(request.environmentId), + encodeURIComponent(request.drainId), + ); + const entries = yield* fs.readDirectory(bundleDir).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "read-bundle", + message: `The drain bundle directory for ${request.drainId} could not be read.`, + cause, + }), + ), + ); + let imported = 0; + let skipped = 0; + const warnings: string[] = []; + for (const file of entries.filter( + (name) => /^thread-.*\.json$/.test(name) && !name.startsWith("thread-worktree%3A"), + )) { + const outcome = yield* Effect.gen(function* () { + const raw = yield* fs.readFileString(path.join(bundleDir, file)).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "read-bundle", + message: `Bundle ${file} could not be read.`, + cause, + }), + ), + ); + const decodedEnvelope = yield* decodeEnvelopeJson(raw).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "decode", + message: `Bundle ${file} is not a drained thread copy.`, + cause, + }), + ), + ); + const envelope: BundleEnvelope = decodedEnvelope; + const threadId = envelope.threadId; + // Checkout artifacts share the thread-*.json naming but hold no thread. + if (threadId !== undefined && threadId.startsWith("worktree:")) return "ignored" as const; + if (threadId === undefined || envelope.body === undefined) { + return yield* new FleetDrainImportError({ + operation: "decode", + message: `Bundle ${file} is not a drained thread copy.`, + }); + } + const body = yield* decodeBodyJson(envelope.body).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "decode", + threadId: ThreadId.make(threadId), + message: `Bundle ${file} does not hold a readable thread snapshot.`, + cause, + }), + ), + ); + if (body.snapshot === undefined) { + return yield* new FleetDrainImportError({ + operation: "decode", + threadId: ThreadId.make(threadId), + message: `Bundle ${file} does not hold a readable thread snapshot.`, + }); + } + const snapshot = yield* decodeSnapshot(body.snapshot).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "decode", + threadId: ThreadId.make(threadId), + message: `Bundle ${file} does not hold a readable thread snapshot.`, + cause, + }), + ), + ); + // A compacted snapshot leads with items omitted (older workers answer + // `compactTurnItems=1`); the drain exports full snapshots, so this + // marker means the bundle cannot be proven complete. Refuse loudly. + if (snapshot.turnItemsOmitLocalVisible === true) { + warnings.push( + `Bundle ${file} holds a compacted snapshot of ${threadId} (local-visible items omitted); it was not imported.`, + ); + return "skipped" as const; + } + // A bundle may name one thread and carry another's records; every + // ownership id must agree with the envelope before anything is written. + // Decode every history item before ownership checks: an unreadable + // page must fail the import rather than disappear from the transcript. + const historyItems: Array = []; + for (const entry of (body.pages ?? []).flatMap((page) => page.items ?? [])) { + if (entry.item === undefined) continue; + historyItems.push( + yield* decodeTurnItem(entry.item).pipe( + Effect.mapError( + (cause) => + new FleetDrainImportError({ + operation: "decode", + threadId: ThreadId.make(threadId), + message: `Bundle ${file} holds an unreadable history item.`, + cause, + }), + ), + ), + ); + } + const foreignIds = [ + snapshot.projection.thread.id, + ...snapshot.projection.turnItems.map((item) => item.threadId), + ...snapshot.projection.messages.map((message) => message.threadId), + ...snapshot.projection.providerThreads.map((entry) => entry.appThreadId), + ...historyItems.map((item) => item.threadId), + ].filter((id) => id !== threadId); + if (foreignIds.length > 0) { + warnings.push( + `Bundle ${file} names ${threadId} but carries records of ${foreignIds[0]}; it was not imported.`, + ); + return "skipped" as const; + } + const snapshotSequence = snapshot.snapshotSequence; + const existing = yield* input.store.find(threadId); + // Sequence numbers only order imports from the same worker. Worker A's + // older snapshot must not authorize replacing A's record with B's. + if (existing !== null && existing.environmentId !== request.environmentId) { + warnings.push( + `Thread ${threadId} was drained from ${existing.environmentId}; worker ${request.environmentId}'s bundle was not imported.`, + ); + return "skipped" as const; + } + if ( + existing !== null && + existing.snapshotSequence !== null && + existing.importedAt !== PENDING_IMPORTED_AT + ) { + if (existing.snapshotSequence === snapshotSequence) return "skipped" as const; + // A bundle older than what was imported is stale, not a supersede. + if (existing.snapshotSequence > snapshotSequence) { + warnings.push( + `Thread ${threadId} already holds a newer snapshot (${existing.snapshotSequence}); the older bundle was not imported.`, + ); + return "skipped" as const; + } + } + const exists = yield* input.store.threadExists(threadId); + if (exists && existing === null) { + // A local thread that was not drain-imported owns this id; archiving + // or overwriting it would destroy local work, so the bundle stays. + warnings.push( + `Thread ${threadId} already exists here and was not drain-imported; its bundle was not imported.`, + ); + return "skipped" as const; + } + const now = DateTime.makeUnsafe(yield* input.now); + // A brand-new thread is claimed before events so a crash cannot make a + // partial import look local. A completed row keeps its watermark until + // the newer import succeeds, so a failed supersede never loses it. + if (existing === null || existing.importedAt === PENDING_IMPORTED_AT) { + yield* input.store.record({ + threadId, + environmentId: request.environmentId, + drainId: request.drainId, + snapshotSequence, + importedAt: PENDING_IMPORTED_AT, + oversizeAttachmentsSkipped: envelope.attachmentsSkipped === true, + }); + } + const payload = archivedThreadPayload({ + thread: snapshot.projection.thread, + environmentId: request.environmentId, + archivedAt: now, + }); + const turnItems = [...snapshot.projection.turnItems, ...historyItems]; + const thread = ThreadId.make(threadId); + const head: OrchestrationV2DomainEvent = { + id: importEventId({ + threadId, + snapshotSequence, + kind: "thread", + ref: String(snapshotSequence ?? "none"), + }), + // The projection decides, not the marker: a resume before the head + // wrote still has to create the thread. + type: exists ? "thread.metadata-updated" : "thread.created", + threadId: thread, + providerInstanceId: payload.providerInstanceId, + occurredAt: now, + payload, + }; + const providerThreadEvents: Array = + snapshot.projection.providerThreads.map( + (providerThread): OrchestrationV2DomainEvent => ({ + id: importEventId({ + threadId, + snapshotSequence, + kind: "provider-thread", + ref: providerThread.id, + }), + type: "provider-thread.updated", + threadId: thread, + providerInstanceId: providerThread.providerInstanceId, + occurredAt: now, + payload: providerThread, + }), + ); + // Restore attachment files before recording, so a crash between the + // two leaves files but no marker and the retry replays idempotently. + yield* restoreAttachments({ + manifest: envelope.attachments ?? [], + bundleDir, + attachmentsDir: input.attachmentsDir, + threadId, + fs, + path, + }); + yield* input.eventSink.write({ + events: [head, ...providerThreadEvents], + }); + const transcript = transcriptEvents({ + threadId: thread, + occurredAt: now, + snapshotSequence, + turnItems, + messages: snapshot.projection.messages, + }); + for (const batch of inBatches(transcript, TRANSCRIPT_EVENT_BATCH_SIZE)) { + yield* input.eventSink.write({ events: [...batch] }); + } + yield* input.store.record({ + threadId, + environmentId: request.environmentId, + drainId: request.drainId, + snapshotSequence, + importedAt: DateTime.formatIso(now), + oversizeAttachmentsSkipped: envelope.attachmentsSkipped === true, + }); + return "imported" as const; + }).pipe( + Effect.tapError((cause) => + Effect.logWarning("Fleet drain bundle import failed", { file, cause }), + ), + Effect.catch((_cause) => + Effect.sync(() => { + warnings.push( + `Bundle ${file} could not be imported; the server log has the bounded failure details.`, + ); + return "failed" as const; + }), + ), + ); + if (outcome === "imported") imported += 1; + else if (outcome === "skipped") skipped += 1; + } + return { imported, skipped, warnings }; + }), + }; +}; + +/** + * The sqlite-backed import-state store over migration 091's table; tests run + * it on the in-memory sqlite layer like the other persistence stores. + */ +export const sqliteFleetDrainImportStore = (sql: SqlClient.SqlClient): FleetDrainImportStore => { + const failed = (message: string, cause: unknown) => + new FleetDrainImportError({ operation: "record", message, cause }); + return { + find: (threadId) => + sql<{ + readonly thread_id: string; + readonly environment_id: string; + readonly drain_id: string; + readonly snapshot_sequence: number | null; + readonly imported_at: string; + readonly oversize_attachments_skipped: number; + }>` + SELECT thread_id, environment_id, drain_id, snapshot_sequence, imported_at, oversize_attachments_skipped + FROM orchestration_v2_fleet_drain_imports + WHERE thread_id = ${threadId} + `.pipe( + NodeSqliteClient.readOnly, + Effect.map((rows) => { + const row = rows[0]; + return row === undefined + ? null + : { + threadId: row.thread_id, + environmentId: row.environment_id, + drainId: row.drain_id, + snapshotSequence: row.snapshot_sequence, + importedAt: row.imported_at, + oversizeAttachmentsSkipped: row.oversize_attachments_skipped === 1, + }; + }), + Effect.mapError((cause) => + failed("The fleet drain import state could not be read.", cause), + ), + ), + threadExists: (threadId) => + sql<{ readonly present: number }>` + SELECT 1 AS present + FROM orchestration_v2_projection_threads + WHERE thread_id = ${threadId} + `.pipe( + NodeSqliteClient.readOnly, + Effect.map((rows) => rows.length > 0), + Effect.mapError((cause) => failed("The thread projection could not be consulted.", cause)), + ), + record: (row) => + sql` + INSERT INTO orchestration_v2_fleet_drain_imports + (thread_id, environment_id, drain_id, snapshot_sequence, imported_at, lossy, oversize_attachments_skipped) + VALUES (${row.threadId}, ${row.environmentId}, ${row.drainId}, ${row.snapshotSequence}, ${row.importedAt}, 0, ${row.oversizeAttachmentsSkipped ? 1 : 0}) + ON CONFLICT(thread_id) DO UPDATE SET + environment_id = excluded.environment_id, + drain_id = excluded.drain_id, + snapshot_sequence = excluded.snapshot_sequence, + imported_at = excluded.imported_at, + oversize_attachments_skipped = excluded.oversize_attachments_skipped + `.pipe( + Effect.asVoid, + Effect.mapError((cause) => + failed(`The fleet drain import state for ${row.threadId} could not be recorded.`, cause), + ), + ), + }; +}; diff --git a/apps/server/src/environment/OpenShellFleetDrain.test.ts b/apps/server/src/environment/OpenShellFleetDrain.test.ts new file mode 100644 index 0000000000000..1c56b3d755125 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetDrain.test.ts @@ -0,0 +1,632 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import { EnvironmentId } from "@t3tools/contracts"; + +import { + type DrainDependencies, + FleetDrainStepError, + makeDrainExecutor, +} from "./OpenShellFleetDrain.ts"; + +const environmentId = EnvironmentId.make("environment:worker"); + +const deps = (input: { + readonly threads: ReadonlyArray; + readonly projects?: ReadonlyArray; + readonly now?: number; + readonly failThread?: string; + readonly failShell?: boolean; + readonly bundleExitCode?: number; + readonly bundleStdout?: string; + readonly failCopy?: boolean; + /** Marks this thread live on the drain's currency re-read. */ + readonly liveAfter?: string; + /** Appears in the currency re-read only. */ + readonly threadAfter?: string; + /** Finishes a turn after its copy (its sequence advances on later reads). */ + readonly sequenceAfterCopy?: string; + readonly liveThread?: string; + readonly sequences?: Readonly>; + readonly attachmentFiles?: Readonly>>; + readonly oversizeThreads?: ReadonlyArray; + readonly dropAttachmentManifest?: boolean; + readonly existingFiles?: ReadonlyMap; + readonly seeded?: ReadonlyArray<{ + readonly drainId: string; + readonly threadId: string; + readonly snapshotSequence: number | null; + readonly attachments?: boolean; + }>; +}): DrainDependencies & { + readonly written: Array<{ drainId: string; threadId: string; body: string }>; + readonly fetched: Array; + readonly fetchedAttachments: Array; + readonly exported: Array; + readonly promoted: Array<{ fromDrainId: string; toDrainId: string; threadId: string }>; + readonly files: ReadonlyMap; +} => { + const manifest = [{ name: "a.bin", hash: "h0", size: 3 }]; + const envelope = (entry: { + threadId: string; + snapshotSequence: number | null; + attachments?: boolean; + }) => + JSON.stringify({ + threadId: entry.threadId, + snapshotSequence: entry.snapshotSequence, + body: `{"threadId":"${entry.threadId}","seeded":true}`, + ...(entry.attachments === true ? { attachments: manifest } : {}), + }); + const files = new Map( + input.existingFiles + ? [...input.existingFiles] + : (input.seeded ?? []).map((entry) => [ + `${entry.drainId}/${entry.threadId}`, + envelope(entry), + ]), + ); + const written: Array<{ drainId: string; threadId: string; body: string }> = []; + const fetched: Array = []; + const fetchedAttachments: Array = []; + const exported: Array = []; + const promoted: Array<{ fromDrainId: string; toDrainId: string; threadId: string }> = []; + return { + written, + fetched, + fetchedAttachments, + exported, + promoted, + files, + now: Effect.succeed(input.now ?? 0), + source: { + shell: (() => { + let calls = 0; + return () => { + calls += 1; + return input.failShell === true + ? Effect.fail( + new FleetDrainStepError({ step: "shell", message: "the worker link is unusable" }), + ) + : Effect.succeed({ + threads: [ + ...input.threads.map((threadId) => ({ + threadId, + live: + input.liveThread === threadId || (calls > 1 && input.liveAfter === threadId), + })), + ...(calls > 1 && input.threadAfter !== undefined + ? [{ threadId: input.threadAfter, live: false }] + : []), + ], + projects: (input.projects ?? []).map((workspaceRoot) => ({ workspaceRoot })), + }); + }; + })(), + threadSequence: (() => { + const reads = new Map(); + return ({ threadId }: { readonly threadId: string }) => { + const count = (reads.get(threadId) ?? 0) + 1; + reads.set(threadId, count); + const base = input.sequences?.[threadId] ?? 1; + return Effect.succeed({ + snapshotSequence: input.sequenceAfterCopy === threadId && count > 1 ? base + 1 : base, + }); + }; + })(), + threadHistory: ({ threadId }) => { + fetched.push(threadId); + return threadId === input.failThread + ? Effect.fail( + new FleetDrainStepError({ step: "history", message: "the worker went away" }), + ) + : Effect.succeed({ + body: `{"threadId":"${threadId}","events":[1,2,3]}`, + snapshotSequence: input.sequences?.[threadId] ?? 1, + }); + }, + attachments: ({ threadId }) => { + fetchedAttachments.push(threadId); + if (input.oversizeThreads?.includes(threadId) === true) { + return Effect.succeed({ files: [], oversizeSkipped: true }); + } + return Effect.succeed({ + files: (input.attachmentFiles?.[threadId] ?? []).map((name) => ({ + name, + bytes: Buffer.from(name, "utf8").toString("base64"), + })), + oversizeSkipped: false, + }); + }, + }, + sink: { + write: ({ drainId, threadId, body, snapshotSequence, attachments }) => + Effect.sync(() => { + const stored = JSON.stringify({ + threadId, + snapshotSequence, + body, + ...(input.dropAttachmentManifest === true + ? {} + : (attachments?.files ?? []).length > 0 + ? { + attachments: (attachments?.files ?? []).map((file) => ({ + name: file.name, + hash: `h-${file.name}`, + size: file.bytes.length, + })), + } + : {}), + ...(attachments?.oversizeSkipped === true ? { attachmentsSkipped: true } : {}), + }); + files.set(`${drainId}/${threadId}`, stored); + written.push({ drainId, threadId, body }); + return { bytes: stored.length }; + }), + read: ({ drainId, threadId }) => + Effect.map( + Effect.sync(() => files.get(`${drainId}/${threadId}`)), + (raw) => { + if (raw === undefined) return null; + const parsed = JSON.parse(raw) as { + threadId: string; + snapshotSequence?: number; + body: string; + attachments?: Array<{ name: string; hash: string; size: number }>; + attachmentsSkipped?: boolean; + }; + return { + threadId: parsed.threadId, + snapshotSequence: + typeof parsed.snapshotSequence === "number" ? parsed.snapshotSequence : null, + body: parsed.body, + ...(parsed.attachments !== undefined && parsed.attachments.length > 0 + ? { attachments: parsed.attachments } + : {}), + ...(parsed.attachmentsSkipped === true ? { attachmentsSkipped: true } : {}), + }; + }, + ), + copy: ({ fromDrainId, toDrainId, threadId }) => + Effect.sync(() => { + if (input.failCopy === true) return false; + const raw = files.get(`${fromDrainId}/${threadId}`); + if (raw === undefined) return false; + files.set(`${toDrainId}/${threadId}`, raw); + promoted.push({ fromDrainId, toDrainId, threadId }); + return true; + }), + }, + git: { + bundle: ({ workspaceRoot }) => + Effect.sync(() => { + exported.push(workspaceRoot); + return { + exitCode: input.bundleExitCode ?? 0, + stdout: input.bundleStdout ?? `QlVORExFOiR7d29ya3NwYWNlUm9vdH0=`, + stderr: "", + }; + }), + }, + }; +}; + +const drain = ( + dependencies: DrainDependencies, + overrides: Partial<{ drainId: string; deadlineAt: number; snapshotOnly: boolean }> = {}, +) => + makeDrainExecutor(dependencies).drain({ + environmentId, + sandboxName: "sandbox-1", + drainId: overrides.drainId ?? "drain:environment:worker:1", + snapshotOnly: overrides.snapshotOnly ?? false, + deadlineAt: overrides.deadlineAt ?? Number.MAX_SAFE_INTEGER, + }); + +describe("fleet drain", () => { + it.effect("acks threads and checkout bundles only when each one landed", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a", "thread:b"], projects: ["/work/a"] }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("acked"); + expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 }); + expect(outcome.worktrees).toEqual({ total: 1, acked: 1, failed: 0 }); + expect(dependencies.exported).toEqual(["/work/a"]); + }), + ); + + it.effect("does not re-fetch a thread whose previous copy is still current", () => + Effect.gen(function* () { + const first = deps({ + threads: ["thread:a", "thread:b"], + failThread: "thread:b", + attachmentFiles: { "thread:a": ["first.bin"] }, + }); + yield* drain(first, { drainId: "drain:environment:worker:1" }); + const retry = deps({ + threads: ["thread:a", "thread:b"], + existingFiles: first.files, + seeded: first.written.map((entry) => ({ + drainId: "drain:environment:worker:1", + threadId: entry.threadId, + snapshotSequence: 1, + attachments: true, + })), + }); + const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("acked"); + // thread:a is still current, so only the failed thread is fetched again. + expect(retry.fetched).toEqual(["thread:b"]); + expect(retry.written.map((entry) => entry.threadId)).toEqual(["thread:b"]); + // The current copy is promoted into the generation the importer reads. + expect(retry.promoted).toEqual([ + { + fromDrainId: "drain:environment:worker:1", + toDrainId: "drain:environment:worker:2", + threadId: "thread:a", + }, + ]); + }), + ); + + it.effect("refuses an ack when the worker starts work during the drain", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a"], liveAfter: "thread:a" }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.warnings?.join(" ")).toContain("changed during the drain"); + }), + ); + + it.effect("refuses an ack when a thread appears during the drain", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a"], threadAfter: "thread:new" }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.threads.acked).toBe(1); + }), + ); + + it.effect("refuses an ack when a copied thread finishes a turn during the drain", () => + Effect.gen(function* () { + const retry = deps({ + threads: ["thread:a"], + sequenceAfterCopy: "thread:a", + seeded: [ + { + drainId: "drain:environment:worker:1", + threadId: "thread:a", + snapshotSequence: 1, + attachments: true, + }, + ], + }); + const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("failed"); + expect(outcome.warnings?.join(" ")).toContain("changed during the drain"); + }), + ); + + it.effect("re-copies a current thread when the previous copy cannot be promoted", () => + Effect.gen(function* () { + const retry = deps({ + threads: ["thread:a"], + failCopy: true, + seeded: [ + { + drainId: "drain:environment:worker:1", + threadId: "thread:a", + snapshotSequence: 1, + attachments: true, + }, + ], + }); + const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("acked"); + expect(retry.promoted).toEqual([]); + expect(retry.fetched).toEqual(["thread:a"]); + }), + ); + + it.effect("re-copies when a previous generation changes during promotion", () => + Effect.gen(function* () { + const retry = deps({ + threads: ["thread:a"], + seeded: [ + { + drainId: "drain:environment:worker:1", + threadId: "thread:a", + snapshotSequence: 1, + attachments: true, + }, + ], + }); + const outcome = yield* drain( + { + ...retry, + sink: { + ...retry.sink, + copy: (input) => + Effect.gen(function* () { + yield* retry.sink.write({ + environmentId: input.environmentId, + drainId: input.toDrainId, + threadId: input.threadId, + body: '{"older":true}', + snapshotSequence: 0, + }); + return true; + }), + }, + }, + { drainId: "drain:environment:worker:2" }, + ); + expect(outcome.state).toBe("acked"); + expect(retry.fetched).toEqual(["thread:a"]); + expect( + (yield* retry.sink.read({ + environmentId, + drainId: "drain:environment:worker:2", + threadId: "thread:a", + }))?.snapshotSequence, + ).toBe(1); + }), + ); + + it.effect("re-copies a thread whose worker state changed since the failed drain", () => + Effect.gen(function* () { + const first = deps({ threads: ["thread:a", "thread:b"], failThread: "thread:b" }); + yield* drain(first, { drainId: "drain:environment:worker:1" }); + const retry = deps({ + threads: ["thread:a", "thread:b"], + sequences: { "thread:a": 2 }, + seeded: first.written.map((entry) => ({ + drainId: "drain:environment:worker:1", + threadId: entry.threadId, + snapshotSequence: 1, + attachments: true, + })), + }); + const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("acked"); + expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 }); + // The changed thread is fetched and written again under the new generation. + expect(retry.fetched).toEqual(["thread:a", "thread:b"]); + expect(retry.written.map((entry) => entry.threadId)).toEqual(["thread:a", "thread:b"]); + }), + ); + + it.effect("never trusts copies left under its own drain id or with a stale marker", () => + Effect.gen(function* () { + // A timed-out run leaves a stale copy under the previous generation and a + // fresh-looking one under the id this retry runs under; neither may count. + const dependencies = deps({ + threads: ["thread:a"], + sequences: { "thread:a": 7 }, + seeded: [ + { + drainId: "drain:environment:worker:1", + threadId: "thread:a", + snapshotSequence: 3, + attachments: true, + }, + { + drainId: "drain:environment:worker:2", + threadId: "thread:a", + snapshotSequence: 7, + attachments: true, + }, + ], + }); + const outcome = yield* drain(dependencies, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("acked"); + expect(dependencies.fetched).toEqual(["thread:a"]); + expect(dependencies.written.map((entry) => entry.drainId)).toEqual([ + "drain:environment:worker:2", + ]); + }), + ); + + it.effect("a timed-out generation still writing cannot corrupt the retry", () => + Effect.gen(function* () { + const base = deps({ + threads: ["thread:a"], + sequences: { "thread:a": 7 }, + seeded: [ + { + drainId: "drain:environment:worker:1", + threadId: "thread:a", + snapshotSequence: 3, + attachments: true, + }, + ], + }); + const sinkWrite = base.sink.write; + // While generation 2 runs, the abandoned generation 1 finishes its own + // (newer) copy into generation 1's directory. + const lateWriter = { + ...base.sink, + write: (input: Parameters[0]) => + sinkWrite(input).pipe( + Effect.tap(() => + sinkWrite({ ...input, drainId: "drain:environment:worker:1", body: '{"late":true}' }), + ), + ), + }; + const outcome = yield* drain( + { ...base, sink: lateWriter }, + { + drainId: "drain:environment:worker:2", + }, + ); + + // The stale generation 1 copy was re-fetched, written under generation 2's + // own id, and verified against that copy — the late generation 1 write, + // which lands in a directory the retry never reads, cannot satisfy it. + expect(outcome.state).toBe("acked"); + expect(base.written.map((entry) => [entry.drainId, entry.threadId])).toEqual([ + ["drain:environment:worker:2", "thread:a"], + ["drain:environment:worker:1", "thread:a"], + ]); + }), + ); + + it.effect("keeps the sandbox when a thread cannot be copied", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a", "thread:b"], failThread: "thread:b" }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.threads).toEqual({ total: 2, acked: 1, failed: 1 }); + }), + ); + + it.effect("refuses to drain a worker whose child is still working", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a"], liveThread: "thread:a" }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.finishedAt).toBeNull(); + expect(outcome.threads.failed).toBe(1); + expect(dependencies.written).toHaveLength(0); + }), + ); + + it.effect("returns failed with no acks when the worker's shell cannot be read", () => + Effect.gen(function* () { + const dependencies = deps({ + threads: ["thread:a"], + projects: ["/work/a"], + failShell: true, + }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.finishedAt).toBeNull(); + expect(outcome.threads).toEqual({ total: 0, acked: 0, failed: 0 }); + expect(outcome.worktrees).toEqual({ total: 0, acked: 0, failed: 0 }); + expect(dependencies.written).toHaveLength(0); + expect(dependencies.exported).toHaveLength(0); + }), + ); + + it.effect("acks a checkout with nothing the mirror lacks without storing a bundle", () => + Effect.gen(function* () { + const dependencies = deps({ + threads: [], + projects: ["/work/a"], + bundleStdout: "", + }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("acked"); + expect(outcome.worktrees).toEqual({ total: 1, acked: 1, failed: 0 }); + expect(dependencies.written).toHaveLength(0); + }), + ); + + it.effect("keeps the sandbox when a checkout bundle was not created", () => + Effect.gen(function* () { + const dependencies = deps({ + threads: ["thread:a"], + projects: ["/work/a"], + bundleExitCode: 1, + }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.worktrees).toEqual({ total: 1, acked: 0, failed: 1 }); + }), + ); + + it.effect("stops at the deadline instead of claiming a partial drain", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a", "thread:b"], now: 5_000 }); + const outcome = yield* drain(dependencies, { deadlineAt: 1_000 }); + + expect(outcome.state).toBe("failed"); + expect(outcome.threads).toEqual({ total: 0, acked: 0, failed: 0 }); + expect(dependencies.written).toHaveLength(0); + }), + ); + + it.effect("copies threads but skips checkout bundles for a forced removal", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a"], projects: ["/work/a"] }); + const outcome = yield* drain(dependencies, { snapshotOnly: true }); + + expect(outcome.state).toBe("acked"); + expect(outcome.threads.acked).toBe(1); + expect(dependencies.exported).toHaveLength(0); + }), + ); + + it.effect("drains attachments with the thread and verifies the manifest", () => + Effect.gen(function* () { + const dependencies = deps({ + threads: ["thread:a"], + attachmentFiles: { "thread:a": ["shot.png", "log.txt"] }, + }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("acked"); + expect(dependencies.fetchedAttachments).toEqual(["thread:a"]); + expect(dependencies.written).toHaveLength(1); + }), + ); + + it.effect("notes an oversize skip in the outcome without failing the drain", () => + Effect.gen(function* () { + const dependencies = deps({ threads: ["thread:a"], oversizeThreads: ["thread:a"] }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("acked"); + expect(outcome.warnings).toEqual([ + "Attachments of thread:a exceeded the drain size cap and were skipped.", + ]); + }), + ); + + it.effect("re-copies a thread whose previous copy predates attachment draining", () => + Effect.gen(function* () { + const first = deps({ threads: ["thread:a"] }); + yield* drain(first, { drainId: "drain:environment:worker:1" }); + const retry = deps({ + threads: ["thread:a"], + attachmentFiles: { "thread:a": ["late.bin"] }, + // The old copy carries no attachment manifest. + seeded: first.written.map((entry) => ({ + drainId: "drain:environment:worker:1", + threadId: entry.threadId, + snapshotSequence: 1, + })), + }); + const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" }); + + expect(outcome.state).toBe("acked"); + expect(retry.fetched).toEqual(["thread:a"]); + }), + ); + + it.effect("fails the copy when the stored manifest does not match what was sent", () => + Effect.gen(function* () { + const dependencies = deps({ + threads: ["thread:a"], + attachmentFiles: { "thread:a": ["shot.png"] }, + dropAttachmentManifest: true, + }); + const outcome = yield* drain(dependencies); + + expect(outcome.state).toBe("failed"); + expect(outcome.threads).toEqual({ total: 1, acked: 0, failed: 1 }); + }), + ); +}); diff --git a/apps/server/src/environment/OpenShellFleetDrain.ts b/apps/server/src/environment/OpenShellFleetDrain.ts new file mode 100644 index 0000000000000..5c23c1075a0a8 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetDrain.ts @@ -0,0 +1,448 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import { + type FleetDrainExecutor as FleetDrainExecutorPort, + type FleetDrainOutcome, +} from "@t3tools/openshell/OpenShellGatewayDriver"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +/** + * Nothing is deleted until its data is here (spec section 7): per-thread copies + * read back for verification, checkout bundle exports, per-item acks keyed by + * drain id. + */ +export class FleetDrainStepError extends Schema.TaggedError< + FleetDrainStepError, + { readonly brand: unique symbol } +>()("FleetDrainStepError", { + step: Schema.Literals(["shell", "thread", "history", "write", "read", "bundle", "attachments"]), + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +export class FleetDrainError extends Schema.TaggedError< + FleetDrainError, + { readonly brand: unique symbol } +>()("FleetDrainError", { + environmentId: Schema.String, + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +/** One stored thread copy plus the currency marker that decides trust on a retry. */ +export interface DrainSinkRecord { + readonly threadId: string; + readonly snapshotSequence: number | null; + readonly body: string; + /** Manifest of the attachment files stored next to the bundle, if any. */ + readonly attachments?: ReadonlyArray; + /** True when the worker's attachments exceeded the drain's size cap. */ + readonly attachmentsSkipped?: boolean; +} + +/** One attachment file in a bundle, base64-encoded. */ +export interface DrainAttachmentFile { + readonly name: string; + readonly bytes: string; +} + +export interface DrainAttachmentManifestEntry { + readonly name: string; + readonly hash: string; + readonly size: number; +} + +/** A thread's attachment files as the worker holds them, plus cap outcome. */ +export interface DrainAttachments { + readonly files: ReadonlyArray; + readonly oversizeSkipped: boolean; +} + +/** Where a drain writes: one directory per run, one file per thread. */ +export interface DrainSink { + readonly write: (input: { + readonly environmentId: EnvironmentId; + readonly drainId: string; + readonly threadId: string; + readonly body: string; + readonly snapshotSequence: number | null; + readonly attachments?: DrainAttachments; + }) => Effect.Effect<{ readonly bytes: number }, FleetDrainStepError>; + readonly read: (input: { + readonly environmentId: EnvironmentId; + readonly drainId: string; + readonly threadId: string; + }) => Effect.Effect; + /** Copies a previous generation's record into this one; false when it is gone. */ + readonly copy: (input: { + readonly environmentId: EnvironmentId; + readonly fromDrainId: string; + readonly toDrainId: string; + readonly threadId: string; + }) => Effect.Effect; +} + +/** Exports one checkout and its checkpoint refs as base64 bundle bytes on exec + * stdout; empty stdout means origin already holds every selected commit. */ +export interface DrainGitBundle { + readonly bundle: (input: { + readonly environmentId: EnvironmentId; + readonly sandboxName: string; + readonly workspaceRoot: string; + }) => Effect.Effect< + { readonly exitCode: number; readonly stdout: string; readonly stderr: string }, + FleetDrainStepError + >; +} + +/** The worker's own view of what it holds, read over its authenticated link. */ +export interface DrainSource { + readonly shell: (environmentId: EnvironmentId) => Effect.Effect< + { + readonly threads: ReadonlyArray<{ readonly threadId: string; readonly live: boolean }>; + readonly projects: ReadonlyArray<{ readonly workspaceRoot: string }>; + }, + FleetDrainStepError + >; + /** Cheap per-thread currency probe: the bounded read's snapshot sequence. */ + readonly threadSequence: (input: { + readonly environmentId: EnvironmentId; + readonly threadId: string; + }) => Effect.Effect<{ readonly snapshotSequence: number | null }, FleetDrainStepError>; + readonly threadHistory: (input: { + readonly environmentId: EnvironmentId; + readonly threadId: string; + }) => Effect.Effect< + { readonly body: string; readonly snapshotSequence: number | null }, + FleetDrainStepError + >; + /** The thread's attachment files; absent, the source cannot serve them. */ + readonly attachments?: (input: { + readonly environmentId: EnvironmentId; + readonly sandboxName: string; + readonly threadId: string; + }) => Effect.Effect; +} + +export interface DrainDependencies { + readonly source: DrainSource; + readonly sink: DrainSink; + readonly git: DrainGitBundle; + readonly now: Effect.Effect; +} + +export interface DrainInput { + readonly environmentId: EnvironmentId; + readonly sandboxName: string; + readonly drainId: string; + readonly snapshotOnly: boolean; + readonly deadlineAt: number; +} + +export interface DrainExecutorShape { + readonly drain: (input: DrainInput) => Effect.Effect; +} + +const zero = () => ({ total: 0, acked: 0, failed: 0 }); + +const reported = (outcome: FleetDrainOutcome): FleetDrainOutcome => outcome; + +/** The immediately preceding generation's id, or null for the first drain. */ +const previousDrainId = (drainId: string): string | null => { + const marker = drainId.lastIndexOf(":"); + if (marker < 0) return null; + const generation = Number.parseInt(drainId.slice(marker + 1), 10); + if (!Number.isFinite(generation) || generation <= 1) return null; + return `${drainId.slice(0, marker)}:${generation - 1}`; +}; + +/** + * Drains one worker. Every step is keyed by `drainId` and per item, so a retry + * after a crash re-reads what already landed instead of fetching it again. + */ +export const makeDrainExecutor = (deps: DrainDependencies): DrainExecutorShape => { + const drain = Effect.fn("FleetDrainExecutor.drain")(function* (input: DrainInput) { + const threads = zero(); + const worktrees = zero(); + const warnings: string[] = []; + /** Sequence each copied thread was read at, re-checked before the ack. */ + const copiedSequences = new Map(); + const copiedWorkspaceRoots = new Set(); + let deadlineHit = false; + let shellFailed = false; + const shell = yield* deps.source.shell(input.environmentId).pipe( + Effect.catch(() => + Effect.sync(() => { + shellFailed = true; + return { threads: [], projects: [] } as { + threads: ReadonlyArray<{ threadId: string; live: boolean }>; + projects: ReadonlyArray<{ workspaceRoot: string }>; + }; + }), + ), + ); + if (shellFailed) { + // Nothing could be listed, so nothing can be acked: the sandbox is kept. + return reported({ + threads: zero(), + worktrees: zero(), + state: "failed", + finishedAt: null, + }); + } + // A child that is still working writes after this snapshot, so copying now + // would ack a stale thread and then delete the sandbox under it. + const stillWorking = shell.threads.filter((thread) => thread.live); + if (stillWorking.length > 0) { + return reported({ + threads: { total: stillWorking.length, acked: 0, failed: stillWorking.length }, + worktrees: zero(), + state: "failed", + finishedAt: null, + }); + } + for (const thread of shell.threads) { + if ((yield* deps.now) >= input.deadlineAt) { + deadlineHit = true; + break; + } + threads.total += 1; + const copied = yield* Effect.gen(function* () { + // A copy from the previous generation is trusted only while its currency + // marker still matches the live worker; otherwise the thread is re-copied. + const previous = previousDrainId(input.drainId); + const prior = + previous === null + ? null + : yield* deps.sink.read({ + environmentId: input.environmentId, + drainId: previous, + threadId: thread.threadId, + }); + if (prior !== null && previous !== null && prior.snapshotSequence !== null) { + const current = yield* deps.source.threadSequence({ + environmentId: input.environmentId, + threadId: thread.threadId, + }); + // A copy from before attachments were drained carries no manifest, so + // it is re-copied once to pick the attachment files up. + if ( + current.snapshotSequence === prior.snapshotSequence && + prior.attachments !== undefined + ) { + // The importer reads this generation's directory only, so the + // verified copy is promoted instead of merely acknowledged. + const promoted = yield* deps.sink + .copy({ + environmentId: input.environmentId, + fromDrainId: previous, + toDrainId: input.drainId, + threadId: thread.threadId, + }) + .pipe(Effect.orElseSucceed(() => false)); + if (promoted) { + const back = yield* deps.sink.read({ + environmentId: input.environmentId, + drainId: input.drainId, + threadId: thread.threadId, + }); + if ( + back !== null && + back.body === prior.body && + back.snapshotSequence === prior.snapshotSequence && + back.attachments?.length === prior.attachments.length && + back.attachments.every((entry, index) => { + const expected = prior.attachments?.[index]; + return ( + entry.name === expected?.name && + entry.hash === expected.hash && + entry.size === expected.size + ); + }) && + back.attachmentsSkipped === prior.attachmentsSkipped + ) { + copiedSequences.set(thread.threadId, prior.snapshotSequence); + return true; + } + } + } + } + const history = yield* deps.source.threadHistory({ + environmentId: input.environmentId, + threadId: thread.threadId, + }); + const attachments = + deps.source.attachments === undefined + ? ({ files: [], oversizeSkipped: false } as const) + : yield* deps.source.attachments({ + environmentId: input.environmentId, + sandboxName: input.sandboxName, + threadId: thread.threadId, + }); + if (attachments.oversizeSkipped) { + warnings.push( + `Attachments of ${thread.threadId} exceeded the drain size cap and were skipped.`, + ); + } + const written = yield* deps.sink.write({ + environmentId: input.environmentId, + drainId: input.drainId, + threadId: thread.threadId, + body: history.body, + snapshotSequence: history.snapshotSequence, + ...(attachments.files.length === 0 && !attachments.oversizeSkipped + ? {} + : { attachments }), + }); + // Read it back: an ack means the bytes are here, not that a write was attempted. + copiedSequences.set(thread.threadId, history.snapshotSequence); + const back = yield* deps.sink.read({ + environmentId: input.environmentId, + drainId: input.drainId, + threadId: thread.threadId, + }); + if ( + back === null || + back.body !== history.body || + written.bytes <= 0 || + (back.attachmentsSkipped === true) !== attachments.oversizeSkipped + ) + return false; + const sent = attachments.files.map((file) => file.name).sort(); + const stored = (back.attachments ?? []).map((entry) => entry.name).sort(); + return sent.length === stored.length && sent.every((name, index) => name === stored[index]); + }).pipe(Effect.orElseSucceed(() => false)); + if (copied) threads.acked += 1; + else threads.failed += 1; + } + // A forced removal still ships the threads, which are the part that cannot be rebuilt. + if (!input.snapshotOnly) { + for (const project of shell.projects) { + if ((yield* deps.now) >= input.deadlineAt) { + deadlineHit = true; + break; + } + worktrees.total += 1; + // One bundle per checkout, keyed by its root, lands beside the thread + // copies and is verified the same way: an ack means the bytes are here. + // An export of nothing (origin has every commit) has no bundle to store. + const worktreeId = `worktree:${project.workspaceRoot}`; + copiedWorkspaceRoots.add(project.workspaceRoot); + const copied = yield* Effect.gen(function* () { + const exported = yield* deps.git.bundle({ + environmentId: input.environmentId, + sandboxName: input.sandboxName, + workspaceRoot: project.workspaceRoot, + }); + if (exported.exitCode !== 0) + return yield* new FleetDrainStepError({ + step: "bundle", + message: exported.stderr.trim() || "The checkout bundle was not created.", + }); + // Nothing the mirror lacks: git refuses an empty bundle, and an ack + // needs no bytes here. + if (exported.stdout === "") return true; + const written = yield* deps.sink.write({ + environmentId: input.environmentId, + drainId: input.drainId, + threadId: worktreeId, + body: exported.stdout, + snapshotSequence: null, + }); + const back = yield* deps.sink.read({ + environmentId: input.environmentId, + drainId: input.drainId, + threadId: worktreeId, + }); + return back !== null && back.body === exported.stdout && written.bytes > 0; + }).pipe(Effect.orElseSucceed(() => false)); + if (copied) worktrees.acked += 1; + else worktrees.failed += 1; + } + } + // Work that started after the enumeration was never copied. The manager's + // `draining` row does not reach the worker, so the ack is re-earned here: + // a live thread, a changed thread set, or a thread that finished a turn + // after it was copied keeps the sandbox. + const after = yield* deps.source + .shell(input.environmentId) + .pipe(Effect.orElseSucceed(() => null)); + let moved = false; + if (after === null) { + moved = true; + } else { + moved = + after.threads.some((thread) => thread.live) || + after.threads.length !== shell.threads.length || + after.threads.some( + (thread) => !shell.threads.some((before) => before.threadId === thread.threadId), + ); + if (!moved && !input.snapshotOnly) { + moved = + after.projects.length !== copiedWorkspaceRoots.size || + after.projects.some((project) => !copiedWorkspaceRoots.has(project.workspaceRoot)); + } + } + if (!moved) { + for (const [threadId, sequence] of copiedSequences) { + if (moved) break; + const current = yield* deps.source + .threadSequence({ environmentId: input.environmentId, threadId }) + .pipe(Effect.orElseSucceed(() => null)); + // A failed currency probe cannot establish that the copy is still current. + // A missing sequence either at copy time or now means currency is not + // proven; null equality is not a match. + if ( + sequence === null || + current === null || + current.snapshotSequence === null || + current.snapshotSequence !== sequence + ) + moved = true; + } + } + if (moved) + warnings.push( + "The worker changed during the drain; its state was copied, nothing was acknowledged.", + ); + // A deadline that has arrived—even after the last item succeeded—is never + // an acknowledgement: there is no budget left to prove currency again. + if ((yield* deps.now) >= input.deadlineAt) deadlineHit = true; + const complete = !moved && !deadlineHit && threads.failed === 0 && worktrees.failed === 0; + return reported({ + threads, + worktrees, + state: complete ? "acked" : "failed", + finishedAt: complete ? yield* deps.now : null, + ...(warnings.length === 0 ? {} : { warnings }), + }); + }); + + return { drain }; +}; + +/** The OpenShell driver calls the drain Promise-shaped, during removal. */ +export const asDriverPort = ( + executor: DrainExecutorShape, + runPromise: (effect: Effect.Effect) => Promise, +): FleetDrainExecutorPort => ({ + drain: (input) => + runPromise( + executor + .drain({ + environmentId: input.environmentId, + sandboxName: input.sandboxName, + drainId: input.drainId, + snapshotOnly: input.snapshotOnly, + deadlineAt: input.deadlineAt, + }) + .pipe( + Effect.orElseSucceed((): FleetDrainOutcome => ({ + threads: zero(), + worktrees: zero(), + state: "failed", + finishedAt: null, + })), + ), + ), +}); diff --git a/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts b/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts new file mode 100644 index 0000000000000..63f4c9890139e --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts @@ -0,0 +1,803 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId } from "@t3tools/contracts"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import { type PlatformError } from "effect/PlatformError"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; + +import { FleetDrainStepError, type DrainSink } from "./OpenShellFleetDrain.ts"; +import * as OpenShellFleetDrain from "./OpenShellFleetDrain.ts"; +import * as OpenShellFleetDrainAdapters from "./OpenShellFleetDrainAdapters.ts"; + +type SinkUse = Effect.Effect< + A, + FleetDrainStepError | PlatformError, + FileSystem.FileSystem | Path.Path +>; + +const refused = ( + effect: Effect.Effect, +): Effect.Effect => + effect.pipe( + Effect.flip, + Effect.catch(() => Effect.die(new Error("expected a refusal"))), + ); + +const withSink = (use: (sink: DrainSink, stateDir: string) => SinkUse) => + Effect.acquireUseRelease( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateDir = yield* fs.makeTempDirectory(); + const sink = yield* OpenShellFleetDrainAdapters.fileDrainSink({ stateDir, fs, path }); + return { fs, path, stateDir, sink }; + }), + ({ sink, stateDir }) => use(sink, stateDir), + ({ fs, stateDir }) => fs.remove(stateDir, { recursive: true }).pipe(Effect.ignore), + ).pipe(Effect.provide(NodeServices.layer)); + +const base64Of = (text: string) => Buffer.from(text, "utf8").toString("base64"); + +const write = ( + sink: DrainSink, + overrides: Partial<{ + environmentId: EnvironmentId; + drainId: string; + threadId: string; + body: string; + }> = {}, +) => + sink.write({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + body: '{"events":[1]}', + snapshotSequence: 4, + ...overrides, + }); + +describe("fileDrainSink", () => { + it.effect("refuses an environment id that climbs out of fleet-drain", () => + withSink((sink) => + Effect.gen(function* () { + const failure = yield* refused(write(sink, { environmentId: EnvironmentId.make("..") })); + expect(failure.step).toBe("write"); + const readFailure = yield* refused( + sink.read({ + environmentId: EnvironmentId.make(".."), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }), + ); + expect(readFailure.step).toBe("read"); + }), + ), + ); + + it.effect("refuses a drain id that climbs out of fleet-drain", () => + withSink((sink) => + Effect.gen(function* () { + const failure = yield* refused(write(sink, { drainId: ".." })); + expect(failure.step).toBe("write"); + const readFailure = yield* refused( + sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "..", + threadId: "thread:a", + }), + ); + expect(readFailure.step).toBe("read"); + }), + ), + ); + + it.effect("neutralizes separators and stores a currency-checked envelope", () => + withSink((sink, stateDir) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const written = yield* write(sink, { + environmentId: EnvironmentId.make("env/../../elsewhere"), + }); + expect(written.bytes).toBeGreaterThan(0); + // The segment cannot name a directory above fleet-drain. + expect(yield* fs.exists(path.join(stateDir, "elsewhere"))).toBe(false); + const record = yield* sink.read({ + environmentId: EnvironmentId.make("env/../../elsewhere"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }); + expect(record).toEqual({ + threadId: "thread:a", + snapshotSequence: 4, + body: '{"events":[1]}', + }); + }), + ), + ); + + it.effect("maps distinct thread ids to distinct files and refuses relative hops", () => + withSink((sink, stateDir) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const drainId = "drain:environment:worker:1"; + const environmentId = EnvironmentId.make("environment:worker"); + const bodies: Record = { + "thread:a": '{"n":1}', + threada: '{"n":2}', + "thread/b": '{"n":3}', + "thread.b": '{"n":4}', + }; + for (const [threadId, body] of Object.entries(bodies)) { + yield* sink.write({ environmentId, drainId, threadId, body, snapshotSequence: null }); + } + // A collision would have overwritten an acked copy with another thread's. + for (const [threadId, body] of Object.entries(bodies)) { + const record = yield* sink.read({ environmentId, drainId, threadId }); + expect(record?.body).toBe(body); + } + const dir = path.join( + stateDir, + "fleet-drain", + "environment%3Aworker", + "drain%3Aenvironment%3Aworker%3A1", + ); + expect(yield* fs.readDirectory(dir)).toHaveLength(4); + for (const hop of ["..", "."]) { + const failure = yield* refused(write(sink, { drainId, threadId: hop, body: "{}" })); + expect(failure.step).toBe("write"); + } + }), + ), + ); + + it.effect("answers null for a thread that was never written", () => + withSink((sink) => + Effect.gen(function* () { + const record = yield* sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:missing", + }); + expect(record).toBeNull(); + }), + ), + ); +}); + +describe("fileDrainSink copy", () => { + it.effect("promotes a previous generation's record with its attachment files", () => + withSink((sink, stateDir) => + Effect.gen(function* () { + const environmentId = EnvironmentId.make("environment:worker"); + yield* sink.write({ + environmentId, + drainId: "drain:environment:worker:1", + threadId: "thread:a", + body: '{"events":[1]}', + snapshotSequence: 2, + attachments: { + files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }], + oversizeSkipped: false, + }, + }); + + const copied = yield* sink.copy({ + environmentId, + fromDrainId: "drain:environment:worker:1", + toDrainId: "drain:environment:worker:2", + threadId: "thread:a", + }); + + expect(copied).toBe(true); + const record = yield* sink.read({ + environmentId, + drainId: "drain:environment:worker:2", + threadId: "thread:a", + }); + expect(record?.body).toBe('{"events":[1]}'); + expect(record?.snapshotSequence).toBe(2); + expect(record?.attachments?.[0]?.name).toBe("attach-1.png"); + + const missing = yield* sink.copy({ + environmentId, + fromDrainId: "drain:environment:worker:1", + toDrainId: "drain:environment:worker:2", + threadId: "thread:missing", + }); + expect(missing).toBe(false); + + // A manifest whose bytes are gone must not look like a promoted copy. + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = path.join( + stateDir, + "fleet-drain", + "environment%3Aworker", + "drain%3Aenvironment%3Aworker%3A1", + ); + for (const file of yield* fs.readDirectory(dir)) { + if (file.endsWith(".bin")) yield* fs.remove(path.join(dir, file)); + } + const broken = yield* refused( + sink.copy({ + environmentId, + fromDrainId: "drain:environment:worker:1", + toDrainId: "drain:environment:worker:3", + threadId: "thread:a", + }), + ); + expect(broken.step).toBe("write"); + }), + ), + ); +}); + +describe("fileDrainSink attachments", () => { + it.effect("stores attachment files beside the bundle and manifests them", () => + withSink((sink, stateDir) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* sink.write({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + body: '{"events":[1]}', + snapshotSequence: 2, + attachments: { + files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }], + oversizeSkipped: false, + }, + }); + const record = yield* sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }); + expect(record?.attachments).toHaveLength(1); + const entry = record?.attachments?.[0]; + expect(entry?.name).toBe("attach-1.png"); + expect(entry?.size).toBe("image-bytes".length); + expect(entry?.hash).toMatch(/^[0-9a-f]{64}$/); + // The sibling file holds the exact bytes, keyed by the same hash. + const dir = path.join( + stateDir, + "fleet-drain", + "environment%3Aworker", + "drain%3Aenvironment%3Aworker%3A1", + ); + const stored = yield* fs.readFile( + path.join(dir, `thread-thread%3Aa-att-${entry?.hash.slice(0, 16)}.bin`), + ); + expect(new TextDecoder().decode(stored)).toBe("image-bytes"); + expect(record?.attachmentsSkipped).toBeUndefined(); + }), + ), + ); + + it.effect("refuses a copy whose attachment bytes changed on disk", () => + withSink((sink, stateDir) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* sink.write({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + body: '{"events":[1]}', + snapshotSequence: 2, + attachments: { + files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }], + oversizeSkipped: false, + }, + }); + const dir = path.join( + stateDir, + "fleet-drain", + "environment%3Aworker", + "drain%3Aenvironment%3Aworker%3A1", + ); + const stored = (yield* fs.readDirectory(dir)).filter((name) => name.endsWith(".bin")); + yield* fs.writeFile(path.join(dir, stored[0]!), new TextEncoder().encode("tampered")); + const record = yield* sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }); + expect(record).toBeNull(); + }), + ), + ); + + it.effect("records an oversize skip without manifest entries", () => + withSink((sink) => + Effect.gen(function* () { + yield* sink.write({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + body: '{"events":[1]}', + snapshotSequence: 2, + attachments: { files: [], oversizeSkipped: true }, + }); + const record = yield* sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }); + expect(record?.attachments).toBeUndefined(); + expect(record?.attachmentsSkipped).toBe(true); + }), + ), + ); + + it.effect("a plain bundle still reads back without manifest fields", () => + withSink((sink) => + Effect.gen(function* () { + yield* write(sink); + const record = yield* sink.read({ + environmentId: EnvironmentId.make("environment:worker"), + drainId: "drain:environment:worker:1", + threadId: "thread:a", + }); + expect(record?.attachments).toBeUndefined(); + expect(record?.attachmentsSkipped).toBeUndefined(); + }), + ), + ); +}); + +describe("execDrainGitBundle", () => { + const execStub = (stdout: string, commands?: Array>, exitCode = 0) => ({ + exec: (_sandboxName: string, command: ReadonlyArray) => + Effect.sync(() => { + commands?.push(command); + return { exitCode, stdout, stderr: "" }; + }), + }); + + it.effect("bundles only the refs the mirror lacks, without contacting a remote", () => + Effect.gen(function* () { + const commands: Array> = []; + const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({ + ...execStub("Qg==", commands), + }); + const result = yield* adapter.bundle({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + workspaceRoot: "/work/a", + }); + expect(result.exitCode).toBe(0); + const line = commands[0]?.join(" ") ?? ""; + // No push: a worker's origin is a read-only mirror, and a branch that + // fell behind would reject a push as non-fast-forward anyway. --all + // would re-export the seeded history, so only what origin lacks travels. + expect(line).toContain("rev-list --count"); + expect(line).toContain("bundle create"); + expect(line).toContain("--branches"); + expect(line).toContain("HEAD --branches"); + expect(line).toContain("--glob='refs/t3/*'"); + expect(line).toContain("--not --remotes=origin"); + expect(line).not.toContain("--all"); + expect(line).not.toContain("--not --remotes "); + expect(line).not.toContain("push"); + expect(line).toContain("'/work/a'"); + }), + ); + + it.effect("passes an empty export through for the drain to acknowledge", () => + Effect.gen(function* () { + const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({ + ...execStub(""), + }); + const result = yield* adapter.bundle({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + workspaceRoot: "/work/a", + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe(""); + }), + ); + + it.effect("returns base64 for a real git bundle stream", () => + Effect.gen(function* () { + const bundle = "# v2 git bundle\nrefnames..."; + const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({ + ...execStub(Buffer.from(bundle, "utf8").toString("base64")), + }); + const result = yield* adapter.bundle({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + workspaceRoot: "/work/a", + }); + expect(result.exitCode).toBe(0); + expect(Buffer.from(result.stdout, "base64").toString("utf8")).toBe(bundle); + expect(Buffer.from(result.stdout, "base64").toString("utf8")).toMatch(/^# v2 git bundle/); + }), + ); + + it.effect("refuses a bundle beyond the size cap before base64 runs", () => + Effect.gen(function* () { + const commands: Array> = []; + const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({ + ...execStub("", commands, 4), + maxBytes: 8, + }); + const outcome = yield* adapter + .bundle({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + workspaceRoot: "/work/a", + }) + .pipe(Effect.flip); + expect(outcome.step).toBe("bundle"); + expect(outcome.message).toContain("size cap"); + const line = commands[0]?.join(" ") ?? ""; + expect(line).toContain("-gt 8"); + expect(line.indexOf("wc -c")).toBeLessThan(line.indexOf("base64")); + }), + ); +}); + +describe("execDrainAttachments", () => { + const listLine = (name: string, size: number) => `==T3ATTL ${name} ${size}\n`; + + const source = (options: { + readonly list: string; + readonly listExit?: number; + readonly files?: Readonly>; + readonly readExit?: number; + readonly maxBytes?: number; + readonly commands?: Array>; + }) => + OpenShellFleetDrainAdapters.execDrainAttachments({ + ...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }), + exec: (sandboxName, command) => { + expect(sandboxName).toBe("sandbox-1"); + options.commands?.push(command); + const script = command[2] ?? ""; + if (script.includes("==T3ATTL")) { + return Effect.succeed({ + exitCode: options.listExit ?? 0, + stdout: options.list, + stderr: "", + }); + } + const name = /'([^']+)'$/.exec(script)?.[1] ?? ""; + return Effect.succeed({ + exitCode: options.readExit ?? 0, + stdout: options.files?.[name] ?? "", + stderr: "", + }); + }, + }); + + it.effect("reads each flat attachment of the thread by its id", () => + Effect.gen(function* () { + const commands: Array> = []; + const attachments = yield* source({ + list: + listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3) + + listLine("thread-a-22222222-2222-4222-8222-222222222222.bin", 3), + files: { + "thread-a-11111111-1111-4111-8111-111111111111.png": base64Of("one"), + "thread-a-22222222-2222-4222-8222-222222222222.bin": base64Of("two"), + }, + commands, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }); + + expect(attachments.oversizeSkipped).toBe(false); + expect(attachments.files.map((file) => file.name)).toEqual([ + "thread-a-11111111-1111-4111-8111-111111111111.png", + "thread-a-22222222-2222-4222-8222-222222222222.bin", + ]); + expect(Buffer.from(attachments.files[0]?.bytes ?? "", "base64").toString()).toBe("one"); + // One listing, then one read per file: no base64 ever rides the listing. + expect(commands).toHaveLength(3); + expect(commands[1]?.join(" ")).toContain("thread-a-11111111-1111-4111-8111-111111111111.png"); + }), + ); + + it.effect("ignores files that are not this thread's attachment ids", () => + Effect.gen(function* () { + const attachments = yield* source({ + list: + listLine("other-a-11111111-1111-4111-8111-111111111111.png", 3) + + listLine("thread-a-..%2Fescape.png", 3) + + listLine("thread-a-33333333-3333-4333-8333-333333333333.png", 3), + files: { "thread-a-33333333-3333-4333-8333-333333333333.png": base64Of("ok") }, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }); + + expect(attachments.files.map((file) => file.name)).toEqual([ + "thread-a-33333333-3333-4333-8333-333333333333.png", + ]); + }), + ); + + it.effect("skips an oversized set from the listing, before reading any bytes", () => + Effect.gen(function* () { + const commands: Array> = []; + const attachments = yield* source({ + list: + listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3) + + listLine("thread-a-2222.png", 3), + maxBytes: 2, + commands, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }); + + expect(commands).toHaveLength(1); + expect(attachments.files).toEqual([]); + }), + ); + + it.effect("fails typed when the worker cannot list attachments", () => + Effect.gen(function* () { + const failure = yield* refused( + source({ list: "", listExit: 7 })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }), + ); + expect(failure.step).toBe("attachments"); + }), + ); + + it.effect("fails typed when one attachment cannot be read", () => + Effect.gen(function* () { + const failure = yield* refused( + source({ + list: listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3), + readExit: 5, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }), + ); + expect(failure.step).toBe("attachments"); + expect(failure.message).toContain("thread-a-11111111-1111-4111-8111-111111111111.png"); + }), + ); + + it.effect("does not let a shorter thread id claim a longer segment's attachment", () => + Effect.gen(function* () { + const followupName = "topic-followup-33333333-3333-4333-8333-333333333333.png"; + const attachments = yield* source({ + list: listLine(followupName, 3), + files: { [followupName]: base64Of("followup") }, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "topic:a", + }); + expect(attachments.files).toEqual([]); + + const exact = yield* source({ + list: listLine(followupName, 3), + files: { [followupName]: base64Of("followup") }, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "topic:followup", + }); + expect(exact.files.map((file) => file.name)).toEqual([followupName]); + }), + ); + + it.effect("checks the decoded attachment bytes against the cap too", () => + Effect.gen(function* () { + const name = "thread-a-11111111-1111-4111-8111-111111111111.png"; + const attachments = yield* source({ + // The listing lies about one byte; the read must enforce the real cap. + list: listLine(name, 2), + files: { [name]: base64Of("three") }, + maxBytes: 2, + })({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + threadId: "thread:a", + }); + expect(attachments.files).toEqual([]); + expect(attachments.oversizeSkipped).toBe(true); + }), + ); +}); + +describe("linkDrainSource", () => { + it.effect("rejects a shell response that omits the archived-thread list", () => + Effect.gen(function* () { + const source = OpenShellFleetDrainAdapters.linkDrainSource({ + resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }), + http: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify( + request.url.includes("/api/orchestration/shell") + ? { threads: [], projects: [] } + : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false }, + ), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + }); + const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker"))); + expect(failure.step).toBe("shell"); + }), + ); + + it.effect("rejects an inventory thread without a usable id", () => + Effect.gen(function* () { + const source = OpenShellFleetDrainAdapters.linkDrainSource({ + resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }), + http: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify( + request.url.includes("/api/orchestration/shell") + ? { + threads: [{ status: "idle" }], + archivedThreads: [], + projects: [], + } + : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false }, + ), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + }); + const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker"))); + expect(failure.step).toBe("shell"); + expect(failure.message).toContain("without a usable id"); + }), + ); + + it.effect("refuses advertised history with no continuation cursor", () => + Effect.gen(function* () { + const source = OpenShellFleetDrainAdapters.linkDrainSource({ + resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }), + http: HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify( + request.url.includes("/api/orchestration/shell") + ? { threads: [], archivedThreads: [], projects: [] } + : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: true }, + ), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + }); + const failure = yield* refused( + source.threadHistory({ + environmentId: EnvironmentId.make("environment:worker"), + threadId: "thread:a", + }), + ); + expect(failure.step).toBe("history"); + expect(failure.message).toContain("without a continuation cursor"); + }), + ); + + it.effect("drains archived threads like live ones", () => + Effect.gen(function* () { + const shellRequests: Array = []; + const shellHeaders: Array> = []; + const http = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify( + request.url.includes("/api/orchestration/shell") + ? (shellRequests.push(request.url), + shellHeaders.push(request.headers as never), + { + threads: [{ threadId: "thread:a", status: "idle" }], + archivedThreads: [{ threadId: "thread:old", status: "archived" }], + projects: [], + }) + : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false }, + ), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ), + ), + ); + const written: Array = []; + const files = new Map< + string, + { threadId: string; snapshotSequence: number | null; body: string } + >(); + const executor = OpenShellFleetDrain.makeDrainExecutor({ + source: OpenShellFleetDrainAdapters.linkDrainSource({ + resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }), + http, + }), + sink: { + write: (input) => + Effect.sync(() => { + written.push(input.threadId); + files.set(input.threadId, { + threadId: input.threadId, + snapshotSequence: input.snapshotSequence, + body: input.body, + }); + return { bytes: input.body.length }; + }), + read: ({ threadId }) => Effect.succeed(files.get(threadId) ?? null), + copy: () => Effect.succeed(false), + }, + git: { bundle: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }) }, + now: Effect.succeed(0), + }); + const outcome = yield* executor.drain({ + environmentId: EnvironmentId.make("environment:worker"), + sandboxName: "sandbox-1", + drainId: "drain:environment:worker:1", + snapshotOnly: true, + deadlineAt: Number.MAX_SAFE_INTEGER, + }); + // The archived thread is copied and acked, not omitted from the drain. + expect(outcome.state).toBe("acked"); + expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 }); + expect(written.sort()).toEqual(["thread:a", "thread:old"]); + // The opt-in asks for archived rows and carries the required protocol header. + expect(shellRequests[0]).toContain("includeArchived=1"); + expect(shellHeaders[0]?.["x-t3-orchestration-protocol"]).toBe("2"); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("refuses an incomplete shell response instead of treating it as empty", () => + Effect.gen(function* () { + const http = HttpClient.make((request) => + Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response("{}", { + status: 200, + headers: { "content-type": "application/json" }, + }), + ), + ), + ); + const source = OpenShellFleetDrainAdapters.linkDrainSource({ + resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }), + http, + }); + const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker"))); + expect(failure.step).toBe("shell"); + expect(failure.message).toContain("incomplete"); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/environment/OpenShellFleetDrainAdapters.ts b/apps/server/src/environment/OpenShellFleetDrainAdapters.ts new file mode 100644 index 0000000000000..aae00fb264a24 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetDrainAdapters.ts @@ -0,0 +1,614 @@ +import { + isLiveSubagentShell, + ORCHESTRATION_PROTOCOL_HEADER, + ORCHESTRATION_PROTOCOL_VERSION_TEXT, + type EnvironmentId, +} from "@t3tools/contracts"; +import * as Digest from "effect/crypto/Digest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Hex from "effect/encoding/Hex"; +import * as Base64 from "effect/encoding/Base64"; +import * as Result from "effect/Result"; +import * as HttpClient from "effect/http/HttpClient"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts"; + +/** The parts of a worker's thread shell that decide whether it is still working. */ +interface DrainShellThread { + readonly id?: string; + readonly threadId?: string; + readonly status?: string; + readonly activityRunStatus?: string | null; + readonly pendingRuntimeRequest?: unknown; + readonly pendingBackgroundTasks?: ReadonlyArray | null; +} + +import { + FleetDrainStepError, + type DrainAttachmentManifestEntry, + type DrainAttachments, + type DrainGitBundle, + type DrainSink, + type DrainSinkRecord, + type DrainSource, +} from "./OpenShellFleetDrain.ts"; + +const encode = (value: string) => encodeURIComponent(value); + +/** Percent-encoding is collision-free ("thread:a" keeps its %3A, so "threada" + * stays distinct, and "thread/b" cannot hop directories because its "/" becomes + * %2F), so the encoding survives whole: refuse only empty names and the relative + * hops "." and "..", which encode to themselves. */ +const segmentOf = (value: string): string | null => { + const encoded = encodeURIComponent(value); + return encoded === "" || encoded === "." || encoded === ".." ? null : encoded; +}; + +const decodeEnvelope = (raw: string): DrainSinkRecord | null => { + try { + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.threadId !== "string" || typeof parsed.body !== "string") return null; + const attachments = Array.isArray(parsed.attachments) + ? parsed.attachments.filter( + (entry): entry is DrainAttachmentManifestEntry => + typeof entry?.name === "string" && + typeof entry?.hash === "string" && + typeof entry?.size === "number", + ) + : undefined; + return { + threadId: parsed.threadId, + snapshotSequence: + typeof parsed.snapshotSequence === "number" ? parsed.snapshotSequence : null, + body: parsed.body, + ...(attachments !== undefined && attachments.length > 0 ? { attachments } : {}), + ...(parsed.attachmentsSkipped === true ? { attachmentsSkipped: true } : {}), + }; + } catch { + return null; + } +}; + +const toEnvelope = Schema.encodeEffect( + Schema.fromJsonString( + Schema.Struct({ + threadId: Schema.String, + snapshotSequence: Schema.NullOr(Schema.Number), + body: Schema.String, + attachments: Schema.optionalKey( + Schema.Array( + Schema.Struct({ name: Schema.String, hash: Schema.String, size: Schema.Number }), + ), + ), + attachmentsSkipped: Schema.optionalKey(Schema.Boolean), + }), + ), +); + +/** `stateDir/fleet-drain///thread-.json`. */ +export const fileDrainSink = Effect.fn("DrainSink.make")(function* (input: { + readonly stateDir: string; + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; +}) { + const { fs, path, stateDir } = input; + // Attachment hashes come from the digest service; the sink is built once + // per drain and carries it. + const digest = yield* Digest.Digest; + return { + write: Effect.fn("DrainSink.write")( + function* (input: { + readonly environmentId: EnvironmentId; + readonly drainId: string; + readonly threadId: string; + readonly body: string; + readonly snapshotSequence: number | null; + readonly attachments?: DrainAttachments; + }) { + const environment = segmentOf(input.environmentId); + const drain = segmentOf(input.drainId); + const thread = segmentOf(input.threadId); + if (environment === null || drain === null || thread === null) { + return yield* new FleetDrainStepError({ + step: "write", + message: `Refused a drain path segment for ${input.environmentId} / ${input.drainId} / ${input.threadId}.`, + }); + } + const dir = path.join(stateDir, "fleet-drain", environment, drain); + const target = path.join(dir, `thread-${thread}.json`); + yield* fs.makeDirectory(dir, { recursive: true }); + // Attachments land as sibling files so the manifest in the envelope stays + // small; the hash keys the file name and is what read-back verifies. + const manifest: Array = []; + for (const file of input.attachments?.files ?? []) { + const decoded = Base64.decode(file.bytes.replace(/\s/g, "")); + if (Result.isFailure(decoded)) { + return yield* new FleetDrainStepError({ + step: "write", + message: `Attachment ${file.name} was not valid base64.`, + }); + } + const bytes = decoded.success; + const hash = Hex.encode(yield* digest.digest(Digest.sha256, bytes)); + yield* fs.writeFile( + path.join(dir, `thread-${thread}-att-${hash.slice(0, 16)}.bin`), + bytes, + ); + manifest.push({ name: file.name, hash, size: bytes.byteLength }); + } + // Write then rename: a retry must never accept a half-written copy as done. + const envelope = yield* toEnvelope({ + threadId: input.threadId, + snapshotSequence: input.snapshotSequence, + body: input.body, + ...(manifest.length === 0 ? {} : { attachments: manifest }), + ...(input.attachments?.oversizeSkipped === true ? { attachmentsSkipped: true } : {}), + }); + yield* fs.writeFileString(`${target}.partial`, envelope); + yield* fs.rename(`${target}.partial`, target); + return { bytes: new TextEncoder().encode(envelope).byteLength }; + }, + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "write", + message: "The drain copy could not be written.", + cause, + }), + ), + ), + read: Effect.fn("DrainSink.read")( + function* (input: { + readonly environmentId: EnvironmentId; + readonly drainId: string; + readonly threadId: string; + }) { + const environment = segmentOf(input.environmentId); + const drain = segmentOf(input.drainId); + const thread = segmentOf(input.threadId); + if (environment === null || drain === null || thread === null) { + return yield* new FleetDrainStepError({ + step: "read", + message: `Refused a drain path segment for ${input.environmentId} / ${input.drainId} / ${input.threadId}.`, + }); + } + const file = path.join( + stateDir, + "fleet-drain", + environment, + drain, + `thread-${thread}.json`, + ); + const raw = yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => null)); + if (raw === null) return null; + const record = decodeEnvelope(raw); + if (record === null) return null; + // A manifest without its bytes is not a copy: verify each hash so a + // corrupted attachment fails the ack instead of being imported later. + for (const entry of record.attachments ?? []) { + const bytes = yield* fs + .readFile( + path.join(path.dirname(file), `thread-${thread}-att-${entry.hash.slice(0, 16)}.bin`), + ) + .pipe(Effect.orElseSucceed(() => null)); + if (bytes === null) return null; + if (Hex.encode(yield* digest.digest(Digest.sha256, bytes)) !== entry.hash) return null; + } + return record; + }, + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "read", + message: "The drain copy could not be read.", + cause, + }), + ), + ), + copy: Effect.fn("DrainSink.copy")( + function* (input: { + readonly environmentId: EnvironmentId; + readonly fromDrainId: string; + readonly toDrainId: string; + readonly threadId: string; + }) { + const environment = segmentOf(input.environmentId); + const from = segmentOf(input.fromDrainId); + const to = segmentOf(input.toDrainId); + const thread = segmentOf(input.threadId); + const fromDir = + environment === null || from === null + ? null + : path.join(stateDir, "fleet-drain", environment, from); + const toDir = + environment === null || to === null + ? null + : path.join(stateDir, "fleet-drain", environment, to); + if (fromDir === null || toDir === null || thread === null) + return yield* new FleetDrainStepError({ + step: "write", + message: `Refused a drain path segment for ${input.fromDrainId} / ${input.toDrainId}.`, + }); + const name = `thread-${thread}.json`; + const raw = yield* fs + .readFileString(path.join(fromDir, name)) + .pipe(Effect.orElseSucceed(() => null)); + const record = raw === null ? null : decodeEnvelope(raw); + if (record === null || raw === null) return false; + yield* fs.makeDirectory(toDir, { recursive: true }); + // Attachment files are keyed by the manifest hash, so they travel with it. + for (const entry of record.attachments ?? []) { + const attachment = `thread-${thread}-att-${entry.hash.slice(0, 16)}.bin`; + const bytes = yield* fs.readFile(path.join(fromDir, attachment)).pipe( + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "write", + message: `Attachment copy failed: ${cause}`, + }), + ), + ); + yield* fs.writeFile(path.join(toDir, attachment), bytes).pipe( + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "write", + message: `Attachment copy failed: ${cause}`, + }), + ), + ); + } + yield* fs.writeFileString(`${path.join(toDir, name)}.partial`, raw); + yield* fs.rename(`${path.join(toDir, name)}.partial`, path.join(toDir, name)); + return true; + }, + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "write", + message: "The drain copy could not be written.", + cause, + }), + ), + ), + } satisfies DrainSink; +}, Effect.provide(Digest.layerWebCrypto)); + +const toJsonBody = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const fromJsonBody = Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown)); + +/** + * Reads the worker over its authenticated link: the shell snapshot lists what it + * holds, and each thread's bounded snapshot plus history is the lossless copy. + */ +export const linkDrainSource = (input: { + readonly resolve: ( + environmentId: EnvironmentId, + ) => Effect.Effect<{ readonly url: string; readonly token: string }, FleetDrainStepError>; + readonly http: HttpClient.HttpClient; +}): DrainSource => { + const http = input.http; + const getJson = (environmentId: EnvironmentId, path: string) => + input.resolve(environmentId).pipe( + Effect.flatMap((link) => + http.get(new URL(path, link.url).toString(), { + // The orchestration endpoints require the protocol header; without it + // the worker answers 400 and the drain would look empty. + headers: { + authorization: `Bearer ${link.token}`, + [ORCHESTRATION_PROTOCOL_HEADER]: ORCHESTRATION_PROTOCOL_VERSION_TEXT, + }, + }), + ), + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "history", + message: "The worker history could not be read.", + cause, + }), + ), + Effect.flatMap((response) => + response.text.pipe( + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "history", + message: "The worker history could not be read.", + cause, + }), + ), + Effect.flatMap((body) => + response.status < 200 || response.status >= 300 + ? Effect.fail( + new FleetDrainStepError({ + step: "history", + message: `The worker answered ${response.status} for ${path}.`, + }), + ) + : fromJsonBody(body).pipe( + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "history", + message: "The worker history could not be read.", + cause, + }), + ), + ), + ), + ), + ), + ); + return { + shell: (environmentId) => + // Archived threads are drainable data: the worker lists them only on request. + getJson(environmentId, "/api/orchestration/shell?includeArchived=1").pipe( + Effect.flatMap((snapshot) => + Effect.gen(function* () { + const shell = snapshot as { + readonly threads?: ReadonlyArray; + readonly archivedThreads?: ReadonlyArray; + readonly projects?: ReadonlyArray<{ + readonly workspaceRoot?: string; + readonly path?: string; + }>; + }; + // A body without both lists is an incomplete answer, not an empty + // worker: an empty ack here would delete unconverted threads. + if ( + !Array.isArray(shell.threads) || + !Array.isArray(shell.projects) || + !Array.isArray(shell.archivedThreads) + ) + return yield* new FleetDrainStepError({ + step: "shell", + message: "The worker's shell response is incomplete; nothing was acknowledged.", + }); + const shellThreads = [...shell.threads, ...shell.archivedThreads]; + const threads: Array<{ readonly threadId: string; readonly live: boolean }> = []; + for (const thread of shellThreads) { + const threadId = thread.threadId ?? thread.id; + if (typeof threadId !== "string" || threadId === "") { + return yield* new FleetDrainStepError({ + step: "shell", + message: "The worker returned a shell thread without a usable id.", + }); + } + // Liveness comes from the shell's own status, and an unreadable one + // counts as live: removing a worker that is still writing loses work. + const live = + thread.status === undefined && thread.activityRunStatus === undefined + ? true + : isLiveSubagentShell({ + status: thread.status ?? "idle", + activityRunStatus: thread.activityRunStatus ?? null, + pendingRuntimeRequest: thread.pendingRuntimeRequest ?? null, + pendingBackgroundTasks: thread.pendingBackgroundTasks ?? null, + } as never); + threads.push({ threadId, live }); + } + const projects = shell.projects.flatMap((project) => { + const workspaceRoot = project.workspaceRoot ?? project.path; + return workspaceRoot === undefined ? [] : [{ workspaceRoot }]; + }); + return { threads, projects }; + }), + ), + ), + threadSequence: (request) => + getJson( + request.environmentId, + `/api/orchestration/threads/${encode(request.threadId)}/bounded`, + ).pipe( + Effect.map((snapshot) => ({ + snapshotSequence: + typeof (snapshot as { snapshotSequence?: unknown }).snapshotSequence === "number" + ? (snapshot as { snapshotSequence: number }).snapshotSequence + : null, + })), + ), + threadHistory: Effect.fn("DrainSource.threadHistory")(function* (request: { + readonly environmentId: EnvironmentId; + readonly threadId: string; + }) { + const encoded = encode(request.threadId); + const snapshot = (yield* getJson( + request.environmentId, + `/api/orchestration/threads/${encoded}/bounded`, + )) as { + readonly snapshotSequence?: unknown; + readonly historyCursor?: string | null; + readonly hasMoreHistory?: boolean; + }; + // The page endpoint needs the worker's own cursor: a page size is rejected, + // and a truncated history is not a drain (spec section 7.2). + const pages: unknown[] = []; + let cursor = snapshot.historyCursor ?? null; + let hasMore = snapshot.hasMoreHistory === true; + if (hasMore && cursor === null) + return yield* new FleetDrainStepError({ + step: "history", + message: "The worker advertised more history without a continuation cursor.", + }); + while (hasMore && cursor !== null) { + const page = (yield* getJson( + request.environmentId, + `/api/orchestration/threads/${encoded}/history?cursor=${encode(cursor)}`, + )) as { readonly nextCursor?: string | null; readonly hasMoreHistory?: boolean }; + pages.push(page); + hasMore = page.hasMoreHistory === true; + cursor = page.nextCursor ?? null; + if (hasMore && cursor === null) + return yield* new FleetDrainStepError({ + step: "history", + message: "The worker advertised more history without a continuation cursor.", + }); + } + const body = yield* toJsonBody({ threadId: request.threadId, snapshot, pages }).pipe( + Effect.mapError( + (cause) => + new FleetDrainStepError({ + step: "history", + message: "The worker history could not be read.", + cause, + }), + ), + ); + return { + body, + snapshotSequence: + typeof snapshot.snapshotSequence === "number" ? snapshot.snapshotSequence : null, + }; + }), + }; +}; + +/** Total decoded bundle bytes a checkout may export before the drain refuses it. */ +const MAX_DRAIN_BUNDLE_BYTES = 256 * 1024 * 1024; + +/** Exports a worker checkout and its checkpoint refs from inside the sandbox. */ +export const execDrainGitBundle = (input: { + readonly exec: ( + sandboxName: string, + command: ReadonlyArray, + ) => Effect.Effect< + { readonly exitCode: number; readonly stdout: string; readonly stderr: string }, + FleetDrainStepError + >; + /** Overridden in tests; the production cap keeps one drain off the OOM path. */ + readonly maxBytes?: number; +}): DrainGitBundle => ({ + bundle: Effect.fn("FleetDrainGitBundle.bundle")( + // A worker cannot push: its origin is the read-only shared mirror, it holds + // no credentials, and a branch behind origin rejects a push anyway. A + // bundle carries the refs (branches and the refs/t3 checkpoints holding + // uncommitted work) the mirror lacks, base64 so exec stdout stays text. + ({ sandboxName, workspaceRoot }) => + Effect.gen(function* () { + const maxBytes = input.maxBytes ?? MAX_DRAIN_BUNDLE_BYTES; + const root = shellQuote(workspaceRoot); + const revs = `HEAD --branches --glob='refs/t3/*'`; + const exported = yield* input.exec(sandboxName, [ + "sh", + "-c", + // HEAD carries a detached commit; subtract only origin, so a commit + // fetched from another remote is not a missing prerequisite. + // An empty export means the mirror has everything and acks. + `b="\${TMPDIR:-/tmp}/t3-fleet-drain-$$.bundle"; ` + + `if [ "$(git -C ${root} rev-list --count ${revs} --not --remotes=origin)" -eq 0 ]; then exit 0; fi; ` + + `git -C ${root} bundle create "$b" ${revs} --not --remotes=origin || exit $?; ` + + // The cap is checked before base64, so an oversized bundle never + // crosses exec as one string. + // shortcut: without origin remote-tracking refs there is no + // baseline, so a large checkout needs force removal. + `if [ "$(wc -c <"$b" | tr -d '[:space:]')" -gt ${maxBytes} ]; then rm -f "$b"; exit 4; fi; ` + + `base64 "$b"; rc=$?; rm -f "$b"; exit $rc`, + ]); + if (exported.exitCode === 4) + return yield* new FleetDrainStepError({ + step: "bundle", + message: `The checkout bundle of ${workspaceRoot} exceeds the drain size cap.`, + }); + return exported; + }), + ), +}); + +/** Single-quotes a value for a shell command line. */ +const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** Total decoded attachment bytes drained per thread before skipping the set. */ +const MAX_DRAIN_ATTACHMENT_BYTES = 50 * 1024 * 1024; + +/** Marks one attachment file's name and size on the listing command's stdout. */ +const ATTACHMENT_LIST_FRAME = "==T3ATTL "; + +/** + * Attachment files over gateway exec (the worker's HTTP API serves no + * attachment bytes): the worker lists the thread's files with their sizes + * first, so an oversized set is skipped before any bytes cross exec, then each + * file is read on its own. Files live flat as `-`. + */ +export const execDrainAttachments = (input: { + readonly exec: ( + sandboxName: string, + command: ReadonlyArray, + ) => Effect.Effect< + { readonly exitCode: number; readonly stdout: string; readonly stderr: string }, + FleetDrainStepError + >; + readonly maxBytes?: number; +}): NonNullable => { + const maxBytes = input.maxBytes ?? MAX_DRAIN_ATTACHMENT_BYTES; + const attachmentsDir = `"\${T3CODE_HOME:-$HOME/.t3}/userdata/attachments"`; + const listCommand = (segment: string) => [ + "sh", + "-c", + `d=${attachmentsDir}; if [ -d "$d" ]; then for f in "$d"/${segment}-*; do ` + + `[ -f "$f" ] || continue; case "$f" in *.part) continue;; esac; ` + + `printf '%s\\n' "${ATTACHMENT_LIST_FRAME}$(basename "$f") $(wc -c <"$f" | tr -d '[:space:]')"; ` + + `done; fi`, + ]; + const readCommand = (name: string) => [ + "sh", + "-c", + `base64 ${attachmentsDir}/${shellQuote(name)}`, + ]; + return ({ sandboxName, threadId }) => + Effect.gen(function* () { + const segment = toSafeThreadAttachmentSegment(threadId); + if (segment === null) return { files: [], oversizeSkipped: false } as const; + const listed = yield* input.exec(sandboxName, listCommand(segment)); + if (listed.exitCode !== 0) { + return yield* new FleetDrainStepError({ + step: "attachments", + message: `The worker did not list attachments for ${threadId} (exit ${listed.exitCode}).`, + }); + } + const entries: Array<{ name: string; size: number }> = []; + for (const line of listed.stdout.split("\n")) { + if (!line.startsWith(ATTACHMENT_LIST_FRAME)) continue; + const rest = line.slice(ATTACHMENT_LIST_FRAME.length).trim(); + const separator = rest.lastIndexOf(" "); + if (separator <= 0) continue; + const name = rest.slice(0, separator); + const size = Number(rest.slice(separator + 1)); + // The worker names files itself; anything else is not one of this thread's. + // The full id pattern prevents `topic` from claiming files that belong + // to a longer segment such as `topic-followup`. + if ( + !new RegExp( + `^${segment}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:\\.[a-z0-9]{1,10})?$`, + "i", + ).test(name) + ) + continue; + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name)) continue; + if (!Number.isSafeInteger(size) || size < 0) continue; + entries.push({ name, size }); + } + if (entries.reduce((total, entry) => total + entry.size, 0) > maxBytes) { + return { files: [], oversizeSkipped: true } as const; + } + const files: Array<{ name: string; bytes: string }> = []; + let copiedBytes = 0; + for (const entry of entries) { + const read = yield* input.exec(sandboxName, readCommand(entry.name)); + if (read.exitCode !== 0) { + return yield* new FleetDrainStepError({ + step: "attachments", + message: `The worker did not read attachment ${entry.name} of ${threadId} (exit ${read.exitCode}).`, + }); + } + const bytes = Buffer.from(read.stdout, "base64"); + copiedBytes += bytes.byteLength; + if (copiedBytes > maxBytes) return { files: [], oversizeSkipped: true } as const; + files.push({ name: entry.name, bytes: read.stdout }); + } + return { files, oversizeSkipped: false } as const; + }); +}; diff --git a/apps/server/src/environment/OpenShellFleetService.test.ts b/apps/server/src/environment/OpenShellFleetService.test.ts new file mode 100644 index 0000000000000..b2db614851781 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetService.test.ts @@ -0,0 +1,220 @@ +import { describe, expect, it } from "@effect/vitest"; +import { vi } from "vite-plus/test"; + +import { EnvironmentId, type OpenShellFleetWorkerStatus } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Context from "effect/Context"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as Layer from "effect/Layer"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as NodeServices from "@effect/platform-node/NodeServices"; + +import * as ServerConfig from "../config.ts"; +import * as PeerEnvironmentRegistry from "../peer/PeerEnvironmentRegistry.ts"; +import * as PeerLinks from "../peer/PeerLinks.ts"; +import * as ServerEnvironment from "./ServerEnvironment.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as OpenShellFleetStore from "./OpenShellFleetStore.ts"; +import * as OpenShellFleetService from "./OpenShellFleetService.ts"; + +const { fakeGatewayClient } = vi.hoisted(() => { + const view = (name: string) => ({ + name, + sandboxId: `sbx-${name}`, + phase: "Ready", + labels: {}, + createdAt: 1_700_000_000_000, + }); + return { + fakeGatewayClient: { + health: async () => {}, + createFromTemplate: async (spec: { name?: string }) => view(spec.name ?? "sbx-x"), + get: async () => null, + listAll: async () => [], + deleteSandbox: async () => "completed" as const, + waitReady: async (name: string) => view(name), + exec: async () => ({ exitCode: 0, stdout: "{}", stderr: "" }), + setPolicy: async () => {}, + listTemplates: async () => [], + createTemplate: async (spec: { name?: string }) => ({ name: spec.name ?? "tpl" }), + }, + }; +}); + +vi.mock("@t3tools/openshell/gatewayClient", () => ({ + connectOpenShellGateway: async () => fakeGatewayClient, +})); + +const fleetOnSettings = { + openshellEnabled: true, + openshell: { gatewayUrl: "https://gateway.example.test", managingInstance: true }, +} as const; + +/** Secret store mock: fails its first `count` reads, then answers empty. */ +const flakySecrets = (failures: number) => { + let calls = 0; + const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: () => { + calls += 1; + return calls <= failures + ? Effect.fail( + new ServerSecretStore.SecretStoreReadError({ + resource: "openshell-gateway-credential", + cause: "locked in this test", + }), + ) + : Effect.succeedNone; + }, + getOrCreateRandom: (name: string) => + Effect.succeed(new TextEncoder().encode(`random-for-${name}`)), + }); + return { secrets, calls: () => calls }; +}; + +const alwaysFailingSecrets = () => { + let calls = 0; + const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: () => { + calls += 1; + return Effect.fail( + new ServerSecretStore.SecretStoreReadError({ + resource: "openshell-gateway-credential", + cause: "locked in this test", + }), + ); + }, + getOrCreateRandom: (name: string) => + Effect.succeed(new TextEncoder().encode(`random-for-${name}`)), + }); + return { secrets, calls: () => calls }; +}; + +const managingEnvironment = Layer.succeed( + ServerEnvironment.ServerEnvironment, + ServerEnvironment.ServerEnvironment.of({ + getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-managing")), + getDescriptor: Effect.die("getDescriptor is unused in this test"), + }), +); + +const layerFleet = ( + settingsContext: Context.Context, + secrets: Layer.Layer, +) => + OpenShellFleetService.layer.pipe( + Layer.provide(Layer.succeedContext(settingsContext)), + Layer.provide(secrets), + Layer.provide(managingEnvironment), + Layer.provide(Layer.mock(PeerEnvironmentRegistry.PeerEnvironmentRegistry)({})), + Layer.provide( + Layer.mock(OpenShellFleetStore.OpenShellFleetStore)({ list: Effect.succeed([]) }), + ), + Layer.provide(ServerConfig.layerTest("/tmp/t3-fleet-test-cwd", "/tmp/t3-fleet-test")), + Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})), + Layer.provide( + Layer.succeed(SecretResolver.SecretResolver, { + label: (provider) => Effect.succeed(provider), + read: (references) => Effect.succeed(references.map(() => undefined)), + invalidate: () => Effect.void, + }), + ), + Layer.provide(NodeServices.layer), + Layer.provide(FetchHttpClient.layer), + ); + +describe("OpenShellFleetService", () => { + it.effect("refuses a spawn once OpenShell is disabled after first use", () => + Effect.scoped( + Effect.gen(function* () { + const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings)); + const settings = Context.get(settingsContext, ServerSettings.ServerSettingsService); + const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe( + Effect.provide( + layerFleet( + settingsContext, + Layer.mock(ServerSecretStore.ServerSecretStore)({ + get: () => Effect.succeedNone, + getOrCreateRandom: (name: string) => + Effect.succeed(new TextEncoder().encode(`random-for-${name}`)), + }), + ), + ), + ); + + expect(yield* fleet.listWorkers({})).toEqual([] as OpenShellFleetWorkerStatus[]); + + yield* settings.updateSettings({ openshellEnabled: false }); + const error = yield* fleet + .spawnWorker({ runtimeModeCeiling: "approval-required", workerClass: "ephemeral" }) + .pipe(Effect.flip); + expect(error).toBeInstanceOf(OpenShellFleetService.OpenShellFleetDisabled); + }), + ), + ); + + it.effect("retries a transient init failure on the next call instead of caching it", () => + Effect.scoped( + Effect.gen(function* () { + const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings)); + const { secrets, calls } = flakySecrets(1); + const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe( + Effect.provide(layerFleet(settingsContext, secrets)), + ); + + const firstError = yield* fleet.listWorkers({}).pipe(Effect.flip); + expect(firstError).toBeInstanceOf(OpenShellFleetService.OpenShellFleetOperationFailed); + + // The same service instance builds the driver on the retry: a failure + // was never cached, so the second call gets a working driver. + expect(yield* fleet.listWorkers({})).toEqual([]); + expect(calls()).toBe(2); + }), + ), + ); + + it.effect("labels an operation's init failure with that operation, not an earlier caller's", () => + Effect.scoped( + Effect.gen(function* () { + const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings)); + const { secrets } = alwaysFailingSecrets(); + const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe( + Effect.provide(layerFleet(settingsContext, secrets)), + ); + + const spawnError = yield* fleet + .spawnWorker({ runtimeModeCeiling: "approval-required", workerClass: "ephemeral" }) + .pipe(Effect.flip); + expect(spawnError).toMatchObject({ + _tag: "OpenShellFleetOperationFailed", + operation: "spawn", + message: "The fleet operation failed.", + }); + + // A failed init leaves nothing cached, so the next operation rebuilds + // and labels the failure with its own operation. + const removeError = yield* fleet + .removeWorker({ targetEnvironmentId: EnvironmentId.make("environment-worker-1") }) + .pipe(Effect.flip); + expect(removeError).toMatchObject({ operation: "remove" }); + }), + ), + ); + it.effect("shares driver initialization across concurrent first calls", () => + Effect.scoped( + Effect.gen(function* () { + const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings)); + const { secrets, calls } = flakySecrets(0); + const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe( + Effect.provide(layerFleet(settingsContext, secrets)), + ); + expect( + yield* Effect.all([fleet.listWorkers({}), fleet.listWorkers({})], { + concurrency: "unbounded", + }), + ).toEqual([[], []]); + expect(calls()).toBe(1); + }), + ), + ); +}); diff --git a/apps/server/src/environment/OpenShellFleetService.ts b/apps/server/src/environment/OpenShellFleetService.ts new file mode 100644 index 0000000000000..0c4a4e7561eb8 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetService.ts @@ -0,0 +1,562 @@ +import { + EnvironmentId, + type GatewayDriverRemoveOptions, + type GatewayDriverRemoveResult, + type GatewayDriverSpawnResult, + isRuntimeModeAbove, + type OpenShellFleetWorkerStatus, + OpenShellWorkerClass, + type OrchestratorMcpRuntimeMode, + type RuntimeMode, + type ServerSettings as ServerSettingsValue, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import type { OpenShellGatewayAuth } from "@t3tools/contracts"; +import * as OpenShellGatewayAuthModule from "./OpenShellGatewayAuth.ts"; +import * as Duration from "effect/Duration"; +import * as FileSystem from "effect/FileSystem"; +import * as HttpClient from "effect/http/HttpClient"; +import * as Path from "effect/Path"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Schedule from "effect/Schedule"; +import * as Semaphore from "effect/Semaphore"; +import * as Schema from "effect/Schema"; + +import { forkParked } from "../serverActivation.ts"; + +import { + connectOpenShellGateway, + type OpenShellGatewayClient, +} from "@t3tools/openshell/gatewayClient"; +import { + OpenShellGatewayDriver, + OpenShellGatewayDriverError, + type GatewayDriverSpawnOptionsWithCeiling, + type OpenShellDriverSettings, + type OrphanSweepResult, + type WorkerPairingRegistrar, + type FleetWorkerIndex, + type FleetWorkerRow, +} from "@t3tools/openshell/OpenShellGatewayDriver"; + +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as OpenShellFleetDrain from "../environment/OpenShellFleetDrain.ts"; +import * as OpenShellFleetDrainAdapters from "../environment/OpenShellFleetDrainAdapters.ts"; +import * as OpenShellFleetBundleImport from "../environment/OpenShellFleetBundleImport.ts"; +import * as ServerConfig from "../config.ts"; +import * as PeerLinks from "../peer/PeerLinks.ts"; +import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as OpenShellFleetStore from "../environment/OpenShellFleetStore.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as PeerEnvironmentRegistry from "../peer/PeerEnvironmentRegistry.ts"; +import * as PeerSessionResolve from "../peer/PeerSessionResolve.ts"; +import * as EventSink from "../orchestration-v2/EventSink.ts"; +import * as SqlClient from "effect/sql/SqlClient"; + +/** ServerSecretStore slot holding the gateway credential (spec section 4.2). */ +const GATEWAY_CREDENTIAL_SLOT = "openshell-gateway-credential"; + +const isDriverError = Schema.is(OpenShellGatewayDriverError); + +const offHere = "OpenShell fleet management is off here."; +const fleetOperationFailed = "The fleet operation failed."; + +export class OpenShellFleetDisabled extends Schema.TaggedError< + OpenShellFleetDisabled, + { readonly brand: unique symbol } +>()("OpenShellFleetDisabled", { message: Schema.String }) {} + +export class OpenShellFleetSpawnRefused extends Schema.TaggedError< + OpenShellFleetSpawnRefused, + { readonly brand: unique symbol } +>()("OpenShellFleetSpawnRefused", { + reason: Schema.Literals(["runtime_mode_above_ceiling", "unknown_template"]), + message: Schema.String, +}) {} + +export class OpenShellFleetOperationFailed extends Schema.TaggedError< + OpenShellFleetOperationFailed, + { readonly brand: unique symbol } +>()("OpenShellFleetOperationFailed", { + operation: Schema.Literals(["spawn", "remove", "list", "sweep"]), + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +export type OpenShellFleetError = + | OpenShellFleetDisabled + | OpenShellFleetSpawnRefused + | OpenShellFleetOperationFailed; + +/** + * The OpenShell fleet on this managing instance (spec sections 5 and 6). Builds + * the drain executor and the post-drain importer per environment: nothing lost + * outranks cleanup. + */ +export class OpenShellFleetService extends Context.Service< + OpenShellFleetService, + { + readonly spawnWorker: (input: { + readonly template?: string; + readonly runtimeMode?: OrchestratorMcpRuntimeMode; + readonly runtimeModeCeiling: RuntimeMode; + readonly workerClass: OpenShellWorkerClass; + readonly label?: string; + readonly ttlMs?: number; + readonly seedFrom?: string; + readonly clientRequestId?: string; + }) => Effect.Effect; + readonly removeWorker: (input: { + readonly targetEnvironmentId: EnvironmentId; + readonly drainTimeoutMs?: number; + readonly force?: boolean; + readonly clientRequestId?: string; + }) => Effect.Effect; + readonly listWorkers: (input: { + readonly workerClass?: OpenShellWorkerClass; + }) => Effect.Effect, OpenShellFleetError>; + /** One orphan-sweep tick: finalizes vanished rows and retires expired ephemeral workers. */ + readonly sweepWorkers: () => Effect.Effect; + } +>()("t3/environment/OpenShellFleetService") {} + +const toIndexRow = (row: OpenShellFleetStore.FleetEnvironment): FleetWorkerRow => ({ + environmentId: row.environmentId, + state: row.state, + workerClass: row.workerClass, + label: row.label, + sandboxName: row.sandboxName, + sandboxId: row.sandboxId, + drain: row.drain, + expiresAt: row.expiresAt, + lastError: row.lastError, +}); + +const make = Effect.gen(function* () { + const settings = yield* ServerSettings.ServerSettingsService; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const secretResolver = yield* SecretResolverCore.SecretResolver; + const environment = yield* ServerEnvironment.ServerEnvironment; + const registry = yield* PeerEnvironmentRegistry.PeerEnvironmentRegistry; + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + const config = yield* ServerConfig.ServerConfig; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const http = yield* HttpClient.HttpClient; + const links = yield* PeerLinks.PeerLinks; + // The post-drain bundle import writes through the same event sink and import + // table the orchestration projections use, so imported threads are ordinary + // archived threads (spec section 7.4). Optional: hosts that do not build the + // orchestration runtime keep draining, and the import warns on the row. + const maybeEventSink = yield* Effect.serviceOption(EventSink.EventSinkV2); + const maybeSql = yield* Effect.serviceOption(SqlClient.SqlClient); + + // Gateway URL changes take effect after restart; failed initialization stays retryable. + const initialization = yield* Semaphore.make(1); + const cachedDriver = yield* Ref.make>(Option.none()); + + const failDisabled = (message: string) => new OpenShellFleetDisabled({ message }); + + /** A stable message for callers; the underlying detail stays in the cause. */ + const failed = (operation: OpenShellFleetOperationFailed["operation"], cause: unknown) => + new OpenShellFleetOperationFailed({ + operation, + message: fleetOperationFailed, + cause, + }); + + /** One driver call, its failures typed for the MCP tools and the driver's own refusals kept. */ + const callDriver = ( + operation: OpenShellFleetOperationFailed["operation"], + call: () => Promise, + refusal?: (error: OpenShellGatewayDriverError) => OpenShellFleetError | undefined, + ): Effect.Effect => + Effect.tryPromise({ + try: call, + catch: (cause) => { + const refused = isDriverError(cause) ? refusal?.(cause) : undefined; + return refused ?? failed(operation, cause); + }, + }); + + /** The gateway block, only on an enabled managing instance with an address. */ + const gatewaySettings: Effect.Effect< + { gatewayUrl: string; openshell: ServerSettingsValue["openshell"] }, + OpenShellFleetError + > = Effect.gen(function* () { + const current = yield* settings.getSettings.pipe( + Effect.mapError(() => failDisabled("The OpenShell fleet settings could not be read here.")), + ); + const openshell = current.openshell; + if (!current.openshellEnabled || !openshell.managingInstance) { + return yield* failDisabled( + "OpenShell fleet management is off here. Turn it on in Settings → Labs and make this environment the managing instance.", + ); + } + const gatewayUrl = openshell.gatewayUrl; + if (gatewayUrl === undefined || gatewayUrl.trim() === "") { + return yield* failDisabled( + "No OpenShell gateway address is configured. Set it in Settings → Labs.", + ); + } + return { gatewayUrl, openshell }; + }); + + /** + * The gateway credential, decoded for the SDK's token provider. A slot that is + * not set means "no credential"; a read that failed is a typed fleet error, + * never silently treated as unset. + */ + const gatewayCredential = Effect.fn("OpenShellFleetService.gatewayCredential")(function* ( + operation: OpenShellFleetOperationFailed["operation"], + ) { + const stored = yield* secrets + .get(GATEWAY_CREDENTIAL_SLOT) + .pipe(Effect.mapError((cause) => failed(operation, cause))); + return Option.isSome(stored) ? new TextDecoder().decode(stored.value) : undefined; + }); + + const openClient = Effect.fn("OpenShellFleetService.openClient")(function* ( + operation: OpenShellFleetOperationFailed["operation"], + gatewayUrl: string, + credential: string | undefined, + auth: OpenShellGatewayAuth = { mode: "static" }, + ) { + const mtlsConnectOptions = yield* Effect.gen(function* () { + if (auth.mode !== "mtls") return undefined; + const provider = yield* OpenShellGatewayAuthModule.makeGatewayAuthTokenProvider({ + auth, + secrets, + resolver: secretResolver, + }).pipe(Effect.catch((cause: unknown) => failed(operation, cause))); + return "connectOptions" in provider ? provider.connectOptions : undefined; + }).pipe(Effect.catch((cause) => failed(operation, cause))); + const oidcTokenProvider = yield* Effect.gen(function* () { + if (auth.mode === "mtls") return undefined; + if (auth.mode !== "static") { + const provider = yield* OpenShellGatewayAuthModule.makeGatewayAuthTokenProvider({ + auth, + secrets, + resolver: secretResolver, + }).pipe(Effect.catch((cause: unknown) => failed(operation, cause))); + return provider.oidcTokenProvider; + } + // Legacy provisioned bearer (spec section 4.2). + return credential === undefined + ? undefined + : (): Promise => Promise.resolve(credential); + }).pipe(Effect.catch((cause) => failed(operation, cause))); + const client: OpenShellGatewayClient = yield* Effect.tryPromise({ + try: () => + connectOpenShellGateway( + oidcTokenProvider === undefined && mtlsConnectOptions === undefined + ? { gatewayUrl } + : { + gatewayUrl, + ...(oidcTokenProvider === undefined ? {} : { oidcTokenProvider }), + ...(mtlsConnectOptions === undefined ? {} : mtlsConnectOptions), + }, + ), + catch: (cause) => failed(operation, cause), + }); + return client; + }); + + /** Nothing is deleted before its data is here (spec section 7). */ + const drainFor = Effect.fn("OpenShellFleetService.drainFor")(function* ( + client: OpenShellGatewayClient, + ) { + const context = yield* Effect.context(); + // A spawned worker has a fleet delegation session, not an ordinary link, so + // the drain resolves it the same way MCP forwarding does. + const resolvePeer = PeerSessionResolve.resolvePeerSession({ + links, + registry: Option.some(registry), + }); + const linkResolve = (environmentId: EnvironmentId) => + resolvePeer(environmentId).pipe( + Effect.mapError( + (failure) => + new OpenShellFleetDrain.FleetDrainStepError({ + step: "shell", + message: "The fleet worker could not be resolved for drain.", + cause: failure, + }), + ), + ); + const bundleImporter = + Option.isSome(maybeEventSink) && Option.isSome(maybeSql) + ? OpenShellFleetBundleImport.makeFleetBundleImporter({ + stateDir: config.stateDir, + attachmentsDir: config.attachmentsDir, + fs, + path, + now: Clock.currentTimeMillis, + eventSink: maybeEventSink.value, + store: OpenShellFleetBundleImport.sqliteFleetDrainImportStore(maybeSql.value), + }) + : null; + const executor = OpenShellFleetDrain.makeDrainExecutor({ + source: { + ...OpenShellFleetDrainAdapters.linkDrainSource({ resolve: linkResolve, http }), + // The worker's environment HTTP API serves no attachment bytes, so + // attachment files travel over gateway exec (see execDrainAttachments). + attachments: OpenShellFleetDrainAdapters.execDrainAttachments({ + exec: (sandboxName, command) => + Effect.tryPromise({ + try: () => client.exec(sandboxName, command), + catch: (cause) => + new OpenShellFleetDrain.FleetDrainStepError({ + step: "attachments", + message: "The worker's attachments could not be read.", + cause, + }), + }), + }), + }, + sink: yield* OpenShellFleetDrainAdapters.fileDrainSink({ + stateDir: config.stateDir, + fs, + path, + }), + git: OpenShellFleetDrainAdapters.execDrainGitBundle({ + exec: (sandboxName, command) => + Effect.tryPromise({ + try: () => client.exec(sandboxName, command), + catch: (cause) => + new OpenShellFleetDrain.FleetDrainStepError({ + step: "bundle", + message: "The checkout bundle could not be exported.", + cause, + }), + }), + }), + now: Clock.currentTimeMillis, + }); + const runBundleImport = (args: { environmentId: EnvironmentId; drainId: string }) => + bundleImporter === null + ? Promise.resolve({ + imported: 0, + skipped: 0, + warnings: [ + "The orchestration runtime is not built here, so drained bundles were not imported.", + ], + }) + : Effect.runPromiseWith(context)( + bundleImporter.importBundle({ + environmentId: args.environmentId, + drainId: args.drainId, + }), + ); + return { + port: OpenShellFleetDrain.asDriverPort(executor, (effect) => + Effect.runPromiseWith(context)(effect), + ), + bundleImporter: { import: runBundleImport }, + }; + }); + + const buildDriver = Effect.fn("OpenShellFleetService.buildDriver")(function* ( + operation: OpenShellFleetOperationFailed["operation"], + ) { + const { gatewayUrl, openshell } = yield* gatewaySettings; + const auth = openshell.auth ?? { mode: "static" }; + // Only static auth reads the legacy gateway slot. A working mTLS/OAuth + // provider must not fail because that optional legacy secret is unreadable. + const credential = auth.mode === "static" ? yield* gatewayCredential(operation) : undefined; + const client = yield* openClient(operation, gatewayUrl, credential, auth); + const context = yield* Effect.context(); + // The driver's collaborators are Promise-shaped because the OpenShell SDK + // is; they run these Effects on the surrounding services rather than a + // fresh runtime, so the sqlite store and registry stay the same instances. + const run = (effect: Effect.Effect): Promise => + Effect.runPromiseWith(context)(effect); + const registrar: WorkerPairingRegistrar = { + register: (input) => run(registry.register(input)), + exchangeBootstrap: (input) => run(registry.exchangeBootstrap(input)), + }; + const index: FleetWorkerIndex = { + upsert: (input) => run(store.upsert(input)).then(toIndexRow), + setState: (environmentId, state, lastError) => + run(store.setState(environmentId, state, lastError)).then((row) => + Option.isSome(row) ? toIndexRow(row.value) : null, + ), + setDrain: (environmentId, drain) => + run(store.setDrain(environmentId, drain)).then((row) => + Option.isSome(row) ? toIndexRow(row.value) : null, + ), + get: (environmentId) => + run(store.get(environmentId)).then((row) => + Option.isSome(row) ? toIndexRow(row.value) : null, + ), + list: () => run(store.list).then((rows) => rows.map(toIndexRow)), + remove: (environmentId) => run(store.remove(environmentId)), + }; + const selfId = yield* environment.getEnvironmentId.pipe(Effect.option); + const driverSettings: OpenShellDriverSettings = { + ...(openshell.defaultSandboxTemplate === undefined + ? {} + : { defaultSandboxTemplate: openshell.defaultSandboxTemplate }), + ...(openshell.workerDefaults === undefined + ? {} + : { workerDefaults: openshell.workerDefaults }), + ...(openshell.sharedCache === undefined ? {} : { sharedCache: openshell.sharedCache }), + }; + const drain = yield* drainFor(client); + return new OpenShellGatewayDriver({ + client, + settings: driverSettings, + ...(Option.isSome(selfId) ? { managingEnvironmentId: selfId.value } : {}), + registrar, + index, + drainExecutor: drain.port, + bundleImporter: drain.bundleImporter, + // Only a caller's clientRequestId makes a spawn repeatable. Without one + // the driver mints a fresh key, so independent spawns never collapse onto + // one sandbox and overwrite its policy. + }); + }); + + /** A cached build is only reused while the settings guard passes on every call. */ + const driverFor = Effect.fn("OpenShellFleetService.driverFor")(function* ( + operation: OpenShellFleetOperationFailed["operation"], + ) { + const cached = yield* Ref.get(cachedDriver); + if (Option.isSome(cached)) return cached.value; + const driver = yield* buildDriver(operation); + yield* Ref.set(cachedDriver, Option.some(driver)); + return driver; + }, initialization.withPermit); + + const spawnWorker: OpenShellFleetService["Service"]["spawnWorker"] = Effect.fn( + "OpenShellFleetService.spawnWorker", + )(function* (input): Effect.fn.Return { + // `inherit` follows the caller; an omitted mode falls back to the conservative + // default the driver documents, never to the caller's ceiling (spec section 5.2). + const runtimeMode = + input.runtimeMode === undefined + ? "approval-required" + : input.runtimeMode === "inherit" + ? input.runtimeModeCeiling + : input.runtimeMode; + if (isRuntimeModeAbove(runtimeMode, input.runtimeModeCeiling)) { + return yield* new OpenShellFleetSpawnRefused({ + reason: "runtime_mode_above_ceiling", + message: `Worker policy ${runtimeMode} is broader than this caller's ${input.runtimeModeCeiling} ceiling.`, + }); + } + // The guard runs per call, so disabling OpenShell after first use also + // stops spawns; the driver itself is cached and would outlive a flip. + yield* gatewaySettings; + const driver = yield* driverFor("spawn"); + const options: GatewayDriverSpawnOptionsWithCeiling = { + ...(input.clientRequestId === undefined ? {} : { idempotencyKey: input.clientRequestId }), + runtimeModeCeiling: input.runtimeModeCeiling, + }; + return yield* callDriver( + "spawn", + () => + driver.spawn( + { + ...(input.template === undefined ? {} : { template: input.template }), + runtimeMode, + workerClass: input.workerClass, + ...(input.label === undefined ? {} : { label: input.label }), + ...(input.ttlMs === undefined ? {} : { ttlMs: input.ttlMs }), + ...(input.seedFrom === undefined ? {} : { seedFrom: input.seedFrom }), + }, + options, + ), + (error) => + error.reason === "runtime_mode_above_ceiling" || error.reason === "unknown_template" + ? new OpenShellFleetSpawnRefused({ reason: error.reason, message: error.message }) + : undefined, + ); + }); + + const removeWorker: OpenShellFleetService["Service"]["removeWorker"] = Effect.fn( + "OpenShellFleetService.removeWorker", + )(function* (input): Effect.fn.Return { + yield* gatewaySettings; + const driver = yield* driverFor("remove"); + const options: GatewayDriverRemoveOptions = { + ...(input.drainTimeoutMs === undefined ? {} : { timeoutMs: input.drainTimeoutMs }), + ...(input.force === undefined ? {} : { force: input.force }), + ...(input.clientRequestId === undefined ? {} : { idempotencyKey: input.clientRequestId }), + }; + return yield* callDriver("remove", () => driver.remove(input.targetEnvironmentId, options)); + }); + + const listWorkers: OpenShellFleetService["Service"]["listWorkers"] = Effect.fn( + "OpenShellFleetService.listWorkers", + )(function* (input): Effect.fn.Return< + ReadonlyArray, + OpenShellFleetError + > { + yield* gatewaySettings; + const driver = yield* driverFor("list"); + return yield* callDriver("list", () => + driver.status(input.workerClass === undefined ? {} : { workerClass: input.workerClass }), + ); + }); + + const sweepWorkers: OpenShellFleetService["Service"]["sweepWorkers"] = Effect.fn( + "OpenShellFleetService.sweepWorkers", + )(function* (): Effect.fn.Return { + // The per-call gate makes an off fleet a no-op tick rather than an error, so + // the sweep below can keep its cadence through settings changes. + yield* gatewaySettings; + const driver = yield* driverFor("sweep"); + return yield* callDriver("sweep", () => driver.orphanSweep()); + }); + + return OpenShellFleetService.of({ spawnWorker, removeWorker, listWorkers, sweepWorkers }); +}); + +export const layer = Layer.effect(OpenShellFleetService, make); + +/** + * Self-retiring ephemeral workers (spec section 5.6): one sweep every 5 minutes + * retires expired ephemeral workers drain-first and finalizes vanished rows. + * Refused drains keep their sandbox, so the next tick retries them. + */ +export const layerSweep = Layer.effectDiscard( + forkParked( + OpenShellFleetService.pipe( + Effect.flatMap((fleet) => fleet.sweepWorkers()), + Effect.catch((cause) => + cause._tag === "OpenShellFleetDisabled" + ? Effect.void + : Effect.logWarning("The OpenShell fleet sweep did not complete", { cause }), + ), + Effect.repeat(Schedule.spaced(Duration.minutes(5))), + ), + ), +); + +/** + * No fleet here: tests and builds that do not run a gateway. Every verb fails + * typed, so a tool that reaches it reports "off" rather than inventing workers. + */ +export const layerDisabled: Layer.Layer = Layer.succeed( + OpenShellFleetService, + OpenShellFleetService.of({ + spawnWorker: Effect.fn("OpenShellFleetService.spawnWorker.disabled")(() => + Effect.fail(new OpenShellFleetDisabled({ message: offHere })), + ), + removeWorker: Effect.fn("OpenShellFleetService.removeWorker.disabled")(() => + Effect.fail(new OpenShellFleetDisabled({ message: offHere })), + ), + listWorkers: Effect.fn("OpenShellFleetService.listWorkers.disabled")(() => + Effect.fail(new OpenShellFleetDisabled({ message: offHere })), + ), + sweepWorkers: Effect.fn("OpenShellFleetService.sweepWorkers.disabled")(() => + Effect.fail(new OpenShellFleetDisabled({ message: offHere })), + ), + }), +); diff --git a/apps/server/src/environment/OpenShellFleetStore.test.ts b/apps/server/src/environment/OpenShellFleetStore.test.ts new file mode 100644 index 0000000000000..20e212404a949 --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetStore.test.ts @@ -0,0 +1,133 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { EnvironmentId } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; + +import * as OpenShellFleetStore from "./OpenShellFleetStore.ts"; +import * as Sqlite from "../persistence/Sqlite.ts"; + +const workerRow = { + environmentId: EnvironmentId.make("environment-worker-1"), + state: "ready" as const, + workerClass: "ephemeral" as const, + label: "Worker One", + sandboxName: "t3-worker-7f3a", + sandboxId: "sandbox-1", + expiresAt: 1_800_000_000_000, +}; + +const drain = { + drainId: "drain:environment-worker-1:1", + threads: { total: 2, acked: 1, failed: 0 }, + worktrees: { total: 1, acked: 0, failed: 0 }, + startedAt: 1_700_000_000_000, + finishedAt: null, +}; + +const storeLayer = (database: Layer.Layer) => + OpenShellFleetStore.layer.pipe(Layer.provideMerge(database)); + +describe("OpenShellFleetStore", () => { + it.effect("keeps one row per environment across state and drain updates", () => + Effect.gen(function* () { + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + const created = yield* store.upsert(workerRow); + assert.equal(created.state, "ready"); + assert.isNull(created.drain); + assert.equal(created.expiresAt, workerRow.expiresAt); + assert.isTrue(DateTime.isDateTime(created.updatedAt)); + + const draining = yield* store.setState(workerRow.environmentId, "draining"); + assert.equal(Option.getOrThrow(draining).state, "draining"); + + const withDrain = yield* store.setDrain(workerRow.environmentId, drain); + assert.deepEqual(Option.getOrThrow(withDrain).drain, drain); + + // The drain clears once every item acked and the deletion settled. + const deleted = yield* store.setState(workerRow.environmentId, "deleting"); + assert.equal(Option.getOrThrow(deleted).state, "deleting"); + const cleared = yield* store.setDrain(workerRow.environmentId, null); + assert.isNull(Option.getOrThrow(cleared).drain); + + assert.equal((yield* store.list).length, 1); + assert.isTrue(yield* store.remove(workerRow.environmentId)); + assert.isTrue(Option.isNone(yield* store.get(workerRow.environmentId))); + assert.isFalse(yield* store.remove(workerRow.environmentId)); + }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))), + ); + + it.effect("a failed drain keeps its last error with the row", () => + Effect.gen(function* () { + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + yield* store.upsert(workerRow); + const failed = yield* store.setState( + workerRow.environmentId, + "failed", + "drain_timeout: 1 worktree unacked", + ); + assert.equal(Option.getOrThrow(failed).lastError, "drain_timeout: 1 worktree unacked"); + // Nothing lost outranks cleanup: a failed row is never interrupted-by-restart. + assert.deepEqual(yield* store.interrupted, []); + }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))), + ); + + it.effect("reports rows a restart interrupted, and only those", () => + Effect.gen(function* () { + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + yield* store.upsert(workerRow); + yield* store.upsert({ + ...workerRow, + environmentId: EnvironmentId.make("environment-worker-2"), + state: "draining" as const, + label: "Worker Two", + }); + const interrupted = yield* store.interrupted; + assert.deepEqual( + interrupted.map((row) => row.environmentId), + [EnvironmentId.make("environment-worker-2")], + ); + assert.equal(interrupted[0]?.state, "draining"); + }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))), + ); + + it.effect("state survives a restart of the managing instance", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const dir = yield* fs.makeTempDirectory({ prefix: "t3-fleet-store-" }); + const dbPath = path.join(dir, "statev2.sqlite"); + const database = Sqlite.layerFromPath(dbPath).pipe( + Layer.provide(NodeServices.layer), + Layer.fresh, + ); + + // One instance writes and exits. + yield* Effect.gen(function* () { + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + yield* store.upsert(workerRow); + yield* store.setState(workerRow.environmentId, "draining"); + yield* store.setDrain(workerRow.environmentId, drain); + }).pipe(Effect.provide(storeLayer(database))); + + // The next instance reads the rows back at startup. + yield* Effect.gen(function* () { + const store = yield* OpenShellFleetStore.OpenShellFleetStore; + const recovered = Option.getOrThrow(yield* store.get(workerRow.environmentId)); + assert.equal(recovered.state, "draining"); + assert.deepEqual(recovered.drain, drain); + assert.deepEqual( + (yield* store.interrupted).map((row) => row.environmentId), + [workerRow.environmentId], + ); + }).pipe(Effect.provide(storeLayer(database))); + + yield* fs.remove(dir, { recursive: true }); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); diff --git a/apps/server/src/environment/OpenShellFleetStore.ts b/apps/server/src/environment/OpenShellFleetStore.ts new file mode 100644 index 0000000000000..ebacf9f21971d --- /dev/null +++ b/apps/server/src/environment/OpenShellFleetStore.ts @@ -0,0 +1,286 @@ +import { + EnvironmentId, + OpenShellDrainProgress, + OpenShellFleetState, + OpenShellWorkerClass, +} from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; + +const decodeDrain = Schema.decodeUnknownEffect( + Schema.NullOr(Schema.fromJsonString(OpenShellDrainProgress)), +); +const encodeDrain = Schema.encodeEffect( + Schema.NullOr(Schema.fromJsonString(OpenShellDrainProgress)), +); + +/** One gateway-managed environment as the fleet lifecycle store keeps it. */ +export interface FleetEnvironment { + readonly environmentId: EnvironmentId; + readonly state: OpenShellFleetState; + readonly workerClass: OpenShellWorkerClass; + readonly label: string | null; + readonly sandboxName: string | null; + readonly sandboxId: string | null; + /** Aggregate drain journal progress; per-item acks live with the drain protocol. */ + readonly drain: OpenShellDrainProgress | null; + /** Ephemeral workers: epoch-ms retirement deadline managed by this instance. */ + readonly expiresAt: number | null; + readonly lastError: string | null; + readonly updatedAt: DateTime.Utc; +} + +export class OpenShellFleetStoreError extends Schema.TaggedError< + OpenShellFleetStoreError, + { readonly brand: unique symbol } +>()("OpenShellFleetStoreError", { + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +const failStorage = (cause: unknown) => + Effect.logWarning("OpenShell fleet state storage failed").pipe( + Effect.andThen( + Effect.fail( + new OpenShellFleetStoreError({ + message: + "The fleet environment state could not be read or saved here. The server log has details.", + cause, + }), + ), + ), + ); + +/** + * Lifecycle states a restart interrupts. The spec's reconciler (section 5.5) + * settles them against the gateway once a driver exists; until then the store + * surfaces them, so nothing is silently dropped and nothing is deleted. + */ +const interruptedStates: ReadonlyArray = [ + "provisioning", + "pairing", + "draining", + "deleting", +]; + +/** + * Fleet lifecycle state for gateway-managed environments (spec section 5.3): + * the row, not the gateway phase, is this instance's truth, so state survives + * a restart. Writes are upserts keyed by environment id. + */ +export class OpenShellFleetStore extends Context.Service< + OpenShellFleetStore, + { + readonly upsert: (input: { + readonly environmentId: EnvironmentId; + readonly state: OpenShellFleetState; + readonly workerClass: OpenShellWorkerClass; + readonly label?: string; + readonly sandboxName?: string; + readonly sandboxId?: string; + readonly expiresAt?: number; + }) => Effect.Effect; + readonly setState: ( + environmentId: EnvironmentId, + state: OpenShellFleetState, + lastError?: string, + ) => Effect.Effect, OpenShellFleetStoreError>; + readonly setDrain: ( + environmentId: EnvironmentId, + drain: OpenShellDrainProgress | null, + ) => Effect.Effect, OpenShellFleetStoreError>; + readonly get: ( + environmentId: EnvironmentId, + ) => Effect.Effect, OpenShellFleetStoreError>; + readonly list: Effect.Effect, OpenShellFleetStoreError>; + /** + * Rows the restart interrupted (spec section 5.5): still owed their drain + * or deletion. OPEN-QUESTION (spec section 5.5): adopt-or-sweep lands + * with the gateway driver; the default keeps every interrupted row. + */ + readonly interrupted: Effect.Effect, OpenShellFleetStoreError>; + /** Forgets the row; the caller drains first (spec section 7) and deletes the sandbox itself. */ + readonly remove: ( + environmentId: EnvironmentId, + ) => Effect.Effect; + } +>()("t3/environment/OpenShellFleetStore") {} + +interface FleetRow { + readonly environment_id: string; + readonly state: string; + readonly worker_class: string; + readonly label: string | null; + readonly sandbox_name: string | null; + readonly sandbox_id: string | null; + readonly drain_json: string | null; + readonly expires_at: number | null; + readonly last_error: string | null; + readonly updated_at: string; +} + +const decodeState = Schema.decodeUnknownEffect(OpenShellFleetState); +const decodeWorkerClass = Schema.decodeUnknownEffect(Schema.Literals(["ephemeral", "persistent"])); + +const toFleetEnvironment = Effect.fn("OpenShellFleetStore.toFleetEnvironment")(function* ( + row: FleetRow, +) { + const state = yield* decodeState(row.state).pipe(Effect.catch(failStorage)); + const workerClass = yield* decodeWorkerClass(row.worker_class).pipe(Effect.catch(failStorage)); + return { + environmentId: EnvironmentId.make(row.environment_id), + state, + workerClass, + label: row.label, + sandboxName: row.sandbox_name, + sandboxId: row.sandbox_id, + drain: yield* decodeDrain(row.drain_json).pipe(Effect.catch(failStorage)), + expiresAt: row.expires_at, + lastError: row.last_error, + updatedAt: DateTime.makeUnsafe(row.updated_at), + } satisfies FleetEnvironment; +}); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + const readRows = sql` + SELECT * FROM openshell_fleet_environments ORDER BY environment_id + `.pipe(NodeSqliteClient.readOnly, Effect.catch(failStorage)); + + const readRow = (environmentId: EnvironmentId) => + sql` + SELECT * FROM openshell_fleet_environments WHERE environment_id = ${environmentId} + `.pipe( + NodeSqliteClient.readOnly, + Effect.catch(failStorage), + Effect.map((rows) => Option.fromNullishOr(rows[0])), + Effect.withSpan("OpenShellFleetStore.readRow"), + ); + + // Crash recovery: what a restart interrupted, read back once at startup so + // the row states this instance owed are visible without a driver (spec + // section 5.5). Nothing is mutated: a kept state is a kept obligation. + yield* Effect.forEach(yield* readRows, toFleetEnvironment).pipe( + Effect.tap((rows) => { + const interruptedRows = rows.filter((row) => interruptedStates.includes(row.state)); + return interruptedRows.length > 0 + ? Effect.logWarning("Fleet environments interrupted by this restart keep their state", { + environments: interruptedRows.map((row) => ({ + environmentId: row.environmentId, + state: row.state, + })), + }) + : Effect.void; + }), + Effect.catch(failStorage), + ); + + const upsert: OpenShellFleetStore["Service"]["upsert"] = Effect.fn("OpenShellFleetStore.upsert")( + function* (input) { + const now = yield* DateTime.now; + yield* sql` + INSERT INTO openshell_fleet_environments ( + environment_id, state, worker_class, label, sandbox_name, sandbox_id, + drain_json, expires_at, last_error, updated_at + ) VALUES ( + ${input.environmentId}, ${input.state}, ${input.workerClass}, + ${input.label ?? null}, ${input.sandboxName ?? null}, ${input.sandboxId ?? null}, + NULL, ${input.expiresAt ?? null}, NULL, ${DateTime.formatIso(now)} + ) + ON CONFLICT (environment_id) DO UPDATE SET + state = excluded.state, + worker_class = excluded.worker_class, + label = excluded.label, + sandbox_name = excluded.sandbox_name, + sandbox_id = excluded.sandbox_id, + expires_at = excluded.expires_at, + last_error = excluded.last_error, + updated_at = excluded.updated_at + `.pipe(Effect.catch(failStorage)); + return Option.getOrThrow(yield* get(input.environmentId)); + }, + ); + + const get: OpenShellFleetStore["Service"]["get"] = (environmentId) => + readRow(environmentId).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeedNone, + onSome: (row) => Effect.asSome(toFleetEnvironment(row)), + }), + ), + Effect.withSpan("OpenShellFleetStore.get"), + ); + + const setState: OpenShellFleetStore["Service"]["setState"] = (environmentId, state, lastError) => + DateTime.now.pipe( + Effect.flatMap((now) => + sql<{ environment_id: string }>` + UPDATE openshell_fleet_environments + SET state = ${state}, last_error = ${lastError ?? null}, + updated_at = ${DateTime.formatIso(now)} + WHERE environment_id = ${environmentId} + RETURNING environment_id + `.pipe(Effect.catch(failStorage)), + ), + Effect.flatMap((updated) => (updated.length > 0 ? get(environmentId) : Effect.succeedNone)), + Effect.withSpan("OpenShellFleetStore.setState"), + ); + + const setDrain: OpenShellFleetStore["Service"]["setDrain"] = (environmentId, drain) => + encodeDrain(drain).pipe( + Effect.catch(failStorage), + Effect.flatMap((drainJson) => + DateTime.now.pipe( + Effect.flatMap((now) => + sql<{ environment_id: string }>` + UPDATE openshell_fleet_environments + SET drain_json = ${drainJson}, updated_at = ${DateTime.formatIso(now)} + WHERE environment_id = ${environmentId} + RETURNING environment_id + `.pipe(Effect.catch(failStorage)), + ), + ), + ), + Effect.flatMap((updated) => (updated.length > 0 ? get(environmentId) : Effect.succeedNone)), + Effect.withSpan("OpenShellFleetStore.setDrain"), + ); + + const list: OpenShellFleetStore["Service"]["list"] = readRows.pipe( + Effect.flatMap((rows) => Effect.forEach(rows, toFleetEnvironment)), + Effect.withSpan("OpenShellFleetStore.list"), + ); + + const interruptedRows = Effect.map(list, (rows) => + rows.filter((row) => interruptedStates.includes(row.state)), + ); + + const remove: OpenShellFleetStore["Service"]["remove"] = (environmentId) => + sql<{ environment_id: string }>` + DELETE FROM openshell_fleet_environments WHERE environment_id = ${environmentId} + RETURNING environment_id + `.pipe( + Effect.map((deleted) => deleted.length > 0), + Effect.catch(failStorage), + Effect.withSpan("OpenShellFleetStore.remove"), + ); + + return OpenShellFleetStore.of({ + upsert, + setState, + setDrain, + get, + list, + interrupted: interruptedRows, + remove, + }); +}); + +export const layer = Layer.effect(OpenShellFleetStore, make); diff --git a/apps/server/src/environment/OpenShellGatewayAuth.ts b/apps/server/src/environment/OpenShellGatewayAuth.ts new file mode 100644 index 0000000000000..9d46cafb5dd85 --- /dev/null +++ b/apps/server/src/environment/OpenShellGatewayAuth.ts @@ -0,0 +1,260 @@ +// @effect-diagnostics globalDate:off - the token provider is a Promise-shaped +// collaborator of the SDK (like the driver and the smoke script), so its cache +// checks wall-clock time outside the Effect runtime. +import type { OpenShellGatewayAuth, SettingsSecretValue } from "@t3tools/contracts"; +import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import * as Schema from "effect/Schema"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts"; +import { openshellGatewayAuthSecretName } from "../settingsSecrets.ts"; + +/** + * Turns the settings' `OpenShellGatewayAuth` into the SDK's `oidcTokenProvider` + * seam — an async bearer getter the client renews through. Secret literals live + * in the server secret store (settings JSON keeps the redacted placeholder) and + * secret-provider references are read through the resolver (spec section 4.2). + */ + +const TOKEN_EXPIRY_MARGIN_MS = 60_000; +const REDACTED = "\u2022\u2022\u2022\u2022\u2022\u2022"; +const GATEWAY_CREDENTIAL_SLOT = "openshell-gateway-credential"; + +export interface GatewayAuthTokenProviderDeps { + readonly auth: OpenShellGatewayAuth; + readonly secrets: ServerSecretStore.ServerSecretStore["Service"]; + readonly resolver: SecretResolverCore.SecretResolver["Service"]; + /** Overridable for tests. */ + readonly fetchImpl?: typeof fetch; +} + +interface CachedToken { + readonly token: string; + readonly expiresAtMs: number; +} + +/** Gateway auth failures: typed so a fleet operation maps them to its own error. */ +export class GatewayAuthError extends Schema.TaggedError()("GatewayAuthError", { + message: Schema.String, + cause: Schema.optional(Schema.Defect()), +}) {} + +const isRedacted = (value: SettingsSecretValue): boolean => + typeof value === "string" && value === REDACTED; + +export const makeGatewayAuthTokenProvider = Effect.fn("OpenShellGatewayAuth.tokenProvider")( + function* (deps: GatewayAuthTokenProviderDeps) { + const { auth, secrets, resolver, fetchImpl } = deps; + const doFetch = fetchImpl ?? fetch; + const decoder = new TextDecoder(); + /** Promise collaborators run with the same services as this Effect. */ + const runtime = yield* Effect.context(); + + const resolveSecretField = Effect.fn("OpenShellGatewayAuth.resolveSecretField")(function* ( + field: string, + value: SettingsSecretValue, + ) { + if (isRedacted(value)) { + const stored = yield* secrets.get(openshellGatewayAuthSecretName(field)); + const secret = Option.isSome(stored) ? decoder.decode(stored.value) : ""; + if (secret === "") + return yield* new GatewayAuthError({ + message: `The gateway auth secret "${field}" is not set.`, + }); + return secret; + } + const secret = yield* requireSettingsSecret(resolver, value).pipe( + Effect.catch( + (cause) => + new GatewayAuthError({ + message: `The gateway auth secret "${field}" could not be read.`, + cause, + }), + ), + ); + if (secret.trim() === "") + return yield* new GatewayAuthError({ + message: `The gateway auth secret "${field}" is empty.`, + }); + return secret; + }); + + let cached: CachedToken | undefined; + let tokenEndpoint: string | undefined; + let refreshInFlight: Promise | undefined; + const usable = (now: number): boolean => + cached !== undefined && cached.expiresAtMs - TOKEN_EXPIRY_MARGIN_MS > now; + const currentToken = (): string => { + const snapshot = cached; + if (snapshot === undefined || !usable(Date.now())) + throw new Error("The gateway token needs renewal."); + return snapshot.token; + }; + + if (auth.mode === "mtls") { + // Lane 1 auth docs: the SDK connect accepts caCert + clientCert + + // clientKey for client-CA gateways. Field shapes still need + // type-level verification against SDK 0.1.7. + const clientKey = yield* resolveSecretField("clientKey", auth.clientKey); + return { + connectOptions: { + ...(auth.caCertPem === undefined ? {} : { caCert: auth.caCertPem }), + clientCert: auth.clientCertPem, + clientKey, + }, + }; + } + + /** OAuth2 client-credentials or refresh-token grant against the OIDC issuer. */ + const fetchToken = async ( + issuer: string, + audience: string | undefined, + form: URLSearchParams, + ): Promise => { + if (tokenEndpoint === undefined) { + // Preserve tenant paths: discovery is relative to the issuer, not its origin. + const issuerUrl = new URL(issuer); + const issuerBase = issuerUrl.pathname.endsWith("/") ? issuer : `${issuer}/`; + const discovery = await doFetch(new URL(".well-known/openid-configuration", issuerBase), { + redirect: "error", + }); + if (!discovery.ok) + throw new Error(`OIDC discovery at ${issuer} failed with status ${discovery.status}.`); + const document = (await discovery.json()) as { + readonly issuer?: unknown; + readonly token_endpoint?: unknown; + }; + if (typeof document.token_endpoint !== "string" || document.token_endpoint === "") + throw new Error(`OIDC discovery at ${issuer} has no token endpoint.`); + const tokenUrl = new URL(document.token_endpoint); + const secureEndpoint = (url: URL) => + url.protocol === "https:" || + (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname)); + if (!secureEndpoint(issuerUrl) || !secureEndpoint(tokenUrl)) + throw new Error("OIDC discovery requires HTTPS endpoints outside local development."); + if ( + typeof document.issuer === "string" && + document.issuer.replace(/\/$/, "") !== issuer.replace(/\/$/, "") + ) + throw new Error(`OIDC discovery at ${issuer} identifies a different issuer.`); + tokenEndpoint = document.token_endpoint; + } + const response = await doFetch(tokenEndpoint, { + redirect: "error", + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: form.toString(), + }); + if (!response.ok) throw new Error(`The gateway token endpoint answered ${response.status}.`); + const payload = (await response.json()) as { + readonly access_token?: unknown; + readonly expires_in?: unknown; + readonly refresh_token?: unknown; + }; + if (typeof payload.access_token !== "string" || payload.access_token === "") + throw new Error("The gateway token endpoint returned no access token."); + const lifetime = + typeof payload.expires_in === "number" && payload.expires_in > 0 + ? payload.expires_in * 1000 + : 300_000; + cached = { token: payload.access_token, expiresAtMs: Date.now() + lifetime }; + // Rotation is opt-in at the protocol level. For a stored literal, rotate + // the server slot immediately; an external secret owner is authoritative + // and cannot be silently overwritten here. + if ( + auth.mode === "oidc-pkce" && + typeof payload.refresh_token === "string" && + payload.refresh_token !== "" && + isRedacted(auth.refreshToken) + ) { + await Effect.runPromiseWith(runtime)( + secrets + .set( + openshellGatewayAuthSecretName("refreshToken"), + new TextEncoder().encode(payload.refresh_token), + ) + .pipe( + Effect.catch( + (cause) => + new GatewayAuthError({ + message: + "The issuer rotated the gateway refresh token, but its server slot could not be updated.", + cause, + }), + ), + ), + ); + } + return cached.token; + }; + + if (auth.mode === "client-credentials") { + const clientSecret = yield* resolveSecretField("clientSecret", auth.clientSecret); + const issuer = auth.issuer; + const audience = auth.audience; + return { + oidcTokenProvider: (): Promise => { + if (usable(Date.now())) return Promise.resolve(currentToken()); + const form = new URLSearchParams({ + grant_type: "client_credentials", + client_id: auth.clientId, + client_secret: clientSecret, + }); + if (audience !== undefined) form.set("audience", audience); + return fetchToken(issuer, audience, form); + }, + }; + } + + if (auth.mode === "oidc-pkce") { + // The desktop performed the browser login; the server holds the refresh + // token and renews with the refresh-token grant (spec section 4.1). + const refreshToken = yield* resolveSecretField("refreshToken", auth.refreshToken); + const configuredIssuer = auth.issuer; + if (configuredIssuer === undefined || configuredIssuer.trim() === "") { + return yield* new GatewayAuthError({ + message: "OIDC refresh authentication requires an issuer.", + }); + } + const issuer = configuredIssuer; + const audience = auth.audience; + return { + oidcTokenProvider: (): Promise => { + if (usable(Date.now())) return Promise.resolve(currentToken()); + // OAuth refresh codes are single-use: serialize callers so concurrent + // operations cannot spend the same token twice. + refreshInFlight ??= (async () => { + try { + const form = new URLSearchParams({ + grant_type: "refresh_token", + refresh_token: refreshToken, + }); + if (issuer !== undefined) form.set("issuer", issuer); + if (audience !== undefined) form.set("audience", audience); + return await fetchToken(issuer, audience, form); + } finally { + refreshInFlight = undefined; + } + })(); + return refreshInFlight; + }, + }; + } + + // static + // Legacy provisioned bearer: read the slot on every token callback so a + // provisioner can replace an expired credential without rebuilding driver. + const oidcTokenProvider = (): Promise => + Effect.runPromiseWith(runtime)( + secrets + .get(GATEWAY_CREDENTIAL_SLOT) + .pipe( + Effect.map((stored) => (Option.isSome(stored) ? decoder.decode(stored.value) : "")), + ), + ); + return { oidcTokenProvider }; + }, +); diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts index e5d47410817f2..c4290d295033d 100644 --- a/apps/server/src/mcp/McpHttpServer.test.ts +++ b/apps/server/src/mcp/McpHttpServer.test.ts @@ -60,6 +60,7 @@ import * as McpToolAccessTestkit from "./McpToolAccess.testkit.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as McpSessionRegistry from "./McpSessionRegistry.ts"; import * as OrchestratorMcpService from "./OrchestratorMcpService.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts"; import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; import { PreviewToolkit } from "./toolkits/preview/tools.ts"; @@ -334,6 +335,7 @@ const layerRecordingTools = (calls: Array) => { Layer.mergeAll( McpToolAccessTestkit.liveThreadsLayer, Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({}), + Layer.mock(OpenShellFleetService.OpenShellFleetService)({}), Layer.mock(ProjectService.ProjectService)({}), Layer.mock(PreviewManager.PreviewManager)({}), Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}), diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts index d7bea75493b73..c31aaeadf67b3 100644 --- a/apps/server/src/mcp/McpHttpServer.ts +++ b/apps/server/src/mcp/McpHttpServer.ts @@ -282,6 +282,17 @@ const makeMcpAuthMiddleware = Effect.gen(function* () { Effect.map((settings) => settings.linkedEnvironmentsEnabled), Effect.orElseSucceed(() => false), ); + // The fleet tools cost every agent schema bytes, so they are hidden where no + // fleet can be managed rather than failing on call. + const fleetAvailable = serverSettings.getSettings.pipe( + Effect.map( + (settings) => + settings.openshellEnabled && + settings.openshell.managingInstance && + (settings.openshell.gatewayUrl ?? "").trim() !== "", + ), + Effect.orElseSucceed(() => false), + ); const pullRequests = yield* Effect.serviceOption(PullRequestService.PullRequestService); const projections = yield* Effect.serviceOption(ProjectionStore.ProjectionStoreV2); // Every request reads it, and a remote added mid-thread shows the tools within a minute. @@ -337,7 +348,13 @@ const makeMcpAuthMiddleware = Effect.gen(function* () { limited.thread === undefined ? undefined : yield* Cache.get(forgeOfThread, limited.thread.threadId); - return { ...limited, linkedEnvironments, origin, ...(forge === undefined ? {} : { forge }) }; + return { + ...limited, + linkedEnvironments, + origin, + fleetAvailable: yield* fleetAvailable, + ...(forge === undefined ? {} : { forge }), + }; }); // Untraced: a span here would wrap the tool call; the HTTP server span already covers it. return Effect.fnUntraced(function* (httpEffect) { diff --git a/apps/server/src/mcp/McpInvocationContext.test.ts b/apps/server/src/mcp/McpInvocationContext.test.ts index 96c71fcd773f9..00e8e0af52a06 100644 --- a/apps/server/src/mcp/McpInvocationContext.test.ts +++ b/apps/server/src/mcp/McpInvocationContext.test.ts @@ -7,8 +7,11 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; +import * as Context from "effect/Context"; +import { McpSchema } from "effect/ai"; import * as McpInvocationContext from "./McpInvocationContext.ts"; +import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts"; it.effect("reports other missing capabilities with the neutral error", () => { const invocation: McpInvocationContext.McpInvocationScope = { @@ -95,3 +98,45 @@ it("acts at an administrator's ceiling for a client approved above it", () => { expect(modes(scope("auto-accept-edits"), "auto").runtimeMode).toBe("auto-accept-edits"); expect(modes(scope("full-access")).runtimeMode).toBe("full-access"); }); + +it.effect("hides fleet tools unless fleet management is explicitly available", () => { + const scope = (fleetAvailable?: boolean): McpInvocationContext.McpInvocationScope => ({ + environmentId: EnvironmentId.make("environment-1"), + requestNamespace: "client:session-1", + thread: undefined, + client: undefined, + capabilities: new Set(["orchestration"]), + issuedAt: 1, + ...(fleetAvailable === undefined ? {} : { fleetAvailable }), + }); + const visible = (fleetAvailable?: boolean) => + Effect.sync(() => McpInvocationContext.hiddenWhileFleetOff()).pipe( + Effect.provideService(McpInvocationContext.McpInvocationContext, scope(fleetAvailable)), + ); + + return Effect.gen(function* () { + expect(yield* visible(false)).toBe(false); + expect(yield* visible(true)).toBe(true); + expect(yield* visible(undefined)).toBe(false); + + const enabledWhen = (name: string): boolean => { + const tool = OrchestratorToolkit.tools[name as keyof typeof OrchestratorToolkit.tools]; + return Context.getOrUndefined(tool.annotations, McpSchema.EnabledWhen) !== undefined; + }; + for (const fleetTool of [ + "t3_environment_spawn", + "t3_environment_remove", + "t3_environment_list", + ]) { + expect(enabledWhen(fleetTool)).toBe(true); + } + for (const coreTool of [ + "orchestrator_capabilities", + "t3_thread_list", + "list_scheduled_tasks", + "t3_thread_wait", + ]) { + expect(enabledWhen(coreTool)).toBe(false); + } + }); +}); diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts index 893c17f23311b..153a4986269f8 100644 --- a/apps/server/src/mcp/McpInvocationContext.ts +++ b/apps/server/src/mcp/McpInvocationContext.ts @@ -152,6 +152,11 @@ export interface McpInvocationScope { readonly client: McpClientCaller | undefined; /** False while the Linked environments beta is off here; `/mcp` reads it per request. */ readonly linkedEnvironments?: boolean; + /** + * Whether this instance can manage a fleet: the OpenShell block is on and this + * environment is the managing instance. Unset means unknown, so tools show. + */ + readonly fleetAvailable?: boolean; /** * The origin this `/mcp` request reached, e.g. a T3 Connect tunnel. Hosted clients cannot * see the URL they were configured with, so links they must fetch carry it. Unset in-process. @@ -193,6 +198,13 @@ export const hiddenFromSubagents = (): boolean => { return scope?.thread?.subagent !== true; }; +/** `McpSchema.EnabledWhen` for the fleet tools, hidden where no fleet can be managed. */ +export const hiddenWhileFleetOff = (): boolean => { + const fiber = Fiber.getCurrent(); + const scope = fiber && Context.getOrUndefined(fiber.context, McpInvocationContext); + return scope?.fleetAvailable === true; +}; + /** * `McpSchema.EnabledWhen` for tools that act as the calling thread: a client * from outside has none. Discovery only; `McpToolAccess` still refuses the diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index 777a5d6df3a56..6e8ac9c415763 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -51,12 +51,14 @@ import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as McpHttpServer from "./McpHttpServer.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as PeerForwarding from "../peer/PeerForwarding.ts"; import * as PeerLinkRequests from "../peer/PeerLinkRequests.ts"; import * as PeerLinks from "../peer/PeerLinks.ts"; const layerOrchestratorToolkit = McpHttpServer.layerOrchestratorToolkit.pipe( + Layer.provide(OpenShellFleetService.layerDisabled), Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})), Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), ); diff --git a/apps/server/src/mcp/linkOrigin.test.ts b/apps/server/src/mcp/linkOrigin.test.ts index 928fda0dd72f8..bf1b6c39d4920 100644 --- a/apps/server/src/mcp/linkOrigin.test.ts +++ b/apps/server/src/mcp/linkOrigin.test.ts @@ -43,6 +43,7 @@ import * as RemoteDelegation from "../peer/RemoteDelegation.ts"; import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as McpHttpServer from "./McpHttpServer.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "./McpInvocationContext.ts"; import * as PeerForwarding from "../peer/PeerForwarding.ts"; import * as ThreadSearch from "../orchestration-v2/ThreadSearch.ts"; @@ -115,7 +116,7 @@ const layerLaunches = Layer.effect( ); const layerTools = Layer.mergeAll( - McpHttpServer.layerOrchestratorToolkit, + McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)), McpHttpServer.layerThreadToolkit, McpHttpServer.layerProjectRegistration, ).pipe( diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts index 1272f367d439d..4bf49566c9e9a 100644 --- a/apps/server/src/mcp/toolkits/core.test.ts +++ b/apps/server/src/mcp/toolkits/core.test.ts @@ -58,6 +58,7 @@ import * as RemoteDelegation from "../../peer/RemoteDelegation.ts"; import * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as McpHttpServer from "../McpHttpServer.ts"; +import * as OpenShellFleetService from "../../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "../McpInvocationContext.ts"; import * as McpToolAccessTestkit from "../McpToolAccess.testkit.ts"; import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; @@ -70,6 +71,7 @@ import * as PeerLinkRequests from "../../peer/PeerLinkRequests.ts"; import * as PeerLinks from "../../peer/PeerLinks.ts"; const layerOrchestratorToolkit = McpHttpServer.layerOrchestratorToolkit.pipe( + Layer.provide(OpenShellFleetService.layerDisabled), Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})), Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})), ); diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts new file mode 100644 index 0000000000000..53b0144ab8451 --- /dev/null +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { + EnvironmentId, + type OrchestratorMcpDelegateTaskInput, + type OrchestratorMcpDelegateTaskResult, + ProviderInstanceId, +} from "@t3tools/contracts"; + +import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts"; +import { OrchestratorToolkit } from "./tools.ts"; +import * as Handlers from "./handlers.ts"; +import * as McpInvocationContext from "../../McpInvocationContext.ts"; +import * as McpToolAccess from "../../McpToolAccess.ts"; +import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts"; +import * as ThreadManagement from "../../../orchestration-v2/ThreadManagementService.ts"; + +const threadId = "thread:parent" as never; + +interface Harness { + readonly spawns: Array< + Parameters[0] + >; + readonly delegated: Array; + readonly failSpawn?: OpenShellFleetService.OpenShellFleetError; +} + +const run = (input: OrchestratorMcpDelegateTaskInput, harness: Harness) => + Effect.gen(function* () { + const caller = { + scope: undefined as never, + threads: {} as never, + caller: { + id: threadId, + runtimeMode: "full-access", + interactionMode: "default", + archivedAt: null, + providerInstanceId: ProviderInstanceId.make("pi"), + activeRunId: "run:parent", + deletedAt: null, + }, + limits: { runtimeMode: "full-access" as const, interactionMode: "default" as const }, + }; + const dependencies = Layer.mergeAll( + Layer.succeed(McpInvocationContext.McpInvocationContext, { + environmentId: EnvironmentId.make("environment"), + requestNamespace: "session", + thread: { + threadId, + providerInstanceId: ProviderInstanceId.make("pi"), + providerSessionId: "session", + }, + client: { + sessionId: "session", + label: "Test", + access: "full-access" as const, + }, + issuedAt: 0, + capabilities: new Set(["orchestration" as const]), + }), + Layer.mock(ThreadManagement.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(caller.caller as never), + }), + Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({ + delegateTask: (_scope, params) => { + harness.delegated.push(params); + return Effect.succeed({ + taskId: "node:task", + childThreadId: null, + remoteChild: { + environmentId: EnvironmentId.make("environment:worker"), + threadId: "thread:child", + label: "worker", + }, + childRunId: null, + childNodeId: "node:task", + status: "running", + workState: "working", + hasPendingChildRuns: false, + latestTerminalRunId: null, + latestTerminalStatus: null, + latestTerminalSummary: null, + latestTerminalResultContextTransferId: null, + providerInstanceId: ProviderInstanceId.make("pi"), + model: null, + summary: null, + resultContextTransferId: null, + hasPendingQuestion: false, + pendingApproval: null, + waitTimedOut: false, + } as unknown as OrchestratorMcpDelegateTaskResult); + }, + }), + Layer.mock(OpenShellFleetService.OpenShellFleetService)({ + spawnWorker: (params) => { + harness.spawns.push(params); + return harness.failSpawn === undefined + ? Effect.succeed({ + environmentId: EnvironmentId.make("environment:worker"), + sandbox: { name: "worker", id: "sandbox-1" }, + workerClass: params.workerClass, + state: "ready", + expiresAt: null, + warnings: [], + } as never) + : Effect.fail(harness.failSpawn); + }, + removeWorker: () => Effect.die("removeWorker is not part of this path"), + listWorkers: () => Effect.succeed([]), + }), + ); + const toolkit = yield* OrchestratorToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(Handlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const events = yield* toolkit + .handle("delegate_task", input as never) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies)); + return events.at(-1)?.result; + }); + +describe("delegate_task openshell-auto isolation", () => { + it.effect("provisions an ephemeral worker and delegates into it", () => + Effect.gen(function* () { + const harness: Harness = { spawns: [], delegated: [] }; + yield* run({ task: "Do the thing", isolation: "openshell-auto" }, harness); + + expect(harness.spawns).toHaveLength(1); + expect(harness.spawns[0]).toMatchObject({ + workerClass: "ephemeral", + runtimeModeCeiling: "full-access", + }); + // The child runs there, not here: the sandbox is the isolation. + expect(harness.delegated[0]?.isolation).toBe("shared"); + expect(harness.delegated[0]?.target).toMatchObject({ + environmentId: EnvironmentId.make("environment:worker"), + }); + }), + ); + + it.effect("delegates locally without provisioning anything", () => + Effect.gen(function* () { + const harness: Harness = { spawns: [], delegated: [] }; + yield* run({ task: "Do the thing" }, harness); + + expect(harness.spawns).toHaveLength(0); + expect(harness.delegated[0]?.target).toBeUndefined(); + }), + ); + + it.effect("reports a refused spawn as a mode-escalation denial", () => + Effect.gen(function* () { + const harness: Harness = { + spawns: [], + delegated: [], + failSpawn: new OpenShellFleetService.OpenShellFleetSpawnRefused({ + reason: "runtime_mode_above_ceiling", + message: + "Worker policy full-access is broader than this caller's approval-required ceiling.", + }), + }; + const result = (yield* run( + { task: "Do the thing", isolation: "openshell-auto" }, + harness, + )) as { readonly code?: string } | undefined; + + expect(harness.delegated).toHaveLength(0); + expect(result?.code).toBe("runtime_mode_escalation_denied"); + }), + ); +}); diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts index d90a118be468e..a0a45a41c6767 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts @@ -1,17 +1,39 @@ import { OrchestratorMcpFailure } from "@t3tools/contracts"; +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeCrypto from "node:crypto"; import { OrchestratorToolkit } from "./tools.ts"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts"; import * as PeerForwarding from "../../../peer/PeerForwarding.ts"; import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts"; import * as PeerLinks from "../../../peer/PeerLinks.ts"; import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts"; -import { readCaller } from "../../threadAccess.ts"; +import { assertNotLinked } from "../../linkOrigin.ts"; +import { assertFullAccess, loadCaller, readCaller } from "../../threadAccess.ts"; import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as McpToolAccess from "../../McpToolAccess.ts"; import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts"; import * as ThreadMetadataMcpService from "../../ThreadMetadataMcpService.ts"; +const fleetFailure = (error: OpenShellFleetService.OpenShellFleetError): OrchestratorMcpFailure => + error._tag === "OpenShellFleetDisabled" + ? new OrchestratorMcpFailure({ code: "capability_denied", message: error.message }) + : error._tag === "OpenShellFleetSpawnRefused" + ? new OrchestratorMcpFailure({ + code: + error.reason === "runtime_mode_above_ceiling" + ? "runtime_mode_escalation_denied" + : "invalid_request", + message: error.message, + }) + : new OrchestratorMcpFailure({ code: "orchestration_error", message: error.message }); + +/** Same caller request id must not collide across MCP sessions/scope owners. */ +const scopedFleetRequestId = (namespace: string, clientRequestId: string): string => + `fleet:${NodeCrypto.createHash("sha256").update(`${namespace}\n${clientRequestId}`).digest("hex")}`; + const handlers = { orchestrator_capabilities: McpToolAccess.reads((input) => Effect.gen(function* () { @@ -24,7 +46,41 @@ const handlers = { Effect.gen(function* () { const scope = yield* McpInvocationContext.McpInvocationContext; const service = yield* OrchestratorMcpService.OrchestratorMcpService; - return yield* service.delegateTask(scope, input); + if (input.isolation !== "openshell-auto") { + return yield* service.delegateTask(scope, input); + } + // Provision the worker here, then hand the task to it like any other + // linked environment. Its retirement is the fleet's own: the deadline + // reaps it drain-first, so nothing is deleted under a running child. + const caller = yield* loadCaller(); + // Provisioning infrastructure is an environment setting, not thread work: + // the same full-access/linked checks as t3_environment_spawn apply here, + // and they run before anything is created. + yield* assertFullAccess(caller, "provision a fleet worker"); + yield* assertNotLinked(caller, "provision a fleet worker"); + const fleet = yield* OpenShellFleetService.OpenShellFleetService; + const spawned = yield* fleet + .spawnWorker({ + workerClass: "ephemeral", + runtimeModeCeiling: caller.limits.runtimeMode, + ...(input.runtimeMode === undefined ? {} : { runtimeMode: input.runtimeMode }), + ...(input.title === undefined ? {} : { label: input.title }), + ...(input.clientRequestId === undefined + ? {} + : { + clientRequestId: scopedFleetRequestId( + scope.requestNamespace, + input.clientRequestId, + ), + }), + }) + .pipe(Effect.mapError(fleetFailure)); + return yield* service.delegateTask(scope, { + ...input, + // The child runs in that environment's checkout; the sandbox is the isolation. + isolation: "shared", + target: { ...input.target, environmentId: spawned.environmentId }, + }); }), ), task_status: McpToolAccess.actsAsCaller(({ taskId }) => @@ -229,6 +285,69 @@ const handlers = { return { environmentId, removed }; }), ), + // Spawning a worker mints infrastructure under this environment's gateway + // credential, so it is an environment setting: full access, never linked work. + t3_environment_spawn: McpToolAccess.writesEnvironment((input, check) => + Effect.gen(function* () { + const fleet = yield* OpenShellFleetService.OpenShellFleetService; + const scope = yield* McpInvocationContext.McpInvocationContext; + const caller = yield* check; + return yield* fleet + .spawnWorker({ + workerClass: input.workerClass, + runtimeModeCeiling: caller.limits.runtimeMode, + ...(input.template === undefined ? {} : { template: input.template }), + ...(input.runtimeMode === undefined ? {} : { runtimeMode: input.runtimeMode }), + ...(input.label === undefined ? {} : { label: input.label }), + ...(input.ttlMs === undefined ? {} : { ttlMs: Duration.toMillis(input.ttlMs) }), + ...(input.seedFrom === undefined ? {} : { seedFrom: input.seedFrom }), + ...(input.clientRequestId === undefined + ? {} + : { + clientRequestId: scopedFleetRequestId( + scope.requestNamespace, + input.clientRequestId, + ), + }), + }) + .pipe(Effect.mapError(fleetFailure)); + }), + ), + // Removal drains worker data back here before any delete, so a refusal is + // the safe outcome; it stays a full-access environment operation. + t3_environment_remove: McpToolAccess.writesEnvironment((input) => + Effect.gen(function* () { + const fleet = yield* OpenShellFleetService.OpenShellFleetService; + const removed = yield* fleet + .removeWorker({ + targetEnvironmentId: input.targetEnvironmentId, + ...(input.drainTimeoutMs === undefined + ? {} + : { drainTimeoutMs: Duration.toMillis(input.drainTimeoutMs) }), + ...(input.force === undefined ? {} : { force: input.force }), + ...(input.clientRequestId === undefined + ? {} + : { clientRequestId: input.clientRequestId }), + }) + .pipe(Effect.mapError(fleetFailure)); + return { + targetEnvironmentId: removed.environmentId, + outcome: removed.outcome, + ...(removed.drain === undefined ? {} : { drain: removed.drain }), + ...(removed.sandboxOutcome === undefined ? {} : { sandboxOutcome: removed.sandboxOutcome }), + ...(removed.reason === undefined ? {} : { reason: removed.reason }), + }; + }), + ), + t3_environment_list: McpToolAccess.reads((input) => + Effect.gen(function* () { + const fleet = yield* OpenShellFleetService.OpenShellFleetService; + const workers = yield* fleet + .listWorkers(input.workerClass === undefined ? {} : { workerClass: input.workerClass }) + .pipe(Effect.mapError(fleetFailure)); + return { workers }; + }), + ), t3_thread_interrupt: McpToolAccess.writesThreads( (input) => [input.threadId], (input) => diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index b2866bb0babc5..7d8c90ca29e99 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -7,6 +7,12 @@ import { OrchestratorMcpEnvironmentLinksResult, OrchestratorMcpEnvironmentUnlinkInput, OrchestratorMcpEnvironmentUnlinkResult, + OrchestratorMcpEnvironmentSpawnInput, + OrchestratorMcpEnvironmentSpawnResult, + OrchestratorMcpEnvironmentRemoveInput, + OrchestratorMcpEnvironmentRemoveResult, + OrchestratorMcpEnvironmentListInput, + OrchestratorMcpEnvironmentListResult, OrchestratorMcpCreateThreadsInput, OrchestratorMcpCreateThreadsResult, OrchestratorMcpDelegateTaskInput, @@ -43,6 +49,7 @@ import * as Schema from "effect/Schema"; import * as Struct from "effect/Struct"; import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts"; +import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts"; import * as PeerForwarding from "../../../peer/PeerForwarding.ts"; import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts"; import * as PeerLinks from "../../../peer/PeerLinks.ts"; @@ -71,6 +78,14 @@ const threadMetadataDependencies = [ ThreadManagementService.ThreadManagementService, ThreadMetadataMcpService.ThreadMetadataMcpService, ]; +/** Delegation that can also provision a fleet worker (spec section 6.3). */ +const delegateDependencies = [...dependencies, OpenShellFleetService.OpenShellFleetService]; +/** Fleet worker lifecycle on the OpenShell gateway (spec section 5.1). */ +const fleetDependencies = [ + McpInvocationContext.McpInvocationContext, + ThreadManagementService.ThreadManagementService, + OpenShellFleetService.OpenShellFleetService, +]; const OrchestratorCapabilitiesTool = Tool.make("orchestrator_capabilities", { description: @@ -88,12 +103,12 @@ const OrchestratorCapabilitiesTool = Tool.make("orchestrator_capabilities", { const DelegateTaskTool = Tool.make("delegate_task", { description: - "Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. Keep `taskId` for `task_status` and `task_cancel`. No secrets in `task`.", + "Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. `isolation:'openshell-auto'` runs the child on a self-retiring ephemeral fleet worker. Keep `taskId` for `task_status` and `task_cancel`. No secrets in `task`.", parameters: OrchestratorMcpDelegateTaskInput, success: OrchestratorMcpDelegateTaskResult, failure: OrchestratorMcpFailure, failureMode: "return", - dependencies, + dependencies: delegateDependencies, }) .annotate(Tool.Title, "Delegate a child task") .annotate(Tool.Destructive, true) @@ -362,6 +377,50 @@ const EnvironmentUnlinkTool = Tool.make("t3_environment_unlink", { .annotate(Tool.Idempotent, true) .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff); +const EnvironmentSpawnTool = Tool.make("t3_environment_spawn", { + description: + "Provision and pair a fleet worker. Ephemeral workers self-retire on the TTL sweep; persistent ones are named lanes. Same `clientRequestId` converges retries and re-pairs a worker whose delegation session expired.", + parameters: OrchestratorMcpEnvironmentSpawnInput, + success: OrchestratorMcpEnvironmentSpawnResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies: fleetDependencies, +}) + .annotate(Tool.Title, "Spawn a fleet worker environment") + .annotate(Tool.Destructive, false) + .annotate(Tool.OpenWorld, true) + .annotate(Tool.Idempotent, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff); + +const EnvironmentRemoveTool = Tool.make("t3_environment_remove", { + description: + "Drain a fleet worker to this instance (thread histories plus a git bundle of the checkout refs origin lacks, committed and uncommitted), then delete its sandbox. Refused while the drain is incomplete; the sandbox is kept. `force` skips the checkout bundles after an explicit user ack: thread histories still export, but the checkouts are lost with the sandbox.", + parameters: OrchestratorMcpEnvironmentRemoveInput, + success: OrchestratorMcpEnvironmentRemoveResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies: fleetDependencies, +}) + .annotate(Tool.Title, "Remove a fleet worker environment") + .annotate(Tool.Destructive, true) + .annotate(Tool.OpenWorld, true) + .annotate(Tool.Idempotent, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff); + +const EnvironmentListTool = Tool.make("t3_environment_list", { + description: "Report this instance's fleet worker state.", + parameters: OrchestratorMcpEnvironmentListInput, + success: OrchestratorMcpEnvironmentListResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies: fleetDependencies, +}) + .annotate(Tool.Title, "List fleet worker environments") + .annotate(Tool.Readonly, true) + .annotate(Tool.Destructive, false) + .annotate(Tool.Idempotent, true) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff); + export const OrchestratorToolkit = Toolkit.make( OrchestratorCapabilitiesTool, DelegateTaskTool, @@ -383,4 +442,7 @@ export const OrchestratorToolkit = Toolkit.make( EnvironmentLinksTool, EnvironmentLinkTool, EnvironmentUnlinkTool, + EnvironmentSpawnTool, + EnvironmentRemoveTool, + EnvironmentListTool, ); diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts index c13a8ab9353cc..ce6269c73de85 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts @@ -626,6 +626,32 @@ it.effect("a launch binds only an existing checkout that is one of the project's }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect("a sandbox workspace is refused here and pointed at delegation", () => + Effect.gen(function* () { + const launched: Array = []; + const { projectId, dependencies } = clientLaunchHarness({ + runtimeModeCeiling: "full-access", + launched, + }); + const toolkit = yield* ProjectToolkit.pipe( + Effect.provide( + McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)), + ), + ); + const result = yield* toolkit + .handle("t3_thread_launch", { + title: "Fix", + projectId, + workspaceStrategy: { type: "sandbox" }, + }) + .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies)); + + expect(result.at(-1)?.result).toMatchObject({ code: "invalid_request" }); + // No thread launched locally: the sandbox would have silently become the root. + expect(launched).toHaveLength(0); + }), +); + it.effect("a launch into a new worktree creates nothing when its base ref has no commit", () => Effect.gen(function* () { const launched: Array = []; diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts index 3966d48ccd090..64034ac0ba753 100644 --- a/apps/server/src/mcp/toolkits/project/handlers.ts +++ b/apps/server/src/mcp/toolkits/project/handlers.ts @@ -377,6 +377,14 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, { } : requestedStrategy; // Reject unresolved base refs and blocked branch names before accepting a launch; worktree provisioning happens later. + // OPEN-QUESTION (spec section 6.3): a sandbox workspace belongs to a fleet + // worker, so v1 launches it as a delegation rather than here. + if (!replaying && strategy?.type === "sandbox") + return yield* new OrchestratorMcpFailure({ + code: "invalid_request", + message: + "workspaceStrategy 'sandbox' is not available on thread launch yet. Delegate the work with isolation:'openshell-auto' to run it in a fleet worker environment.", + }); if (!replaying && strategy?.type === "worktree") yield* ThreadLaunch.ThreadLaunchService.pipe( Effect.flatMap((launches) => diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index df965d891487c..77398d460f704 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -2597,6 +2597,7 @@ export const CLAUDE_READ_ONLY_T3_MCP_ALLOWED_TOOLS: ReadonlyArray = [ "mcp__t3-code__t3_preview_list", "mcp__t3-code__t3_environment_read", "mcp__t3-code__t3_environment_links", + "mcp__t3-code__t3_environment_list", "mcp__t3-code__t3_theme_read", "mcp__t3-code__html_preview", "mcp__t3-code__html_render", diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 00939843c13f1..32bccce82e3db 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -1010,6 +1010,16 @@ function delegatedTaskTerminalStatus( function worktreeSetupPending( projection: Pick, ): boolean { + // A sandbox run can never resolve to a local workspace, so it holds only + // while it is the latest run: a newer run supersedes it. A worktree holds + // until its path exists, whatever came after it. + const latest = latestExecutedRun(projection.runs); + if ( + latest !== null && + (latest.status === "failed" || latest.status === "interrupted") && + latest.workspacePreparation?.type === "sandbox" + ) + return true; return ( projection.thread.worktreePath === null && projection.runs.some( @@ -2561,8 +2571,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const worktreeMissing = endedRunId !== undefined && failedRun?.id === endedRunId && - failedRun?.workspacePreparation?.type === "worktree" && - projection.thread.worktreePath === null; + (failedRun?.workspacePreparation?.type === "sandbox" || + (failedRun?.workspacePreparation?.type === "worktree" && + projection.thread.worktreePath === null)); const settings = yield* serverSettings.getSettings.pipe(Effect.orDie); const projectOverride = settings.projectSettingsOverrides[projection.thread.projectId]?.providerInstances?.[ @@ -7848,8 +7859,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio activeRun === undefined && (dispatchMode.type === "queue_after_active" || dispatchMode.type === "start_immediately") && (latestRun?.status === "failed" || latestRun?.status === "interrupted") && - latestRun.workspacePreparation?.type === "worktree" && - projection.thread.worktreePath === null + (latestRun.workspacePreparation?.type === "sandbox" || + (latestRun.workspacePreparation?.type === "worktree" && + projection.thread.worktreePath === null)) ? latestRun : undefined; // A steer whose run already ended still keeps its place in the queue. @@ -15574,6 +15586,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio "orchestration_v2.command_type": command.type, "orchestration_v2.thread_id": commandThreadId(command), }); + // A sandbox workspace is provisioned by delegation, never by a dispatch; + // refuse before any event records it on a run. Replays short-circuit above. + if ( + (command.type === "message.dispatch" && + command.dispatchMode.type === "defer_start" && + command.dispatchMode.workspaceStrategy?.type === "sandbox") || + (command.type === "delegated_task.request" && + command.workspaceStrategy?.type === "sandbox") || + (command.type === "thread.create" && + command.initialMessage?.workspaceStrategy?.type === "sandbox") + ) + return yield* new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause: + "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.", + }); const events = yield* Ref.make>([]); const effects = yield* Ref.make>([]); diff --git a/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts b/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts new file mode 100644 index 0000000000000..c14840ebaa130 --- /dev/null +++ b/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts @@ -0,0 +1,108 @@ +import { assert, it } from "@effect/vitest"; +import { CommandId, MessageId, ProjectId, ThreadId } from "@t3tools/contracts"; +import * as Layer from "effect/Layer"; +import * as Effect from "effect/Effect"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts"; +import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; + +const instanceId = ProviderInstanceId.make("codex"); +const adapter = { + instanceId, + driver: ProviderDriverKind.make("codex"), + getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }), + openSession: () => Effect.die("No provider process needed for metadata commands"), +} as ProviderAdapter.ProviderAdapterV2["Service"]; +const layerDatabase = SqlitePersistence.layerMemory; +const layerTest = Layer.mergeAll( + layerDatabase, + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), + ProviderReplayHarness.layerWithRegistry( + { name: "sandbox-refusal" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { databaseLayer: layerDatabase, runEffectWorker: false }, + ), +); + +const createThread = (threadId: ThreadId) => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("project:sandbox-refusal"), + title: "Sandbox refusal", + modelSelection: { instanceId, model: "gpt-5" }, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: null, + createdBy: "user", + creationSource: "web", + }); + }); + +it.layer(layerTest)("sandbox workspace refusals", (it) => { + it.effect("refuses a defer_start dispatch with a sandbox strategy and records no run", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:sandbox-defer"); + yield* createThread(threadId); + const refused = yield* orchestrator + .dispatch({ + type: "message.dispatch", + commandId: CommandId.make("dispatch:sandbox-defer"), + threadId, + messageId: MessageId.make("message:sandbox-defer"), + text: "Run this in a worker", + attachments: [], + modelSelection: { instanceId, model: "gpt-5" }, + dispatchMode: { + type: "defer_start", + workspaceStrategy: { type: "sandbox" }, + }, + createdBy: "user", + creationSource: "web", + }) + .pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + assert.match(String(refused.cause), /isolation 'openshell-auto'/); + const records = yield* projections.getThreadRecords(threadId, ["runs"]); + assert.equal(records.runs.length, 0); + }), + ); + + it.effect("refuses a delegated task with a sandbox strategy", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const parentThreadId = ThreadId.make("thread:sandbox-delegate-parent"); + yield* createThread(parentThreadId); + const refused = yield* orchestrator + .dispatch({ + type: "delegated_task.request", + commandId: CommandId.make("dispatch:sandbox-delegate"), + parentThreadId, + parentRunId: "run:sandbox-delegate" as never, + parentNodeId: "node:sandbox-delegate" as never, + task: "Build this in a worker", + modelSelection: { instanceId, model: "gpt-5" }, + runtimeMode: "full-access", + interactionMode: "default", + workspaceStrategy: { type: "sandbox" }, + createdBy: "user", + creationSource: "web", + }) + .pipe(Effect.flip); + assert.equal(refused._tag, "OrchestratorDispatchError"); + assert.match(String(refused.cause), /isolation 'openshell-auto'/); + }), + ); +}); diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index ba91ba94d2520..4f43cf6d3eb70 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -490,7 +490,7 @@ export interface ProjectionTimelinePage { } export interface ShellSnapshotOptions { - readonly location?: "active" | "archive"; + readonly location?: "active" | "archive" | "all"; readonly projectId?: ProjectId; readonly unsettledOnly?: boolean; /** diff --git a/apps/server/src/orchestration-v2/ShellStream.ts b/apps/server/src/orchestration-v2/ShellStream.ts index 5ee1005b16830..282dcbf03f49f 100644 --- a/apps/server/src/orchestration-v2/ShellStream.ts +++ b/apps/server/src/orchestration-v2/ShellStream.ts @@ -167,13 +167,15 @@ function buildActiveShellSnapshot(input: { readonly projects: ReadonlyArray; readonly threads: OrchestrationV2ThreadShellSnapshot; readonly snapshotSequence: number; + /** A drain has to copy archived threads too before it may delete them. */ + readonly includeArchived?: boolean; }): OrchestrationV2ShellSnapshot { return { schemaVersion: input.threads.schemaVersion, snapshotSequence: input.snapshotSequence, projects: input.projects, threads: input.threads.threads, - archivedThreads: [], + archivedThreads: input.includeArchived === true ? input.threads.archivedThreads : [], }; } @@ -347,6 +349,7 @@ export function omitOlderSettledShells(input: { interface ShellSubscriberOptions { readonly omitIdleSubagents: boolean; readonly omitOlderSettled: boolean; + readonly includeArchived?: boolean; } function trimShellSnapshot( @@ -426,11 +429,16 @@ export const loadSubscriberShellSnapshot = Effect.fn("ShellStream.loadSubscriber .withTransaction( Effect.gen(function* () { const index = trims ? yield* input.listShellIndex : null; + // Active index IDs are a trim filter, not an inventory filter. An + // archive-inclusive snapshot must still read archived threads, so it + // ignores the active-ID filter and applies trims after the full read. + const subscriberIds = + index === null || input.options.includeArchived === true + ? undefined + : shellSubscriberThreadIds(index, input.options); return { index, - decodeThreads: yield* input.readThreads( - index === null ? undefined : shellSubscriberThreadIds(index, input.options), - ), + decodeThreads: yield* input.readThreads(subscriberIds), projects: yield* input.listProjects, snapshotSequence: yield* input.latestSequence, }; @@ -441,6 +449,7 @@ export const loadSubscriberShellSnapshot = Effect.fn("ShellStream.loadSubscriber projects: read.projects, threads: yield* read.decodeThreads, snapshotSequence: read.snapshotSequence, + includeArchived: input.options.includeArchived === true, }); const facts = shellTrimFacts(read.index ?? base.threads); return { diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts index 2a3a8e5510550..02cf1737ff63f 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts @@ -1189,6 +1189,55 @@ const preparationSettled = (threadId: ThreadId) => return yield* threads.getThreadProjection(threadId); }); +it.effect("refuses a sandbox launch before creating a thread or run", () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const failed = yield* launches + .launch( + launchInput({ + command: "command:launch:sandbox", + thread: "thread:launch:sandbox", + message: "Run in a worker", + workspace: { type: "sandbox" }, + }), + ) + .pipe(Effect.flip); + if (failed._tag !== "ThreadLaunchError") { + assert.fail(`expected ThreadLaunchError, got ${failed._tag}`); + } + assert.equal(failed.operation, "create-thread"); + assert.match(String(failed.cause), /isolation 'openshell-auto'/); + assert.equal(harness.runSetup.mock.calls.length, 0); + assert.equal(yield* threads.getThreadShell(ThreadId.make("thread:launch:sandbox")), null); + }).pipe(Effect.provide(harness.layer)); +}); + +it.effect("refuses a sandbox launch without an initial message too", () => { + const harness = makeHarness(); + return Effect.gen(function* () { + const launches = yield* ThreadLaunch.ThreadLaunchService; + const threads = yield* ThreadManagement.ThreadManagementService; + const failed = yield* launches + .launch( + launchInput({ + command: "command:launch:sandbox-silent", + thread: "thread:launch:sandbox-silent", + workspace: { type: "sandbox" }, + }), + ) + .pipe(Effect.flip); + assert.equal(failed._tag, "ThreadLaunchError"); + assert.match(String(failed.cause), /isolation 'openshell-auto'/); + assert.equal(harness.runSetup.mock.calls.length, 0); + assert.equal( + yield* threads.getThreadShell(ThreadId.make("thread:launch:sandbox-silent")), + null, + ); + }).pipe(Effect.provide(harness.layer)); +}); + it.effect.each([ { prefix: "my-prefix", collision: null, expected: /^my-prefix\/[0-9a-f]{8}$/ }, { prefix: "my-prefix", collision: "my-prefix", expected: /^my-prefix-[0-9a-f]{8}$/ }, diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index e0ae516a190b5..eb2e2b81dfd80 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -5,6 +5,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, type ChatAttachment, + EnvironmentId, type MessageId, type ModelSelection, type OrchestrationV2Actor, @@ -83,6 +84,12 @@ export type ThreadLaunchWorkspaceStrategy = readonly branch?: string | undefined; readonly startFromOrigin?: boolean | undefined; readonly repositories?: ReadonlyArray<{ readonly path: string }> | undefined; + } + /** A fleet worker's workspace. Only delegation provisions it, so a local launch fails its run. */ + | { + readonly type: "sandbox"; + readonly workerClass?: "ephemeral" | "persistent" | undefined; + readonly branch?: string | undefined; }; export interface ThreadLaunchInitialMessage { @@ -117,6 +124,10 @@ export interface ThreadLaunchInput { readonly metadata?: OrchestrationV2ProviderThreadNativeMetadata; }; readonly createdBy: OrchestrationV2Actor; + /** Transport-supplied self id; the launch service owns the local-only rule. */ + readonly currentEnvironmentId?: EnvironmentId; + /** Requested launch target; `undefined` means this environment. */ + readonly environmentId?: EnvironmentId; readonly creationSource: OrchestrationV2CreationSource; /** The linked environment this launch is for, if one asked for it. */ readonly linkOrigin?: OrchestrationV2LinkOrigin; @@ -556,6 +567,19 @@ const make = Effect.gen(function* () { ), ); + // A sandbox workspace belongs to a fleet worker, which is created by a + // delegation. Launching it here would quietly run in the project root. + if (input.workspaceStrategy.type === "sandbox") { + return yield* new ThreadLaunchError({ + operation: "provision-worktree", + commandId: input.commandId, + projectId: input.projectId, + ...(threadId === undefined ? {} : { threadId }), + cause: new Error( + "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.", + ), + }); + } const reused = input.reusedWorktree; const tracked = input.workspaceStrategy.type === "worktree" || reused !== undefined; let createdWorktreePath: string | null = null; @@ -1338,6 +1362,16 @@ const make = Effect.gen(function* () { (yield* managedFolders.isScratchProject(request.projectId)) ? { ...request, workspaceStrategy: { type: "root" } } : request; + // A foreign launch has no local projection to return; delegate_task owns + // remote dispatch. Keep the rule here rather than in WebSocket transport. + if (input.environmentId !== undefined && input.environmentId !== input.currentEnvironmentId) { + return yield* mapError( + input, + "create-thread", + )( + "A thread launches in this environment. To run work in another environment, use delegate_task with target.environmentId; its result arrives when the task settles.", + ); + } yield* ProjectCloneTracker.rejectCommandsDuringClone(cloneTracker, { type: "thread.create", projectId: input.projectId, @@ -1359,6 +1393,17 @@ const make = Effect.gen(function* () { } const launchReceipt = yield* readReceipt(input, input.commandId); + // A sandbox workspace belongs to a fleet worker, which only a delegation + // provisions. Refuse before any id or thread mutation; an accepted + // launch still replays through its receipt. + if (input.workspaceStrategy.type === "sandbox" && Option.isNone(launchReceipt)) { + return yield* mapError( + input, + "create-thread", + )( + "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.", + ); + } yield* threads.assertProviderInstanceEnabled({ projectId: input.projectId, commandId: input.commandId, @@ -1570,15 +1615,17 @@ const make = Effect.gen(function* () { Option.isSome(launchReceipt) && (accepted?.type === "worktree" || accepted?.type === "existing_worktree") ? accepted - : Option.isSome(launchReceipt) && - workspaceStrategy.type === "root" && - boundWorktreePath !== null - ? { - type: "existing_worktree", - worktreePath: boundWorktreePath, - branch: workspaceStrategy.branch, - } - : workspaceStrategy; + : Option.isSome(launchReceipt) && accepted?.type === "root" + ? accepted + : Option.isSome(launchReceipt) && + workspaceStrategy.type === "root" && + boundWorktreePath !== null + ? { + type: "existing_worktree", + worktreePath: boundWorktreePath, + branch: workspaceStrategy.branch, + } + : workspaceStrategy; yield* schedulePreparation( { ...input, workspaceStrategy: preparationStrategy }, threadId, diff --git a/apps/server/src/orchestration-v2/http.ts b/apps/server/src/orchestration-v2/http.ts index b58ce1b143119..4449ac3677876 100644 --- a/apps/server/src/orchestration-v2/http.ts +++ b/apps/server/src/orchestration-v2/http.ts @@ -84,13 +84,14 @@ export const layer = HttpApiBuilder.group( function* (options: { readonly omitIdleSubagents: boolean; readonly omitOlderSettled: boolean; + readonly includeArchived: boolean; }) { const { snapshot } = yield* loadSubscriberShellSnapshot({ sql, listShellIndex: threadManagement.listActiveShellIndex(), readThreads: (threadIds) => threadManagement.readShellSnapshot({ - location: "active", + location: options.includeArchived ? "all" : "active", ...(threadIds === undefined ? {} : { threadIds }), }), listProjects: projectStore.listShells(), @@ -106,15 +107,21 @@ export const layer = HttpApiBuilder.group( const answerShellSnapshot = Effect.fn("http.orchestration.answerShellSnapshot")((query: { readonly omitIdleSubagents?: string; readonly omitOlderSettled?: string; + readonly includeArchived?: string; readonly resumeFrom?: string; }) => { const resumeFrom = query.resumeFrom === undefined ? Number.NaN : Number(query.resumeFrom); + // Archive-inclusive replay is not fully reconstructed by the active-only + // event replay yet, so always send a complete archive-inclusive snapshot. + const canResume = query.includeArchived !== "1"; return loadShellUnlessResumable({ events: applicationEvents, - resumeFrom: Number.isSafeInteger(resumeFrom) && resumeFrom >= 0 ? resumeFrom : undefined, + resumeFrom: + canResume && Number.isSafeInteger(resumeFrom) && resumeFrom >= 0 ? resumeFrom : undefined, loadSnapshot: loadShellSnapshot({ omitIdleSubagents: query.omitIdleSubagents === "1", omitOlderSettled: query.omitOlderSettled === "1", + includeArchived: query.includeArchived === "1", }), }); }); diff --git a/apps/server/src/peer/PeerEnvironmentRegistry.test.ts b/apps/server/src/peer/PeerEnvironmentRegistry.test.ts new file mode 100644 index 0000000000000..65076f410adce --- /dev/null +++ b/apps/server/src/peer/PeerEnvironmentRegistry.test.ts @@ -0,0 +1,497 @@ +// @effect-diagnostics nodeBuiltinImport:off - the redirect test exercises the real fetch boundary. +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { type ExecutionEnvironmentDescriptor } from "@t3tools/contracts"; +import { assert, describe, it } from "@effect/vitest"; +import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as TestClock from "effect/testing/TestClock"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpClientResponse from "effect/http/HttpClientResponse"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as NodeHttp from "node:http"; +import * as NodeNet from "node:net"; + +import * as ServerConfig from "../config.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as Sqlite from "../persistence/Sqlite.ts"; +import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts"; +import { PeerEnvironmentRegistryError } from "./PeerEnvironmentRegistry.ts"; +import { descriptorOf } from "./PeerLinks.testkit.ts"; + +const self = descriptorOf("environment-self", "Self Here"); + +const layerSelfEnvironment = Layer.succeed(ServerEnvironment.ServerEnvironment, { + getEnvironmentId: Effect.succeed(self.environmentId), + getDescriptor: Effect.succeed(self), +}); + +/** A paired peer's delegation session, as the exchange grants it. */ +const GRANTED_SCOPES = "orchestration:read orchestration:operate terminal:operate"; + +interface FakePeer { + readonly http: HttpClient.HttpClient; + /** How often its descriptor was asked for. */ + readonly probes: () => number; + /** The form bodies of the token-exchange requests it answered. */ + readonly exchanges: () => ReadonlyArray>; +} + +interface FakeToken { + readonly scope?: string; + /** The access token it mints; defaults to one fixed value. */ + readonly accessToken?: string | (() => string); + readonly status?: number; + /** Only this origin answers the descriptor probe; default: every origin. */ + readonly answersAt?: string; +} + +/** + * The peer as the registry sees it over its HTTP client: a descriptor probe + * and the bootstrap token exchange. No socket is opened. + */ +const fakePeer = (descriptor: ExecutionEnvironmentDescriptor, token: FakeToken = {}): FakePeer => { + let probes = 0; + const exchanges: Array> = []; + const http = HttpClient.make((request) => { + const url = new URL(request.url); + if (url.pathname === "/.well-known/t3/environment") { + probes += 1; + if (token.answersAt !== undefined && url.origin !== token.answersAt) { + return Effect.succeed( + HttpClientResponse.fromWeb(request, new Response(null, { status: 404 })), + ); + } + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(JSON.stringify(descriptor), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ), + ); + } + if (url.pathname === "/oauth/token") { + // The exchange posts the payload as urlencoded text. + const raw = + request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : ""; + exchanges.push(Object.fromEntries(new URLSearchParams(raw))); + const accessToken = + typeof token.accessToken === "function" + ? token.accessToken() + : (token.accessToken ?? "session-token"); + return Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response( + JSON.stringify({ + access_token: accessToken, + issued_token_type: "urn:ietf:params:oauth:token-type:access_token", + token_type: "Bearer", + expires_in: 3600, + scope: token.scope ?? GRANTED_SCOPES, + }), + { + status: token.status ?? 200, + headers: { "content-type": "application/json" }, + }, + ), + ), + ); + } + return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 404 }))); + }); + return { http, probes: () => probes, exchanges: () => exchanges }; +}; + +const fakeContext = ( + descriptor: ExecutionEnvironmentDescriptor, + token: FakeToken = {}, + openshellEnabled = true, +) => + buildContext( + Layer.succeed(HttpClient.HttpClient, fakePeer(descriptor, token).http), + openshellEnabled, + ); + +const buildContext = (http: Layer.Layer, openshellEnabled = true) => + Layer.build( + PeerEnvironmentRegistry.layer.pipe( + Layer.provide(http), + Layer.provideMerge(layerSelfEnvironment), + Layer.provideMerge(ServerSettings.layerTest({ openshellEnabled })), + Layer.provideMerge(Sqlite.layerMemory), + Layer.provideMerge(ServerSecretStore.layer), + Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-fleet-registry-" })), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + Layer.fresh, + ), + ); + +const workerPeer = descriptorOf("environment-worker-1", "Worker One"); + +const worker = (urls: ReadonlyArray = ["http://127.0.0.1:8111"]) => ({ + environmentId: workerPeer.environmentId, + role: "worker" as const, + label: "Worker One", + urls, + workerClass: "ephemeral" as const, +}); + +const reasonOf = (error: unknown) => (error as PeerEnvironmentRegistryError).reason; + +/** A peer whose descriptor answers but whose token exchange never does. */ +const silentExchangeContext = (descriptor: ExecutionEnvironmentDescriptor) => + buildContext( + Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + request.url.endsWith("/.well-known/t3/environment") + ? Effect.succeed( + HttpClientResponse.fromWeb( + request, + new Response(JSON.stringify(descriptor), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ), + ) + : Effect.never, + ), + ), + ); + +describe("PeerEnvironmentRegistry", () => { + it.effect("registers, lists and forgets a peer worker", () => + Effect.gen(function* () { + const registry = Context.get( + yield* fakeContext(self), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + assert.equal(registered.environmentId, workerPeer.environmentId); + assert.equal(registered.role, "worker"); + assert.equal(registered.workerClass, "ephemeral"); + assert.deepEqual(registered.urls, ["http://127.0.0.1:8111"]); + assert.isTrue(Option.isNone(registered.session)); + + assert.equal((yield* registry.list).length, 1); + + assert.isTrue(yield* registry.unregister(registered.environmentId)); + assert.deepEqual(yield* registry.list, []); + assert.isTrue(Option.isNone(yield* registry.get(registered.environmentId))); + }).pipe(Effect.scoped), + ); + + it.effect("registers a managing instance without a worker class", () => + Effect.gen(function* () { + const registry = Context.get( + yield* fakeContext(self), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const managing = yield* registry.register({ + environmentId: descriptorOf("environment-manager", "Manager").environmentId, + role: "managing", + label: "Manager", + urls: ["http://127.0.0.1:8222"], + }); + assert.isNull(managing.workerClass); + }).pipe(Effect.scoped), + ); + + it.effect("refuses to manage peers while the fleet beta is off", () => + Effect.gen(function* () { + const registry = Context.get( + yield* fakeContext(self, {}, false), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const refused = yield* registry.register(worker()).pipe(Effect.flip); + assert.equal(reasonOf(refused), "disabled"); + }).pipe(Effect.scoped), + ); + + it.effect("refuses to register this environment as its own peer", () => + Effect.gen(function* () { + const registry = Context.get( + yield* fakeContext(self), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const refused = yield* registry + .register({ ...worker(), environmentId: self.environmentId }) + .pipe(Effect.flip); + assert.equal(reasonOf(refused), "self"); + }).pipe(Effect.scoped), + ); + + it.effect("exchanges a bootstrap credential for a stored delegation session", () => + Effect.gen(function* () { + const peer = fakePeer(workerPeer); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + + const paired = yield* registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "one-time-bootstrap-credential", + }); + const session = paired.session; + assert.isTrue(Option.isSome(session)); + assert.deepEqual(Option.getOrThrow(session).scopes, [ + "orchestration:read", + "orchestration:operate", + "terminal:operate", + ]); + assert.equal(paired.label, "Worker One"); + // The probe proved the address, then the exchange used it once. + assert.equal(peer.probes(), 1); + assert.equal(peer.exchanges().length, 1); + const exchange = peer.exchanges()[0]!; + assert.equal(exchange.grant_type, "urn:ietf:params:oauth:grant-type:token-exchange"); + assert.equal(exchange.subject_token, "one-time-bootstrap-credential"); + assert.equal( + exchange.subject_token_type, + "urn:t3:params:oauth:token-type:environment-bootstrap", + ); + assert.equal(exchange.scope, GRANTED_SCOPES); + + // The bridge resolves the stored session, with the token from the peer. + const resolved = yield* registry.resolveSession(registered.environmentId); + assert.equal(resolved.token, "session-token"); + assert.equal(resolved.url, "http://127.0.0.1:8111"); + assert.equal(resolved.label, "Worker One"); + + yield* TestClock.adjust(Duration.seconds(3600)); + const expired = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip); + assert.equal(expired.reason, "not_paired"); + }).pipe(Effect.scoped), + ); + + it.effect("binds the session to the verified address and drops it when addresses change", () => + Effect.gen(function* () { + const peer = fakePeer(workerPeer, { answersAt: "http://127.0.0.1:8112" }); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const urls = ["http://127.0.0.1:8111", "http://127.0.0.1:8112"]; + const registered = yield* registry.register(worker(urls)); + yield* registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "one-time-bootstrap-credential", + }); + assert.equal( + (yield* registry.resolveSession(registered.environmentId)).url, + "http://127.0.0.1:8112", + ); + + // Re-registering the same addresses keeps the session and its verified address. + yield* registry.register(worker(urls)); + assert.equal( + (yield* registry.resolveSession(registered.environmentId)).url, + "http://127.0.0.1:8112", + ); + + yield* registry.register(worker(["http://127.0.0.1:9999"])); + const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip); + assert.equal(refused.reason, "not_paired"); + }).pipe(Effect.scoped), + ); + + it.effect("compares address sets without duplicates when keeping a session", () => + Effect.gen(function* () { + const peer = fakePeer(workerPeer, { answersAt: "http://127.0.0.1:8112" }); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const urls = ["http://127.0.0.1:8111", "http://127.0.0.1:8112"]; + const registered = yield* registry.register(worker(urls)); + yield* registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "one-time-bootstrap-credential", + }); + + // [A, A, B] is the same SET as the stored [A, B]: the session stays. + yield* registry.register(worker([urls[0]!, urls[0]!, urls[1]!])); + assert.equal( + (yield* registry.resolveSession(registered.environmentId)).url, + "http://127.0.0.1:8112", + ); + + // [A, A] is a DIFFERENT set from the stored [A, B]: the session is dropped. + yield* registry.register(worker([urls[0]!, urls[0]!])); + const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip); + assert.equal(refused.reason, "not_paired"); + }).pipe(Effect.scoped), + ); + + it.effect("re-pairing replaces the stored session token", () => + Effect.gen(function* () { + let issued = 0; + const peer = fakePeer(workerPeer, { accessToken: () => `session-token-${(issued += 1)}` }); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + yield* registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "first-credential", + }); + assert.equal( + (yield* registry.resolveSession(registered.environmentId)).token, + "session-token-1", + ); + + yield* registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "second-credential", + }); + const resolved = yield* registry.resolveSession(registered.environmentId); + assert.equal(resolved.token, "session-token-2"); + // Re-pairing updates the row, not a second one. + assert.equal((yield* registry.list).length, 1); + assert.equal(peer.exchanges().length, 2); + }).pipe(Effect.scoped), + ); + + it.effect("refuses an exchange narrowed below the delegation scopes", () => + Effect.gen(function* () { + const peer = fakePeer(workerPeer, { scope: "orchestration:read orchestration:operate" }); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + const refused = yield* registry + .exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "narrow-credential", + }) + .pipe(Effect.flip); + assert.equal(reasonOf(refused), "exchange_rejected"); + const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId)); + assert.isTrue(Option.isNone(unpaired.session)); + assert.equal(peer.exchanges().length, 1); + }).pipe(Effect.scoped), + ); + + it.effect("refuses an exchange at an address that answers as another environment", () => + Effect.gen(function* () { + const peer = fakePeer(descriptorOf("environment-stranger", "Stranger")); + const registry = Context.get( + yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + const refused = yield* registry + .exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "credential", + }) + .pipe(Effect.flip); + assert.equal(reasonOf(refused), "unreachable"); + const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId)); + assert.isTrue(Option.isNone(unpaired.session)); + assert.equal(peer.exchanges().length, 0); + }).pipe(Effect.scoped), + ); + + it.effect("refuses to resolve a peer that never paired", () => + Effect.gen(function* () { + const registry = Context.get( + yield* fakeContext(self), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip); + assert.equal(reasonOf(refused), "not_paired"); + const unknown = yield* registry + .resolveSession(descriptorOf("environment-nowhere", "Nowhere").environmentId) + .pipe(Effect.flip); + assert.equal(reasonOf(unknown), "unknown_peer"); + }).pipe(Effect.scoped), + ); + + it.effect("fails the exchange with exchange_timeout when the verified peer hangs", () => + Effect.gen(function* () { + const registry = Context.get( + yield* silentExchangeContext(workerPeer), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker()); + const fiber = yield* Effect.forkScoped( + registry.exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "credential", + }), + ); + yield* TestClock.adjust(Duration.seconds(16)); + const refused = yield* Fiber.join(fiber).pipe(Effect.flip); + assert.equal(reasonOf(refused), "exchange_timeout"); + const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId)); + assert.isTrue(Option.isNone(unpaired.session)); + }).pipe(Effect.scoped), + ); + + it.effect("refuses a redirect after a verified descriptor and forwards no token", () => + Effect.gen(function* () { + const received: Array = []; + let serverPort = 0; + const server = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const listening = NodeHttp.createServer((request, response) => { + const url = request.url ?? ""; + if (url === "/.well-known/t3/environment") { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(workerPeer)); + return; + } + if (url === "/oauth/token") { + // A 307 after the descriptor: the redirect target must get nothing. + response.writeHead(307, { + location: `http://127.0.0.1:${serverPort}/evil/token`, + }); + response.end(); + return; + } + received.push(request.headers.authorization ?? ""); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ access_token: "stolen", scope: GRANTED_SCOPES })); + }); + listening.listen(0, "127.0.0.1", () => { + serverPort = (listening.address() as NodeNet.AddressInfo).port; + resume(Effect.succeed(listening)); + }); + }), + (listening) => Effect.sync(() => listening.close()), + ); + const port = (server.address() as NodeNet.AddressInfo).port; + const registry = Context.get( + yield* buildContext(FetchHttpClient.layer), + PeerEnvironmentRegistry.PeerEnvironmentRegistry, + ); + const registered = yield* registry.register(worker([`http://127.0.0.1:${port}`])); + const refused = yield* registry + .exchangeBootstrap({ + environmentId: registered.environmentId, + bootstrapCredential: "credential", + }) + .pipe(Effect.flip); + assert.equal(reasonOf(refused), "exchange_rejected"); + // The redirect target received no request, so no credential reached it. + assert.deepEqual(received, []); + const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId)); + assert.isTrue(Option.isNone(unpaired.session)); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/peer/PeerEnvironmentRegistry.ts b/apps/server/src/peer/PeerEnvironmentRegistry.ts new file mode 100644 index 0000000000000..7078959413100 --- /dev/null +++ b/apps/server/src/peer/PeerEnvironmentRegistry.ts @@ -0,0 +1,588 @@ +import { + AuthAccessTokenType, + AuthEnvironmentBootstrapTokenType, + AuthTokenExchangeGrantType, + AuthOrchestrationOperateScope, + AuthOrchestrationReadScope, + AuthTerminalOperateScope, + EnvironmentHttpApi, + EnvironmentId, + type ExecutionEnvironmentDescriptor, + type PeerLinkError, + OpenShellWorkerClass, +} from "@t3tools/contracts"; +import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; +import * as Context from "effect/Context"; +import * as SecureRandom from "effect/crypto/SecureRandom"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Hex from "effect/encoding/Hex"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as Schema from "effect/Schema"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; +import * as HttpClient from "effect/http/HttpClient"; +import * as HttpApi from "effect/http-api/HttpApi"; +import * as HttpApiClient from "effect/http-api/HttpApiClient"; +import { isLocalLoopbackHost } from "@t3tools/shared/hostClassification"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { peerOrigin } from "./PeerLinks.ts"; + +const PeerHttpApi = HttpApi.make(EnvironmentHttpApi.identifier) + .add(EnvironmentHttpApi.groups.metadata) + .add(EnvironmentHttpApi.groups.auth) + .annotateMerge(EnvironmentHttpApi.annotations); + +/** How long a peer's descriptor may take before an address counts as unreachable. */ +const probeTimeout = (origin: string) => { + // An unparseable origin is just an unreachable address; it gets the default. + const hostname = (() => { + try { + return new URL(origin).hostname; + } catch { + return null; + } + })(); + return hostname !== null && isLocalLoopbackHost(hostname) + ? Duration.seconds(3) + : Duration.seconds(10); +}; + +/** The bootstrap exchange must not park a spawn on a silent verified peer. */ +const exchangeTimeout = Duration.seconds(15); + +/** + * Scopes a delegation session asks for: the narrowest set that launches, + * follows and steers threads on the peer; the peer may grant narrower. + * OPEN-QUESTION (spec section 5.2, lanes 2 and 7): final scope set. + */ +const DelegationSessionScopes = [ + AuthOrchestrationReadScope, + AuthOrchestrationOperateScope, + AuthTerminalOperateScope, +] as const; + +const OPENSHELL_FLEET_OFF = + "OpenShell fleet management is off on this machine. Turn it on in Settings → Labs."; + +export type PeerEnvironmentRole = "managing" | "worker"; + +export interface PeerEnvironmentSession { + /** The scopes the peer actually granted, which never exceed the ask. */ + readonly scopes: ReadonlyArray; + readonly expiresAt: DateTime.Utc; +} + +/** One registered peer t3 instance: the managing instance or a worker. */ +export interface PeerEnvironment { + readonly environmentId: EnvironmentId; + readonly role: PeerEnvironmentRole; + readonly label: string; + readonly urls: ReadonlyArray; + /** Workers only; the managing instance has none. */ + readonly workerClass: OpenShellWorkerClass | null; + readonly registeredAt: DateTime.Utc; + /** Present once a bootstrap exchange paired this environment with the peer. */ + readonly session: Option.Option; + readonly lastError: string | null; +} + +/** A delegation session ready to call the peer's bridge with. */ +export interface ResolvedPeerSession { + readonly environmentId: EnvironmentId; + readonly label: string; + /** The first registered address, which pairing proved answers as this peer. */ + readonly url: string; + readonly token: string; +} + +export class PeerEnvironmentRegistryError extends Schema.TaggedError()( + "PeerEnvironmentRegistryError", + { + reason: Schema.Literals([ + "storage", + "disabled", + "unknown_peer", + "self", + "unreachable", + "not_paired", + "exchange_rejected", + "exchange_timeout", + ]), + message: Schema.String, + cause: Schema.optional(Schema.Defect()), + }, +) {} + +const isRegistryError = Schema.is(PeerEnvironmentRegistryError); + +/** Fails with a fixed storage error that keeps its cause; the log gets the details. */ +const failStorage = (cause: unknown) => + Effect.logWarning("Fleet peer environment storage failed").pipe( + Effect.andThen( + Effect.fail( + new PeerEnvironmentRegistryError({ + reason: "storage", + message: + "The fleet peer registry could not be read or saved here. The server log has details.", + cause, + }), + ), + ), + ); + +const decodeUrls = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.String))); +const encodeUrls = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(Schema.String))); +const decodeWorkerClass = Schema.decodeUnknownEffect(Schema.NullOr(OpenShellWorkerClass)); +const decodeScopes = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.String))); +const encodeScopes = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(Schema.String))); + +/** A peer address that may receive a credential, or why it may not. */ +const originOf = (url: string): Effect.Effect => + // peerOrigin's returns infer as a union of effects; one assertion narrows the + // error, which is only ever a PeerLinkError. + Effect.suspend(() => peerOrigin(url) as Effect.Effect).pipe( + Effect.mapError( + (error) => + new PeerEnvironmentRegistryError({ reason: "unreachable", message: error.message }), + ), + ); + +/** + * Registry of peer t3 instances this environment knows as a fleet: itself as + * the managing instance plus the workers it delegates to. Pairing trades a + * peer's one-time bootstrap credential for a delegation session; the token + * lives in the server secret store, the row keeps the rest. + */ +export class PeerEnvironmentRegistry extends Context.Service< + PeerEnvironmentRegistry, + { + /** Whether the OpenShell fleet beta (Settings → Labs) is on here. */ + readonly enabled: Effect.Effect; + readonly register: (input: { + readonly environmentId: EnvironmentId; + readonly role: PeerEnvironmentRole; + readonly label: string; + readonly urls: ReadonlyArray; + readonly workerClass?: OpenShellWorkerClass; + }) => Effect.Effect; + /** + * Forgets the peer here and removes its delegation session. The peer still + * lists this environment's session until it is revoked there, like + * unlinking a peer link. + */ + readonly unregister: ( + environmentId: EnvironmentId, + ) => Effect.Effect; + readonly list: Effect.Effect, PeerEnvironmentRegistryError>; + readonly get: ( + environmentId: EnvironmentId, + ) => Effect.Effect, PeerEnvironmentRegistryError>; + /** + * Pairing bootstrap exchange: the peer's one-time bootstrap credential + * becomes a delegation session scoped to `DelegationSessionScopes`. The + * credential goes only to an address whose descriptor proves it is the + * registered peer. Re-pairing replaces the stored token. + */ + readonly exchangeBootstrap: (input: { + readonly environmentId: EnvironmentId; + readonly bootstrapCredential: string; + }) => Effect.Effect; + /** The stored delegation session, ready to call the peer's bridge with. */ + readonly resolveSession: ( + environmentId: EnvironmentId, + ) => Effect.Effect; + } +>()("t3/peer/PeerEnvironmentRegistry") {} + +interface PeerEnvironmentRow { + readonly environment_id: string; + readonly role: string; + readonly label: string; + readonly urls_json: string; + readonly worker_class: string | null; + readonly secret_name: string | null; + readonly session_scopes_json: string | null; + readonly session_expires_at: string | null; + readonly registered_at: string; + readonly last_error: string | null; +} + +const toPeerEnvironment = Effect.fn("PeerEnvironmentRegistry.toPeerEnvironment")(function* ( + row: PeerEnvironmentRow, +) { + const session = + row.secret_name !== null && row.session_scopes_json !== null && row.session_expires_at !== null + ? Option.some({ + scopes: yield* decodeScopes(row.session_scopes_json).pipe(Effect.catch(failStorage)), + expiresAt: DateTime.makeUnsafe(row.session_expires_at), + } satisfies PeerEnvironmentSession) + : Option.none(); + return { + environmentId: EnvironmentId.make(row.environment_id), + role: row.role as PeerEnvironmentRole, + label: row.label, + urls: yield* decodeUrls(row.urls_json).pipe(Effect.catch(failStorage)), + workerClass: yield* decodeWorkerClass(row.worker_class).pipe(Effect.catch(failStorage)), + registeredAt: DateTime.makeUnsafe(row.registered_at), + session, + lastError: row.last_error, + } satisfies PeerEnvironment; +}); + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const secrets = yield* ServerSecretStore.ServerSecretStore; + const random = yield* SecureRandom.SecureRandom; + const httpClient = yield* HttpClient.HttpClient; + const environment = yield* ServerEnvironment.ServerEnvironment; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const enabled = serverSettings.getSettings.pipe( + Effect.map((settings) => settings.openshellEnabled), + Effect.orElseSucceed(() => false), + ); + + /** A fresh slot name for one peer's delegation session token; the row records it. */ + const newSecretName = random.randomBytes(16).pipe( + Effect.map((bytes) => `fleet-peer-${Hex.encode(bytes)}`), + Effect.orDie, + ); + + // Only the origin `peerOrigin` checked may receive a bootstrap credential or + // token: a redirect could forward them anywhere, including plain http. + const peerHttpClient = HttpClient.transform(httpClient, (response) => + Effect.provideService(response, FetchHttpClient.RequestInit, { redirect: "manual" }), + ); + + const clientFor = (origin: string) => + HttpApiClient.make(PeerHttpApi, { baseUrl: origin }).pipe( + Effect.provideService(HttpClient.HttpClient, peerHttpClient), + ); + + /** The environment answering at `origin`, or nothing when it does not answer as T3 Code. */ + const describe = (origin: string) => + clientFor(origin).pipe( + Effect.flatMap((client) => client.metadata.descriptor()), + Effect.timeout(probeTimeout(origin)), + Effect.option, + ); + + const readRows = sql` + SELECT * FROM fleet_peer_environments ORDER BY label, environment_id + `.pipe(NodeSqliteClient.readOnly, Effect.catch(failStorage)); + + const readRow = (environmentId: EnvironmentId) => + sql` + SELECT * FROM fleet_peer_environments WHERE environment_id = ${environmentId} + `.pipe( + NodeSqliteClient.readOnly, + Effect.catch(failStorage), + Effect.map((rows) => Option.fromNullishOr(rows[0])), + ); + + const requireRow = (environmentId: EnvironmentId) => + readRow(environmentId).pipe( + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail( + new PeerEnvironmentRegistryError({ + reason: "unknown_peer", + message: `${environmentId} is not a registered fleet peer.`, + }), + ), + onSome: Effect.succeed, + }), + ), + ); + + const selfEnvironmentId = environment.getEnvironmentId.pipe( + Effect.orElseSucceed(() => EnvironmentId.make("environment-self")), + ); + + const register: PeerEnvironmentRegistry["Service"]["register"] = Effect.fn( + "PeerEnvironmentRegistry.register", + )(function* (input) { + if (!(yield* enabled)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "disabled", + message: OPENSHELL_FLEET_OFF, + }); + } + if (input.environmentId === (yield* selfEnvironmentId)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "self", + message: "That environment is this one.", + }); + } + const urls = yield* Effect.forEach(input.urls, originOf); + const uniqueUrls = [...new Set(urls)]; + const previous = yield* readRow(input.environmentId); + const previousUrls = Option.isSome(previous) + ? yield* decodeUrls(previous.value.urls_json).pipe(Effect.catch(failStorage)) + : []; + // The same address SET (deduped) keeps the pairing: the session and its + // verified-first order stay. In the upsert below the keepSession branch + // leaves urls_json alone, so it cannot undo a bootstrap exchange that + // committed while this register ran. + const addressSet = new Set(uniqueUrls); + const previousSet = new Set(previousUrls); + const keepSession = + Option.isSome(previous) && + addressSet.size === previousSet.size && + [...addressSet].every((url) => previousSet.has(url)); + const urlsJson = yield* encodeUrls(uniqueUrls).pipe(Effect.catch(failStorage)); + const now = yield* DateTime.now; + yield* sql` + INSERT INTO fleet_peer_environments ( + environment_id, role, label, urls_json, worker_class, secret_name, + session_scopes_json, session_expires_at, registered_at, last_error + ) VALUES ( + ${input.environmentId}, ${input.role}, ${input.label}, + ${urlsJson}, + ${input.workerClass ?? null}, NULL, NULL, NULL, ${DateTime.formatIso(now)}, NULL + ) + ON CONFLICT (environment_id) DO UPDATE SET + role = excluded.role, + label = excluded.label, + urls_json = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN urls_json ELSE excluded.urls_json END, + worker_class = excluded.worker_class, + secret_name = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN secret_name END, + session_scopes_json = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN session_scopes_json END, + session_expires_at = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN session_expires_at END, + last_error = NULL + `.pipe(Effect.catch(failStorage)); + // New addresses were never checked against this peer, so its token must not follow them. + if (Option.isSome(previous) && !keepSession && previous.value.secret_name !== null) { + yield* secrets.remove(previous.value.secret_name).pipe(Effect.ignore); + } + return Option.getOrThrow(yield* get(input.environmentId)); + }); + + const unregister: PeerEnvironmentRegistry["Service"]["unregister"] = Effect.fn( + "PeerEnvironmentRegistry.unregister", + )(function* (environmentId) { + if (!(yield* enabled)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "disabled", + message: OPENSHELL_FLEET_OFF, + }); + } + const row = yield* requireRow(environmentId); + // Remove the credential before forgetting the row. If removal fails, the + // row keeps secret_name so a later unregister retries credential cleanup. + if (row.secret_name !== null) { + yield* secrets.remove(row.secret_name).pipe(Effect.catch(failStorage)); + } + const deleted = yield* sql<{ environment_id: string }>` + DELETE FROM fleet_peer_environments WHERE environment_id = ${environmentId} + RETURNING environment_id + `.pipe(Effect.catch(failStorage)); + return deleted.length > 0; + }); + + const list: PeerEnvironmentRegistry["Service"]["list"] = Effect.gen(function* () { + const rows = yield* readRows; + return yield* Effect.forEach(rows, toPeerEnvironment); + }); + + const get: PeerEnvironmentRegistry["Service"]["get"] = (environmentId) => + readRow(environmentId).pipe( + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeedNone, + onSome: (row) => Effect.asSome(toPeerEnvironment(row)), + }), + ), + ); + + const exchangeBootstrap: PeerEnvironmentRegistry["Service"]["exchangeBootstrap"] = Effect.fn( + "PeerEnvironmentRegistry.exchangeBootstrap", + )(function* (input) { + if (!(yield* enabled)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "disabled", + message: OPENSHELL_FLEET_OFF, + }); + } + const row = yield* requireRow(input.environmentId); + const urls = yield* decodeUrls(row.urls_json).pipe(Effect.catch(failStorage)); + // The credential goes only to an address whose descriptor proves it is + // this peer, so a reused address never receives it. + const checked = yield* Effect.forEach( + urls, + (url) => + Effect.gen(function* () { + const origin = yield* originOf(url); + return { origin, descriptor: yield* describe(origin) }; + }).pipe(Effect.result), + { concurrency: "unbounded" }, + ); + const answering = checked.flatMap((result) => { + if (!Result.isSuccess(result) || Option.isNone(result.success.descriptor)) return []; + const descriptor = result.success.descriptor.value; + const modeLimitsHold = + descriptor.capabilities.mcpModeLimitHeader === true && + descriptor.capabilities.linkFence === true; + return descriptor.environmentId === input.environmentId && modeLimitsHold + ? [{ origin: result.success.origin, descriptor }] + : []; + }); + const found = answering[0]; + if (found === undefined) { + return yield* new PeerEnvironmentRegistryError({ + reason: "unreachable", + message: `${input.environmentId} did not answer as itself at ${urls.join(", ")}. No session was stored.`, + }); + } + const descriptor = found.descriptor as ExecutionEnvironmentDescriptor; + // resolveSession sends the token to the first address, so that must be the verified one. + const verifiedUrlsJson = yield* encodeUrls([ + found.origin, + ...urls.filter((url) => url !== found.origin), + ]).pipe(Effect.catch(failStorage)); + const exchanged = yield* clientFor(found.origin).pipe( + Effect.flatMap((client) => + client.auth.token({ + headers: {}, + payload: { + grant_type: AuthTokenExchangeGrantType, + subject_token: input.bootstrapCredential, + subject_token_type: AuthEnvironmentBootstrapTokenType, + requested_token_type: AuthAccessTokenType, + scope: [...DelegationSessionScopes].join(" "), + }, + }), + ), + Effect.timeoutOption(exchangeTimeout), + Effect.flatMap( + Option.match({ + onNone: () => + Effect.fail( + new PeerEnvironmentRegistryError({ + reason: "exchange_timeout", + message: `${input.environmentId} did not answer the bootstrap exchange in time. No session was stored.`, + }), + ), + onSome: Effect.succeed, + }), + ), + Effect.mapError((cause) => + isRegistryError(cause) + ? cause + : new PeerEnvironmentRegistryError({ + reason: "exchange_rejected", + message: `${descriptor.label} refused the delegation session exchange. Check its pairing credential and try again.`, + cause, + }), + ), + ); + const granted = exchanged.scope.split(" ").filter((scope) => scope.length > 0); + const missing = DelegationSessionScopes.filter((scope) => !granted.includes(scope)); + if (granted.length === 0 || missing.length > 0) { + return yield* new PeerEnvironmentRegistryError({ + reason: "exchange_rejected", + message: `${descriptor.label} granted a session without ${ + missing.length > 0 ? missing.join(", ") : "any delegated scope" + }. Issue a pairing credential that carries them.`, + }); + } + const expiresAt = DateTime.add(yield* DateTime.now, { seconds: exchanged.expires_in }); + // The token goes under a name of its own before the row points at it, so + // the row and its token are replaced together. + const secretName = yield* newSecretName; + yield* secrets + .set(secretName, new TextEncoder().encode(exchanged.access_token)) + .pipe(Effect.catch(failStorage)); + // `RETURNING` closes the race: a peer unregistered mid-exchange yields no row, + // and the fresh secret is discarded instead of leaving an orphan token behind. + const updated = yield* sql<{ readonly secret_name: string | null }>` + UPDATE fleet_peer_environments + SET urls_json = ${verifiedUrlsJson}, + secret_name = ${secretName}, session_scopes_json = ${yield* encodeScopes(granted).pipe( + Effect.catch(failStorage), + )}, + session_expires_at = ${DateTime.formatIso(expiresAt)}, label = ${descriptor.label}, + last_error = NULL + WHERE environment_id = ${input.environmentId} AND urls_json = ${row.urls_json} + RETURNING secret_name + `.pipe( + Effect.catch((cause) => + secrets.remove(secretName).pipe(Effect.ignore, Effect.andThen(failStorage(cause))), + ), + ); + if (updated.length === 0) { + yield* secrets.remove(secretName).pipe(Effect.ignore); + return yield* new PeerEnvironmentRegistryError({ + reason: "unknown_peer", + message: `${input.environmentId} was unregistered or changed addresses during pairing, so its session was not stored.`, + }); + } + const previous = row; + // Pairing again replaces the old token; a racing exchange may leave its + // own behind, unused either way. + if (previous.secret_name !== null && previous.secret_name !== secretName) { + yield* secrets.remove(previous.secret_name).pipe(Effect.ignore); + } + return Option.getOrThrow(yield* get(input.environmentId)); + }); + + const resolveSession: PeerEnvironmentRegistry["Service"]["resolveSession"] = Effect.fn( + "PeerEnvironmentRegistry.resolveSession", + )(function* (environmentId) { + // Resolution stays available while the beta is off, so work already + // running on a peer still finishes, like linked environments do. + const row = yield* requireRow(environmentId); + const peer = yield* toPeerEnvironment(row); + if (Option.isNone(peer.session) || row.secret_name === null) { + return yield* new PeerEnvironmentRegistryError({ + reason: "not_paired", + message: `${environmentId} has no delegation session yet. Pair it with a bootstrap credential from the peer.`, + }); + } + if (DateTime.isLessThanOrEqualTo(peer.session.value.expiresAt, yield* DateTime.now)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "not_paired", + message: `The delegation session for ${peer.label} expired. Re-run t3_environment_spawn with the same clientRequestId (or without one, if it was spawned without) to re-pair the existing worker.`, + }); + } + const stored = yield* secrets.get(row.secret_name).pipe(Effect.catch(failStorage)); + if (Option.isNone(stored)) { + return yield* new PeerEnvironmentRegistryError({ + reason: "unknown_peer", + message: `The delegation session for ${peer.label} lost its credential. Pair it again.`, + }); + } + const url = peer.urls[0]; + if (url === undefined) { + return yield* new PeerEnvironmentRegistryError({ + reason: "unreachable", + message: `${peer.label} has no address to reach it at.`, + }); + } + return { + environmentId: peer.environmentId, + label: peer.label, + url, + token: new TextDecoder().decode(stored.value), + }; + }); + + return PeerEnvironmentRegistry.of({ + enabled, + register, + unregister, + list, + get, + exchangeBootstrap, + resolveSession, + }); +}); + +export const layer = Layer.effect(PeerEnvironmentRegistry, make); diff --git a/apps/server/src/peer/PeerForwarding.test.ts b/apps/server/src/peer/PeerForwarding.test.ts index 1d3f563f35b08..1c13ef330be69 100644 --- a/apps/server/src/peer/PeerForwarding.test.ts +++ b/apps/server/src/peer/PeerForwarding.test.ts @@ -40,6 +40,7 @@ import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as RemoteDelegation from "./RemoteDelegation.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { liveThreadShell } from "../mcp/McpToolAccess.testkit.ts"; import * as PeerForwarding from "./PeerForwarding.ts"; @@ -239,7 +240,9 @@ const serveBox = (seen: Ref.Ref, linkSession: Ref.Ref) => servePeer( box, Layer.mergeAll( - McpHttpServer.layerOrchestratorToolkit, + McpHttpServer.layerOrchestratorToolkit.pipe( + Layer.provide(OpenShellFleetService.layerDisabled), + ), McpHttpServer.layerProjectRegistration, McpHttpServer.layerThreadToolkit, ).pipe( @@ -294,7 +297,7 @@ const mcpClient = McpSchema.McpServerClient.of({ const makeLaptop = Effect.gen(function* () { const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); const layerHere = Layer.mergeAll( - McpHttpServer.layerOrchestratorToolkit, + McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)), McpHttpServer.layerProjectRegistration, McpHttpServer.layerThreadToolkit, ).pipe( diff --git a/apps/server/src/peer/PeerLinkRequests.test.ts b/apps/server/src/peer/PeerLinkRequests.test.ts index 2eec22a90bfa0..12223f8390342 100644 --- a/apps/server/src/peer/PeerLinkRequests.test.ts +++ b/apps/server/src/peer/PeerLinkRequests.test.ts @@ -35,6 +35,7 @@ import { McpSchema, McpServer, Tool, Toolkit } from "effect/ai"; import * as ServerConfig from "../config.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import * as McpToolAccess from "../mcp/McpToolAccess.ts"; import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; @@ -164,6 +165,7 @@ const makeLaptopWith = ( Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())), ); const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe( + Layer.provide(OpenShellFleetService.layerDisabled), Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), Layer.provideMerge(PeerLinkRequests.layer), Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})), diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 5325e5745c5d7..6684c3c31f7e7 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -172,7 +172,7 @@ const failStorage = (cause: unknown) => * code and then the link's token travel to it, so plain http is accepted only * where the network already encrypts: this machine, or a Tailscale address. */ -const peerOrigin = (url: string) => { +export const peerOrigin = (url: string) => { const parsed = URL.parse(url); if (parsed === null || (parsed.protocol !== "https:" && parsed.protocol !== "http:")) { return Effect.fail( diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts index da28f01e375b9..f65cfe248fc56 100644 --- a/apps/server/src/peer/PeerMcpClient.test.ts +++ b/apps/server/src/peer/PeerMcpClient.test.ts @@ -1,9 +1,10 @@ import { NodeHttpServer } from "@effect/platform-node"; -import { expect, it } from "@effect/vitest"; +import { assert, expect, it } from "@effect/vitest"; import { EnvironmentId, OrchestrationV2LinkedSender, OrchestratorMcpFailure, + PeerLinkError, ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; @@ -24,6 +25,7 @@ import { HttpClient, HttpClientResponse } from "effect/http"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import * as McpToolAccess from "../mcp/McpToolAccess.ts"; +import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts"; import * as PeerLinks from "./PeerLinks.ts"; import * as PeerMcpClient from "./PeerMcpClient.ts"; import { descriptorOf, layerLinkingEnvironment, linkTo, servePeer } from "./PeerLinks.testkit.ts"; @@ -447,3 +449,102 @@ it.effect("bounds the session handshake, not a tool call that waits longer", () expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" }); }), ); + +it.effect("reaches an unlinked worker through its fleet delegation session", () => + Effect.gen(function* () { + const requests: Array<{ readonly url: string; readonly authorization: string | undefined }> = + []; + let calls = 0; + const http = HttpClient.make((request) => + Effect.sync(() => { + requests.push({ url: request.url, authorization: request.headers.authorization }); + const step = requests.length; + const toolCallBody = JSON.stringify({ + result: { content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }] }, + }); + if (step === 1) { + return HttpClientResponse.fromWeb( + request, + new Response(toolCallBody, { + status: 200, + headers: { "mcp-session-id": "fleet-session" }, + }), + ); + } + calls += 1; + return HttpClientResponse.fromWeb( + request, + new Response(step === 2 ? "" : toolCallBody, { + status: step === 2 ? 202 : 200, + headers: { "mcp-session-id": "fleet-session" }, + }), + ); + }), + ); + const fleet = descriptorOf("environment-fleet", "Fleet Worker"); + const context = yield* Layer.build( + PeerMcpClient.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.succeed(HttpClient.HttpClient, http), + Layer.mock(PeerLinks.PeerLinks)({ + resolve: (environmentId) => + environmentId === fleet.environmentId + ? Effect.fail( + new PeerLinkError({ + reason: "unknown_link", + message: "This environment is not linked.", + }), + ) + : Effect.fail( + new PeerLinkError({ + reason: "storage", + message: "Not part of this test.", + }), + ), + recordOutcome: () => Effect.void, + }), + Layer.succeed(PeerEnvironmentRegistry.PeerEnvironmentRegistry, { + enabled: Effect.succeed(true), + register: () => Effect.die("not part of this test"), + unregister: () => Effect.die("not part of this test"), + list: Effect.die("not part of this test"), + get: () => Effect.die("not part of this test"), + exchangeBootstrap: () => Effect.die("not part of this test"), + resolveSession: (environmentId) => + environmentId === fleet.environmentId + ? Effect.succeed({ + environmentId: fleet.environmentId, + label: fleet.label, + url: "http://127.0.0.1:8333", + token: "fleet-delegation-session-token", + }) + : Effect.fail( + new PeerEnvironmentRegistry.PeerEnvironmentRegistryError({ + reason: "unknown_peer", + message: "Not a registered peer.", + }), + ), + } satisfies Partial< + PeerEnvironmentRegistry.PeerEnvironmentRegistry["Service"] + > as never), + ), + ), + ), + ); + const client = Context.get(context, PeerMcpClient.PeerMcpClient); + assert.deepEqual( + yield* client.call({ + environmentId: fleet.environmentId, + tool: "probe", + arguments: {}, + limits: { runtimeMode: "approval-required", interactionMode: "plan" }, + success: Result, + }), + { modes: "approval-required/plan" }, + ); + // The worker's session carried the call; the registry session was its credential. + assert.include(requests[0]?.authorization ?? "", "fleet-delegation-session-token"); + assert.equal(calls, 2); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index 25c3b70287ab1..8798f2e3f4463 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -1,5 +1,5 @@ import { - type EnvironmentId, + EnvironmentId, OrchestratorMcpFailure, type PeerLinkError, type ProviderInteractionMode, @@ -19,7 +19,10 @@ import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts"; +import type { ResolvedPeerSession } from "./PeerSessionResolve.ts"; import * as PeerLinks from "./PeerLinks.ts"; +import * as PeerSessionResolve from "./PeerSessionResolve.ts"; /** The only MCP protocol version a T3 Code `/mcp` registers. */ const PROTOCOL_VERSION = "2025-06-18"; @@ -85,6 +88,29 @@ const fromLinkError = (error: PeerLinkError) => message: error.message, }); +/** How the agent hears a fleet registry problem when it asked to call a peer. */ +const fromRegistryError = (error: PeerEnvironmentRegistry.PeerEnvironmentRegistryError) => + new OrchestratorMcpFailure({ + code: + error.reason === "unknown_peer" || error.reason === "not_paired" + ? "invalid_request" + : "orchestration_error", + message: error.message, + }); + +/** + * A worker's credential comes from pairing, not from the user's links, so the + * recovery differs: a link is re-linked in Settings, a worker is re-paired. + */ +const rejected = (from: "link" | "registry") => + new OrchestratorMcpFailure({ + code: "capability_denied", + message: + from === "registry" + ? "This fleet worker no longer accepts its delegation session. It may have been removed or re-paired; spawn a replacement, or re-pair it with a fresh bootstrap credential." + : "The linked environment no longer accepts this link. It may have been revoked there; link it again.", + }); + /** Not final, so a task already running there resumes once the beta is on there again. */ const linkedEnvironmentsOffThere = new OrchestratorMcpFailure({ code: "orchestration_error", @@ -109,6 +135,9 @@ const responseJson = (contentType: string | undefined, text: string) => const make = Effect.gen(function* () { const links = yield* PeerLinks.PeerLinks; + // Optional so environments without a fleet registry build without it; where + // the registry is provided, its delegation sessions answer for their peers. + const registry = yield* Effect.serviceOption(PeerEnvironmentRegistry.PeerEnvironmentRegistry); const httpClient = yield* HttpClient.HttpClient; // One MCP session per peer and address, reused across calls. The peer keeps // sessions in memory, so a restart there answers 404 and we start a new one. @@ -128,26 +157,24 @@ const make = Effect.gen(function* () { // A peer that renews a used link says when it now ends; one that does not sends nothing. const recordLinkExpiry = ( - link: PeerLinks.ResolvedPeerLink["link"], + environmentId: EnvironmentId, headers: Readonly>, ) => Option.match( - Option.filter( - Option.flatMap( - Option.fromNullishOr(headers[McpInvocationContext.LINK_EXPIRES_AT_HEADER]), - DateTime.make, - ), - (expiresAt) => !DateTime.Equivalence(expiresAt, link.expiresAt), + Option.flatMap( + Option.fromNullishOr(headers[McpInvocationContext.LINK_EXPIRES_AT_HEADER]), + DateTime.make, ), { onNone: () => Effect.void, - onSome: (expiresAt) => links.recordExpiry(link.environmentId, expiresAt), + onSome: (expiresAt) => links.recordExpiry(environmentId, expiresAt), }, ); const openSession = Effect.fn("PeerMcpClient.openSession")( function* ( - { link, url, token }: PeerLinks.ResolvedPeerLink, + { url, token, from }: ResolvedPeerSession, + environmentId: EnvironmentId, limitHeader: Record, ) { const initialized = yield* post( @@ -168,15 +195,17 @@ const make = Effect.gen(function* () { const sessionId = initialized.headers["mcp-session-id"]; if (initialized.status === 403) return yield* linkedEnvironmentsOffThere; if (initialized.status !== 200 || sessionId === undefined) { - return yield* new OrchestratorMcpFailure({ - code: initialized.status === 401 ? "capability_denied" : "orchestration_error", - message: - initialized.status === 401 - ? "The linked environment no longer accepts this link. It may have been revoked there; link it again." - : `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`, - }); + return yield* initialized.status === 401 + ? rejected(from) + : new OrchestratorMcpFailure({ + code: "orchestration_error", + message: `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`, + }); + } + // Only link-sourced sessions renew their link's expiry on use. + if (from === "link") { + yield* recordLinkExpiry(environmentId, initialized.headers); } - yield* recordLinkExpiry(link, initialized.headers); yield* post( url, token, @@ -189,11 +218,22 @@ const make = Effect.gen(function* () { (effect, { url }) => Effect.timeout(effect, PeerLinks.probeTimeout(url)), ); + /** + * Where a peer is reached and with what credential: its link if one is + * stored, else its fleet registry delegation session. Both spellings reach + * the peer's `/mcp`, so the bridge, forwarding and delegation work the same + * for linked environments and registered workers. + */ + const resolvePeer = (environmentId: EnvironmentId) => + PeerSessionResolve.resolvePeerSession({ links, registry })(environmentId).pipe( + Effect.mapError((failure) => + failure._tag === "link" ? fromLinkError(failure.error) : fromRegistryError(failure.error), + ), + ); + const call: PeerMcpClient["Service"]["call"] = Effect.fn("PeerMcpClient.call")( function* (input) { - const resolved = yield* links - .resolve(input.environmentId) - .pipe(Effect.mapError(fromLinkError)); + const resolved = yield* resolvePeer(input.environmentId); const limitHeader = { [McpInvocationContext.MODE_LIMIT_HEADER]: `${input.limits.runtimeMode}/${input.limits.interactionMode}`, }; @@ -230,24 +270,19 @@ const make = Effect.gen(function* () { }, ); const existing = (yield* Ref.get(sessions)).get(sessionKey); - const sessionId = existing ?? (yield* openSession(resolved, limitHeader)); + const sessionId = + existing ?? (yield* openSession(resolved, input.environmentId, limitHeader)); // The initialized session outlives a tool request, including a long wait or cancellation. if (existing === undefined) { yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, sessionId)); } let response = yield* callOnce(sessionId); if (response.status === 404) { - const renewed = yield* openSession(resolved, limitHeader); + const renewed = yield* openSession(resolved, input.environmentId, limitHeader); yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, renewed)); response = yield* callOnce(renewed); } - if (response.status === 401) { - return yield* new OrchestratorMcpFailure({ - code: "capability_denied", - message: - "The linked environment no longer accepts this link. It may have been revoked there; link it again.", - }); - } + if (response.status === 401) return yield* rejected(resolved.from); if (response.status === 403) return yield* linkedEnvironmentsOffThere; if (response.status !== 200) { return yield* new OrchestratorMcpFailure({ @@ -255,7 +290,9 @@ const make = Effect.gen(function* () { message: `The linked environment answered HTTP ${response.status}.`, }); } - yield* recordLinkExpiry(resolved.link, response.headers); + if (resolved.from === "link") { + yield* recordLinkExpiry(input.environmentId, response.headers); + } const body = yield* decodeToolCallResponse( responseJson(response.headers["content-type"], yield* response.text), ).pipe( diff --git a/apps/server/src/peer/PeerSessionResolve.test.ts b/apps/server/src/peer/PeerSessionResolve.test.ts new file mode 100644 index 0000000000000..8c73d52da232c --- /dev/null +++ b/apps/server/src/peer/PeerSessionResolve.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "@effect/vitest"; +import { EnvironmentId, PeerLinkError } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import { PeerEnvironmentRegistryError } from "./PeerEnvironmentRegistry.ts"; +import { resolvePeerSession } from "./PeerSessionResolve.ts"; + +const environmentId = EnvironmentId.make("environment:worker"); + +const linkError = (reason: "unknown_link" | "expired") => + new PeerLinkError({ reason, message: reason }); + +const sessionError = () => + new PeerEnvironmentRegistryError({ reason: "not_paired", message: "not paired" }); + +const deps = (input: { + readonly link: Effect.Effect<{ url: string; token: string }, PeerLinkError>; + readonly session?: Effect.Effect<{ url: string; token: string }, PeerEnvironmentRegistryError>; +}) => ({ + links: { resolve: () => input.link } as never, + registry: + input.session === undefined + ? Option.none() + : Option.some({ resolveSession: () => input.session! } as never), +}); + +describe("resolvePeerSession", () => { + it.effect("prefers the ordinary link", () => + Effect.gen(function* () { + const resolved = yield* resolvePeerSession( + deps({ link: Effect.succeed({ url: "http://link.test", token: "link-token" }) }), + )(environmentId); + expect(resolved).toEqual({ url: "http://link.test", token: "link-token", from: "link" }); + }), + ); + + it.effect("falls back to the fleet delegation session for an unlinked worker", () => + Effect.gen(function* () { + const resolved = yield* resolvePeerSession( + deps({ + link: Effect.fail(linkError("unknown_link")), + session: Effect.succeed({ url: "http://worker.test", token: "session-token" }), + }), + )(environmentId); + expect(resolved).toEqual({ + url: "http://worker.test", + token: "session-token", + from: "registry", + }); + }), + ); + + it.effect("reports an expired link instead of using another credential", () => + Effect.gen(function* () { + const failure = yield* resolvePeerSession( + deps({ + link: Effect.fail(linkError("expired")), + session: Effect.succeed({ url: "http://worker.test", token: "session-token" }), + }), + )(environmentId).pipe(Effect.flip); + expect(failure._tag).toBe("link"); + expect(failure._tag === "link" ? failure.error.reason : undefined).toBe("expired"); + }), + ); + + it.effect("has no fallback without a registry", () => + Effect.gen(function* () { + const failure = yield* resolvePeerSession( + deps({ link: Effect.fail(linkError("unknown_link")) }), + )(environmentId).pipe(Effect.flip); + expect(failure._tag).toBe("link"); + }), + ); + + it.effect("reports a registry failure when the worker has no session", () => + Effect.gen(function* () { + const failure = yield* resolvePeerSession( + deps({ + link: Effect.fail(linkError("unknown_link")), + session: Effect.fail(sessionError()), + }), + )(environmentId).pipe(Effect.flip); + expect(failure._tag).toBe("registry"); + }), + ); +}); diff --git a/apps/server/src/peer/PeerSessionResolve.ts b/apps/server/src/peer/PeerSessionResolve.ts new file mode 100644 index 0000000000000..831495011e138 --- /dev/null +++ b/apps/server/src/peer/PeerSessionResolve.ts @@ -0,0 +1,49 @@ +import type { EnvironmentId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; + +import type * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts"; +import type * as PeerLinks from "./PeerLinks.ts"; + +/** Where a peer answer will come from, so a 401 can name the right recovery. */ +export interface ResolvedPeerSession { + readonly url: string; + readonly token: string; + readonly from: "link" | "registry"; +} + +/** + * The credential a peer answers with: its ordinary link first, else the fleet + * delegation session it was paired with. Both spellings reach the peer's + * `/mcp` and `/api`; only an `unknown_link` falls through, so an expired or + * unreachable link is reported instead of silently using another credential. + */ +export const resolvePeerSession = (deps: { + readonly links: PeerLinks.PeerLinks["Service"]; + readonly registry: Option.Option; +}) => + Effect.fn("resolvePeerSession")(function* (environmentId: EnvironmentId) { + const fromLink = yield* Effect.result(deps.links.resolve(environmentId)); + if (Result.isSuccess(fromLink)) { + return { + url: fromLink.success.url, + token: fromLink.success.token, + from: "link", + } satisfies ResolvedPeerSession; + } + if (Option.isNone(deps.registry) || fromLink.failure.reason !== "unknown_link") { + return yield* Effect.fail({ + _tag: "link" as const, + error: fromLink.failure, + }); + } + const session = yield* deps.registry.value + .resolveSession(environmentId) + .pipe(Effect.mapError((error) => ({ _tag: "registry" as const, error }))); + return { + url: session.url, + token: session.token, + from: "registry", + } satisfies ResolvedPeerSession; + }); diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index 3c14fb427fc2e..fbd8d2ebbda6f 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -42,6 +42,7 @@ import { McpSchema, McpServer } from "effect/ai"; import * as ServerConfig from "../config.ts"; import * as McpHttpServer from "../mcp/McpHttpServer.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { idleThreadProjection, liveThreadShell } from "../mcp/McpToolAccess.testkit.ts"; import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts"; @@ -570,7 +571,7 @@ const boxToolkitLayer = (thread: BoxThread) => { ), }); return Layer.merge( - McpHttpServer.layerOrchestratorToolkit, + McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)), McpHttpServer.layerProjectRegistration, ).pipe( Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))), @@ -699,6 +700,7 @@ const makeLaptop = ( Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())), ); const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe( + Layer.provide(OpenShellFleetService.layerDisabled), Layer.provideMerge(McpHttpServer.layerInProcessMcpServer), Layer.provideMerge( RemoteDelegation.layer.pipe( diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 91b521fdb6036..ea5a82b2c6860 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -113,6 +113,9 @@ import Migration0102 from "./Migrations/102_ThreadItemCounts.ts"; import Migration0103 from "./Migrations/103_IssueTrackerAdditionalScopes.ts"; import Migration0104 from "./Migrations/104_ProjectionProjectsTrackedFolders.ts"; import Migration0105 from "./Migrations/105_TaskFolders.ts"; +import Migration0109 from "./Migrations/109_FleetPeerEnvironments.ts"; +import Migration0110 from "./Migrations/110_OpenShellFleetState.ts"; +import Migration0111 from "./Migrations/111_FleetDrainImports.ts"; const migrationEntries = [ [1, "OrchestrationEvents", Migration0001], @@ -221,6 +224,9 @@ const migrationEntries = [ [103, "IssueTrackerAdditionalScopes", Migration0103], [104, "ProjectionProjectsTrackedFolders", Migration0104], [105, "TaskFolders", Migration0105], + [109, "FleetPeerEnvironments", Migration0109], + [110, "OpenShellFleetState", Migration0110], + [111, "FleetDrainImports", Migration0111], ] as const; export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); diff --git a/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts b/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts new file mode 100644 index 0000000000000..48e1de0172738 --- /dev/null +++ b/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts @@ -0,0 +1,24 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Peer t3 instances this environment knows as a fleet: the managing instance + // itself and the workers it delegates to. The delegation session token is + // kept in the server secret store under `secret_name`, like a peer link. + yield* sql` + CREATE TABLE IF NOT EXISTS fleet_peer_environments ( + environment_id TEXT PRIMARY KEY, + role TEXT NOT NULL, + label TEXT NOT NULL, + urls_json TEXT NOT NULL, + worker_class TEXT, + secret_name TEXT, + session_scopes_json TEXT, + session_expires_at TEXT, + registered_at TEXT NOT NULL, + last_error TEXT + ) + `; +}); diff --git a/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts b/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts new file mode 100644 index 0000000000000..f5a81cbfb4983 --- /dev/null +++ b/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts @@ -0,0 +1,25 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Lifecycle of gateway-managed worker environments, tracked here so state + // survives a restart of the managing instance (spec section 5.5). Drain + // progress is the aggregate journal view; per-item acks land with the drain + // protocol itself. + yield* sql` + CREATE TABLE IF NOT EXISTS openshell_fleet_environments ( + environment_id TEXT PRIMARY KEY, + state TEXT NOT NULL, + worker_class TEXT NOT NULL, + label TEXT, + sandbox_name TEXT, + sandbox_id TEXT, + drain_json TEXT, + expires_at INTEGER, + last_error TEXT, + updated_at TEXT NOT NULL + ) + `; +}); diff --git a/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts b/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts new file mode 100644 index 0000000000000..f247a22d3c9ff --- /dev/null +++ b/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts @@ -0,0 +1,21 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + + // Drained thread bundles imported as archived threads: one row per imported + // thread, keyed by thread id, carrying the currency marker (environment, + // drain id, snapshot sequence) that decides idempotency and supersession. + yield* sql` + CREATE TABLE IF NOT EXISTS orchestration_v2_fleet_drain_imports ( + thread_id TEXT PRIMARY KEY, + environment_id TEXT NOT NULL, + drain_id TEXT NOT NULL, + snapshot_sequence INTEGER, + imported_at TEXT NOT NULL, + lossy INTEGER NOT NULL DEFAULT 0, + oversize_attachments_skipped INTEGER NOT NULL DEFAULT 0 + ) + `; +}); diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts index d2b0dafab8731..7c788c4c49d77 100644 --- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts +++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts @@ -85,6 +85,9 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [109, "FleetPeerEnvironments"], + [110, "OpenShellFleetState"], + [111, "FleetDrainImports"], ]); assert.deepStrictEqual(yield* runMigrations(), []); assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports); @@ -213,6 +216,9 @@ describe("V2 preview upgrade", () => { [103, "IssueTrackerAdditionalScopes"], [104, "ProjectionProjectsTrackedFolders"], [105, "TaskFolders"], + [109, "FleetPeerEnvironments"], + [110, "OpenShellFleetState"], + [111, "FleetDrainImports"], ]); }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))), ); diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts index 3b3f1459cb9dd..46828d0b0f9ad 100644 --- a/apps/server/src/provider/AgentContextBudget.test.ts +++ b/apps/server/src/provider/AgentContextBudget.test.ts @@ -11,6 +11,7 @@ import { AssetSigner } from "../assets/AssetSigner.ts"; import { ServerSecretStore } from "../auth/ServerSecretStore.ts"; import { ServerConfig } from "../config.ts"; import { DeviceService } from "../device/DeviceService.ts"; +import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts"; import { ServerEnvironment } from "../environment/ServerEnvironment.ts"; import { GitManager } from "../git/GitManager.ts"; import { GitWorkflowService } from "../git/GitWorkflowService.ts"; @@ -66,6 +67,7 @@ const toolHandlerServices = Layer.mergeAll( Layer.mock(McpProviderSessions)({}), Layer.mock(OrchestratorV2)({}), Layer.mock(PeerForwarding)({}), + Layer.mock(OpenShellFleetService.OpenShellFleetService)({}), Layer.mock(PeerLinkRequests)({}), Layer.mock(PeerLinks)({}), Layer.mock(PreviewAutomationBroker)({}), diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index e089e5a84218a..e560ee52f0b12 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { "claudeSystemAppendBytes": 6245, "codexAdditionalContextBytes": 7639, - "mcpToolListJsonBytes": 59026 + "mcpToolListJsonBytes": 60962 } diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts index 86b13922946b0..90d5d0060cbc4 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts @@ -1159,6 +1159,50 @@ it.effect("checks a task's destination thread only when the destination changes" }).pipe(Effect.provide(SqlitePersistence.layerMemory)), ); +it.effect("refuses to schedule a sandbox workspace", () => + Effect.gen(function* () { + const layer = ScheduledTaskService.layer.pipe( + Layer.provide( + Layer.mergeAll( + ManagedPolicy.layerTest(), + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}), + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadShell: () => Effect.succeed(null), + }), + Layer.mock(SecretRequests.SecretRequests)({ + consume: () => Effect.succeed("secret"), + }), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: "/projects", + isScratchProject: () => Effect.succeed(false), + }), + NodeCrypto.layer, + Scheduler.layer, + ), + ), + ); + yield* Effect.gen(function* () { + const service = yield* ScheduledTaskService.ScheduledTaskService; + const refused = yield* service + .upsert({ + title: "Run in a worker", + prompt: "Check progress", + enabled: true, + schedule: { type: "interval", everyMs: 60_000 }, + projectId: ProjectId.make("project:test"), + threadId: null, + workspaceStrategy: { type: "sandbox" }, + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + }) + .pipe(Effect.flip); + assert.match(refused.message, /isolation 'openshell-auto'/); + assert.lengthOf((yield* service.list()).tasks, 0); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.provide(SqlitePersistence.layerMemory)), +); + it.effect("a run interrupted right after it reserves the task does not block the next run", () => Effect.gen(function* () { const launches = yield* Ref.make(0); diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts index 68c95748d0ce8..ae6bde4d5e2ba 100644 --- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts +++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts @@ -1479,6 +1479,18 @@ export const layer = Layer.effect( // keep their run history, and so real load failures propagate instead // of silently resetting an existing row. const existingTask = yield* findTask(id); + // A sandbox workspace belongs to a fleet worker; a scheduled launch would + // run in the project root. Existing sandbox tasks stay editable, but a + // new task or a strategy change to sandbox is still refused. + if ( + input.workspaceStrategy.type === "sandbox" && + existingTask?.workspaceStrategy.type !== "sandbox" + ) { + return yield* taskError( + "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of scheduling it here.", + input.id === undefined ? {} : { taskId: input.id }, + ); + } const threadMode = resolveThreadMode(input, existingTask); const keptThreadId = input.threadId ?? (input.unbindThread === true ? null : (existingTask?.threadId ?? null)); diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 720834730e99b..e4708c2e9e489 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -95,9 +95,12 @@ import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; import * as PeerClientPairing from "./peer/PeerClientPairing.ts"; import * as PeerClientPairingHttp from "./peer/http.ts"; import * as PeerLinkRequests from "./peer/PeerLinkRequests.ts"; +import * as PeerEnvironmentRegistry from "./peer/PeerEnvironmentRegistry.ts"; import * as PeerLinks from "./peer/PeerLinks.ts"; import * as PeerMcpClient from "./peer/PeerMcpClient.ts"; import * as PeerThreadRelay from "./peer/PeerThreadRelay.ts"; +import * as OpenShellFleetStore from "./environment/OpenShellFleetStore.ts"; +import * as OpenShellFleetService from "./environment/OpenShellFleetService.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts"; import * as DeviceService from "./device/DeviceService.ts"; @@ -462,6 +465,18 @@ const layerIssueService = Layer.merge(IssueService.layer, IssueTrackerConnection // One instance serves MCP link calls and refreshes snapshots on thread events. const layerThreadIssueService = ThreadIssueService.layer; +// One fleet service instance is shared by the environment routes and sweep. +const layerConfiguredOpenShellFleetService = OpenShellFleetService.layer.pipe( + Layer.provide(PeerEnvironmentRegistry.layer), + Layer.provide(OpenShellFleetStore.layer), + Layer.provide(PeerLinks.layer), + Layer.provide(NodeServices.layer), + // Post-drain bundle imports write through the orchestration event sink; + // shared layer references keep one instance across the app. + Layer.provideMerge(RuntimeLayer.layerEventSink), + Layer.provideMerge(SqlitePersistence.layerConfig), +); + const layerGitManager = GitManager.layer.pipe( Layer.provide(Layer.merge(ProjectionStoreV2.layer, ProjectStore.layer)), Layer.provide(layerPullRequestService.pipe(Layer.provide(RuntimeLayer.layerProjectService))), @@ -922,13 +937,24 @@ const layerMakeRoutes = Layer.mergeAll( ), ), Layer.provide(PreviewBrowser.layer), - // Links to other environments, shared by clients (WebSocket) and agents (/mcp). + // Links to other environments, shared by Settings (WebSocket) and agents (/mcp). + // The fleet registry answers for registered workers, so the same bridge reaches them. + // Built here so the fleet state store re-reads what a restart interrupted at startup. Layer.provide( Layer.mergeAll(PeerForwarding.layer, PeerThreadRelay.layer, PeerClientPairing.layer).pipe( Layer.provideMerge(PeerMcpClient.layer), Layer.provideMerge(PeerLinks.layer), + Layer.provideMerge(PeerEnvironmentRegistry.layer), + Layer.provide(OpenShellFleetStore.layer), ), ), + // Fleet worker lifecycle behind the environment MCP tools (spec section 5). + Layer.provide(layerConfiguredOpenShellFleetService), + // Ephemeral workers self-retire: the periodic sweep shares the service + // instance above through layer memoization. + Layer.provide( + OpenShellFleetService.layerSweep.pipe(Layer.provide(layerConfiguredOpenShellFleetService)), + ), Layer.provide( SpeechService.layer.pipe( Layer.provide(layerServerSettings), diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index f837fd8ca9722..fbe69821deb94 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -1973,6 +1973,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" "cloudSpeechCredentials.openrouter", "easDeviceHosts.[].accessToken", "simlockDeviceHosts.[].agentToken", + "openshell.auth.clientSecret", + "openshell.auth.refreshToken", + "openshell.auth.clientKey", "usageLimitSources.*.managementKey", "textGenerators.*.apiKey", ]); diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts index d433621f3b1d7..cac36b9d004ee 100644 --- a/apps/server/src/settingsSecrets.ts +++ b/apps/server/src/settingsSecrets.ts @@ -63,6 +63,17 @@ export interface SettingsSecret { readonly environmentVariable?: string; } +/** Secret fields the gateway auth union holds, per mode; the other fields are not secrets. */ +const OPEN_SHELL_AUTH_SECRET_FIELDS: Readonly>> = { + "client-credentials": ["clientSecret"], + "oidc-pkce": ["refreshToken"], + mtls: ["clientKey"], +}; + +/** The slot a gateway auth secret lives in (spec section 4.2: never settings JSON). */ +export const openshellGatewayAuthSecretName = (field: string): string => + `openshell-gateway-auth-${encode(field)}`; + type SecretValue = Pick; type SecretSlotName = (field: string, serverHost: string | null) => string; type SecretFieldsMapper = ( @@ -247,8 +258,30 @@ function mapSecretFieldGroups( ) as SimlockDeviceHostConfig, ); + // Gateway auth secrets live one level down (openshell.auth.), and which + // fields are secret depends on the mode. A literal is stored to its slot and + // redacted; a stored placeholder or a secret-provider reference passes through. + const openshell = settings.openshell; + const openshellAuth = openshell?.auth; + const openshellMapped = + openshell === undefined + ? settings.openshell + : { + ...openshell, + ...(openshellAuth === undefined + ? {} + : ({ + auth: mapFields( + openshellAuth as Record, + OPEN_SHELL_AUTH_SECRET_FIELDS[openshellAuth.mode] ?? [], + (field) => openshellGatewayAuthSecretName(field), + ), + } as ServerSettings["openshell"])), + }; + return { ...settings, + openshell: openshellMapped, cloudSpeechCredentials: mapFields( settings.cloudSpeechCredentials, voiceSecretFields, diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index d96b91f30d62a..f981995d83c91 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2198,6 +2198,20 @@ export const layerWsRpc = ( "orchestration_v2.command_id": input.commandId, "orchestration_v2.project_id": input.projectId, }).pipe( + Effect.andThen( + Effect.flatMap(serverEnvironment.getEnvironmentId, (selfId) => + input.environmentId === undefined || input.environmentId === selfId + ? Effect.void + : Effect.fail( + new OrchestrationV2ThreadLaunchError({ + commandId: input.commandId, + projectId: input.projectId, + message: + "A thread launches in this environment. To run work in another environment, use delegate_task with target.environmentId; its result arrives when the task settles.", + }), + ), + ), + ), Effect.andThen( startup .enqueueCommand( diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx index 293435976974a..2187207c941ea 100644 --- a/apps/web/src/components/settings/LabsSettingsPanel.tsx +++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx @@ -2,10 +2,10 @@ import { managedCapabilityDenial } from "@t3tools/client-runtime/contract-labels import { useAuth } from "@clerk/react"; import { Link } from "@tanstack/react-router"; import { T, useGT, useMessages } from "gt-react"; - import { hasCloudPublicConfig } from "~/cloud/publicConfig"; import { useT3ConnectAuthPrompt } from "../clerk/useT3ConnectAuthPrompt"; import { Button } from "../ui/button"; +import { Input } from "../ui/input"; import { Switch } from "../ui/switch"; import { ScopedSwitch } from "./ScopedSwitch"; import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout"; @@ -28,10 +28,165 @@ export function LabsSettingsPanel() { + ); } +function OpenShellFleetSection() { + const gt = useGT(); + return ( + + + + ); +} + +function OpenShellFleetRow() { + const searchableSetting = useSearchableSetting(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const gt = useGT(); + return ( + + Worker environments on an OpenShell gateway. This machine becomes the managing instance + that spawns, pairs, drains, and deletes fleet workers. + + } + control={ + updateSettings({ openshellEnabled: Boolean(checked) })} + aria-label={gt("OpenShell fleet")} + /> + } + > + {settings.openshellEnabled ? ( +
+ + + +
+ ) : null} +
+ ); +} + +function OpenShellGatewayUrlRow() { + const searchableSetting = useSearchableSetting(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const gt = useGT(); + const stored = settings.openshell?.gatewayUrl ?? ""; + // `key` re-seeds the uncontrolled input when the stored value changes. + const commit = (raw: string) => { + const value = raw.trim(); + if (value === stored) return; + if (value === "") return; + updateSettings({ openshell: { gatewayUrl: value } }); + }; + return ( + + commit(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + commit(event.currentTarget.value); + } + }} + /> + + } + /> + ); +} + +function OpenShellManagingRow() { + const searchableSetting = useSearchableSetting(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const gt = useGT(); + return ( + + Make this environment the managing instance. Only one managing instance can exist per + gateway; linked environments can never hold this role. + + } + control={ + + updateSettings({ openshell: { managingInstance: Boolean(checked) } }) + } + aria-label={gt("Managing instance")} + /> + } + /> + ); +} + +function OpenShellTemplateRow() { + const searchableSetting = useSearchableSetting(); + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + const gt = useGT(); + const stored = settings.openshell?.defaultSandboxTemplate ?? ""; + // `key` re-seeds the uncontrolled input when the stored value changes. + const commit = (raw: string) => { + const value = raw.trim(); + if (value === stored) return; + if (value === "") return; + updateSettings({ openshell: { defaultSandboxTemplate: value } }); + }; + return ( + + commit(event.currentTarget.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + commit(event.currentTarget.value); + } + }} + /> + + } + /> + ); +} + function BoringSidebarRow() { const searchableSetting = useSearchableSetting(); const settings = useScopedSettings(); diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx index 471dcdf0fe0df..3a8995619022d 100644 --- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx +++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx @@ -107,6 +107,7 @@ import { scheduleFromDraft, scheduleLabel, scheduledTaskDefaultModel, + leavesSandboxWorkspace, weekdayShortLabel, taskToDraft, unbindDraftFromThread, @@ -172,6 +173,7 @@ const WORKSPACE_MODE_LABELS: Record = { worktree: msg("Create a new worktree"), root: msg("Use the project checkout"), existing_worktree: msg("Use a specific checkout"), + sandbox: msg("Fleet worker sandbox"), }; const SCOPE_SUMMARY_TEMPLATES: Record = { @@ -197,6 +199,7 @@ const EMPTY_DRAFT: DraftState = { savedThreadId: "", threadMode: "new_thread", workspaceMode: "worktree", + storedSandbox: false, baseRef: "", startFromOrigin: true, existingWorktreePath: "", @@ -1667,11 +1670,13 @@ function ScheduledTaskEditorForm({ ? { type: "root" } : draft.workspaceMode === "existing_worktree" ? { type: "existing_worktree", worktreePath: draft.existingWorktreePath.trim() } - : { - type: "worktree", - baseRef: worktreeBaseRef, - startFromOrigin: draft.startFromOrigin, - }; + : draft.workspaceMode === "sandbox" + ? { type: "sandbox" } + : { + type: "worktree", + baseRef: worktreeBaseRef, + startFromOrigin: draft.startFromOrigin, + }; const input: ScheduledTaskUpsertInput = { ...(draft.editingId ? { id: draft.editingId as ScheduledTaskId, requireExisting: true } : {}), title: draft.title.trim(), @@ -1869,14 +1874,45 @@ function ScheduledTaskEditorForm({