diff --git a/.github/workflows/openshell-smoke.yml b/.github/workflows/openshell-smoke.yml
new file mode 100644
index 0000000000000..5d52145f2fcba
--- /dev/null
+++ b/.github/workflows/openshell-smoke.yml
@@ -0,0 +1,74 @@
+name: OpenShell gateway smoke
+
+# Opt-in live check of the OpenShell fleet integration against a REAL gateway.
+# The job runs only when a repository administrator configures gateway access;
+# otherwise every step skips and pull requests stay green.
+#
+# How to enable (repository secrets, set once by an administrator):
+# OPENSHELL_SMOKE_GATEWAY_URL the gateway control-plane base URL (https)
+# OPENSHELL_SMOKE_GATEWAY_CREDENTIAL the bearer credential the gateway's OIDC
+# token provider returns to the OpenShell SDK
+# Optional repository variable:
+# OPENSHELL_SMOKE_TEMPLATE sandbox template to spawn; unset falls
+# back to a worker-labeled template listed
+# by the gateway.
+# What the gateway must expose: health, sandbox template listing, sandbox
+# create/wait/get/delete, and label filtering on `t3-managed-by`.
+# Budget: the script defaults to 900 seconds (OPENSHELL_SMOKE_TIMEOUT_SECS).
+
+on:
+ workflow_dispatch:
+ schedule:
+ # Avoid minute zero, when GitHub scheduled jobs are busiest.
+ - cron: "17 4 * * 1"
+ pull_request:
+ paths:
+ - "packages/openshell/**"
+ - "apps/server/src/environment/**"
+ - "scripts/openshell-smoke.ts"
+ - ".github/workflows/openshell-smoke.yml"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: openshell-smoke-${{ github.event.pull_request.number || github.sha }}
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+ smoke:
+ name: Gateway smoke
+ runs-on: ubuntu-24.04
+ timeout-minutes: 25
+ steps:
+ # Same presence pattern as the EXPO_TOKEN check in mobile-eas-production.yml.
+ - id: smoke-secrets
+ name: Check for gateway secrets
+ env:
+ SMOKE_URL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_URL }}
+ SMOKE_CREDENTIAL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_CREDENTIAL }}
+ run: |
+ if [ -n "$SMOKE_URL" ] && [ -n "$SMOKE_CREDENTIAL" ]; then
+ echo "present=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "present=false" >> "$GITHUB_OUTPUT"
+ echo "OPENSHELL_SMOKE_GATEWAY_URL and OPENSHELL_SMOKE_GATEWAY_CREDENTIAL are not configured; skipping the live gateway smoke."
+ fi
+
+ - name: Checkout
+ if: steps.smoke-secrets.outputs.present == 'true'
+ uses: actions/checkout@v6
+
+ - name: Setup Vite+
+ if: steps.smoke-secrets.outputs.present == 'true'
+ uses: ./.github/actions/setup-ci-deps
+ with:
+ task-cache-read-token: ${{ secrets.T3_TASK_CACHE_READ_TOKEN }}
+
+ - name: Run the gateway smoke
+ if: steps.smoke-secrets.outputs.present == 'true'
+ env:
+ OPENSHELL_SMOKE_GATEWAY_URL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_URL }}
+ OPENSHELL_SMOKE_GATEWAY_CREDENTIAL: ${{ secrets.OPENSHELL_SMOKE_GATEWAY_CREDENTIAL }}
+ OPENSHELL_SMOKE_TEMPLATE: ${{ vars.OPENSHELL_SMOKE_TEMPLATE }}
+ run: node scripts/openshell-smoke.ts
diff --git a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx
index 8e15dad08ac23..66bddc72673c4 100644
--- a/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx
+++ b/apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx
@@ -80,6 +80,7 @@ import { useSettingsEnvironmentFilter, type SettingsTarget } from "./settings-en
import {
changeDraftProject,
editDraft,
+ leavesSandboxWorkspace,
parseIntervalMinutes,
scheduledTaskDefaultModel,
scheduleFromDraft,
@@ -891,11 +892,13 @@ function TaskForm({
? { type: "root" }
: draft.workspace === "existing_worktree"
? { type: "existing_worktree", worktreePath: draft.checkoutPath.trim() }
- : {
- type: "worktree",
- baseRef: worktreeBaseRef,
- startFromOrigin: draft.startFromOrigin,
- },
+ : draft.workspace === "sandbox"
+ ? { type: "sandbox" }
+ : {
+ type: "worktree",
+ baseRef: worktreeBaseRef,
+ startFromOrigin: draft.startFromOrigin,
+ },
// Tasks have no mode picker; a mode above the administrator's ceiling is refused.
runtimeMode:
clampManagedRuntimeMode(config?.managedPolicy, draft.runtimeMode) ?? draft.runtimeMode,
@@ -1091,9 +1094,20 @@ function TaskForm({
? gt("New worktree")
: draft.workspace === "root"
? gt("Project checkout")
- : gt("Specific checkout")
+ : draft.workspace === "sandbox"
+ ? gt("Fleet worker sandbox")
+ : gt("Specific checkout")
}
actions={[
+ ...(draft.storedSandbox
+ ? [
+ {
+ id: "sandbox",
+ title: gt("Fleet worker sandbox"),
+ state: draft.workspace === "sandbox" ? ("on" as const) : undefined,
+ },
+ ]
+ : []),
{
id: "worktree",
title: gt("New worktree"),
@@ -1111,10 +1125,22 @@ function TaskForm({
},
]}
onSelect={(id) => {
- if (id === "worktree" || id === "root" || id === "existing_worktree")
+ if (
+ id === "worktree" ||
+ id === "root" ||
+ id === "existing_worktree" ||
+ id === "sandbox"
+ )
setDraft((current) => ({ ...current, workspace: id }));
}}
/>
+ {leavesSandboxWorkspace(draft) ? (
+
+
+ Future runs will no longer use the fleet worker sandbox this task is paired with.
+
+
+ ) : null}
{draft.workspace === "worktree" ? (
{
).toBe("release");
});
});
+
+describe("sandbox workspace retention", () => {
+ const task = (workspaceStrategy: ScheduledTask["workspaceStrategy"]): ScheduledTask => ({
+ id: ScheduledTaskId.make("task"),
+ title: "Nightly review",
+ prompt: "Review open issues",
+ enabled: true,
+ schedule: { type: "interval", everyMs: 60_000 },
+ projectId: ProjectId.make("project"),
+ threadId: null,
+ workspaceStrategy,
+ modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ createdBy: "user",
+ creationSource: "mobile",
+ createdAt: "2026-09-17T00:00:00.000Z",
+ updatedAt: "2026-09-17T00:00:00.000Z",
+ nextRunAt: null,
+ lastRunAt: null,
+ lastRunStatus: "never",
+ lastRunError: null,
+ runCount: 0,
+ });
+
+ it("keeps a stored sandbox strategy when editing an unrelated field", () => {
+ const initial = editDraft(task({ type: "sandbox" }));
+ expect(initial.workspace).toBe("sandbox");
+ expect(initial.storedSandbox).toBe(true);
+ expect(leavesSandboxWorkspace(initial)).toBe(false);
+ // Renaming the task does not move it off its fleet worker.
+ const renamed = { ...initial, title: "Renamed" };
+ expect(hasScheduledTaskDraftChanges(initial, renamed)).toBe(true);
+ expect(leavesSandboxWorkspace(renamed)).toBe(false);
+ });
+
+ it("surfaces a warning only when the workspace control leaves the sandbox", () => {
+ const initial = editDraft(task({ type: "sandbox" }));
+ const moved = { ...initial, workspace: "worktree" as const };
+ expect(leavesSandboxWorkspace(moved)).toBe(true);
+ expect(hasScheduledTaskDraftChanges(initial, moved)).toBe(true);
+ // A task that never ran in a sandbox is never flagged.
+ const worktreeDraft = editDraft(
+ task({ type: "worktree", baseRef: "release", startFromOrigin: false }),
+ );
+ expect(worktreeDraft.storedSandbox).toBe(false);
+ expect(leavesSandboxWorkspace({ ...worktreeDraft, workspace: "root" as const })).toBe(false);
+ });
+});
diff --git a/apps/mobile/src/features/settings/scheduledTaskDraft.ts b/apps/mobile/src/features/settings/scheduledTaskDraft.ts
index 5bacd4be83c81..2b3f01282934a 100644
--- a/apps/mobile/src/features/settings/scheduledTaskDraft.ts
+++ b/apps/mobile/src/features/settings/scheduledTaskDraft.ts
@@ -197,7 +197,7 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
};
}
-type Workspace = "worktree" | "root" | "existing_worktree";
+type Workspace = "worktree" | "root" | "existing_worktree" | "sandbox";
export type ScheduledTaskDraft = {
readonly task: ScheduledTask | null;
readonly title: string;
@@ -209,6 +209,8 @@ export type ScheduledTaskDraft = {
readonly schedule: ScheduleDraft;
readonly threadMode: ScheduledTaskThreadMode;
readonly workspace: Workspace;
+ /** The stored task runs in a fleet worker sandbox; leaving it is surfaced. */
+ readonly storedSandbox: boolean;
readonly baseRef: string;
readonly checkoutPath: string;
readonly enabled: boolean;
@@ -301,6 +303,7 @@ export function createDraft(
schedule: DEFAULT_SCHEDULE,
threadMode: "new_thread",
workspace: "worktree",
+ storedSandbox: false,
baseRef: "",
checkoutPath: "",
enabled: true,
@@ -309,6 +312,11 @@ export function createDraft(
};
}
+/** An edit that moves a stored sandbox task off its fleet worker; the editor surfaces it. */
+export function leavesSandboxWorkspace(draft: ScheduledTaskDraft): boolean {
+ return draft.storedSandbox && draft.workspace !== "sandbox";
+}
+
export function editDraft(task: ScheduledTask): ScheduledTaskDraft {
return {
task,
@@ -320,7 +328,11 @@ export function editDraft(task: ScheduledTask): ScheduledTaskDraft {
modelSelectionIsExplicit: true,
schedule: scheduleDraftForTask(task),
threadMode: scheduledTaskThreadMode(task),
+ // A sandbox workspace stays: editing an unrelated field must not silently
+ // move the task off its fleet worker.
workspace: task.workspaceStrategy.type,
+ storedSandbox: task.workspaceStrategy.type === "sandbox",
+ // Main's newer editor keeps the base automatic unless a worktree pinned one.
baseRef: task.workspaceStrategy.type === "worktree" ? task.workspaceStrategy.baseRef : "",
checkoutPath:
task.workspaceStrategy.type === "existing_worktree"
diff --git a/apps/mobile/src/i18n/translations/de.json b/apps/mobile/src/i18n/translations/de.json
index 8777f08e663ce..1ebd3a20d6d72 100644
--- a/apps/mobile/src/i18n/translations/de.json
+++ b/apps/mobile/src/i18n/translations/de.json
@@ -3595,6 +3595,7 @@
"85728e592594c929": "Ausführen in",
"c7fe4313def0ef2f": "Neuer Worktree",
"e91a39ca1b5d9bbd": "Projekt-Checkout",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Bestimmter Checkout",
"8b724804e55e48e6": "Checkout-Pfad",
"b325f97ceab8bc64": "Zeitplan",
@@ -3653,6 +3654,7 @@
"3c5185df378edef9": "Wird nur ausgeführt, wenn alle Filter zutreffen. Andere Anfragen werden als gefiltert protokolliert. Trenne „ist eines von“-Werte mit Kommas; ein Wert darf kein Komma enthalten.",
"d1a74724f70edb0b": "Diese Aufgabe existiert nicht mehr.",
"c191069dded46bcc": "Diese Umgebung ist getrennt. Verbinde sie erneut, bevor du speicherst.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Anfragen mit demselben Schlüssel werden innerhalb von 7 Tagen nur einmal ausgeführt.",
"a5bbe09b0c89d86c": "Beim Zusammenfassen wird die neueste Payload ausgeführt, wenn das Zeitfenster ab der ersten Zustellung endet. Leere Schlüssel werden normal ausgeführt.",
"0e2233fb297f7c07": "Intervalle müssen mindestens 1 Minute betragen. Passe dieses Intervall vor dem Speichern an.",
diff --git a/apps/mobile/src/i18n/translations/en.json b/apps/mobile/src/i18n/translations/en.json
index dfe3a473fe248..4d0f4917942e4 100644
--- a/apps/mobile/src/i18n/translations/en.json
+++ b/apps/mobile/src/i18n/translations/en.json
@@ -3597,6 +3597,7 @@
"85728e592594c929": "Run in",
"c7fe4313def0ef2f": "New worktree",
"e91a39ca1b5d9bbd": "Project checkout",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Specific checkout",
"8b724804e55e48e6": "Checkout path",
"b325f97ceab8bc64": "Schedule",
@@ -3655,6 +3656,7 @@
"3c5185df378edef9": "Runs only when every filter matches. Other requests are logged as filtered. Separate “is one of” values with commas; a value can't contain a comma.",
"d1a74724f70edb0b": "This task no longer exists.",
"c191069dded46bcc": "This environment is disconnected. Reconnect before saving.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Requests with the same key run only once within 7 days.",
"a5bbe09b0c89d86c": "Coalescing runs the latest payload when the window from the first delivery closes. Empty keys run normally.",
"0e2233fb297f7c07": "Intervals must be at least 1 minute. Update this interval before saving.",
diff --git a/apps/mobile/src/i18n/translations/es.json b/apps/mobile/src/i18n/translations/es.json
index 05f58a63227d3..001c976cdf1cf 100644
--- a/apps/mobile/src/i18n/translations/es.json
+++ b/apps/mobile/src/i18n/translations/es.json
@@ -3590,6 +3590,7 @@
"85728e592594c929": "Ejecutar en",
"c7fe4313def0ef2f": "Nuevo worktree",
"e91a39ca1b5d9bbd": "Checkout del proyecto",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Checkout concreto",
"8b724804e55e48e6": "Ruta del checkout",
"b325f97ceab8bc64": "Programación",
@@ -3648,6 +3649,7 @@
"3c5185df378edef9": "Se ejecuta solo cuando coinciden todos los filtros. Las demás solicitudes se registran como filtradas. Separa los valores de «es uno de» con comas; un valor no puede contener una coma.",
"d1a74724f70edb0b": "Esta tarea ya no existe.",
"c191069dded46bcc": "Este entorno está desconectado. Vuelve a conectarlo antes de guardar.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Las solicitudes con la misma clave solo se ejecutan una vez en 7 días.",
"a5bbe09b0c89d86c": "La agrupación ejecuta la carga más reciente cuando se cierra la ventana abierta por la primera entrega. Las claves vacías se ejecutan con normalidad.",
"0e2233fb297f7c07": "Los intervalos deben ser de al menos 1 minuto. Actualiza este intervalo antes de guardar.",
diff --git a/apps/mobile/src/i18n/translations/fr.json b/apps/mobile/src/i18n/translations/fr.json
index f661c315215c0..7d1497e707dec 100644
--- a/apps/mobile/src/i18n/translations/fr.json
+++ b/apps/mobile/src/i18n/translations/fr.json
@@ -3590,6 +3590,7 @@
"85728e592594c929": "Exécuter dans",
"c7fe4313def0ef2f": "Nouveau worktree",
"e91a39ca1b5d9bbd": "Checkout du projet",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Checkout précis",
"8b724804e55e48e6": "Chemin du checkout",
"b325f97ceab8bc64": "Planification",
@@ -3648,6 +3649,7 @@
"3c5185df378edef9": "S’exécute uniquement si tous les filtres correspondent. Les autres requêtes sont journalisées comme filtrées. Séparez les valeurs « fait partie de » par des virgules ; une valeur ne peut pas contenir de virgule.",
"d1a74724f70edb0b": "Cette tâche n’existe plus.",
"c191069dded46bcc": "Cet environnement est déconnecté. Reconnectez-le avant d’enregistrer.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Les requêtes avec la même clé ne s’exécutent qu’une fois en 7 jours.",
"a5bbe09b0c89d86c": "Le regroupement exécute la dernière charge utile quand la fenêtre ouverte par la première livraison se ferme. Les clés vides s’exécutent normalement.",
"0e2233fb297f7c07": "Les intervalles doivent être d’au moins 1 minute. Modifiez cet intervalle avant d’enregistrer.",
diff --git a/apps/mobile/src/i18n/translations/ja.json b/apps/mobile/src/i18n/translations/ja.json
index 56616809052d3..69e48be8e8448 100644
--- a/apps/mobile/src/i18n/translations/ja.json
+++ b/apps/mobile/src/i18n/translations/ja.json
@@ -3586,6 +3586,7 @@
"85728e592594c929": "実行場所",
"c7fe4313def0ef2f": "新しいワークツリー",
"e91a39ca1b5d9bbd": "プロジェクトのチェックアウト",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "特定のチェックアウト",
"8b724804e55e48e6": "チェックアウトのパス",
"b325f97ceab8bc64": "スケジュール",
@@ -3644,6 +3645,7 @@
"3c5185df378edef9": "すべてのフィルターに一致した場合のみ実行します。それ以外のリクエストはフィルター済みとして記録されます。「いずれかに一致」の値はカンマで区切ってください。値にカンマを含めることはできません。",
"d1a74724f70edb0b": "このタスクは存在しません。",
"c191069dded46bcc": "この環境は切断されています。保存する前に再接続してください。",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "同じキーのリクエストは 7 日以内に 1 回だけ実行されます。",
"a5bbe09b0c89d86c": "集約すると、最初の配信からの期間が終了したときに最新のペイロードで実行します。キーが空の場合は通常どおり実行します。",
"0e2233fb297f7c07": "間隔は 1 分以上にする必要があります。保存する前にこの間隔を更新してください。",
diff --git a/apps/mobile/src/i18n/translations/ko.json b/apps/mobile/src/i18n/translations/ko.json
index 443285f8664c4..bddfcba0eeb05 100644
--- a/apps/mobile/src/i18n/translations/ko.json
+++ b/apps/mobile/src/i18n/translations/ko.json
@@ -3590,6 +3590,7 @@
"85728e592594c929": "실행 위치",
"c7fe4313def0ef2f": "새 워크트리",
"e91a39ca1b5d9bbd": "프로젝트 체크아웃",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "특정 체크아웃",
"8b724804e55e48e6": "체크아웃 경로",
"b325f97ceab8bc64": "일정",
@@ -3648,6 +3649,7 @@
"3c5185df378edef9": "모든 필터가 일치할 때만 실행합니다. 다른 요청은 필터링됨으로 기록됩니다. “다음 중 하나” 값은 쉼표로 구분하며, 값에는 쉼표를 넣을 수 없습니다.",
"d1a74724f70edb0b": "이 작업은 더 이상 존재하지 않습니다.",
"c191069dded46bcc": "이 환경의 연결이 끊어졌습니다. 저장하기 전에 다시 연결하세요.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "키가 같은 요청은 7일 이내에 한 번만 실행됩니다.",
"a5bbe09b0c89d86c": "병합하면 첫 전송부터 시작된 기간이 끝날 때 최신 페이로드로 실행합니다. 키가 비어 있으면 정상적으로 실행됩니다.",
"0e2233fb297f7c07": "간격은 1분 이상이어야 합니다. 저장하기 전에 간격을 업데이트하세요.",
diff --git a/apps/mobile/src/i18n/translations/pt-BR.json b/apps/mobile/src/i18n/translations/pt-BR.json
index 33793cc553d34..bf75b05b8fcc9 100644
--- a/apps/mobile/src/i18n/translations/pt-BR.json
+++ b/apps/mobile/src/i18n/translations/pt-BR.json
@@ -3590,6 +3590,7 @@
"85728e592594c929": "Executar em",
"c7fe4313def0ef2f": "Novo worktree",
"e91a39ca1b5d9bbd": "Checkout do projeto",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Checkout específico",
"8b724804e55e48e6": "Caminho do checkout",
"b325f97ceab8bc64": "Agendamento",
@@ -3648,6 +3649,7 @@
"3c5185df378edef9": "Executa só quando todos os filtros correspondem. As outras requisições são registradas como filtradas. Separe os valores de “é um de” com vírgulas; um valor não pode conter vírgula.",
"d1a74724f70edb0b": "Esta tarefa não existe mais.",
"c191069dded46bcc": "Este ambiente está desconectado. Reconecte antes de salvar.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Requisições com a mesma chave executam só uma vez em 7 dias.",
"a5bbe09b0c89d86c": "O agrupamento executa o payload mais recente quando a janela iniciada pela primeira entrega fecha. Chaves vazias executam normalmente.",
"0e2233fb297f7c07": "Os intervalos devem ser de pelo menos 1 minuto. Atualize este intervalo antes de salvar.",
diff --git a/apps/mobile/src/i18n/translations/ru.json b/apps/mobile/src/i18n/translations/ru.json
index d0e2d9577a9ab..fc450831f12a4 100644
--- a/apps/mobile/src/i18n/translations/ru.json
+++ b/apps/mobile/src/i18n/translations/ru.json
@@ -3644,6 +3644,7 @@
"85728e592594c929": "Выполнять в",
"c7fe4313def0ef2f": "Новое рабочее дерево",
"e91a39ca1b5d9bbd": "Копия проекта",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Конкретная копия",
"8b724804e55e48e6": "Путь к копии",
"b325f97ceab8bc64": "Расписание",
@@ -3702,6 +3703,7 @@
"3c5185df378edef9": "Выполняется, только если совпадают все фильтры. Остальные запросы записываются как отфильтрованные. Значения для «одно из» разделяйте запятыми; значение не может содержать запятую.",
"d1a74724f70edb0b": "Эта задача больше не существует.",
"c191069dded46bcc": "Это окружение отключено. Переподключитесь перед сохранением.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Запросы с одинаковым ключом выполняются только один раз в течение 7 дней.",
"a5bbe09b0c89d86c": "При объединении последний payload выполняется, когда закрывается окно, отсчитанное от первой доставки. Пустые ключи выполняются как обычно.",
"0e2233fb297f7c07": "Интервал должен быть не меньше 1 минуты. Измените интервал перед сохранением.",
diff --git a/apps/mobile/src/i18n/translations/sv.json b/apps/mobile/src/i18n/translations/sv.json
index 7fd2f945cc033..fbe2eeba444dc 100644
--- a/apps/mobile/src/i18n/translations/sv.json
+++ b/apps/mobile/src/i18n/translations/sv.json
@@ -3596,6 +3596,7 @@
"85728e592594c929": "Kör i",
"c7fe4313def0ef2f": "Ny worktree",
"e91a39ca1b5d9bbd": "Projektets utcheckning",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "Specifik utcheckning",
"8b724804e55e48e6": "Sökväg till utcheckning",
"b325f97ceab8bc64": "Schema",
@@ -3654,6 +3655,7 @@
"3c5185df378edef9": "Körs bara när alla filter matchar. Andra begäranden loggas som filtrerade. Separera värden för ”är en av” med kommatecken; ett värde får inte innehålla kommatecken.",
"d1a74724f70edb0b": "Uppgiften finns inte längre.",
"c191069dded46bcc": "Miljön är frånkopplad. Återanslut innan du sparar.",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "Begäranden med samma nyckel körs bara en gång inom 7 dagar.",
"a5bbe09b0c89d86c": "Sammanslagning kör den senaste nyttolasten när fönstret från den första leveransen stängs. Tomma nycklar körs som vanligt.",
"0e2233fb297f7c07": "Intervall måste vara minst 1 minut. Uppdatera intervallet innan du sparar.",
diff --git a/apps/mobile/src/i18n/translations/zh.json b/apps/mobile/src/i18n/translations/zh.json
index 54c72efe3b4fb..bff9aea153025 100644
--- a/apps/mobile/src/i18n/translations/zh.json
+++ b/apps/mobile/src/i18n/translations/zh.json
@@ -3592,6 +3592,7 @@
"85728e592594c929": "运行位置",
"c7fe4313def0ef2f": "新工作树",
"e91a39ca1b5d9bbd": "项目工作副本",
+ "c1b7be381b2a2df1": "Fleet worker sandbox",
"27a6259d0e7c054e": "指定工作副本",
"8b724804e55e48e6": "工作副本路径",
"b325f97ceab8bc64": "定时规则",
@@ -3650,6 +3651,7 @@
"3c5185df378edef9": "仅在所有筛选条件都匹配时运行。其他请求会记录为已筛除。“属于其中之一”的多个值用逗号分隔;单个值不能包含逗号。",
"d1a74724f70edb0b": "此任务已不存在。",
"c191069dded46bcc": "此环境已断开连接。保存前请重新连接。",
+ "46cea643e5ae4196": "Future runs will no longer use the fleet worker sandbox this task is paired with.",
"0d15db4c81851479": "键相同的请求在 7 天内只运行一次。",
"a5bbe09b0c89d86c": "合并会在自首次投递起的窗口结束时,用最新的负载运行。空键正常运行。",
"0e2233fb297f7c07": "间隔必须至少为 1 分钟。保存前请更新此间隔。",
diff --git a/apps/server/integration/transferBudgetV2.integration.test.ts b/apps/server/integration/transferBudgetV2.integration.test.ts
index 4aaac3b9e30e9..e4df9fd314ca2 100644
--- a/apps/server/integration/transferBudgetV2.integration.test.ts
+++ b/apps/server/integration/transferBudgetV2.integration.test.ts
@@ -109,7 +109,9 @@ const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Req
const CLIENT_SHELL_OPT_INS = {
omitIdleSubagents: "1",
omitOlderSettled: "1",
-} satisfies Required>;
+} satisfies Required<
+ Omit
+>;
const CLIENT_THREAD_SUBSCRIPTION = {
requestCompletionMarker: true,
acceptBoundedSnapshot: true,
@@ -133,7 +135,9 @@ const clientShellResumeQuery = (resumeFrom: number) =>
query({
...CLIENT_SHELL_OPT_INS,
resumeFrom: String(resumeFrom),
- } satisfies Required);
+ } satisfies Required<
+ Omit
+ >);
const decodeThreadSnapshot = Schema.decodeUnknownEffect(
Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)),
diff --git a/apps/server/package.json b/apps/server/package.json
index 9b9b3c022cacb..0942d991fcd5c 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -86,6 +86,7 @@
"@handy-computer/models": "0.2.3",
"@opencode-ai/sdk": "^1.3.15",
"@t3tools/contracts": "workspace:*",
+ "@t3tools/openshell": "workspace:*",
"@t3tools/provider-testing": "workspace:*",
"@t3tools/secret-provider-core": "workspace:*",
"@t3tools/secret-provider-definitions": "workspace:*",
diff --git a/apps/server/src/environment/OpenShellFleetBundleImport.test.ts b/apps/server/src/environment/OpenShellFleetBundleImport.test.ts
new file mode 100644
index 0000000000000..12503179b8bf3
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetBundleImport.test.ts
@@ -0,0 +1,665 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { describe, expect, it } from "@effect/vitest";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Path from "effect/Path";
+import * as SqlClient from "effect/sql/SqlClient";
+
+import { EnvironmentId } from "@t3tools/contracts";
+
+import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts";
+import * as Sqlite from "../persistence/Sqlite.ts";
+import Migration0111 from "../persistence/Migrations/111_FleetDrainImports.ts";
+import * as OpenShellFleetDrainAdapters from "./OpenShellFleetDrainAdapters.ts";
+import * as OpenShellFleetBundleImport from "./OpenShellFleetBundleImport.ts";
+
+const ENVIRONMENT = "environment:worker";
+const DRAIN_ID = "drain:environment:worker:1";
+
+const base64 = (text: string) => Buffer.from(text, "utf8").toString("base64");
+
+/** A minimal wire-format bounded snapshot for one archived-candidate thread. */
+const snapshotJson = (input: {
+ readonly threadId: string;
+ readonly snapshotSequence: number;
+ readonly title?: string;
+ readonly itemText?: string;
+ readonly compacted?: boolean;
+}) => ({
+ snapshotSequence: input.snapshotSequence,
+ ...(input.compacted === true ? { turnItemsOmitLocalVisible: true } : {}),
+ projection: {
+ thread: {
+ createdBy: "system",
+ creationSource: "server",
+ id: input.threadId,
+ projectId: "project:drain-test",
+ title: input.title ?? "Worker thread",
+ providerInstanceId: "codex",
+ modelSelection: { instanceId: "codex", model: "gpt-5.4" },
+ runtimeMode: "auto",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ activeProviderThreadId: null,
+ lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: input.threadId },
+ forkedFrom: null,
+ createdAt: "2026-11-06T10:00:00.000Z",
+ updatedAt: "2026-11-06T11:00:00.000Z",
+ archivedAt: null,
+ deletedAt: null,
+ },
+ runs: [],
+ attempts: [],
+ nodes: [],
+ subagents: [],
+ providerSessions: [],
+ providerThreads: [
+ {
+ id: "provider-thread:1",
+ driver: "codex",
+ providerInstanceId: "codex",
+ providerSessionId: null,
+ appThreadId: input.threadId,
+ ownerNodeId: null,
+ nativeThreadRef: { driver: "codex", nativeId: "native-1", strength: "strong" },
+ nativeConversationHeadRef: null,
+ status: "idle",
+ firstRunOrdinal: null,
+ lastRunOrdinal: null,
+ handoffIds: [],
+ forkedFrom: null,
+ pendingBackgroundTasks: [],
+ createdAt: "2026-11-06T10:00:00.000Z",
+ updatedAt: "2026-11-06T11:00:00.000Z",
+ },
+ ],
+ providerTurns: [],
+ runtimeRequests: [],
+ messages: [
+ {
+ createdBy: "user",
+ creationSource: "server",
+ id: `${input.threadId}:msg:1`,
+ threadId: input.threadId,
+ runId: null,
+ nodeId: null,
+ role: "user",
+ text: "hello from the worker",
+ attachments: [],
+ streaming: false,
+ createdAt: "2026-11-06T10:00:00.000Z",
+ updatedAt: "2026-11-06T10:00:00.000Z",
+ },
+ ],
+ plans: [],
+ turnItems: [
+ {
+ id: `${input.threadId}:item:1`,
+ threadId: input.threadId,
+ runId: null,
+ nodeId: null,
+ providerThreadId: null,
+ providerTurnId: null,
+ nativeItemRef: null,
+ parentItemId: null,
+ ordinal: 1,
+ status: "completed",
+ title: null,
+ startedAt: "2026-11-06T10:00:00.000Z",
+ completedAt: "2026-11-06T10:00:00.000Z",
+ updatedAt: "2026-11-06T10:00:00.000Z",
+ createdBy: "user",
+ creationSource: "server",
+ type: "user_message",
+ messageId: `${input.threadId}:msg:1`,
+ inputIntent: "turn_start",
+ text: input.itemText ?? "hello from the worker",
+ attachments: [],
+ },
+ ],
+ checkpointScopes: [],
+ checkpoints: [],
+ contextHandoffs: [],
+ contextTransfers: [],
+ visibleTurnItems: [],
+ updatedAt: "2026-11-06T11:00:00.000Z",
+ },
+ historyCursor: null,
+ hasMoreHistory: false,
+ latestLocalTurnOrdinal: 1,
+});
+
+interface WrittenEvents {
+ readonly writes: Array>;
+}
+
+interface FakeStoreState {
+ readonly rows: Map;
+ readonly exists: Set;
+}
+
+const harness = Effect.acquireUseRelease(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const stateDir = yield* fs.makeTempDirectory();
+ const attachmentsDir = yield* fs.makeTempDirectory();
+ return { fs, path, stateDir, attachmentsDir };
+ }),
+ (resources) =>
+ Effect.gen(function* () {
+ const { fs, path, stateDir, attachmentsDir } = resources;
+ const written: WrittenEvents["writes"] = [];
+ const store: FakeStoreState = { rows: new Map(), exists: new Set() };
+ let writesUntilFailure: number | undefined;
+ const eventSink = {
+ write: (input: { readonly events: ReadonlyArray<{ id: string; type: string }> }) =>
+ Effect.suspend(() => {
+ if (writesUntilFailure !== undefined) {
+ if (writesUntilFailure === 0) {
+ return Effect.fail(
+ new OpenShellFleetBundleImport.FleetDrainImportError({
+ operation: "record",
+ message: "the event sink is unavailable",
+ }),
+ );
+ }
+ writesUntilFailure -= 1;
+ }
+ return Effect.sync(() => {
+ written.push(input.events);
+ // The fake projection learns about the thread from its creation event,
+ // as the real event sink does.
+ for (const event of input.events as ReadonlyArray<{
+ readonly type?: string;
+ readonly threadId?: unknown;
+ }>)
+ if (event.type === "thread.created") store.exists.add(String(event.threadId));
+ return input.events;
+ });
+ }),
+ };
+ const importer = OpenShellFleetBundleImport.makeFleetBundleImporter({
+ stateDir,
+ attachmentsDir,
+ fs,
+ path,
+ now: Effect.succeed(1_700_000_000_000),
+ eventSink: eventSink as never,
+ store: {
+ find: (threadId) => Effect.succeed(store.rows.get(threadId) ?? null),
+ threadExists: (threadId) => Effect.succeed(store.exists.has(threadId)),
+ record: (row) =>
+ Effect.sync(() => {
+ store.rows.set(row.threadId, row);
+ }),
+ },
+ });
+ const writeBundle = (input: {
+ readonly threadId: string;
+ readonly snapshotSequence: number;
+ readonly title?: string;
+ readonly compacted?: boolean;
+ readonly attachments?: ReadonlyArray<{ name: string; bytes: string }>;
+ readonly attachmentsSkipped?: boolean;
+ }) =>
+ Effect.gen(function* () {
+ const snapshot = snapshotJson(input);
+ const sink = yield* OpenShellFleetDrainAdapters.fileDrainSink({ stateDir, fs, path });
+ yield* sink.write({
+ environmentId: EnvironmentId.make(ENVIRONMENT),
+ drainId: DRAIN_ID,
+ threadId: input.threadId,
+ body: JSON.stringify({ threadId: input.threadId, snapshot, pages: [] }),
+ snapshotSequence: input.snapshotSequence,
+ ...(input.attachments === undefined && !input.attachmentsSkipped
+ ? {}
+ : {
+ attachments: {
+ files: input.attachments ?? [],
+ oversizeSkipped: input.attachmentsSkipped === true,
+ },
+ }),
+ });
+ });
+ return {
+ importer,
+ written,
+ store,
+ stateDir,
+ attachmentsDir,
+ fs,
+ path,
+ writeBundle,
+ failWrites: (count?: number) => {
+ writesUntilFailure = count;
+ },
+ };
+ }),
+ ({ fs, stateDir, attachmentsDir }) =>
+ Effect.all([
+ fs.remove(stateDir, { recursive: true }),
+ fs.remove(attachmentsDir, { recursive: true }),
+ ]).pipe(Effect.ignore),
+);
+
+const eventTypes = (written: WrittenEvents["writes"]) =>
+ written.flatMap((batch) => batch.map((event) => event.type));
+
+const importedThreadIds = (written: WrittenEvents["writes"]) =>
+ written.flatMap((batch) =>
+ batch.filter((event) => event.type === "thread.created").map((event) => event.id),
+ );
+
+describe("fleet bundle import", () => {
+ it.effect("imports a drained thread as an archived thread with a visible marker", () =>
+ Effect.gen(function* () {
+ const { importer, written, store, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ expect(result.skipped).toBe(0);
+ expect(eventTypes(written)).toContain("thread.created");
+ expect(eventTypes(written)).toContain("turn-item.updated");
+ expect(eventTypes(written)).toContain("message.updated");
+ expect(eventTypes(written)).toContain("provider-thread.updated");
+ const head = written
+ .flatMap((batch) => batch)
+ .find((event) => event.type === "thread.created") as unknown as {
+ payload: { archivedAt: string | null; title: string; tags?: ReadonlyArray };
+ };
+ expect(head.payload.archivedAt).not.toBeNull();
+ expect(head.payload.title).toContain("(drained from environment:worker)");
+ expect(head.payload.tags).toContain("fleet-drain:environment:worker");
+ expect(store.rows.get("thread:a")?.snapshotSequence).toBe(3);
+ expect(store.rows.get("thread:a")?.drainId).toBe(DRAIN_ID);
+ expect(importedThreadIds(written)).toHaveLength(1);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("a checkout artifact is not imported as a thread", () =>
+ Effect.gen(function* () {
+ const { importer, writeBundle, stateDir, fs, path } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+ // A git bundle lands beside the thread copies under the worktree: key.
+ yield* fs.writeFileString(
+ path.join(
+ stateDir,
+ "fleet-drain",
+ encodeURIComponent(ENVIRONMENT),
+ encodeURIComponent(DRAIN_ID),
+ `thread-checkout.json`,
+ ),
+ JSON.stringify({ threadId: "worktree:/work/a", snapshotSequence: null, body: "QlVORExF" }),
+ );
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ expect(result.skipped).toBe(0);
+ expect(result.warnings).toHaveLength(0);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("the same threadId and snapshotSequence imports only once", () =>
+ Effect.gen(function* () {
+ const { importer, written, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+ yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID });
+ const firstCount = written.flat().length;
+
+ const again = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(again.imported).toBe(0);
+ expect(again.skipped).toBe(1);
+ // No new events and no duplicate thread creation.
+ expect(written.flat().length).toBe(firstCount);
+ expect(importedThreadIds(written)).toHaveLength(1);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("a changed snapshotSequence supersedes the previous import", () =>
+ Effect.gen(function* () {
+ const { importer, written, store, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3, title: "Before" });
+ yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID });
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9, title: "After" });
+ written.length = 0;
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ expect(result.skipped).toBe(0);
+ // The thread shell is refreshed, not re-created.
+ expect(eventTypes(written)).not.toContain("thread.created");
+ expect(eventTypes(written)).toContain("thread.metadata-updated");
+ expect(eventTypes(written)).toContain("turn-item.updated");
+ const head = written
+ .flatMap((batch) => batch)
+ .find((event) => event.type === "thread.metadata-updated") as unknown as {
+ payload: { title: string };
+ };
+ expect(head.payload.title).toContain("After");
+ expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("does not let another worker's sequence supersede the prior owner", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9 });
+ yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID });
+ const owned = store.rows.get("thread:a");
+ if (owned === undefined) throw new Error("expected the first import record");
+ store.rows.set("thread:a", { ...owned, environmentId: "environment:worker-a" });
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 10 });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(0);
+ expect(result.skipped).toBe(1);
+ expect(result.warnings.join(" ")).toContain("drained from environment:worker-a");
+ expect(store.rows.get("thread:a")?.environmentId).toBe("environment:worker-a");
+ expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("never overwrites a local thread that was not drain-imported", () =>
+ Effect.gen(function* () {
+ const { importer, written, store, writeBundle } = yield* harness;
+ store.exists.add("thread:local");
+ yield* writeBundle({ threadId: "thread:local", snapshotSequence: 1 });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(0);
+ expect(result.skipped).toBe(1);
+ expect(result.warnings).toHaveLength(1);
+ expect(written).toHaveLength(0);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("a broken bundle warns and never fails the whole import", () =>
+ Effect.gen(function* () {
+ const { importer, written, stateDir, fs, path, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 1 });
+ yield* fs.writeFileString(
+ path.join(
+ stateDir,
+ "fleet-drain",
+ encodeURIComponent(ENVIRONMENT),
+ encodeURIComponent(DRAIN_ID),
+ "thread-broken.json",
+ ),
+ '{"threadId":"thread:broken"}',
+ );
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ expect(result.warnings).toHaveLength(1);
+ expect(written.flat().length).toBeGreaterThan(0);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("a compacted snapshot is refused instead of silently losing its omitted items", () =>
+ Effect.gen(function* () {
+ const { importer, written, store, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 1, compacted: true });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(0);
+ expect(result.skipped).toBe(1);
+ expect(result.warnings).toHaveLength(1);
+ expect(result.warnings[0]).toContain("compacted snapshot");
+ expect(written).toHaveLength(0);
+ expect(store.rows.get("thread:a")).toBeUndefined();
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
+
+describe("import resumability", () => {
+ it.effect("resumes an import that failed after claiming the thread", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle, failWrites, written } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+ // The head lands, the first transcript batch fails.
+ failWrites(1);
+ const first = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+ expect(first.imported).toBe(0);
+ expect(first.warnings).toHaveLength(1);
+ expect(store.rows.get("thread:a")?.importedAt).toBe("");
+
+ failWrites(undefined);
+ written.length = 0;
+ const second = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(second.imported).toBe(1);
+ expect(store.rows.get("thread:a")?.importedAt).not.toBe("");
+ // The retry replays the thread instead of treating its own head as a collision.
+ expect(eventTypes(written)).toContain("turn-item.updated");
+ expect(eventTypes(written)).toContain("message.updated");
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("refuses a bundle whose records belong to another thread", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle, written, stateDir, fs, path } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+ const file = path.join(
+ stateDir,
+ "fleet-drain",
+ encodeURIComponent(ENVIRONMENT),
+ encodeURIComponent(DRAIN_ID),
+ `thread-${encodeURIComponent("thread:a")}.json`,
+ );
+ const envelope = JSON.parse(yield* fs.readFileString(file)) as { body: string };
+ envelope.body = envelope.body.replaceAll("thread:a", "thread:victim");
+ yield* fs.writeFileString(file, JSON.stringify(envelope));
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(0);
+ expect(result.skipped).toBe(1);
+ expect(result.warnings).toHaveLength(1);
+ expect(written).toEqual([]);
+ expect(store.rows.size).toBe(0);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+ it.effect("refuses a bundle older than the imported snapshot", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 9 });
+ yield* importer.importBundle({ environmentId: ENVIRONMENT, drainId: DRAIN_ID });
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+
+ const older = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(older.imported).toBe(0);
+ expect(older.skipped).toBe(1);
+ expect(older.warnings.join(" ")).toContain("newer snapshot");
+ expect(store.rows.get("thread:a")?.snapshotSequence).toBe(9);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("refuses an unreadable history page before writing anything", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle, written, stateDir, fs, path } = yield* harness;
+ yield* writeBundle({ threadId: "thread:a", snapshotSequence: 3 });
+ const file = path.join(
+ stateDir,
+ "fleet-drain",
+ encodeURIComponent(ENVIRONMENT),
+ encodeURIComponent(DRAIN_ID),
+ `thread-${encodeURIComponent("thread:a")}.json`,
+ );
+ const envelope = JSON.parse(yield* fs.readFileString(file)) as { body: string };
+ const body = JSON.parse(envelope.body) as { pages: unknown };
+ body.pages = [{ items: [{ item: { threadId: "thread:victim" } }] }];
+ envelope.body = JSON.stringify(body);
+ yield* fs.writeFileString(file, JSON.stringify(envelope));
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(0);
+ expect(result.skipped).toBe(0);
+ expect(result.warnings).toHaveLength(1);
+ expect(written).toEqual([]);
+ expect(store.rows.size).toBe(0);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
+
+describe("attachment restore", () => {
+ it.effect("restores manifest attachments flat, keyed by the attachment id", () =>
+ Effect.gen(function* () {
+ const { importer, store, attachmentsDir, fs, writeBundle } = yield* harness;
+ const segment = toSafeThreadAttachmentSegment("thread:a");
+ const name = `${segment}-1111aaaa.png`;
+ yield* writeBundle({
+ threadId: "thread:a",
+ snapshotSequence: 2,
+ attachments: [{ name, bytes: base64("png-bytes") }],
+ });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ const restored = yield* fs.readFile(`${attachmentsDir}/${name}`);
+ expect(new TextDecoder().decode(restored)).toBe("png-bytes");
+ expect(store.rows.get("thread:a")?.oversizeAttachmentsSkipped).toBe(false);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("refuses an attachment named for another thread", () =>
+ Effect.gen(function* () {
+ const { importer, store, attachmentsDir, fs, writeBundle } = yield* harness;
+ yield* writeBundle({
+ threadId: "thread:a",
+ snapshotSequence: 2,
+ attachments: [{ name: "other-thread-1111aaaa.png", bytes: base64("png-bytes") }],
+ });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ const entries = yield* fs.readDirectory(attachmentsDir);
+ expect(entries).toEqual([]);
+ expect(store.rows.get("thread:a")?.snapshotSequence).toBe(2);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("records an oversize skip in the import marker", () =>
+ Effect.gen(function* () {
+ const { importer, store, writeBundle } = yield* harness;
+ yield* writeBundle({
+ threadId: "thread:big",
+ snapshotSequence: 2,
+ attachmentsSkipped: true,
+ });
+
+ const result = yield* importer.importBundle({
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ });
+
+ expect(result.imported).toBe(1);
+ expect(store.rows.get("thread:big")?.oversizeAttachmentsSkipped).toBe(true);
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
+
+describe("sqlite import store", () => {
+ it.effect("round-trips rows and supersedes on conflict", () =>
+ Effect.gen(function* () {
+ const context = yield* Layer.build(Sqlite.layerMemory);
+ const sql = Context.get(context, SqlClient.SqlClient);
+ yield* Migration0111.pipe(Effect.provideService(SqlClient.SqlClient, sql));
+ const store = OpenShellFleetBundleImport.sqliteFleetDrainImportStore(sql);
+
+ expect(yield* store.find("thread:x")).toBeNull();
+ yield* store.record({
+ threadId: "thread:x",
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ snapshotSequence: 4,
+ importedAt: "2026-11-06T10:00:00.000Z",
+ oversizeAttachmentsSkipped: true,
+ });
+ const row = yield* store.find("thread:x");
+ expect(row).toMatchObject({
+ threadId: "thread:x",
+ environmentId: ENVIRONMENT,
+ drainId: DRAIN_ID,
+ snapshotSequence: 4,
+ oversizeAttachmentsSkipped: true,
+ });
+ expect(yield* store.threadExists("thread:x")).toBe(false);
+
+ yield* store.record({
+ threadId: "thread:x",
+ environmentId: ENVIRONMENT,
+ drainId: "drain:environment:worker:2",
+ snapshotSequence: 5,
+ importedAt: "2026-11-06T12:00:00.000Z",
+ oversizeAttachmentsSkipped: false,
+ });
+ expect(yield* store.find("thread:x")).toMatchObject({
+ snapshotSequence: 5,
+ drainId: "drain:environment:worker:2",
+ oversizeAttachmentsSkipped: false,
+ });
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
diff --git a/apps/server/src/environment/OpenShellFleetBundleImport.ts b/apps/server/src/environment/OpenShellFleetBundleImport.ts
new file mode 100644
index 0000000000000..d2a5d524b7f9c
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetBundleImport.ts
@@ -0,0 +1,598 @@
+import {
+ EventId,
+ type OrchestrationV2AppThread,
+ type OrchestrationV2ConversationMessage,
+ type OrchestrationV2DomainEvent,
+ OrchestrationV2ThreadBoundedSnapshot,
+ OrchestrationV2TurnItem,
+ ThreadId,
+} from "@t3tools/contracts";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+
+import { normalizeAttachmentRelativePath } from "../attachmentPaths.ts";
+import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts";
+import * as EventSink from "../orchestration-v2/EventSink.ts";
+import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
+import * as SqlClient from "effect/sql/SqlClient";
+
+/**
+ * Drains a worker bundle into an ARCHIVED read-only thread (spec 7.4): the snapshot's projection
+ * replays through the EventSink (lossless; no lossy AgentSessionImporter fallback). Idempotency:
+ * migration-091 rows mark {environmentId, drainId, snapshotSequence} — same sequence skips,
+ * changed supersedes via projection upserts; crashes before the marker only warn (content stays).
+ */ /** One attachment file recorded in a bundle envelope's manifest. */
+export interface BundleAttachmentManifestEntry {
+ readonly name: string;
+ readonly hash: string;
+ readonly size: number;
+}
+
+/** Transcription of one imports-table row; `snapshotSequence` decides currency. */
+export interface FleetDrainImportRow {
+ readonly threadId: string;
+ readonly environmentId: string;
+ readonly drainId: string;
+ readonly snapshotSequence: number | null;
+ readonly importedAt: string;
+ readonly oversizeAttachmentsSkipped: boolean;
+}
+
+export class FleetDrainImportError extends Schema.TaggedError<
+ FleetDrainImportError,
+ { readonly brand: unique symbol }
+>()("FleetDrainImportError", {
+ operation: Schema.Literals(["read-bundle", "decode", "record", "restore-attachments"]),
+ threadId: Schema.optional(ThreadId),
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+/** Where import state lives; sqlite-backed in production, faked in tests. */
+export interface FleetDrainImportStore {
+ readonly find: (
+ threadId: string,
+ ) => Effect.Effect;
+ readonly threadExists: (threadId: string) => Effect.Effect;
+ readonly record: (row: FleetDrainImportRow) => Effect.Effect;
+}
+
+const EVENT_PREFIX = "fleet-drain";
+/** `importedAt` value of a row claimed before its events were written. */
+const PENDING_IMPORTED_AT = "";
+const TRANSCRIPT_EVENT_BATCH_SIZE = 100;
+const TITLE_SUFFIX_OF = (environmentId: string) => ` (drained from ${environmentId})`;
+
+const decodeSnapshot = Schema.decodeUnknownEffect(
+ Schema.toCodecJson(OrchestrationV2ThreadBoundedSnapshot),
+);
+const decodeTurnItem = Schema.decodeUnknownEffect(Schema.toCodecJson(OrchestrationV2TurnItem));
+
+const decodeEnvelopeJson = Schema.decodeEffect(
+ Schema.fromJsonString(
+ Schema.Struct({
+ threadId: Schema.optional(Schema.String),
+ /** The copied thread as JSON: `{ threadId, snapshot, pages }`. */
+ body: Schema.optional(Schema.String),
+ attachments: Schema.optional(
+ Schema.Array(
+ Schema.Struct({
+ name: Schema.String,
+ hash: Schema.String,
+ size: Schema.Number,
+ }),
+ ),
+ ),
+ attachmentsSkipped: Schema.optional(Schema.Boolean),
+ }),
+ ),
+);
+
+const decodeBodyJson = Schema.decodeEffect(
+ Schema.fromJsonString(
+ Schema.Struct({
+ snapshot: Schema.optional(Schema.Unknown),
+ pages: Schema.optional(
+ Schema.Array(
+ Schema.Struct({
+ items: Schema.optional(
+ Schema.Array(Schema.Struct({ item: Schema.optional(Schema.Unknown) })),
+ ),
+ }),
+ ),
+ ),
+ }),
+ ),
+);
+
+const archivedThreadPayload = (input: {
+ readonly thread: OrchestrationV2AppThread;
+ readonly environmentId: string;
+ readonly archivedAt: DateTime.Utc;
+}): OrchestrationV2AppThread => ({
+ ...input.thread,
+ title: input.thread.title.endsWith(TITLE_SUFFIX_OF(input.environmentId))
+ ? input.thread.title
+ : `${input.thread.title}${TITLE_SUFFIX_OF(input.environmentId)}`,
+ tags: [...new Set([...(input.thread.tags ?? []), `${EVENT_PREFIX}:${input.environmentId}`])],
+ archivedAt: input.archivedAt,
+ updatedAt: input.archivedAt,
+});
+
+const importEventId = (parts: {
+ readonly threadId: string;
+ readonly snapshotSequence: number | null;
+ readonly kind: string;
+ readonly ref: string;
+}): EventId =>
+ EventId.make(
+ `${EVENT_PREFIX}:${parts.threadId}:s${parts.snapshotSequence ?? "none"}:${parts.kind}:${parts.ref}`,
+ );
+
+const transcriptEvents = (input: {
+ readonly threadId: ThreadId;
+ readonly occurredAt: DateTime.Utc;
+ readonly snapshotSequence: number | null;
+ readonly turnItems: ReadonlyArray;
+ readonly messages: ReadonlyArray;
+}): Array => [
+ ...input.turnItems.map((item): OrchestrationV2DomainEvent => ({
+ id: importEventId({
+ threadId: input.threadId,
+ snapshotSequence: input.snapshotSequence,
+ kind: "turn-item",
+ ref: item.id,
+ }),
+ type: "turn-item.updated",
+ threadId: input.threadId,
+ occurredAt: input.occurredAt,
+ payload: item,
+ })),
+ ...input.messages.map((message): OrchestrationV2DomainEvent => ({
+ id: importEventId({
+ threadId: input.threadId,
+ snapshotSequence: input.snapshotSequence,
+ kind: "message",
+ ref: message.id,
+ }),
+ type: "message.updated",
+ threadId: input.threadId,
+ occurredAt: input.occurredAt,
+ payload: message,
+ })),
+];
+
+const inBatches = (items: ReadonlyArray, size: number): Array> => {
+ const batches: Array> = [];
+ for (let index = 0; index < items.length; index += size) {
+ batches.push(items.slice(index, index + size));
+ }
+ return batches;
+};
+
+const restoreAttachments = Effect.fn("FleetBundleImport.restoreAttachments")(function* (input: {
+ readonly manifest: ReadonlyArray;
+ readonly bundleDir: string;
+ readonly attachmentsDir: string;
+ readonly threadId: string;
+ readonly fs: FileSystem.FileSystem;
+ readonly path: Path.Path;
+}) {
+ const segment = toSafeThreadAttachmentSegment(input.threadId);
+ if (segment === null) return;
+ yield* input.fs.makeDirectory(input.attachmentsDir, { recursive: true }).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "restore-attachments",
+ threadId: ThreadId.make(input.threadId),
+ message: `Attachment directory for ${input.threadId} could not be created.`,
+ cause,
+ }),
+ ),
+ );
+ for (const entry of input.manifest) {
+ // The store keeps attachments flat as `-`; only
+ // such a name may land in the attachments root.
+ const name = normalizeAttachmentRelativePath(entry.name);
+ if (name === null || name.includes("/") || name !== entry.name) continue;
+ if (!name.startsWith(`${segment}-`)) continue;
+ const source = input.path.join(
+ input.bundleDir,
+ `thread-${encodeURIComponent(input.threadId)}-att-${entry.hash.slice(0, 16)}.bin`,
+ );
+ const bytes = yield* input.fs.readFile(source).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "restore-attachments",
+ threadId: ThreadId.make(input.threadId),
+ message: `Attachment ${entry.name} could not be read back from the bundle.`,
+ cause,
+ }),
+ ),
+ );
+ const target = input.path.join(input.attachmentsDir, name);
+ yield* input.fs.writeFile(target, bytes).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "restore-attachments",
+ threadId: ThreadId.make(input.threadId),
+ message: `Attachment ${entry.name} could not be restored.`,
+ cause,
+ }),
+ ),
+ );
+ }
+});
+
+export interface FleetBundleImporterShape {
+ /** Imports every bundle in a drain directory; per-thread problems warn, never throw. */
+ readonly importBundle: (input: {
+ readonly environmentId: string;
+ readonly drainId: string;
+ }) => Effect.Effect<
+ {
+ readonly imported: number;
+ readonly skipped: number;
+ readonly warnings: ReadonlyArray;
+ },
+ FleetDrainImportError
+ >;
+}
+
+interface BundleEnvelope {
+ readonly threadId?: string | undefined;
+ /** The copied thread as JSON: `{ threadId, snapshot, pages }`. */
+ readonly body?: string | undefined;
+ readonly attachments?: ReadonlyArray | undefined;
+ readonly attachmentsSkipped?: boolean | undefined;
+}
+
+export const makeFleetBundleImporter = (input: {
+ readonly stateDir: string;
+ readonly attachmentsDir: string;
+ readonly fs: FileSystem.FileSystem;
+ readonly path: Path.Path;
+ readonly now: Effect.Effect;
+ readonly eventSink: Pick;
+ readonly store: FleetDrainImportStore;
+}): FleetBundleImporterShape => {
+ const { fs, path, stateDir } = input;
+ return {
+ importBundle: Effect.fn("FleetBundleImport.importBundle")(function* (request) {
+ const bundleDir = path.join(
+ stateDir,
+ "fleet-drain",
+ encodeURIComponent(request.environmentId),
+ encodeURIComponent(request.drainId),
+ );
+ const entries = yield* fs.readDirectory(bundleDir).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "read-bundle",
+ message: `The drain bundle directory for ${request.drainId} could not be read.`,
+ cause,
+ }),
+ ),
+ );
+ let imported = 0;
+ let skipped = 0;
+ const warnings: string[] = [];
+ for (const file of entries.filter(
+ (name) => /^thread-.*\.json$/.test(name) && !name.startsWith("thread-worktree%3A"),
+ )) {
+ const outcome = yield* Effect.gen(function* () {
+ const raw = yield* fs.readFileString(path.join(bundleDir, file)).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "read-bundle",
+ message: `Bundle ${file} could not be read.`,
+ cause,
+ }),
+ ),
+ );
+ const decodedEnvelope = yield* decodeEnvelopeJson(raw).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "decode",
+ message: `Bundle ${file} is not a drained thread copy.`,
+ cause,
+ }),
+ ),
+ );
+ const envelope: BundleEnvelope = decodedEnvelope;
+ const threadId = envelope.threadId;
+ // Checkout artifacts share the thread-*.json naming but hold no thread.
+ if (threadId !== undefined && threadId.startsWith("worktree:")) return "ignored" as const;
+ if (threadId === undefined || envelope.body === undefined) {
+ return yield* new FleetDrainImportError({
+ operation: "decode",
+ message: `Bundle ${file} is not a drained thread copy.`,
+ });
+ }
+ const body = yield* decodeBodyJson(envelope.body).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "decode",
+ threadId: ThreadId.make(threadId),
+ message: `Bundle ${file} does not hold a readable thread snapshot.`,
+ cause,
+ }),
+ ),
+ );
+ if (body.snapshot === undefined) {
+ return yield* new FleetDrainImportError({
+ operation: "decode",
+ threadId: ThreadId.make(threadId),
+ message: `Bundle ${file} does not hold a readable thread snapshot.`,
+ });
+ }
+ const snapshot = yield* decodeSnapshot(body.snapshot).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "decode",
+ threadId: ThreadId.make(threadId),
+ message: `Bundle ${file} does not hold a readable thread snapshot.`,
+ cause,
+ }),
+ ),
+ );
+ // A compacted snapshot leads with items omitted (older workers answer
+ // `compactTurnItems=1`); the drain exports full snapshots, so this
+ // marker means the bundle cannot be proven complete. Refuse loudly.
+ if (snapshot.turnItemsOmitLocalVisible === true) {
+ warnings.push(
+ `Bundle ${file} holds a compacted snapshot of ${threadId} (local-visible items omitted); it was not imported.`,
+ );
+ return "skipped" as const;
+ }
+ // A bundle may name one thread and carry another's records; every
+ // ownership id must agree with the envelope before anything is written.
+ // Decode every history item before ownership checks: an unreadable
+ // page must fail the import rather than disappear from the transcript.
+ const historyItems: Array = [];
+ for (const entry of (body.pages ?? []).flatMap((page) => page.items ?? [])) {
+ if (entry.item === undefined) continue;
+ historyItems.push(
+ yield* decodeTurnItem(entry.item).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainImportError({
+ operation: "decode",
+ threadId: ThreadId.make(threadId),
+ message: `Bundle ${file} holds an unreadable history item.`,
+ cause,
+ }),
+ ),
+ ),
+ );
+ }
+ const foreignIds = [
+ snapshot.projection.thread.id,
+ ...snapshot.projection.turnItems.map((item) => item.threadId),
+ ...snapshot.projection.messages.map((message) => message.threadId),
+ ...snapshot.projection.providerThreads.map((entry) => entry.appThreadId),
+ ...historyItems.map((item) => item.threadId),
+ ].filter((id) => id !== threadId);
+ if (foreignIds.length > 0) {
+ warnings.push(
+ `Bundle ${file} names ${threadId} but carries records of ${foreignIds[0]}; it was not imported.`,
+ );
+ return "skipped" as const;
+ }
+ const snapshotSequence = snapshot.snapshotSequence;
+ const existing = yield* input.store.find(threadId);
+ // Sequence numbers only order imports from the same worker. Worker A's
+ // older snapshot must not authorize replacing A's record with B's.
+ if (existing !== null && existing.environmentId !== request.environmentId) {
+ warnings.push(
+ `Thread ${threadId} was drained from ${existing.environmentId}; worker ${request.environmentId}'s bundle was not imported.`,
+ );
+ return "skipped" as const;
+ }
+ if (
+ existing !== null &&
+ existing.snapshotSequence !== null &&
+ existing.importedAt !== PENDING_IMPORTED_AT
+ ) {
+ if (existing.snapshotSequence === snapshotSequence) return "skipped" as const;
+ // A bundle older than what was imported is stale, not a supersede.
+ if (existing.snapshotSequence > snapshotSequence) {
+ warnings.push(
+ `Thread ${threadId} already holds a newer snapshot (${existing.snapshotSequence}); the older bundle was not imported.`,
+ );
+ return "skipped" as const;
+ }
+ }
+ const exists = yield* input.store.threadExists(threadId);
+ if (exists && existing === null) {
+ // A local thread that was not drain-imported owns this id; archiving
+ // or overwriting it would destroy local work, so the bundle stays.
+ warnings.push(
+ `Thread ${threadId} already exists here and was not drain-imported; its bundle was not imported.`,
+ );
+ return "skipped" as const;
+ }
+ const now = DateTime.makeUnsafe(yield* input.now);
+ // A brand-new thread is claimed before events so a crash cannot make a
+ // partial import look local. A completed row keeps its watermark until
+ // the newer import succeeds, so a failed supersede never loses it.
+ if (existing === null || existing.importedAt === PENDING_IMPORTED_AT) {
+ yield* input.store.record({
+ threadId,
+ environmentId: request.environmentId,
+ drainId: request.drainId,
+ snapshotSequence,
+ importedAt: PENDING_IMPORTED_AT,
+ oversizeAttachmentsSkipped: envelope.attachmentsSkipped === true,
+ });
+ }
+ const payload = archivedThreadPayload({
+ thread: snapshot.projection.thread,
+ environmentId: request.environmentId,
+ archivedAt: now,
+ });
+ const turnItems = [...snapshot.projection.turnItems, ...historyItems];
+ const thread = ThreadId.make(threadId);
+ const head: OrchestrationV2DomainEvent = {
+ id: importEventId({
+ threadId,
+ snapshotSequence,
+ kind: "thread",
+ ref: String(snapshotSequence ?? "none"),
+ }),
+ // The projection decides, not the marker: a resume before the head
+ // wrote still has to create the thread.
+ type: exists ? "thread.metadata-updated" : "thread.created",
+ threadId: thread,
+ providerInstanceId: payload.providerInstanceId,
+ occurredAt: now,
+ payload,
+ };
+ const providerThreadEvents: Array =
+ snapshot.projection.providerThreads.map(
+ (providerThread): OrchestrationV2DomainEvent => ({
+ id: importEventId({
+ threadId,
+ snapshotSequence,
+ kind: "provider-thread",
+ ref: providerThread.id,
+ }),
+ type: "provider-thread.updated",
+ threadId: thread,
+ providerInstanceId: providerThread.providerInstanceId,
+ occurredAt: now,
+ payload: providerThread,
+ }),
+ );
+ // Restore attachment files before recording, so a crash between the
+ // two leaves files but no marker and the retry replays idempotently.
+ yield* restoreAttachments({
+ manifest: envelope.attachments ?? [],
+ bundleDir,
+ attachmentsDir: input.attachmentsDir,
+ threadId,
+ fs,
+ path,
+ });
+ yield* input.eventSink.write({
+ events: [head, ...providerThreadEvents],
+ });
+ const transcript = transcriptEvents({
+ threadId: thread,
+ occurredAt: now,
+ snapshotSequence,
+ turnItems,
+ messages: snapshot.projection.messages,
+ });
+ for (const batch of inBatches(transcript, TRANSCRIPT_EVENT_BATCH_SIZE)) {
+ yield* input.eventSink.write({ events: [...batch] });
+ }
+ yield* input.store.record({
+ threadId,
+ environmentId: request.environmentId,
+ drainId: request.drainId,
+ snapshotSequence,
+ importedAt: DateTime.formatIso(now),
+ oversizeAttachmentsSkipped: envelope.attachmentsSkipped === true,
+ });
+ return "imported" as const;
+ }).pipe(
+ Effect.tapError((cause) =>
+ Effect.logWarning("Fleet drain bundle import failed", { file, cause }),
+ ),
+ Effect.catch((_cause) =>
+ Effect.sync(() => {
+ warnings.push(
+ `Bundle ${file} could not be imported; the server log has the bounded failure details.`,
+ );
+ return "failed" as const;
+ }),
+ ),
+ );
+ if (outcome === "imported") imported += 1;
+ else if (outcome === "skipped") skipped += 1;
+ }
+ return { imported, skipped, warnings };
+ }),
+ };
+};
+
+/**
+ * The sqlite-backed import-state store over migration 091's table; tests run
+ * it on the in-memory sqlite layer like the other persistence stores.
+ */
+export const sqliteFleetDrainImportStore = (sql: SqlClient.SqlClient): FleetDrainImportStore => {
+ const failed = (message: string, cause: unknown) =>
+ new FleetDrainImportError({ operation: "record", message, cause });
+ return {
+ find: (threadId) =>
+ sql<{
+ readonly thread_id: string;
+ readonly environment_id: string;
+ readonly drain_id: string;
+ readonly snapshot_sequence: number | null;
+ readonly imported_at: string;
+ readonly oversize_attachments_skipped: number;
+ }>`
+ SELECT thread_id, environment_id, drain_id, snapshot_sequence, imported_at, oversize_attachments_skipped
+ FROM orchestration_v2_fleet_drain_imports
+ WHERE thread_id = ${threadId}
+ `.pipe(
+ NodeSqliteClient.readOnly,
+ Effect.map((rows) => {
+ const row = rows[0];
+ return row === undefined
+ ? null
+ : {
+ threadId: row.thread_id,
+ environmentId: row.environment_id,
+ drainId: row.drain_id,
+ snapshotSequence: row.snapshot_sequence,
+ importedAt: row.imported_at,
+ oversizeAttachmentsSkipped: row.oversize_attachments_skipped === 1,
+ };
+ }),
+ Effect.mapError((cause) =>
+ failed("The fleet drain import state could not be read.", cause),
+ ),
+ ),
+ threadExists: (threadId) =>
+ sql<{ readonly present: number }>`
+ SELECT 1 AS present
+ FROM orchestration_v2_projection_threads
+ WHERE thread_id = ${threadId}
+ `.pipe(
+ NodeSqliteClient.readOnly,
+ Effect.map((rows) => rows.length > 0),
+ Effect.mapError((cause) => failed("The thread projection could not be consulted.", cause)),
+ ),
+ record: (row) =>
+ sql`
+ INSERT INTO orchestration_v2_fleet_drain_imports
+ (thread_id, environment_id, drain_id, snapshot_sequence, imported_at, lossy, oversize_attachments_skipped)
+ VALUES (${row.threadId}, ${row.environmentId}, ${row.drainId}, ${row.snapshotSequence}, ${row.importedAt}, 0, ${row.oversizeAttachmentsSkipped ? 1 : 0})
+ ON CONFLICT(thread_id) DO UPDATE SET
+ environment_id = excluded.environment_id,
+ drain_id = excluded.drain_id,
+ snapshot_sequence = excluded.snapshot_sequence,
+ imported_at = excluded.imported_at,
+ oversize_attachments_skipped = excluded.oversize_attachments_skipped
+ `.pipe(
+ Effect.asVoid,
+ Effect.mapError((cause) =>
+ failed(`The fleet drain import state for ${row.threadId} could not be recorded.`, cause),
+ ),
+ ),
+ };
+};
diff --git a/apps/server/src/environment/OpenShellFleetDrain.test.ts b/apps/server/src/environment/OpenShellFleetDrain.test.ts
new file mode 100644
index 0000000000000..1c56b3d755125
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetDrain.test.ts
@@ -0,0 +1,632 @@
+import { describe, expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import { EnvironmentId } from "@t3tools/contracts";
+
+import {
+ type DrainDependencies,
+ FleetDrainStepError,
+ makeDrainExecutor,
+} from "./OpenShellFleetDrain.ts";
+
+const environmentId = EnvironmentId.make("environment:worker");
+
+const deps = (input: {
+ readonly threads: ReadonlyArray;
+ readonly projects?: ReadonlyArray;
+ readonly now?: number;
+ readonly failThread?: string;
+ readonly failShell?: boolean;
+ readonly bundleExitCode?: number;
+ readonly bundleStdout?: string;
+ readonly failCopy?: boolean;
+ /** Marks this thread live on the drain's currency re-read. */
+ readonly liveAfter?: string;
+ /** Appears in the currency re-read only. */
+ readonly threadAfter?: string;
+ /** Finishes a turn after its copy (its sequence advances on later reads). */
+ readonly sequenceAfterCopy?: string;
+ readonly liveThread?: string;
+ readonly sequences?: Readonly>;
+ readonly attachmentFiles?: Readonly>>;
+ readonly oversizeThreads?: ReadonlyArray;
+ readonly dropAttachmentManifest?: boolean;
+ readonly existingFiles?: ReadonlyMap;
+ readonly seeded?: ReadonlyArray<{
+ readonly drainId: string;
+ readonly threadId: string;
+ readonly snapshotSequence: number | null;
+ readonly attachments?: boolean;
+ }>;
+}): DrainDependencies & {
+ readonly written: Array<{ drainId: string; threadId: string; body: string }>;
+ readonly fetched: Array;
+ readonly fetchedAttachments: Array;
+ readonly exported: Array;
+ readonly promoted: Array<{ fromDrainId: string; toDrainId: string; threadId: string }>;
+ readonly files: ReadonlyMap;
+} => {
+ const manifest = [{ name: "a.bin", hash: "h0", size: 3 }];
+ const envelope = (entry: {
+ threadId: string;
+ snapshotSequence: number | null;
+ attachments?: boolean;
+ }) =>
+ JSON.stringify({
+ threadId: entry.threadId,
+ snapshotSequence: entry.snapshotSequence,
+ body: `{"threadId":"${entry.threadId}","seeded":true}`,
+ ...(entry.attachments === true ? { attachments: manifest } : {}),
+ });
+ const files = new Map(
+ input.existingFiles
+ ? [...input.existingFiles]
+ : (input.seeded ?? []).map((entry) => [
+ `${entry.drainId}/${entry.threadId}`,
+ envelope(entry),
+ ]),
+ );
+ const written: Array<{ drainId: string; threadId: string; body: string }> = [];
+ const fetched: Array = [];
+ const fetchedAttachments: Array = [];
+ const exported: Array = [];
+ const promoted: Array<{ fromDrainId: string; toDrainId: string; threadId: string }> = [];
+ return {
+ written,
+ fetched,
+ fetchedAttachments,
+ exported,
+ promoted,
+ files,
+ now: Effect.succeed(input.now ?? 0),
+ source: {
+ shell: (() => {
+ let calls = 0;
+ return () => {
+ calls += 1;
+ return input.failShell === true
+ ? Effect.fail(
+ new FleetDrainStepError({ step: "shell", message: "the worker link is unusable" }),
+ )
+ : Effect.succeed({
+ threads: [
+ ...input.threads.map((threadId) => ({
+ threadId,
+ live:
+ input.liveThread === threadId || (calls > 1 && input.liveAfter === threadId),
+ })),
+ ...(calls > 1 && input.threadAfter !== undefined
+ ? [{ threadId: input.threadAfter, live: false }]
+ : []),
+ ],
+ projects: (input.projects ?? []).map((workspaceRoot) => ({ workspaceRoot })),
+ });
+ };
+ })(),
+ threadSequence: (() => {
+ const reads = new Map();
+ return ({ threadId }: { readonly threadId: string }) => {
+ const count = (reads.get(threadId) ?? 0) + 1;
+ reads.set(threadId, count);
+ const base = input.sequences?.[threadId] ?? 1;
+ return Effect.succeed({
+ snapshotSequence: input.sequenceAfterCopy === threadId && count > 1 ? base + 1 : base,
+ });
+ };
+ })(),
+ threadHistory: ({ threadId }) => {
+ fetched.push(threadId);
+ return threadId === input.failThread
+ ? Effect.fail(
+ new FleetDrainStepError({ step: "history", message: "the worker went away" }),
+ )
+ : Effect.succeed({
+ body: `{"threadId":"${threadId}","events":[1,2,3]}`,
+ snapshotSequence: input.sequences?.[threadId] ?? 1,
+ });
+ },
+ attachments: ({ threadId }) => {
+ fetchedAttachments.push(threadId);
+ if (input.oversizeThreads?.includes(threadId) === true) {
+ return Effect.succeed({ files: [], oversizeSkipped: true });
+ }
+ return Effect.succeed({
+ files: (input.attachmentFiles?.[threadId] ?? []).map((name) => ({
+ name,
+ bytes: Buffer.from(name, "utf8").toString("base64"),
+ })),
+ oversizeSkipped: false,
+ });
+ },
+ },
+ sink: {
+ write: ({ drainId, threadId, body, snapshotSequence, attachments }) =>
+ Effect.sync(() => {
+ const stored = JSON.stringify({
+ threadId,
+ snapshotSequence,
+ body,
+ ...(input.dropAttachmentManifest === true
+ ? {}
+ : (attachments?.files ?? []).length > 0
+ ? {
+ attachments: (attachments?.files ?? []).map((file) => ({
+ name: file.name,
+ hash: `h-${file.name}`,
+ size: file.bytes.length,
+ })),
+ }
+ : {}),
+ ...(attachments?.oversizeSkipped === true ? { attachmentsSkipped: true } : {}),
+ });
+ files.set(`${drainId}/${threadId}`, stored);
+ written.push({ drainId, threadId, body });
+ return { bytes: stored.length };
+ }),
+ read: ({ drainId, threadId }) =>
+ Effect.map(
+ Effect.sync(() => files.get(`${drainId}/${threadId}`)),
+ (raw) => {
+ if (raw === undefined) return null;
+ const parsed = JSON.parse(raw) as {
+ threadId: string;
+ snapshotSequence?: number;
+ body: string;
+ attachments?: Array<{ name: string; hash: string; size: number }>;
+ attachmentsSkipped?: boolean;
+ };
+ return {
+ threadId: parsed.threadId,
+ snapshotSequence:
+ typeof parsed.snapshotSequence === "number" ? parsed.snapshotSequence : null,
+ body: parsed.body,
+ ...(parsed.attachments !== undefined && parsed.attachments.length > 0
+ ? { attachments: parsed.attachments }
+ : {}),
+ ...(parsed.attachmentsSkipped === true ? { attachmentsSkipped: true } : {}),
+ };
+ },
+ ),
+ copy: ({ fromDrainId, toDrainId, threadId }) =>
+ Effect.sync(() => {
+ if (input.failCopy === true) return false;
+ const raw = files.get(`${fromDrainId}/${threadId}`);
+ if (raw === undefined) return false;
+ files.set(`${toDrainId}/${threadId}`, raw);
+ promoted.push({ fromDrainId, toDrainId, threadId });
+ return true;
+ }),
+ },
+ git: {
+ bundle: ({ workspaceRoot }) =>
+ Effect.sync(() => {
+ exported.push(workspaceRoot);
+ return {
+ exitCode: input.bundleExitCode ?? 0,
+ stdout: input.bundleStdout ?? `QlVORExFOiR7d29ya3NwYWNlUm9vdH0=`,
+ stderr: "",
+ };
+ }),
+ },
+ };
+};
+
+const drain = (
+ dependencies: DrainDependencies,
+ overrides: Partial<{ drainId: string; deadlineAt: number; snapshotOnly: boolean }> = {},
+) =>
+ makeDrainExecutor(dependencies).drain({
+ environmentId,
+ sandboxName: "sandbox-1",
+ drainId: overrides.drainId ?? "drain:environment:worker:1",
+ snapshotOnly: overrides.snapshotOnly ?? false,
+ deadlineAt: overrides.deadlineAt ?? Number.MAX_SAFE_INTEGER,
+ });
+
+describe("fleet drain", () => {
+ it.effect("acks threads and checkout bundles only when each one landed", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a", "thread:b"], projects: ["/work/a"] });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("acked");
+ expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 });
+ expect(outcome.worktrees).toEqual({ total: 1, acked: 1, failed: 0 });
+ expect(dependencies.exported).toEqual(["/work/a"]);
+ }),
+ );
+
+ it.effect("does not re-fetch a thread whose previous copy is still current", () =>
+ Effect.gen(function* () {
+ const first = deps({
+ threads: ["thread:a", "thread:b"],
+ failThread: "thread:b",
+ attachmentFiles: { "thread:a": ["first.bin"] },
+ });
+ yield* drain(first, { drainId: "drain:environment:worker:1" });
+ const retry = deps({
+ threads: ["thread:a", "thread:b"],
+ existingFiles: first.files,
+ seeded: first.written.map((entry) => ({
+ drainId: "drain:environment:worker:1",
+ threadId: entry.threadId,
+ snapshotSequence: 1,
+ attachments: true,
+ })),
+ });
+ const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("acked");
+ // thread:a is still current, so only the failed thread is fetched again.
+ expect(retry.fetched).toEqual(["thread:b"]);
+ expect(retry.written.map((entry) => entry.threadId)).toEqual(["thread:b"]);
+ // The current copy is promoted into the generation the importer reads.
+ expect(retry.promoted).toEqual([
+ {
+ fromDrainId: "drain:environment:worker:1",
+ toDrainId: "drain:environment:worker:2",
+ threadId: "thread:a",
+ },
+ ]);
+ }),
+ );
+
+ it.effect("refuses an ack when the worker starts work during the drain", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a"], liveAfter: "thread:a" });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.warnings?.join(" ")).toContain("changed during the drain");
+ }),
+ );
+
+ it.effect("refuses an ack when a thread appears during the drain", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a"], threadAfter: "thread:new" });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.threads.acked).toBe(1);
+ }),
+ );
+
+ it.effect("refuses an ack when a copied thread finishes a turn during the drain", () =>
+ Effect.gen(function* () {
+ const retry = deps({
+ threads: ["thread:a"],
+ sequenceAfterCopy: "thread:a",
+ seeded: [
+ {
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ snapshotSequence: 1,
+ attachments: true,
+ },
+ ],
+ });
+ const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.warnings?.join(" ")).toContain("changed during the drain");
+ }),
+ );
+
+ it.effect("re-copies a current thread when the previous copy cannot be promoted", () =>
+ Effect.gen(function* () {
+ const retry = deps({
+ threads: ["thread:a"],
+ failCopy: true,
+ seeded: [
+ {
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ snapshotSequence: 1,
+ attachments: true,
+ },
+ ],
+ });
+ const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("acked");
+ expect(retry.promoted).toEqual([]);
+ expect(retry.fetched).toEqual(["thread:a"]);
+ }),
+ );
+
+ it.effect("re-copies when a previous generation changes during promotion", () =>
+ Effect.gen(function* () {
+ const retry = deps({
+ threads: ["thread:a"],
+ seeded: [
+ {
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ snapshotSequence: 1,
+ attachments: true,
+ },
+ ],
+ });
+ const outcome = yield* drain(
+ {
+ ...retry,
+ sink: {
+ ...retry.sink,
+ copy: (input) =>
+ Effect.gen(function* () {
+ yield* retry.sink.write({
+ environmentId: input.environmentId,
+ drainId: input.toDrainId,
+ threadId: input.threadId,
+ body: '{"older":true}',
+ snapshotSequence: 0,
+ });
+ return true;
+ }),
+ },
+ },
+ { drainId: "drain:environment:worker:2" },
+ );
+ expect(outcome.state).toBe("acked");
+ expect(retry.fetched).toEqual(["thread:a"]);
+ expect(
+ (yield* retry.sink.read({
+ environmentId,
+ drainId: "drain:environment:worker:2",
+ threadId: "thread:a",
+ }))?.snapshotSequence,
+ ).toBe(1);
+ }),
+ );
+
+ it.effect("re-copies a thread whose worker state changed since the failed drain", () =>
+ Effect.gen(function* () {
+ const first = deps({ threads: ["thread:a", "thread:b"], failThread: "thread:b" });
+ yield* drain(first, { drainId: "drain:environment:worker:1" });
+ const retry = deps({
+ threads: ["thread:a", "thread:b"],
+ sequences: { "thread:a": 2 },
+ seeded: first.written.map((entry) => ({
+ drainId: "drain:environment:worker:1",
+ threadId: entry.threadId,
+ snapshotSequence: 1,
+ attachments: true,
+ })),
+ });
+ const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("acked");
+ expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 });
+ // The changed thread is fetched and written again under the new generation.
+ expect(retry.fetched).toEqual(["thread:a", "thread:b"]);
+ expect(retry.written.map((entry) => entry.threadId)).toEqual(["thread:a", "thread:b"]);
+ }),
+ );
+
+ it.effect("never trusts copies left under its own drain id or with a stale marker", () =>
+ Effect.gen(function* () {
+ // A timed-out run leaves a stale copy under the previous generation and a
+ // fresh-looking one under the id this retry runs under; neither may count.
+ const dependencies = deps({
+ threads: ["thread:a"],
+ sequences: { "thread:a": 7 },
+ seeded: [
+ {
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ snapshotSequence: 3,
+ attachments: true,
+ },
+ {
+ drainId: "drain:environment:worker:2",
+ threadId: "thread:a",
+ snapshotSequence: 7,
+ attachments: true,
+ },
+ ],
+ });
+ const outcome = yield* drain(dependencies, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("acked");
+ expect(dependencies.fetched).toEqual(["thread:a"]);
+ expect(dependencies.written.map((entry) => entry.drainId)).toEqual([
+ "drain:environment:worker:2",
+ ]);
+ }),
+ );
+
+ it.effect("a timed-out generation still writing cannot corrupt the retry", () =>
+ Effect.gen(function* () {
+ const base = deps({
+ threads: ["thread:a"],
+ sequences: { "thread:a": 7 },
+ seeded: [
+ {
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ snapshotSequence: 3,
+ attachments: true,
+ },
+ ],
+ });
+ const sinkWrite = base.sink.write;
+ // While generation 2 runs, the abandoned generation 1 finishes its own
+ // (newer) copy into generation 1's directory.
+ const lateWriter = {
+ ...base.sink,
+ write: (input: Parameters[0]) =>
+ sinkWrite(input).pipe(
+ Effect.tap(() =>
+ sinkWrite({ ...input, drainId: "drain:environment:worker:1", body: '{"late":true}' }),
+ ),
+ ),
+ };
+ const outcome = yield* drain(
+ { ...base, sink: lateWriter },
+ {
+ drainId: "drain:environment:worker:2",
+ },
+ );
+
+ // The stale generation 1 copy was re-fetched, written under generation 2's
+ // own id, and verified against that copy — the late generation 1 write,
+ // which lands in a directory the retry never reads, cannot satisfy it.
+ expect(outcome.state).toBe("acked");
+ expect(base.written.map((entry) => [entry.drainId, entry.threadId])).toEqual([
+ ["drain:environment:worker:2", "thread:a"],
+ ["drain:environment:worker:1", "thread:a"],
+ ]);
+ }),
+ );
+
+ it.effect("keeps the sandbox when a thread cannot be copied", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a", "thread:b"], failThread: "thread:b" });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.threads).toEqual({ total: 2, acked: 1, failed: 1 });
+ }),
+ );
+
+ it.effect("refuses to drain a worker whose child is still working", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a"], liveThread: "thread:a" });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.finishedAt).toBeNull();
+ expect(outcome.threads.failed).toBe(1);
+ expect(dependencies.written).toHaveLength(0);
+ }),
+ );
+
+ it.effect("returns failed with no acks when the worker's shell cannot be read", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({
+ threads: ["thread:a"],
+ projects: ["/work/a"],
+ failShell: true,
+ });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.finishedAt).toBeNull();
+ expect(outcome.threads).toEqual({ total: 0, acked: 0, failed: 0 });
+ expect(outcome.worktrees).toEqual({ total: 0, acked: 0, failed: 0 });
+ expect(dependencies.written).toHaveLength(0);
+ expect(dependencies.exported).toHaveLength(0);
+ }),
+ );
+
+ it.effect("acks a checkout with nothing the mirror lacks without storing a bundle", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({
+ threads: [],
+ projects: ["/work/a"],
+ bundleStdout: "",
+ });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("acked");
+ expect(outcome.worktrees).toEqual({ total: 1, acked: 1, failed: 0 });
+ expect(dependencies.written).toHaveLength(0);
+ }),
+ );
+
+ it.effect("keeps the sandbox when a checkout bundle was not created", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({
+ threads: ["thread:a"],
+ projects: ["/work/a"],
+ bundleExitCode: 1,
+ });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.worktrees).toEqual({ total: 1, acked: 0, failed: 1 });
+ }),
+ );
+
+ it.effect("stops at the deadline instead of claiming a partial drain", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a", "thread:b"], now: 5_000 });
+ const outcome = yield* drain(dependencies, { deadlineAt: 1_000 });
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.threads).toEqual({ total: 0, acked: 0, failed: 0 });
+ expect(dependencies.written).toHaveLength(0);
+ }),
+ );
+
+ it.effect("copies threads but skips checkout bundles for a forced removal", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a"], projects: ["/work/a"] });
+ const outcome = yield* drain(dependencies, { snapshotOnly: true });
+
+ expect(outcome.state).toBe("acked");
+ expect(outcome.threads.acked).toBe(1);
+ expect(dependencies.exported).toHaveLength(0);
+ }),
+ );
+
+ it.effect("drains attachments with the thread and verifies the manifest", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({
+ threads: ["thread:a"],
+ attachmentFiles: { "thread:a": ["shot.png", "log.txt"] },
+ });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("acked");
+ expect(dependencies.fetchedAttachments).toEqual(["thread:a"]);
+ expect(dependencies.written).toHaveLength(1);
+ }),
+ );
+
+ it.effect("notes an oversize skip in the outcome without failing the drain", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({ threads: ["thread:a"], oversizeThreads: ["thread:a"] });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("acked");
+ expect(outcome.warnings).toEqual([
+ "Attachments of thread:a exceeded the drain size cap and were skipped.",
+ ]);
+ }),
+ );
+
+ it.effect("re-copies a thread whose previous copy predates attachment draining", () =>
+ Effect.gen(function* () {
+ const first = deps({ threads: ["thread:a"] });
+ yield* drain(first, { drainId: "drain:environment:worker:1" });
+ const retry = deps({
+ threads: ["thread:a"],
+ attachmentFiles: { "thread:a": ["late.bin"] },
+ // The old copy carries no attachment manifest.
+ seeded: first.written.map((entry) => ({
+ drainId: "drain:environment:worker:1",
+ threadId: entry.threadId,
+ snapshotSequence: 1,
+ })),
+ });
+ const outcome = yield* drain(retry, { drainId: "drain:environment:worker:2" });
+
+ expect(outcome.state).toBe("acked");
+ expect(retry.fetched).toEqual(["thread:a"]);
+ }),
+ );
+
+ it.effect("fails the copy when the stored manifest does not match what was sent", () =>
+ Effect.gen(function* () {
+ const dependencies = deps({
+ threads: ["thread:a"],
+ attachmentFiles: { "thread:a": ["shot.png"] },
+ dropAttachmentManifest: true,
+ });
+ const outcome = yield* drain(dependencies);
+
+ expect(outcome.state).toBe("failed");
+ expect(outcome.threads).toEqual({ total: 1, acked: 0, failed: 1 });
+ }),
+ );
+});
diff --git a/apps/server/src/environment/OpenShellFleetDrain.ts b/apps/server/src/environment/OpenShellFleetDrain.ts
new file mode 100644
index 0000000000000..5c23c1075a0a8
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetDrain.ts
@@ -0,0 +1,448 @@
+import type { EnvironmentId } from "@t3tools/contracts";
+import {
+ type FleetDrainExecutor as FleetDrainExecutorPort,
+ type FleetDrainOutcome,
+} from "@t3tools/openshell/OpenShellGatewayDriver";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+/**
+ * Nothing is deleted until its data is here (spec section 7): per-thread copies
+ * read back for verification, checkout bundle exports, per-item acks keyed by
+ * drain id.
+ */
+export class FleetDrainStepError extends Schema.TaggedError<
+ FleetDrainStepError,
+ { readonly brand: unique symbol }
+>()("FleetDrainStepError", {
+ step: Schema.Literals(["shell", "thread", "history", "write", "read", "bundle", "attachments"]),
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+export class FleetDrainError extends Schema.TaggedError<
+ FleetDrainError,
+ { readonly brand: unique symbol }
+>()("FleetDrainError", {
+ environmentId: Schema.String,
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+/** One stored thread copy plus the currency marker that decides trust on a retry. */
+export interface DrainSinkRecord {
+ readonly threadId: string;
+ readonly snapshotSequence: number | null;
+ readonly body: string;
+ /** Manifest of the attachment files stored next to the bundle, if any. */
+ readonly attachments?: ReadonlyArray;
+ /** True when the worker's attachments exceeded the drain's size cap. */
+ readonly attachmentsSkipped?: boolean;
+}
+
+/** One attachment file in a bundle, base64-encoded. */
+export interface DrainAttachmentFile {
+ readonly name: string;
+ readonly bytes: string;
+}
+
+export interface DrainAttachmentManifestEntry {
+ readonly name: string;
+ readonly hash: string;
+ readonly size: number;
+}
+
+/** A thread's attachment files as the worker holds them, plus cap outcome. */
+export interface DrainAttachments {
+ readonly files: ReadonlyArray;
+ readonly oversizeSkipped: boolean;
+}
+
+/** Where a drain writes: one directory per run, one file per thread. */
+export interface DrainSink {
+ readonly write: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly drainId: string;
+ readonly threadId: string;
+ readonly body: string;
+ readonly snapshotSequence: number | null;
+ readonly attachments?: DrainAttachments;
+ }) => Effect.Effect<{ readonly bytes: number }, FleetDrainStepError>;
+ readonly read: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly drainId: string;
+ readonly threadId: string;
+ }) => Effect.Effect;
+ /** Copies a previous generation's record into this one; false when it is gone. */
+ readonly copy: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly fromDrainId: string;
+ readonly toDrainId: string;
+ readonly threadId: string;
+ }) => Effect.Effect;
+}
+
+/** Exports one checkout and its checkpoint refs as base64 bundle bytes on exec
+ * stdout; empty stdout means origin already holds every selected commit. */
+export interface DrainGitBundle {
+ readonly bundle: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly sandboxName: string;
+ readonly workspaceRoot: string;
+ }) => Effect.Effect<
+ { readonly exitCode: number; readonly stdout: string; readonly stderr: string },
+ FleetDrainStepError
+ >;
+}
+
+/** The worker's own view of what it holds, read over its authenticated link. */
+export interface DrainSource {
+ readonly shell: (environmentId: EnvironmentId) => Effect.Effect<
+ {
+ readonly threads: ReadonlyArray<{ readonly threadId: string; readonly live: boolean }>;
+ readonly projects: ReadonlyArray<{ readonly workspaceRoot: string }>;
+ },
+ FleetDrainStepError
+ >;
+ /** Cheap per-thread currency probe: the bounded read's snapshot sequence. */
+ readonly threadSequence: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly threadId: string;
+ }) => Effect.Effect<{ readonly snapshotSequence: number | null }, FleetDrainStepError>;
+ readonly threadHistory: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly threadId: string;
+ }) => Effect.Effect<
+ { readonly body: string; readonly snapshotSequence: number | null },
+ FleetDrainStepError
+ >;
+ /** The thread's attachment files; absent, the source cannot serve them. */
+ readonly attachments?: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly sandboxName: string;
+ readonly threadId: string;
+ }) => Effect.Effect;
+}
+
+export interface DrainDependencies {
+ readonly source: DrainSource;
+ readonly sink: DrainSink;
+ readonly git: DrainGitBundle;
+ readonly now: Effect.Effect;
+}
+
+export interface DrainInput {
+ readonly environmentId: EnvironmentId;
+ readonly sandboxName: string;
+ readonly drainId: string;
+ readonly snapshotOnly: boolean;
+ readonly deadlineAt: number;
+}
+
+export interface DrainExecutorShape {
+ readonly drain: (input: DrainInput) => Effect.Effect;
+}
+
+const zero = () => ({ total: 0, acked: 0, failed: 0 });
+
+const reported = (outcome: FleetDrainOutcome): FleetDrainOutcome => outcome;
+
+/** The immediately preceding generation's id, or null for the first drain. */
+const previousDrainId = (drainId: string): string | null => {
+ const marker = drainId.lastIndexOf(":");
+ if (marker < 0) return null;
+ const generation = Number.parseInt(drainId.slice(marker + 1), 10);
+ if (!Number.isFinite(generation) || generation <= 1) return null;
+ return `${drainId.slice(0, marker)}:${generation - 1}`;
+};
+
+/**
+ * Drains one worker. Every step is keyed by `drainId` and per item, so a retry
+ * after a crash re-reads what already landed instead of fetching it again.
+ */
+export const makeDrainExecutor = (deps: DrainDependencies): DrainExecutorShape => {
+ const drain = Effect.fn("FleetDrainExecutor.drain")(function* (input: DrainInput) {
+ const threads = zero();
+ const worktrees = zero();
+ const warnings: string[] = [];
+ /** Sequence each copied thread was read at, re-checked before the ack. */
+ const copiedSequences = new Map();
+ const copiedWorkspaceRoots = new Set();
+ let deadlineHit = false;
+ let shellFailed = false;
+ const shell = yield* deps.source.shell(input.environmentId).pipe(
+ Effect.catch(() =>
+ Effect.sync(() => {
+ shellFailed = true;
+ return { threads: [], projects: [] } as {
+ threads: ReadonlyArray<{ threadId: string; live: boolean }>;
+ projects: ReadonlyArray<{ workspaceRoot: string }>;
+ };
+ }),
+ ),
+ );
+ if (shellFailed) {
+ // Nothing could be listed, so nothing can be acked: the sandbox is kept.
+ return reported({
+ threads: zero(),
+ worktrees: zero(),
+ state: "failed",
+ finishedAt: null,
+ });
+ }
+ // A child that is still working writes after this snapshot, so copying now
+ // would ack a stale thread and then delete the sandbox under it.
+ const stillWorking = shell.threads.filter((thread) => thread.live);
+ if (stillWorking.length > 0) {
+ return reported({
+ threads: { total: stillWorking.length, acked: 0, failed: stillWorking.length },
+ worktrees: zero(),
+ state: "failed",
+ finishedAt: null,
+ });
+ }
+ for (const thread of shell.threads) {
+ if ((yield* deps.now) >= input.deadlineAt) {
+ deadlineHit = true;
+ break;
+ }
+ threads.total += 1;
+ const copied = yield* Effect.gen(function* () {
+ // A copy from the previous generation is trusted only while its currency
+ // marker still matches the live worker; otherwise the thread is re-copied.
+ const previous = previousDrainId(input.drainId);
+ const prior =
+ previous === null
+ ? null
+ : yield* deps.sink.read({
+ environmentId: input.environmentId,
+ drainId: previous,
+ threadId: thread.threadId,
+ });
+ if (prior !== null && previous !== null && prior.snapshotSequence !== null) {
+ const current = yield* deps.source.threadSequence({
+ environmentId: input.environmentId,
+ threadId: thread.threadId,
+ });
+ // A copy from before attachments were drained carries no manifest, so
+ // it is re-copied once to pick the attachment files up.
+ if (
+ current.snapshotSequence === prior.snapshotSequence &&
+ prior.attachments !== undefined
+ ) {
+ // The importer reads this generation's directory only, so the
+ // verified copy is promoted instead of merely acknowledged.
+ const promoted = yield* deps.sink
+ .copy({
+ environmentId: input.environmentId,
+ fromDrainId: previous,
+ toDrainId: input.drainId,
+ threadId: thread.threadId,
+ })
+ .pipe(Effect.orElseSucceed(() => false));
+ if (promoted) {
+ const back = yield* deps.sink.read({
+ environmentId: input.environmentId,
+ drainId: input.drainId,
+ threadId: thread.threadId,
+ });
+ if (
+ back !== null &&
+ back.body === prior.body &&
+ back.snapshotSequence === prior.snapshotSequence &&
+ back.attachments?.length === prior.attachments.length &&
+ back.attachments.every((entry, index) => {
+ const expected = prior.attachments?.[index];
+ return (
+ entry.name === expected?.name &&
+ entry.hash === expected.hash &&
+ entry.size === expected.size
+ );
+ }) &&
+ back.attachmentsSkipped === prior.attachmentsSkipped
+ ) {
+ copiedSequences.set(thread.threadId, prior.snapshotSequence);
+ return true;
+ }
+ }
+ }
+ }
+ const history = yield* deps.source.threadHistory({
+ environmentId: input.environmentId,
+ threadId: thread.threadId,
+ });
+ const attachments =
+ deps.source.attachments === undefined
+ ? ({ files: [], oversizeSkipped: false } as const)
+ : yield* deps.source.attachments({
+ environmentId: input.environmentId,
+ sandboxName: input.sandboxName,
+ threadId: thread.threadId,
+ });
+ if (attachments.oversizeSkipped) {
+ warnings.push(
+ `Attachments of ${thread.threadId} exceeded the drain size cap and were skipped.`,
+ );
+ }
+ const written = yield* deps.sink.write({
+ environmentId: input.environmentId,
+ drainId: input.drainId,
+ threadId: thread.threadId,
+ body: history.body,
+ snapshotSequence: history.snapshotSequence,
+ ...(attachments.files.length === 0 && !attachments.oversizeSkipped
+ ? {}
+ : { attachments }),
+ });
+ // Read it back: an ack means the bytes are here, not that a write was attempted.
+ copiedSequences.set(thread.threadId, history.snapshotSequence);
+ const back = yield* deps.sink.read({
+ environmentId: input.environmentId,
+ drainId: input.drainId,
+ threadId: thread.threadId,
+ });
+ if (
+ back === null ||
+ back.body !== history.body ||
+ written.bytes <= 0 ||
+ (back.attachmentsSkipped === true) !== attachments.oversizeSkipped
+ )
+ return false;
+ const sent = attachments.files.map((file) => file.name).sort();
+ const stored = (back.attachments ?? []).map((entry) => entry.name).sort();
+ return sent.length === stored.length && sent.every((name, index) => name === stored[index]);
+ }).pipe(Effect.orElseSucceed(() => false));
+ if (copied) threads.acked += 1;
+ else threads.failed += 1;
+ }
+ // A forced removal still ships the threads, which are the part that cannot be rebuilt.
+ if (!input.snapshotOnly) {
+ for (const project of shell.projects) {
+ if ((yield* deps.now) >= input.deadlineAt) {
+ deadlineHit = true;
+ break;
+ }
+ worktrees.total += 1;
+ // One bundle per checkout, keyed by its root, lands beside the thread
+ // copies and is verified the same way: an ack means the bytes are here.
+ // An export of nothing (origin has every commit) has no bundle to store.
+ const worktreeId = `worktree:${project.workspaceRoot}`;
+ copiedWorkspaceRoots.add(project.workspaceRoot);
+ const copied = yield* Effect.gen(function* () {
+ const exported = yield* deps.git.bundle({
+ environmentId: input.environmentId,
+ sandboxName: input.sandboxName,
+ workspaceRoot: project.workspaceRoot,
+ });
+ if (exported.exitCode !== 0)
+ return yield* new FleetDrainStepError({
+ step: "bundle",
+ message: exported.stderr.trim() || "The checkout bundle was not created.",
+ });
+ // Nothing the mirror lacks: git refuses an empty bundle, and an ack
+ // needs no bytes here.
+ if (exported.stdout === "") return true;
+ const written = yield* deps.sink.write({
+ environmentId: input.environmentId,
+ drainId: input.drainId,
+ threadId: worktreeId,
+ body: exported.stdout,
+ snapshotSequence: null,
+ });
+ const back = yield* deps.sink.read({
+ environmentId: input.environmentId,
+ drainId: input.drainId,
+ threadId: worktreeId,
+ });
+ return back !== null && back.body === exported.stdout && written.bytes > 0;
+ }).pipe(Effect.orElseSucceed(() => false));
+ if (copied) worktrees.acked += 1;
+ else worktrees.failed += 1;
+ }
+ }
+ // Work that started after the enumeration was never copied. The manager's
+ // `draining` row does not reach the worker, so the ack is re-earned here:
+ // a live thread, a changed thread set, or a thread that finished a turn
+ // after it was copied keeps the sandbox.
+ const after = yield* deps.source
+ .shell(input.environmentId)
+ .pipe(Effect.orElseSucceed(() => null));
+ let moved = false;
+ if (after === null) {
+ moved = true;
+ } else {
+ moved =
+ after.threads.some((thread) => thread.live) ||
+ after.threads.length !== shell.threads.length ||
+ after.threads.some(
+ (thread) => !shell.threads.some((before) => before.threadId === thread.threadId),
+ );
+ if (!moved && !input.snapshotOnly) {
+ moved =
+ after.projects.length !== copiedWorkspaceRoots.size ||
+ after.projects.some((project) => !copiedWorkspaceRoots.has(project.workspaceRoot));
+ }
+ }
+ if (!moved) {
+ for (const [threadId, sequence] of copiedSequences) {
+ if (moved) break;
+ const current = yield* deps.source
+ .threadSequence({ environmentId: input.environmentId, threadId })
+ .pipe(Effect.orElseSucceed(() => null));
+ // A failed currency probe cannot establish that the copy is still current.
+ // A missing sequence either at copy time or now means currency is not
+ // proven; null equality is not a match.
+ if (
+ sequence === null ||
+ current === null ||
+ current.snapshotSequence === null ||
+ current.snapshotSequence !== sequence
+ )
+ moved = true;
+ }
+ }
+ if (moved)
+ warnings.push(
+ "The worker changed during the drain; its state was copied, nothing was acknowledged.",
+ );
+ // A deadline that has arrived—even after the last item succeeded—is never
+ // an acknowledgement: there is no budget left to prove currency again.
+ if ((yield* deps.now) >= input.deadlineAt) deadlineHit = true;
+ const complete = !moved && !deadlineHit && threads.failed === 0 && worktrees.failed === 0;
+ return reported({
+ threads,
+ worktrees,
+ state: complete ? "acked" : "failed",
+ finishedAt: complete ? yield* deps.now : null,
+ ...(warnings.length === 0 ? {} : { warnings }),
+ });
+ });
+
+ return { drain };
+};
+
+/** The OpenShell driver calls the drain Promise-shaped, during removal. */
+export const asDriverPort = (
+ executor: DrainExecutorShape,
+ runPromise: (effect: Effect.Effect) => Promise,
+): FleetDrainExecutorPort => ({
+ drain: (input) =>
+ runPromise(
+ executor
+ .drain({
+ environmentId: input.environmentId,
+ sandboxName: input.sandboxName,
+ drainId: input.drainId,
+ snapshotOnly: input.snapshotOnly,
+ deadlineAt: input.deadlineAt,
+ })
+ .pipe(
+ Effect.orElseSucceed((): FleetDrainOutcome => ({
+ threads: zero(),
+ worktrees: zero(),
+ state: "failed",
+ finishedAt: null,
+ })),
+ ),
+ ),
+});
diff --git a/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts b/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts
new file mode 100644
index 0000000000000..63f4c9890139e
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetDrainAdapters.test.ts
@@ -0,0 +1,803 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { EnvironmentId } from "@t3tools/contracts";
+import { describe, expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Path from "effect/Path";
+import { type PlatformError } from "effect/PlatformError";
+import * as HttpClient from "effect/http/HttpClient";
+import * as HttpClientResponse from "effect/http/HttpClientResponse";
+
+import { FleetDrainStepError, type DrainSink } from "./OpenShellFleetDrain.ts";
+import * as OpenShellFleetDrain from "./OpenShellFleetDrain.ts";
+import * as OpenShellFleetDrainAdapters from "./OpenShellFleetDrainAdapters.ts";
+
+type SinkUse = Effect.Effect<
+ A,
+ FleetDrainStepError | PlatformError,
+ FileSystem.FileSystem | Path.Path
+>;
+
+const refused = (
+ effect: Effect.Effect,
+): Effect.Effect =>
+ effect.pipe(
+ Effect.flip,
+ Effect.catch(() => Effect.die(new Error("expected a refusal"))),
+ );
+
+const withSink = (use: (sink: DrainSink, stateDir: string) => SinkUse) =>
+ Effect.acquireUseRelease(
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const stateDir = yield* fs.makeTempDirectory();
+ const sink = yield* OpenShellFleetDrainAdapters.fileDrainSink({ stateDir, fs, path });
+ return { fs, path, stateDir, sink };
+ }),
+ ({ sink, stateDir }) => use(sink, stateDir),
+ ({ fs, stateDir }) => fs.remove(stateDir, { recursive: true }).pipe(Effect.ignore),
+ ).pipe(Effect.provide(NodeServices.layer));
+
+const base64Of = (text: string) => Buffer.from(text, "utf8").toString("base64");
+
+const write = (
+ sink: DrainSink,
+ overrides: Partial<{
+ environmentId: EnvironmentId;
+ drainId: string;
+ threadId: string;
+ body: string;
+ }> = {},
+) =>
+ sink.write({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ body: '{"events":[1]}',
+ snapshotSequence: 4,
+ ...overrides,
+ });
+
+describe("fileDrainSink", () => {
+ it.effect("refuses an environment id that climbs out of fleet-drain", () =>
+ withSink((sink) =>
+ Effect.gen(function* () {
+ const failure = yield* refused(write(sink, { environmentId: EnvironmentId.make("..") }));
+ expect(failure.step).toBe("write");
+ const readFailure = yield* refused(
+ sink.read({
+ environmentId: EnvironmentId.make(".."),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ }),
+ );
+ expect(readFailure.step).toBe("read");
+ }),
+ ),
+ );
+
+ it.effect("refuses a drain id that climbs out of fleet-drain", () =>
+ withSink((sink) =>
+ Effect.gen(function* () {
+ const failure = yield* refused(write(sink, { drainId: ".." }));
+ expect(failure.step).toBe("write");
+ const readFailure = yield* refused(
+ sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "..",
+ threadId: "thread:a",
+ }),
+ );
+ expect(readFailure.step).toBe("read");
+ }),
+ ),
+ );
+
+ it.effect("neutralizes separators and stores a currency-checked envelope", () =>
+ withSink((sink, stateDir) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const written = yield* write(sink, {
+ environmentId: EnvironmentId.make("env/../../elsewhere"),
+ });
+ expect(written.bytes).toBeGreaterThan(0);
+ // The segment cannot name a directory above fleet-drain.
+ expect(yield* fs.exists(path.join(stateDir, "elsewhere"))).toBe(false);
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("env/../../elsewhere"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ });
+ expect(record).toEqual({
+ threadId: "thread:a",
+ snapshotSequence: 4,
+ body: '{"events":[1]}',
+ });
+ }),
+ ),
+ );
+
+ it.effect("maps distinct thread ids to distinct files and refuses relative hops", () =>
+ withSink((sink, stateDir) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const drainId = "drain:environment:worker:1";
+ const environmentId = EnvironmentId.make("environment:worker");
+ const bodies: Record = {
+ "thread:a": '{"n":1}',
+ threada: '{"n":2}',
+ "thread/b": '{"n":3}',
+ "thread.b": '{"n":4}',
+ };
+ for (const [threadId, body] of Object.entries(bodies)) {
+ yield* sink.write({ environmentId, drainId, threadId, body, snapshotSequence: null });
+ }
+ // A collision would have overwritten an acked copy with another thread's.
+ for (const [threadId, body] of Object.entries(bodies)) {
+ const record = yield* sink.read({ environmentId, drainId, threadId });
+ expect(record?.body).toBe(body);
+ }
+ const dir = path.join(
+ stateDir,
+ "fleet-drain",
+ "environment%3Aworker",
+ "drain%3Aenvironment%3Aworker%3A1",
+ );
+ expect(yield* fs.readDirectory(dir)).toHaveLength(4);
+ for (const hop of ["..", "."]) {
+ const failure = yield* refused(write(sink, { drainId, threadId: hop, body: "{}" }));
+ expect(failure.step).toBe("write");
+ }
+ }),
+ ),
+ );
+
+ it.effect("answers null for a thread that was never written", () =>
+ withSink((sink) =>
+ Effect.gen(function* () {
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:missing",
+ });
+ expect(record).toBeNull();
+ }),
+ ),
+ );
+});
+
+describe("fileDrainSink copy", () => {
+ it.effect("promotes a previous generation's record with its attachment files", () =>
+ withSink((sink, stateDir) =>
+ Effect.gen(function* () {
+ const environmentId = EnvironmentId.make("environment:worker");
+ yield* sink.write({
+ environmentId,
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ body: '{"events":[1]}',
+ snapshotSequence: 2,
+ attachments: {
+ files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }],
+ oversizeSkipped: false,
+ },
+ });
+
+ const copied = yield* sink.copy({
+ environmentId,
+ fromDrainId: "drain:environment:worker:1",
+ toDrainId: "drain:environment:worker:2",
+ threadId: "thread:a",
+ });
+
+ expect(copied).toBe(true);
+ const record = yield* sink.read({
+ environmentId,
+ drainId: "drain:environment:worker:2",
+ threadId: "thread:a",
+ });
+ expect(record?.body).toBe('{"events":[1]}');
+ expect(record?.snapshotSequence).toBe(2);
+ expect(record?.attachments?.[0]?.name).toBe("attach-1.png");
+
+ const missing = yield* sink.copy({
+ environmentId,
+ fromDrainId: "drain:environment:worker:1",
+ toDrainId: "drain:environment:worker:2",
+ threadId: "thread:missing",
+ });
+ expect(missing).toBe(false);
+
+ // A manifest whose bytes are gone must not look like a promoted copy.
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const dir = path.join(
+ stateDir,
+ "fleet-drain",
+ "environment%3Aworker",
+ "drain%3Aenvironment%3Aworker%3A1",
+ );
+ for (const file of yield* fs.readDirectory(dir)) {
+ if (file.endsWith(".bin")) yield* fs.remove(path.join(dir, file));
+ }
+ const broken = yield* refused(
+ sink.copy({
+ environmentId,
+ fromDrainId: "drain:environment:worker:1",
+ toDrainId: "drain:environment:worker:3",
+ threadId: "thread:a",
+ }),
+ );
+ expect(broken.step).toBe("write");
+ }),
+ ),
+ );
+});
+
+describe("fileDrainSink attachments", () => {
+ it.effect("stores attachment files beside the bundle and manifests them", () =>
+ withSink((sink, stateDir) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ yield* sink.write({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ body: '{"events":[1]}',
+ snapshotSequence: 2,
+ attachments: {
+ files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }],
+ oversizeSkipped: false,
+ },
+ });
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ });
+ expect(record?.attachments).toHaveLength(1);
+ const entry = record?.attachments?.[0];
+ expect(entry?.name).toBe("attach-1.png");
+ expect(entry?.size).toBe("image-bytes".length);
+ expect(entry?.hash).toMatch(/^[0-9a-f]{64}$/);
+ // The sibling file holds the exact bytes, keyed by the same hash.
+ const dir = path.join(
+ stateDir,
+ "fleet-drain",
+ "environment%3Aworker",
+ "drain%3Aenvironment%3Aworker%3A1",
+ );
+ const stored = yield* fs.readFile(
+ path.join(dir, `thread-thread%3Aa-att-${entry?.hash.slice(0, 16)}.bin`),
+ );
+ expect(new TextDecoder().decode(stored)).toBe("image-bytes");
+ expect(record?.attachmentsSkipped).toBeUndefined();
+ }),
+ ),
+ );
+
+ it.effect("refuses a copy whose attachment bytes changed on disk", () =>
+ withSink((sink, stateDir) =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ yield* sink.write({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ body: '{"events":[1]}',
+ snapshotSequence: 2,
+ attachments: {
+ files: [{ name: "attach-1.png", bytes: base64Of("image-bytes") }],
+ oversizeSkipped: false,
+ },
+ });
+ const dir = path.join(
+ stateDir,
+ "fleet-drain",
+ "environment%3Aworker",
+ "drain%3Aenvironment%3Aworker%3A1",
+ );
+ const stored = (yield* fs.readDirectory(dir)).filter((name) => name.endsWith(".bin"));
+ yield* fs.writeFile(path.join(dir, stored[0]!), new TextEncoder().encode("tampered"));
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ });
+ expect(record).toBeNull();
+ }),
+ ),
+ );
+
+ it.effect("records an oversize skip without manifest entries", () =>
+ withSink((sink) =>
+ Effect.gen(function* () {
+ yield* sink.write({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ body: '{"events":[1]}',
+ snapshotSequence: 2,
+ attachments: { files: [], oversizeSkipped: true },
+ });
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ });
+ expect(record?.attachments).toBeUndefined();
+ expect(record?.attachmentsSkipped).toBe(true);
+ }),
+ ),
+ );
+
+ it.effect("a plain bundle still reads back without manifest fields", () =>
+ withSink((sink) =>
+ Effect.gen(function* () {
+ yield* write(sink);
+ const record = yield* sink.read({
+ environmentId: EnvironmentId.make("environment:worker"),
+ drainId: "drain:environment:worker:1",
+ threadId: "thread:a",
+ });
+ expect(record?.attachments).toBeUndefined();
+ expect(record?.attachmentsSkipped).toBeUndefined();
+ }),
+ ),
+ );
+});
+
+describe("execDrainGitBundle", () => {
+ const execStub = (stdout: string, commands?: Array>, exitCode = 0) => ({
+ exec: (_sandboxName: string, command: ReadonlyArray) =>
+ Effect.sync(() => {
+ commands?.push(command);
+ return { exitCode, stdout, stderr: "" };
+ }),
+ });
+
+ it.effect("bundles only the refs the mirror lacks, without contacting a remote", () =>
+ Effect.gen(function* () {
+ const commands: Array> = [];
+ const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({
+ ...execStub("Qg==", commands),
+ });
+ const result = yield* adapter.bundle({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ workspaceRoot: "/work/a",
+ });
+ expect(result.exitCode).toBe(0);
+ const line = commands[0]?.join(" ") ?? "";
+ // No push: a worker's origin is a read-only mirror, and a branch that
+ // fell behind would reject a push as non-fast-forward anyway. --all
+ // would re-export the seeded history, so only what origin lacks travels.
+ expect(line).toContain("rev-list --count");
+ expect(line).toContain("bundle create");
+ expect(line).toContain("--branches");
+ expect(line).toContain("HEAD --branches");
+ expect(line).toContain("--glob='refs/t3/*'");
+ expect(line).toContain("--not --remotes=origin");
+ expect(line).not.toContain("--all");
+ expect(line).not.toContain("--not --remotes ");
+ expect(line).not.toContain("push");
+ expect(line).toContain("'/work/a'");
+ }),
+ );
+
+ it.effect("passes an empty export through for the drain to acknowledge", () =>
+ Effect.gen(function* () {
+ const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({
+ ...execStub(""),
+ });
+ const result = yield* adapter.bundle({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ workspaceRoot: "/work/a",
+ });
+ expect(result.exitCode).toBe(0);
+ expect(result.stdout).toBe("");
+ }),
+ );
+
+ it.effect("returns base64 for a real git bundle stream", () =>
+ Effect.gen(function* () {
+ const bundle = "# v2 git bundle\nrefnames...";
+ const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({
+ ...execStub(Buffer.from(bundle, "utf8").toString("base64")),
+ });
+ const result = yield* adapter.bundle({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ workspaceRoot: "/work/a",
+ });
+ expect(result.exitCode).toBe(0);
+ expect(Buffer.from(result.stdout, "base64").toString("utf8")).toBe(bundle);
+ expect(Buffer.from(result.stdout, "base64").toString("utf8")).toMatch(/^# v2 git bundle/);
+ }),
+ );
+
+ it.effect("refuses a bundle beyond the size cap before base64 runs", () =>
+ Effect.gen(function* () {
+ const commands: Array> = [];
+ const adapter = OpenShellFleetDrainAdapters.execDrainGitBundle({
+ ...execStub("", commands, 4),
+ maxBytes: 8,
+ });
+ const outcome = yield* adapter
+ .bundle({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ workspaceRoot: "/work/a",
+ })
+ .pipe(Effect.flip);
+ expect(outcome.step).toBe("bundle");
+ expect(outcome.message).toContain("size cap");
+ const line = commands[0]?.join(" ") ?? "";
+ expect(line).toContain("-gt 8");
+ expect(line.indexOf("wc -c")).toBeLessThan(line.indexOf("base64"));
+ }),
+ );
+});
+
+describe("execDrainAttachments", () => {
+ const listLine = (name: string, size: number) => `==T3ATTL ${name} ${size}\n`;
+
+ const source = (options: {
+ readonly list: string;
+ readonly listExit?: number;
+ readonly files?: Readonly>;
+ readonly readExit?: number;
+ readonly maxBytes?: number;
+ readonly commands?: Array>;
+ }) =>
+ OpenShellFleetDrainAdapters.execDrainAttachments({
+ ...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }),
+ exec: (sandboxName, command) => {
+ expect(sandboxName).toBe("sandbox-1");
+ options.commands?.push(command);
+ const script = command[2] ?? "";
+ if (script.includes("==T3ATTL")) {
+ return Effect.succeed({
+ exitCode: options.listExit ?? 0,
+ stdout: options.list,
+ stderr: "",
+ });
+ }
+ const name = /'([^']+)'$/.exec(script)?.[1] ?? "";
+ return Effect.succeed({
+ exitCode: options.readExit ?? 0,
+ stdout: options.files?.[name] ?? "",
+ stderr: "",
+ });
+ },
+ });
+
+ it.effect("reads each flat attachment of the thread by its id", () =>
+ Effect.gen(function* () {
+ const commands: Array> = [];
+ const attachments = yield* source({
+ list:
+ listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3) +
+ listLine("thread-a-22222222-2222-4222-8222-222222222222.bin", 3),
+ files: {
+ "thread-a-11111111-1111-4111-8111-111111111111.png": base64Of("one"),
+ "thread-a-22222222-2222-4222-8222-222222222222.bin": base64Of("two"),
+ },
+ commands,
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ });
+
+ expect(attachments.oversizeSkipped).toBe(false);
+ expect(attachments.files.map((file) => file.name)).toEqual([
+ "thread-a-11111111-1111-4111-8111-111111111111.png",
+ "thread-a-22222222-2222-4222-8222-222222222222.bin",
+ ]);
+ expect(Buffer.from(attachments.files[0]?.bytes ?? "", "base64").toString()).toBe("one");
+ // One listing, then one read per file: no base64 ever rides the listing.
+ expect(commands).toHaveLength(3);
+ expect(commands[1]?.join(" ")).toContain("thread-a-11111111-1111-4111-8111-111111111111.png");
+ }),
+ );
+
+ it.effect("ignores files that are not this thread's attachment ids", () =>
+ Effect.gen(function* () {
+ const attachments = yield* source({
+ list:
+ listLine("other-a-11111111-1111-4111-8111-111111111111.png", 3) +
+ listLine("thread-a-..%2Fescape.png", 3) +
+ listLine("thread-a-33333333-3333-4333-8333-333333333333.png", 3),
+ files: { "thread-a-33333333-3333-4333-8333-333333333333.png": base64Of("ok") },
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ });
+
+ expect(attachments.files.map((file) => file.name)).toEqual([
+ "thread-a-33333333-3333-4333-8333-333333333333.png",
+ ]);
+ }),
+ );
+
+ it.effect("skips an oversized set from the listing, before reading any bytes", () =>
+ Effect.gen(function* () {
+ const commands: Array> = [];
+ const attachments = yield* source({
+ list:
+ listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3) +
+ listLine("thread-a-2222.png", 3),
+ maxBytes: 2,
+ commands,
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ });
+
+ expect(commands).toHaveLength(1);
+ expect(attachments.files).toEqual([]);
+ }),
+ );
+
+ it.effect("fails typed when the worker cannot list attachments", () =>
+ Effect.gen(function* () {
+ const failure = yield* refused(
+ source({ list: "", listExit: 7 })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ }),
+ );
+ expect(failure.step).toBe("attachments");
+ }),
+ );
+
+ it.effect("fails typed when one attachment cannot be read", () =>
+ Effect.gen(function* () {
+ const failure = yield* refused(
+ source({
+ list: listLine("thread-a-11111111-1111-4111-8111-111111111111.png", 3),
+ readExit: 5,
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ }),
+ );
+ expect(failure.step).toBe("attachments");
+ expect(failure.message).toContain("thread-a-11111111-1111-4111-8111-111111111111.png");
+ }),
+ );
+
+ it.effect("does not let a shorter thread id claim a longer segment's attachment", () =>
+ Effect.gen(function* () {
+ const followupName = "topic-followup-33333333-3333-4333-8333-333333333333.png";
+ const attachments = yield* source({
+ list: listLine(followupName, 3),
+ files: { [followupName]: base64Of("followup") },
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "topic:a",
+ });
+ expect(attachments.files).toEqual([]);
+
+ const exact = yield* source({
+ list: listLine(followupName, 3),
+ files: { [followupName]: base64Of("followup") },
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "topic:followup",
+ });
+ expect(exact.files.map((file) => file.name)).toEqual([followupName]);
+ }),
+ );
+
+ it.effect("checks the decoded attachment bytes against the cap too", () =>
+ Effect.gen(function* () {
+ const name = "thread-a-11111111-1111-4111-8111-111111111111.png";
+ const attachments = yield* source({
+ // The listing lies about one byte; the read must enforce the real cap.
+ list: listLine(name, 2),
+ files: { [name]: base64Of("three") },
+ maxBytes: 2,
+ })({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ threadId: "thread:a",
+ });
+ expect(attachments.files).toEqual([]);
+ expect(attachments.oversizeSkipped).toBe(true);
+ }),
+ );
+});
+
+describe("linkDrainSource", () => {
+ it.effect("rejects a shell response that omits the archived-thread list", () =>
+ Effect.gen(function* () {
+ const source = OpenShellFleetDrainAdapters.linkDrainSource({
+ resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }),
+ http: HttpClient.make((request) =>
+ Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ JSON.stringify(
+ request.url.includes("/api/orchestration/shell")
+ ? { threads: [], projects: [] }
+ : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false },
+ ),
+ { status: 200, headers: { "content-type": "application/json" } },
+ ),
+ ),
+ ),
+ ),
+ });
+ const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker")));
+ expect(failure.step).toBe("shell");
+ }),
+ );
+
+ it.effect("rejects an inventory thread without a usable id", () =>
+ Effect.gen(function* () {
+ const source = OpenShellFleetDrainAdapters.linkDrainSource({
+ resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }),
+ http: HttpClient.make((request) =>
+ Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ JSON.stringify(
+ request.url.includes("/api/orchestration/shell")
+ ? {
+ threads: [{ status: "idle" }],
+ archivedThreads: [],
+ projects: [],
+ }
+ : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false },
+ ),
+ { status: 200, headers: { "content-type": "application/json" } },
+ ),
+ ),
+ ),
+ ),
+ });
+ const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker")));
+ expect(failure.step).toBe("shell");
+ expect(failure.message).toContain("without a usable id");
+ }),
+ );
+
+ it.effect("refuses advertised history with no continuation cursor", () =>
+ Effect.gen(function* () {
+ const source = OpenShellFleetDrainAdapters.linkDrainSource({
+ resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }),
+ http: HttpClient.make((request) =>
+ Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ JSON.stringify(
+ request.url.includes("/api/orchestration/shell")
+ ? { threads: [], archivedThreads: [], projects: [] }
+ : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: true },
+ ),
+ { status: 200, headers: { "content-type": "application/json" } },
+ ),
+ ),
+ ),
+ ),
+ });
+ const failure = yield* refused(
+ source.threadHistory({
+ environmentId: EnvironmentId.make("environment:worker"),
+ threadId: "thread:a",
+ }),
+ );
+ expect(failure.step).toBe("history");
+ expect(failure.message).toContain("without a continuation cursor");
+ }),
+ );
+
+ it.effect("drains archived threads like live ones", () =>
+ Effect.gen(function* () {
+ const shellRequests: Array = [];
+ const shellHeaders: Array> = [];
+ const http = HttpClient.make((request) =>
+ Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ JSON.stringify(
+ request.url.includes("/api/orchestration/shell")
+ ? (shellRequests.push(request.url),
+ shellHeaders.push(request.headers as never),
+ {
+ threads: [{ threadId: "thread:a", status: "idle" }],
+ archivedThreads: [{ threadId: "thread:old", status: "archived" }],
+ projects: [],
+ })
+ : { snapshotSequence: 1, historyCursor: null, hasMoreHistory: false },
+ ),
+ { status: 200, headers: { "content-type": "application/json" } },
+ ),
+ ),
+ ),
+ );
+ const written: Array = [];
+ const files = new Map<
+ string,
+ { threadId: string; snapshotSequence: number | null; body: string }
+ >();
+ const executor = OpenShellFleetDrain.makeDrainExecutor({
+ source: OpenShellFleetDrainAdapters.linkDrainSource({
+ resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }),
+ http,
+ }),
+ sink: {
+ write: (input) =>
+ Effect.sync(() => {
+ written.push(input.threadId);
+ files.set(input.threadId, {
+ threadId: input.threadId,
+ snapshotSequence: input.snapshotSequence,
+ body: input.body,
+ });
+ return { bytes: input.body.length };
+ }),
+ read: ({ threadId }) => Effect.succeed(files.get(threadId) ?? null),
+ copy: () => Effect.succeed(false),
+ },
+ git: { bundle: () => Effect.succeed({ exitCode: 0, stdout: "", stderr: "" }) },
+ now: Effect.succeed(0),
+ });
+ const outcome = yield* executor.drain({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandboxName: "sandbox-1",
+ drainId: "drain:environment:worker:1",
+ snapshotOnly: true,
+ deadlineAt: Number.MAX_SAFE_INTEGER,
+ });
+ // The archived thread is copied and acked, not omitted from the drain.
+ expect(outcome.state).toBe("acked");
+ expect(outcome.threads).toEqual({ total: 2, acked: 2, failed: 0 });
+ expect(written.sort()).toEqual(["thread:a", "thread:old"]);
+ // The opt-in asks for archived rows and carries the required protocol header.
+ expect(shellRequests[0]).toContain("includeArchived=1");
+ expect(shellHeaders[0]?.["x-t3-orchestration-protocol"]).toBe("2");
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+
+ it.effect("refuses an incomplete shell response instead of treating it as empty", () =>
+ Effect.gen(function* () {
+ const http = HttpClient.make((request) =>
+ Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response("{}", {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ ),
+ ),
+ );
+ const source = OpenShellFleetDrainAdapters.linkDrainSource({
+ resolve: () => Effect.succeed({ url: "http://worker.test", token: "t" }),
+ http,
+ });
+ const failure = yield* refused(source.shell(EnvironmentId.make("environment:worker")));
+ expect(failure.step).toBe("shell");
+ expect(failure.message).toContain("incomplete");
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
diff --git a/apps/server/src/environment/OpenShellFleetDrainAdapters.ts b/apps/server/src/environment/OpenShellFleetDrainAdapters.ts
new file mode 100644
index 0000000000000..aae00fb264a24
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetDrainAdapters.ts
@@ -0,0 +1,614 @@
+import {
+ isLiveSubagentShell,
+ ORCHESTRATION_PROTOCOL_HEADER,
+ ORCHESTRATION_PROTOCOL_VERSION_TEXT,
+ type EnvironmentId,
+} from "@t3tools/contracts";
+import * as Digest from "effect/crypto/Digest";
+import * as Effect from "effect/Effect";
+import * as FileSystem from "effect/FileSystem";
+import * as Hex from "effect/encoding/Hex";
+import * as Base64 from "effect/encoding/Base64";
+import * as Result from "effect/Result";
+import * as HttpClient from "effect/http/HttpClient";
+import * as Path from "effect/Path";
+import * as Schema from "effect/Schema";
+
+import { toSafeThreadAttachmentSegment } from "../attachmentStore.ts";
+
+/** The parts of a worker's thread shell that decide whether it is still working. */
+interface DrainShellThread {
+ readonly id?: string;
+ readonly threadId?: string;
+ readonly status?: string;
+ readonly activityRunStatus?: string | null;
+ readonly pendingRuntimeRequest?: unknown;
+ readonly pendingBackgroundTasks?: ReadonlyArray | null;
+}
+
+import {
+ FleetDrainStepError,
+ type DrainAttachmentManifestEntry,
+ type DrainAttachments,
+ type DrainGitBundle,
+ type DrainSink,
+ type DrainSinkRecord,
+ type DrainSource,
+} from "./OpenShellFleetDrain.ts";
+
+const encode = (value: string) => encodeURIComponent(value);
+
+/** Percent-encoding is collision-free ("thread:a" keeps its %3A, so "threada"
+ * stays distinct, and "thread/b" cannot hop directories because its "/" becomes
+ * %2F), so the encoding survives whole: refuse only empty names and the relative
+ * hops "." and "..", which encode to themselves. */
+const segmentOf = (value: string): string | null => {
+ const encoded = encodeURIComponent(value);
+ return encoded === "" || encoded === "." || encoded === ".." ? null : encoded;
+};
+
+const decodeEnvelope = (raw: string): DrainSinkRecord | null => {
+ try {
+ const parsed = JSON.parse(raw) as Partial;
+ if (typeof parsed.threadId !== "string" || typeof parsed.body !== "string") return null;
+ const attachments = Array.isArray(parsed.attachments)
+ ? parsed.attachments.filter(
+ (entry): entry is DrainAttachmentManifestEntry =>
+ typeof entry?.name === "string" &&
+ typeof entry?.hash === "string" &&
+ typeof entry?.size === "number",
+ )
+ : undefined;
+ return {
+ threadId: parsed.threadId,
+ snapshotSequence:
+ typeof parsed.snapshotSequence === "number" ? parsed.snapshotSequence : null,
+ body: parsed.body,
+ ...(attachments !== undefined && attachments.length > 0 ? { attachments } : {}),
+ ...(parsed.attachmentsSkipped === true ? { attachmentsSkipped: true } : {}),
+ };
+ } catch {
+ return null;
+ }
+};
+
+const toEnvelope = Schema.encodeEffect(
+ Schema.fromJsonString(
+ Schema.Struct({
+ threadId: Schema.String,
+ snapshotSequence: Schema.NullOr(Schema.Number),
+ body: Schema.String,
+ attachments: Schema.optionalKey(
+ Schema.Array(
+ Schema.Struct({ name: Schema.String, hash: Schema.String, size: Schema.Number }),
+ ),
+ ),
+ attachmentsSkipped: Schema.optionalKey(Schema.Boolean),
+ }),
+ ),
+);
+
+/** `stateDir/fleet-drain///thread-.json`. */
+export const fileDrainSink = Effect.fn("DrainSink.make")(function* (input: {
+ readonly stateDir: string;
+ readonly fs: FileSystem.FileSystem;
+ readonly path: Path.Path;
+}) {
+ const { fs, path, stateDir } = input;
+ // Attachment hashes come from the digest service; the sink is built once
+ // per drain and carries it.
+ const digest = yield* Digest.Digest;
+ return {
+ write: Effect.fn("DrainSink.write")(
+ function* (input: {
+ readonly environmentId: EnvironmentId;
+ readonly drainId: string;
+ readonly threadId: string;
+ readonly body: string;
+ readonly snapshotSequence: number | null;
+ readonly attachments?: DrainAttachments;
+ }) {
+ const environment = segmentOf(input.environmentId);
+ const drain = segmentOf(input.drainId);
+ const thread = segmentOf(input.threadId);
+ if (environment === null || drain === null || thread === null) {
+ return yield* new FleetDrainStepError({
+ step: "write",
+ message: `Refused a drain path segment for ${input.environmentId} / ${input.drainId} / ${input.threadId}.`,
+ });
+ }
+ const dir = path.join(stateDir, "fleet-drain", environment, drain);
+ const target = path.join(dir, `thread-${thread}.json`);
+ yield* fs.makeDirectory(dir, { recursive: true });
+ // Attachments land as sibling files so the manifest in the envelope stays
+ // small; the hash keys the file name and is what read-back verifies.
+ const manifest: Array = [];
+ for (const file of input.attachments?.files ?? []) {
+ const decoded = Base64.decode(file.bytes.replace(/\s/g, ""));
+ if (Result.isFailure(decoded)) {
+ return yield* new FleetDrainStepError({
+ step: "write",
+ message: `Attachment ${file.name} was not valid base64.`,
+ });
+ }
+ const bytes = decoded.success;
+ const hash = Hex.encode(yield* digest.digest(Digest.sha256, bytes));
+ yield* fs.writeFile(
+ path.join(dir, `thread-${thread}-att-${hash.slice(0, 16)}.bin`),
+ bytes,
+ );
+ manifest.push({ name: file.name, hash, size: bytes.byteLength });
+ }
+ // Write then rename: a retry must never accept a half-written copy as done.
+ const envelope = yield* toEnvelope({
+ threadId: input.threadId,
+ snapshotSequence: input.snapshotSequence,
+ body: input.body,
+ ...(manifest.length === 0 ? {} : { attachments: manifest }),
+ ...(input.attachments?.oversizeSkipped === true ? { attachmentsSkipped: true } : {}),
+ });
+ yield* fs.writeFileString(`${target}.partial`, envelope);
+ yield* fs.rename(`${target}.partial`, target);
+ return { bytes: new TextEncoder().encode(envelope).byteLength };
+ },
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "write",
+ message: "The drain copy could not be written.",
+ cause,
+ }),
+ ),
+ ),
+ read: Effect.fn("DrainSink.read")(
+ function* (input: {
+ readonly environmentId: EnvironmentId;
+ readonly drainId: string;
+ readonly threadId: string;
+ }) {
+ const environment = segmentOf(input.environmentId);
+ const drain = segmentOf(input.drainId);
+ const thread = segmentOf(input.threadId);
+ if (environment === null || drain === null || thread === null) {
+ return yield* new FleetDrainStepError({
+ step: "read",
+ message: `Refused a drain path segment for ${input.environmentId} / ${input.drainId} / ${input.threadId}.`,
+ });
+ }
+ const file = path.join(
+ stateDir,
+ "fleet-drain",
+ environment,
+ drain,
+ `thread-${thread}.json`,
+ );
+ const raw = yield* fs.readFileString(file).pipe(Effect.orElseSucceed(() => null));
+ if (raw === null) return null;
+ const record = decodeEnvelope(raw);
+ if (record === null) return null;
+ // A manifest without its bytes is not a copy: verify each hash so a
+ // corrupted attachment fails the ack instead of being imported later.
+ for (const entry of record.attachments ?? []) {
+ const bytes = yield* fs
+ .readFile(
+ path.join(path.dirname(file), `thread-${thread}-att-${entry.hash.slice(0, 16)}.bin`),
+ )
+ .pipe(Effect.orElseSucceed(() => null));
+ if (bytes === null) return null;
+ if (Hex.encode(yield* digest.digest(Digest.sha256, bytes)) !== entry.hash) return null;
+ }
+ return record;
+ },
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "read",
+ message: "The drain copy could not be read.",
+ cause,
+ }),
+ ),
+ ),
+ copy: Effect.fn("DrainSink.copy")(
+ function* (input: {
+ readonly environmentId: EnvironmentId;
+ readonly fromDrainId: string;
+ readonly toDrainId: string;
+ readonly threadId: string;
+ }) {
+ const environment = segmentOf(input.environmentId);
+ const from = segmentOf(input.fromDrainId);
+ const to = segmentOf(input.toDrainId);
+ const thread = segmentOf(input.threadId);
+ const fromDir =
+ environment === null || from === null
+ ? null
+ : path.join(stateDir, "fleet-drain", environment, from);
+ const toDir =
+ environment === null || to === null
+ ? null
+ : path.join(stateDir, "fleet-drain", environment, to);
+ if (fromDir === null || toDir === null || thread === null)
+ return yield* new FleetDrainStepError({
+ step: "write",
+ message: `Refused a drain path segment for ${input.fromDrainId} / ${input.toDrainId}.`,
+ });
+ const name = `thread-${thread}.json`;
+ const raw = yield* fs
+ .readFileString(path.join(fromDir, name))
+ .pipe(Effect.orElseSucceed(() => null));
+ const record = raw === null ? null : decodeEnvelope(raw);
+ if (record === null || raw === null) return false;
+ yield* fs.makeDirectory(toDir, { recursive: true });
+ // Attachment files are keyed by the manifest hash, so they travel with it.
+ for (const entry of record.attachments ?? []) {
+ const attachment = `thread-${thread}-att-${entry.hash.slice(0, 16)}.bin`;
+ const bytes = yield* fs.readFile(path.join(fromDir, attachment)).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "write",
+ message: `Attachment copy failed: ${cause}`,
+ }),
+ ),
+ );
+ yield* fs.writeFile(path.join(toDir, attachment), bytes).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "write",
+ message: `Attachment copy failed: ${cause}`,
+ }),
+ ),
+ );
+ }
+ yield* fs.writeFileString(`${path.join(toDir, name)}.partial`, raw);
+ yield* fs.rename(`${path.join(toDir, name)}.partial`, path.join(toDir, name));
+ return true;
+ },
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "write",
+ message: "The drain copy could not be written.",
+ cause,
+ }),
+ ),
+ ),
+ } satisfies DrainSink;
+}, Effect.provide(Digest.layerWebCrypto));
+
+const toJsonBody = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
+const fromJsonBody = Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown));
+
+/**
+ * Reads the worker over its authenticated link: the shell snapshot lists what it
+ * holds, and each thread's bounded snapshot plus history is the lossless copy.
+ */
+export const linkDrainSource = (input: {
+ readonly resolve: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect<{ readonly url: string; readonly token: string }, FleetDrainStepError>;
+ readonly http: HttpClient.HttpClient;
+}): DrainSource => {
+ const http = input.http;
+ const getJson = (environmentId: EnvironmentId, path: string) =>
+ input.resolve(environmentId).pipe(
+ Effect.flatMap((link) =>
+ http.get(new URL(path, link.url).toString(), {
+ // The orchestration endpoints require the protocol header; without it
+ // the worker answers 400 and the drain would look empty.
+ headers: {
+ authorization: `Bearer ${link.token}`,
+ [ORCHESTRATION_PROTOCOL_HEADER]: ORCHESTRATION_PROTOCOL_VERSION_TEXT,
+ },
+ }),
+ ),
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "history",
+ message: "The worker history could not be read.",
+ cause,
+ }),
+ ),
+ Effect.flatMap((response) =>
+ response.text.pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "history",
+ message: "The worker history could not be read.",
+ cause,
+ }),
+ ),
+ Effect.flatMap((body) =>
+ response.status < 200 || response.status >= 300
+ ? Effect.fail(
+ new FleetDrainStepError({
+ step: "history",
+ message: `The worker answered ${response.status} for ${path}.`,
+ }),
+ )
+ : fromJsonBody(body).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "history",
+ message: "The worker history could not be read.",
+ cause,
+ }),
+ ),
+ ),
+ ),
+ ),
+ ),
+ );
+ return {
+ shell: (environmentId) =>
+ // Archived threads are drainable data: the worker lists them only on request.
+ getJson(environmentId, "/api/orchestration/shell?includeArchived=1").pipe(
+ Effect.flatMap((snapshot) =>
+ Effect.gen(function* () {
+ const shell = snapshot as {
+ readonly threads?: ReadonlyArray;
+ readonly archivedThreads?: ReadonlyArray;
+ readonly projects?: ReadonlyArray<{
+ readonly workspaceRoot?: string;
+ readonly path?: string;
+ }>;
+ };
+ // A body without both lists is an incomplete answer, not an empty
+ // worker: an empty ack here would delete unconverted threads.
+ if (
+ !Array.isArray(shell.threads) ||
+ !Array.isArray(shell.projects) ||
+ !Array.isArray(shell.archivedThreads)
+ )
+ return yield* new FleetDrainStepError({
+ step: "shell",
+ message: "The worker's shell response is incomplete; nothing was acknowledged.",
+ });
+ const shellThreads = [...shell.threads, ...shell.archivedThreads];
+ const threads: Array<{ readonly threadId: string; readonly live: boolean }> = [];
+ for (const thread of shellThreads) {
+ const threadId = thread.threadId ?? thread.id;
+ if (typeof threadId !== "string" || threadId === "") {
+ return yield* new FleetDrainStepError({
+ step: "shell",
+ message: "The worker returned a shell thread without a usable id.",
+ });
+ }
+ // Liveness comes from the shell's own status, and an unreadable one
+ // counts as live: removing a worker that is still writing loses work.
+ const live =
+ thread.status === undefined && thread.activityRunStatus === undefined
+ ? true
+ : isLiveSubagentShell({
+ status: thread.status ?? "idle",
+ activityRunStatus: thread.activityRunStatus ?? null,
+ pendingRuntimeRequest: thread.pendingRuntimeRequest ?? null,
+ pendingBackgroundTasks: thread.pendingBackgroundTasks ?? null,
+ } as never);
+ threads.push({ threadId, live });
+ }
+ const projects = shell.projects.flatMap((project) => {
+ const workspaceRoot = project.workspaceRoot ?? project.path;
+ return workspaceRoot === undefined ? [] : [{ workspaceRoot }];
+ });
+ return { threads, projects };
+ }),
+ ),
+ ),
+ threadSequence: (request) =>
+ getJson(
+ request.environmentId,
+ `/api/orchestration/threads/${encode(request.threadId)}/bounded`,
+ ).pipe(
+ Effect.map((snapshot) => ({
+ snapshotSequence:
+ typeof (snapshot as { snapshotSequence?: unknown }).snapshotSequence === "number"
+ ? (snapshot as { snapshotSequence: number }).snapshotSequence
+ : null,
+ })),
+ ),
+ threadHistory: Effect.fn("DrainSource.threadHistory")(function* (request: {
+ readonly environmentId: EnvironmentId;
+ readonly threadId: string;
+ }) {
+ const encoded = encode(request.threadId);
+ const snapshot = (yield* getJson(
+ request.environmentId,
+ `/api/orchestration/threads/${encoded}/bounded`,
+ )) as {
+ readonly snapshotSequence?: unknown;
+ readonly historyCursor?: string | null;
+ readonly hasMoreHistory?: boolean;
+ };
+ // The page endpoint needs the worker's own cursor: a page size is rejected,
+ // and a truncated history is not a drain (spec section 7.2).
+ const pages: unknown[] = [];
+ let cursor = snapshot.historyCursor ?? null;
+ let hasMore = snapshot.hasMoreHistory === true;
+ if (hasMore && cursor === null)
+ return yield* new FleetDrainStepError({
+ step: "history",
+ message: "The worker advertised more history without a continuation cursor.",
+ });
+ while (hasMore && cursor !== null) {
+ const page = (yield* getJson(
+ request.environmentId,
+ `/api/orchestration/threads/${encoded}/history?cursor=${encode(cursor)}`,
+ )) as { readonly nextCursor?: string | null; readonly hasMoreHistory?: boolean };
+ pages.push(page);
+ hasMore = page.hasMoreHistory === true;
+ cursor = page.nextCursor ?? null;
+ if (hasMore && cursor === null)
+ return yield* new FleetDrainStepError({
+ step: "history",
+ message: "The worker advertised more history without a continuation cursor.",
+ });
+ }
+ const body = yield* toJsonBody({ threadId: request.threadId, snapshot, pages }).pipe(
+ Effect.mapError(
+ (cause) =>
+ new FleetDrainStepError({
+ step: "history",
+ message: "The worker history could not be read.",
+ cause,
+ }),
+ ),
+ );
+ return {
+ body,
+ snapshotSequence:
+ typeof snapshot.snapshotSequence === "number" ? snapshot.snapshotSequence : null,
+ };
+ }),
+ };
+};
+
+/** Total decoded bundle bytes a checkout may export before the drain refuses it. */
+const MAX_DRAIN_BUNDLE_BYTES = 256 * 1024 * 1024;
+
+/** Exports a worker checkout and its checkpoint refs from inside the sandbox. */
+export const execDrainGitBundle = (input: {
+ readonly exec: (
+ sandboxName: string,
+ command: ReadonlyArray,
+ ) => Effect.Effect<
+ { readonly exitCode: number; readonly stdout: string; readonly stderr: string },
+ FleetDrainStepError
+ >;
+ /** Overridden in tests; the production cap keeps one drain off the OOM path. */
+ readonly maxBytes?: number;
+}): DrainGitBundle => ({
+ bundle: Effect.fn("FleetDrainGitBundle.bundle")(
+ // A worker cannot push: its origin is the read-only shared mirror, it holds
+ // no credentials, and a branch behind origin rejects a push anyway. A
+ // bundle carries the refs (branches and the refs/t3 checkpoints holding
+ // uncommitted work) the mirror lacks, base64 so exec stdout stays text.
+ ({ sandboxName, workspaceRoot }) =>
+ Effect.gen(function* () {
+ const maxBytes = input.maxBytes ?? MAX_DRAIN_BUNDLE_BYTES;
+ const root = shellQuote(workspaceRoot);
+ const revs = `HEAD --branches --glob='refs/t3/*'`;
+ const exported = yield* input.exec(sandboxName, [
+ "sh",
+ "-c",
+ // HEAD carries a detached commit; subtract only origin, so a commit
+ // fetched from another remote is not a missing prerequisite.
+ // An empty export means the mirror has everything and acks.
+ `b="\${TMPDIR:-/tmp}/t3-fleet-drain-$$.bundle"; ` +
+ `if [ "$(git -C ${root} rev-list --count ${revs} --not --remotes=origin)" -eq 0 ]; then exit 0; fi; ` +
+ `git -C ${root} bundle create "$b" ${revs} --not --remotes=origin || exit $?; ` +
+ // The cap is checked before base64, so an oversized bundle never
+ // crosses exec as one string.
+ // shortcut: without origin remote-tracking refs there is no
+ // baseline, so a large checkout needs force removal.
+ `if [ "$(wc -c <"$b" | tr -d '[:space:]')" -gt ${maxBytes} ]; then rm -f "$b"; exit 4; fi; ` +
+ `base64 "$b"; rc=$?; rm -f "$b"; exit $rc`,
+ ]);
+ if (exported.exitCode === 4)
+ return yield* new FleetDrainStepError({
+ step: "bundle",
+ message: `The checkout bundle of ${workspaceRoot} exceeds the drain size cap.`,
+ });
+ return exported;
+ }),
+ ),
+});
+
+/** Single-quotes a value for a shell command line. */
+const shellQuote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`;
+
+/** Total decoded attachment bytes drained per thread before skipping the set. */
+const MAX_DRAIN_ATTACHMENT_BYTES = 50 * 1024 * 1024;
+
+/** Marks one attachment file's name and size on the listing command's stdout. */
+const ATTACHMENT_LIST_FRAME = "==T3ATTL ";
+
+/**
+ * Attachment files over gateway exec (the worker's HTTP API serves no
+ * attachment bytes): the worker lists the thread's files with their sizes
+ * first, so an oversized set is skipped before any bytes cross exec, then each
+ * file is read on its own. Files live flat as `-`.
+ */
+export const execDrainAttachments = (input: {
+ readonly exec: (
+ sandboxName: string,
+ command: ReadonlyArray,
+ ) => Effect.Effect<
+ { readonly exitCode: number; readonly stdout: string; readonly stderr: string },
+ FleetDrainStepError
+ >;
+ readonly maxBytes?: number;
+}): NonNullable => {
+ const maxBytes = input.maxBytes ?? MAX_DRAIN_ATTACHMENT_BYTES;
+ const attachmentsDir = `"\${T3CODE_HOME:-$HOME/.t3}/userdata/attachments"`;
+ const listCommand = (segment: string) => [
+ "sh",
+ "-c",
+ `d=${attachmentsDir}; if [ -d "$d" ]; then for f in "$d"/${segment}-*; do ` +
+ `[ -f "$f" ] || continue; case "$f" in *.part) continue;; esac; ` +
+ `printf '%s\\n' "${ATTACHMENT_LIST_FRAME}$(basename "$f") $(wc -c <"$f" | tr -d '[:space:]')"; ` +
+ `done; fi`,
+ ];
+ const readCommand = (name: string) => [
+ "sh",
+ "-c",
+ `base64 ${attachmentsDir}/${shellQuote(name)}`,
+ ];
+ return ({ sandboxName, threadId }) =>
+ Effect.gen(function* () {
+ const segment = toSafeThreadAttachmentSegment(threadId);
+ if (segment === null) return { files: [], oversizeSkipped: false } as const;
+ const listed = yield* input.exec(sandboxName, listCommand(segment));
+ if (listed.exitCode !== 0) {
+ return yield* new FleetDrainStepError({
+ step: "attachments",
+ message: `The worker did not list attachments for ${threadId} (exit ${listed.exitCode}).`,
+ });
+ }
+ const entries: Array<{ name: string; size: number }> = [];
+ for (const line of listed.stdout.split("\n")) {
+ if (!line.startsWith(ATTACHMENT_LIST_FRAME)) continue;
+ const rest = line.slice(ATTACHMENT_LIST_FRAME.length).trim();
+ const separator = rest.lastIndexOf(" ");
+ if (separator <= 0) continue;
+ const name = rest.slice(0, separator);
+ const size = Number(rest.slice(separator + 1));
+ // The worker names files itself; anything else is not one of this thread's.
+ // The full id pattern prevents `topic` from claiming files that belong
+ // to a longer segment such as `topic-followup`.
+ if (
+ !new RegExp(
+ `^${segment}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?:\\.[a-z0-9]{1,10})?$`,
+ "i",
+ ).test(name)
+ )
+ continue;
+ if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name)) continue;
+ if (!Number.isSafeInteger(size) || size < 0) continue;
+ entries.push({ name, size });
+ }
+ if (entries.reduce((total, entry) => total + entry.size, 0) > maxBytes) {
+ return { files: [], oversizeSkipped: true } as const;
+ }
+ const files: Array<{ name: string; bytes: string }> = [];
+ let copiedBytes = 0;
+ for (const entry of entries) {
+ const read = yield* input.exec(sandboxName, readCommand(entry.name));
+ if (read.exitCode !== 0) {
+ return yield* new FleetDrainStepError({
+ step: "attachments",
+ message: `The worker did not read attachment ${entry.name} of ${threadId} (exit ${read.exitCode}).`,
+ });
+ }
+ const bytes = Buffer.from(read.stdout, "base64");
+ copiedBytes += bytes.byteLength;
+ if (copiedBytes > maxBytes) return { files: [], oversizeSkipped: true } as const;
+ files.push({ name: entry.name, bytes: read.stdout });
+ }
+ return { files, oversizeSkipped: false } as const;
+ });
+};
diff --git a/apps/server/src/environment/OpenShellFleetService.test.ts b/apps/server/src/environment/OpenShellFleetService.test.ts
new file mode 100644
index 0000000000000..b2db614851781
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetService.test.ts
@@ -0,0 +1,220 @@
+import { describe, expect, it } from "@effect/vitest";
+import { vi } from "vite-plus/test";
+
+import { EnvironmentId, type OpenShellFleetWorkerStatus } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Context from "effect/Context";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as Layer from "effect/Layer";
+import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+
+import * as ServerConfig from "../config.ts";
+import * as PeerEnvironmentRegistry from "../peer/PeerEnvironmentRegistry.ts";
+import * as PeerLinks from "../peer/PeerLinks.ts";
+import * as ServerEnvironment from "./ServerEnvironment.ts";
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import * as OpenShellFleetStore from "./OpenShellFleetStore.ts";
+import * as OpenShellFleetService from "./OpenShellFleetService.ts";
+
+const { fakeGatewayClient } = vi.hoisted(() => {
+ const view = (name: string) => ({
+ name,
+ sandboxId: `sbx-${name}`,
+ phase: "Ready",
+ labels: {},
+ createdAt: 1_700_000_000_000,
+ });
+ return {
+ fakeGatewayClient: {
+ health: async () => {},
+ createFromTemplate: async (spec: { name?: string }) => view(spec.name ?? "sbx-x"),
+ get: async () => null,
+ listAll: async () => [],
+ deleteSandbox: async () => "completed" as const,
+ waitReady: async (name: string) => view(name),
+ exec: async () => ({ exitCode: 0, stdout: "{}", stderr: "" }),
+ setPolicy: async () => {},
+ listTemplates: async () => [],
+ createTemplate: async (spec: { name?: string }) => ({ name: spec.name ?? "tpl" }),
+ },
+ };
+});
+
+vi.mock("@t3tools/openshell/gatewayClient", () => ({
+ connectOpenShellGateway: async () => fakeGatewayClient,
+}));
+
+const fleetOnSettings = {
+ openshellEnabled: true,
+ openshell: { gatewayUrl: "https://gateway.example.test", managingInstance: true },
+} as const;
+
+/** Secret store mock: fails its first `count` reads, then answers empty. */
+const flakySecrets = (failures: number) => {
+ let calls = 0;
+ const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
+ get: () => {
+ calls += 1;
+ return calls <= failures
+ ? Effect.fail(
+ new ServerSecretStore.SecretStoreReadError({
+ resource: "openshell-gateway-credential",
+ cause: "locked in this test",
+ }),
+ )
+ : Effect.succeedNone;
+ },
+ getOrCreateRandom: (name: string) =>
+ Effect.succeed(new TextEncoder().encode(`random-for-${name}`)),
+ });
+ return { secrets, calls: () => calls };
+};
+
+const alwaysFailingSecrets = () => {
+ let calls = 0;
+ const secrets = Layer.mock(ServerSecretStore.ServerSecretStore)({
+ get: () => {
+ calls += 1;
+ return Effect.fail(
+ new ServerSecretStore.SecretStoreReadError({
+ resource: "openshell-gateway-credential",
+ cause: "locked in this test",
+ }),
+ );
+ },
+ getOrCreateRandom: (name: string) =>
+ Effect.succeed(new TextEncoder().encode(`random-for-${name}`)),
+ });
+ return { secrets, calls: () => calls };
+};
+
+const managingEnvironment = Layer.succeed(
+ ServerEnvironment.ServerEnvironment,
+ ServerEnvironment.ServerEnvironment.of({
+ getEnvironmentId: Effect.succeed(EnvironmentId.make("environment-managing")),
+ getDescriptor: Effect.die("getDescriptor is unused in this test"),
+ }),
+);
+
+const layerFleet = (
+ settingsContext: Context.Context,
+ secrets: Layer.Layer,
+) =>
+ OpenShellFleetService.layer.pipe(
+ Layer.provide(Layer.succeedContext(settingsContext)),
+ Layer.provide(secrets),
+ Layer.provide(managingEnvironment),
+ Layer.provide(Layer.mock(PeerEnvironmentRegistry.PeerEnvironmentRegistry)({})),
+ Layer.provide(
+ Layer.mock(OpenShellFleetStore.OpenShellFleetStore)({ list: Effect.succeed([]) }),
+ ),
+ Layer.provide(ServerConfig.layerTest("/tmp/t3-fleet-test-cwd", "/tmp/t3-fleet-test")),
+ Layer.provide(Layer.mock(PeerLinks.PeerLinks)({})),
+ Layer.provide(
+ Layer.succeed(SecretResolver.SecretResolver, {
+ label: (provider) => Effect.succeed(provider),
+ read: (references) => Effect.succeed(references.map(() => undefined)),
+ invalidate: () => Effect.void,
+ }),
+ ),
+ Layer.provide(NodeServices.layer),
+ Layer.provide(FetchHttpClient.layer),
+ );
+
+describe("OpenShellFleetService", () => {
+ it.effect("refuses a spawn once OpenShell is disabled after first use", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings));
+ const settings = Context.get(settingsContext, ServerSettings.ServerSettingsService);
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe(
+ Effect.provide(
+ layerFleet(
+ settingsContext,
+ Layer.mock(ServerSecretStore.ServerSecretStore)({
+ get: () => Effect.succeedNone,
+ getOrCreateRandom: (name: string) =>
+ Effect.succeed(new TextEncoder().encode(`random-for-${name}`)),
+ }),
+ ),
+ ),
+ );
+
+ expect(yield* fleet.listWorkers({})).toEqual([] as OpenShellFleetWorkerStatus[]);
+
+ yield* settings.updateSettings({ openshellEnabled: false });
+ const error = yield* fleet
+ .spawnWorker({ runtimeModeCeiling: "approval-required", workerClass: "ephemeral" })
+ .pipe(Effect.flip);
+ expect(error).toBeInstanceOf(OpenShellFleetService.OpenShellFleetDisabled);
+ }),
+ ),
+ );
+
+ it.effect("retries a transient init failure on the next call instead of caching it", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings));
+ const { secrets, calls } = flakySecrets(1);
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe(
+ Effect.provide(layerFleet(settingsContext, secrets)),
+ );
+
+ const firstError = yield* fleet.listWorkers({}).pipe(Effect.flip);
+ expect(firstError).toBeInstanceOf(OpenShellFleetService.OpenShellFleetOperationFailed);
+
+ // The same service instance builds the driver on the retry: a failure
+ // was never cached, so the second call gets a working driver.
+ expect(yield* fleet.listWorkers({})).toEqual([]);
+ expect(calls()).toBe(2);
+ }),
+ ),
+ );
+
+ it.effect("labels an operation's init failure with that operation, not an earlier caller's", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings));
+ const { secrets } = alwaysFailingSecrets();
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe(
+ Effect.provide(layerFleet(settingsContext, secrets)),
+ );
+
+ const spawnError = yield* fleet
+ .spawnWorker({ runtimeModeCeiling: "approval-required", workerClass: "ephemeral" })
+ .pipe(Effect.flip);
+ expect(spawnError).toMatchObject({
+ _tag: "OpenShellFleetOperationFailed",
+ operation: "spawn",
+ message: "The fleet operation failed.",
+ });
+
+ // A failed init leaves nothing cached, so the next operation rebuilds
+ // and labels the failure with its own operation.
+ const removeError = yield* fleet
+ .removeWorker({ targetEnvironmentId: EnvironmentId.make("environment-worker-1") })
+ .pipe(Effect.flip);
+ expect(removeError).toMatchObject({ operation: "remove" });
+ }),
+ ),
+ );
+ it.effect("shares driver initialization across concurrent first calls", () =>
+ Effect.scoped(
+ Effect.gen(function* () {
+ const settingsContext = yield* Layer.build(ServerSettings.layerTest(fleetOnSettings));
+ const { secrets, calls } = flakySecrets(0);
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService.pipe(
+ Effect.provide(layerFleet(settingsContext, secrets)),
+ );
+ expect(
+ yield* Effect.all([fleet.listWorkers({}), fleet.listWorkers({})], {
+ concurrency: "unbounded",
+ }),
+ ).toEqual([[], []]);
+ expect(calls()).toBe(1);
+ }),
+ ),
+ );
+});
diff --git a/apps/server/src/environment/OpenShellFleetService.ts b/apps/server/src/environment/OpenShellFleetService.ts
new file mode 100644
index 0000000000000..0c4a4e7561eb8
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetService.ts
@@ -0,0 +1,562 @@
+import {
+ EnvironmentId,
+ type GatewayDriverRemoveOptions,
+ type GatewayDriverRemoveResult,
+ type GatewayDriverSpawnResult,
+ isRuntimeModeAbove,
+ type OpenShellFleetWorkerStatus,
+ OpenShellWorkerClass,
+ type OrchestratorMcpRuntimeMode,
+ type RuntimeMode,
+ type ServerSettings as ServerSettingsValue,
+} from "@t3tools/contracts";
+import * as Clock from "effect/Clock";
+import type { OpenShellGatewayAuth } from "@t3tools/contracts";
+import * as OpenShellGatewayAuthModule from "./OpenShellGatewayAuth.ts";
+import * as Duration from "effect/Duration";
+import * as FileSystem from "effect/FileSystem";
+import * as HttpClient from "effect/http/HttpClient";
+import * as Path from "effect/Path";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Ref from "effect/Ref";
+import * as Schedule from "effect/Schedule";
+import * as Semaphore from "effect/Semaphore";
+import * as Schema from "effect/Schema";
+
+import { forkParked } from "../serverActivation.ts";
+
+import {
+ connectOpenShellGateway,
+ type OpenShellGatewayClient,
+} from "@t3tools/openshell/gatewayClient";
+import {
+ OpenShellGatewayDriver,
+ OpenShellGatewayDriverError,
+ type GatewayDriverSpawnOptionsWithCeiling,
+ type OpenShellDriverSettings,
+ type OrphanSweepResult,
+ type WorkerPairingRegistrar,
+ type FleetWorkerIndex,
+ type FleetWorkerRow,
+} from "@t3tools/openshell/OpenShellGatewayDriver";
+
+import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
+import * as OpenShellFleetDrain from "../environment/OpenShellFleetDrain.ts";
+import * as OpenShellFleetDrainAdapters from "../environment/OpenShellFleetDrainAdapters.ts";
+import * as OpenShellFleetBundleImport from "../environment/OpenShellFleetBundleImport.ts";
+import * as ServerConfig from "../config.ts";
+import * as PeerLinks from "../peer/PeerLinks.ts";
+import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as OpenShellFleetStore from "../environment/OpenShellFleetStore.ts";
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import * as PeerEnvironmentRegistry from "../peer/PeerEnvironmentRegistry.ts";
+import * as PeerSessionResolve from "../peer/PeerSessionResolve.ts";
+import * as EventSink from "../orchestration-v2/EventSink.ts";
+import * as SqlClient from "effect/sql/SqlClient";
+
+/** ServerSecretStore slot holding the gateway credential (spec section 4.2). */
+const GATEWAY_CREDENTIAL_SLOT = "openshell-gateway-credential";
+
+const isDriverError = Schema.is(OpenShellGatewayDriverError);
+
+const offHere = "OpenShell fleet management is off here.";
+const fleetOperationFailed = "The fleet operation failed.";
+
+export class OpenShellFleetDisabled extends Schema.TaggedError<
+ OpenShellFleetDisabled,
+ { readonly brand: unique symbol }
+>()("OpenShellFleetDisabled", { message: Schema.String }) {}
+
+export class OpenShellFleetSpawnRefused extends Schema.TaggedError<
+ OpenShellFleetSpawnRefused,
+ { readonly brand: unique symbol }
+>()("OpenShellFleetSpawnRefused", {
+ reason: Schema.Literals(["runtime_mode_above_ceiling", "unknown_template"]),
+ message: Schema.String,
+}) {}
+
+export class OpenShellFleetOperationFailed extends Schema.TaggedError<
+ OpenShellFleetOperationFailed,
+ { readonly brand: unique symbol }
+>()("OpenShellFleetOperationFailed", {
+ operation: Schema.Literals(["spawn", "remove", "list", "sweep"]),
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+export type OpenShellFleetError =
+ | OpenShellFleetDisabled
+ | OpenShellFleetSpawnRefused
+ | OpenShellFleetOperationFailed;
+
+/**
+ * The OpenShell fleet on this managing instance (spec sections 5 and 6). Builds
+ * the drain executor and the post-drain importer per environment: nothing lost
+ * outranks cleanup.
+ */
+export class OpenShellFleetService extends Context.Service<
+ OpenShellFleetService,
+ {
+ readonly spawnWorker: (input: {
+ readonly template?: string;
+ readonly runtimeMode?: OrchestratorMcpRuntimeMode;
+ readonly runtimeModeCeiling: RuntimeMode;
+ readonly workerClass: OpenShellWorkerClass;
+ readonly label?: string;
+ readonly ttlMs?: number;
+ readonly seedFrom?: string;
+ readonly clientRequestId?: string;
+ }) => Effect.Effect;
+ readonly removeWorker: (input: {
+ readonly targetEnvironmentId: EnvironmentId;
+ readonly drainTimeoutMs?: number;
+ readonly force?: boolean;
+ readonly clientRequestId?: string;
+ }) => Effect.Effect;
+ readonly listWorkers: (input: {
+ readonly workerClass?: OpenShellWorkerClass;
+ }) => Effect.Effect, OpenShellFleetError>;
+ /** One orphan-sweep tick: finalizes vanished rows and retires expired ephemeral workers. */
+ readonly sweepWorkers: () => Effect.Effect;
+ }
+>()("t3/environment/OpenShellFleetService") {}
+
+const toIndexRow = (row: OpenShellFleetStore.FleetEnvironment): FleetWorkerRow => ({
+ environmentId: row.environmentId,
+ state: row.state,
+ workerClass: row.workerClass,
+ label: row.label,
+ sandboxName: row.sandboxName,
+ sandboxId: row.sandboxId,
+ drain: row.drain,
+ expiresAt: row.expiresAt,
+ lastError: row.lastError,
+});
+
+const make = Effect.gen(function* () {
+ const settings = yield* ServerSettings.ServerSettingsService;
+ const secrets = yield* ServerSecretStore.ServerSecretStore;
+ const secretResolver = yield* SecretResolverCore.SecretResolver;
+ const environment = yield* ServerEnvironment.ServerEnvironment;
+ const registry = yield* PeerEnvironmentRegistry.PeerEnvironmentRegistry;
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ const config = yield* ServerConfig.ServerConfig;
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const http = yield* HttpClient.HttpClient;
+ const links = yield* PeerLinks.PeerLinks;
+ // The post-drain bundle import writes through the same event sink and import
+ // table the orchestration projections use, so imported threads are ordinary
+ // archived threads (spec section 7.4). Optional: hosts that do not build the
+ // orchestration runtime keep draining, and the import warns on the row.
+ const maybeEventSink = yield* Effect.serviceOption(EventSink.EventSinkV2);
+ const maybeSql = yield* Effect.serviceOption(SqlClient.SqlClient);
+
+ // Gateway URL changes take effect after restart; failed initialization stays retryable.
+ const initialization = yield* Semaphore.make(1);
+ const cachedDriver = yield* Ref.make>(Option.none());
+
+ const failDisabled = (message: string) => new OpenShellFleetDisabled({ message });
+
+ /** A stable message for callers; the underlying detail stays in the cause. */
+ const failed = (operation: OpenShellFleetOperationFailed["operation"], cause: unknown) =>
+ new OpenShellFleetOperationFailed({
+ operation,
+ message: fleetOperationFailed,
+ cause,
+ });
+
+ /** One driver call, its failures typed for the MCP tools and the driver's own refusals kept. */
+ const callDriver = (
+ operation: OpenShellFleetOperationFailed["operation"],
+ call: () => Promise,
+ refusal?: (error: OpenShellGatewayDriverError) => OpenShellFleetError | undefined,
+ ): Effect.Effect =>
+ Effect.tryPromise({
+ try: call,
+ catch: (cause) => {
+ const refused = isDriverError(cause) ? refusal?.(cause) : undefined;
+ return refused ?? failed(operation, cause);
+ },
+ });
+
+ /** The gateway block, only on an enabled managing instance with an address. */
+ const gatewaySettings: Effect.Effect<
+ { gatewayUrl: string; openshell: ServerSettingsValue["openshell"] },
+ OpenShellFleetError
+ > = Effect.gen(function* () {
+ const current = yield* settings.getSettings.pipe(
+ Effect.mapError(() => failDisabled("The OpenShell fleet settings could not be read here.")),
+ );
+ const openshell = current.openshell;
+ if (!current.openshellEnabled || !openshell.managingInstance) {
+ return yield* failDisabled(
+ "OpenShell fleet management is off here. Turn it on in Settings → Labs and make this environment the managing instance.",
+ );
+ }
+ const gatewayUrl = openshell.gatewayUrl;
+ if (gatewayUrl === undefined || gatewayUrl.trim() === "") {
+ return yield* failDisabled(
+ "No OpenShell gateway address is configured. Set it in Settings → Labs.",
+ );
+ }
+ return { gatewayUrl, openshell };
+ });
+
+ /**
+ * The gateway credential, decoded for the SDK's token provider. A slot that is
+ * not set means "no credential"; a read that failed is a typed fleet error,
+ * never silently treated as unset.
+ */
+ const gatewayCredential = Effect.fn("OpenShellFleetService.gatewayCredential")(function* (
+ operation: OpenShellFleetOperationFailed["operation"],
+ ) {
+ const stored = yield* secrets
+ .get(GATEWAY_CREDENTIAL_SLOT)
+ .pipe(Effect.mapError((cause) => failed(operation, cause)));
+ return Option.isSome(stored) ? new TextDecoder().decode(stored.value) : undefined;
+ });
+
+ const openClient = Effect.fn("OpenShellFleetService.openClient")(function* (
+ operation: OpenShellFleetOperationFailed["operation"],
+ gatewayUrl: string,
+ credential: string | undefined,
+ auth: OpenShellGatewayAuth = { mode: "static" },
+ ) {
+ const mtlsConnectOptions = yield* Effect.gen(function* () {
+ if (auth.mode !== "mtls") return undefined;
+ const provider = yield* OpenShellGatewayAuthModule.makeGatewayAuthTokenProvider({
+ auth,
+ secrets,
+ resolver: secretResolver,
+ }).pipe(Effect.catch((cause: unknown) => failed(operation, cause)));
+ return "connectOptions" in provider ? provider.connectOptions : undefined;
+ }).pipe(Effect.catch((cause) => failed(operation, cause)));
+ const oidcTokenProvider = yield* Effect.gen(function* () {
+ if (auth.mode === "mtls") return undefined;
+ if (auth.mode !== "static") {
+ const provider = yield* OpenShellGatewayAuthModule.makeGatewayAuthTokenProvider({
+ auth,
+ secrets,
+ resolver: secretResolver,
+ }).pipe(Effect.catch((cause: unknown) => failed(operation, cause)));
+ return provider.oidcTokenProvider;
+ }
+ // Legacy provisioned bearer (spec section 4.2).
+ return credential === undefined
+ ? undefined
+ : (): Promise => Promise.resolve(credential);
+ }).pipe(Effect.catch((cause) => failed(operation, cause)));
+ const client: OpenShellGatewayClient = yield* Effect.tryPromise({
+ try: () =>
+ connectOpenShellGateway(
+ oidcTokenProvider === undefined && mtlsConnectOptions === undefined
+ ? { gatewayUrl }
+ : {
+ gatewayUrl,
+ ...(oidcTokenProvider === undefined ? {} : { oidcTokenProvider }),
+ ...(mtlsConnectOptions === undefined ? {} : mtlsConnectOptions),
+ },
+ ),
+ catch: (cause) => failed(operation, cause),
+ });
+ return client;
+ });
+
+ /** Nothing is deleted before its data is here (spec section 7). */
+ const drainFor = Effect.fn("OpenShellFleetService.drainFor")(function* (
+ client: OpenShellGatewayClient,
+ ) {
+ const context = yield* Effect.context();
+ // A spawned worker has a fleet delegation session, not an ordinary link, so
+ // the drain resolves it the same way MCP forwarding does.
+ const resolvePeer = PeerSessionResolve.resolvePeerSession({
+ links,
+ registry: Option.some(registry),
+ });
+ const linkResolve = (environmentId: EnvironmentId) =>
+ resolvePeer(environmentId).pipe(
+ Effect.mapError(
+ (failure) =>
+ new OpenShellFleetDrain.FleetDrainStepError({
+ step: "shell",
+ message: "The fleet worker could not be resolved for drain.",
+ cause: failure,
+ }),
+ ),
+ );
+ const bundleImporter =
+ Option.isSome(maybeEventSink) && Option.isSome(maybeSql)
+ ? OpenShellFleetBundleImport.makeFleetBundleImporter({
+ stateDir: config.stateDir,
+ attachmentsDir: config.attachmentsDir,
+ fs,
+ path,
+ now: Clock.currentTimeMillis,
+ eventSink: maybeEventSink.value,
+ store: OpenShellFleetBundleImport.sqliteFleetDrainImportStore(maybeSql.value),
+ })
+ : null;
+ const executor = OpenShellFleetDrain.makeDrainExecutor({
+ source: {
+ ...OpenShellFleetDrainAdapters.linkDrainSource({ resolve: linkResolve, http }),
+ // The worker's environment HTTP API serves no attachment bytes, so
+ // attachment files travel over gateway exec (see execDrainAttachments).
+ attachments: OpenShellFleetDrainAdapters.execDrainAttachments({
+ exec: (sandboxName, command) =>
+ Effect.tryPromise({
+ try: () => client.exec(sandboxName, command),
+ catch: (cause) =>
+ new OpenShellFleetDrain.FleetDrainStepError({
+ step: "attachments",
+ message: "The worker's attachments could not be read.",
+ cause,
+ }),
+ }),
+ }),
+ },
+ sink: yield* OpenShellFleetDrainAdapters.fileDrainSink({
+ stateDir: config.stateDir,
+ fs,
+ path,
+ }),
+ git: OpenShellFleetDrainAdapters.execDrainGitBundle({
+ exec: (sandboxName, command) =>
+ Effect.tryPromise({
+ try: () => client.exec(sandboxName, command),
+ catch: (cause) =>
+ new OpenShellFleetDrain.FleetDrainStepError({
+ step: "bundle",
+ message: "The checkout bundle could not be exported.",
+ cause,
+ }),
+ }),
+ }),
+ now: Clock.currentTimeMillis,
+ });
+ const runBundleImport = (args: { environmentId: EnvironmentId; drainId: string }) =>
+ bundleImporter === null
+ ? Promise.resolve({
+ imported: 0,
+ skipped: 0,
+ warnings: [
+ "The orchestration runtime is not built here, so drained bundles were not imported.",
+ ],
+ })
+ : Effect.runPromiseWith(context)(
+ bundleImporter.importBundle({
+ environmentId: args.environmentId,
+ drainId: args.drainId,
+ }),
+ );
+ return {
+ port: OpenShellFleetDrain.asDriverPort(executor, (effect) =>
+ Effect.runPromiseWith(context)(effect),
+ ),
+ bundleImporter: { import: runBundleImport },
+ };
+ });
+
+ const buildDriver = Effect.fn("OpenShellFleetService.buildDriver")(function* (
+ operation: OpenShellFleetOperationFailed["operation"],
+ ) {
+ const { gatewayUrl, openshell } = yield* gatewaySettings;
+ const auth = openshell.auth ?? { mode: "static" };
+ // Only static auth reads the legacy gateway slot. A working mTLS/OAuth
+ // provider must not fail because that optional legacy secret is unreadable.
+ const credential = auth.mode === "static" ? yield* gatewayCredential(operation) : undefined;
+ const client = yield* openClient(operation, gatewayUrl, credential, auth);
+ const context = yield* Effect.context();
+ // The driver's collaborators are Promise-shaped because the OpenShell SDK
+ // is; they run these Effects on the surrounding services rather than a
+ // fresh runtime, so the sqlite store and registry stay the same instances.
+ const run = (effect: Effect.Effect): Promise =>
+ Effect.runPromiseWith(context)(effect);
+ const registrar: WorkerPairingRegistrar = {
+ register: (input) => run(registry.register(input)),
+ exchangeBootstrap: (input) => run(registry.exchangeBootstrap(input)),
+ };
+ const index: FleetWorkerIndex = {
+ upsert: (input) => run(store.upsert(input)).then(toIndexRow),
+ setState: (environmentId, state, lastError) =>
+ run(store.setState(environmentId, state, lastError)).then((row) =>
+ Option.isSome(row) ? toIndexRow(row.value) : null,
+ ),
+ setDrain: (environmentId, drain) =>
+ run(store.setDrain(environmentId, drain)).then((row) =>
+ Option.isSome(row) ? toIndexRow(row.value) : null,
+ ),
+ get: (environmentId) =>
+ run(store.get(environmentId)).then((row) =>
+ Option.isSome(row) ? toIndexRow(row.value) : null,
+ ),
+ list: () => run(store.list).then((rows) => rows.map(toIndexRow)),
+ remove: (environmentId) => run(store.remove(environmentId)),
+ };
+ const selfId = yield* environment.getEnvironmentId.pipe(Effect.option);
+ const driverSettings: OpenShellDriverSettings = {
+ ...(openshell.defaultSandboxTemplate === undefined
+ ? {}
+ : { defaultSandboxTemplate: openshell.defaultSandboxTemplate }),
+ ...(openshell.workerDefaults === undefined
+ ? {}
+ : { workerDefaults: openshell.workerDefaults }),
+ ...(openshell.sharedCache === undefined ? {} : { sharedCache: openshell.sharedCache }),
+ };
+ const drain = yield* drainFor(client);
+ return new OpenShellGatewayDriver({
+ client,
+ settings: driverSettings,
+ ...(Option.isSome(selfId) ? { managingEnvironmentId: selfId.value } : {}),
+ registrar,
+ index,
+ drainExecutor: drain.port,
+ bundleImporter: drain.bundleImporter,
+ // Only a caller's clientRequestId makes a spawn repeatable. Without one
+ // the driver mints a fresh key, so independent spawns never collapse onto
+ // one sandbox and overwrite its policy.
+ });
+ });
+
+ /** A cached build is only reused while the settings guard passes on every call. */
+ const driverFor = Effect.fn("OpenShellFleetService.driverFor")(function* (
+ operation: OpenShellFleetOperationFailed["operation"],
+ ) {
+ const cached = yield* Ref.get(cachedDriver);
+ if (Option.isSome(cached)) return cached.value;
+ const driver = yield* buildDriver(operation);
+ yield* Ref.set(cachedDriver, Option.some(driver));
+ return driver;
+ }, initialization.withPermit);
+
+ const spawnWorker: OpenShellFleetService["Service"]["spawnWorker"] = Effect.fn(
+ "OpenShellFleetService.spawnWorker",
+ )(function* (input): Effect.fn.Return {
+ // `inherit` follows the caller; an omitted mode falls back to the conservative
+ // default the driver documents, never to the caller's ceiling (spec section 5.2).
+ const runtimeMode =
+ input.runtimeMode === undefined
+ ? "approval-required"
+ : input.runtimeMode === "inherit"
+ ? input.runtimeModeCeiling
+ : input.runtimeMode;
+ if (isRuntimeModeAbove(runtimeMode, input.runtimeModeCeiling)) {
+ return yield* new OpenShellFleetSpawnRefused({
+ reason: "runtime_mode_above_ceiling",
+ message: `Worker policy ${runtimeMode} is broader than this caller's ${input.runtimeModeCeiling} ceiling.`,
+ });
+ }
+ // The guard runs per call, so disabling OpenShell after first use also
+ // stops spawns; the driver itself is cached and would outlive a flip.
+ yield* gatewaySettings;
+ const driver = yield* driverFor("spawn");
+ const options: GatewayDriverSpawnOptionsWithCeiling = {
+ ...(input.clientRequestId === undefined ? {} : { idempotencyKey: input.clientRequestId }),
+ runtimeModeCeiling: input.runtimeModeCeiling,
+ };
+ return yield* callDriver(
+ "spawn",
+ () =>
+ driver.spawn(
+ {
+ ...(input.template === undefined ? {} : { template: input.template }),
+ runtimeMode,
+ workerClass: input.workerClass,
+ ...(input.label === undefined ? {} : { label: input.label }),
+ ...(input.ttlMs === undefined ? {} : { ttlMs: input.ttlMs }),
+ ...(input.seedFrom === undefined ? {} : { seedFrom: input.seedFrom }),
+ },
+ options,
+ ),
+ (error) =>
+ error.reason === "runtime_mode_above_ceiling" || error.reason === "unknown_template"
+ ? new OpenShellFleetSpawnRefused({ reason: error.reason, message: error.message })
+ : undefined,
+ );
+ });
+
+ const removeWorker: OpenShellFleetService["Service"]["removeWorker"] = Effect.fn(
+ "OpenShellFleetService.removeWorker",
+ )(function* (input): Effect.fn.Return {
+ yield* gatewaySettings;
+ const driver = yield* driverFor("remove");
+ const options: GatewayDriverRemoveOptions = {
+ ...(input.drainTimeoutMs === undefined ? {} : { timeoutMs: input.drainTimeoutMs }),
+ ...(input.force === undefined ? {} : { force: input.force }),
+ ...(input.clientRequestId === undefined ? {} : { idempotencyKey: input.clientRequestId }),
+ };
+ return yield* callDriver("remove", () => driver.remove(input.targetEnvironmentId, options));
+ });
+
+ const listWorkers: OpenShellFleetService["Service"]["listWorkers"] = Effect.fn(
+ "OpenShellFleetService.listWorkers",
+ )(function* (input): Effect.fn.Return<
+ ReadonlyArray,
+ OpenShellFleetError
+ > {
+ yield* gatewaySettings;
+ const driver = yield* driverFor("list");
+ return yield* callDriver("list", () =>
+ driver.status(input.workerClass === undefined ? {} : { workerClass: input.workerClass }),
+ );
+ });
+
+ const sweepWorkers: OpenShellFleetService["Service"]["sweepWorkers"] = Effect.fn(
+ "OpenShellFleetService.sweepWorkers",
+ )(function* (): Effect.fn.Return {
+ // The per-call gate makes an off fleet a no-op tick rather than an error, so
+ // the sweep below can keep its cadence through settings changes.
+ yield* gatewaySettings;
+ const driver = yield* driverFor("sweep");
+ return yield* callDriver("sweep", () => driver.orphanSweep());
+ });
+
+ return OpenShellFleetService.of({ spawnWorker, removeWorker, listWorkers, sweepWorkers });
+});
+
+export const layer = Layer.effect(OpenShellFleetService, make);
+
+/**
+ * Self-retiring ephemeral workers (spec section 5.6): one sweep every 5 minutes
+ * retires expired ephemeral workers drain-first and finalizes vanished rows.
+ * Refused drains keep their sandbox, so the next tick retries them.
+ */
+export const layerSweep = Layer.effectDiscard(
+ forkParked(
+ OpenShellFleetService.pipe(
+ Effect.flatMap((fleet) => fleet.sweepWorkers()),
+ Effect.catch((cause) =>
+ cause._tag === "OpenShellFleetDisabled"
+ ? Effect.void
+ : Effect.logWarning("The OpenShell fleet sweep did not complete", { cause }),
+ ),
+ Effect.repeat(Schedule.spaced(Duration.minutes(5))),
+ ),
+ ),
+);
+
+/**
+ * No fleet here: tests and builds that do not run a gateway. Every verb fails
+ * typed, so a tool that reaches it reports "off" rather than inventing workers.
+ */
+export const layerDisabled: Layer.Layer = Layer.succeed(
+ OpenShellFleetService,
+ OpenShellFleetService.of({
+ spawnWorker: Effect.fn("OpenShellFleetService.spawnWorker.disabled")(() =>
+ Effect.fail(new OpenShellFleetDisabled({ message: offHere })),
+ ),
+ removeWorker: Effect.fn("OpenShellFleetService.removeWorker.disabled")(() =>
+ Effect.fail(new OpenShellFleetDisabled({ message: offHere })),
+ ),
+ listWorkers: Effect.fn("OpenShellFleetService.listWorkers.disabled")(() =>
+ Effect.fail(new OpenShellFleetDisabled({ message: offHere })),
+ ),
+ sweepWorkers: Effect.fn("OpenShellFleetService.sweepWorkers.disabled")(() =>
+ Effect.fail(new OpenShellFleetDisabled({ message: offHere })),
+ ),
+ }),
+);
diff --git a/apps/server/src/environment/OpenShellFleetStore.test.ts b/apps/server/src/environment/OpenShellFleetStore.test.ts
new file mode 100644
index 0000000000000..20e212404a949
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetStore.test.ts
@@ -0,0 +1,133 @@
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { EnvironmentId } from "@t3tools/contracts";
+import { assert, describe, it } from "@effect/vitest";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+import * as FileSystem from "effect/FileSystem";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Path from "effect/Path";
+
+import * as OpenShellFleetStore from "./OpenShellFleetStore.ts";
+import * as Sqlite from "../persistence/Sqlite.ts";
+
+const workerRow = {
+ environmentId: EnvironmentId.make("environment-worker-1"),
+ state: "ready" as const,
+ workerClass: "ephemeral" as const,
+ label: "Worker One",
+ sandboxName: "t3-worker-7f3a",
+ sandboxId: "sandbox-1",
+ expiresAt: 1_800_000_000_000,
+};
+
+const drain = {
+ drainId: "drain:environment-worker-1:1",
+ threads: { total: 2, acked: 1, failed: 0 },
+ worktrees: { total: 1, acked: 0, failed: 0 },
+ startedAt: 1_700_000_000_000,
+ finishedAt: null,
+};
+
+const storeLayer = (database: Layer.Layer) =>
+ OpenShellFleetStore.layer.pipe(Layer.provideMerge(database));
+
+describe("OpenShellFleetStore", () => {
+ it.effect("keeps one row per environment across state and drain updates", () =>
+ Effect.gen(function* () {
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ const created = yield* store.upsert(workerRow);
+ assert.equal(created.state, "ready");
+ assert.isNull(created.drain);
+ assert.equal(created.expiresAt, workerRow.expiresAt);
+ assert.isTrue(DateTime.isDateTime(created.updatedAt));
+
+ const draining = yield* store.setState(workerRow.environmentId, "draining");
+ assert.equal(Option.getOrThrow(draining).state, "draining");
+
+ const withDrain = yield* store.setDrain(workerRow.environmentId, drain);
+ assert.deepEqual(Option.getOrThrow(withDrain).drain, drain);
+
+ // The drain clears once every item acked and the deletion settled.
+ const deleted = yield* store.setState(workerRow.environmentId, "deleting");
+ assert.equal(Option.getOrThrow(deleted).state, "deleting");
+ const cleared = yield* store.setDrain(workerRow.environmentId, null);
+ assert.isNull(Option.getOrThrow(cleared).drain);
+
+ assert.equal((yield* store.list).length, 1);
+ assert.isTrue(yield* store.remove(workerRow.environmentId));
+ assert.isTrue(Option.isNone(yield* store.get(workerRow.environmentId)));
+ assert.isFalse(yield* store.remove(workerRow.environmentId));
+ }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))),
+ );
+
+ it.effect("a failed drain keeps its last error with the row", () =>
+ Effect.gen(function* () {
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ yield* store.upsert(workerRow);
+ const failed = yield* store.setState(
+ workerRow.environmentId,
+ "failed",
+ "drain_timeout: 1 worktree unacked",
+ );
+ assert.equal(Option.getOrThrow(failed).lastError, "drain_timeout: 1 worktree unacked");
+ // Nothing lost outranks cleanup: a failed row is never interrupted-by-restart.
+ assert.deepEqual(yield* store.interrupted, []);
+ }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))),
+ );
+
+ it.effect("reports rows a restart interrupted, and only those", () =>
+ Effect.gen(function* () {
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ yield* store.upsert(workerRow);
+ yield* store.upsert({
+ ...workerRow,
+ environmentId: EnvironmentId.make("environment-worker-2"),
+ state: "draining" as const,
+ label: "Worker Two",
+ });
+ const interrupted = yield* store.interrupted;
+ assert.deepEqual(
+ interrupted.map((row) => row.environmentId),
+ [EnvironmentId.make("environment-worker-2")],
+ );
+ assert.equal(interrupted[0]?.state, "draining");
+ }).pipe(Effect.provide(storeLayer(Sqlite.layerMemory))),
+ );
+
+ it.effect("state survives a restart of the managing instance", () =>
+ Effect.gen(function* () {
+ const fs = yield* FileSystem.FileSystem;
+ const path = yield* Path.Path;
+ const dir = yield* fs.makeTempDirectory({ prefix: "t3-fleet-store-" });
+ const dbPath = path.join(dir, "statev2.sqlite");
+ const database = Sqlite.layerFromPath(dbPath).pipe(
+ Layer.provide(NodeServices.layer),
+ Layer.fresh,
+ );
+
+ // One instance writes and exits.
+ yield* Effect.gen(function* () {
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ yield* store.upsert(workerRow);
+ yield* store.setState(workerRow.environmentId, "draining");
+ yield* store.setDrain(workerRow.environmentId, drain);
+ }).pipe(Effect.provide(storeLayer(database)));
+
+ // The next instance reads the rows back at startup.
+ yield* Effect.gen(function* () {
+ const store = yield* OpenShellFleetStore.OpenShellFleetStore;
+ const recovered = Option.getOrThrow(yield* store.get(workerRow.environmentId));
+ assert.equal(recovered.state, "draining");
+ assert.deepEqual(recovered.drain, drain);
+ assert.deepEqual(
+ (yield* store.interrupted).map((row) => row.environmentId),
+ [workerRow.environmentId],
+ );
+ }).pipe(Effect.provide(storeLayer(database)));
+
+ yield* fs.remove(dir, { recursive: true });
+ }).pipe(Effect.provide(NodeServices.layer)),
+ );
+});
diff --git a/apps/server/src/environment/OpenShellFleetStore.ts b/apps/server/src/environment/OpenShellFleetStore.ts
new file mode 100644
index 0000000000000..ebacf9f21971d
--- /dev/null
+++ b/apps/server/src/environment/OpenShellFleetStore.ts
@@ -0,0 +1,286 @@
+import {
+ EnvironmentId,
+ OpenShellDrainProgress,
+ OpenShellFleetState,
+ OpenShellWorkerClass,
+} from "@t3tools/contracts";
+import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
+import * as Context from "effect/Context";
+import * as DateTime from "effect/DateTime";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Schema from "effect/Schema";
+import * as SqlClient from "effect/sql/SqlClient";
+
+const decodeDrain = Schema.decodeUnknownEffect(
+ Schema.NullOr(Schema.fromJsonString(OpenShellDrainProgress)),
+);
+const encodeDrain = Schema.encodeEffect(
+ Schema.NullOr(Schema.fromJsonString(OpenShellDrainProgress)),
+);
+
+/** One gateway-managed environment as the fleet lifecycle store keeps it. */
+export interface FleetEnvironment {
+ readonly environmentId: EnvironmentId;
+ readonly state: OpenShellFleetState;
+ readonly workerClass: OpenShellWorkerClass;
+ readonly label: string | null;
+ readonly sandboxName: string | null;
+ readonly sandboxId: string | null;
+ /** Aggregate drain journal progress; per-item acks live with the drain protocol. */
+ readonly drain: OpenShellDrainProgress | null;
+ /** Ephemeral workers: epoch-ms retirement deadline managed by this instance. */
+ readonly expiresAt: number | null;
+ readonly lastError: string | null;
+ readonly updatedAt: DateTime.Utc;
+}
+
+export class OpenShellFleetStoreError extends Schema.TaggedError<
+ OpenShellFleetStoreError,
+ { readonly brand: unique symbol }
+>()("OpenShellFleetStoreError", {
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+const failStorage = (cause: unknown) =>
+ Effect.logWarning("OpenShell fleet state storage failed").pipe(
+ Effect.andThen(
+ Effect.fail(
+ new OpenShellFleetStoreError({
+ message:
+ "The fleet environment state could not be read or saved here. The server log has details.",
+ cause,
+ }),
+ ),
+ ),
+ );
+
+/**
+ * Lifecycle states a restart interrupts. The spec's reconciler (section 5.5)
+ * settles them against the gateway once a driver exists; until then the store
+ * surfaces them, so nothing is silently dropped and nothing is deleted.
+ */
+const interruptedStates: ReadonlyArray = [
+ "provisioning",
+ "pairing",
+ "draining",
+ "deleting",
+];
+
+/**
+ * Fleet lifecycle state for gateway-managed environments (spec section 5.3):
+ * the row, not the gateway phase, is this instance's truth, so state survives
+ * a restart. Writes are upserts keyed by environment id.
+ */
+export class OpenShellFleetStore extends Context.Service<
+ OpenShellFleetStore,
+ {
+ readonly upsert: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly state: OpenShellFleetState;
+ readonly workerClass: OpenShellWorkerClass;
+ readonly label?: string;
+ readonly sandboxName?: string;
+ readonly sandboxId?: string;
+ readonly expiresAt?: number;
+ }) => Effect.Effect;
+ readonly setState: (
+ environmentId: EnvironmentId,
+ state: OpenShellFleetState,
+ lastError?: string,
+ ) => Effect.Effect, OpenShellFleetStoreError>;
+ readonly setDrain: (
+ environmentId: EnvironmentId,
+ drain: OpenShellDrainProgress | null,
+ ) => Effect.Effect, OpenShellFleetStoreError>;
+ readonly get: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect, OpenShellFleetStoreError>;
+ readonly list: Effect.Effect, OpenShellFleetStoreError>;
+ /**
+ * Rows the restart interrupted (spec section 5.5): still owed their drain
+ * or deletion. OPEN-QUESTION (spec section 5.5): adopt-or-sweep lands
+ * with the gateway driver; the default keeps every interrupted row.
+ */
+ readonly interrupted: Effect.Effect, OpenShellFleetStoreError>;
+ /** Forgets the row; the caller drains first (spec section 7) and deletes the sandbox itself. */
+ readonly remove: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect;
+ }
+>()("t3/environment/OpenShellFleetStore") {}
+
+interface FleetRow {
+ readonly environment_id: string;
+ readonly state: string;
+ readonly worker_class: string;
+ readonly label: string | null;
+ readonly sandbox_name: string | null;
+ readonly sandbox_id: string | null;
+ readonly drain_json: string | null;
+ readonly expires_at: number | null;
+ readonly last_error: string | null;
+ readonly updated_at: string;
+}
+
+const decodeState = Schema.decodeUnknownEffect(OpenShellFleetState);
+const decodeWorkerClass = Schema.decodeUnknownEffect(Schema.Literals(["ephemeral", "persistent"]));
+
+const toFleetEnvironment = Effect.fn("OpenShellFleetStore.toFleetEnvironment")(function* (
+ row: FleetRow,
+) {
+ const state = yield* decodeState(row.state).pipe(Effect.catch(failStorage));
+ const workerClass = yield* decodeWorkerClass(row.worker_class).pipe(Effect.catch(failStorage));
+ return {
+ environmentId: EnvironmentId.make(row.environment_id),
+ state,
+ workerClass,
+ label: row.label,
+ sandboxName: row.sandbox_name,
+ sandboxId: row.sandbox_id,
+ drain: yield* decodeDrain(row.drain_json).pipe(Effect.catch(failStorage)),
+ expiresAt: row.expires_at,
+ lastError: row.last_error,
+ updatedAt: DateTime.makeUnsafe(row.updated_at),
+ } satisfies FleetEnvironment;
+});
+
+const make = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ const readRows = sql`
+ SELECT * FROM openshell_fleet_environments ORDER BY environment_id
+ `.pipe(NodeSqliteClient.readOnly, Effect.catch(failStorage));
+
+ const readRow = (environmentId: EnvironmentId) =>
+ sql`
+ SELECT * FROM openshell_fleet_environments WHERE environment_id = ${environmentId}
+ `.pipe(
+ NodeSqliteClient.readOnly,
+ Effect.catch(failStorage),
+ Effect.map((rows) => Option.fromNullishOr(rows[0])),
+ Effect.withSpan("OpenShellFleetStore.readRow"),
+ );
+
+ // Crash recovery: what a restart interrupted, read back once at startup so
+ // the row states this instance owed are visible without a driver (spec
+ // section 5.5). Nothing is mutated: a kept state is a kept obligation.
+ yield* Effect.forEach(yield* readRows, toFleetEnvironment).pipe(
+ Effect.tap((rows) => {
+ const interruptedRows = rows.filter((row) => interruptedStates.includes(row.state));
+ return interruptedRows.length > 0
+ ? Effect.logWarning("Fleet environments interrupted by this restart keep their state", {
+ environments: interruptedRows.map((row) => ({
+ environmentId: row.environmentId,
+ state: row.state,
+ })),
+ })
+ : Effect.void;
+ }),
+ Effect.catch(failStorage),
+ );
+
+ const upsert: OpenShellFleetStore["Service"]["upsert"] = Effect.fn("OpenShellFleetStore.upsert")(
+ function* (input) {
+ const now = yield* DateTime.now;
+ yield* sql`
+ INSERT INTO openshell_fleet_environments (
+ environment_id, state, worker_class, label, sandbox_name, sandbox_id,
+ drain_json, expires_at, last_error, updated_at
+ ) VALUES (
+ ${input.environmentId}, ${input.state}, ${input.workerClass},
+ ${input.label ?? null}, ${input.sandboxName ?? null}, ${input.sandboxId ?? null},
+ NULL, ${input.expiresAt ?? null}, NULL, ${DateTime.formatIso(now)}
+ )
+ ON CONFLICT (environment_id) DO UPDATE SET
+ state = excluded.state,
+ worker_class = excluded.worker_class,
+ label = excluded.label,
+ sandbox_name = excluded.sandbox_name,
+ sandbox_id = excluded.sandbox_id,
+ expires_at = excluded.expires_at,
+ last_error = excluded.last_error,
+ updated_at = excluded.updated_at
+ `.pipe(Effect.catch(failStorage));
+ return Option.getOrThrow(yield* get(input.environmentId));
+ },
+ );
+
+ const get: OpenShellFleetStore["Service"]["get"] = (environmentId) =>
+ readRow(environmentId).pipe(
+ Effect.flatMap(
+ Option.match({
+ onNone: () => Effect.succeedNone,
+ onSome: (row) => Effect.asSome(toFleetEnvironment(row)),
+ }),
+ ),
+ Effect.withSpan("OpenShellFleetStore.get"),
+ );
+
+ const setState: OpenShellFleetStore["Service"]["setState"] = (environmentId, state, lastError) =>
+ DateTime.now.pipe(
+ Effect.flatMap((now) =>
+ sql<{ environment_id: string }>`
+ UPDATE openshell_fleet_environments
+ SET state = ${state}, last_error = ${lastError ?? null},
+ updated_at = ${DateTime.formatIso(now)}
+ WHERE environment_id = ${environmentId}
+ RETURNING environment_id
+ `.pipe(Effect.catch(failStorage)),
+ ),
+ Effect.flatMap((updated) => (updated.length > 0 ? get(environmentId) : Effect.succeedNone)),
+ Effect.withSpan("OpenShellFleetStore.setState"),
+ );
+
+ const setDrain: OpenShellFleetStore["Service"]["setDrain"] = (environmentId, drain) =>
+ encodeDrain(drain).pipe(
+ Effect.catch(failStorage),
+ Effect.flatMap((drainJson) =>
+ DateTime.now.pipe(
+ Effect.flatMap((now) =>
+ sql<{ environment_id: string }>`
+ UPDATE openshell_fleet_environments
+ SET drain_json = ${drainJson}, updated_at = ${DateTime.formatIso(now)}
+ WHERE environment_id = ${environmentId}
+ RETURNING environment_id
+ `.pipe(Effect.catch(failStorage)),
+ ),
+ ),
+ ),
+ Effect.flatMap((updated) => (updated.length > 0 ? get(environmentId) : Effect.succeedNone)),
+ Effect.withSpan("OpenShellFleetStore.setDrain"),
+ );
+
+ const list: OpenShellFleetStore["Service"]["list"] = readRows.pipe(
+ Effect.flatMap((rows) => Effect.forEach(rows, toFleetEnvironment)),
+ Effect.withSpan("OpenShellFleetStore.list"),
+ );
+
+ const interruptedRows = Effect.map(list, (rows) =>
+ rows.filter((row) => interruptedStates.includes(row.state)),
+ );
+
+ const remove: OpenShellFleetStore["Service"]["remove"] = (environmentId) =>
+ sql<{ environment_id: string }>`
+ DELETE FROM openshell_fleet_environments WHERE environment_id = ${environmentId}
+ RETURNING environment_id
+ `.pipe(
+ Effect.map((deleted) => deleted.length > 0),
+ Effect.catch(failStorage),
+ Effect.withSpan("OpenShellFleetStore.remove"),
+ );
+
+ return OpenShellFleetStore.of({
+ upsert,
+ setState,
+ setDrain,
+ get,
+ list,
+ interrupted: interruptedRows,
+ remove,
+ });
+});
+
+export const layer = Layer.effect(OpenShellFleetStore, make);
diff --git a/apps/server/src/environment/OpenShellGatewayAuth.ts b/apps/server/src/environment/OpenShellGatewayAuth.ts
new file mode 100644
index 0000000000000..9d46cafb5dd85
--- /dev/null
+++ b/apps/server/src/environment/OpenShellGatewayAuth.ts
@@ -0,0 +1,260 @@
+// @effect-diagnostics globalDate:off - the token provider is a Promise-shaped
+// collaborator of the SDK (like the driver and the smoke script), so its cache
+// checks wall-clock time outside the Effect runtime.
+import type { OpenShellGatewayAuth, SettingsSecretValue } from "@t3tools/contracts";
+import * as SecretResolverCore from "@t3tools/secret-provider-core/server/SecretResolver";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+
+import * as Schema from "effect/Schema";
+
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
+import { openshellGatewayAuthSecretName } from "../settingsSecrets.ts";
+
+/**
+ * Turns the settings' `OpenShellGatewayAuth` into the SDK's `oidcTokenProvider`
+ * seam — an async bearer getter the client renews through. Secret literals live
+ * in the server secret store (settings JSON keeps the redacted placeholder) and
+ * secret-provider references are read through the resolver (spec section 4.2).
+ */
+
+const TOKEN_EXPIRY_MARGIN_MS = 60_000;
+const REDACTED = "\u2022\u2022\u2022\u2022\u2022\u2022";
+const GATEWAY_CREDENTIAL_SLOT = "openshell-gateway-credential";
+
+export interface GatewayAuthTokenProviderDeps {
+ readonly auth: OpenShellGatewayAuth;
+ readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
+ readonly resolver: SecretResolverCore.SecretResolver["Service"];
+ /** Overridable for tests. */
+ readonly fetchImpl?: typeof fetch;
+}
+
+interface CachedToken {
+ readonly token: string;
+ readonly expiresAtMs: number;
+}
+
+/** Gateway auth failures: typed so a fleet operation maps them to its own error. */
+export class GatewayAuthError extends Schema.TaggedError()("GatewayAuthError", {
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+}) {}
+
+const isRedacted = (value: SettingsSecretValue): boolean =>
+ typeof value === "string" && value === REDACTED;
+
+export const makeGatewayAuthTokenProvider = Effect.fn("OpenShellGatewayAuth.tokenProvider")(
+ function* (deps: GatewayAuthTokenProviderDeps) {
+ const { auth, secrets, resolver, fetchImpl } = deps;
+ const doFetch = fetchImpl ?? fetch;
+ const decoder = new TextDecoder();
+ /** Promise collaborators run with the same services as this Effect. */
+ const runtime = yield* Effect.context();
+
+ const resolveSecretField = Effect.fn("OpenShellGatewayAuth.resolveSecretField")(function* (
+ field: string,
+ value: SettingsSecretValue,
+ ) {
+ if (isRedacted(value)) {
+ const stored = yield* secrets.get(openshellGatewayAuthSecretName(field));
+ const secret = Option.isSome(stored) ? decoder.decode(stored.value) : "";
+ if (secret === "")
+ return yield* new GatewayAuthError({
+ message: `The gateway auth secret "${field}" is not set.`,
+ });
+ return secret;
+ }
+ const secret = yield* requireSettingsSecret(resolver, value).pipe(
+ Effect.catch(
+ (cause) =>
+ new GatewayAuthError({
+ message: `The gateway auth secret "${field}" could not be read.`,
+ cause,
+ }),
+ ),
+ );
+ if (secret.trim() === "")
+ return yield* new GatewayAuthError({
+ message: `The gateway auth secret "${field}" is empty.`,
+ });
+ return secret;
+ });
+
+ let cached: CachedToken | undefined;
+ let tokenEndpoint: string | undefined;
+ let refreshInFlight: Promise | undefined;
+ const usable = (now: number): boolean =>
+ cached !== undefined && cached.expiresAtMs - TOKEN_EXPIRY_MARGIN_MS > now;
+ const currentToken = (): string => {
+ const snapshot = cached;
+ if (snapshot === undefined || !usable(Date.now()))
+ throw new Error("The gateway token needs renewal.");
+ return snapshot.token;
+ };
+
+ if (auth.mode === "mtls") {
+ // Lane 1 auth docs: the SDK connect accepts caCert + clientCert +
+ // clientKey for client-CA gateways. Field shapes still need
+ // type-level verification against SDK 0.1.7.
+ const clientKey = yield* resolveSecretField("clientKey", auth.clientKey);
+ return {
+ connectOptions: {
+ ...(auth.caCertPem === undefined ? {} : { caCert: auth.caCertPem }),
+ clientCert: auth.clientCertPem,
+ clientKey,
+ },
+ };
+ }
+
+ /** OAuth2 client-credentials or refresh-token grant against the OIDC issuer. */
+ const fetchToken = async (
+ issuer: string,
+ audience: string | undefined,
+ form: URLSearchParams,
+ ): Promise => {
+ if (tokenEndpoint === undefined) {
+ // Preserve tenant paths: discovery is relative to the issuer, not its origin.
+ const issuerUrl = new URL(issuer);
+ const issuerBase = issuerUrl.pathname.endsWith("/") ? issuer : `${issuer}/`;
+ const discovery = await doFetch(new URL(".well-known/openid-configuration", issuerBase), {
+ redirect: "error",
+ });
+ if (!discovery.ok)
+ throw new Error(`OIDC discovery at ${issuer} failed with status ${discovery.status}.`);
+ const document = (await discovery.json()) as {
+ readonly issuer?: unknown;
+ readonly token_endpoint?: unknown;
+ };
+ if (typeof document.token_endpoint !== "string" || document.token_endpoint === "")
+ throw new Error(`OIDC discovery at ${issuer} has no token endpoint.`);
+ const tokenUrl = new URL(document.token_endpoint);
+ const secureEndpoint = (url: URL) =>
+ url.protocol === "https:" ||
+ (url.protocol === "http:" && ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname));
+ if (!secureEndpoint(issuerUrl) || !secureEndpoint(tokenUrl))
+ throw new Error("OIDC discovery requires HTTPS endpoints outside local development.");
+ if (
+ typeof document.issuer === "string" &&
+ document.issuer.replace(/\/$/, "") !== issuer.replace(/\/$/, "")
+ )
+ throw new Error(`OIDC discovery at ${issuer} identifies a different issuer.`);
+ tokenEndpoint = document.token_endpoint;
+ }
+ const response = await doFetch(tokenEndpoint, {
+ redirect: "error",
+ method: "POST",
+ headers: { "content-type": "application/x-www-form-urlencoded" },
+ body: form.toString(),
+ });
+ if (!response.ok) throw new Error(`The gateway token endpoint answered ${response.status}.`);
+ const payload = (await response.json()) as {
+ readonly access_token?: unknown;
+ readonly expires_in?: unknown;
+ readonly refresh_token?: unknown;
+ };
+ if (typeof payload.access_token !== "string" || payload.access_token === "")
+ throw new Error("The gateway token endpoint returned no access token.");
+ const lifetime =
+ typeof payload.expires_in === "number" && payload.expires_in > 0
+ ? payload.expires_in * 1000
+ : 300_000;
+ cached = { token: payload.access_token, expiresAtMs: Date.now() + lifetime };
+ // Rotation is opt-in at the protocol level. For a stored literal, rotate
+ // the server slot immediately; an external secret owner is authoritative
+ // and cannot be silently overwritten here.
+ if (
+ auth.mode === "oidc-pkce" &&
+ typeof payload.refresh_token === "string" &&
+ payload.refresh_token !== "" &&
+ isRedacted(auth.refreshToken)
+ ) {
+ await Effect.runPromiseWith(runtime)(
+ secrets
+ .set(
+ openshellGatewayAuthSecretName("refreshToken"),
+ new TextEncoder().encode(payload.refresh_token),
+ )
+ .pipe(
+ Effect.catch(
+ (cause) =>
+ new GatewayAuthError({
+ message:
+ "The issuer rotated the gateway refresh token, but its server slot could not be updated.",
+ cause,
+ }),
+ ),
+ ),
+ );
+ }
+ return cached.token;
+ };
+
+ if (auth.mode === "client-credentials") {
+ const clientSecret = yield* resolveSecretField("clientSecret", auth.clientSecret);
+ const issuer = auth.issuer;
+ const audience = auth.audience;
+ return {
+ oidcTokenProvider: (): Promise => {
+ if (usable(Date.now())) return Promise.resolve(currentToken());
+ const form = new URLSearchParams({
+ grant_type: "client_credentials",
+ client_id: auth.clientId,
+ client_secret: clientSecret,
+ });
+ if (audience !== undefined) form.set("audience", audience);
+ return fetchToken(issuer, audience, form);
+ },
+ };
+ }
+
+ if (auth.mode === "oidc-pkce") {
+ // The desktop performed the browser login; the server holds the refresh
+ // token and renews with the refresh-token grant (spec section 4.1).
+ const refreshToken = yield* resolveSecretField("refreshToken", auth.refreshToken);
+ const configuredIssuer = auth.issuer;
+ if (configuredIssuer === undefined || configuredIssuer.trim() === "") {
+ return yield* new GatewayAuthError({
+ message: "OIDC refresh authentication requires an issuer.",
+ });
+ }
+ const issuer = configuredIssuer;
+ const audience = auth.audience;
+ return {
+ oidcTokenProvider: (): Promise => {
+ if (usable(Date.now())) return Promise.resolve(currentToken());
+ // OAuth refresh codes are single-use: serialize callers so concurrent
+ // operations cannot spend the same token twice.
+ refreshInFlight ??= (async () => {
+ try {
+ const form = new URLSearchParams({
+ grant_type: "refresh_token",
+ refresh_token: refreshToken,
+ });
+ if (issuer !== undefined) form.set("issuer", issuer);
+ if (audience !== undefined) form.set("audience", audience);
+ return await fetchToken(issuer, audience, form);
+ } finally {
+ refreshInFlight = undefined;
+ }
+ })();
+ return refreshInFlight;
+ },
+ };
+ }
+
+ // static
+ // Legacy provisioned bearer: read the slot on every token callback so a
+ // provisioner can replace an expired credential without rebuilding driver.
+ const oidcTokenProvider = (): Promise =>
+ Effect.runPromiseWith(runtime)(
+ secrets
+ .get(GATEWAY_CREDENTIAL_SLOT)
+ .pipe(
+ Effect.map((stored) => (Option.isSome(stored) ? decoder.decode(stored.value) : "")),
+ ),
+ );
+ return { oidcTokenProvider };
+ },
+);
diff --git a/apps/server/src/mcp/McpHttpServer.test.ts b/apps/server/src/mcp/McpHttpServer.test.ts
index e5d47410817f2..c4290d295033d 100644
--- a/apps/server/src/mcp/McpHttpServer.test.ts
+++ b/apps/server/src/mcp/McpHttpServer.test.ts
@@ -60,6 +60,7 @@ import * as McpToolAccessTestkit from "./McpToolAccess.testkit.ts";
import * as McpInvocationContext from "./McpInvocationContext.ts";
import * as McpSessionRegistry from "./McpSessionRegistry.ts";
import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as PreviewAutomationBroker from "./PreviewAutomationBroker.ts";
import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts";
import { PreviewToolkit } from "./toolkits/preview/tools.ts";
@@ -334,6 +335,7 @@ const layerRecordingTools = (calls: Array) => {
Layer.mergeAll(
McpToolAccessTestkit.liveThreadsLayer,
Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({}),
+ Layer.mock(OpenShellFleetService.OpenShellFleetService)({}),
Layer.mock(ProjectService.ProjectService)({}),
Layer.mock(PreviewManager.PreviewManager)({}),
Layer.mock(PreviewAutomationBroker.PreviewAutomationBroker)({}),
diff --git a/apps/server/src/mcp/McpHttpServer.ts b/apps/server/src/mcp/McpHttpServer.ts
index d7bea75493b73..c31aaeadf67b3 100644
--- a/apps/server/src/mcp/McpHttpServer.ts
+++ b/apps/server/src/mcp/McpHttpServer.ts
@@ -282,6 +282,17 @@ const makeMcpAuthMiddleware = Effect.gen(function* () {
Effect.map((settings) => settings.linkedEnvironmentsEnabled),
Effect.orElseSucceed(() => false),
);
+ // The fleet tools cost every agent schema bytes, so they are hidden where no
+ // fleet can be managed rather than failing on call.
+ const fleetAvailable = serverSettings.getSettings.pipe(
+ Effect.map(
+ (settings) =>
+ settings.openshellEnabled &&
+ settings.openshell.managingInstance &&
+ (settings.openshell.gatewayUrl ?? "").trim() !== "",
+ ),
+ Effect.orElseSucceed(() => false),
+ );
const pullRequests = yield* Effect.serviceOption(PullRequestService.PullRequestService);
const projections = yield* Effect.serviceOption(ProjectionStore.ProjectionStoreV2);
// Every request reads it, and a remote added mid-thread shows the tools within a minute.
@@ -337,7 +348,13 @@ const makeMcpAuthMiddleware = Effect.gen(function* () {
limited.thread === undefined
? undefined
: yield* Cache.get(forgeOfThread, limited.thread.threadId);
- return { ...limited, linkedEnvironments, origin, ...(forge === undefined ? {} : { forge }) };
+ return {
+ ...limited,
+ linkedEnvironments,
+ origin,
+ fleetAvailable: yield* fleetAvailable,
+ ...(forge === undefined ? {} : { forge }),
+ };
});
// Untraced: a span here would wrap the tool call; the HTTP server span already covers it.
return Effect.fnUntraced(function* (httpEffect) {
diff --git a/apps/server/src/mcp/McpInvocationContext.test.ts b/apps/server/src/mcp/McpInvocationContext.test.ts
index 96c71fcd773f9..00e8e0af52a06 100644
--- a/apps/server/src/mcp/McpInvocationContext.test.ts
+++ b/apps/server/src/mcp/McpInvocationContext.test.ts
@@ -7,8 +7,11 @@ import {
ThreadId,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
+import * as Context from "effect/Context";
+import { McpSchema } from "effect/ai";
import * as McpInvocationContext from "./McpInvocationContext.ts";
+import { OrchestratorToolkit } from "./toolkits/orchestrator/tools.ts";
it.effect("reports other missing capabilities with the neutral error", () => {
const invocation: McpInvocationContext.McpInvocationScope = {
@@ -95,3 +98,45 @@ it("acts at an administrator's ceiling for a client approved above it", () => {
expect(modes(scope("auto-accept-edits"), "auto").runtimeMode).toBe("auto-accept-edits");
expect(modes(scope("full-access")).runtimeMode).toBe("full-access");
});
+
+it.effect("hides fleet tools unless fleet management is explicitly available", () => {
+ const scope = (fleetAvailable?: boolean): McpInvocationContext.McpInvocationScope => ({
+ environmentId: EnvironmentId.make("environment-1"),
+ requestNamespace: "client:session-1",
+ thread: undefined,
+ client: undefined,
+ capabilities: new Set(["orchestration"]),
+ issuedAt: 1,
+ ...(fleetAvailable === undefined ? {} : { fleetAvailable }),
+ });
+ const visible = (fleetAvailable?: boolean) =>
+ Effect.sync(() => McpInvocationContext.hiddenWhileFleetOff()).pipe(
+ Effect.provideService(McpInvocationContext.McpInvocationContext, scope(fleetAvailable)),
+ );
+
+ return Effect.gen(function* () {
+ expect(yield* visible(false)).toBe(false);
+ expect(yield* visible(true)).toBe(true);
+ expect(yield* visible(undefined)).toBe(false);
+
+ const enabledWhen = (name: string): boolean => {
+ const tool = OrchestratorToolkit.tools[name as keyof typeof OrchestratorToolkit.tools];
+ return Context.getOrUndefined(tool.annotations, McpSchema.EnabledWhen) !== undefined;
+ };
+ for (const fleetTool of [
+ "t3_environment_spawn",
+ "t3_environment_remove",
+ "t3_environment_list",
+ ]) {
+ expect(enabledWhen(fleetTool)).toBe(true);
+ }
+ for (const coreTool of [
+ "orchestrator_capabilities",
+ "t3_thread_list",
+ "list_scheduled_tasks",
+ "t3_thread_wait",
+ ]) {
+ expect(enabledWhen(coreTool)).toBe(false);
+ }
+ });
+});
diff --git a/apps/server/src/mcp/McpInvocationContext.ts b/apps/server/src/mcp/McpInvocationContext.ts
index 893c17f23311b..153a4986269f8 100644
--- a/apps/server/src/mcp/McpInvocationContext.ts
+++ b/apps/server/src/mcp/McpInvocationContext.ts
@@ -152,6 +152,11 @@ export interface McpInvocationScope {
readonly client: McpClientCaller | undefined;
/** False while the Linked environments beta is off here; `/mcp` reads it per request. */
readonly linkedEnvironments?: boolean;
+ /**
+ * Whether this instance can manage a fleet: the OpenShell block is on and this
+ * environment is the managing instance. Unset means unknown, so tools show.
+ */
+ readonly fleetAvailable?: boolean;
/**
* The origin this `/mcp` request reached, e.g. a T3 Connect tunnel. Hosted clients cannot
* see the URL they were configured with, so links they must fetch carry it. Unset in-process.
@@ -193,6 +198,13 @@ export const hiddenFromSubagents = (): boolean => {
return scope?.thread?.subagent !== true;
};
+/** `McpSchema.EnabledWhen` for the fleet tools, hidden where no fleet can be managed. */
+export const hiddenWhileFleetOff = (): boolean => {
+ const fiber = Fiber.getCurrent();
+ const scope = fiber && Context.getOrUndefined(fiber.context, McpInvocationContext);
+ return scope?.fleetAvailable === true;
+};
+
/**
* `McpSchema.EnabledWhen` for tools that act as the calling thread: a client
* from outside has none. Discovery only; `McpToolAccess` still refuses the
diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts
index 777a5d6df3a56..6e8ac9c415763 100644
--- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts
+++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts
@@ -51,12 +51,14 @@ import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as RemoteDelegation from "../peer/RemoteDelegation.ts";
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import * as McpHttpServer from "./McpHttpServer.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "./McpInvocationContext.ts";
import * as PeerForwarding from "../peer/PeerForwarding.ts";
import * as PeerLinkRequests from "../peer/PeerLinkRequests.ts";
import * as PeerLinks from "../peer/PeerLinks.ts";
const layerOrchestratorToolkit = McpHttpServer.layerOrchestratorToolkit.pipe(
+ Layer.provide(OpenShellFleetService.layerDisabled),
Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})),
Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
);
diff --git a/apps/server/src/mcp/linkOrigin.test.ts b/apps/server/src/mcp/linkOrigin.test.ts
index 928fda0dd72f8..bf1b6c39d4920 100644
--- a/apps/server/src/mcp/linkOrigin.test.ts
+++ b/apps/server/src/mcp/linkOrigin.test.ts
@@ -43,6 +43,7 @@ import * as RemoteDelegation from "../peer/RemoteDelegation.ts";
import * as SourceControlRepositoryService from "../sourceControl/SourceControlRepositoryService.ts";
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
import * as McpHttpServer from "./McpHttpServer.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "./McpInvocationContext.ts";
import * as PeerForwarding from "../peer/PeerForwarding.ts";
import * as ThreadSearch from "../orchestration-v2/ThreadSearch.ts";
@@ -115,7 +116,7 @@ const layerLaunches = Layer.effect(
);
const layerTools = Layer.mergeAll(
- McpHttpServer.layerOrchestratorToolkit,
+ McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)),
McpHttpServer.layerThreadToolkit,
McpHttpServer.layerProjectRegistration,
).pipe(
diff --git a/apps/server/src/mcp/toolkits/core.test.ts b/apps/server/src/mcp/toolkits/core.test.ts
index 1272f367d439d..4bf49566c9e9a 100644
--- a/apps/server/src/mcp/toolkits/core.test.ts
+++ b/apps/server/src/mcp/toolkits/core.test.ts
@@ -58,6 +58,7 @@ import * as RemoteDelegation from "../../peer/RemoteDelegation.ts";
import * as ScheduledTaskService from "../../scheduledTasks/ScheduledTaskService.ts";
import * as ServerSettings from "../../serverSettings.ts";
import * as McpHttpServer from "../McpHttpServer.ts";
+import * as OpenShellFleetService from "../../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "../McpInvocationContext.ts";
import * as McpToolAccessTestkit from "../McpToolAccess.testkit.ts";
import * as AnalyticsService from "../../telemetry/AnalyticsService.ts";
@@ -70,6 +71,7 @@ import * as PeerLinkRequests from "../../peer/PeerLinkRequests.ts";
import * as PeerLinks from "../../peer/PeerLinks.ts";
const layerOrchestratorToolkit = McpHttpServer.layerOrchestratorToolkit.pipe(
+ Layer.provide(OpenShellFleetService.layerDisabled),
Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})),
Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
);
diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts
new file mode 100644
index 0000000000000..53b0144ab8451
--- /dev/null
+++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.test.ts
@@ -0,0 +1,174 @@
+import { describe, expect, it } from "@effect/vitest";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Stream from "effect/Stream";
+import {
+ EnvironmentId,
+ type OrchestratorMcpDelegateTaskInput,
+ type OrchestratorMcpDelegateTaskResult,
+ ProviderInstanceId,
+} from "@t3tools/contracts";
+
+import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts";
+import { OrchestratorToolkit } from "./tools.ts";
+import * as Handlers from "./handlers.ts";
+import * as McpInvocationContext from "../../McpInvocationContext.ts";
+import * as McpToolAccess from "../../McpToolAccess.ts";
+import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts";
+import * as ThreadManagement from "../../../orchestration-v2/ThreadManagementService.ts";
+
+const threadId = "thread:parent" as never;
+
+interface Harness {
+ readonly spawns: Array<
+ Parameters[0]
+ >;
+ readonly delegated: Array;
+ readonly failSpawn?: OpenShellFleetService.OpenShellFleetError;
+}
+
+const run = (input: OrchestratorMcpDelegateTaskInput, harness: Harness) =>
+ Effect.gen(function* () {
+ const caller = {
+ scope: undefined as never,
+ threads: {} as never,
+ caller: {
+ id: threadId,
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ archivedAt: null,
+ providerInstanceId: ProviderInstanceId.make("pi"),
+ activeRunId: "run:parent",
+ deletedAt: null,
+ },
+ limits: { runtimeMode: "full-access" as const, interactionMode: "default" as const },
+ };
+ const dependencies = Layer.mergeAll(
+ Layer.succeed(McpInvocationContext.McpInvocationContext, {
+ environmentId: EnvironmentId.make("environment"),
+ requestNamespace: "session",
+ thread: {
+ threadId,
+ providerInstanceId: ProviderInstanceId.make("pi"),
+ providerSessionId: "session",
+ },
+ client: {
+ sessionId: "session",
+ label: "Test",
+ access: "full-access" as const,
+ },
+ issuedAt: 0,
+ capabilities: new Set(["orchestration" as const]),
+ }),
+ Layer.mock(ThreadManagement.ThreadManagementService)({
+ getThreadShell: () => Effect.succeed(caller.caller as never),
+ }),
+ Layer.mock(OrchestratorMcpService.OrchestratorMcpService)({
+ delegateTask: (_scope, params) => {
+ harness.delegated.push(params);
+ return Effect.succeed({
+ taskId: "node:task",
+ childThreadId: null,
+ remoteChild: {
+ environmentId: EnvironmentId.make("environment:worker"),
+ threadId: "thread:child",
+ label: "worker",
+ },
+ childRunId: null,
+ childNodeId: "node:task",
+ status: "running",
+ workState: "working",
+ hasPendingChildRuns: false,
+ latestTerminalRunId: null,
+ latestTerminalStatus: null,
+ latestTerminalSummary: null,
+ latestTerminalResultContextTransferId: null,
+ providerInstanceId: ProviderInstanceId.make("pi"),
+ model: null,
+ summary: null,
+ resultContextTransferId: null,
+ hasPendingQuestion: false,
+ pendingApproval: null,
+ waitTimedOut: false,
+ } as unknown as OrchestratorMcpDelegateTaskResult);
+ },
+ }),
+ Layer.mock(OpenShellFleetService.OpenShellFleetService)({
+ spawnWorker: (params) => {
+ harness.spawns.push(params);
+ return harness.failSpawn === undefined
+ ? Effect.succeed({
+ environmentId: EnvironmentId.make("environment:worker"),
+ sandbox: { name: "worker", id: "sandbox-1" },
+ workerClass: params.workerClass,
+ state: "ready",
+ expiresAt: null,
+ warnings: [],
+ } as never)
+ : Effect.fail(harness.failSpawn);
+ },
+ removeWorker: () => Effect.die("removeWorker is not part of this path"),
+ listWorkers: () => Effect.succeed([]),
+ }),
+ );
+ const toolkit = yield* OrchestratorToolkit.pipe(
+ Effect.provide(
+ McpToolAccess.HandlersLayer.layer(Handlers.layer).pipe(Layer.provide(dependencies)),
+ ),
+ );
+ const events = yield* toolkit
+ .handle("delegate_task", input as never)
+ .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies));
+ return events.at(-1)?.result;
+ });
+
+describe("delegate_task openshell-auto isolation", () => {
+ it.effect("provisions an ephemeral worker and delegates into it", () =>
+ Effect.gen(function* () {
+ const harness: Harness = { spawns: [], delegated: [] };
+ yield* run({ task: "Do the thing", isolation: "openshell-auto" }, harness);
+
+ expect(harness.spawns).toHaveLength(1);
+ expect(harness.spawns[0]).toMatchObject({
+ workerClass: "ephemeral",
+ runtimeModeCeiling: "full-access",
+ });
+ // The child runs there, not here: the sandbox is the isolation.
+ expect(harness.delegated[0]?.isolation).toBe("shared");
+ expect(harness.delegated[0]?.target).toMatchObject({
+ environmentId: EnvironmentId.make("environment:worker"),
+ });
+ }),
+ );
+
+ it.effect("delegates locally without provisioning anything", () =>
+ Effect.gen(function* () {
+ const harness: Harness = { spawns: [], delegated: [] };
+ yield* run({ task: "Do the thing" }, harness);
+
+ expect(harness.spawns).toHaveLength(0);
+ expect(harness.delegated[0]?.target).toBeUndefined();
+ }),
+ );
+
+ it.effect("reports a refused spawn as a mode-escalation denial", () =>
+ Effect.gen(function* () {
+ const harness: Harness = {
+ spawns: [],
+ delegated: [],
+ failSpawn: new OpenShellFleetService.OpenShellFleetSpawnRefused({
+ reason: "runtime_mode_above_ceiling",
+ message:
+ "Worker policy full-access is broader than this caller's approval-required ceiling.",
+ }),
+ };
+ const result = (yield* run(
+ { task: "Do the thing", isolation: "openshell-auto" },
+ harness,
+ )) as { readonly code?: string } | undefined;
+
+ expect(harness.delegated).toHaveLength(0);
+ expect(result?.code).toBe("runtime_mode_escalation_denied");
+ }),
+ );
+});
diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts
index d90a118be468e..a0a45a41c6767 100644
--- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts
+++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts
@@ -1,17 +1,39 @@
import { OrchestratorMcpFailure } from "@t3tools/contracts";
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeCrypto from "node:crypto";
import { OrchestratorToolkit } from "./tools.ts";
+import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
+import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts";
import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
import * as PeerLinks from "../../../peer/PeerLinks.ts";
import * as RemoteDelegation from "../../../peer/RemoteDelegation.ts";
-import { readCaller } from "../../threadAccess.ts";
+import { assertNotLinked } from "../../linkOrigin.ts";
+import { assertFullAccess, loadCaller, readCaller } from "../../threadAccess.ts";
import * as McpInvocationContext from "../../McpInvocationContext.ts";
import * as McpToolAccess from "../../McpToolAccess.ts";
import * as OrchestratorMcpService from "../../OrchestratorMcpService.ts";
import * as ThreadMetadataMcpService from "../../ThreadMetadataMcpService.ts";
+const fleetFailure = (error: OpenShellFleetService.OpenShellFleetError): OrchestratorMcpFailure =>
+ error._tag === "OpenShellFleetDisabled"
+ ? new OrchestratorMcpFailure({ code: "capability_denied", message: error.message })
+ : error._tag === "OpenShellFleetSpawnRefused"
+ ? new OrchestratorMcpFailure({
+ code:
+ error.reason === "runtime_mode_above_ceiling"
+ ? "runtime_mode_escalation_denied"
+ : "invalid_request",
+ message: error.message,
+ })
+ : new OrchestratorMcpFailure({ code: "orchestration_error", message: error.message });
+
+/** Same caller request id must not collide across MCP sessions/scope owners. */
+const scopedFleetRequestId = (namespace: string, clientRequestId: string): string =>
+ `fleet:${NodeCrypto.createHash("sha256").update(`${namespace}\n${clientRequestId}`).digest("hex")}`;
+
const handlers = {
orchestrator_capabilities: McpToolAccess.reads((input) =>
Effect.gen(function* () {
@@ -24,7 +46,41 @@ const handlers = {
Effect.gen(function* () {
const scope = yield* McpInvocationContext.McpInvocationContext;
const service = yield* OrchestratorMcpService.OrchestratorMcpService;
- return yield* service.delegateTask(scope, input);
+ if (input.isolation !== "openshell-auto") {
+ return yield* service.delegateTask(scope, input);
+ }
+ // Provision the worker here, then hand the task to it like any other
+ // linked environment. Its retirement is the fleet's own: the deadline
+ // reaps it drain-first, so nothing is deleted under a running child.
+ const caller = yield* loadCaller();
+ // Provisioning infrastructure is an environment setting, not thread work:
+ // the same full-access/linked checks as t3_environment_spawn apply here,
+ // and they run before anything is created.
+ yield* assertFullAccess(caller, "provision a fleet worker");
+ yield* assertNotLinked(caller, "provision a fleet worker");
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService;
+ const spawned = yield* fleet
+ .spawnWorker({
+ workerClass: "ephemeral",
+ runtimeModeCeiling: caller.limits.runtimeMode,
+ ...(input.runtimeMode === undefined ? {} : { runtimeMode: input.runtimeMode }),
+ ...(input.title === undefined ? {} : { label: input.title }),
+ ...(input.clientRequestId === undefined
+ ? {}
+ : {
+ clientRequestId: scopedFleetRequestId(
+ scope.requestNamespace,
+ input.clientRequestId,
+ ),
+ }),
+ })
+ .pipe(Effect.mapError(fleetFailure));
+ return yield* service.delegateTask(scope, {
+ ...input,
+ // The child runs in that environment's checkout; the sandbox is the isolation.
+ isolation: "shared",
+ target: { ...input.target, environmentId: spawned.environmentId },
+ });
}),
),
task_status: McpToolAccess.actsAsCaller(({ taskId }) =>
@@ -229,6 +285,69 @@ const handlers = {
return { environmentId, removed };
}),
),
+ // Spawning a worker mints infrastructure under this environment's gateway
+ // credential, so it is an environment setting: full access, never linked work.
+ t3_environment_spawn: McpToolAccess.writesEnvironment((input, check) =>
+ Effect.gen(function* () {
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService;
+ const scope = yield* McpInvocationContext.McpInvocationContext;
+ const caller = yield* check;
+ return yield* fleet
+ .spawnWorker({
+ workerClass: input.workerClass,
+ runtimeModeCeiling: caller.limits.runtimeMode,
+ ...(input.template === undefined ? {} : { template: input.template }),
+ ...(input.runtimeMode === undefined ? {} : { runtimeMode: input.runtimeMode }),
+ ...(input.label === undefined ? {} : { label: input.label }),
+ ...(input.ttlMs === undefined ? {} : { ttlMs: Duration.toMillis(input.ttlMs) }),
+ ...(input.seedFrom === undefined ? {} : { seedFrom: input.seedFrom }),
+ ...(input.clientRequestId === undefined
+ ? {}
+ : {
+ clientRequestId: scopedFleetRequestId(
+ scope.requestNamespace,
+ input.clientRequestId,
+ ),
+ }),
+ })
+ .pipe(Effect.mapError(fleetFailure));
+ }),
+ ),
+ // Removal drains worker data back here before any delete, so a refusal is
+ // the safe outcome; it stays a full-access environment operation.
+ t3_environment_remove: McpToolAccess.writesEnvironment((input) =>
+ Effect.gen(function* () {
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService;
+ const removed = yield* fleet
+ .removeWorker({
+ targetEnvironmentId: input.targetEnvironmentId,
+ ...(input.drainTimeoutMs === undefined
+ ? {}
+ : { drainTimeoutMs: Duration.toMillis(input.drainTimeoutMs) }),
+ ...(input.force === undefined ? {} : { force: input.force }),
+ ...(input.clientRequestId === undefined
+ ? {}
+ : { clientRequestId: input.clientRequestId }),
+ })
+ .pipe(Effect.mapError(fleetFailure));
+ return {
+ targetEnvironmentId: removed.environmentId,
+ outcome: removed.outcome,
+ ...(removed.drain === undefined ? {} : { drain: removed.drain }),
+ ...(removed.sandboxOutcome === undefined ? {} : { sandboxOutcome: removed.sandboxOutcome }),
+ ...(removed.reason === undefined ? {} : { reason: removed.reason }),
+ };
+ }),
+ ),
+ t3_environment_list: McpToolAccess.reads((input) =>
+ Effect.gen(function* () {
+ const fleet = yield* OpenShellFleetService.OpenShellFleetService;
+ const workers = yield* fleet
+ .listWorkers(input.workerClass === undefined ? {} : { workerClass: input.workerClass })
+ .pipe(Effect.mapError(fleetFailure));
+ return { workers };
+ }),
+ ),
t3_thread_interrupt: McpToolAccess.writesThreads(
(input) => [input.threadId],
(input) =>
diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts
index b2866bb0babc5..7d8c90ca29e99 100644
--- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts
+++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts
@@ -7,6 +7,12 @@ import {
OrchestratorMcpEnvironmentLinksResult,
OrchestratorMcpEnvironmentUnlinkInput,
OrchestratorMcpEnvironmentUnlinkResult,
+ OrchestratorMcpEnvironmentSpawnInput,
+ OrchestratorMcpEnvironmentSpawnResult,
+ OrchestratorMcpEnvironmentRemoveInput,
+ OrchestratorMcpEnvironmentRemoveResult,
+ OrchestratorMcpEnvironmentListInput,
+ OrchestratorMcpEnvironmentListResult,
OrchestratorMcpCreateThreadsInput,
OrchestratorMcpCreateThreadsResult,
OrchestratorMcpDelegateTaskInput,
@@ -43,6 +49,7 @@ import * as Schema from "effect/Schema";
import * as Struct from "effect/Struct";
import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts";
+import * as OpenShellFleetService from "../../../environment/OpenShellFleetService.ts";
import * as PeerForwarding from "../../../peer/PeerForwarding.ts";
import * as PeerLinkRequests from "../../../peer/PeerLinkRequests.ts";
import * as PeerLinks from "../../../peer/PeerLinks.ts";
@@ -71,6 +78,14 @@ const threadMetadataDependencies = [
ThreadManagementService.ThreadManagementService,
ThreadMetadataMcpService.ThreadMetadataMcpService,
];
+/** Delegation that can also provision a fleet worker (spec section 6.3). */
+const delegateDependencies = [...dependencies, OpenShellFleetService.OpenShellFleetService];
+/** Fleet worker lifecycle on the OpenShell gateway (spec section 5.1). */
+const fleetDependencies = [
+ McpInvocationContext.McpInvocationContext,
+ ThreadManagementService.ThreadManagementService,
+ OpenShellFleetService.OpenShellFleetService,
+];
const OrchestratorCapabilitiesTool = Tool.make("orchestrator_capabilities", {
description:
@@ -88,12 +103,12 @@ const OrchestratorCapabilitiesTool = Tool.make("orchestrator_capabilities", {
const DelegateTaskTool = Tool.make("delegate_task", {
description:
- "Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. Keep `taskId` for `task_status` and `task_cancel`. No secrets in `task`.",
+ "Start one child agent with only `task`. No parent history. Separate checkout or PR per child: `isolation:'worktree'`. `isolation:'openshell-auto'` runs the child on a self-retiring ephemeral fleet worker. Keep `taskId` for `task_status` and `task_cancel`. No secrets in `task`.",
parameters: OrchestratorMcpDelegateTaskInput,
success: OrchestratorMcpDelegateTaskResult,
failure: OrchestratorMcpFailure,
failureMode: "return",
- dependencies,
+ dependencies: delegateDependencies,
})
.annotate(Tool.Title, "Delegate a child task")
.annotate(Tool.Destructive, true)
@@ -362,6 +377,50 @@ const EnvironmentUnlinkTool = Tool.make("t3_environment_unlink", {
.annotate(Tool.Idempotent, true)
.annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileLinksOff);
+const EnvironmentSpawnTool = Tool.make("t3_environment_spawn", {
+ description:
+ "Provision and pair a fleet worker. Ephemeral workers self-retire on the TTL sweep; persistent ones are named lanes. Same `clientRequestId` converges retries and re-pairs a worker whose delegation session expired.",
+ parameters: OrchestratorMcpEnvironmentSpawnInput,
+ success: OrchestratorMcpEnvironmentSpawnResult,
+ failure: OrchestratorMcpFailure,
+ failureMode: "return",
+ dependencies: fleetDependencies,
+})
+ .annotate(Tool.Title, "Spawn a fleet worker environment")
+ .annotate(Tool.Destructive, false)
+ .annotate(Tool.OpenWorld, true)
+ .annotate(Tool.Idempotent, true)
+ .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff);
+
+const EnvironmentRemoveTool = Tool.make("t3_environment_remove", {
+ description:
+ "Drain a fleet worker to this instance (thread histories plus a git bundle of the checkout refs origin lacks, committed and uncommitted), then delete its sandbox. Refused while the drain is incomplete; the sandbox is kept. `force` skips the checkout bundles after an explicit user ack: thread histories still export, but the checkouts are lost with the sandbox.",
+ parameters: OrchestratorMcpEnvironmentRemoveInput,
+ success: OrchestratorMcpEnvironmentRemoveResult,
+ failure: OrchestratorMcpFailure,
+ failureMode: "return",
+ dependencies: fleetDependencies,
+})
+ .annotate(Tool.Title, "Remove a fleet worker environment")
+ .annotate(Tool.Destructive, true)
+ .annotate(Tool.OpenWorld, true)
+ .annotate(Tool.Idempotent, true)
+ .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff);
+
+const EnvironmentListTool = Tool.make("t3_environment_list", {
+ description: "Report this instance's fleet worker state.",
+ parameters: OrchestratorMcpEnvironmentListInput,
+ success: OrchestratorMcpEnvironmentListResult,
+ failure: OrchestratorMcpFailure,
+ failureMode: "return",
+ dependencies: fleetDependencies,
+})
+ .annotate(Tool.Title, "List fleet worker environments")
+ .annotate(Tool.Readonly, true)
+ .annotate(Tool.Destructive, false)
+ .annotate(Tool.Idempotent, true)
+ .annotate(McpSchema.EnabledWhen, McpInvocationContext.hiddenWhileFleetOff);
+
export const OrchestratorToolkit = Toolkit.make(
OrchestratorCapabilitiesTool,
DelegateTaskTool,
@@ -383,4 +442,7 @@ export const OrchestratorToolkit = Toolkit.make(
EnvironmentLinksTool,
EnvironmentLinkTool,
EnvironmentUnlinkTool,
+ EnvironmentSpawnTool,
+ EnvironmentRemoveTool,
+ EnvironmentListTool,
);
diff --git a/apps/server/src/mcp/toolkits/project/handlers.test.ts b/apps/server/src/mcp/toolkits/project/handlers.test.ts
index c13a8ab9353cc..ce6269c73de85 100644
--- a/apps/server/src/mcp/toolkits/project/handlers.test.ts
+++ b/apps/server/src/mcp/toolkits/project/handlers.test.ts
@@ -626,6 +626,32 @@ it.effect("a launch binds only an existing checkout that is one of the project's
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);
+it.effect("a sandbox workspace is refused here and pointed at delegation", () =>
+ Effect.gen(function* () {
+ const launched: Array = [];
+ const { projectId, dependencies } = clientLaunchHarness({
+ runtimeModeCeiling: "full-access",
+ launched,
+ });
+ const toolkit = yield* ProjectToolkit.pipe(
+ Effect.provide(
+ McpToolAccess.HandlersLayer.layer(ProjectHandlers.layer).pipe(Layer.provide(dependencies)),
+ ),
+ );
+ const result = yield* toolkit
+ .handle("t3_thread_launch", {
+ title: "Fix",
+ projectId,
+ workspaceStrategy: { type: "sandbox" },
+ })
+ .pipe(Stream.unwrap, Stream.runCollect, Effect.provide(dependencies));
+
+ expect(result.at(-1)?.result).toMatchObject({ code: "invalid_request" });
+ // No thread launched locally: the sandbox would have silently become the root.
+ expect(launched).toHaveLength(0);
+ }),
+);
+
it.effect("a launch into a new worktree creates nothing when its base ref has no commit", () =>
Effect.gen(function* () {
const launched: Array = [];
diff --git a/apps/server/src/mcp/toolkits/project/handlers.ts b/apps/server/src/mcp/toolkits/project/handlers.ts
index 3966d48ccd090..64034ac0ba753 100644
--- a/apps/server/src/mcp/toolkits/project/handlers.ts
+++ b/apps/server/src/mcp/toolkits/project/handlers.ts
@@ -377,6 +377,14 @@ export const layer = McpToolAccess.toLayer(ProjectToolkit, {
}
: requestedStrategy;
// Reject unresolved base refs and blocked branch names before accepting a launch; worktree provisioning happens later.
+ // OPEN-QUESTION (spec section 6.3): a sandbox workspace belongs to a fleet
+ // worker, so v1 launches it as a delegation rather than here.
+ if (!replaying && strategy?.type === "sandbox")
+ return yield* new OrchestratorMcpFailure({
+ code: "invalid_request",
+ message:
+ "workspaceStrategy 'sandbox' is not available on thread launch yet. Delegate the work with isolation:'openshell-auto' to run it in a fleet worker environment.",
+ });
if (!replaying && strategy?.type === "worktree")
yield* ThreadLaunch.ThreadLaunchService.pipe(
Effect.flatMap((launches) =>
diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
index df965d891487c..77398d460f704 100644
--- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
@@ -2597,6 +2597,7 @@ export const CLAUDE_READ_ONLY_T3_MCP_ALLOWED_TOOLS: ReadonlyArray = [
"mcp__t3-code__t3_preview_list",
"mcp__t3-code__t3_environment_read",
"mcp__t3-code__t3_environment_links",
+ "mcp__t3-code__t3_environment_list",
"mcp__t3-code__t3_theme_read",
"mcp__t3-code__html_preview",
"mcp__t3-code__html_render",
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts
index 00939843c13f1..32bccce82e3db 100644
--- a/apps/server/src/orchestration-v2/Orchestrator.ts
+++ b/apps/server/src/orchestration-v2/Orchestrator.ts
@@ -1010,6 +1010,16 @@ function delegatedTaskTerminalStatus(
function worktreeSetupPending(
projection: Pick,
): boolean {
+ // A sandbox run can never resolve to a local workspace, so it holds only
+ // while it is the latest run: a newer run supersedes it. A worktree holds
+ // until its path exists, whatever came after it.
+ const latest = latestExecutedRun(projection.runs);
+ if (
+ latest !== null &&
+ (latest.status === "failed" || latest.status === "interrupted") &&
+ latest.workspacePreparation?.type === "sandbox"
+ )
+ return true;
return (
projection.thread.worktreePath === null &&
projection.runs.some(
@@ -2561,8 +2571,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
const worktreeMissing =
endedRunId !== undefined &&
failedRun?.id === endedRunId &&
- failedRun?.workspacePreparation?.type === "worktree" &&
- projection.thread.worktreePath === null;
+ (failedRun?.workspacePreparation?.type === "sandbox" ||
+ (failedRun?.workspacePreparation?.type === "worktree" &&
+ projection.thread.worktreePath === null));
const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
const projectOverride =
settings.projectSettingsOverrides[projection.thread.projectId]?.providerInstances?.[
@@ -7848,8 +7859,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
activeRun === undefined &&
(dispatchMode.type === "queue_after_active" || dispatchMode.type === "start_immediately") &&
(latestRun?.status === "failed" || latestRun?.status === "interrupted") &&
- latestRun.workspacePreparation?.type === "worktree" &&
- projection.thread.worktreePath === null
+ (latestRun.workspacePreparation?.type === "sandbox" ||
+ (latestRun.workspacePreparation?.type === "worktree" &&
+ projection.thread.worktreePath === null))
? latestRun
: undefined;
// A steer whose run already ended still keeps its place in the queue.
@@ -15574,6 +15586,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
"orchestration_v2.command_type": command.type,
"orchestration_v2.thread_id": commandThreadId(command),
});
+ // A sandbox workspace is provisioned by delegation, never by a dispatch;
+ // refuse before any event records it on a run. Replays short-circuit above.
+ if (
+ (command.type === "message.dispatch" &&
+ command.dispatchMode.type === "defer_start" &&
+ command.dispatchMode.workspaceStrategy?.type === "sandbox") ||
+ (command.type === "delegated_task.request" &&
+ command.workspaceStrategy?.type === "sandbox") ||
+ (command.type === "thread.create" &&
+ command.initialMessage?.workspaceStrategy?.type === "sandbox")
+ )
+ return yield* new OrchestratorDispatchError({
+ commandId: command.commandId,
+ commandType: command.type,
+ cause:
+ "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.",
+ });
const events = yield* Ref.make>([]);
const effects = yield* Ref.make>([]);
diff --git a/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts b/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts
new file mode 100644
index 0000000000000..c14840ebaa130
--- /dev/null
+++ b/apps/server/src/orchestration-v2/OrchestratorSandboxRefusal.test.ts
@@ -0,0 +1,108 @@
+import { assert, it } from "@effect/vitest";
+import { CommandId, MessageId, ProjectId, ThreadId } from "@t3tools/contracts";
+import * as Layer from "effect/Layer";
+import * as Effect from "effect/Effect";
+import * as Orchestrator from "./Orchestrator.ts";
+import * as ProjectionStore from "./ProjectionStore.ts";
+import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
+import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
+import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
+import { ProviderDriverKind, ProviderInstanceId } from "@t3tools/contracts";
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
+
+const instanceId = ProviderInstanceId.make("codex");
+const adapter = {
+ instanceId,
+ driver: ProviderDriverKind.make("codex"),
+ getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
+ planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
+ openSession: () => Effect.die("No provider process needed for metadata commands"),
+} as ProviderAdapter.ProviderAdapterV2["Service"];
+const layerDatabase = SqlitePersistence.layerMemory;
+const layerTest = Layer.mergeAll(
+ layerDatabase,
+ ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
+ ProviderReplayHarness.layerWithRegistry(
+ { name: "sandbox-refusal" },
+ ProviderAdapterRegistry.layerFromAdapters([adapter]),
+ { databaseLayer: layerDatabase, runEffectWorker: false },
+ ),
+);
+
+const createThread = (threadId: ThreadId) =>
+ Effect.gen(function* () {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ yield* orchestrator.dispatch({
+ type: "thread.create",
+ commandId: CommandId.make(`create:${threadId}`),
+ threadId,
+ projectId: ProjectId.make("project:sandbox-refusal"),
+ title: "Sandbox refusal",
+ modelSelection: { instanceId, model: "gpt-5" },
+ runtimeMode: "approval-required",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ createdBy: "user",
+ creationSource: "web",
+ });
+ });
+
+it.layer(layerTest)("sandbox workspace refusals", (it) => {
+ it.effect("refuses a defer_start dispatch with a sandbox strategy and records no run", () =>
+ Effect.gen(function* () {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const threadId = ThreadId.make("thread:sandbox-defer");
+ yield* createThread(threadId);
+ const refused = yield* orchestrator
+ .dispatch({
+ type: "message.dispatch",
+ commandId: CommandId.make("dispatch:sandbox-defer"),
+ threadId,
+ messageId: MessageId.make("message:sandbox-defer"),
+ text: "Run this in a worker",
+ attachments: [],
+ modelSelection: { instanceId, model: "gpt-5" },
+ dispatchMode: {
+ type: "defer_start",
+ workspaceStrategy: { type: "sandbox" },
+ },
+ createdBy: "user",
+ creationSource: "web",
+ })
+ .pipe(Effect.flip);
+ assert.equal(refused._tag, "OrchestratorDispatchError");
+ assert.match(String(refused.cause), /isolation 'openshell-auto'/);
+ const records = yield* projections.getThreadRecords(threadId, ["runs"]);
+ assert.equal(records.runs.length, 0);
+ }),
+ );
+
+ it.effect("refuses a delegated task with a sandbox strategy", () =>
+ Effect.gen(function* () {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ const parentThreadId = ThreadId.make("thread:sandbox-delegate-parent");
+ yield* createThread(parentThreadId);
+ const refused = yield* orchestrator
+ .dispatch({
+ type: "delegated_task.request",
+ commandId: CommandId.make("dispatch:sandbox-delegate"),
+ parentThreadId,
+ parentRunId: "run:sandbox-delegate" as never,
+ parentNodeId: "node:sandbox-delegate" as never,
+ task: "Build this in a worker",
+ modelSelection: { instanceId, model: "gpt-5" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ workspaceStrategy: { type: "sandbox" },
+ createdBy: "user",
+ creationSource: "web",
+ })
+ .pipe(Effect.flip);
+ assert.equal(refused._tag, "OrchestratorDispatchError");
+ assert.match(String(refused.cause), /isolation 'openshell-auto'/);
+ }),
+ );
+});
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts
index ba91ba94d2520..4f43cf6d3eb70 100644
--- a/apps/server/src/orchestration-v2/ProjectionStore.ts
+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts
@@ -490,7 +490,7 @@ export interface ProjectionTimelinePage {
}
export interface ShellSnapshotOptions {
- readonly location?: "active" | "archive";
+ readonly location?: "active" | "archive" | "all";
readonly projectId?: ProjectId;
readonly unsettledOnly?: boolean;
/**
diff --git a/apps/server/src/orchestration-v2/ShellStream.ts b/apps/server/src/orchestration-v2/ShellStream.ts
index 5ee1005b16830..282dcbf03f49f 100644
--- a/apps/server/src/orchestration-v2/ShellStream.ts
+++ b/apps/server/src/orchestration-v2/ShellStream.ts
@@ -167,13 +167,15 @@ function buildActiveShellSnapshot(input: {
readonly projects: ReadonlyArray;
readonly threads: OrchestrationV2ThreadShellSnapshot;
readonly snapshotSequence: number;
+ /** A drain has to copy archived threads too before it may delete them. */
+ readonly includeArchived?: boolean;
}): OrchestrationV2ShellSnapshot {
return {
schemaVersion: input.threads.schemaVersion,
snapshotSequence: input.snapshotSequence,
projects: input.projects,
threads: input.threads.threads,
- archivedThreads: [],
+ archivedThreads: input.includeArchived === true ? input.threads.archivedThreads : [],
};
}
@@ -347,6 +349,7 @@ export function omitOlderSettledShells(input: {
interface ShellSubscriberOptions {
readonly omitIdleSubagents: boolean;
readonly omitOlderSettled: boolean;
+ readonly includeArchived?: boolean;
}
function trimShellSnapshot(
@@ -426,11 +429,16 @@ export const loadSubscriberShellSnapshot = Effect.fn("ShellStream.loadSubscriber
.withTransaction(
Effect.gen(function* () {
const index = trims ? yield* input.listShellIndex : null;
+ // Active index IDs are a trim filter, not an inventory filter. An
+ // archive-inclusive snapshot must still read archived threads, so it
+ // ignores the active-ID filter and applies trims after the full read.
+ const subscriberIds =
+ index === null || input.options.includeArchived === true
+ ? undefined
+ : shellSubscriberThreadIds(index, input.options);
return {
index,
- decodeThreads: yield* input.readThreads(
- index === null ? undefined : shellSubscriberThreadIds(index, input.options),
- ),
+ decodeThreads: yield* input.readThreads(subscriberIds),
projects: yield* input.listProjects,
snapshotSequence: yield* input.latestSequence,
};
@@ -441,6 +449,7 @@ export const loadSubscriberShellSnapshot = Effect.fn("ShellStream.loadSubscriber
projects: read.projects,
threads: yield* read.decodeThreads,
snapshotSequence: read.snapshotSequence,
+ includeArchived: input.options.includeArchived === true,
});
const facts = shellTrimFacts(read.index ?? base.threads);
return {
diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
index 2a3a8e5510550..02cf1737ff63f 100644
--- a/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
+++ b/apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
@@ -1189,6 +1189,55 @@ const preparationSettled = (threadId: ThreadId) =>
return yield* threads.getThreadProjection(threadId);
});
+it.effect("refuses a sandbox launch before creating a thread or run", () => {
+ const harness = makeHarness();
+ return Effect.gen(function* () {
+ const launches = yield* ThreadLaunch.ThreadLaunchService;
+ const threads = yield* ThreadManagement.ThreadManagementService;
+ const failed = yield* launches
+ .launch(
+ launchInput({
+ command: "command:launch:sandbox",
+ thread: "thread:launch:sandbox",
+ message: "Run in a worker",
+ workspace: { type: "sandbox" },
+ }),
+ )
+ .pipe(Effect.flip);
+ if (failed._tag !== "ThreadLaunchError") {
+ assert.fail(`expected ThreadLaunchError, got ${failed._tag}`);
+ }
+ assert.equal(failed.operation, "create-thread");
+ assert.match(String(failed.cause), /isolation 'openshell-auto'/);
+ assert.equal(harness.runSetup.mock.calls.length, 0);
+ assert.equal(yield* threads.getThreadShell(ThreadId.make("thread:launch:sandbox")), null);
+ }).pipe(Effect.provide(harness.layer));
+});
+
+it.effect("refuses a sandbox launch without an initial message too", () => {
+ const harness = makeHarness();
+ return Effect.gen(function* () {
+ const launches = yield* ThreadLaunch.ThreadLaunchService;
+ const threads = yield* ThreadManagement.ThreadManagementService;
+ const failed = yield* launches
+ .launch(
+ launchInput({
+ command: "command:launch:sandbox-silent",
+ thread: "thread:launch:sandbox-silent",
+ workspace: { type: "sandbox" },
+ }),
+ )
+ .pipe(Effect.flip);
+ assert.equal(failed._tag, "ThreadLaunchError");
+ assert.match(String(failed.cause), /isolation 'openshell-auto'/);
+ assert.equal(harness.runSetup.mock.calls.length, 0);
+ assert.equal(
+ yield* threads.getThreadShell(ThreadId.make("thread:launch:sandbox-silent")),
+ null,
+ );
+ }).pipe(Effect.provide(harness.layer));
+});
+
it.effect.each([
{ prefix: "my-prefix", collision: null, expected: /^my-prefix\/[0-9a-f]{8}$/ },
{ prefix: "my-prefix", collision: "my-prefix", expected: /^my-prefix-[0-9a-f]{8}$/ },
diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts
index e0ae516a190b5..eb2e2b81dfd80 100644
--- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts
+++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts
@@ -5,6 +5,7 @@ import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import {
CommandId,
type ChatAttachment,
+ EnvironmentId,
type MessageId,
type ModelSelection,
type OrchestrationV2Actor,
@@ -83,6 +84,12 @@ export type ThreadLaunchWorkspaceStrategy =
readonly branch?: string | undefined;
readonly startFromOrigin?: boolean | undefined;
readonly repositories?: ReadonlyArray<{ readonly path: string }> | undefined;
+ }
+ /** A fleet worker's workspace. Only delegation provisions it, so a local launch fails its run. */
+ | {
+ readonly type: "sandbox";
+ readonly workerClass?: "ephemeral" | "persistent" | undefined;
+ readonly branch?: string | undefined;
};
export interface ThreadLaunchInitialMessage {
@@ -117,6 +124,10 @@ export interface ThreadLaunchInput {
readonly metadata?: OrchestrationV2ProviderThreadNativeMetadata;
};
readonly createdBy: OrchestrationV2Actor;
+ /** Transport-supplied self id; the launch service owns the local-only rule. */
+ readonly currentEnvironmentId?: EnvironmentId;
+ /** Requested launch target; `undefined` means this environment. */
+ readonly environmentId?: EnvironmentId;
readonly creationSource: OrchestrationV2CreationSource;
/** The linked environment this launch is for, if one asked for it. */
readonly linkOrigin?: OrchestrationV2LinkOrigin;
@@ -556,6 +567,19 @@ const make = Effect.gen(function* () {
),
);
+ // A sandbox workspace belongs to a fleet worker, which is created by a
+ // delegation. Launching it here would quietly run in the project root.
+ if (input.workspaceStrategy.type === "sandbox") {
+ return yield* new ThreadLaunchError({
+ operation: "provision-worktree",
+ commandId: input.commandId,
+ projectId: input.projectId,
+ ...(threadId === undefined ? {} : { threadId }),
+ cause: new Error(
+ "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.",
+ ),
+ });
+ }
const reused = input.reusedWorktree;
const tracked = input.workspaceStrategy.type === "worktree" || reused !== undefined;
let createdWorktreePath: string | null = null;
@@ -1338,6 +1362,16 @@ const make = Effect.gen(function* () {
(yield* managedFolders.isScratchProject(request.projectId))
? { ...request, workspaceStrategy: { type: "root" } }
: request;
+ // A foreign launch has no local projection to return; delegate_task owns
+ // remote dispatch. Keep the rule here rather than in WebSocket transport.
+ if (input.environmentId !== undefined && input.environmentId !== input.currentEnvironmentId) {
+ return yield* mapError(
+ input,
+ "create-thread",
+ )(
+ "A thread launches in this environment. To run work in another environment, use delegate_task with target.environmentId; its result arrives when the task settles.",
+ );
+ }
yield* ProjectCloneTracker.rejectCommandsDuringClone(cloneTracker, {
type: "thread.create",
projectId: input.projectId,
@@ -1359,6 +1393,17 @@ const make = Effect.gen(function* () {
}
const launchReceipt = yield* readReceipt(input, input.commandId);
+ // A sandbox workspace belongs to a fleet worker, which only a delegation
+ // provisions. Refuse before any id or thread mutation; an accepted
+ // launch still replays through its receipt.
+ if (input.workspaceStrategy.type === "sandbox" && Option.isNone(launchReceipt)) {
+ return yield* mapError(
+ input,
+ "create-thread",
+ )(
+ "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of launching it here.",
+ );
+ }
yield* threads.assertProviderInstanceEnabled({
projectId: input.projectId,
commandId: input.commandId,
@@ -1570,15 +1615,17 @@ const make = Effect.gen(function* () {
Option.isSome(launchReceipt) &&
(accepted?.type === "worktree" || accepted?.type === "existing_worktree")
? accepted
- : Option.isSome(launchReceipt) &&
- workspaceStrategy.type === "root" &&
- boundWorktreePath !== null
- ? {
- type: "existing_worktree",
- worktreePath: boundWorktreePath,
- branch: workspaceStrategy.branch,
- }
- : workspaceStrategy;
+ : Option.isSome(launchReceipt) && accepted?.type === "root"
+ ? accepted
+ : Option.isSome(launchReceipt) &&
+ workspaceStrategy.type === "root" &&
+ boundWorktreePath !== null
+ ? {
+ type: "existing_worktree",
+ worktreePath: boundWorktreePath,
+ branch: workspaceStrategy.branch,
+ }
+ : workspaceStrategy;
yield* schedulePreparation(
{ ...input, workspaceStrategy: preparationStrategy },
threadId,
diff --git a/apps/server/src/orchestration-v2/http.ts b/apps/server/src/orchestration-v2/http.ts
index b58ce1b143119..4449ac3677876 100644
--- a/apps/server/src/orchestration-v2/http.ts
+++ b/apps/server/src/orchestration-v2/http.ts
@@ -84,13 +84,14 @@ export const layer = HttpApiBuilder.group(
function* (options: {
readonly omitIdleSubagents: boolean;
readonly omitOlderSettled: boolean;
+ readonly includeArchived: boolean;
}) {
const { snapshot } = yield* loadSubscriberShellSnapshot({
sql,
listShellIndex: threadManagement.listActiveShellIndex(),
readThreads: (threadIds) =>
threadManagement.readShellSnapshot({
- location: "active",
+ location: options.includeArchived ? "all" : "active",
...(threadIds === undefined ? {} : { threadIds }),
}),
listProjects: projectStore.listShells(),
@@ -106,15 +107,21 @@ export const layer = HttpApiBuilder.group(
const answerShellSnapshot = Effect.fn("http.orchestration.answerShellSnapshot")((query: {
readonly omitIdleSubagents?: string;
readonly omitOlderSettled?: string;
+ readonly includeArchived?: string;
readonly resumeFrom?: string;
}) => {
const resumeFrom = query.resumeFrom === undefined ? Number.NaN : Number(query.resumeFrom);
+ // Archive-inclusive replay is not fully reconstructed by the active-only
+ // event replay yet, so always send a complete archive-inclusive snapshot.
+ const canResume = query.includeArchived !== "1";
return loadShellUnlessResumable({
events: applicationEvents,
- resumeFrom: Number.isSafeInteger(resumeFrom) && resumeFrom >= 0 ? resumeFrom : undefined,
+ resumeFrom:
+ canResume && Number.isSafeInteger(resumeFrom) && resumeFrom >= 0 ? resumeFrom : undefined,
loadSnapshot: loadShellSnapshot({
omitIdleSubagents: query.omitIdleSubagents === "1",
omitOlderSettled: query.omitOlderSettled === "1",
+ includeArchived: query.includeArchived === "1",
}),
});
});
diff --git a/apps/server/src/peer/PeerEnvironmentRegistry.test.ts b/apps/server/src/peer/PeerEnvironmentRegistry.test.ts
new file mode 100644
index 0000000000000..65076f410adce
--- /dev/null
+++ b/apps/server/src/peer/PeerEnvironmentRegistry.test.ts
@@ -0,0 +1,497 @@
+// @effect-diagnostics nodeBuiltinImport:off - the redirect test exercises the real fetch boundary.
+import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { type ExecutionEnvironmentDescriptor } from "@t3tools/contracts";
+import { assert, describe, it } from "@effect/vitest";
+import * as Context from "effect/Context";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as TestClock from "effect/testing/TestClock";
+import * as HttpClient from "effect/http/HttpClient";
+import * as HttpClientResponse from "effect/http/HttpClientResponse";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as NodeHttp from "node:http";
+import * as NodeNet from "node:net";
+
+import * as ServerConfig from "../config.ts";
+import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as Sqlite from "../persistence/Sqlite.ts";
+import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts";
+import { PeerEnvironmentRegistryError } from "./PeerEnvironmentRegistry.ts";
+import { descriptorOf } from "./PeerLinks.testkit.ts";
+
+const self = descriptorOf("environment-self", "Self Here");
+
+const layerSelfEnvironment = Layer.succeed(ServerEnvironment.ServerEnvironment, {
+ getEnvironmentId: Effect.succeed(self.environmentId),
+ getDescriptor: Effect.succeed(self),
+});
+
+/** A paired peer's delegation session, as the exchange grants it. */
+const GRANTED_SCOPES = "orchestration:read orchestration:operate terminal:operate";
+
+interface FakePeer {
+ readonly http: HttpClient.HttpClient;
+ /** How often its descriptor was asked for. */
+ readonly probes: () => number;
+ /** The form bodies of the token-exchange requests it answered. */
+ readonly exchanges: () => ReadonlyArray>;
+}
+
+interface FakeToken {
+ readonly scope?: string;
+ /** The access token it mints; defaults to one fixed value. */
+ readonly accessToken?: string | (() => string);
+ readonly status?: number;
+ /** Only this origin answers the descriptor probe; default: every origin. */
+ readonly answersAt?: string;
+}
+
+/**
+ * The peer as the registry sees it over its HTTP client: a descriptor probe
+ * and the bootstrap token exchange. No socket is opened.
+ */
+const fakePeer = (descriptor: ExecutionEnvironmentDescriptor, token: FakeToken = {}): FakePeer => {
+ let probes = 0;
+ const exchanges: Array> = [];
+ const http = HttpClient.make((request) => {
+ const url = new URL(request.url);
+ if (url.pathname === "/.well-known/t3/environment") {
+ probes += 1;
+ if (token.answersAt !== undefined && url.origin !== token.answersAt) {
+ return Effect.succeed(
+ HttpClientResponse.fromWeb(request, new Response(null, { status: 404 })),
+ );
+ }
+ return Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(JSON.stringify(descriptor), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ ),
+ );
+ }
+ if (url.pathname === "/oauth/token") {
+ // The exchange posts the payload as urlencoded text.
+ const raw =
+ request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "";
+ exchanges.push(Object.fromEntries(new URLSearchParams(raw)));
+ const accessToken =
+ typeof token.accessToken === "function"
+ ? token.accessToken()
+ : (token.accessToken ?? "session-token");
+ return Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(
+ JSON.stringify({
+ access_token: accessToken,
+ issued_token_type: "urn:ietf:params:oauth:token-type:access_token",
+ token_type: "Bearer",
+ expires_in: 3600,
+ scope: token.scope ?? GRANTED_SCOPES,
+ }),
+ {
+ status: token.status ?? 200,
+ headers: { "content-type": "application/json" },
+ },
+ ),
+ ),
+ );
+ }
+ return Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 404 })));
+ });
+ return { http, probes: () => probes, exchanges: () => exchanges };
+};
+
+const fakeContext = (
+ descriptor: ExecutionEnvironmentDescriptor,
+ token: FakeToken = {},
+ openshellEnabled = true,
+) =>
+ buildContext(
+ Layer.succeed(HttpClient.HttpClient, fakePeer(descriptor, token).http),
+ openshellEnabled,
+ );
+
+const buildContext = (http: Layer.Layer, openshellEnabled = true) =>
+ Layer.build(
+ PeerEnvironmentRegistry.layer.pipe(
+ Layer.provide(http),
+ Layer.provideMerge(layerSelfEnvironment),
+ Layer.provideMerge(ServerSettings.layerTest({ openshellEnabled })),
+ Layer.provideMerge(Sqlite.layerMemory),
+ Layer.provideMerge(ServerSecretStore.layer),
+ Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-fleet-registry-" })),
+ Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
+ Layer.fresh,
+ ),
+ );
+
+const workerPeer = descriptorOf("environment-worker-1", "Worker One");
+
+const worker = (urls: ReadonlyArray = ["http://127.0.0.1:8111"]) => ({
+ environmentId: workerPeer.environmentId,
+ role: "worker" as const,
+ label: "Worker One",
+ urls,
+ workerClass: "ephemeral" as const,
+});
+
+const reasonOf = (error: unknown) => (error as PeerEnvironmentRegistryError).reason;
+
+/** A peer whose descriptor answers but whose token exchange never does. */
+const silentExchangeContext = (descriptor: ExecutionEnvironmentDescriptor) =>
+ buildContext(
+ Layer.succeed(
+ HttpClient.HttpClient,
+ HttpClient.make((request) =>
+ request.url.endsWith("/.well-known/t3/environment")
+ ? Effect.succeed(
+ HttpClientResponse.fromWeb(
+ request,
+ new Response(JSON.stringify(descriptor), {
+ status: 200,
+ headers: { "content-type": "application/json" },
+ }),
+ ),
+ )
+ : Effect.never,
+ ),
+ ),
+ );
+
+describe("PeerEnvironmentRegistry", () => {
+ it.effect("registers, lists and forgets a peer worker", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* fakeContext(self),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ assert.equal(registered.environmentId, workerPeer.environmentId);
+ assert.equal(registered.role, "worker");
+ assert.equal(registered.workerClass, "ephemeral");
+ assert.deepEqual(registered.urls, ["http://127.0.0.1:8111"]);
+ assert.isTrue(Option.isNone(registered.session));
+
+ assert.equal((yield* registry.list).length, 1);
+
+ assert.isTrue(yield* registry.unregister(registered.environmentId));
+ assert.deepEqual(yield* registry.list, []);
+ assert.isTrue(Option.isNone(yield* registry.get(registered.environmentId)));
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("registers a managing instance without a worker class", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* fakeContext(self),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const managing = yield* registry.register({
+ environmentId: descriptorOf("environment-manager", "Manager").environmentId,
+ role: "managing",
+ label: "Manager",
+ urls: ["http://127.0.0.1:8222"],
+ });
+ assert.isNull(managing.workerClass);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses to manage peers while the fleet beta is off", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* fakeContext(self, {}, false),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const refused = yield* registry.register(worker()).pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "disabled");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses to register this environment as its own peer", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* fakeContext(self),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const refused = yield* registry
+ .register({ ...worker(), environmentId: self.environmentId })
+ .pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "self");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("exchanges a bootstrap credential for a stored delegation session", () =>
+ Effect.gen(function* () {
+ const peer = fakePeer(workerPeer);
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+
+ const paired = yield* registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "one-time-bootstrap-credential",
+ });
+ const session = paired.session;
+ assert.isTrue(Option.isSome(session));
+ assert.deepEqual(Option.getOrThrow(session).scopes, [
+ "orchestration:read",
+ "orchestration:operate",
+ "terminal:operate",
+ ]);
+ assert.equal(paired.label, "Worker One");
+ // The probe proved the address, then the exchange used it once.
+ assert.equal(peer.probes(), 1);
+ assert.equal(peer.exchanges().length, 1);
+ const exchange = peer.exchanges()[0]!;
+ assert.equal(exchange.grant_type, "urn:ietf:params:oauth:grant-type:token-exchange");
+ assert.equal(exchange.subject_token, "one-time-bootstrap-credential");
+ assert.equal(
+ exchange.subject_token_type,
+ "urn:t3:params:oauth:token-type:environment-bootstrap",
+ );
+ assert.equal(exchange.scope, GRANTED_SCOPES);
+
+ // The bridge resolves the stored session, with the token from the peer.
+ const resolved = yield* registry.resolveSession(registered.environmentId);
+ assert.equal(resolved.token, "session-token");
+ assert.equal(resolved.url, "http://127.0.0.1:8111");
+ assert.equal(resolved.label, "Worker One");
+
+ yield* TestClock.adjust(Duration.seconds(3600));
+ const expired = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip);
+ assert.equal(expired.reason, "not_paired");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("binds the session to the verified address and drops it when addresses change", () =>
+ Effect.gen(function* () {
+ const peer = fakePeer(workerPeer, { answersAt: "http://127.0.0.1:8112" });
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const urls = ["http://127.0.0.1:8111", "http://127.0.0.1:8112"];
+ const registered = yield* registry.register(worker(urls));
+ yield* registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "one-time-bootstrap-credential",
+ });
+ assert.equal(
+ (yield* registry.resolveSession(registered.environmentId)).url,
+ "http://127.0.0.1:8112",
+ );
+
+ // Re-registering the same addresses keeps the session and its verified address.
+ yield* registry.register(worker(urls));
+ assert.equal(
+ (yield* registry.resolveSession(registered.environmentId)).url,
+ "http://127.0.0.1:8112",
+ );
+
+ yield* registry.register(worker(["http://127.0.0.1:9999"]));
+ const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip);
+ assert.equal(refused.reason, "not_paired");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("compares address sets without duplicates when keeping a session", () =>
+ Effect.gen(function* () {
+ const peer = fakePeer(workerPeer, { answersAt: "http://127.0.0.1:8112" });
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const urls = ["http://127.0.0.1:8111", "http://127.0.0.1:8112"];
+ const registered = yield* registry.register(worker(urls));
+ yield* registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "one-time-bootstrap-credential",
+ });
+
+ // [A, A, B] is the same SET as the stored [A, B]: the session stays.
+ yield* registry.register(worker([urls[0]!, urls[0]!, urls[1]!]));
+ assert.equal(
+ (yield* registry.resolveSession(registered.environmentId)).url,
+ "http://127.0.0.1:8112",
+ );
+
+ // [A, A] is a DIFFERENT set from the stored [A, B]: the session is dropped.
+ yield* registry.register(worker([urls[0]!, urls[0]!]));
+ const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip);
+ assert.equal(refused.reason, "not_paired");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("re-pairing replaces the stored session token", () =>
+ Effect.gen(function* () {
+ let issued = 0;
+ const peer = fakePeer(workerPeer, { accessToken: () => `session-token-${(issued += 1)}` });
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ yield* registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "first-credential",
+ });
+ assert.equal(
+ (yield* registry.resolveSession(registered.environmentId)).token,
+ "session-token-1",
+ );
+
+ yield* registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "second-credential",
+ });
+ const resolved = yield* registry.resolveSession(registered.environmentId);
+ assert.equal(resolved.token, "session-token-2");
+ // Re-pairing updates the row, not a second one.
+ assert.equal((yield* registry.list).length, 1);
+ assert.equal(peer.exchanges().length, 2);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses an exchange narrowed below the delegation scopes", () =>
+ Effect.gen(function* () {
+ const peer = fakePeer(workerPeer, { scope: "orchestration:read orchestration:operate" });
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ const refused = yield* registry
+ .exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "narrow-credential",
+ })
+ .pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "exchange_rejected");
+ const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId));
+ assert.isTrue(Option.isNone(unpaired.session));
+ assert.equal(peer.exchanges().length, 1);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses an exchange at an address that answers as another environment", () =>
+ Effect.gen(function* () {
+ const peer = fakePeer(descriptorOf("environment-stranger", "Stranger"));
+ const registry = Context.get(
+ yield* buildContext(Layer.succeed(HttpClient.HttpClient, peer.http)),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ const refused = yield* registry
+ .exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "credential",
+ })
+ .pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "unreachable");
+ const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId));
+ assert.isTrue(Option.isNone(unpaired.session));
+ assert.equal(peer.exchanges().length, 0);
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses to resolve a peer that never paired", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* fakeContext(self),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ const refused = yield* registry.resolveSession(registered.environmentId).pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "not_paired");
+ const unknown = yield* registry
+ .resolveSession(descriptorOf("environment-nowhere", "Nowhere").environmentId)
+ .pipe(Effect.flip);
+ assert.equal(reasonOf(unknown), "unknown_peer");
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("fails the exchange with exchange_timeout when the verified peer hangs", () =>
+ Effect.gen(function* () {
+ const registry = Context.get(
+ yield* silentExchangeContext(workerPeer),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker());
+ const fiber = yield* Effect.forkScoped(
+ registry.exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "credential",
+ }),
+ );
+ yield* TestClock.adjust(Duration.seconds(16));
+ const refused = yield* Fiber.join(fiber).pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "exchange_timeout");
+ const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId));
+ assert.isTrue(Option.isNone(unpaired.session));
+ }).pipe(Effect.scoped),
+ );
+
+ it.effect("refuses a redirect after a verified descriptor and forwards no token", () =>
+ Effect.gen(function* () {
+ const received: Array = [];
+ let serverPort = 0;
+ const server = yield* Effect.acquireRelease(
+ Effect.callback((resume) => {
+ const listening = NodeHttp.createServer((request, response) => {
+ const url = request.url ?? "";
+ if (url === "/.well-known/t3/environment") {
+ response.writeHead(200, { "content-type": "application/json" });
+ response.end(JSON.stringify(workerPeer));
+ return;
+ }
+ if (url === "/oauth/token") {
+ // A 307 after the descriptor: the redirect target must get nothing.
+ response.writeHead(307, {
+ location: `http://127.0.0.1:${serverPort}/evil/token`,
+ });
+ response.end();
+ return;
+ }
+ received.push(request.headers.authorization ?? "");
+ response.writeHead(200, { "content-type": "application/json" });
+ response.end(JSON.stringify({ access_token: "stolen", scope: GRANTED_SCOPES }));
+ });
+ listening.listen(0, "127.0.0.1", () => {
+ serverPort = (listening.address() as NodeNet.AddressInfo).port;
+ resume(Effect.succeed(listening));
+ });
+ }),
+ (listening) => Effect.sync(() => listening.close()),
+ );
+ const port = (server.address() as NodeNet.AddressInfo).port;
+ const registry = Context.get(
+ yield* buildContext(FetchHttpClient.layer),
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry,
+ );
+ const registered = yield* registry.register(worker([`http://127.0.0.1:${port}`]));
+ const refused = yield* registry
+ .exchangeBootstrap({
+ environmentId: registered.environmentId,
+ bootstrapCredential: "credential",
+ })
+ .pipe(Effect.flip);
+ assert.equal(reasonOf(refused), "exchange_rejected");
+ // The redirect target received no request, so no credential reached it.
+ assert.deepEqual(received, []);
+ const unpaired = Option.getOrThrow(yield* registry.get(registered.environmentId));
+ assert.isTrue(Option.isNone(unpaired.session));
+ }).pipe(Effect.scoped),
+ );
+});
diff --git a/apps/server/src/peer/PeerEnvironmentRegistry.ts b/apps/server/src/peer/PeerEnvironmentRegistry.ts
new file mode 100644
index 0000000000000..7078959413100
--- /dev/null
+++ b/apps/server/src/peer/PeerEnvironmentRegistry.ts
@@ -0,0 +1,588 @@
+import {
+ AuthAccessTokenType,
+ AuthEnvironmentBootstrapTokenType,
+ AuthTokenExchangeGrantType,
+ AuthOrchestrationOperateScope,
+ AuthOrchestrationReadScope,
+ AuthTerminalOperateScope,
+ EnvironmentHttpApi,
+ EnvironmentId,
+ type ExecutionEnvironmentDescriptor,
+ type PeerLinkError,
+ OpenShellWorkerClass,
+} from "@t3tools/contracts";
+import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
+import * as Context from "effect/Context";
+import * as SecureRandom from "effect/crypto/SecureRandom";
+import * as DateTime from "effect/DateTime";
+import * as Duration from "effect/Duration";
+import * as Effect from "effect/Effect";
+import * as Hex from "effect/encoding/Hex";
+import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as Result from "effect/Result";
+import * as Schema from "effect/Schema";
+import * as SqlClient from "effect/sql/SqlClient";
+import * as FetchHttpClient from "effect/http/FetchHttpClient";
+import * as HttpClient from "effect/http/HttpClient";
+import * as HttpApi from "effect/http-api/HttpApi";
+import * as HttpApiClient from "effect/http-api/HttpApiClient";
+import { isLocalLoopbackHost } from "@t3tools/shared/hostClassification";
+
+import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
+import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
+import * as ServerSettings from "../serverSettings.ts";
+import { peerOrigin } from "./PeerLinks.ts";
+
+const PeerHttpApi = HttpApi.make(EnvironmentHttpApi.identifier)
+ .add(EnvironmentHttpApi.groups.metadata)
+ .add(EnvironmentHttpApi.groups.auth)
+ .annotateMerge(EnvironmentHttpApi.annotations);
+
+/** How long a peer's descriptor may take before an address counts as unreachable. */
+const probeTimeout = (origin: string) => {
+ // An unparseable origin is just an unreachable address; it gets the default.
+ const hostname = (() => {
+ try {
+ return new URL(origin).hostname;
+ } catch {
+ return null;
+ }
+ })();
+ return hostname !== null && isLocalLoopbackHost(hostname)
+ ? Duration.seconds(3)
+ : Duration.seconds(10);
+};
+
+/** The bootstrap exchange must not park a spawn on a silent verified peer. */
+const exchangeTimeout = Duration.seconds(15);
+
+/**
+ * Scopes a delegation session asks for: the narrowest set that launches,
+ * follows and steers threads on the peer; the peer may grant narrower.
+ * OPEN-QUESTION (spec section 5.2, lanes 2 and 7): final scope set.
+ */
+const DelegationSessionScopes = [
+ AuthOrchestrationReadScope,
+ AuthOrchestrationOperateScope,
+ AuthTerminalOperateScope,
+] as const;
+
+const OPENSHELL_FLEET_OFF =
+ "OpenShell fleet management is off on this machine. Turn it on in Settings → Labs.";
+
+export type PeerEnvironmentRole = "managing" | "worker";
+
+export interface PeerEnvironmentSession {
+ /** The scopes the peer actually granted, which never exceed the ask. */
+ readonly scopes: ReadonlyArray;
+ readonly expiresAt: DateTime.Utc;
+}
+
+/** One registered peer t3 instance: the managing instance or a worker. */
+export interface PeerEnvironment {
+ readonly environmentId: EnvironmentId;
+ readonly role: PeerEnvironmentRole;
+ readonly label: string;
+ readonly urls: ReadonlyArray;
+ /** Workers only; the managing instance has none. */
+ readonly workerClass: OpenShellWorkerClass | null;
+ readonly registeredAt: DateTime.Utc;
+ /** Present once a bootstrap exchange paired this environment with the peer. */
+ readonly session: Option.Option;
+ readonly lastError: string | null;
+}
+
+/** A delegation session ready to call the peer's bridge with. */
+export interface ResolvedPeerSession {
+ readonly environmentId: EnvironmentId;
+ readonly label: string;
+ /** The first registered address, which pairing proved answers as this peer. */
+ readonly url: string;
+ readonly token: string;
+}
+
+export class PeerEnvironmentRegistryError extends Schema.TaggedError()(
+ "PeerEnvironmentRegistryError",
+ {
+ reason: Schema.Literals([
+ "storage",
+ "disabled",
+ "unknown_peer",
+ "self",
+ "unreachable",
+ "not_paired",
+ "exchange_rejected",
+ "exchange_timeout",
+ ]),
+ message: Schema.String,
+ cause: Schema.optional(Schema.Defect()),
+ },
+) {}
+
+const isRegistryError = Schema.is(PeerEnvironmentRegistryError);
+
+/** Fails with a fixed storage error that keeps its cause; the log gets the details. */
+const failStorage = (cause: unknown) =>
+ Effect.logWarning("Fleet peer environment storage failed").pipe(
+ Effect.andThen(
+ Effect.fail(
+ new PeerEnvironmentRegistryError({
+ reason: "storage",
+ message:
+ "The fleet peer registry could not be read or saved here. The server log has details.",
+ cause,
+ }),
+ ),
+ ),
+ );
+
+const decodeUrls = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.String)));
+const encodeUrls = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(Schema.String)));
+const decodeWorkerClass = Schema.decodeUnknownEffect(Schema.NullOr(OpenShellWorkerClass));
+const decodeScopes = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.String)));
+const encodeScopes = Schema.encodeEffect(Schema.fromJsonString(Schema.Array(Schema.String)));
+
+/** A peer address that may receive a credential, or why it may not. */
+const originOf = (url: string): Effect.Effect =>
+ // peerOrigin's returns infer as a union of effects; one assertion narrows the
+ // error, which is only ever a PeerLinkError.
+ Effect.suspend(() => peerOrigin(url) as Effect.Effect).pipe(
+ Effect.mapError(
+ (error) =>
+ new PeerEnvironmentRegistryError({ reason: "unreachable", message: error.message }),
+ ),
+ );
+
+/**
+ * Registry of peer t3 instances this environment knows as a fleet: itself as
+ * the managing instance plus the workers it delegates to. Pairing trades a
+ * peer's one-time bootstrap credential for a delegation session; the token
+ * lives in the server secret store, the row keeps the rest.
+ */
+export class PeerEnvironmentRegistry extends Context.Service<
+ PeerEnvironmentRegistry,
+ {
+ /** Whether the OpenShell fleet beta (Settings → Labs) is on here. */
+ readonly enabled: Effect.Effect;
+ readonly register: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly role: PeerEnvironmentRole;
+ readonly label: string;
+ readonly urls: ReadonlyArray;
+ readonly workerClass?: OpenShellWorkerClass;
+ }) => Effect.Effect;
+ /**
+ * Forgets the peer here and removes its delegation session. The peer still
+ * lists this environment's session until it is revoked there, like
+ * unlinking a peer link.
+ */
+ readonly unregister: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect;
+ readonly list: Effect.Effect, PeerEnvironmentRegistryError>;
+ readonly get: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect, PeerEnvironmentRegistryError>;
+ /**
+ * Pairing bootstrap exchange: the peer's one-time bootstrap credential
+ * becomes a delegation session scoped to `DelegationSessionScopes`. The
+ * credential goes only to an address whose descriptor proves it is the
+ * registered peer. Re-pairing replaces the stored token.
+ */
+ readonly exchangeBootstrap: (input: {
+ readonly environmentId: EnvironmentId;
+ readonly bootstrapCredential: string;
+ }) => Effect.Effect;
+ /** The stored delegation session, ready to call the peer's bridge with. */
+ readonly resolveSession: (
+ environmentId: EnvironmentId,
+ ) => Effect.Effect;
+ }
+>()("t3/peer/PeerEnvironmentRegistry") {}
+
+interface PeerEnvironmentRow {
+ readonly environment_id: string;
+ readonly role: string;
+ readonly label: string;
+ readonly urls_json: string;
+ readonly worker_class: string | null;
+ readonly secret_name: string | null;
+ readonly session_scopes_json: string | null;
+ readonly session_expires_at: string | null;
+ readonly registered_at: string;
+ readonly last_error: string | null;
+}
+
+const toPeerEnvironment = Effect.fn("PeerEnvironmentRegistry.toPeerEnvironment")(function* (
+ row: PeerEnvironmentRow,
+) {
+ const session =
+ row.secret_name !== null && row.session_scopes_json !== null && row.session_expires_at !== null
+ ? Option.some({
+ scopes: yield* decodeScopes(row.session_scopes_json).pipe(Effect.catch(failStorage)),
+ expiresAt: DateTime.makeUnsafe(row.session_expires_at),
+ } satisfies PeerEnvironmentSession)
+ : Option.none();
+ return {
+ environmentId: EnvironmentId.make(row.environment_id),
+ role: row.role as PeerEnvironmentRole,
+ label: row.label,
+ urls: yield* decodeUrls(row.urls_json).pipe(Effect.catch(failStorage)),
+ workerClass: yield* decodeWorkerClass(row.worker_class).pipe(Effect.catch(failStorage)),
+ registeredAt: DateTime.makeUnsafe(row.registered_at),
+ session,
+ lastError: row.last_error,
+ } satisfies PeerEnvironment;
+});
+
+const make = Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+ const secrets = yield* ServerSecretStore.ServerSecretStore;
+ const random = yield* SecureRandom.SecureRandom;
+ const httpClient = yield* HttpClient.HttpClient;
+ const environment = yield* ServerEnvironment.ServerEnvironment;
+ const serverSettings = yield* ServerSettings.ServerSettingsService;
+ const enabled = serverSettings.getSettings.pipe(
+ Effect.map((settings) => settings.openshellEnabled),
+ Effect.orElseSucceed(() => false),
+ );
+
+ /** A fresh slot name for one peer's delegation session token; the row records it. */
+ const newSecretName = random.randomBytes(16).pipe(
+ Effect.map((bytes) => `fleet-peer-${Hex.encode(bytes)}`),
+ Effect.orDie,
+ );
+
+ // Only the origin `peerOrigin` checked may receive a bootstrap credential or
+ // token: a redirect could forward them anywhere, including plain http.
+ const peerHttpClient = HttpClient.transform(httpClient, (response) =>
+ Effect.provideService(response, FetchHttpClient.RequestInit, { redirect: "manual" }),
+ );
+
+ const clientFor = (origin: string) =>
+ HttpApiClient.make(PeerHttpApi, { baseUrl: origin }).pipe(
+ Effect.provideService(HttpClient.HttpClient, peerHttpClient),
+ );
+
+ /** The environment answering at `origin`, or nothing when it does not answer as T3 Code. */
+ const describe = (origin: string) =>
+ clientFor(origin).pipe(
+ Effect.flatMap((client) => client.metadata.descriptor()),
+ Effect.timeout(probeTimeout(origin)),
+ Effect.option,
+ );
+
+ const readRows = sql`
+ SELECT * FROM fleet_peer_environments ORDER BY label, environment_id
+ `.pipe(NodeSqliteClient.readOnly, Effect.catch(failStorage));
+
+ const readRow = (environmentId: EnvironmentId) =>
+ sql`
+ SELECT * FROM fleet_peer_environments WHERE environment_id = ${environmentId}
+ `.pipe(
+ NodeSqliteClient.readOnly,
+ Effect.catch(failStorage),
+ Effect.map((rows) => Option.fromNullishOr(rows[0])),
+ );
+
+ const requireRow = (environmentId: EnvironmentId) =>
+ readRow(environmentId).pipe(
+ Effect.flatMap(
+ Option.match({
+ onNone: () =>
+ Effect.fail(
+ new PeerEnvironmentRegistryError({
+ reason: "unknown_peer",
+ message: `${environmentId} is not a registered fleet peer.`,
+ }),
+ ),
+ onSome: Effect.succeed,
+ }),
+ ),
+ );
+
+ const selfEnvironmentId = environment.getEnvironmentId.pipe(
+ Effect.orElseSucceed(() => EnvironmentId.make("environment-self")),
+ );
+
+ const register: PeerEnvironmentRegistry["Service"]["register"] = Effect.fn(
+ "PeerEnvironmentRegistry.register",
+ )(function* (input) {
+ if (!(yield* enabled)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "disabled",
+ message: OPENSHELL_FLEET_OFF,
+ });
+ }
+ if (input.environmentId === (yield* selfEnvironmentId)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "self",
+ message: "That environment is this one.",
+ });
+ }
+ const urls = yield* Effect.forEach(input.urls, originOf);
+ const uniqueUrls = [...new Set(urls)];
+ const previous = yield* readRow(input.environmentId);
+ const previousUrls = Option.isSome(previous)
+ ? yield* decodeUrls(previous.value.urls_json).pipe(Effect.catch(failStorage))
+ : [];
+ // The same address SET (deduped) keeps the pairing: the session and its
+ // verified-first order stay. In the upsert below the keepSession branch
+ // leaves urls_json alone, so it cannot undo a bootstrap exchange that
+ // committed while this register ran.
+ const addressSet = new Set(uniqueUrls);
+ const previousSet = new Set(previousUrls);
+ const keepSession =
+ Option.isSome(previous) &&
+ addressSet.size === previousSet.size &&
+ [...addressSet].every((url) => previousSet.has(url));
+ const urlsJson = yield* encodeUrls(uniqueUrls).pipe(Effect.catch(failStorage));
+ const now = yield* DateTime.now;
+ yield* sql`
+ INSERT INTO fleet_peer_environments (
+ environment_id, role, label, urls_json, worker_class, secret_name,
+ session_scopes_json, session_expires_at, registered_at, last_error
+ ) VALUES (
+ ${input.environmentId}, ${input.role}, ${input.label},
+ ${urlsJson},
+ ${input.workerClass ?? null}, NULL, NULL, NULL, ${DateTime.formatIso(now)}, NULL
+ )
+ ON CONFLICT (environment_id) DO UPDATE SET
+ role = excluded.role,
+ label = excluded.label,
+ urls_json = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN urls_json ELSE excluded.urls_json END,
+ worker_class = excluded.worker_class,
+ secret_name = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN secret_name END,
+ session_scopes_json = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN session_scopes_json END,
+ session_expires_at = CASE WHEN ${keepSession ? 1 : 0} = 1 THEN session_expires_at END,
+ last_error = NULL
+ `.pipe(Effect.catch(failStorage));
+ // New addresses were never checked against this peer, so its token must not follow them.
+ if (Option.isSome(previous) && !keepSession && previous.value.secret_name !== null) {
+ yield* secrets.remove(previous.value.secret_name).pipe(Effect.ignore);
+ }
+ return Option.getOrThrow(yield* get(input.environmentId));
+ });
+
+ const unregister: PeerEnvironmentRegistry["Service"]["unregister"] = Effect.fn(
+ "PeerEnvironmentRegistry.unregister",
+ )(function* (environmentId) {
+ if (!(yield* enabled)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "disabled",
+ message: OPENSHELL_FLEET_OFF,
+ });
+ }
+ const row = yield* requireRow(environmentId);
+ // Remove the credential before forgetting the row. If removal fails, the
+ // row keeps secret_name so a later unregister retries credential cleanup.
+ if (row.secret_name !== null) {
+ yield* secrets.remove(row.secret_name).pipe(Effect.catch(failStorage));
+ }
+ const deleted = yield* sql<{ environment_id: string }>`
+ DELETE FROM fleet_peer_environments WHERE environment_id = ${environmentId}
+ RETURNING environment_id
+ `.pipe(Effect.catch(failStorage));
+ return deleted.length > 0;
+ });
+
+ const list: PeerEnvironmentRegistry["Service"]["list"] = Effect.gen(function* () {
+ const rows = yield* readRows;
+ return yield* Effect.forEach(rows, toPeerEnvironment);
+ });
+
+ const get: PeerEnvironmentRegistry["Service"]["get"] = (environmentId) =>
+ readRow(environmentId).pipe(
+ Effect.flatMap(
+ Option.match({
+ onNone: () => Effect.succeedNone,
+ onSome: (row) => Effect.asSome(toPeerEnvironment(row)),
+ }),
+ ),
+ );
+
+ const exchangeBootstrap: PeerEnvironmentRegistry["Service"]["exchangeBootstrap"] = Effect.fn(
+ "PeerEnvironmentRegistry.exchangeBootstrap",
+ )(function* (input) {
+ if (!(yield* enabled)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "disabled",
+ message: OPENSHELL_FLEET_OFF,
+ });
+ }
+ const row = yield* requireRow(input.environmentId);
+ const urls = yield* decodeUrls(row.urls_json).pipe(Effect.catch(failStorage));
+ // The credential goes only to an address whose descriptor proves it is
+ // this peer, so a reused address never receives it.
+ const checked = yield* Effect.forEach(
+ urls,
+ (url) =>
+ Effect.gen(function* () {
+ const origin = yield* originOf(url);
+ return { origin, descriptor: yield* describe(origin) };
+ }).pipe(Effect.result),
+ { concurrency: "unbounded" },
+ );
+ const answering = checked.flatMap((result) => {
+ if (!Result.isSuccess(result) || Option.isNone(result.success.descriptor)) return [];
+ const descriptor = result.success.descriptor.value;
+ const modeLimitsHold =
+ descriptor.capabilities.mcpModeLimitHeader === true &&
+ descriptor.capabilities.linkFence === true;
+ return descriptor.environmentId === input.environmentId && modeLimitsHold
+ ? [{ origin: result.success.origin, descriptor }]
+ : [];
+ });
+ const found = answering[0];
+ if (found === undefined) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "unreachable",
+ message: `${input.environmentId} did not answer as itself at ${urls.join(", ")}. No session was stored.`,
+ });
+ }
+ const descriptor = found.descriptor as ExecutionEnvironmentDescriptor;
+ // resolveSession sends the token to the first address, so that must be the verified one.
+ const verifiedUrlsJson = yield* encodeUrls([
+ found.origin,
+ ...urls.filter((url) => url !== found.origin),
+ ]).pipe(Effect.catch(failStorage));
+ const exchanged = yield* clientFor(found.origin).pipe(
+ Effect.flatMap((client) =>
+ client.auth.token({
+ headers: {},
+ payload: {
+ grant_type: AuthTokenExchangeGrantType,
+ subject_token: input.bootstrapCredential,
+ subject_token_type: AuthEnvironmentBootstrapTokenType,
+ requested_token_type: AuthAccessTokenType,
+ scope: [...DelegationSessionScopes].join(" "),
+ },
+ }),
+ ),
+ Effect.timeoutOption(exchangeTimeout),
+ Effect.flatMap(
+ Option.match({
+ onNone: () =>
+ Effect.fail(
+ new PeerEnvironmentRegistryError({
+ reason: "exchange_timeout",
+ message: `${input.environmentId} did not answer the bootstrap exchange in time. No session was stored.`,
+ }),
+ ),
+ onSome: Effect.succeed,
+ }),
+ ),
+ Effect.mapError((cause) =>
+ isRegistryError(cause)
+ ? cause
+ : new PeerEnvironmentRegistryError({
+ reason: "exchange_rejected",
+ message: `${descriptor.label} refused the delegation session exchange. Check its pairing credential and try again.`,
+ cause,
+ }),
+ ),
+ );
+ const granted = exchanged.scope.split(" ").filter((scope) => scope.length > 0);
+ const missing = DelegationSessionScopes.filter((scope) => !granted.includes(scope));
+ if (granted.length === 0 || missing.length > 0) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "exchange_rejected",
+ message: `${descriptor.label} granted a session without ${
+ missing.length > 0 ? missing.join(", ") : "any delegated scope"
+ }. Issue a pairing credential that carries them.`,
+ });
+ }
+ const expiresAt = DateTime.add(yield* DateTime.now, { seconds: exchanged.expires_in });
+ // The token goes under a name of its own before the row points at it, so
+ // the row and its token are replaced together.
+ const secretName = yield* newSecretName;
+ yield* secrets
+ .set(secretName, new TextEncoder().encode(exchanged.access_token))
+ .pipe(Effect.catch(failStorage));
+ // `RETURNING` closes the race: a peer unregistered mid-exchange yields no row,
+ // and the fresh secret is discarded instead of leaving an orphan token behind.
+ const updated = yield* sql<{ readonly secret_name: string | null }>`
+ UPDATE fleet_peer_environments
+ SET urls_json = ${verifiedUrlsJson},
+ secret_name = ${secretName}, session_scopes_json = ${yield* encodeScopes(granted).pipe(
+ Effect.catch(failStorage),
+ )},
+ session_expires_at = ${DateTime.formatIso(expiresAt)}, label = ${descriptor.label},
+ last_error = NULL
+ WHERE environment_id = ${input.environmentId} AND urls_json = ${row.urls_json}
+ RETURNING secret_name
+ `.pipe(
+ Effect.catch((cause) =>
+ secrets.remove(secretName).pipe(Effect.ignore, Effect.andThen(failStorage(cause))),
+ ),
+ );
+ if (updated.length === 0) {
+ yield* secrets.remove(secretName).pipe(Effect.ignore);
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "unknown_peer",
+ message: `${input.environmentId} was unregistered or changed addresses during pairing, so its session was not stored.`,
+ });
+ }
+ const previous = row;
+ // Pairing again replaces the old token; a racing exchange may leave its
+ // own behind, unused either way.
+ if (previous.secret_name !== null && previous.secret_name !== secretName) {
+ yield* secrets.remove(previous.secret_name).pipe(Effect.ignore);
+ }
+ return Option.getOrThrow(yield* get(input.environmentId));
+ });
+
+ const resolveSession: PeerEnvironmentRegistry["Service"]["resolveSession"] = Effect.fn(
+ "PeerEnvironmentRegistry.resolveSession",
+ )(function* (environmentId) {
+ // Resolution stays available while the beta is off, so work already
+ // running on a peer still finishes, like linked environments do.
+ const row = yield* requireRow(environmentId);
+ const peer = yield* toPeerEnvironment(row);
+ if (Option.isNone(peer.session) || row.secret_name === null) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "not_paired",
+ message: `${environmentId} has no delegation session yet. Pair it with a bootstrap credential from the peer.`,
+ });
+ }
+ if (DateTime.isLessThanOrEqualTo(peer.session.value.expiresAt, yield* DateTime.now)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "not_paired",
+ message: `The delegation session for ${peer.label} expired. Re-run t3_environment_spawn with the same clientRequestId (or without one, if it was spawned without) to re-pair the existing worker.`,
+ });
+ }
+ const stored = yield* secrets.get(row.secret_name).pipe(Effect.catch(failStorage));
+ if (Option.isNone(stored)) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "unknown_peer",
+ message: `The delegation session for ${peer.label} lost its credential. Pair it again.`,
+ });
+ }
+ const url = peer.urls[0];
+ if (url === undefined) {
+ return yield* new PeerEnvironmentRegistryError({
+ reason: "unreachable",
+ message: `${peer.label} has no address to reach it at.`,
+ });
+ }
+ return {
+ environmentId: peer.environmentId,
+ label: peer.label,
+ url,
+ token: new TextDecoder().decode(stored.value),
+ };
+ });
+
+ return PeerEnvironmentRegistry.of({
+ enabled,
+ register,
+ unregister,
+ list,
+ get,
+ exchangeBootstrap,
+ resolveSession,
+ });
+});
+
+export const layer = Layer.effect(PeerEnvironmentRegistry, make);
diff --git a/apps/server/src/peer/PeerForwarding.test.ts b/apps/server/src/peer/PeerForwarding.test.ts
index 1d3f563f35b08..1c13ef330be69 100644
--- a/apps/server/src/peer/PeerForwarding.test.ts
+++ b/apps/server/src/peer/PeerForwarding.test.ts
@@ -40,6 +40,7 @@ import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as RemoteDelegation from "./RemoteDelegation.ts";
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
import { liveThreadShell } from "../mcp/McpToolAccess.testkit.ts";
import * as PeerForwarding from "./PeerForwarding.ts";
@@ -239,7 +240,9 @@ const serveBox = (seen: Ref.Ref, linkSession: Ref.Ref) =>
servePeer(
box,
Layer.mergeAll(
- McpHttpServer.layerOrchestratorToolkit,
+ McpHttpServer.layerOrchestratorToolkit.pipe(
+ Layer.provide(OpenShellFleetService.layerDisabled),
+ ),
McpHttpServer.layerProjectRegistration,
McpHttpServer.layerThreadToolkit,
).pipe(
@@ -294,7 +297,7 @@ const mcpClient = McpSchema.McpServerClient.of({
const makeLaptop = Effect.gen(function* () {
const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build);
const layerHere = Layer.mergeAll(
- McpHttpServer.layerOrchestratorToolkit,
+ McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)),
McpHttpServer.layerProjectRegistration,
McpHttpServer.layerThreadToolkit,
).pipe(
diff --git a/apps/server/src/peer/PeerLinkRequests.test.ts b/apps/server/src/peer/PeerLinkRequests.test.ts
index 2eec22a90bfa0..12223f8390342 100644
--- a/apps/server/src/peer/PeerLinkRequests.test.ts
+++ b/apps/server/src/peer/PeerLinkRequests.test.ts
@@ -35,6 +35,7 @@ import { McpSchema, McpServer, Tool, Toolkit } from "effect/ai";
import * as ServerConfig from "../config.ts";
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
import * as McpToolAccess from "../mcp/McpToolAccess.ts";
import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
@@ -164,6 +165,7 @@ const makeLaptopWith = (
Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())),
);
const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe(
+ Layer.provide(OpenShellFleetService.layerDisabled),
Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
Layer.provideMerge(PeerLinkRequests.layer),
Layer.provide(Layer.mock(PeerForwarding.PeerForwarding)({})),
diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts
index 5325e5745c5d7..6684c3c31f7e7 100644
--- a/apps/server/src/peer/PeerLinks.ts
+++ b/apps/server/src/peer/PeerLinks.ts
@@ -172,7 +172,7 @@ const failStorage = (cause: unknown) =>
* code and then the link's token travel to it, so plain http is accepted only
* where the network already encrypts: this machine, or a Tailscale address.
*/
-const peerOrigin = (url: string) => {
+export const peerOrigin = (url: string) => {
const parsed = URL.parse(url);
if (parsed === null || (parsed.protocol !== "https:" && parsed.protocol !== "http:")) {
return Effect.fail(
diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts
index da28f01e375b9..f65cfe248fc56 100644
--- a/apps/server/src/peer/PeerMcpClient.test.ts
+++ b/apps/server/src/peer/PeerMcpClient.test.ts
@@ -1,9 +1,10 @@
import { NodeHttpServer } from "@effect/platform-node";
-import { expect, it } from "@effect/vitest";
+import { assert, expect, it } from "@effect/vitest";
import {
EnvironmentId,
OrchestrationV2LinkedSender,
OrchestratorMcpFailure,
+ PeerLinkError,
ThreadId,
} from "@t3tools/contracts";
import * as Context from "effect/Context";
@@ -24,6 +25,7 @@ import { HttpClient, HttpClientResponse } from "effect/http";
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
import * as McpToolAccess from "../mcp/McpToolAccess.ts";
+import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts";
import * as PeerLinks from "./PeerLinks.ts";
import * as PeerMcpClient from "./PeerMcpClient.ts";
import { descriptorOf, layerLinkingEnvironment, linkTo, servePeer } from "./PeerLinks.testkit.ts";
@@ -447,3 +449,102 @@ it.effect("bounds the session handshake, not a tool call that waits longer", ()
expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" });
}),
);
+
+it.effect("reaches an unlinked worker through its fleet delegation session", () =>
+ Effect.gen(function* () {
+ const requests: Array<{ readonly url: string; readonly authorization: string | undefined }> =
+ [];
+ let calls = 0;
+ const http = HttpClient.make((request) =>
+ Effect.sync(() => {
+ requests.push({ url: request.url, authorization: request.headers.authorization });
+ const step = requests.length;
+ const toolCallBody = JSON.stringify({
+ result: { content: [{ type: "text", text: '{"modes":"approval-required/plan"}' }] },
+ });
+ if (step === 1) {
+ return HttpClientResponse.fromWeb(
+ request,
+ new Response(toolCallBody, {
+ status: 200,
+ headers: { "mcp-session-id": "fleet-session" },
+ }),
+ );
+ }
+ calls += 1;
+ return HttpClientResponse.fromWeb(
+ request,
+ new Response(step === 2 ? "" : toolCallBody, {
+ status: step === 2 ? 202 : 200,
+ headers: { "mcp-session-id": "fleet-session" },
+ }),
+ );
+ }),
+ );
+ const fleet = descriptorOf("environment-fleet", "Fleet Worker");
+ const context = yield* Layer.build(
+ PeerMcpClient.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.succeed(HttpClient.HttpClient, http),
+ Layer.mock(PeerLinks.PeerLinks)({
+ resolve: (environmentId) =>
+ environmentId === fleet.environmentId
+ ? Effect.fail(
+ new PeerLinkError({
+ reason: "unknown_link",
+ message: "This environment is not linked.",
+ }),
+ )
+ : Effect.fail(
+ new PeerLinkError({
+ reason: "storage",
+ message: "Not part of this test.",
+ }),
+ ),
+ recordOutcome: () => Effect.void,
+ }),
+ Layer.succeed(PeerEnvironmentRegistry.PeerEnvironmentRegistry, {
+ enabled: Effect.succeed(true),
+ register: () => Effect.die("not part of this test"),
+ unregister: () => Effect.die("not part of this test"),
+ list: Effect.die("not part of this test"),
+ get: () => Effect.die("not part of this test"),
+ exchangeBootstrap: () => Effect.die("not part of this test"),
+ resolveSession: (environmentId) =>
+ environmentId === fleet.environmentId
+ ? Effect.succeed({
+ environmentId: fleet.environmentId,
+ label: fleet.label,
+ url: "http://127.0.0.1:8333",
+ token: "fleet-delegation-session-token",
+ })
+ : Effect.fail(
+ new PeerEnvironmentRegistry.PeerEnvironmentRegistryError({
+ reason: "unknown_peer",
+ message: "Not a registered peer.",
+ }),
+ ),
+ } satisfies Partial<
+ PeerEnvironmentRegistry.PeerEnvironmentRegistry["Service"]
+ > as never),
+ ),
+ ),
+ ),
+ );
+ const client = Context.get(context, PeerMcpClient.PeerMcpClient);
+ assert.deepEqual(
+ yield* client.call({
+ environmentId: fleet.environmentId,
+ tool: "probe",
+ arguments: {},
+ limits: { runtimeMode: "approval-required", interactionMode: "plan" },
+ success: Result,
+ }),
+ { modes: "approval-required/plan" },
+ );
+ // The worker's session carried the call; the registry session was its credential.
+ assert.include(requests[0]?.authorization ?? "", "fleet-delegation-session-token");
+ assert.equal(calls, 2);
+ }).pipe(Effect.scoped),
+);
diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts
index 25c3b70287ab1..8798f2e3f4463 100644
--- a/apps/server/src/peer/PeerMcpClient.ts
+++ b/apps/server/src/peer/PeerMcpClient.ts
@@ -1,5 +1,5 @@
import {
- type EnvironmentId,
+ EnvironmentId,
OrchestratorMcpFailure,
type PeerLinkError,
type ProviderInteractionMode,
@@ -19,7 +19,10 @@ import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
+import * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts";
+import type { ResolvedPeerSession } from "./PeerSessionResolve.ts";
import * as PeerLinks from "./PeerLinks.ts";
+import * as PeerSessionResolve from "./PeerSessionResolve.ts";
/** The only MCP protocol version a T3 Code `/mcp` registers. */
const PROTOCOL_VERSION = "2025-06-18";
@@ -85,6 +88,29 @@ const fromLinkError = (error: PeerLinkError) =>
message: error.message,
});
+/** How the agent hears a fleet registry problem when it asked to call a peer. */
+const fromRegistryError = (error: PeerEnvironmentRegistry.PeerEnvironmentRegistryError) =>
+ new OrchestratorMcpFailure({
+ code:
+ error.reason === "unknown_peer" || error.reason === "not_paired"
+ ? "invalid_request"
+ : "orchestration_error",
+ message: error.message,
+ });
+
+/**
+ * A worker's credential comes from pairing, not from the user's links, so the
+ * recovery differs: a link is re-linked in Settings, a worker is re-paired.
+ */
+const rejected = (from: "link" | "registry") =>
+ new OrchestratorMcpFailure({
+ code: "capability_denied",
+ message:
+ from === "registry"
+ ? "This fleet worker no longer accepts its delegation session. It may have been removed or re-paired; spawn a replacement, or re-pair it with a fresh bootstrap credential."
+ : "The linked environment no longer accepts this link. It may have been revoked there; link it again.",
+ });
+
/** Not final, so a task already running there resumes once the beta is on there again. */
const linkedEnvironmentsOffThere = new OrchestratorMcpFailure({
code: "orchestration_error",
@@ -109,6 +135,9 @@ const responseJson = (contentType: string | undefined, text: string) =>
const make = Effect.gen(function* () {
const links = yield* PeerLinks.PeerLinks;
+ // Optional so environments without a fleet registry build without it; where
+ // the registry is provided, its delegation sessions answer for their peers.
+ const registry = yield* Effect.serviceOption(PeerEnvironmentRegistry.PeerEnvironmentRegistry);
const httpClient = yield* HttpClient.HttpClient;
// One MCP session per peer and address, reused across calls. The peer keeps
// sessions in memory, so a restart there answers 404 and we start a new one.
@@ -128,26 +157,24 @@ const make = Effect.gen(function* () {
// A peer that renews a used link says when it now ends; one that does not sends nothing.
const recordLinkExpiry = (
- link: PeerLinks.ResolvedPeerLink["link"],
+ environmentId: EnvironmentId,
headers: Readonly>,
) =>
Option.match(
- Option.filter(
- Option.flatMap(
- Option.fromNullishOr(headers[McpInvocationContext.LINK_EXPIRES_AT_HEADER]),
- DateTime.make,
- ),
- (expiresAt) => !DateTime.Equivalence(expiresAt, link.expiresAt),
+ Option.flatMap(
+ Option.fromNullishOr(headers[McpInvocationContext.LINK_EXPIRES_AT_HEADER]),
+ DateTime.make,
),
{
onNone: () => Effect.void,
- onSome: (expiresAt) => links.recordExpiry(link.environmentId, expiresAt),
+ onSome: (expiresAt) => links.recordExpiry(environmentId, expiresAt),
},
);
const openSession = Effect.fn("PeerMcpClient.openSession")(
function* (
- { link, url, token }: PeerLinks.ResolvedPeerLink,
+ { url, token, from }: ResolvedPeerSession,
+ environmentId: EnvironmentId,
limitHeader: Record,
) {
const initialized = yield* post(
@@ -168,15 +195,17 @@ const make = Effect.gen(function* () {
const sessionId = initialized.headers["mcp-session-id"];
if (initialized.status === 403) return yield* linkedEnvironmentsOffThere;
if (initialized.status !== 200 || sessionId === undefined) {
- return yield* new OrchestratorMcpFailure({
- code: initialized.status === 401 ? "capability_denied" : "orchestration_error",
- message:
- initialized.status === 401
- ? "The linked environment no longer accepts this link. It may have been revoked there; link it again."
- : `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`,
- });
+ return yield* initialized.status === 401
+ ? rejected(from)
+ : new OrchestratorMcpFailure({
+ code: "orchestration_error",
+ message: `The linked environment refused to start an MCP session (HTTP ${initialized.status}).`,
+ });
+ }
+ // Only link-sourced sessions renew their link's expiry on use.
+ if (from === "link") {
+ yield* recordLinkExpiry(environmentId, initialized.headers);
}
- yield* recordLinkExpiry(link, initialized.headers);
yield* post(
url,
token,
@@ -189,11 +218,22 @@ const make = Effect.gen(function* () {
(effect, { url }) => Effect.timeout(effect, PeerLinks.probeTimeout(url)),
);
+ /**
+ * Where a peer is reached and with what credential: its link if one is
+ * stored, else its fleet registry delegation session. Both spellings reach
+ * the peer's `/mcp`, so the bridge, forwarding and delegation work the same
+ * for linked environments and registered workers.
+ */
+ const resolvePeer = (environmentId: EnvironmentId) =>
+ PeerSessionResolve.resolvePeerSession({ links, registry })(environmentId).pipe(
+ Effect.mapError((failure) =>
+ failure._tag === "link" ? fromLinkError(failure.error) : fromRegistryError(failure.error),
+ ),
+ );
+
const call: PeerMcpClient["Service"]["call"] = Effect.fn("PeerMcpClient.call")(
function* (input) {
- const resolved = yield* links
- .resolve(input.environmentId)
- .pipe(Effect.mapError(fromLinkError));
+ const resolved = yield* resolvePeer(input.environmentId);
const limitHeader = {
[McpInvocationContext.MODE_LIMIT_HEADER]: `${input.limits.runtimeMode}/${input.limits.interactionMode}`,
};
@@ -230,24 +270,19 @@ const make = Effect.gen(function* () {
},
);
const existing = (yield* Ref.get(sessions)).get(sessionKey);
- const sessionId = existing ?? (yield* openSession(resolved, limitHeader));
+ const sessionId =
+ existing ?? (yield* openSession(resolved, input.environmentId, limitHeader));
// The initialized session outlives a tool request, including a long wait or cancellation.
if (existing === undefined) {
yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, sessionId));
}
let response = yield* callOnce(sessionId);
if (response.status === 404) {
- const renewed = yield* openSession(resolved, limitHeader);
+ const renewed = yield* openSession(resolved, input.environmentId, limitHeader);
yield* Ref.update(sessions, (current) => new Map(current).set(sessionKey, renewed));
response = yield* callOnce(renewed);
}
- if (response.status === 401) {
- return yield* new OrchestratorMcpFailure({
- code: "capability_denied",
- message:
- "The linked environment no longer accepts this link. It may have been revoked there; link it again.",
- });
- }
+ if (response.status === 401) return yield* rejected(resolved.from);
if (response.status === 403) return yield* linkedEnvironmentsOffThere;
if (response.status !== 200) {
return yield* new OrchestratorMcpFailure({
@@ -255,7 +290,9 @@ const make = Effect.gen(function* () {
message: `The linked environment answered HTTP ${response.status}.`,
});
}
- yield* recordLinkExpiry(resolved.link, response.headers);
+ if (resolved.from === "link") {
+ yield* recordLinkExpiry(input.environmentId, response.headers);
+ }
const body = yield* decodeToolCallResponse(
responseJson(response.headers["content-type"], yield* response.text),
).pipe(
diff --git a/apps/server/src/peer/PeerSessionResolve.test.ts b/apps/server/src/peer/PeerSessionResolve.test.ts
new file mode 100644
index 0000000000000..8c73d52da232c
--- /dev/null
+++ b/apps/server/src/peer/PeerSessionResolve.test.ts
@@ -0,0 +1,87 @@
+import { describe, expect, it } from "@effect/vitest";
+import { EnvironmentId, PeerLinkError } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+
+import { PeerEnvironmentRegistryError } from "./PeerEnvironmentRegistry.ts";
+import { resolvePeerSession } from "./PeerSessionResolve.ts";
+
+const environmentId = EnvironmentId.make("environment:worker");
+
+const linkError = (reason: "unknown_link" | "expired") =>
+ new PeerLinkError({ reason, message: reason });
+
+const sessionError = () =>
+ new PeerEnvironmentRegistryError({ reason: "not_paired", message: "not paired" });
+
+const deps = (input: {
+ readonly link: Effect.Effect<{ url: string; token: string }, PeerLinkError>;
+ readonly session?: Effect.Effect<{ url: string; token: string }, PeerEnvironmentRegistryError>;
+}) => ({
+ links: { resolve: () => input.link } as never,
+ registry:
+ input.session === undefined
+ ? Option.none()
+ : Option.some({ resolveSession: () => input.session! } as never),
+});
+
+describe("resolvePeerSession", () => {
+ it.effect("prefers the ordinary link", () =>
+ Effect.gen(function* () {
+ const resolved = yield* resolvePeerSession(
+ deps({ link: Effect.succeed({ url: "http://link.test", token: "link-token" }) }),
+ )(environmentId);
+ expect(resolved).toEqual({ url: "http://link.test", token: "link-token", from: "link" });
+ }),
+ );
+
+ it.effect("falls back to the fleet delegation session for an unlinked worker", () =>
+ Effect.gen(function* () {
+ const resolved = yield* resolvePeerSession(
+ deps({
+ link: Effect.fail(linkError("unknown_link")),
+ session: Effect.succeed({ url: "http://worker.test", token: "session-token" }),
+ }),
+ )(environmentId);
+ expect(resolved).toEqual({
+ url: "http://worker.test",
+ token: "session-token",
+ from: "registry",
+ });
+ }),
+ );
+
+ it.effect("reports an expired link instead of using another credential", () =>
+ Effect.gen(function* () {
+ const failure = yield* resolvePeerSession(
+ deps({
+ link: Effect.fail(linkError("expired")),
+ session: Effect.succeed({ url: "http://worker.test", token: "session-token" }),
+ }),
+ )(environmentId).pipe(Effect.flip);
+ expect(failure._tag).toBe("link");
+ expect(failure._tag === "link" ? failure.error.reason : undefined).toBe("expired");
+ }),
+ );
+
+ it.effect("has no fallback without a registry", () =>
+ Effect.gen(function* () {
+ const failure = yield* resolvePeerSession(
+ deps({ link: Effect.fail(linkError("unknown_link")) }),
+ )(environmentId).pipe(Effect.flip);
+ expect(failure._tag).toBe("link");
+ }),
+ );
+
+ it.effect("reports a registry failure when the worker has no session", () =>
+ Effect.gen(function* () {
+ const failure = yield* resolvePeerSession(
+ deps({
+ link: Effect.fail(linkError("unknown_link")),
+ session: Effect.fail(sessionError()),
+ }),
+ )(environmentId).pipe(Effect.flip);
+ expect(failure._tag).toBe("registry");
+ }),
+ );
+});
diff --git a/apps/server/src/peer/PeerSessionResolve.ts b/apps/server/src/peer/PeerSessionResolve.ts
new file mode 100644
index 0000000000000..831495011e138
--- /dev/null
+++ b/apps/server/src/peer/PeerSessionResolve.ts
@@ -0,0 +1,49 @@
+import type { EnvironmentId } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Result from "effect/Result";
+
+import type * as PeerEnvironmentRegistry from "./PeerEnvironmentRegistry.ts";
+import type * as PeerLinks from "./PeerLinks.ts";
+
+/** Where a peer answer will come from, so a 401 can name the right recovery. */
+export interface ResolvedPeerSession {
+ readonly url: string;
+ readonly token: string;
+ readonly from: "link" | "registry";
+}
+
+/**
+ * The credential a peer answers with: its ordinary link first, else the fleet
+ * delegation session it was paired with. Both spellings reach the peer's
+ * `/mcp` and `/api`; only an `unknown_link` falls through, so an expired or
+ * unreachable link is reported instead of silently using another credential.
+ */
+export const resolvePeerSession = (deps: {
+ readonly links: PeerLinks.PeerLinks["Service"];
+ readonly registry: Option.Option;
+}) =>
+ Effect.fn("resolvePeerSession")(function* (environmentId: EnvironmentId) {
+ const fromLink = yield* Effect.result(deps.links.resolve(environmentId));
+ if (Result.isSuccess(fromLink)) {
+ return {
+ url: fromLink.success.url,
+ token: fromLink.success.token,
+ from: "link",
+ } satisfies ResolvedPeerSession;
+ }
+ if (Option.isNone(deps.registry) || fromLink.failure.reason !== "unknown_link") {
+ return yield* Effect.fail({
+ _tag: "link" as const,
+ error: fromLink.failure,
+ });
+ }
+ const session = yield* deps.registry.value
+ .resolveSession(environmentId)
+ .pipe(Effect.mapError((error) => ({ _tag: "registry" as const, error })));
+ return {
+ url: session.url,
+ token: session.token,
+ from: "registry",
+ } satisfies ResolvedPeerSession;
+ });
diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts
index 3c14fb427fc2e..fbd8d2ebbda6f 100644
--- a/apps/server/src/peer/RemoteDelegation.test.ts
+++ b/apps/server/src/peer/RemoteDelegation.test.ts
@@ -42,6 +42,7 @@ import { McpSchema, McpServer } from "effect/ai";
import * as ServerConfig from "../config.ts";
import * as McpHttpServer from "../mcp/McpHttpServer.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import * as McpInvocationContext from "../mcp/McpInvocationContext.ts";
import { idleThreadProjection, liveThreadShell } from "../mcp/McpToolAccess.testkit.ts";
import { CodexProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/CodexAdapterV2.ts";
@@ -570,7 +571,7 @@ const boxToolkitLayer = (thread: BoxThread) => {
),
});
return Layer.merge(
- McpHttpServer.layerOrchestratorToolkit,
+ McpHttpServer.layerOrchestratorToolkit.pipe(Layer.provide(OpenShellFleetService.layerDisabled)),
McpHttpServer.layerProjectRegistration,
).pipe(
Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
@@ -699,6 +700,7 @@ const makeLaptop = (
Layer.provideMerge(Layer.merge(layerOrchestrator, ServerSettings.layerTest())),
);
const layerHere = McpHttpServer.layerOrchestratorToolkit.pipe(
+ Layer.provide(OpenShellFleetService.layerDisabled),
Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
Layer.provideMerge(
RemoteDelegation.layer.pipe(
diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts
index 91b521fdb6036..ea5a82b2c6860 100644
--- a/apps/server/src/persistence/Migrations.ts
+++ b/apps/server/src/persistence/Migrations.ts
@@ -113,6 +113,9 @@ import Migration0102 from "./Migrations/102_ThreadItemCounts.ts";
import Migration0103 from "./Migrations/103_IssueTrackerAdditionalScopes.ts";
import Migration0104 from "./Migrations/104_ProjectionProjectsTrackedFolders.ts";
import Migration0105 from "./Migrations/105_TaskFolders.ts";
+import Migration0109 from "./Migrations/109_FleetPeerEnvironments.ts";
+import Migration0110 from "./Migrations/110_OpenShellFleetState.ts";
+import Migration0111 from "./Migrations/111_FleetDrainImports.ts";
const migrationEntries = [
[1, "OrchestrationEvents", Migration0001],
@@ -221,6 +224,9 @@ const migrationEntries = [
[103, "IssueTrackerAdditionalScopes", Migration0103],
[104, "ProjectionProjectsTrackedFolders", Migration0104],
[105, "TaskFolders", Migration0105],
+ [109, "FleetPeerEnvironments", Migration0109],
+ [110, "OpenShellFleetState", Migration0110],
+ [111, "FleetDrainImports", Migration0111],
] as const;
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
diff --git a/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts b/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts
new file mode 100644
index 0000000000000..48e1de0172738
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/109_FleetPeerEnvironments.ts
@@ -0,0 +1,24 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // Peer t3 instances this environment knows as a fleet: the managing instance
+ // itself and the workers it delegates to. The delegation session token is
+ // kept in the server secret store under `secret_name`, like a peer link.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS fleet_peer_environments (
+ environment_id TEXT PRIMARY KEY,
+ role TEXT NOT NULL,
+ label TEXT NOT NULL,
+ urls_json TEXT NOT NULL,
+ worker_class TEXT,
+ secret_name TEXT,
+ session_scopes_json TEXT,
+ session_expires_at TEXT,
+ registered_at TEXT NOT NULL,
+ last_error TEXT
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts b/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts
new file mode 100644
index 0000000000000..f5a81cbfb4983
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/110_OpenShellFleetState.ts
@@ -0,0 +1,25 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // Lifecycle of gateway-managed worker environments, tracked here so state
+ // survives a restart of the managing instance (spec section 5.5). Drain
+ // progress is the aggregate journal view; per-item acks land with the drain
+ // protocol itself.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS openshell_fleet_environments (
+ environment_id TEXT PRIMARY KEY,
+ state TEXT NOT NULL,
+ worker_class TEXT NOT NULL,
+ label TEXT,
+ sandbox_name TEXT,
+ sandbox_id TEXT,
+ drain_json TEXT,
+ expires_at INTEGER,
+ last_error TEXT,
+ updated_at TEXT NOT NULL
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts b/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts
new file mode 100644
index 0000000000000..f247a22d3c9ff
--- /dev/null
+++ b/apps/server/src/persistence/Migrations/111_FleetDrainImports.ts
@@ -0,0 +1,21 @@
+import * as Effect from "effect/Effect";
+import * as SqlClient from "effect/sql/SqlClient";
+
+export default Effect.gen(function* () {
+ const sql = yield* SqlClient.SqlClient;
+
+ // Drained thread bundles imported as archived threads: one row per imported
+ // thread, keyed by thread id, carrying the currency marker (environment,
+ // drain id, snapshot sequence) that decides idempotency and supersession.
+ yield* sql`
+ CREATE TABLE IF NOT EXISTS orchestration_v2_fleet_drain_imports (
+ thread_id TEXT PRIMARY KEY,
+ environment_id TEXT NOT NULL,
+ drain_id TEXT NOT NULL,
+ snapshot_sequence INTEGER,
+ imported_at TEXT NOT NULL,
+ lossy INTEGER NOT NULL DEFAULT 0,
+ oversize_attachments_skipped INTEGER NOT NULL DEFAULT 0
+ )
+ `;
+});
diff --git a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
index d2b0dafab8731..7c788c4c49d77 100644
--- a/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+++ b/apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
@@ -85,6 +85,9 @@ describe("V2 preview upgrade", () => {
[103, "IssueTrackerAdditionalScopes"],
[104, "ProjectionProjectsTrackedFolders"],
[105, "TaskFolders"],
+ [109, "FleetPeerEnvironments"],
+ [110, "OpenShellFleetState"],
+ [111, "FleetDrainImports"],
]);
assert.deepStrictEqual(yield* runMigrations(), []);
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -213,6 +216,9 @@ describe("V2 preview upgrade", () => {
[103, "IssueTrackerAdditionalScopes"],
[104, "ProjectionProjectsTrackedFolders"],
[105, "TaskFolders"],
+ [109, "FleetPeerEnvironments"],
+ [110, "OpenShellFleetState"],
+ [111, "FleetDrainImports"],
]);
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
);
diff --git a/apps/server/src/provider/AgentContextBudget.test.ts b/apps/server/src/provider/AgentContextBudget.test.ts
index 3b3f1459cb9dd..46828d0b0f9ad 100644
--- a/apps/server/src/provider/AgentContextBudget.test.ts
+++ b/apps/server/src/provider/AgentContextBudget.test.ts
@@ -11,6 +11,7 @@ import { AssetSigner } from "../assets/AssetSigner.ts";
import { ServerSecretStore } from "../auth/ServerSecretStore.ts";
import { ServerConfig } from "../config.ts";
import { DeviceService } from "../device/DeviceService.ts";
+import * as OpenShellFleetService from "../environment/OpenShellFleetService.ts";
import { ServerEnvironment } from "../environment/ServerEnvironment.ts";
import { GitManager } from "../git/GitManager.ts";
import { GitWorkflowService } from "../git/GitWorkflowService.ts";
@@ -66,6 +67,7 @@ const toolHandlerServices = Layer.mergeAll(
Layer.mock(McpProviderSessions)({}),
Layer.mock(OrchestratorV2)({}),
Layer.mock(PeerForwarding)({}),
+ Layer.mock(OpenShellFleetService.OpenShellFleetService)({}),
Layer.mock(PeerLinkRequests)({}),
Layer.mock(PeerLinks)({}),
Layer.mock(PreviewAutomationBroker)({}),
diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json
index e089e5a84218a..e560ee52f0b12 100644
--- a/apps/server/src/provider/agentContextBudget.json
+++ b/apps/server/src/provider/agentContextBudget.json
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 6245,
"codexAdditionalContextBytes": 7639,
- "mcpToolListJsonBytes": 59026
+ "mcpToolListJsonBytes": 60962
}
diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
index 86b13922946b0..90d5d0060cbc4 100644
--- a/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+++ b/apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
@@ -1159,6 +1159,50 @@ it.effect("checks a task's destination thread only when the destination changes"
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
);
+it.effect("refuses to schedule a sandbox workspace", () =>
+ Effect.gen(function* () {
+ const layer = ScheduledTaskService.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ ManagedPolicy.layerTest(),
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
+ Layer.mock(ThreadManagementService.ThreadManagementService)({
+ getThreadShell: () => Effect.succeed(null),
+ }),
+ Layer.mock(SecretRequests.SecretRequests)({
+ consume: () => Effect.succeed("secret"),
+ }),
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
+ namedProjectsRoot: "/projects",
+ isScratchProject: () => Effect.succeed(false),
+ }),
+ NodeCrypto.layer,
+ Scheduler.layer,
+ ),
+ ),
+ );
+ yield* Effect.gen(function* () {
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
+ const refused = yield* service
+ .upsert({
+ title: "Run in a worker",
+ prompt: "Check progress",
+ enabled: true,
+ schedule: { type: "interval", everyMs: 60_000 },
+ projectId: ProjectId.make("project:test"),
+ threadId: null,
+ workspaceStrategy: { type: "sandbox" },
+ modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ })
+ .pipe(Effect.flip);
+ assert.match(refused.message, /isolation 'openshell-auto'/);
+ assert.lengthOf((yield* service.list()).tasks, 0);
+ }).pipe(Effect.provide(layer));
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
+);
+
it.effect("a run interrupted right after it reserves the task does not block the next run", () =>
Effect.gen(function* () {
const launches = yield* Ref.make(0);
diff --git a/apps/server/src/scheduledTasks/ScheduledTaskService.ts b/apps/server/src/scheduledTasks/ScheduledTaskService.ts
index 68c95748d0ce8..ae6bde4d5e2ba 100644
--- a/apps/server/src/scheduledTasks/ScheduledTaskService.ts
+++ b/apps/server/src/scheduledTasks/ScheduledTaskService.ts
@@ -1479,6 +1479,18 @@ export const layer = Layer.effect(
// keep their run history, and so real load failures propagate instead
// of silently resetting an existing row.
const existingTask = yield* findTask(id);
+ // A sandbox workspace belongs to a fleet worker; a scheduled launch would
+ // run in the project root. Existing sandbox tasks stay editable, but a
+ // new task or a strategy change to sandbox is still refused.
+ if (
+ input.workspaceStrategy.type === "sandbox" &&
+ existingTask?.workspaceStrategy.type !== "sandbox"
+ ) {
+ return yield* taskError(
+ "A sandbox workspace runs in a fleet worker, so delegate the work with isolation 'openshell-auto' instead of scheduling it here.",
+ input.id === undefined ? {} : { taskId: input.id },
+ );
+ }
const threadMode = resolveThreadMode(input, existingTask);
const keptThreadId =
input.threadId ?? (input.unbindThread === true ? null : (existingTask?.threadId ?? null));
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index 720834730e99b..e4708c2e9e489 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -95,9 +95,12 @@ import * as RemoteDelegation from "./peer/RemoteDelegation.ts";
import * as PeerClientPairing from "./peer/PeerClientPairing.ts";
import * as PeerClientPairingHttp from "./peer/http.ts";
import * as PeerLinkRequests from "./peer/PeerLinkRequests.ts";
+import * as PeerEnvironmentRegistry from "./peer/PeerEnvironmentRegistry.ts";
import * as PeerLinks from "./peer/PeerLinks.ts";
import * as PeerMcpClient from "./peer/PeerMcpClient.ts";
import * as PeerThreadRelay from "./peer/PeerThreadRelay.ts";
+import * as OpenShellFleetStore from "./environment/OpenShellFleetStore.ts";
+import * as OpenShellFleetService from "./environment/OpenShellFleetService.ts";
import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts";
import * as PreviewAutomationBroker from "./mcp/PreviewAutomationBroker.ts";
import * as DeviceService from "./device/DeviceService.ts";
@@ -462,6 +465,18 @@ const layerIssueService = Layer.merge(IssueService.layer, IssueTrackerConnection
// One instance serves MCP link calls and refreshes snapshots on thread events.
const layerThreadIssueService = ThreadIssueService.layer;
+// One fleet service instance is shared by the environment routes and sweep.
+const layerConfiguredOpenShellFleetService = OpenShellFleetService.layer.pipe(
+ Layer.provide(PeerEnvironmentRegistry.layer),
+ Layer.provide(OpenShellFleetStore.layer),
+ Layer.provide(PeerLinks.layer),
+ Layer.provide(NodeServices.layer),
+ // Post-drain bundle imports write through the orchestration event sink;
+ // shared layer references keep one instance across the app.
+ Layer.provideMerge(RuntimeLayer.layerEventSink),
+ Layer.provideMerge(SqlitePersistence.layerConfig),
+);
+
const layerGitManager = GitManager.layer.pipe(
Layer.provide(Layer.merge(ProjectionStoreV2.layer, ProjectStore.layer)),
Layer.provide(layerPullRequestService.pipe(Layer.provide(RuntimeLayer.layerProjectService))),
@@ -922,13 +937,24 @@ const layerMakeRoutes = Layer.mergeAll(
),
),
Layer.provide(PreviewBrowser.layer),
- // Links to other environments, shared by clients (WebSocket) and agents (/mcp).
+ // Links to other environments, shared by Settings (WebSocket) and agents (/mcp).
+ // The fleet registry answers for registered workers, so the same bridge reaches them.
+ // Built here so the fleet state store re-reads what a restart interrupted at startup.
Layer.provide(
Layer.mergeAll(PeerForwarding.layer, PeerThreadRelay.layer, PeerClientPairing.layer).pipe(
Layer.provideMerge(PeerMcpClient.layer),
Layer.provideMerge(PeerLinks.layer),
+ Layer.provideMerge(PeerEnvironmentRegistry.layer),
+ Layer.provide(OpenShellFleetStore.layer),
),
),
+ // Fleet worker lifecycle behind the environment MCP tools (spec section 5).
+ Layer.provide(layerConfiguredOpenShellFleetService),
+ // Ephemeral workers self-retire: the periodic sweep shares the service
+ // instance above through layer memoization.
+ Layer.provide(
+ OpenShellFleetService.layerSweep.pipe(Layer.provide(layerConfiguredOpenShellFleetService)),
+ ),
Layer.provide(
SpeechService.layer.pipe(
Layer.provide(layerServerSettings),
diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts
index f837fd8ca9722..fbe69821deb94 100644
--- a/apps/server/src/serverSettings.test.ts
+++ b/apps/server/src/serverSettings.test.ts
@@ -1973,6 +1973,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
"cloudSpeechCredentials.openrouter",
"easDeviceHosts.[].accessToken",
"simlockDeviceHosts.[].agentToken",
+ "openshell.auth.clientSecret",
+ "openshell.auth.refreshToken",
+ "openshell.auth.clientKey",
"usageLimitSources.*.managementKey",
"textGenerators.*.apiKey",
]);
diff --git a/apps/server/src/settingsSecrets.ts b/apps/server/src/settingsSecrets.ts
index d433621f3b1d7..cac36b9d004ee 100644
--- a/apps/server/src/settingsSecrets.ts
+++ b/apps/server/src/settingsSecrets.ts
@@ -63,6 +63,17 @@ export interface SettingsSecret {
readonly environmentVariable?: string;
}
+/** Secret fields the gateway auth union holds, per mode; the other fields are not secrets. */
+const OPEN_SHELL_AUTH_SECRET_FIELDS: Readonly>> = {
+ "client-credentials": ["clientSecret"],
+ "oidc-pkce": ["refreshToken"],
+ mtls: ["clientKey"],
+};
+
+/** The slot a gateway auth secret lives in (spec section 4.2: never settings JSON). */
+export const openshellGatewayAuthSecretName = (field: string): string =>
+ `openshell-gateway-auth-${encode(field)}`;
+
type SecretValue = Pick;
type SecretSlotName = (field: string, serverHost: string | null) => string;
type SecretFieldsMapper = (
@@ -247,8 +258,30 @@ function mapSecretFieldGroups(
) as SimlockDeviceHostConfig,
);
+ // Gateway auth secrets live one level down (openshell.auth.), and which
+ // fields are secret depends on the mode. A literal is stored to its slot and
+ // redacted; a stored placeholder or a secret-provider reference passes through.
+ const openshell = settings.openshell;
+ const openshellAuth = openshell?.auth;
+ const openshellMapped =
+ openshell === undefined
+ ? settings.openshell
+ : {
+ ...openshell,
+ ...(openshellAuth === undefined
+ ? {}
+ : ({
+ auth: mapFields(
+ openshellAuth as Record,
+ OPEN_SHELL_AUTH_SECRET_FIELDS[openshellAuth.mode] ?? [],
+ (field) => openshellGatewayAuthSecretName(field),
+ ),
+ } as ServerSettings["openshell"])),
+ };
+
return {
...settings,
+ openshell: openshellMapped,
cloudSpeechCredentials: mapFields(
settings.cloudSpeechCredentials,
voiceSecretFields,
diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts
index d96b91f30d62a..f981995d83c91 100644
--- a/apps/server/src/ws.ts
+++ b/apps/server/src/ws.ts
@@ -2198,6 +2198,20 @@ export const layerWsRpc = (
"orchestration_v2.command_id": input.commandId,
"orchestration_v2.project_id": input.projectId,
}).pipe(
+ Effect.andThen(
+ Effect.flatMap(serverEnvironment.getEnvironmentId, (selfId) =>
+ input.environmentId === undefined || input.environmentId === selfId
+ ? Effect.void
+ : Effect.fail(
+ new OrchestrationV2ThreadLaunchError({
+ commandId: input.commandId,
+ projectId: input.projectId,
+ message:
+ "A thread launches in this environment. To run work in another environment, use delegate_task with target.environmentId; its result arrives when the task settles.",
+ }),
+ ),
+ ),
+ ),
Effect.andThen(
startup
.enqueueCommand(
diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx
index 293435976974a..2187207c941ea 100644
--- a/apps/web/src/components/settings/LabsSettingsPanel.tsx
+++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx
@@ -2,10 +2,10 @@ import { managedCapabilityDenial } from "@t3tools/client-runtime/contract-labels
import { useAuth } from "@clerk/react";
import { Link } from "@tanstack/react-router";
import { T, useGT, useMessages } from "gt-react";
-
import { hasCloudPublicConfig } from "~/cloud/publicConfig";
import { useT3ConnectAuthPrompt } from "../clerk/useT3ConnectAuthPrompt";
import { Button } from "../ui/button";
+import { Input } from "../ui/input";
import { Switch } from "../ui/switch";
import { ScopedSwitch } from "./ScopedSwitch";
import { SettingsPageContainer, SettingsRow, SettingsSection } from "./settingsLayout";
@@ -28,10 +28,165 @@ export function LabsSettingsPanel() {
+
);
}
+function OpenShellFleetSection() {
+ const gt = useGT();
+ return (
+
+
+
+ );
+}
+
+function OpenShellFleetRow() {
+ const searchableSetting = useSearchableSetting();
+ const settings = useScopedSettings();
+ const updateSettings = useUpdateScopedSettings();
+ const gt = useGT();
+ return (
+
+ Worker environments on an OpenShell gateway. This machine becomes the managing instance
+ that spawns, pairs, drains, and deletes fleet workers.
+
+ }
+ control={
+ updateSettings({ openshellEnabled: Boolean(checked) })}
+ aria-label={gt("OpenShell fleet")}
+ />
+ }
+ >
+ {settings.openshellEnabled ? (
+
+
+
+
+
+ ) : null}
+
+ );
+}
+
+function OpenShellGatewayUrlRow() {
+ const searchableSetting = useSearchableSetting();
+ const settings = useScopedSettings();
+ const updateSettings = useUpdateScopedSettings();
+ const gt = useGT();
+ const stored = settings.openshell?.gatewayUrl ?? "";
+ // `key` re-seeds the uncontrolled input when the stored value changes.
+ const commit = (raw: string) => {
+ const value = raw.trim();
+ if (value === stored) return;
+ if (value === "") return;
+ updateSettings({ openshell: { gatewayUrl: value } });
+ };
+ return (
+
+ commit(event.currentTarget.value)}
+ onKeyDown={(event) => {
+ if (event.key === "Enter") {
+ event.preventDefault();
+ commit(event.currentTarget.value);
+ }
+ }}
+ />
+
+ }
+ />
+ );
+}
+
+function OpenShellManagingRow() {
+ const searchableSetting = useSearchableSetting();
+ const settings = useScopedSettings();
+ const updateSettings = useUpdateScopedSettings();
+ const gt = useGT();
+ return (
+
+ Make this environment the managing instance. Only one managing instance can exist per
+ gateway; linked environments can never hold this role.
+
+ }
+ control={
+
+ updateSettings({ openshell: { managingInstance: Boolean(checked) } })
+ }
+ aria-label={gt("Managing instance")}
+ />
+ }
+ />
+ );
+}
+
+function OpenShellTemplateRow() {
+ const searchableSetting = useSearchableSetting();
+ const settings = useScopedSettings();
+ const updateSettings = useUpdateScopedSettings();
+ const gt = useGT();
+ const stored = settings.openshell?.defaultSandboxTemplate ?? "";
+ // `key` re-seeds the uncontrolled input when the stored value changes.
+ const commit = (raw: string) => {
+ const value = raw.trim();
+ if (value === stored) return;
+ if (value === "") return;
+ updateSettings({ openshell: { defaultSandboxTemplate: value } });
+ };
+ return (
+
+ commit(event.currentTarget.value)}
+ onKeyDown={(event) => {
+ if (event.key === "Enter") {
+ event.preventDefault();
+ commit(event.currentTarget.value);
+ }
+ }}
+ />
+
+ }
+ />
+ );
+}
+
function BoringSidebarRow() {
const searchableSetting = useSearchableSetting();
const settings = useScopedSettings();
diff --git a/apps/web/src/components/settings/ScheduledTasksSettings.tsx b/apps/web/src/components/settings/ScheduledTasksSettings.tsx
index 471dcdf0fe0df..3a8995619022d 100644
--- a/apps/web/src/components/settings/ScheduledTasksSettings.tsx
+++ b/apps/web/src/components/settings/ScheduledTasksSettings.tsx
@@ -107,6 +107,7 @@ import {
scheduleFromDraft,
scheduleLabel,
scheduledTaskDefaultModel,
+ leavesSandboxWorkspace,
weekdayShortLabel,
taskToDraft,
unbindDraftFromThread,
@@ -172,6 +173,7 @@ const WORKSPACE_MODE_LABELS: Record = {
worktree: msg("Create a new worktree"),
root: msg("Use the project checkout"),
existing_worktree: msg("Use a specific checkout"),
+ sandbox: msg("Fleet worker sandbox"),
};
const SCOPE_SUMMARY_TEMPLATES: Record = {
@@ -197,6 +199,7 @@ const EMPTY_DRAFT: DraftState = {
savedThreadId: "",
threadMode: "new_thread",
workspaceMode: "worktree",
+ storedSandbox: false,
baseRef: "",
startFromOrigin: true,
existingWorktreePath: "",
@@ -1667,11 +1670,13 @@ function ScheduledTaskEditorForm({
? { type: "root" }
: draft.workspaceMode === "existing_worktree"
? { type: "existing_worktree", worktreePath: draft.existingWorktreePath.trim() }
- : {
- type: "worktree",
- baseRef: worktreeBaseRef,
- startFromOrigin: draft.startFromOrigin,
- };
+ : draft.workspaceMode === "sandbox"
+ ? { type: "sandbox" }
+ : {
+ type: "worktree",
+ baseRef: worktreeBaseRef,
+ startFromOrigin: draft.startFromOrigin,
+ };
const input: ScheduledTaskUpsertInput = {
...(draft.editingId ? { id: draft.editingId as ScheduledTaskId, requireExisting: true } : {}),
title: draft.title.trim(),
@@ -1869,14 +1874,45 @@ function ScheduledTaskEditorForm({