diff --git a/apps/server/src/fork/homeProvisioner/codexOverlayControl.test.ts b/apps/server/src/fork/homeProvisioner/codexOverlayControl.test.ts new file mode 100644 index 0000000000000..865b149303ef0 --- /dev/null +++ b/apps/server/src/fork/homeProvisioner/codexOverlayControl.test.ts @@ -0,0 +1,110 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { CodexSettings } from "@t3tools/contracts"; +import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +import { + materializeCodexShadowHome, + resolveCodexHomeLayout, +} from "../../provider/Drivers/CodexHomeLayout.ts"; + +const decodeCodexSettings = Schema.decodeSync(CodexSettings); + +const linkTarget = Effect.fn("codexOverlayControl.test.linkTarget")(function* (path: string) { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.readLink(path).pipe(Effect.orElseSucceed(() => null)); +}); + +const setup = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-codex-control-" }); + const base = path.join(root, "base"); + const control = path.join(base, "app-server-control"); + yield* fileSystem.makeDirectory(control, { recursive: true }); + yield* fileSystem.writeFileString(path.join(control, "daemon.sock"), ""); + yield* fileSystem.writeFileString(path.join(base, "hooks.json"), "{}\n"); + const overlay = (name: string) => path.join(root, "overlays", name); + const materialize = (shadow: string) => + resolveCodexHomeLayout(decodeCodexSettings({ homePath: base, shadowHomePath: shadow })).pipe( + Effect.flatMap(materializeCodexShadowHome), + ); + return { fileSystem, path, base, control, overlay, materialize }; +}); + +it.layer(NodeServices.layer)("Codex overlay app-server-control", (it) => { + it.effect.skipIf(!symlinksSupported)( + "never links a new overlay's app-server-control to the shared home", + () => + Effect.gen(function* () { + const { path, base, overlay, materialize } = yield* setup; + const fresh = overlay("fresh"); + yield* materialize(fresh); + assert.strictEqual(yield* linkTarget(path.join(fresh, "app-server-control")), null); + assert.strictEqual( + yield* linkTarget(path.join(fresh, "hooks.json")), + path.join(base, "hooks.json"), + ); + }).pipe(Effect.scoped), + ); + + it.effect.skipIf(!symlinksSupported)( + "drops an older overlay's link into the shared home and keeps the shared daemon's files", + () => + Effect.gen(function* () { + const { fileSystem, path, control, overlay, materialize } = yield* setup; + const old = overlay("old"); + yield* fileSystem.makeDirectory(old, { recursive: true }); + yield* fileSystem.symlink(control, path.join(old, "app-server-control")); + yield* materialize(old); + yield* materialize(old); + assert.strictEqual(yield* linkTarget(path.join(old, "app-server-control")), null); + assert.isFalse(yield* fileSystem.exists(path.join(old, "app-server-control"))); + assert.isTrue(yield* fileSystem.exists(path.join(control, "daemon.sock"))); + }).pipe(Effect.scoped), + ); + + it.effect.skipIf(!symlinksSupported)( + "drops an overlay's link to an earlier base or through an aliased path", + () => + Effect.gen(function* () { + const { fileSystem, path, base, control, overlay, materialize } = yield* setup; + const root = path.dirname(base); + const earlier = path.join(root, "earlier-base", "app-server-control"); + yield* fileSystem.makeDirectory(earlier, { recursive: true }); + const alias = path.join(root, "base-alias"); + yield* fileSystem.symlink(base, alias); + const links = { + earlier, + aliased: path.join(alias, "app-server-control"), + }; + for (const [name, target] of Object.entries(links)) { + const shadow = overlay(name); + yield* fileSystem.makeDirectory(shadow, { recursive: true }); + yield* fileSystem.symlink(target, path.join(shadow, "app-server-control")); + yield* materialize(shadow); + assert.strictEqual(yield* linkTarget(path.join(shadow, "app-server-control")), null); + assert.isFalse(yield* fileSystem.exists(path.join(shadow, "app-server-control")), name); + } + assert.isTrue(yield* fileSystem.exists(path.join(control, "daemon.sock"))); + assert.isTrue(yield* fileSystem.exists(earlier)); + }).pipe(Effect.scoped), + ); + + it.effect.skipIf(!symlinksSupported)("keeps an overlay's own app-server-control", () => + Effect.gen(function* () { + const { fileSystem, path, overlay, materialize } = yield* setup; + const local = overlay("local"); + yield* fileSystem.makeDirectory(path.join(local, "app-server-control"), { + recursive: true, + }); + yield* fileSystem.writeFileString(path.join(local, "app-server-control", "own.sock"), ""); + yield* materialize(local); + assert.isTrue(yield* fileSystem.exists(path.join(local, "app-server-control", "own.sock"))); + }).pipe(Effect.scoped), + ); +}); diff --git a/apps/server/src/fork/homeProvisioner/homeProvisioner.test.ts b/apps/server/src/fork/homeProvisioner/homeProvisioner.test.ts new file mode 100644 index 0000000000000..60f27425767e9 --- /dev/null +++ b/apps/server/src/fork/homeProvisioner/homeProvisioner.test.ts @@ -0,0 +1,514 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + SecretProviderId, + type HomeDeclarationWireV1, + type ManagedPolicySummary, + type ServerSettings, +} from "@t3tools/contracts"; +import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; +import type { AnyProviderDriver } from "@t3tools/provider-core/server/driver"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; +import * as TestClock from "effect/testing/TestClock"; + +import { CodexSettings, DEFAULT_SERVER_SETTINGS } from "@t3tools/contracts"; +import { resolveAntigravityInstanceDirectories } from "../../provider/antigravityAuthSupport.ts"; +import { + materializeCodexShadowHome, + resolveCodexHomeLayout, +} from "../../provider/Drivers/CodexHomeLayout.ts"; +import { + HOOK_STATE_DB_ENV, + PROVIDER_INSTANCE_ID_ENV, + withProviderProcessIdentity, + withProviderProcessIdentityEnv, +} from "../instanceIdentity.ts"; +import { deriveProviderInstanceConfigMap } from "../../provider/ProviderInstanceRegistryHydration.ts"; +import { + deriveDeclaredHomes, + makeHomeProvisionerRunner, + serverDeclarerId, + spawnHomeProvisioner, + type ProvisionerOutcome, + type ProvisionerRequest, + type ProvisionerWait, +} from "./homeProvisioner.ts"; + +const claude = ProviderDriverKind.make("claudeAgent"); +const codex = ProviderDriverKind.make("codex"); +const antigravity = ProviderDriverKind.make("antigravity"); +const instance = ProviderInstanceId.make; +const STATE_DB = "/srv/t3/userdata/statev2.sqlite"; +const decodeCodexSettings = Schema.decodeSync(CodexSettings); + +const settingsWith = ( + providerInstances: Record, + overrides: Partial = {}, +): ServerSettings => ({ + ...DEFAULT_SERVER_SETTINGS, + providerInstances: providerInstances as ServerSettings["providerInstances"], + ...overrides, +}); + +const request = (label: string): ProvisionerRequest => ({ + command: ["provisioner"], + declaration: { + version: 1, + declarer: "t3code:test", + homes: [{ home: { agent: "claude", configDir: label } }], + }, +}); + +describe("withProviderProcessIdentityEnv", () => { + it("adds the instance id and state database and never T3CODE_HOME", () => { + const environment = withProviderProcessIdentityEnv(instance("claude_work"), STATE_DB, [ + { name: "ANTHROPIC_BASE_URL", value: "http://proxy", sensitive: false }, + { name: PROVIDER_INSTANCE_ID_ENV, value: "spoofed", sensitive: false }, + ]); + assert.deepStrictEqual(environment, [ + { name: "ANTHROPIC_BASE_URL", value: "http://proxy", sensitive: false }, + { name: PROVIDER_INSTANCE_ID_ENV, value: "claude_work", sensitive: false }, + { name: HOOK_STATE_DB_ENV, value: STATE_DB, sensitive: false }, + ]); + assert.notInclude( + environment.map((variable) => variable.name), + "T3CODE_HOME", + ); + }); + + it.effect("reaches each driver's create call and a process spawned from it", () => + Effect.gen(function* () { + const created = yield* Ref.make>([]); + const driver = { + driverKind: claude, + create: (input: Parameters[0]) => + Ref.update(created, (names) => [ + ...names, + ...input.environment.map((variable) => `${variable.name}=${variable.value}`), + ]).pipe(Effect.as({} as never)), + } as unknown as AnyProviderDriver; + const [wrapped] = withProviderProcessIdentity([driver], STATE_DB); + const environment = withProviderProcessIdentityEnv(instance("claude_b"), STATE_DB, []); + yield* wrapped!.create({ + instanceId: instance("claude_b"), + displayName: undefined, + environment: [], + enabled: true, + config: {}, + }); + assert.deepStrictEqual(yield* Ref.get(created), [ + `${PROVIDER_INSTANCE_ID_ENV}=claude_b`, + `${HOOK_STATE_DB_ENV}=${STATE_DB}`, + ]); + + const processEnvironment = yield* mergeProviderInstanceEnvironment(environment, { + PATH: "/usr/bin:/bin", + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const printed = yield* spawner.string( + ChildProcess.make( + process.execPath, + [ + "-e", + "process.stdout.write(JSON.stringify([process.env.T3CODE_PROVIDER_INSTANCE_ID, process.env.T3CODE_HOOK_STATE_DB, process.env.T3CODE_HOME ?? null]))", + ], + { env: processEnvironment, extendEnv: false }, + ), + ); + assert.deepStrictEqual(JSON.parse(printed), ["claude_b", STATE_DB, null]); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("deriveDeclaredHomes", () => { + const derive = ( + settings: ServerSettings, + environment: NodeJS.ProcessEnv = {}, + managedPolicy?: ManagedPolicySummary, + ) => + deriveDeclaredHomes(deriveProviderInstanceConfigMap(settings), { + stateDir: "/srv/t3/userdata", + managedPolicy, + }).pipe( + Effect.provideService(HostProcess.HomeDirectory, "/home/agent"), + Effect.provideService(HostProcess.Environment, environment), + Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ); + + it.effect("lists each enabled instance's home from settings, including unavailable ones", () => + Effect.gen(function* () { + const derived = yield* derive( + settingsWith({ + claude_work: { driver: claude, config: { homePath: "~/slots/work/.claude" } }, + claude_locked: { + driver: claude, + environment: [ + { name: "CLAUDE_CONFIG_DIR", value: "/slots/locked", sensitive: false }, + { + name: "ANTHROPIC_API_KEY", + value: "", + sensitive: true, + secret: { provider: SecretProviderId.make("vault"), secretName: "missing" }, + }, + ], + }, + claude_off: { driver: claude, enabled: false, config: { homePath: "/slots/off" } }, + codex_overlay: { + driver: codex, + config: { homePath: "/codex/base", shadowHomePath: "/codex/overlays/a" }, + }, + codex_env: { + driver: codex, + environment: [{ name: "CODEX_HOME", value: "/codex/from-env", sensitive: false }], + }, + agy: { driver: antigravity, enabled: true }, + broken: { driver: claude, config: { homePath: 42 } }, + }), + ); + const agyProfile = (yield* resolveAntigravityInstanceDirectories( + "/srv/t3/userdata", + instance("agy"), + ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)))).profile; + + assert.deepStrictEqual(derived.homes, [ + { agent: "claude", configDir: "/home/agent/slots/work/.claude" }, + { agent: "claude", configDir: "/slots/locked" }, + { agent: "codex", codexHome: "/codex/base" }, + { agent: "codex", codexHome: "/codex/overlays/a" }, + { agent: "codex", codexHome: "/codex/from-env" }, + { agent: "agy", geminiDir: agyProfile }, + { agent: "codex", codexHome: "/home/agent/.codex" }, + { agent: "claude", configDir: "/home/agent/.claude" }, + ]); + assert.deepStrictEqual(derived.skipped, ["broken: its Claude settings do not decode."]); + }), + ); + + it.effect("declares a project-enabled instance and drops duplicate homes", () => + Effect.gen(function* () { + const derived = yield* derive( + settingsWith( + { + codex: { driver: codex, enabled: false }, + claudeAgent: { driver: claude, enabled: false }, + claude_project: { driver: claude, enabled: false, config: { homePath: "/p/.claude" } }, + claude_same: { driver: claude, config: { homePath: "/p/.claude" } }, + }, + { + projectSettingsOverrides: { + project: { providerInstances: { claude_project: { enabled: true } } }, + } as unknown as ServerSettings["projectSettingsOverrides"], + }, + ), + ); + assert.deepStrictEqual(derived.homes, [{ agent: "claude", configDir: "/p/.claude" }]); + }), + ); + + it.effect( + "declares a default instance a project turns on and skips drivers a policy blocks", + () => + Effect.gen(function* () { + const settings = settingsWith( + { codex: { driver: codex, enabled: false } }, + { + projectSettingsOverrides: { + project: { providerInstances: { antigravity: { enabled: true } } }, + } as unknown as ServerSettings["projectSettingsOverrides"], + }, + ); + const agyProfile = (yield* resolveAntigravityInstanceDirectories( + "/srv/t3/userdata", + instance("antigravity"), + ).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)))).profile; + assert.deepStrictEqual((yield* derive(settings)).homes, [ + { agent: "claude", configDir: "/home/agent/.claude" }, + { agent: "agy", geminiDir: agyProfile }, + ]); + const claudeOnly: ManagedPolicySummary = { + organization: "Example", + lockedSettings: [], + limits: { providers: { allowedDrivers: [claude] } }, + }; + assert.deepStrictEqual((yield* derive(settings, {}, claudeOnly)).homes, [ + { agent: "claude", configDir: "/home/agent/.claude" }, + ]); + }), + ); +}); + +describe("serverDeclarerId", () => { + it.effect("names the server by its real base folder", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-declarer-" }); + const base = path.join(root, "base"); + yield* fileSystem.makeDirectory(base); + const id = yield* serverDeclarerId(base); + assert.match(id, /^t3code:[0-9a-f]{8}$/); + if (symlinksSupported) { + yield* fileSystem.symlink(base, path.join(root, "alias")); + assert.strictEqual(yield* serverDeclarerId(path.join(root, "alias")), id); + } + assert.notStrictEqual(yield* serverDeclarerId(root), id); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); +}); + +describe("makeHomeProvisionerRunner", () => { + const gatedApply = Effect.gen(function* () { + const calls = yield* Ref.make>([]); + const started = yield* Deferred.make(); + const gate = yield* Deferred.make(); + const finished = yield* Ref.make(0); + const apply = (next: ProvisionerRequest) => + Effect.gen(function* () { + const label = + next.declaration.homes[0]?.home.agent === "claude" + ? next.declaration.homes[0].home.configDir + : ""; + yield* Ref.update(calls, (all) => [...all, label]); + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(gate); + yield* Ref.update(finished, (count) => count + 1); + return { kind: "applied", output: label } satisfies ProvisionerOutcome; + }); + return { calls, started, gate, finished, apply }; + }); + + it.effect("merges requests made during a run into one trailing run with the newest", () => + Effect.gen(function* () { + const fake = yield* gatedApply; + const runner = yield* makeHomeProvisionerRunner(fake.apply); + const first = yield* Effect.forkChild(runner.request(request("a"))); + yield* Deferred.await(fake.started); + const second = yield* Effect.forkChild(runner.request(request("b"))); + const third = yield* Effect.forkChild(runner.request(request("c"))); + // Past the cap every request has joined the runner, so the order below is fixed. + yield* TestClock.adjust("5 seconds"); + for (const waiting of [first, second, third]) { + assert.deepStrictEqual(yield* Fiber.join(waiting), { kind: "stillRunning" }); + } + yield* Deferred.succeed(fake.gate, undefined); + const after: ProvisionerWait = yield* runner.request(request("d")); + assert.deepStrictEqual(after, { + kind: "finished", + outcome: { kind: "applied", output: "d" }, + }); + assert.deepStrictEqual(yield* Ref.get(fake.calls), ["a", "c", "d"]); + }).pipe(Effect.scoped), + ); + + it.effect("stops waiting after the cap and lets the run finish", () => + Effect.gen(function* () { + const fake = yield* gatedApply; + const runner = yield* makeHomeProvisionerRunner(fake.apply); + const waiting = yield* Effect.forkChild(runner.request(request("slow"))); + yield* Deferred.await(fake.started); + yield* TestClock.adjust("5 seconds"); + assert.deepStrictEqual(yield* Fiber.join(waiting), { kind: "stillRunning" }); + assert.strictEqual(yield* Ref.get(fake.finished), 0); + + const next = yield* Effect.forkChild(runner.request(request("next"))); + yield* Deferred.succeed(fake.gate, undefined); + assert.deepStrictEqual(yield* Fiber.join(next), { + kind: "finished", + outcome: { kind: "applied", output: "next" }, + }); + assert.strictEqual(yield* Ref.get(fake.finished), 2); + assert.deepStrictEqual(yield* Ref.get(fake.calls), ["slow", "next"]); + }).pipe(Effect.scoped), + ); + + it.effect("turns a run that dies into a failed outcome and keeps taking requests", () => + Effect.gen(function* () { + const runs = yield* Ref.make(0); + const runner = yield* makeHomeProvisionerRunner((next) => + Ref.getAndUpdate(runs, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 0 + ? Effect.die(new Error("provisioner defect")) + : Effect.succeed({ + kind: "applied", + output: next.declaration.declarer, + }), + ), + ), + ); + const died = yield* runner.request(request("a")); + assert.strictEqual(died.kind === "finished" ? died.outcome.kind : died.kind, "failed"); + assert.include( + died.kind === "finished" && died.outcome.kind === "failed" ? died.outcome.detail : "", + "provisioner defect", + ); + assert.deepStrictEqual(yield* runner.request(request("b")), { + kind: "finished", + outcome: { kind: "applied", output: "t3code:test" }, + }); + }).pipe(Effect.scoped), + ); + + it.effect("frees the runner and ends every wait when its worker is interrupted", () => + Effect.gen(function* () { + const fake = yield* gatedApply; + const runnerScope = yield* Scope.make(); + const runner = yield* makeHomeProvisionerRunner(fake.apply).pipe(Scope.provide(runnerScope)); + const first = yield* Effect.forkChild(runner.request(request("a"))); + yield* Deferred.await(fake.started); + const second = yield* Effect.forkChild(runner.request(request("b"))); + yield* Effect.yieldNow; + yield* Scope.close(runnerScope, Exit.void); + for (const waiting of [first, second]) { + assert.deepStrictEqual(yield* Fiber.join(waiting), { + kind: "finished", + outcome: { kind: "failed", detail: "was stopped." }, + }); + } + }), + ); +}); + +describe("spawnHomeProvisioner", () => { + it.effect("writes the declaration to the command's stdin and reports its exit", () => + Effect.gen(function* () { + const echo = yield* spawnHomeProvisioner({ + command: [process.execPath, "-e", "process.stdin.pipe(process.stdout)"], + declaration: request("/x").declaration, + }); + assert.strictEqual(echo.kind, "applied"); + assert.deepStrictEqual( + JSON.parse(echo.kind === "applied" ? echo.output : "null") as HomeDeclarationWireV1, + request("/x").declaration, + ); + const failed = yield* spawnHomeProvisioner({ + command: [process.execPath, "-e", "console.error('no such declarer'); process.exit(3)"], + declaration: request("/x").declaration, + }); + assert.deepStrictEqual(failed, { kind: "failed", detail: "exited with 3: no such declarer" }); + }).pipe(Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps only the first 16 KiB of a command's output", () => + Effect.gen(function* () { + const loud = yield* spawnHomeProvisioner({ + command: [process.execPath, "-e", "process.stdout.write('x'.repeat(1 << 20))"], + declaration: request("/x").declaration, + }); + assert.strictEqual(loud.kind === "applied" ? loud.output.length : 0, 16 * 1024); + }).pipe(Effect.provide(NodeServices.layer)), + ); +}); + +describe("end-to-end home declaration delivery", () => { + it.live.skipIf(!symlinksSupported)( + "materializes Codex overlays and delivers Claude, Codex, and Antigravity homes to a stub provisioner", + () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-provision-" }); + const osHome = path.join(root, "home"); + const claudeHome = path.join(root, "slots", "claude-work"); + const codexBase = path.join(root, "codex-base"); + const overlayFresh = path.join(root, "overlays", "fresh"); + const overlayOld = path.join(root, "overlays", "old"); + const stateDir = path.join(root, "userdata"); + const recordedPath = path.join(root, "received-declaration.json"); + for (const directory of [osHome, claudeHome, codexBase, stateDir]) { + yield* fileSystem.makeDirectory(directory, { recursive: true }); + } + yield* fileSystem.writeFileString(path.join(codexBase, "config.toml"), ""); + yield* fileSystem.writeFileString(path.join(codexBase, "hooks.json"), "{}\n"); + yield* fileSystem.makeDirectory(path.join(codexBase, "app-server-control")); + yield* fileSystem.makeDirectory(overlayOld, { recursive: true }); + yield* fileSystem.writeFileString(path.join(overlayOld, "auth.json"), "{}\n"); + yield* fileSystem.symlink( + path.join(codexBase, "app-server-control"), + path.join(overlayOld, "app-server-control"), + ); + for (const overlay of [overlayFresh, overlayOld]) { + const layout = yield* resolveCodexHomeLayout( + decodeCodexSettings({ homePath: codexBase, shadowHomePath: overlay }), + ); + yield* materializeCodexShadowHome(layout); + assert.strictEqual( + yield* fileSystem.readLink(path.join(overlay, "hooks.json")), + path.join(codexBase, "hooks.json"), + ); + assert.isFalse( + yield* fileSystem.readLink(path.join(overlay, "app-server-control")).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ), + ); + } + + const settings = settingsWith({ + claudeAgent: { driver: claude, enabled: false }, + codex: { driver: codex, enabled: false }, + claude_work: { driver: claude, config: { homePath: claudeHome } }, + codex_fresh: { + driver: codex, + config: { homePath: codexBase, shadowHomePath: overlayFresh }, + }, + codex_old: { driver: codex, config: { homePath: codexBase, shadowHomePath: overlayOld } }, + agy: { driver: antigravity, enabled: true }, + }); + const derived = yield* deriveDeclaredHomes(deriveProviderInstanceConfigMap(settings), { + stateDir, + }).pipe( + Effect.provideService(HostProcess.HomeDirectory, osHome), + Effect.provideService(HostProcess.Environment, {}), + ); + + const stubScript = [ + "const fs = require('node:fs');", + "let data = '';", + "process.stdin.setEncoding('utf8');", + "process.stdin.on('data', (chunk) => { data += chunk; });", + "process.stdin.on('end', () => {", + " const parsed = JSON.parse(data);", + " if (parsed.version !== 1 || !Array.isArray(parsed.homes)) process.exit(2);", + " fs.writeFileSync(process.argv[1], JSON.stringify(parsed));", + "});", + ].join("\n"); + const declarer = yield* serverDeclarerId(stateDir); + const outcome = yield* spawnHomeProvisioner({ + command: [process.execPath, "-e", stubScript, recordedPath], + declaration: { + version: 1, + declarer, + homes: derived.homes.map((home) => ({ home })), + }, + }); + assert.strictEqual(outcome.kind, "applied", JSON.stringify(outcome)); + + const received = JSON.parse( + yield* fileSystem.readFileString(recordedPath), + ) as HomeDeclarationWireV1; + assert.strictEqual(received.version, 1); + assert.strictEqual(received.declarer, declarer); + assert.deepStrictEqual( + received.homes.map((entry) => entry.home), + derived.homes, + ); + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), + ); +}); diff --git a/apps/server/src/fork/homeProvisioner/homeProvisioner.ts b/apps/server/src/fork/homeProvisioner/homeProvisioner.ts new file mode 100644 index 0000000000000..9613ca882f2e7 --- /dev/null +++ b/apps/server/src/fork/homeProvisioner/homeProvisioner.ts @@ -0,0 +1,322 @@ +import { + ClaudeSettings, + CodexSettings, + ProviderInstanceId, + resolveProviderInstanceEnabled, + type DeclaredHome, + HomeDeclarationWireV1, + type HomeProvisionerCommand, + type ManagedPolicySummary, + type ProviderInstanceConfigMap, +} from "@t3tools/contracts"; +import { mergeProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Cause from "effect/Cause"; +import * as Digest from "effect/crypto/Digest"; +import * as Deferred from "effect/Deferred"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Hex from "effect/encoding/Hex"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; +import * as ChildProcess from "effect/process/ChildProcess"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import { disallowedDriverReason } from "../../policy/managedProviders.ts"; +import { resolveAntigravityInstanceDirectories } from "../../provider/antigravityAuthSupport.ts"; +import { resolveManagedCodexHomeLayout } from "../../provider/CodexManagedHome.ts"; +import { resolveClaudeHomePath } from "../../provider/Drivers/ClaudeHome.ts"; +import { resolveCodexHomeLayout } from "../../provider/Drivers/CodexHomeLayout.ts"; + +const decodeClaudeSettings = Schema.decodeUnknownOption(ClaudeSettings); +const decodeCodexSettings = Schema.decodeUnknownOption(CodexSettings); +const encodeDeclaration = Schema.encodeEffect(Schema.fromJsonString(HomeDeclarationWireV1)); + +const homeKey = (home: DeclaredHome): string => + home.agent === "claude" + ? `claude:${home.configDir}` + : home.agent === "codex" + ? `codex:${home.codexHome}` + : `agy:${home.geminiDir}`; + +export interface DerivedHomes { + readonly homes: ReadonlyArray; + /** Instances whose homes could not be read from their settings, one line each. */ + readonly skipped: ReadonlyArray; +} + +/** + * The homes of the enabled instances in the instance map the registry runs from, minus drivers + * the managed policy blocks. Live instances are not used: an instance waiting on a secret, or one + * whose driver failed to start, still owns its home, and dropping it would retire that home's + * hooks. Upstream resolvers name each home, so the list matches what each driver uses. + */ +export const deriveDeclaredHomes = Effect.fn("homeProvisioner.deriveDeclaredHomes")(function* ( + instances: ProviderInstanceConfigMap, + options: { + readonly stateDir: string; + readonly managedPolicy?: ManagedPolicySummary | undefined; + }, +) { + const hostEnvironment = yield* HostProcess.Environment; + const homes = new Map(); + const skipped: Array = []; + const add = (home: DeclaredHome) => { + if (!homes.has(homeKey(home))) homes.set(homeKey(home), home); + }; + + for (const [rawId, entry] of Object.entries(instances)) { + if (!resolveProviderInstanceEnabled(entry)) continue; + if (disallowedDriverReason(options.managedPolicy, entry.driver) !== undefined) continue; + const instanceId = ProviderInstanceId.make(rawId); + const environment = yield* mergeProviderInstanceEnvironment(entry.environment, hostEnvironment); + switch (entry.driver) { + case "claudeAgent": { + const config = decodeClaudeSettings(entry.config ?? {}); + if (Option.isNone(config)) { + skipped.push(`${instanceId}: its Claude settings do not decode.`); + break; + } + add({ + agent: "claude", + configDir: yield* resolveClaudeHomePath(config.value, environment), + }); + break; + } + case "codex": { + const decoded = decodeCodexSettings(entry.config ?? {}); + if (Option.isNone(decoded)) { + skipped.push(`${instanceId}: its Codex settings do not decode.`); + break; + } + const config = decoded.value; + // Same fallback as the Codex driver: an unset home path follows CODEX_HOME. + const layout = + config.setupMode === "managed" + ? yield* resolveManagedCodexHomeLayout(options.stateDir, instanceId, config) + : yield* resolveCodexHomeLayout( + config.homePath.trim().length === 0 && environment.CODEX_HOME?.trim() + ? { ...config, homePath: environment.CODEX_HOME } + : config, + ); + // The base owns the hooks.json and config.toml an overlay links, so it is declared first. + if (layout.mode === "authOverlay") + add({ agent: "codex", codexHome: layout.sharedHomePath }); + add({ agent: "codex", codexHome: layout.effectiveHomePath ?? layout.sharedHomePath }); + break; + } + case "antigravity": { + const directories = yield* resolveAntigravityInstanceDirectories( + options.stateDir, + instanceId, + ); + add({ agent: "agy", geminiDir: directories.profile }); + break; + } + } + } + return { homes: [...homes.values()], skipped } satisfies DerivedHomes; +}); + +/** Dev and packaged servers keep different base folders, so their declarations never clash. */ +export const serverDeclarerId = Effect.fn("homeProvisioner.serverDeclarerId")(function* ( + baseDir: string, +) { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const digest = yield* Digest.Digest; + const real = yield* fileSystem + .realPath(baseDir) + .pipe(Effect.orElseSucceed(() => path.resolve(baseDir))); + const hash = yield* digest.digest(Digest.sha256, new TextEncoder().encode(real)); + return `t3code:${Hex.encode(hash).slice(0, 8)}`; +}); + +export type ProvisionerOutcome = + | { readonly kind: "applied"; readonly output: string } + | { readonly kind: "failed"; readonly detail: string }; + +export interface ProvisionerRequest { + readonly command: HomeProvisionerCommand; + readonly declaration: HomeDeclarationWireV1; +} + +export type ProvisionerWait = + | { readonly kind: "finished"; readonly outcome: ProvisionerOutcome } + /** The run goes on in the background; a later request still waits for its own run. */ + | { readonly kind: "stillRunning" }; + +export interface HomeProvisionerRunner { + readonly request: (request: ProvisionerRequest) => Effect.Effect; +} + +const HARD_LIMIT = Duration.minutes(2); +const OUTPUT_LIMIT_BYTES = 16 * 1024; + +/** Reads the whole stream, so the child never blocks on a full pipe, and keeps its first 16 KiB. */ +const readText = (stream: Stream.Stream) => + Stream.runFold( + stream, + () => ({ chunks: [] as Array, size: 0 }), + (kept, chunk) => { + const room = OUTPUT_LIMIT_BYTES - kept.size; + if (room > 0) { + const part = chunk.subarray(0, room); + kept.chunks.push(part); + kept.size += part.length; + } + return kept; + }, + ).pipe(Effect.map((kept) => new TextDecoder().decode(Buffer.concat(kept.chunks)))); + +/** + * Runs the command once with the declaration on stdin. The 2-minute limit stops the command even + * halfway through writing a home. Never fails: errors become an outcome. + */ +export const spawnHomeProvisioner = Effect.fn("homeProvisioner.spawn")(function* ( + request: ProvisionerRequest, +) { + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const [command, ...args] = request.command; + return yield* Effect.scoped( + Effect.gen(function* () { + const input = yield* encodeDeclaration(request.declaration); + const child = yield* spawner.spawn( + ChildProcess.make(command!, args, { + stdin: { stream: Stream.encodeText(Stream.make(input)) }, + forceKillAfter: "5 seconds", + }), + ); + const [output, errors, exitCode] = yield* Effect.all( + [readText(child.stdout), readText(child.stderr), child.exitCode], + { concurrency: "unbounded" }, + ); + return exitCode === 0 + ? ({ kind: "applied", output: output.trim() } satisfies ProvisionerOutcome) + : ({ + kind: "failed", + detail: `exited with ${exitCode}: ${(errors.trim() || output.trim()).slice(0, 2000)}`, + } satisfies ProvisionerOutcome); + }), + ).pipe( + Effect.timeoutOrElse({ + duration: HARD_LIMIT, + orElse: () => + Effect.succeed({ + kind: "failed", + detail: `did not finish within ${Duration.format(HARD_LIMIT)}.`, + }), + }), + Effect.catch((cause) => + Effect.succeed({ kind: "failed", detail: cause.message }), + ), + ); +}); + +interface PendingRun { + readonly request: ProvisionerRequest; + readonly done: Deferred.Deferred; +} + +interface RunnerState { + readonly current: PendingRun | null; + /** The run that starts when the current one ends; later requests replace its request. */ + readonly trailing: PendingRun | null; +} + +const IDLE: RunnerState = { current: null, trailing: null }; + +/** + * One run at a time; requests made during a run merge into one trailing run with the newest + * request. A caller waits up to `awaitCap`, and the run goes on in the runner's scope after that. + * A run that dies becomes a failed outcome and an interrupted worker frees the runner. + */ +export const makeHomeProvisionerRunner = Effect.fn("homeProvisioner.makeRunner")(function* ( + apply: (request: ProvisionerRequest) => Effect.Effect, + options: { readonly awaitCap: Duration.Input } = { awaitCap: "5 seconds" }, +) { + const scope = yield* Effect.scope; + const state = yield* Ref.make(IDLE); + + const runOne = (run: PendingRun) => + apply(run.request).pipe( + Effect.catchCause((cause) => + Effect.logError("The home provisioner run failed", cause).pipe( + Effect.as({ kind: "failed", detail: Cause.pretty(cause) }), + ), + ), + Effect.flatMap((outcome) => Deferred.succeed(run.done, outcome)), + ); + + const release = Ref.getAndSet(state, IDLE).pipe( + Effect.flatMap(({ current, trailing }) => + Effect.forEach( + [current, trailing], + (run) => + run === null + ? Effect.void + : Deferred.succeed(run.done, { kind: "failed", detail: "was stopped." }), + { discard: true }, + ), + ), + ); + + const drain = Effect.gen(function* () { + let current = (yield* Ref.get(state)).current; + while (current !== null) { + yield* runOne(current); + current = yield* Ref.modify(state, (now): readonly [PendingRun | null, RunnerState] => [ + now.trailing, + { current: now.trailing, trailing: null }, + ]); + } + }).pipe(Effect.onExit((exit) => (Exit.isSuccess(exit) ? Effect.void : release))); + + const request = Effect.fn("homeProvisioner.request")(function* (next: ProvisionerRequest) { + const fresh = yield* Deferred.make(); + // Claim and fork together, so an interrupt cannot leave a claimed runner with no worker. + const done = yield* Effect.uninterruptible( + Ref.modify( + state, + ( + now, + ): readonly [ + { readonly start: boolean; readonly done: PendingRun["done"] }, + RunnerState, + ] => { + if (now.current === null) { + return [ + { start: true, done: fresh }, + { current: { request: next, done: fresh }, trailing: null }, + ]; + } + const done = now.trailing?.done ?? fresh; + return [ + { start: false, done }, + { current: now.current, trailing: { request: next, done } }, + ]; + }, + ).pipe( + Effect.tap((action) => + action.start ? Effect.forkIn(Effect.interruptible(drain), scope) : Effect.void, + ), + Effect.map((action) => action.done), + ), + ); + return yield* Deferred.await(done).pipe( + Effect.map((outcome): ProvisionerWait => ({ kind: "finished", outcome })), + Effect.timeoutOrElse({ + duration: options.awaitCap, + orElse: () => Effect.succeed({ kind: "stillRunning" }), + }), + ); + }); + + return { request } satisfies HomeProvisionerRunner; +}); diff --git a/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.test.ts b/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.test.ts new file mode 100644 index 0000000000000..f4899d2f67c5e --- /dev/null +++ b/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.test.ts @@ -0,0 +1,106 @@ +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import { + DEFAULT_SERVER_SETTINGS, + ProviderDriverKind, + type ServerSettings, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as ServerConfig from "../../config.ts"; +import * as ProviderInstanceRegistry from "../../provider/ProviderInstanceRegistry.ts"; +import { deriveProviderInstanceConfigMap } from "../../provider/ProviderInstanceRegistryHydration.ts"; +import * as Settings from "../../serverSettings.ts"; +import { layer } from "./homeProvisionerLayer.ts"; + +const claude = ProviderDriverKind.make("claudeAgent"); + +it.effect("runs on a registry change, and on a settings change only when the request changed", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-provision-layer-" }); + const settings = yield* Ref.make(DEFAULT_SERVER_SETTINGS); + const settingsChanges = yield* PubSub.unbounded(); + const registryChanges = yield* PubSub.unbounded(); + const settingsReads = yield* Queue.unbounded(); + const readPermits = yield* Queue.unbounded(); + const settingsRead = yield* Queue.unbounded(); + const runs = yield* Ref.make(0); + const spawner = ChildProcessSpawner.make(() => + Effect.succeed({ + stdout: Stream.empty, + stderr: Stream.empty, + exitCode: Ref.update(runs, (count) => count + 1).pipe(Effect.as(0)), + } as unknown as ChildProcessSpawner.ChildProcessHandle), + ); + + yield* Layer.build( + layer(deriveProviderInstanceConfigMap).pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(Settings.ServerSettingsService)({ + getSettings: Queue.offer(settingsReads, undefined).pipe( + Effect.andThen(Queue.take(readPermits)), + Effect.andThen(Ref.get(settings)), + Effect.tap(() => Queue.offer(settingsRead, undefined)), + ), + subscribeChanges: Effect.succeed(Stream.fromPubSub(settingsChanges)), + }), + Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({ + subscribeChanges: PubSub.subscribe(registryChanges), + }), + ServerConfig.layerTest(root, root), + Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + Layer.succeed(HostProcess.HomeDirectory, path.join(root, "home")), + Layer.succeed(HostProcess.Environment, {}), + ), + ), + ), + ); + // Each change pauses at its settings read. The layer handles one change at a time, so a paused + // change means the one before it is done; releasing it lets it read settings and go on. + const awaitPaused = Queue.take(settingsReads); + const release = Queue.offer(readPermits, undefined).pipe( + Effect.andThen(Queue.take(settingsRead)), + ); + const changeSettings = (patch: Partial) => + Ref.updateAndGet(settings, (current) => ({ ...current, ...patch })).pipe( + Effect.flatMap((next) => PubSub.publish(settingsChanges, next)), + Effect.andThen(awaitPaused), + ); + const unrelatedChange = changeSettings({ + defaultTheme: "an-unrelated-theme", + } as Partial); + const runsSoFar = Ref.get(runs); + + yield* awaitPaused; + yield* release; + yield* changeSettings({ homeProvisioner: ["provisioner"] }); + yield* release; + yield* unrelatedChange; + assert.strictEqual(yield* runsSoFar, 1, "setting the command runs it"); + yield* release; + yield* changeSettings({ + providerInstances: { claude_work: { driver: claude } } as ServerSettings["providerInstances"], + }); + assert.strictEqual(yield* runsSoFar, 1, "an unrelated settings change is skipped"); + yield* release; + yield* PubSub.publish(registryChanges, undefined).pipe(Effect.andThen(awaitPaused)); + assert.strictEqual(yield* runsSoFar, 2, "a new home runs"); + yield* release; + yield* unrelatedChange; + assert.strictEqual(yield* runsSoFar, 3, "a registry change runs with the same request"); + yield* release; + }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), +); diff --git a/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.ts b/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.ts new file mode 100644 index 0000000000000..4cd17305e1e0d --- /dev/null +++ b/apps/server/src/fork/homeProvisioner/homeProvisionerLayer.ts @@ -0,0 +1,81 @@ +import type { ProviderInstanceConfigMap, ServerSettings } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; +import * as Stream from "effect/Stream"; +import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner"; + +import * as ServerConfig from "../../config.ts"; +import * as ProviderInstanceRegistry from "../../provider/ProviderInstanceRegistry.ts"; +import * as Settings from "../../serverSettings.ts"; +import { + deriveDeclaredHomes, + makeHomeProvisionerRunner, + serverDeclarerId, + spawnHomeProvisioner, + type ProvisionerRequest, +} from "./homeProvisioner.ts"; + +/** + * Sends the homes of `instanceConfigs`, the registry's instance map, to the configured + * `homeProvisioner` command; with no command set it runs nothing. A settings change runs it only + * when the request differs from the last applied one. A registry change always runs it, because a + * rebuilt instance can create its overlay or profile while the request stays the same. + */ +export const layer = (instanceConfigs: (settings: ServerSettings) => ProviderInstanceConfigMap) => + Layer.effectDiscard( + Effect.gen(function* () { + const settingsService = yield* Settings.ServerSettingsService; + const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry; + const config = yield* ServerConfig.ServerConfig; + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const declarer = yield* serverDeclarerId(config.baseDir); + const lastApplied = yield* Ref.make(null); + const requestKey = (request: ProvisionerRequest) => JSON.stringify(request); + const runner = yield* makeHomeProvisionerRunner((request) => + spawnHomeProvisioner(request).pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.tap((outcome) => + outcome.kind === "applied" + ? Ref.set(lastApplied, requestKey(request)) + : Effect.logWarning("The home provisioner failed", { detail: outcome.detail }), + ), + ), + ); + + const sync = Effect.fn("homeProvisioner.sync")( + function* (always: boolean) { + const settings = yield* settingsService.getSettings; + if (settings.homeProvisioner === undefined) return; + const derived = yield* deriveDeclaredHomes(instanceConfigs(settings), { + stateDir: config.stateDir, + managedPolicy: settingsService.managedPolicy, + }); + const request: ProvisionerRequest = { + command: settings.homeProvisioner, + declaration: { + version: 1, + declarer, + homes: derived.homes.map((home) => ({ home })), + }, + }; + if (!always && requestKey(request) === (yield* Ref.get(lastApplied))) return; + if (derived.skipped.length > 0) { + yield* Effect.logWarning("Some provider homes were left out of the declaration", { + skipped: derived.skipped, + }); + } + yield* runner.request(request); + }, + Effect.catchCause((cause) => Effect.logError("Declaring provider homes failed", cause)), + ); + + const registryChanges = yield* registry.subscribeChanges; + const settingsChanges = yield* settingsService.subscribeChanges; + yield* Stream.merge( + Stream.fromSubscription(registryChanges).pipe(Stream.map(() => true)), + settingsChanges.pipe(Stream.map(() => false)), + ).pipe(Stream.runForEach(sync), Effect.forkScoped); + yield* Effect.forkScoped(sync(true)); + }), + ); diff --git a/apps/server/src/fork/instanceIdentity.ts b/apps/server/src/fork/instanceIdentity.ts new file mode 100644 index 0000000000000..03c73a43ff213 --- /dev/null +++ b/apps/server/src/fork/instanceIdentity.ts @@ -0,0 +1,39 @@ +import type { ProviderInstanceId, ResolvedProviderInstanceEnvironment } from "@t3tools/contracts"; +import type { AnyProviderDriver } from "@t3tools/provider-core/server/driver"; + +export const PROVIDER_INSTANCE_ID_ENV = "T3CODE_PROVIDER_INSTANCE_ID"; +export const HOOK_STATE_DB_ENV = "T3CODE_HOOK_STATE_DB"; + +/** + * Tells hooks inside a provider process which instance started it and where the server's state + * database is, for read-only lookups. It never sets T3CODE_HOME: a `t3` command an agent runs + * reads that variable as its own data directory. + */ +export const withProviderProcessIdentityEnv = ( + instanceId: ProviderInstanceId, + stateDbPath: string, + environment: ResolvedProviderInstanceEnvironment, +): ResolvedProviderInstanceEnvironment => [ + ...environment.filter( + (variable) => variable.name !== PROVIDER_INSTANCE_ID_ENV && variable.name !== HOOK_STATE_DB_ENV, + ), + { name: PROVIDER_INSTANCE_ID_ENV, value: instanceId, sensitive: false }, + { name: HOOK_STATE_DB_ENV, value: stateDbPath, sensitive: false }, +]; + +export const withProviderProcessIdentity = ( + drivers: ReadonlyArray>, + stateDbPath: string, +): ReadonlyArray> => + drivers.map((driver) => ({ + ...driver, + create: (input) => + driver.create({ + ...input, + environment: withProviderProcessIdentityEnv( + input.instanceId, + stateDbPath, + input.environment, + ), + }), + })); diff --git a/apps/server/src/fork/providerPools/poolDispatch.test.ts b/apps/server/src/fork/providerPools/poolDispatch.test.ts new file mode 100644 index 0000000000000..897810ebf2365 --- /dev/null +++ b/apps/server/src/fork/providerPools/poolDispatch.test.ts @@ -0,0 +1,94 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ThreadId, + type OrchestrationV2ServerCommand, + type OrchestrationV2ThreadProjection, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import * as CommandReceiptStore from "../../orchestration-v2/CommandReceiptStore.ts"; +import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; +import * as ProviderSessionManager from "../../orchestration-v2/ProviderSessionManager.ts"; +import * as ThreadManagementService from "../../orchestration-v2/ThreadManagementService.ts"; +import * as SqlitePersistence from "../../persistence/Sqlite.ts"; +import * as ServerSettings from "../../serverSettings.ts"; + +const id = ProviderInstanceId.make; +const claude = ProviderDriverKind.make("claudeAgent"); +const projectId = ProjectId.make("project:pool-dispatch"); +const threadId = ThreadId.make("thread:pool-dispatch"); + +const layerWith = (dispatched: Array) => + ThreadManagementService.layer.pipe( + Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), + Layer.provide( + Layer.mock(Orchestrator.OrchestratorV2)({ + getThreadRecords: () => + Effect.succeed({ + thread: { + id: threadId, + projectId, + deletedAt: null, + modelSelection: { instanceId: id("codex"), model: "gpt" }, + }, + } as unknown as OrchestrationV2ThreadProjection), + dispatch: (command) => + Effect.sync(() => { + dispatched.push(command); + return { sequence: 1, storedEvents: [] } as never; + }), + }), + ), + Layer.provide( + ServerSettings.layerTest({ + providerInstances: { + [id("claude_a")]: { driver: claude }, + [id("claude_b")]: { driver: claude }, + [id("claude_off")]: { driver: claude, enabled: false }, + }, + providerPools: { claude_auto: { members: ["claude_a", "claude_b"] } }, + }), + ), + Layer.provide(CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))), + ); + +const switchingMessage = (instanceId: string): OrchestrationV2ServerCommand => ({ + type: "message.dispatch", + commandId: CommandId.make(`command:pool-dispatch:${instanceId}`), + threadId, + messageId: MessageId.make(`message:pool-dispatch:${instanceId}`), + text: "Continue on the pool.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + modelSelection: { instanceId: id(instanceId), model: "opus" }, + createdBy: "user", + creationSource: "web", +}); + +it.effect( + "thread management hands a message that switches to a pool to the orchestrator to place", + () => { + const dispatched: Array = []; + return Effect.gen(function* () { + const threads = yield* ThreadManagementService.ThreadManagementService; + yield* threads.dispatch(switchingMessage("claude_auto")); + assert.deepStrictEqual( + dispatched.map((command) => command.commandId), + ["command:pool-dispatch:claude_auto"], + ); + + const error = yield* threads.dispatch(switchingMessage("claude_off")).pipe(Effect.flip); + assert.instanceOf( + error, + ThreadManagementService.ThreadManagementProviderInstanceDisabledError, + ); + assert.strictEqual(dispatched.length, 1); + }).pipe(Effect.provide(layerWith(dispatched))); + }, +); diff --git a/apps/server/src/fork/providerPools/poolPlacement.test.ts b/apps/server/src/fork/providerPools/poolPlacement.test.ts new file mode 100644 index 0000000000000..78bfb9ba6c5a1 --- /dev/null +++ b/apps/server/src/fork/providerPools/poolPlacement.test.ts @@ -0,0 +1,586 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + CommandId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ServerSettingsError, + ThreadId, + type OrchestrationV2ServerCommand, + type ServerProvider, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; + +import { BUILT_IN_DRIVERS } from "../../provider/builtInDrivers.ts"; +import * as ProviderRegistry from "../../provider/ProviderRegistry.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import { + BUILT_IN_DRIVER_KINDS, + ProviderPoolPlacementError, + ProviderPoolUnresolvedError, + providerSupportsModel, + refreshRequestedInstances, + resolveCommandPool, + resolvePoolTarget, + wrapOrchestratorDispatch, +} from "./poolPlacement.ts"; + +const id = ProviderInstanceId.make; +const claude = ProviderDriverKind.make("claudeAgent"); +const antigravity = ProviderDriverKind.make("antigravity"); +const projectId = ProjectId.make("project:pools"); +const projectWithMembersOff = ProjectId.make("project:pools-off"); +const threadId = ThreadId.make("thread:pools"); +const NOW = "2026-11-20T12:00:00.000Z"; + +const settingsLayer = ServerSettings.ServerSettingsService.layerTest({ + providerInstances: { + [id("claude_a")]: { driver: claude }, + [id("claude_b")]: { driver: claude }, + [id("codex_a")]: { driver: ProviderDriverKind.make("codex") }, + [id("agy_a")]: { driver: antigravity, enabled: true }, + [id("agy_b")]: { driver: antigravity, enabled: true }, + }, + providerPools: { + claude_auto: { displayName: "Claude Auto", members: ["claude_a", "claude_b"] }, + agy_auto: { displayName: "Antigravity Auto", members: ["agy_a", "agy_b"] }, + broken: { members: ["claude_a", "codex_a"] }, + codex_a: { members: ["claude_a"] }, + codex: { members: ["mixed", 1] }, + }, + projectSettingsOverrides: { + [projectWithMembersOff]: { + providerInstances: { + [id("claude_a")]: { enabled: false }, + [id("claude_b")]: { enabled: false }, + }, + }, + }, +}); + +const unreadableSettings = Effect.fail( + new ServerSettingsError({ settingsPath: "settings.json", operation: "normalize" }), +); + +const provider = (instanceId: string, overrides: Partial = {}): ServerProvider => ({ + instanceId: id(instanceId), + driver: claude, + enabled: true, + installed: true, + version: "test", + status: "ready", + auth: { status: "authenticated" }, + checkedAt: NOW, + models: [{ slug: "opus", name: "Opus", isCustom: false, capabilities: null }], + slashCommands: [], + skills: [], + ...overrides, +}); + +const fullUntil = (resetsAt: string): Partial => ({ + usageLimits: { + checkedAt: NOW, + windows: [{ id: "week", kind: "weekly", label: "Week", usedPercent: 100, resetsAt }], + }, +}); + +const atNow = TestClock.setTime(Date.parse(NOW)); + +const getSettings = Effect.gen(function* () { + return yield* (yield* ServerSettings.ServerSettingsService).getSettings; +}); + +const launchFrom = ( + requested: string, + providers: ReadonlyArray, + seedKey = "command:launch", +) => + resolvePoolTarget({ + requestedInstanceId: id(requested), + projectId, + model: "opus", + seedKey, + getSettings, + providers: Effect.succeed(providers), + }); + +const switchCommand = (instanceId: string): OrchestrationV2ServerCommand => ({ + type: "provider.switch", + commandId: CommandId.make("command:switch"), + threadId, + modelSelection: { instanceId: id(instanceId), model: "opus" }, +}); + +const orchestratorWith = (current: string | null) => ({ + getSettings, + threadShell: () => + Effect.succeed(current === null ? null : { projectId, providerInstanceId: id(current) }), + registeredInstanceIds: Effect.succeed([id("claude_a"), id("claude_b"), id("codex_a")]), + hasReceipt: () => Effect.succeed(false), +}); + +const createCommand = (instanceId: string): OrchestrationV2ServerCommand => ({ + type: "thread.create", + createdBy: "user", + creationSource: "web", + commandId: CommandId.make("command:create"), + threadId, + projectId, + title: "Pooled", + modelSelection: { instanceId: id(instanceId), model: "opus" }, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, +}); + +it.layer(settingsLayer)("provider pool placement", (it) => { + describe("launch and delegate targets", () => { + it.effect("passes a concrete instance through without loading providers", () => + Effect.gen(function* () { + const placed = yield* resolvePoolTarget({ + requestedInstanceId: id("claude_b"), + projectId, + seedKey: "command:x", + getSettings, + providers: Effect.die("providers are not needed for a concrete instance"), + }); + assert.strictEqual(placed, "claude_b"); + }), + ); + + it.effect("places a pool on one member, the same one on retry", () => + Effect.gen(function* () { + yield* atNow; + const providers = [provider("claude_a"), provider("claude_b")]; + const first = yield* launchFrom("claude_auto", providers); + assert.include(["claude_a", "claude_b"], first); + assert.strictEqual(yield* launchFrom("claude_auto", providers), first); + }), + ); + + it.effect("skips a member that is signed out or at its usage limit", () => + Effect.gen(function* () { + yield* atNow; + const later = "2026-11-21T00:00:00.000Z"; + for (const seedKey of ["k1", "k2", "k3"]) { + assert.strictEqual( + yield* launchFrom( + "claude_auto", + [provider("claude_a", fullUntil(later)), provider("claude_b")], + seedKey, + ), + "claude_b", + ); + assert.strictEqual( + yield* launchFrom( + "claude_auto", + [provider("claude_a"), provider("claude_b", { auth: { status: "unauthenticated" } })], + seedKey, + ), + "claude_a", + ); + } + }), + ); + + it.effect("fails with each member's reason when no member can run", () => + Effect.gen(function* () { + yield* atNow; + const error = yield* launchFrom("claude_auto", [ + provider("claude_a", { enabled: false }), + ]).pipe(Effect.flip); + assert.instanceOf(error, ProviderPoolPlacementError); + assert.deepStrictEqual(error.reasons, [ + "claude_a: is turned off.", + "claude_b: is not registered.", + ]); + }), + ); + + it.effect("fails a delegate whose driver or own check rules out the pool", () => + Effect.gen(function* () { + yield* atNow; + const providers = [provider("claude_a"), provider("claude_b")]; + const wrongDriver = yield* resolvePoolTarget({ + requestedInstanceId: id("claude_auto"), + projectId, + driver: ProviderDriverKind.make("codex"), + seedKey: "command:delegate", + getSettings, + providers: Effect.succeed(providers), + }).pipe(Effect.flip); + assert.include(wrongDriver.message, "runs claudeAgent, not codex."); + + const placed = yield* resolvePoolTarget({ + requestedInstanceId: id("claude_auto"), + projectId, + seedKey: "command:delegate", + getSettings, + providers: Effect.succeed(providers), + unusableReason: (candidate) => + candidate.instanceId === "claude_a" ? "has no V2 adapter." : null, + }); + assert.strictEqual(placed, "claude_b"); + }), + ); + + it.effect("fails an invalid pool with the reason it was dropped", () => + Effect.gen(function* () { + const error = yield* launchFrom("broken", [provider("claude_a")]).pipe(Effect.flip); + assert.deepStrictEqual(error.reasons, [ + "Provider pool 'broken' mixes members of different drivers.", + ]); + }), + ); + + it.effect("sends an id shared by an instance and an invalid pool to the instance", () => + Effect.gen(function* () { + for (const requested of ["codex_a", "codex"]) { + const placed = yield* resolvePoolTarget({ + requestedInstanceId: id(requested), + projectId, + seedKey: "command:collide", + getSettings, + providers: Effect.die("providers are not needed for a concrete instance"), + }); + assert.strictEqual(placed, requested); + } + }), + ); + + it.effect("ignores the project's enabled instances on a receipt replay", () => + Effect.gen(function* () { + yield* atNow; + const providers = [provider("claude_a"), provider("claude_b")]; + const replay = (skipProjectEnablementFilter: boolean) => + resolvePoolTarget({ + requestedInstanceId: id("claude_auto"), + projectId: projectWithMembersOff, + seedKey: "command:replay", + getSettings, + providers: Effect.succeed(providers), + skipProjectEnablementFilter, + }); + const refused = yield* replay(false).pipe(Effect.flip); + assert.deepStrictEqual(refused.reasons, [ + "claude_a: is turned off for this project.", + "claude_b: is turned off for this project.", + ]); + assert.include(["claude_a", "claude_b"], yield* replay(true)); + }), + ); + + it.effect("reuses the instance a retried launch committed without placing again", () => + Effect.gen(function* () { + const placed = yield* resolvePoolTarget({ + requestedInstanceId: id("claude_auto"), + projectId, + seedKey: "command:retry", + getSettings, + providers: Effect.die("a committed retry does not place"), + committedInstanceId: Effect.succeed(id("claude_b")), + }); + assert.strictEqual(placed, "claude_b"); + }), + ); + + it.effect("re-probes every member of a pool, or the one instance named", () => + Effect.gen(function* () { + const probed: Array = []; + const refresh = (requested: string) => + refreshRequestedInstances({ + requestedInstanceId: id(requested), + getSettings, + refreshInstance: (instanceId) => + Effect.sync(() => { + probed.push(instanceId); + return [provider(instanceId)]; + }), + }); + yield* refresh("claude_auto"); + yield* refresh("codex_a"); + assert.deepStrictEqual(probed, ["claude_a", "claude_b", "codex_a"]); + }), + ); + }); + + describe("orchestrator commands", () => { + it.effect("keeps the thread's instance on a switch to its own pool", () => + Effect.gen(function* () { + const resolved = yield* resolveCommandPool( + switchCommand("claude_auto"), + orchestratorWith("claude_b"), + ); + assert.deepStrictEqual(resolved, switchCommand("claude_b")); + }), + ); + + it.effect("places a switch from another driver on a member", () => + Effect.gen(function* () { + const resolved = yield* resolveCommandPool( + switchCommand("claude_auto"), + orchestratorWith("codex_a"), + ); + assert.strictEqual(resolved.type, "provider.switch"); + const instanceId = + resolved.type === "provider.switch" ? resolved.modelSelection.instanceId : ""; + assert.include(["claude_a", "claude_b"], instanceId); + }), + ); + + it.effect("leaves concrete selections and commands without one alone", () => + Effect.gen(function* () { + const concrete = switchCommand("codex_a"); + assert.strictEqual( + yield* resolveCommandPool(concrete, orchestratorWith("claude_a")), + concrete, + ); + const stop: OrchestrationV2ServerCommand = { + type: "thread.stop", + commandId: CommandId.make("command:stop"), + threadId, + }; + assert.strictEqual(yield* resolveCommandPool(stop, orchestratorWith(null)), stop); + }), + ); + + it.effect("refuses a thread create that still names a pool", () => + Effect.gen(function* () { + const error = yield* resolveCommandPool( + createCommand("claude_auto"), + orchestratorWith(null), + ).pipe(Effect.flip); + assert.instanceOf(error, ProviderPoolUnresolvedError); + const concrete = createCommand("claude_a"); + assert.strictEqual(yield* resolveCommandPool(concrete, orchestratorWith(null)), concrete); + }), + ); + + it.effect("keeps a concrete instance whose id an invalid pool shares", () => + Effect.gen(function* () { + for (const command of [switchCommand("codex_a"), createCommand("codex")]) { + assert.strictEqual( + yield* resolveCommandPool(command, orchestratorWith("claude_a")), + command, + ); + } + }), + ); + + it.effect("passes only a registered instance when settings cannot be read", () => + Effect.gen(function* () { + const orchestrator = { ...orchestratorWith(null), getSettings: unreadableSettings }; + const registered = createCommand("claude_a"); + assert.strictEqual(yield* resolveCommandPool(registered, orchestrator), registered); + const error = yield* resolveCommandPool(createCommand("claude_auto"), orchestrator).pipe( + Effect.flip, + ); + assert.instanceOf(error, ServerSettingsError); + }), + ); + + it.effect("replays an accepted switch without placing it again", () => + Effect.gen(function* () { + const command = switchCommand("claude_auto"); + const replayed = yield* resolveCommandPool(command, { + ...orchestratorWith("codex_a"), + registeredInstanceIds: Effect.succeed([]), + hasReceipt: () => Effect.succeed(true), + }); + assert.strictEqual(replayed, command); + }), + ); + + it.effect( + "uses provider snapshots to skip signed-out, incompatible, and limited members on switch", + () => + Effect.gen(function* () { + yield* atNow; + const signedOutCurrent = yield* resolveCommandPool(switchCommand("claude_auto"), { + ...orchestratorWith("claude_a"), + providers: Effect.succeed([ + provider("claude_a", { auth: { status: "unauthenticated" } }), + provider("claude_b"), + ]), + }); + assert.strictEqual( + signedOutCurrent.type === "provider.switch" + ? signedOutCurrent.modelSelection.instanceId + : "", + "claude_b", + ); + + const limitedAndWrongModel = yield* resolveCommandPool(switchCommand("claude_auto"), { + ...orchestratorWith("codex_a"), + providers: Effect.succeed([ + provider("claude_a", fullUntil("2026-11-20T15:00:00.000Z")), + provider("claude_b"), + ]), + }); + assert.strictEqual( + limitedAndWrongModel.type === "provider.switch" + ? limitedAndWrongModel.modelSelection.instanceId + : "", + "claude_b", + ); + + const missingModel = yield* resolveCommandPool(switchCommand("claude_auto"), { + ...orchestratorWith("claude_a"), + providers: Effect.succeed([ + provider("claude_a", { + models: [{ slug: "sonnet", name: "Sonnet", isCustom: false, capabilities: null }], + }), + provider("claude_b"), + ]), + }); + assert.strictEqual( + missingModel.type === "provider.switch" ? missingModel.modelSelection.instanceId : "", + "claude_b", + ); + }), + ); + + it.effect("wrapOrchestratorDispatch reads ProviderRegistry snapshots from context", () => + Effect.gen(function* () { + const received: Array = []; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const dispatch = yield* wrapOrchestratorDispatch( + (command) => + Effect.sync(() => { + received.push(command); + return command; + }), + { + serverSettings, + projectionStore: { + getThreadShell: () => + Effect.succeed({ projectId, providerInstanceId: id("claude_a") }), + }, + providerAdapters: { + list: () => Effect.succeed([id("claude_a"), id("claude_b")]), + }, + commandReceipts: { + getByCommandId: () => Effect.succeedNone, + }, + mapError: (_command, cause) => cause, + }, + ); + yield* dispatch(switchCommand("claude_auto")); + assert.strictEqual(received.length, 1); + const placed = received[0]!; + assert.strictEqual( + placed.type === "provider.switch" ? placed.modelSelection.instanceId : "", + "claude_b", + ); + }).pipe( + Effect.provide( + Layer.mock(ProviderRegistry.ProviderRegistry)({ + getProviders: Effect.succeed([ + provider("claude_a", { auth: { status: "unauthenticated" } }), + provider("claude_b"), + ]), + }), + ), + ), + ); + }); + + describe("model matching semantics", () => { + it.effect("accepts aliases, Antigravity thinking suffixes, and empty model catalogs", () => + Effect.gen(function* () { + yield* atNow; + const aliasedProvider = provider("claude_a", { + models: [ + { + slug: "opus", + name: "Opus", + aliases: ["claude-opus"], + isCustom: false, + capabilities: null, + }, + ], + usageLimits: { + checkedAt: NOW, + windows: [ + { + id: "week", + kind: "weekly", + label: "Week", + usedPercent: 100, + resetsAt: "2026-11-20T15:00:00.000Z", + modelIds: ["opus"], + }, + ], + }, + }); + const emptyCatalogProvider = provider("claude_b", { models: [] }); + + assert.isTrue(providerSupportsModel(aliasedProvider, "CLAUDE-OPUS")); + assert.isTrue(providerSupportsModel(emptyCatalogProvider, "any-custom-slug")); + assert.isFalse(providerSupportsModel(aliasedProvider, "unknown-model")); + + const aliasPlacement = yield* resolvePoolTarget({ + requestedInstanceId: id("claude_auto"), + projectId, + model: "CLAUDE-OPUS", + seedKey: "command:alias", + getSettings, + providers: Effect.succeed([aliasedProvider, emptyCatalogProvider]), + }); + assert.strictEqual(aliasPlacement, id("claude_b")); + + const agyWithThinking = provider("agy_a", { + driver: antigravity, + models: [ + { + slug: "gemini-3.8-flash", + name: "Gemini 3.8 Flash", + isCustom: false, + capabilities: { + optionDescriptors: [ + { + id: "thought_level", + label: "Thinking", + type: "select", + role: "reasoning", + options: [ + { id: "low", label: "Low" }, + { id: "high", label: "High" }, + ], + }, + ], + }, + }, + ], + }); + const agyWithout = provider("agy_b", { + driver: antigravity, + models: [{ slug: "other", name: "Other", isCustom: false, capabilities: null }], + }); + + assert.isTrue(providerSupportsModel(agyWithThinking, "gemini-3.8-flash-high")); + assert.isFalse(providerSupportsModel(agyWithThinking, "gemini-3.8-flash-bogus")); + + const agyPlacement = yield* resolvePoolTarget({ + requestedInstanceId: id("agy_auto"), + projectId, + model: "gemini-3.8-flash-high", + seedKey: "command:agy", + getSettings, + providers: Effect.succeed([agyWithThinking, agyWithout]), + }); + assert.strictEqual(agyPlacement, id("agy_a")); + }), + ); + }); + + it("lists the same driver kinds as the built-in drivers", () => { + assert.deepStrictEqual( + [...BUILT_IN_DRIVER_KINDS].toSorted(), + BUILT_IN_DRIVERS.map((driver) => driver.driverKind as string).toSorted(), + ); + }); +}); diff --git a/apps/server/src/fork/providerPools/poolPlacement.ts b/apps/server/src/fork/providerPools/poolPlacement.ts new file mode 100644 index 0000000000000..8e82322ec5e87 --- /dev/null +++ b/apps/server/src/fork/providerPools/poolPlacement.ts @@ -0,0 +1,435 @@ +import { + isProviderAvailable, + type CommandId, + resolveProjectProviderInstanceEnabled, + type OrchestrationV2ServerCommand, + type ProjectId, + type ProviderDriverKind, + type ProviderInstanceId, + type ServerProvider, + type ServerSettings, + type ServerSettingsError, + type ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import * as ProviderRegistry from "../../provider/ProviderRegistry.ts"; +import { + buildPoolTable, + resolvePoolSelection, + type PoolCandidate, + type PoolTable, +} from "./pools.ts"; + +/** Pool ids may not take these, since each also names a driver's default instance. */ +export const BUILT_IN_DRIVER_KINDS: ReadonlySet = new Set([ + "codex", + "claudeAgent", + "cursor", + "grok", + "fx", + "opencode", + "antigravity", + "kiro", + "pi", + "muse", + "acpRegistry", +]); + +export class ProviderPoolPlacementError extends Schema.TaggedError()( + "ProviderPoolPlacementError", + { poolId: Schema.String, reasons: Schema.Array(Schema.String) }, +) { + override get message(): string { + return `No member of provider pool '${this.poolId}' can take this request. ${this.reasons.join(" ")}`; + } +} + +export class ProviderPoolUnresolvedError extends Schema.TaggedError()( + "ProviderPoolUnresolvedError", + { poolId: Schema.String, commandType: Schema.String }, +) { + override get message(): string { + return `Provider pool '${this.poolId}' reached ${this.commandType} without being placed on one of its instances.`; + } +} + +const advertisedModel = (provider: ServerProvider, model: string) => + provider.models.find((candidate) => candidate.slug === model) ?? + provider.models.find((candidate) => + candidate.aliases?.some((alias) => alias.toLowerCase() === model.toLowerCase()), + ); + +const antigravityBaseModelSlug = (provider: ServerProvider, model: string): string | undefined => { + if (provider.driver !== "antigravity" || advertisedModel(provider, model)) return undefined; + const separator = model.lastIndexOf("-"); + if (separator === -1) return undefined; + const family = model.slice(0, separator); + const value = model.slice(separator + 1); + const thinking = provider.models + .find((candidate) => candidate.slug === family) + ?.capabilities?.optionDescriptors?.find( + (descriptor) => descriptor.type === "select" && descriptor.role === "reasoning", + ); + if (thinking?.type === "select" && thinking.options.some((choice) => choice.id === value)) { + return family; + } + return undefined; +}; + +/** Accepts exact slugs, case-insensitive aliases, Antigravity thinking suffixes, and empty catalogs. */ +export const providerSupportsModel = (provider: ServerProvider, model: string): boolean => + provider.models.length === 0 || + advertisedModel(provider, model) !== undefined || + antigravityBaseModelSlug(provider, model) !== undefined; + +/** The latest reset among full usage windows that cover the model, or null when none is full. */ +const providerLimitedUntil = ( + provider: ServerProvider, + model: string | undefined, +): number | null => { + const canonicalModel = + model === undefined + ? undefined + : (advertisedModel(provider, model)?.slug ?? antigravityBaseModelSlug(provider, model)); + let until: number | null = null; + for (const window of provider.usageLimits?.windows ?? []) { + if (window.usedPercent < 100 || window.resetsAt === undefined) continue; + if ( + model !== undefined && + window.modelIds !== undefined && + !window.modelIds.includes(model) && + (canonicalModel === undefined || !window.modelIds.includes(canonicalModel)) + ) { + continue; + } + const resetsAt = Date.parse(window.resetsAt); + if (Number.isFinite(resetsAt) && (until === null || resetsAt > until)) until = resetsAt; + } + return until; +}; + +const providerUnusableReason = (provider: ServerProvider): string | null => { + if (!isProviderAvailable(provider)) { + return provider.unavailableReason ?? "is unavailable."; + } + if (!provider.enabled) return "is turned off."; + if (!provider.installed) return "is not installed."; + if (provider.status === "error" || provider.status === "disabled") { + return provider.message ?? `has status ${provider.status}.`; + } + if (provider.auth.status === "unauthenticated") return "is not signed in."; + return null; +}; + +const candidatesFromProviders = (input: { + readonly settings: ServerSettings; + readonly projectId: ProjectId | null; + readonly model: string | undefined; + readonly providers: ReadonlyArray; + readonly unusableReason: (provider: ServerProvider) => string | null; + readonly skipProjectEnablementFilter: boolean; +}): ReadonlyArray => + input.providers.map((provider) => ({ + instanceId: provider.instanceId, + unusableReason: input.unusableReason(provider), + enabledForProject: + input.skipProjectEnablementFilter || + resolveProjectProviderInstanceEnabled(input.settings, input.projectId, provider.instanceId), + supportsModel: (requested) => providerSupportsModel(provider, requested), + limitedUntil: providerLimitedUntil(provider, input.model), + })); + +type GetSettings = Effect.Effect; + +const poolTable = (settings: ServerSettings): PoolTable => + buildPoolTable(settings, BUILT_IN_DRIVER_KINDS); + +/** Unreadable settings name no pools, so a settings fault never blocks an instance selection. */ +const readPools = (getSettings: GetSettings) => + getSettings.pipe( + Effect.map((settings) => ({ settings, table: poolTable(settings) })), + Effect.orElseSucceed(() => undefined), + ); + +/** A valid or dropped pool. An instance id never names a pool, even when a pool entry shares it. */ +const namesPool = (table: PoolTable, instanceId: ProviderInstanceId): boolean => + table.pools.has(instanceId) || table.dropped.has(instanceId); + +const placeInPool = Effect.fn("providerPools.placeInPool")(function* (input: { + readonly table: PoolTable; + readonly kind: "launch" | "switch"; + readonly requestedInstanceId: ProviderInstanceId; + readonly currentInstanceId: ProviderInstanceId | null; + readonly driver: ProviderDriverKind | undefined; + readonly model: string | undefined; + readonly seedKey: string; + readonly candidates: ReadonlyArray; +}) { + const { table } = input; + const dropped = table.dropped.get(input.requestedInstanceId); + if (dropped !== undefined) { + return yield* new ProviderPoolPlacementError({ + poolId: input.requestedInstanceId, + reasons: [dropped], + }); + } + const base = { + requestedInstanceId: input.requestedInstanceId, + driver: input.driver, + model: input.model, + seedKey: input.seedKey, + table, + candidates: input.candidates, + now: yield* Clock.currentTimeMillis, + }; + const selection = resolvePoolSelection( + input.kind === "launch" + ? { ...base, kind: "launch" } + : { ...base, kind: "switch", currentInstanceId: input.currentInstanceId }, + ); + if (selection.kind === "noEligibleMember") { + return yield* new ProviderPoolPlacementError({ + poolId: selection.poolId, + reasons: selection.reasons, + }); + } + yield* Effect.logInfo("Provider pool placed a request", { + poolId: input.requestedInstanceId, + instanceId: selection.instanceId, + kind: selection.kind, + seedKey: input.seedKey, + }); + return selection.instanceId; +}); + +/** + * Where a new thread or child task starts from a requested selection: an instance id passes + * through, a pool id becomes one of its members. Providers load only when a pool is named. + */ +export const resolvePoolTarget = Effect.fn("providerPools.resolvePoolTarget")(function* (input: { + readonly requestedInstanceId: ProviderInstanceId; + readonly projectId: ProjectId | null; + readonly driver?: ProviderDriverKind | undefined; + readonly model?: string | undefined; + readonly seedKey: string; + readonly getSettings: GetSettings; + readonly providers: Effect.Effect>; + readonly unusableReason?: ((provider: ServerProvider) => string | null) | undefined; + /** A receipt replay starts no work, so it ignores the project's enabled instances. */ + readonly skipProjectEnablementFilter?: boolean | undefined; + /** On a retry of an accepted command, the instance its first try committed, which skips placement. */ + readonly committedInstanceId?: Effect.Effect | undefined; +}) { + const pools = yield* readPools(input.getSettings); + if (pools === undefined || !namesPool(pools.table, input.requestedInstanceId)) { + return input.requestedInstanceId; + } + const committed = + input.committedInstanceId === undefined ? null : yield* input.committedInstanceId; + if (committed !== null) return committed; + const { settings, table } = pools; + return yield* placeInPool({ + table, + kind: "launch", + requestedInstanceId: input.requestedInstanceId, + currentInstanceId: null, + driver: input.driver, + model: input.model, + seedKey: input.seedKey, + candidates: candidatesFromProviders({ + settings, + projectId: input.projectId, + model: input.model, + providers: yield* input.providers, + unusableReason: input.unusableReason ?? providerUnusableReason, + skipProjectEnablementFilter: input.skipProjectEnablementFilter === true, + }), + }); +}); + +/** Re-probes the requested instance, or each member when the id names a pool. */ +export const refreshRequestedInstances = Effect.fn("providerPools.refreshRequestedInstances")( + function* (input: { + readonly requestedInstanceId: ProviderInstanceId; + readonly getSettings: GetSettings; + readonly refreshInstance: ( + instanceId: ProviderInstanceId, + ) => Effect.Effect>; + }) { + const pools = yield* readPools(input.getSettings); + const instanceIds = pools?.table.pools.get(input.requestedInstanceId)?.members ?? [ + input.requestedInstanceId, + ]; + const snapshots = yield* Effect.forEach(instanceIds, input.refreshInstance); + return snapshots.at(-1) ?? []; + }, +); + +type SwitchCommand = Extract< + OrchestrationV2ServerCommand, + { readonly type: "provider.switch" | "thread.model-selection.set" | "message.dispatch" } +>; +type CreateCommand = Extract< + OrchestrationV2ServerCommand, + { readonly type: "thread.create" | "delegated_task.request" } +>; + +const isSwitchCommand = (command: OrchestrationV2ServerCommand): command is SwitchCommand => + command.type === "provider.switch" || + command.type === "thread.model-selection.set" || + command.type === "message.dispatch"; + +const isCreateCommand = (command: OrchestrationV2ServerCommand): command is CreateCommand => + command.type === "thread.create" || command.type === "delegated_task.request"; + +const commandSelection = (command: OrchestrationV2ServerCommand) => + isCreateCommand(command) || isSwitchCommand(command) ? command.modelSelection : undefined; + +/** + * Whether a command's own selection names a pool. Thread management skips its project check for + * one: the orchestrator places a pool only on a member enabled for the project. + */ +export const commandNamesPool = ( + command: OrchestrationV2ServerCommand, + getSettings: GetSettings, +) => { + const selection = commandSelection(command); + return selection === undefined + ? Effect.succeed(false) + : readPools(getSettings).pipe( + Effect.map((pools) => pools !== undefined && namesPool(pools.table, selection.instanceId)), + ); +}; + +/** + * The orchestrator's last check before a selection is persisted. A switch to a pool is placed + * here, keeping the thread's instance when it is an eligible member. A create that still names a + * pool is refused, because its caller had to place it first. When settings cannot be read, only + * a registered instance passes, so a pool id is never persisted. + */ +export const resolveCommandPool = Effect.fn("providerPools.resolveCommandPool")(function* ( + command: OrchestrationV2ServerCommand, + orchestrator: { + readonly getSettings: GetSettings; + readonly threadShell: (threadId: ThreadId) => Effect.Effect<{ + readonly projectId: ProjectId; + readonly providerInstanceId: ProviderInstanceId; + } | null>; + readonly registeredInstanceIds: Effect.Effect>; + readonly providers?: Effect.Effect> | undefined; + readonly hasReceipt: (commandId: CommandId) => Effect.Effect; + }, +) { + const selection = commandSelection(command); + if (selection === undefined) return command; + const settings = yield* orchestrator.getSettings.pipe( + Effect.catch((error) => + orchestrator.registeredInstanceIds.pipe( + Effect.flatMap((registered) => + registered.includes(selection.instanceId) ? Effect.undefined : Effect.fail(error), + ), + ), + ), + ); + if (settings === undefined) return command; + const table = poolTable(settings); + if (!namesPool(table, selection.instanceId)) return command; + if (!isSwitchCommand(command)) { + return yield* new ProviderPoolUnresolvedError({ + poolId: selection.instanceId, + commandType: command.type, + }); + } + // A retry of an accepted command replays its first result, so it is not placed again. + if (yield* orchestrator.hasReceipt(command.commandId)) return command; + const shell = yield* orchestrator.threadShell(command.threadId); + const registered = yield* orchestrator.registeredInstanceIds; + const candidates = + orchestrator.providers !== undefined + ? candidatesFromProviders({ + settings, + projectId: shell?.projectId ?? null, + model: selection.model, + providers: (yield* orchestrator.providers).filter((provider) => + registered.includes(provider.instanceId), + ), + unusableReason: providerUnusableReason, + skipProjectEnablementFilter: false, + }) + : registered.map((registeredId) => ({ + instanceId: registeredId, + unusableReason: null, + enabledForProject: resolveProjectProviderInstanceEnabled( + settings, + shell?.projectId ?? null, + registeredId, + ), + supportsModel: null, + limitedUntil: null, + })); + const instanceId = yield* placeInPool({ + table, + kind: "switch", + requestedInstanceId: selection.instanceId, + currentInstanceId: shell?.providerInstanceId ?? null, + driver: undefined, + model: selection.model, + seedKey: command.commandId, + candidates, + }); + return { ...command, modelSelection: { ...selection, instanceId } }; +}); + +/** Wraps orchestrator dispatch so pool switches and creates resolve without inline upstream wiring. */ +export const wrapOrchestratorDispatch = Effect.fn("providerPools.wrapOrchestratorDispatch")( + function* ( + dispatchWithReceipt: (command: OrchestrationV2ServerCommand) => Effect.Effect, + deps: { + readonly serverSettings: { readonly getSettings: GetSettings }; + readonly projectionStore: { + readonly getThreadShell: (threadId: ThreadId) => Effect.Effect< + { + readonly projectId: ProjectId; + readonly providerInstanceId: ProviderInstanceId; + } | null, + ShellE + >; + }; + readonly providerAdapters: { + readonly list: () => Effect.Effect>; + }; + readonly commandReceipts: { + readonly getByCommandId: ( + commandId: CommandId, + ) => Effect.Effect, ReceiptE>; + }; + readonly mapError: ( + command: OrchestrationV2ServerCommand, + cause: ProviderPoolPlacementError | ProviderPoolUnresolvedError | ServerSettingsError, + ) => E; + }, + ) { + const providerRegistry = yield* Effect.serviceOption(ProviderRegistry.ProviderRegistry); + return Effect.fnUntraced(function* (command: OrchestrationV2ServerCommand) { + const resolved = yield* resolveCommandPool(command, { + getSettings: deps.serverSettings.getSettings, + threadShell: (threadId) => + deps.projectionStore.getThreadShell(threadId).pipe(Effect.orElseSucceed(() => null)), + registeredInstanceIds: deps.providerAdapters.list(), + providers: Option.isSome(providerRegistry) + ? providerRegistry.value.getProviders + : undefined, + hasReceipt: (commandId) => + deps.commandReceipts.getByCommandId(commandId).pipe( + Effect.map(Option.isSome), + Effect.orElseSucceed(() => false), + ), + }).pipe(Effect.mapError((cause) => deps.mapError(command, cause))); + return yield* dispatchWithReceipt(resolved); + }); + }, +); diff --git a/apps/server/src/fork/providerPools/pools.test.ts b/apps/server/src/fork/providerPools/pools.test.ts new file mode 100644 index 0000000000000..5226de3937496 --- /dev/null +++ b/apps/server/src/fork/providerPools/pools.test.ts @@ -0,0 +1,296 @@ +import { assert, describe, it } from "@effect/vitest"; +import { + ProviderDriverKind, + ProviderInstanceId, + ProviderPoolId, + type ProviderInstanceConfig, + type ServerSettings, +} from "@t3tools/contracts"; + +import { + buildPoolTable, + place, + resolvePoolSelection, + type MemberState, + type PoolCandidate, + type PoolSelectionRequest, + type ProviderPool, +} from "./pools.ts"; + +const id = ProviderInstanceId.make; +const DRIVER_KINDS: ReadonlySet = new Set(["claudeAgent", "codex", "antigravity"]); +const NOW = Date.parse("2026-11-20T12:00:00.000Z"); +const HOUR = 3_600_000; + +const instances: Record = { + claude_a: { driver: ProviderDriverKind.make("claudeAgent") }, + claude_b: { driver: ProviderDriverKind.make("claudeAgent") }, + claude_c: { driver: ProviderDriverKind.make("claudeAgent") }, + codex_a: { driver: ProviderDriverKind.make("codex") }, +}; + +const settingsWith = ( + providerPools: Record, +): Pick => ({ + providerPools, + providerInstances: instances as ServerSettings["providerInstances"], +}); + +const claudeAuto = { + displayName: "Claude Auto", + members: ["claude_a", "claude_b", "claude_c"], +}; + +const pool: ProviderPool = { + id: ProviderPoolId.make("claude_auto"), + displayName: "Claude Auto", + driver: ProviderDriverKind.make("claudeAgent"), + members: [id("claude_a"), id("claude_b"), id("claude_c")], +}; + +const eligible = (instanceId: string, limitedUntil: number | null = null): MemberState => ({ + instanceId: id(instanceId), + eligibility: { kind: "eligible", limitedUntil }, +}); + +const candidate = (instanceId: string, overrides: Partial = {}): PoolCandidate => ({ + instanceId: id(instanceId), + unusableReason: null, + enabledForProject: true, + supportsModel: (model) => model === "opus" || model === "sonnet", + limitedUntil: null, + ...overrides, +}); + +const table = buildPoolTable(settingsWith({ claude_auto: claudeAuto }), DRIVER_KINDS); + +const launch = (overrides: Partial = {}): PoolSelectionRequest => + ({ + kind: "launch", + requestedInstanceId: id("claude_auto"), + seedKey: "command:1", + table, + candidates: [candidate("claude_a"), candidate("claude_b"), candidate("claude_c")], + now: NOW, + ...overrides, + }) as PoolSelectionRequest; + +describe("buildPoolTable", () => { + it("keeps valid pools, drops each invalid pool with its reason and ignores instance ids", () => { + const built = buildPoolTable( + settingsWith({ + claude_auto: claudeAuto, + claude_a: { members: ["claude_b"] }, + codex: { members: ["codex_a"] }, + empty: { members: [] }, + twice: { members: ["claude_a", "claude_a"] }, + nested: { members: ["claude_auto"] }, + missing: { members: ["claude_z"] }, + mixed: { members: ["claude_a", "codex_a"] }, + malformed: { members: "claude_a" }, + "bad id!": { members: ["claude_a"] }, + defaults: { members: ["claudeAgent", "claude_b"] }, + }), + DRIVER_KINDS, + ); + + assert.deepStrictEqual([...built.pools.keys()].toSorted(), ["claude_auto", "defaults"]); + assert.deepStrictEqual(built.pools.get("claude_auto"), pool); + assert.strictEqual(built.pools.get("defaults")?.driver, "claudeAgent"); + assert.deepStrictEqual( + [...built.dropped.values()], + [ + "Provider pool 'bad id!' has an id that is not a valid instance id.", + "Provider pool 'empty' has no members.", + "Provider pool 'malformed' is not an object with a members list of instance ids.", + "Provider pool 'missing' lists 'claude_z', which is not a provider instance.", + "Provider pool 'mixed' mixes members of different drivers.", + "Provider pool 'nested' lists the pool 'claude_auto' as a member; pools cannot contain pools.", + "Provider pool 'twice' lists a member more than once.", + ], + ); + }); + + it("gives an empty table when no pools are set", () => { + const built = buildPoolTable( + { providerInstances: instances as ServerSettings["providerInstances"] }, + DRIVER_KINDS, + ); + assert.strictEqual(built.pools.size, 0); + assert.strictEqual(built.dropped.size, 0); + }); +}); + +describe("place", () => { + it("lands a retried key on the same member and spreads different keys", () => { + const members = [eligible("claude_a"), eligible("claude_b"), eligible("claude_c")]; + const first = place(pool, members, "command:retry", NOW); + for (const attempt of [1, 2, 3]) { + assert.deepStrictEqual(place(pool, members, "command:retry", NOW), first, `try ${attempt}`); + } + const chosen = new Set( + Array.from({ length: 30 }, (_, index) => { + const placement = place(pool, members, `command:${index}`, NOW); + return placement.kind === "placed" ? placement.instanceId : null; + }), + ); + assert.deepStrictEqual([...chosen].map(String).toSorted(), [ + "claude_a", + "claude_b", + "claude_c", + ]); + }); + + it("skips limited members while another member is free", () => { + const members = [ + eligible("claude_a", NOW + HOUR), + eligible("claude_b"), + eligible("claude_c", NOW + 2 * HOUR), + ]; + for (const key of ["k1", "k2", "k3", "k4"]) { + assert.deepStrictEqual(place(pool, members, key, NOW), { + kind: "placed", + poolId: pool.id, + instanceId: id("claude_b"), + limited: false, + }); + } + }); + + it("treats a limit that has passed as free", () => { + const placement = place(pool, [eligible("claude_a", NOW - 1)], "k", NOW); + assert.deepStrictEqual(placement, { + kind: "placed", + poolId: pool.id, + instanceId: id("claude_a"), + limited: false, + }); + }); + + it("picks the member whose limit ends first when all are limited", () => { + const members = [ + eligible("claude_a", NOW + 3 * HOUR), + eligible("claude_b", NOW + HOUR), + eligible("claude_c", NOW + 2 * HOUR), + ]; + assert.deepStrictEqual(place(pool, members, "any", NOW), { + kind: "placed", + poolId: pool.id, + instanceId: id("claude_b"), + limited: true, + }); + }); + + it("reports every member's reason when none is eligible", () => { + const placement = place( + pool, + [ + { instanceId: id("claude_a"), eligibility: { kind: "ineligible", reason: "is off." } }, + { instanceId: id("claude_b"), eligibility: { kind: "ineligible", reason: "no auth." } }, + ], + "k", + NOW, + ); + assert.deepStrictEqual(placement, { + kind: "noEligibleMember", + poolId: pool.id, + reasons: ["claude_a: is off.", "claude_b: no auth."], + }); + }); +}); + +describe("resolvePoolSelection", () => { + it("passes an explicit instance through, including a pool member", () => { + for (const requested of ["claude_b", "codex_a"]) { + assert.deepStrictEqual(resolvePoolSelection(launch({ requestedInstanceId: id(requested) })), { + kind: "concrete", + instanceId: id(requested), + }); + } + }); + + it("places a launch on a pool and gives the same member on retry", () => { + const first = resolvePoolSelection(launch()); + assert.strictEqual(first.kind, "placed"); + assert.deepStrictEqual(resolvePoolSelection(launch()), first); + }); + + it("filters members by registration, health, project, driver and model", () => { + const candidates = [ + candidate("claude_a", { enabledForProject: false }), + candidate("claude_b", { supportsModel: (model) => model === "sonnet" }), + ]; + assert.deepStrictEqual(resolvePoolSelection(launch({ candidates, model: "opus" })), { + kind: "noEligibleMember", + poolId: pool.id, + reasons: [ + "claude_a: is turned off for this project.", + "claude_b: does not offer model opus.", + "claude_c: is not registered.", + ], + }); + assert.deepStrictEqual( + resolvePoolSelection( + launch({ + candidates: [ + candidate("claude_a", { unusableReason: "is not signed in." }), + candidate("claude_b", { supportsModel: null }), + candidate("claude_c", { supportsModel: (model) => model === "sonnet" }), + ], + model: "opus", + }), + ), + { kind: "placed", poolId: pool.id, instanceId: id("claude_b"), limited: false }, + ); + const wrongDriver = resolvePoolSelection(launch({ driver: ProviderDriverKind.make("codex") })); + assert.strictEqual(wrongDriver.kind, "noEligibleMember"); + assert.include( + wrongDriver.kind === "noEligibleMember" ? wrongDriver.reasons[0] : "", + "runs claudeAgent, not codex.", + ); + }); + + it("keeps the current instance on a switch to the pool that holds it", () => { + assert.deepStrictEqual( + resolvePoolSelection({ ...launch(), kind: "switch", currentInstanceId: id("claude_c") }), + { kind: "kept", poolId: pool.id, instanceId: id("claude_c") }, + ); + }); + + it("moves a switch off a current member that is no longer eligible", () => { + for (const current of [ + candidate("claude_c", { enabledForProject: false }), + candidate("claude_c", { unusableReason: "is not signed in." }), + ]) { + const selection = resolvePoolSelection({ + ...launch(), + kind: "switch", + currentInstanceId: id("claude_c"), + candidates: [candidate("claude_a"), candidate("claude_b"), current], + }); + assert.strictEqual(selection.kind, "placed"); + assert.include( + ["claude_a", "claude_b"], + selection.kind === "placed" ? selection.instanceId : "", + ); + } + assert.strictEqual( + resolvePoolSelection({ + ...launch(), + kind: "switch", + currentInstanceId: id("claude_c"), + candidates: [], + }).kind, + "noEligibleMember", + ); + }); + + it("places a switch from outside the pool like a launch", () => { + const fromCodex = resolvePoolSelection({ + ...launch(), + kind: "switch", + currentInstanceId: id("codex_a"), + }); + assert.deepStrictEqual(fromCodex, resolvePoolSelection(launch())); + }); +}); diff --git a/apps/server/src/fork/providerPools/pools.ts b/apps/server/src/fork/providerPools/pools.ts new file mode 100644 index 0000000000000..ab561bf967e59 --- /dev/null +++ b/apps/server/src/fork/providerPools/pools.ts @@ -0,0 +1,265 @@ +import { + ProviderPoolConfig, + ProviderPoolId, + type ProviderDriverKind, + type ProviderInstanceId, + type ServerSettings, +} from "@t3tools/contracts"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +export interface ProviderPool { + readonly id: ProviderPoolId; + readonly displayName: string; + readonly driver: ProviderDriverKind; + /** Never empty, no repeats, every member an instance of `driver`. */ + readonly members: ReadonlyArray; +} + +export interface PoolTable { + readonly pools: ReadonlyMap; + /** Each dropped pool's id and the reason it was dropped. */ + readonly dropped: ReadonlyMap; +} + +const decodePoolId = Schema.decodeUnknownOption(ProviderPoolId); +const decodePoolConfig = Schema.decodeUnknownOption(ProviderPoolConfig); + +/** + * Checks every pool and keeps only the valid ones, so a bad pool never blocks the rest. Members + * must be distinct existing instances of one driver, never pools. A pool whose id equals an + * instance id or a driver kind (the driver kinds name the default instances) is ignored, not + * dropped: that id names the instance. + */ +export function buildPoolTable( + settings: Pick, + driverKinds: ReadonlySet, +): PoolTable { + const raw = settings.providerPools ?? {}; + const pools = new Map(); + const dropped = new Map(); + const driverOf = (instanceId: string): ProviderDriverKind | undefined => + Object.hasOwn(settings.providerInstances, instanceId) + ? settings.providerInstances[instanceId as ProviderInstanceId]?.driver + : driverKinds.has(instanceId) + ? (instanceId as ProviderDriverKind) + : undefined; + const poolIds = new Set(Object.keys(raw).filter((rawId) => driverOf(rawId) === undefined)); + + for (const rawId of [...poolIds].toSorted()) { + const problem = (detail: string) => dropped.set(rawId, `Provider pool '${rawId}' ${detail}`); + const id = decodePoolId(rawId); + if (Option.isNone(id)) { + problem("has an id that is not a valid instance id."); + continue; + } + const config = decodePoolConfig(raw[rawId]); + if (Option.isNone(config)) { + problem("is not an object with a members list of instance ids."); + continue; + } + const members = config.value.members; + if (members.length === 0) { + problem("has no members."); + continue; + } + if (new Set(members).size !== members.length) { + problem("lists a member more than once."); + continue; + } + const nested = members.find((member) => poolIds.has(member)); + if (nested !== undefined) { + problem(`lists the pool '${nested}' as a member; pools cannot contain pools.`); + continue; + } + const missing = members.find((member) => driverOf(member) === undefined); + if (missing !== undefined) { + problem(`lists '${missing}', which is not a provider instance.`); + continue; + } + const drivers = new Set(members.map((member) => driverOf(member))); + if (drivers.size !== 1) { + problem("mixes members of different drivers."); + continue; + } + pools.set(rawId, { + id: id.value, + displayName: config.value.displayName ?? rawId, + driver: driverOf(members[0]!)!, + members, + }); + } + return { pools, dropped }; +} + +export type MemberEligibility = + | { readonly kind: "eligible"; readonly limitedUntil: number | null } + | { readonly kind: "ineligible"; readonly reason: string }; + +export interface MemberState { + readonly instanceId: ProviderInstanceId; + readonly eligibility: MemberEligibility; +} + +export type Placement = + | { + readonly kind: "placed"; + readonly poolId: ProviderPoolId; + readonly instanceId: ProviderInstanceId; + /** True when every eligible member was limited and this one resets first. */ + readonly limited: boolean; + } + | { + readonly kind: "noEligibleMember"; + readonly poolId: ProviderPoolId; + readonly reasons: ReadonlyArray; + }; + +/** + * 32-bit FNV-1a. Same key and same eligible list give the same member, so a retried command lands + * where its first try did. + */ +const seedIndex = (seedKey: string, size: number): number => { + let hash = 0x811c9dc5; + for (const byte of new TextEncoder().encode(seedKey)) { + hash = Math.imul(hash ^ byte, 0x01000193); + } + return (hash >>> 0) % size; +}; + +/** + * Picks among eligible members that are not limited, by the seed. When every eligible member is + * limited it picks the one whose limit ends first. + */ +export function place( + pool: ProviderPool, + members: ReadonlyArray, + seedKey: string, + now: number, +): Placement { + const eligible = members.flatMap((member) => + member.eligibility.kind === "eligible" + ? [{ instanceId: member.instanceId, limitedUntil: member.eligibility.limitedUntil }] + : [], + ); + if (eligible.length === 0) { + return { + kind: "noEligibleMember", + poolId: pool.id, + reasons: members.map((member) => + member.eligibility.kind === "ineligible" + ? `${member.instanceId}: ${member.eligibility.reason}` + : `${member.instanceId}: eligible`, + ), + }; + } + const free = eligible.filter( + (member) => member.limitedUntil === null || member.limitedUntil <= now, + ); + if (free.length > 0) { + return { + kind: "placed", + poolId: pool.id, + instanceId: free[seedIndex(seedKey, free.length)]!.instanceId, + limited: false, + }; + } + const soonest = eligible.reduce((best, member) => + member.limitedUntil! < best.limitedUntil! ? member : best, + ); + return { kind: "placed", poolId: pool.id, instanceId: soonest.instanceId, limited: true }; +} + +/** What the caller knows about one concrete instance for this request. */ +export interface PoolCandidate { + readonly instanceId: ProviderInstanceId; + /** Why this caller cannot run on it now, or null when it can. */ + readonly unusableReason: string | null; + readonly enabledForProject: boolean; + /** Null when the caller does not know the models, which skips the model check. */ + readonly supportsModel: ((model: string) => boolean) | null; + readonly limitedUntil: number | null; +} + +interface PoolSelectionRequestBase { + readonly requestedInstanceId: ProviderInstanceId; + readonly driver?: ProviderDriverKind | undefined; + readonly model?: string | undefined; + readonly seedKey: string; + readonly table: PoolTable; + readonly candidates: ReadonlyArray; + readonly now: number; +} + +export type PoolSelectionRequest = + | (PoolSelectionRequestBase & { readonly kind: "launch" }) + | (PoolSelectionRequestBase & { + readonly kind: "switch"; + readonly currentInstanceId: ProviderInstanceId | null; + }); + +export type PoolSelection = + | { readonly kind: "concrete"; readonly instanceId: ProviderInstanceId } + | { + readonly kind: "placed"; + readonly poolId: ProviderPoolId; + readonly instanceId: ProviderInstanceId; + readonly limited: boolean; + } + | { + readonly kind: "kept"; + readonly poolId: ProviderPoolId; + readonly instanceId: ProviderInstanceId; + } + | { + readonly kind: "noEligibleMember"; + readonly poolId: ProviderPoolId; + readonly reasons: ReadonlyArray; + }; + +const memberEligibility = ( + pool: ProviderPool, + instanceId: ProviderInstanceId, + request: PoolSelectionRequestBase, +): MemberEligibility => { + if (request.driver !== undefined && request.driver !== pool.driver) { + return { kind: "ineligible", reason: `runs ${pool.driver}, not ${request.driver}.` }; + } + const candidate = request.candidates.find((entry) => entry.instanceId === instanceId); + if (candidate === undefined) return { kind: "ineligible", reason: "is not registered." }; + if (candidate.unusableReason !== null) { + return { kind: "ineligible", reason: candidate.unusableReason }; + } + if (!candidate.enabledForProject) { + return { kind: "ineligible", reason: "is turned off for this project." }; + } + if ( + request.model !== undefined && + candidate.supportsModel !== null && + !candidate.supportsModel(request.model) + ) { + return { kind: "ineligible", reason: `does not offer model ${request.model}.` }; + } + return { kind: "eligible", limitedUntil: candidate.limitedUntil }; +}; + +/** + * Turns a requested selection into a concrete instance. An instance id passes through. A pool id + * is placed in one member, except that a switch to the pool already holding the thread's instance + * keeps that instance while it is eligible, so nothing is handed off. + */ +export function resolvePoolSelection(request: PoolSelectionRequest): PoolSelection { + const pool = request.table.pools.get(request.requestedInstanceId); + if (pool === undefined) return { kind: "concrete", instanceId: request.requestedInstanceId }; + const members = pool.members.map((instanceId) => ({ + instanceId, + eligibility: memberEligibility(pool, instanceId, request), + })); + if (request.kind === "switch") { + const current = members.find((member) => member.instanceId === request.currentInstanceId); + if (current?.eligibility.kind === "eligible") { + return { kind: "kept", poolId: pool.id, instanceId: current.instanceId }; + } + } + return place(pool, members, request.seedKey, request.now); +} diff --git a/apps/server/src/mcp/OrchestratorMcpService.ts b/apps/server/src/mcp/OrchestratorMcpService.ts index 50ef6d73c3b09..415a8c3e21545 100644 --- a/apps/server/src/mcp/OrchestratorMcpService.ts +++ b/apps/server/src/mcp/OrchestratorMcpService.ts @@ -119,6 +119,10 @@ import * as ProjectService from "../project/ProjectService.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { + refreshRequestedInstances, + resolvePoolTarget, +} from "../fork/providerPools/poolPlacement.ts"; import { clientModeCeiling, type McpInvocationScope, @@ -1635,12 +1639,16 @@ const make = Effect.gen(function* () { Effect.catchIf( (error) => error.code === "provider_unavailable" || error.code === "model_unavailable", () => - (input.cwd === undefined - ? providerRegistry.refreshInstance(instanceId) - : providerRegistry - .refreshInstance(instanceId) - .pipe(Effect.andThen(scanWorkspace(instanceId, input.cwd))) - ).pipe( + refreshRequestedInstances({ + requestedInstanceId: instanceId, + getSettings: serverSettings.getSettings, + refreshInstance: (id) => + input.cwd === undefined + ? providerRegistry.refreshInstance(id) + : providerRegistry + .refreshInstance(id) + .pipe(Effect.andThen(scanWorkspace(id, input.cwd))), + }).pipe( Effect.flatMap((providers) => resolveTarget({ ...input, providers: withWorkspaceModels(providers, input.cwd) }), ), @@ -1655,10 +1663,28 @@ const make = Effect.gen(function* () { readonly providers: ReadonlyArray; /** Receipt replays bypass current settings because they start no work. Dispatch must reuse the checked command ID. */ readonly skipProjectEnablementFilter?: boolean; + readonly seedKey: string; }): Effect.fn.Return { - const requestedInstanceId = input.target?.providerInstanceId; const requestedDriver = input.target?.driverKind; const orchestrationCapableInstanceIds = yield* loadOrchestrationCapableInstanceIds(); + const requestedInstanceId = + input.target?.providerInstanceId === undefined + ? undefined + : yield* resolvePoolTarget({ + requestedInstanceId: input.target.providerInstanceId, + projectId: input.parent.thread.projectId, + driver: requestedDriver, + model: input.target.model, + seedKey: input.seedKey, + getSettings: serverSettings.getSettings, + providers: Effect.succeed(input.providers), + skipProjectEnablementFilter: input.skipProjectEnablementFilter, + unusableReason: (provider) => + providerConstraints( + provider, + orchestrationCapableInstanceIds.has(provider.instanceId), + ).join(" ") || null, + }).pipe(Effect.mapError((error) => failure("provider_unavailable", error.message))); let instanceId = requestedInstanceId; if (instanceId === undefined && requestedDriver !== undefined) { @@ -2850,6 +2876,7 @@ const make = Effect.gen(function* () { providers, cwd: yield* parentWorkspaceCwd(parent), skipProjectEnablementFilter: isRetryOfAcceptedDelegation, + seedKey: commandId, }); const runtimeMode = yield* resolveRuntimeMode( parent.thread.runtimeMode, @@ -3225,6 +3252,7 @@ const make = Effect.gen(function* () { providers, cwd, skipProjectEnablementFilter: isRetryOfAcceptedCreation, + seedKey: commandId, }); const runtimeMode = yield* resolveRuntimeMode( parent.thread.runtimeMode, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 8603a47378c5a..1a3d73ebd8dd5 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -215,6 +215,7 @@ import { import { planThreadDeletion } from "./ThreadDeletion.ts"; import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ServerSettings from "../serverSettings.ts"; +import { wrapOrchestratorDispatch } from "../fork/providerPools/poolPlacement.ts"; const isMessageQuestionAnswer = Schema.is( Schema.Union([TrimmedNonEmptyString, Schema.NonEmptyArray(TrimmedNonEmptyString)]), @@ -16893,7 +16894,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio awaitRestartContinuation: Effect.fn("OrchestratorV2.awaitRestartContinuation")(function (run) { return effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore); }), - dispatch: dispatchWithReceipt, + dispatch: yield* wrapOrchestratorDispatch(dispatchWithReceipt, { + serverSettings, + projectionStore, + providerAdapters, + commandReceipts, + mapError: (command, cause) => + new OrchestratorDispatchError({ + commandId: command.commandId, + commandType: command.type, + cause, + }), + }), autoArmLimitRecovery: Effect.fn("OrchestratorV2.autoArmLimitRecovery")(function (input) { return threadDispatch.withLock( input.threadId, diff --git a/apps/server/src/orchestration-v2/ThreadLaunchService.ts b/apps/server/src/orchestration-v2/ThreadLaunchService.ts index 6bf0513b00756..433beb1ab4fc9 100644 --- a/apps/server/src/orchestration-v2/ThreadLaunchService.ts +++ b/apps/server/src/orchestration-v2/ThreadLaunchService.ts @@ -51,6 +51,7 @@ import * as ProjectService from "../project/ProjectService.ts"; import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; +import { resolvePoolTarget } from "../fork/providerPools/poolPlacement.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as TextGeneration from "../textGeneration/TextGeneration.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; @@ -1250,11 +1251,30 @@ const make = Effect.gen(function* () { function* (request) { // Scratch has no Git repository. Normalize before folder allocation so // durable preparation and retries use the thread's own Scratch folder. - const input: ThreadLaunchInput = + const normalized: ThreadLaunchInput = request.workspaceStrategy.type === "worktree" && (yield* managedFolders.isScratchProject(request.projectId)) ? { ...request, workspaceStrategy: { type: "root" } } : request; + const launchReceipt = yield* readReceipt(normalized, normalized.commandId); + const instanceId = yield* resolvePoolTarget({ + requestedInstanceId: normalized.modelSelection.instanceId, + projectId: normalized.projectId, + model: normalized.modelSelection.model, + seedKey: normalized.commandId, + getSettings: serverSettings.getSettings, + providers: providerRegistry.getProviders, + committedInstanceId: Option.isSome(launchReceipt) + ? threads.getThreadShell(launchReceipt.value.threadId).pipe( + Effect.map((shell) => shell?.providerInstanceId ?? null), + Effect.orElseSucceed(() => null), + ) + : undefined, + }).pipe(Effect.mapError(mapError(normalized, "create-thread"))); + const input: ThreadLaunchInput = { + ...normalized, + modelSelection: { ...normalized.modelSelection, instanceId }, + }; yield* ProjectCloneTracker.rejectCommandsDuringClone(cloneTracker, { type: "thread.create", projectId: input.projectId, @@ -1275,7 +1295,6 @@ const make = Effect.gen(function* () { )("Reusing an existing thread requires a thread id."); } - const launchReceipt = yield* readReceipt(input, input.commandId); yield* threads.assertProviderInstanceEnabled({ projectId: input.projectId, commandId: input.commandId, diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 20e15f9c7033f..333687aab63d6 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -71,6 +71,7 @@ import { projectTurnItemForDetail } from "./WireProjection.ts"; import { cancelledBeforeStart } from "./RestartBackgroundNote.ts"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; import * as ServerSettings from "../serverSettings.ts"; +import { commandNamesPool } from "../fork/providerPools/poolPlacement.ts"; import { ThreadWorktreeRestore } from "../workspace/restoreMissingWorktree.ts"; export type ThreadManagementSendMode = "auto" | "queue" | "steer" | "restart"; @@ -828,7 +829,8 @@ const make = Effect.gen(function* () { const dispatch: ThreadManagementServiceShape["dispatch"] = (command) => ensureCommandTranscripts(command).pipe( - Effect.andThen(assertDispatchTargetEnabled(command)), + Effect.andThen(commandNamesPool(command, serverSettings.getSettings)), + Effect.flatMap((pool) => (pool ? Effect.void : assertDispatchTargetEnabled(command))), Effect.andThen(orchestrator.dispatch(command)), ); diff --git a/apps/server/src/provider/Drivers/CodexHomeLayout.ts b/apps/server/src/provider/Drivers/CodexHomeLayout.ts index d8a7270ab7130..79f72a0a52657 100644 --- a/apps/server/src/provider/Drivers/CodexHomeLayout.ts +++ b/apps/server/src/provider/Drivers/CodexHomeLayout.ts @@ -31,8 +31,15 @@ const KNOWN_SHARED_DIRECTORIES = [ // auth.lock guards the shadow home's private auth.json. const PRIVATE_ENTRY_NAMES = new Set(["auth.json", "auth.lock", "models_cache.json"]); -// Finder maintains metadata separately for each directory, including shadow homes. -const SHADOW_LOCAL_ENTRY_NAMES = new Set(["log", "memories", "tmp", ".DS_Store"]); +// Finder maintains metadata separately for each directory, including shadow homes. A shared +// app-server-control would connect a shadow home's clients to a daemon on the shared account. +const SHADOW_LOCAL_ENTRY_NAMES = new Set([ + "log", + "memories", + "tmp", + ".DS_Store", + "app-server-control", +]); // Shadow homes may create local runtime locks first; replace them with shared links because they guard shared state. const REPLACEABLE_SHARED_RUNTIME_ENTRIES = new Set([ "mcp-oauth-locks", @@ -436,6 +443,13 @@ export const materializeCodexShadowHome = Effect.fn("materializeCodexShadowHome" }), { discard: true }, ); + // Older shadow homes linked app-server-control to a shared home; only a local one isolates. + yield* removePrivateSymlink({ + fileSystem, + sharedHomePath: layout.sharedHomePath, + effectiveHomePath, + entryName: "app-server-control", + }); yield* Effect.forEach( entries, diff --git a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts index 1e882e3796c35..4cbd5fb188767 100644 --- a/apps/server/src/provider/ProviderInstanceRegistryHydration.ts +++ b/apps/server/src/provider/ProviderInstanceRegistryHydration.ts @@ -32,6 +32,8 @@ import type * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import type * as BackgroundPolicy from "../background/BackgroundPolicy.ts"; import * as ServerConfig from "../config.ts"; import type * as PortScanner from "../preview/PortScanner.ts"; +import * as HomeProvisioner from "../fork/homeProvisioner/homeProvisionerLayer.ts"; +import { withProviderProcessIdentity } from "../fork/instanceIdentity.ts"; type ProviderInstanceRegistryHydrationEnv = | Exclude< @@ -200,7 +202,10 @@ export const layer: Layer.Layer< // Instances of drivers the policy leaves out stay listed, as unavailable, so their threads // explain why instead of losing their provider. const layerMutable = ProviderInstanceRegistry.layer({ - drivers: BUILT_IN_DRIVERS, + drivers: withProviderProcessIdentity( + BUILT_IN_DRIVERS, + (yield* ServerConfig.ServerConfig).dbPath, + ), configMap: initialConfigMap, blockedReason: (driver) => ManagedProviders.disallowedDriverReason(serverSettings.managedPolicy, driver), @@ -212,9 +217,10 @@ export const layer: Layer.Layer< Layer.provide(SecretResolver.layer), ); - return layerSettingsWatcher(settingsChanges, initialSettings).pipe( - Layer.provideMerge(layerMutable), - ); + return Layer.merge( + layerSettingsWatcher(settingsChanges, initialSettings), + HomeProvisioner.layer(deriveProviderInstanceConfigMap), + ).pipe(Layer.provideMerge(layerMutable)); }), ) as Layer.Layer< | ProviderInstanceRegistry.ProviderInstanceRegistry diff --git a/packages/contracts/src/fork/homeProvisioner.ts b/packages/contracts/src/fork/homeProvisioner.ts new file mode 100644 index 0000000000000..c0ffe4cb79c2a --- /dev/null +++ b/packages/contracts/src/fork/homeProvisioner.ts @@ -0,0 +1,33 @@ +import * as Schema from "effect/Schema"; + +import { ForwardCompatibleOptional, TrimmedNonEmptyString } from "../baseSchemas.ts"; + +/** The argv of a command that reads a `HomeDeclarationWireV1` as JSON on stdin. */ +export const HomeProvisionerCommand = Schema.Array(TrimmedNonEmptyString).check( + Schema.isMinLength(1), +); +export type HomeProvisionerCommand = typeof HomeProvisionerCommand.Type; + +export const HomeProvisionerSetting = ForwardCompatibleOptional(HomeProvisionerCommand); + +export const DeclaredHome = Schema.Union([ + Schema.Struct({ agent: Schema.Literal("claude"), configDir: TrimmedNonEmptyString }), + Schema.Struct({ agent: Schema.Literal("codex"), codexHome: TrimmedNonEmptyString }), + Schema.Struct({ agent: Schema.Literal("agy"), geminiDir: TrimmedNonEmptyString }), +]); +export type DeclaredHome = typeof DeclaredHome.Type; + +const HomeDeclarationEntry = Schema.Struct({ + home: DeclaredHome, +}); + +/** + * Each declaration replaces the declarer's whole list of provider homes. The server writes this + * form; the provisioner owns its parser. + */ +export const HomeDeclarationWireV1 = Schema.Struct({ + version: Schema.Literal(1), + declarer: TrimmedNonEmptyString, + homes: Schema.Array(HomeDeclarationEntry), +}); +export type HomeDeclarationWireV1 = typeof HomeDeclarationWireV1.Type; diff --git a/packages/contracts/src/fork/providerPools.ts b/packages/contracts/src/fork/providerPools.ts new file mode 100644 index 0000000000000..0a021fd98c868 --- /dev/null +++ b/packages/contracts/src/fork/providerPools.ts @@ -0,0 +1,26 @@ +import * as Schema from "effect/Schema"; + +import { ForwardCompatibleOptional, TrimmedNonEmptyString } from "../baseSchemas.ts"; +import { ProviderInstanceId } from "../providerInstance.ts"; + +/** + * A pool id travels where an instance id does (a model selection's `instanceId`), so it keeps the + * instance id rules. The server swaps it for one member before anything is persisted. + */ +export const ProviderPoolId = ProviderInstanceId.pipe(Schema.brand("ProviderPoolId")); +export type ProviderPoolId = typeof ProviderPoolId.Type; + +export const ProviderPoolConfig = Schema.Struct({ + displayName: Schema.optionalKey(TrimmedNonEmptyString), + members: Schema.Array(ProviderInstanceId), +}); +export type ProviderPoolConfig = typeof ProviderPoolConfig.Type; + +/** + * Entries stay unparsed so one bad pool never fails the whole settings file. The server parses and + * checks each entry, and drops the ones that fail. + */ +export const ProviderPools = Schema.Record(Schema.String, Schema.Unknown); +export type ProviderPools = typeof ProviderPools.Type; + +export const ProviderPoolsSetting = ForwardCompatibleOptional(ProviderPools); diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index a254f6c62afce..cf9bf48f34c91 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -41,6 +41,8 @@ export * from "./fork/threadSubThreads.ts"; export * from "./fork/threadUserView.ts"; export * from "./fork/fyi.ts"; export * from "./fork/rightPanelMenu.ts"; +export * from "./fork/providerPools.ts"; +export * from "./fork/homeProvisioner.ts"; export * from "./managedPolicy.ts"; export * from "./git.ts"; export * from "./vcs.ts"; diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 29ad1d512f009..c5aad50fb56b4 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -33,6 +33,8 @@ import { CompactWhenIdleSettingsPatch, } from "./fork/idleCompactionLevels.ts"; import { RightPanelMenuOverrides } from "./fork/rightPanelMenu.ts"; +import { HomeProvisionerSetting } from "./fork/homeProvisioner.ts"; +import { ProviderPoolsSetting } from "./fork/providerPools.ts"; import { CustomModelSetting, DEFAULT_TEXT_GENERATION_MODEL, @@ -1595,6 +1597,8 @@ export const ServerSettings = Schema.Struct({ providerInstances: Schema.Record(ProviderInstanceId, ProviderInstanceConfig).pipe( Schema.withDecodingDefault(Effect.succeed({})), ), + providerPools: ProviderPoolsSetting, + homeProvisioner: HomeProvisionerSetting, observability: ObservabilitySettings.pipe(Schema.withDecodingDefault(Effect.succeed({}))), sourceControlHosts: Schema.Record(Schema.String, SourceControlHostSettings).pipe( Schema.withDecodingDefault(Effect.succeed({})),