diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts index fd820a2c478d1..8e2c51ee96f21 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.test.ts @@ -229,8 +229,12 @@ describe("DesktopBackendConfiguration", () => { const environment = yield* DesktopEnvironment.DesktopEnvironment; const configuration = yield* DesktopBackendConfiguration.DesktopBackendConfiguration; - const first = yield* configuration.resolvePrimary; - const second = yield* configuration.resolvePrimary; + const first = yield* configuration.resolvePrimary.pipe( + Effect.provideService(HostProcess.Environment, {}), + ); + const second = yield* configuration.resolvePrimary.pipe( + Effect.provideService(HostProcess.Environment, { UV_THREADPOOL_SIZE: "" }), + ); assert.equal(first.executablePath, process.execPath); assert.deepEqual(first.args.slice(0, 3), [ @@ -242,6 +246,8 @@ describe("DesktopBackendConfiguration", () => { assert.equal(first.cwd, environment.backendCwd); assert.equal(first.captureOutput, true); assert.equal(first.env.ELECTRON_RUN_AS_NODE, "1"); + assert.equal(first.env.UV_THREADPOOL_SIZE, "16"); + assert.equal(second.env.UV_THREADPOOL_SIZE, ""); assert.equal(first.env.T3CODE_DESKTOP_SHELL_DISCOVERY_PENDING, "1"); assert.isUndefined(first.env.T3CODE_PORT); assert.isUndefined(first.env.T3CODE_MODE); diff --git a/apps/desktop/src/backend/DesktopBackendConfiguration.ts b/apps/desktop/src/backend/DesktopBackendConfiguration.ts index 9c12690f31351..350422fe39262 100644 --- a/apps/desktop/src/backend/DesktopBackendConfiguration.ts +++ b/apps/desktop/src/backend/DesktopBackendConfiguration.ts @@ -535,6 +535,7 @@ const resolvePrimaryStartConfig = Effect.fn("desktop.backendConfiguration.resolv const environment = yield* DesktopEnvironment.DesktopEnvironment; const serverExposure = yield* DesktopServerExposure.DesktopServerExposure; const backendExposure = yield* serverExposure.backendConfig; + const hostEnv = yield* HostProcess.Environment; const bootstrap = { mode: "desktop" as const, @@ -573,6 +574,8 @@ const resolvePrimaryStartConfig = Effect.fn("desktop.backendConfiguration.resolv env: { ...backendChildEnvPatch(), ELECTRON_RUN_AS_NODE: "1", + // A hung getaddrinfo holds a libuv thread for good; the default four starve everything else. + UV_THREADPOOL_SIZE: hostEnv.UV_THREADPOOL_SIZE ?? "16", T3CODE_DESKTOP_SHELL_DISCOVERY_PENDING: "1", T3CODE_CLI_PATH: Option.getOrUndefined(input.cliPath), [DESKTOP_HOST_SCOPE_ENV]: input.hostScope, diff --git a/apps/server/src/binCli.ts b/apps/server/src/binCli.ts index 995b207e7c781..244653829c32e 100644 --- a/apps/server/src/binCli.ts +++ b/apps/server/src/binCli.ts @@ -4,6 +4,7 @@ import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import * as Effect from "effect/Effect"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Layer from "effect/Layer"; import { Argument, Command } from "effect/cli"; import * as CliError from "effect/cli/CliError"; @@ -121,16 +122,21 @@ export const cli = makeCli(); export function runCli() { extendConnectionAttemptTimeout(); - keepOutboundFetchOnHttp1(); - offerCliInstall({ - args: process.argv.slice(2), - // A child started over IPC (a service launcher) has no one to answer. - interactive: Boolean(process.stdin.isTTY && process.stdout.isTTY) && process.send === undefined, - foreground: inTerminalForeground, - confirm: confirmCliInstall, - }).pipe( - // The offer handles its own errors; even an unexpected defect must not stop the command. - Effect.catchDefect(() => Effect.succeed(true)), + HostProcess.Platform.pipe( + Effect.map(keepOutboundFetchOnHttp1), + Effect.andThen( + offerCliInstall({ + args: process.argv.slice(2), + // A child started over IPC (a service launcher) has no one to answer. + interactive: + Boolean(process.stdin.isTTY && process.stdout.isTTY) && process.send === undefined, + foreground: inTerminalForeground, + confirm: confirmCliInstall, + }).pipe( + // The offer handles its own errors; even an unexpected defect must not stop the command. + Effect.catchDefect(() => Effect.succeed(true)), + ), + ), Effect.flatMap((proceed) => proceed ? Command.run(cli, { version: packageJson.version }) diff --git a/apps/server/src/outboundHttp.test.ts b/apps/server/src/outboundHttp.test.ts index ed3b09abaa5e6..00325019cd205 100644 --- a/apps/server/src/outboundHttp.test.ts +++ b/apps/server/src/outboundHttp.test.ts @@ -1,4 +1,6 @@ // @effect-diagnostics nodeBuiltinImport:off - the test runs a real TLS h2 server to see which protocol fetch negotiates. +import * as NodeDns from "node:dns"; +import * as NodeHttp from "node:http"; import * as NodeHttp2 from "node:http2"; import * as NodeNet from "node:net"; import * as NodeTls from "node:tls"; @@ -10,12 +12,13 @@ import * as Stream from "effect/Stream"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { FetchHttpClient, HttpClient } from "effect/http"; import * as Undici from "undici"; -import { afterAll, afterEach, beforeAll, describe, expect } from "vite-plus/test"; +import { afterAll, afterEach, beforeAll, describe, expect, vi } from "vite-plus/test"; import { builtinFetchUsesHttp2, ignoreMacosTypeOfServiceEinval, installHttp1FetchDispatcher, + makeBoundedLookup, } from "./outboundHttp.ts"; // Non-secret, self-signed 127.0.0.1 certificate, valid until 2126; ALPN h2 requires TLS. @@ -55,6 +58,7 @@ beforeAll(async () => { }); afterEach(() => { + vi.useRealTimers(); Undici.setGlobalDispatcher(defaultDispatcher); NodeNet.setDefaultAutoSelectFamilyAttemptTimeout(defaultAttemptTimeout); }); @@ -164,3 +168,159 @@ if (calls === 0) throw new Error("fetch did not call setTypeOfService"); }).pipe(Effect.provide(NodeServices.layer)), ); }); + +describe("makeBoundedLookup", () => { + type Callback = ( + error: NodeJS.ErrnoException | null, + address?: string | Array, + family?: number, + ) => void; + const hangingLookup = () => {}; + const resolverOf = (v4: Array, v6: Array = []) => ({ + resolve4: (_host: string, cb: (error: Error | null, found?: Array) => void) => + cb(null, v4), + resolve6: (_host: string, cb: (error: Error | null, found?: Array) => void) => + cb(null, v6), + }); + const run = ( + lookup: ReturnType, + options: Parameters[1], + ) => + new Promise<{ + error: NodeJS.ErrnoException | null; + address?: string | Array | undefined; + family?: number | undefined; + }>((resolve) => + lookup("example.test", options, (error, address, family) => + resolve({ error, address, family }), + ), + ); + + const afterDeadline = (pending: Promise) => { + vi.advanceTimersByTime(10); + return pending; + }; + + it("passes a prompt dns.lookup answer through untouched", async () => { + const lookup = makeBoundedLookup({ + timeoutMs: 50, + lookup: (_host, _options, cb) => cb(null, "10.0.0.1", 4), + resolver: resolverOf(["9.9.9.9"]), + }); + expect(await run(lookup, {})).toEqual({ error: null, address: "10.0.0.1", family: 4 }); + }); + + it("falls back to c-ares when getaddrinfo hangs, honoring family and all", async () => { + vi.useFakeTimers(); + const lookup = makeBoundedLookup({ + timeoutMs: 10, + lookup: hangingLookup, + resolver: resolverOf(["192.0.2.1"], ["2001:db8::1"]), + }); + expect(await afterDeadline(run(lookup, { family: 4 }))).toEqual({ + error: null, + address: "192.0.2.1", + family: 4, + }); + expect(await afterDeadline(run(lookup, { family: 6 }))).toEqual({ + error: null, + address: "2001:db8::1", + family: 6, + }); + expect(await afterDeadline(run(lookup, { all: true }))).toEqual({ + error: null, + address: [ + { address: "192.0.2.1", family: 4 }, + { address: "2001:db8::1", family: 6 }, + ], + family: undefined, + }); + }); + + it("ignores a late getaddrinfo answer after the fallback replied", async () => { + vi.useFakeTimers(); + let late: Callback = () => {}; + const lookup = makeBoundedLookup({ + timeoutMs: 10, + lookup: (_host, _options, cb) => { + late = cb; + }, + resolver: resolverOf(["192.0.2.1"]), + }); + const calls: Array = []; + await afterDeadline( + new Promise((resolve) => + lookup("example.test", { family: 4 }, (error, address) => { + calls.push(error ?? address); + resolve(); + }), + ), + ); + late(null, "10.9.9.9", 4); + expect(calls).toEqual(["192.0.2.1"]); + }); + + it("reports ENOTFOUND when the fallback finds nothing", async () => { + vi.useFakeTimers(); + const lookup = makeBoundedLookup({ + timeoutMs: 10, + lookup: hangingLookup, + resolver: resolverOf([]), + }); + expect((await afterDeadline(run(lookup, { family: 4 }))).error?.code).toBe("ENOTFOUND"); + }); + + it("lets undici connect through the bounded lookup while getaddrinfo hangs", async () => { + installHttp1FetchDispatcher({ + lookup: makeBoundedLookup({ + timeoutMs: 10, + lookup: hangingLookup, + resolver: resolverOf(["127.0.0.1"]), + }), + }); + const plain = NodeHttp.createServer((_request, response) => response.end("resolved")); + await new Promise((resolve) => plain.listen(0, "127.0.0.1", resolve)); + try { + const port = (plain.address() as NodeNet.AddressInfo).port; + const response = await Undici.request(`http://hung-dns.test:${port}/`); + expect(await response.body.text()).toBe("resolved"); + } finally { + await new Promise((resolve) => plain.close(() => resolve())); + } + }); + + it("resolves an env proxy hostname through the bounded lookup for HTTPS requests", async () => { + const proxy = NodeHttp.createServer(); + // CONNECT sockets leave the server's connection tracking, so closeAllConnections misses them. + const tunnels: Array = []; + proxy.on("connect", (request: NodeHttp.IncomingMessage, client: NodeNet.Socket) => { + const [host, port] = (request.url ?? "").split(":"); + const upstream = NodeNet.connect(Number(port), host, () => { + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + upstream.pipe(client); + client.pipe(upstream); + }); + tunnels.push(client, upstream); + }); + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)); + try { + const proxyUrl = `http://hung-proxy.test:${(proxy.address() as NodeNet.AddressInfo).port}`; + for (const name of ["https_proxy", "HTTPS_PROXY"]) vi.stubEnv(name, proxyUrl); + for (const name of ["no_proxy", "NO_PROXY"]) vi.stubEnv(name, ""); + Undici.setGlobalDispatcher(new Undici.EnvHttpProxyAgent()); + installHttp1FetchDispatcher({ + lookup: makeBoundedLookup({ + timeoutMs: 10, + lookup: hangingLookup, + resolver: resolverOf(["127.0.0.1"]), + }), + }); + expect(await versionViaGlobalDispatcher()).toBe("1.1"); + } finally { + vi.unstubAllEnvs(); + for (const socket of tunnels) socket.destroy(); + proxy.closeAllConnections(); + await new Promise((resolve) => proxy.close(() => resolve())); + } + }); +}); diff --git a/apps/server/src/outboundHttp.ts b/apps/server/src/outboundHttp.ts index ead14cb5d2035..7d52c5b427a38 100644 --- a/apps/server/src/outboundHttp.ts +++ b/apps/server/src/outboundHttp.ts @@ -1,3 +1,5 @@ +// @effect-diagnostics globalTimers:off -- A libuv getaddrinfo cannot be cancelled, so the lookup deadline is a plain timer cleared on every completion path. +import * as NodeDns from "node:dns"; import * as NodeModule from "node:module"; import * as NodeNet from "node:net"; @@ -10,25 +12,134 @@ const requireUndici = NodeModule.createRequire(import.meta.url); export const builtinFetchUsesHttp2 = (bundledUndiciVersion: string | undefined) => Number.parseInt(bundledUndiciVersion ?? "", 10) >= 8; +const LOOKUP_TIMEOUT_MS = 3_000; + +type LookupCallback = ( + error: NodeJS.ErrnoException | null, + address?: string | Array, + family?: number, +) => void; +type Lookup = (hostname: string, options: NodeDns.LookupOptions, callback: LookupCallback) => void; +type ResolveCallback = (error: Error | null, addresses?: Array) => void; +type ResolverLike = { + readonly resolve4: (hostname: string, callback: ResolveCallback) => void; + readonly resolve6: (hostname: string, callback: ResolveCallback) => void; +}; + +const notFound = (hostname: string): NodeJS.ErrnoException => + Object.assign(new Error(`getaddrinfo ENOTFOUND ${hostname}`), { + code: "ENOTFOUND", + syscall: "getaddrinfo", + hostname, + }); + +const wantedFamily = (family: NodeDns.LookupOptions["family"]): 0 | 4 | 6 => + family === 4 || family === "IPv4" ? 4 : family === 6 || family === "IPv6" ? 6 : 0; + +/** + * getaddrinfo has no timeout and every hung call permanently holds one of the four libuv + * threadpool threads (macOS mDNSResponder can stop answering one process for good). Bound the + * wait, then answer from c-ares, which does its own socket I/O and uses no pool thread. + */ +export function makeBoundedLookup( + options: { + readonly timeoutMs?: number; + readonly lookup?: Lookup; + readonly resolver?: ResolverLike; + } = {}, +): NodeNet.LookupFunction { + const timeoutMs = options.timeoutMs ?? LOOKUP_TIMEOUT_MS; + const lookup = options.lookup ?? (NodeDns.lookup as unknown as Lookup); + let resolver = options.resolver; + + return (( + hostname: string, + lookupOptions: NodeDns.LookupOptions | number, + callback: LookupCallback, + ) => { + const normalized: NodeDns.LookupOptions = + typeof lookupOptions === "number" ? { family: lookupOptions } : (lookupOptions ?? {}); + let settled = false; + const deliver = ( + error: NodeJS.ErrnoException | null, + addresses: Array, + ) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error !== null) return callback(error); + if (normalized.all) return callback(null, addresses); + const first = addresses[0]; + return first === undefined + ? callback(notFound(hostname)) + : callback(null, first.address, first.family); + }; + + const timer = setTimeout(() => { + if (settled) return; + resolver ??= new NodeDns.Resolver({ timeout: 2_000, tries: 2 }); + const dns = resolver; + const family = wantedFamily(normalized.family); + const versions: Array<4 | 6> = family === 0 ? [4, 6] : [family]; + const collected: Partial>> = {}; + let pending = versions.length; + for (const version of versions) { + const onResult: ResolveCallback = (error, found) => { + collected[version] = + error || !found ? [] : found.map((address) => ({ address, family: version })); + if (--pending > 0) return; + const all = [...(collected[4] ?? []), ...(collected[6] ?? [])]; + deliver(all.length === 0 ? notFound(hostname) : null, all); + }; + if (version === 4) dns.resolve4(hostname, onResult); + else dns.resolve6(hostname, onResult); + } + }, timeoutMs); + timer.unref(); + + lookup(hostname, normalized, (error, address, resolvedFamily) => { + if (error) return deliver(error, []); + deliver( + null, + Array.isArray(address) + ? address + : [{ address: address ?? "", family: resolvedFamily ?? 4 }], + ); + }); + }) as NodeNet.LookupFunction; +} + /** Preserves the EnvHttpProxyAgent installed by --use-env-proxy / NODE_USE_ENV_PROXY. */ -export function installHttp1FetchDispatcher(): void { +export function installHttp1FetchDispatcher( + options: { readonly lookup?: NodeNet.LookupFunction } = {}, +): void { const undici = requireUndici("undici") as typeof Undici; + const connectOptions = + options.lookup === undefined ? {} : { connect: { lookup: options.lookup } }; // Node's agent comes from its internal undici copy, so `instanceof` against the npm class is false. const usesEnvProxy = undici.getGlobalDispatcher().constructor.name === "EnvHttpProxyAgent"; undici.setGlobalDispatcher( usesEnvProxy ? new undici.EnvHttpProxyAgent({ allowH2: false, + ...connectOptions, + ...(options.lookup === undefined ? {} : { proxyTls: { lookup: options.lookup } }), // ProxyAgent's connector forces h2 and ignores allowH2; replace it for HTTPS proxies. - clientFactory: (origin) => new undici.Pool(origin, { allowH2: false }), + clientFactory: (origin) => new undici.Pool(origin, { allowH2: false, ...connectOptions }), }) - : new undici.Agent({ allowH2: false }), + : new undici.Agent({ allowH2: false, ...connectOptions }), ); } -/** Avoids uncaught idle-timeout errors on h2 streams (https://github.com/nodejs/undici/issues/5936). */ -export function keepOutboundFetchOnHttp1(): void { - if (builtinFetchUsesHttp2(process.versions.undici)) installHttp1FetchDispatcher(); +/** + * Avoids uncaught idle-timeout errors on h2 streams (https://github.com/nodejs/undici/issues/5936), + * and on macOS bounds name resolution so a hung getaddrinfo cannot starve the libuv threadpool. + */ +export function keepOutboundFetchOnHttp1(platform: NodeJS.Platform): void { + const bounded = platform === "darwin"; + if (bounded || builtinFetchUsesHttp2(process.versions.undici)) { + installHttp1FetchDispatcher(bounded ? { lookup: makeBoundedLookup() } : {}); + } } // @types/node does not declare setTypeOfService yet. diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 213036b8f2ff6..66848e6815d7d 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -544,6 +544,7 @@ const layerTerminal = TerminalManager.layer( Layer.provide(layerPtyAdapter), Layer.provide(layerPortScanner), Layer.provide(layerNativeTelemetry), + Layer.provide(SecretResolver.layer), ); const layerPreview = Layer.empty.pipe( diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index c7259b4bd84fb..5ae0d5932f1c8 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -1323,6 +1323,7 @@ if (context.update?.status === "pending") { yield* fs.writeFileString(envPath, "T3CODE_TEST_SERVICE_TOKEN=initial\n"); vi.stubEnv("T3CODE_HOME", root); vi.stubEnv("T3CODE_TEST_SERVICE_TOKEN", "inherited"); + vi.stubEnv("UV_THREADPOOL_SIZE", undefined); yield* Effect.addFinalizer(() => Effect.sync(() => vi.unstubAllEnvs())); yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); yield* fs.writeFileString(databasePath, "before trial"); @@ -1332,6 +1333,7 @@ import { appendFileSync, writeFileSync } from "node:fs"; const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); appendFileSync(${JSON.stringify(observationsPath)}, JSON.stringify({ token: process.env.T3CODE_TEST_SERVICE_TOKEN, + threadpool: process.env.UV_THREADPOOL_SIZE, update: context.update?.status ?? "active", }) + "\\n"); if (context.update?.status === "pending") { @@ -1392,9 +1394,9 @@ if (context.update?.status === "pending") { .split("\n") .map((line) => JSON.parse(line)); assert.deepEqual(observations, [ - { token: "initial", update: "active" }, - { token: "initial", update: "pending" }, - { token: "initial", update: "rolled-back" }, + { token: "initial", threadpool: "16", update: "active" }, + { token: "initial", threadpool: "16", update: "pending" }, + { token: "initial", threadpool: "16", update: "rolled-back" }, ]); assert.equal((yield* HostProcess.Environment).T3CODE_TEST_SERVICE_TOKEN, "inherited"); assert.equal(yield* fs.readFileString(envPath), "T3CODE_TEST_SERVICE_TOKEN=changed\n"); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index 8dcd7886fce51..653ad580a0a79 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -1215,10 +1215,19 @@ export class Launcher { ...(update === undefined ? {} : { update }), }; const spawnArguments = runtimeSpawnArguments(paths); + // Windows reads names without regard to case, so any spelling there is the user's override. + const threadpoolSizeSet = Object.keys(this.#environment).some( + (name) => name.toUpperCase() === "UV_THREADPOOL_SIZE", + ); const appendStdout = this.#output === undefined && isRegularFile(1); const appendStderr = this.#output === undefined && isRegularFile(2); const child = NodeChildProcess.spawn(spawnArguments.command, spawnArguments.args, { - env: { ...this.#environment, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }, + env: { + ...this.#environment, + // A hung getaddrinfo holds a libuv thread for good; the default four starve everything else. + ...(threadpoolSizeSet ? {} : { UV_THREADPOOL_SIZE: "16" }), + [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context), + }, stdio: this.#output === undefined ? ["inherit", appendStdout ? "pipe" : "inherit", appendStderr ? "pipe" : "inherit", "ipc"] diff --git a/apps/server/src/terminal/Manager.test.ts b/apps/server/src/terminal/Manager.test.ts index c6f45a60ab75e..4202446ea1df2 100644 --- a/apps/server/src/terminal/Manager.test.ts +++ b/apps/server/src/terminal/Manager.test.ts @@ -12,11 +12,13 @@ import { type TerminalOpenInput, ProviderDriverKind, ProviderInstanceId, + SecretProviderId, retainedTerminalHistory, terminalSequenceEnd, terminalSequenceStateAfter, TerminalProviderInstanceNotFoundError, } from "@t3tools/contracts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as HostProcess from "@t3tools/shared/HostProcess"; import { SpawnExecutableResolution } from "@t3tools/shared/shell"; import * as NodePtyAdapter from "./NodePtyAdapter.ts"; @@ -40,7 +42,7 @@ import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as TestClock from "effect/testing/TestClock"; -import { expect, vi } from "vite-plus/test"; +import { describe, expect, vi } from "vite-plus/test"; import * as ServerConfig from "../config.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; @@ -2804,6 +2806,82 @@ it.layer( ), ); + describe("secret-backed provider environment", () => { + const secret = { provider: SecretProviderId.make("vault"), reference: "kv/api#key" }; + const settingsWithSecret = ServerSettings.layerTest({ + providerInstances: { + [ProviderInstanceId.make("codex_secret")]: { + driver: ProviderDriverKind.make("codex"), + environment: [{ name: "PROVIDER_API_KEY", value: secret, sensitive: true }], + }, + [ProviderInstanceId.make("codex_secret_shadowed")]: { + driver: ProviderDriverKind.make("codex"), + environment: [ + { name: "PROVIDER_API_KEY", value: secret, sensitive: true }, + { name: "PROVIDER_API_KEY", value: "configured-later", sensitive: false }, + ], + }, + }, + }); + const resolveWith = ( + read: SecretResolver.SecretResolver["Service"]["read"] | undefined, + invalidate: SecretResolver.SecretResolver["Service"]["invalidate"] = () => Effect.void, + rawProviderInstanceId = "codex_secret", + ) => + Effect.gen(function* () { + const path = yield* Path.Path; + const serverSettings = yield* ServerSettings.ServerSettingsService; + return yield* TerminalManager.resolveProviderInstanceTerminalEnvironment({ + deriveProviderInstanceConfigMap, + serverSettings, + path, + rawProviderInstanceId, + env: undefined, + secrets: + read === undefined + ? undefined + : SecretResolver.SecretResolver.of({ + read, + invalidate, + label: (provider) => Effect.succeed(provider), + }), + }); + }).pipe(Effect.provide(settingsWithSecret)); + + it.effect("reads secrets into the terminal environment in configured order", () => + Effect.gen(function* () { + const environment = yield* resolveWith(() => Effect.succeed(["sk-from-vault"])); + expect(environment.PROVIDER_API_KEY).toBe("sk-from-vault"); + const shadowed = yield* resolveWith( + () => Effect.succeed(["sk-from-vault"]), + undefined, + "codex_secret_shadowed", + ); + expect(shadowed.PROVIDER_API_KEY).toBe("configured-later"); + }), + ); + + it.effect("fails the launch naming the variable and forgets the failed read", () => + Effect.gen(function* () { + const invalidated: Array | undefined> = []; + const error = yield* resolveWith( + () => Effect.succeed([undefined]), + (references) => Effect.sync(() => void invalidated.push(references)), + ).pipe(Effect.flip); + expect(error._tag).toBe("TerminalProviderEnvironmentError"); + expect(error.message).toContain("PROVIDER_API_KEY (vault:kv/api#key)"); + expect(invalidated).toEqual([[secret]]); + }), + ); + + it.effect("fails the launch when no secret resolver is available", () => + Effect.gen(function* () { + const error = yield* resolveWith(undefined).pipe(Effect.flip); + expect(error._tag).toBe("TerminalProviderEnvironmentError"); + }), + ); + }); + it.effect("restarts a running terminal when the resolved provider environment changes", () => Effect.gen(function* () { const providerInstanceId = ProviderInstanceId.make("codex_work"); diff --git a/apps/server/src/terminal/Manager.ts b/apps/server/src/terminal/Manager.ts index 99a15c28f9e7f..5165753fbe645 100644 --- a/apps/server/src/terminal/Manager.ts +++ b/apps/server/src/terminal/Manager.ts @@ -68,13 +68,11 @@ import * as SynchronizedRef from "effect/SynchronizedRef"; import packageJson from "../../package.json" with { type: "json" }; import * as ServerConfig from "../config.ts"; -import { - literalProviderInstanceEnvironment, - overlayProviderInstanceEnvironment, -} from "@t3tools/provider-core/server/instanceEnvironment"; +import { overlayProviderInstanceEnvironment } from "@t3tools/provider-core/server/instanceEnvironment"; import { resolveCodexHomeLayout } from "../provider/Drivers/CodexHomeLayout.ts"; import { makeClaudeEnvironment } from "../provider/Drivers/ClaudeHome.ts"; import type { deriveProviderInstanceConfigMap } from "../provider/ProviderInstanceRegistryHydration.ts"; +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver"; import * as ServerSettings from "../serverSettings.ts"; import { increment, terminalSessionsTotal } from "../observability/Metrics.ts"; import { expandHomeVariable } from "@t3tools/provider-core/server/pathExpansion"; @@ -1736,6 +1734,7 @@ export const resolveProviderInstanceTerminalEnvironment = Effect.fn( readonly path: Path.Path; readonly rawProviderInstanceId: string; readonly env: Record | undefined; + readonly secrets?: SecretResolver.SecretResolver["Service"] | undefined; }) { const providerInstanceId = ProviderInstanceId.make(input.rawProviderInstanceId); const settings = yield* input.serverSettings.getSettings.pipe( @@ -1746,8 +1745,39 @@ export const resolveProviderInstanceTerminalEnvironment = Effect.fn( return yield* new TerminalProviderInstanceNotFoundError({ providerInstanceId }); } + // A shell missing a configured credential would fall back to whatever the server inherited, + // so an unreadable secret fails the launch instead of dropping the variable. + const secretVariables = (instance.environment ?? []).flatMap((variable) => + typeof variable.value === "string" ? [] : [{ name: variable.name, reference: variable.value }], + ); + const secretValues = + secretVariables.length === 0 + ? [] + : input.secrets === undefined + ? secretVariables.map(() => undefined) + : yield* input.secrets.read(secretVariables.map(({ reference }) => reference)); + const unreadable = secretVariables.filter( + (_, index) => (secretValues[index] ?? "").trim() === "", + ); + if (unreadable.length > 0) { + // The resolver caches failed reads too, so a retry after unlocking must ask the provider again. + if (input.secrets !== undefined) { + yield* input.secrets.invalidate(unreadable.map(({ reference }) => reference)); + } + return yield* new TerminalProviderEnvironmentError({ + providerInstanceId, + unreadableSecrets: unreadable, + cause: new Error("A configured secret could not be read from its secret provider."), + }); + } + + let secretIndex = 0; let resolved = yield* overlayProviderInstanceEnvironment( - literalProviderInstanceEnvironment(instance.environment), + (instance.environment ?? []).map((variable) => + typeof variable.value === "string" + ? { ...variable, value: variable.value } + : { ...variable, value: secretValues[secretIndex++] ?? "", sensitive: true }, + ), input.env ?? {}, ); if (instance.driver === "codex") { @@ -1782,6 +1812,7 @@ export const make = Effect.fn("TerminalManager.make")(function* ( const portDiscovery = yield* PortScanner.PortDiscovery; const nativeTelemetry = yield* NativeTelemetryClient.NativeTelemetryClient; const serverSettings = yield* ServerSettings.ServerSettingsService; + const secrets = yield* SecretResolver.SecretResolver; const path = yield* Path.Path; const resolveProviderInstanceEnvironment = Effect.fn( "terminal.resolveProviderInstanceEnvironment", @@ -1792,6 +1823,7 @@ export const make = Effect.fn("TerminalManager.make")(function* ( path, rawProviderInstanceId, env, + secrets, }), ); return yield* makeWithOptions({ diff --git a/packages/contracts/src/terminal.ts b/packages/contracts/src/terminal.ts index 09678600764b2..e1badce09caa5 100644 --- a/packages/contracts/src/terminal.ts +++ b/packages/contracts/src/terminal.ts @@ -1,6 +1,7 @@ import * as Schema from "effect/Schema"; import { TrimmedNonEmptyString } from "./baseSchemas.ts"; import { ProviderInstanceId } from "./providerInstance.ts"; +import { SecretReference } from "./secretReference.ts"; export const DEFAULT_TERMINAL_ID = "term-1"; export const DEFAULT_MAX_TERMINAL_BUFFER_BYTES = 512 * 1024; @@ -470,11 +471,19 @@ export class TerminalProviderEnvironmentError extends Schema.TaggedError `${name} (${reference.provider}:${reference.reference})`, + ); + return unreadable.length === 0 + ? `Could not prepare the terminal environment for provider instance: ${this.providerInstanceId}` + : `Could not read ${unreadable.join(", ")} for provider instance: ${this.providerInstanceId}. Unlock its secret provider, then try again.`; } }