diff --git a/apps/server/src/auth/EnvironmentAuth.ts b/apps/server/src/auth/EnvironmentAuth.ts index 384afb2d72c1c..2731d660ad806 100644 --- a/apps/server/src/auth/EnvironmentAuth.ts +++ b/apps/server/src/auth/EnvironmentAuth.ts @@ -621,6 +621,19 @@ export class EnvironmentAuth extends Context.Service< }, ServerAuthInternalError >; + /** + * The calling link's own session, signed again for another full lifetime. The session id stays, + * so the work the link started here stays its own. Only a linked environment's session renews: + * an outside agent still signs in again when its session ends. None when it is revoked or gone. + */ + readonly renewMcpClientSession: (input: { + readonly sessionId: AuthSessionId; + readonly access: AuthMcpClientAccess; + readonly peerLink: boolean; + }) => Effect.Effect< + Option.Option<{ readonly token: string; readonly expiresAt: DateTime.Utc }>, + ServerAuthInternalError + >; /** * Spends a code covering all granted scopes and returns the level it approves. A level-bound * code must name `access`. Rejects desktop grants and leaves refused codes unspent. @@ -1567,6 +1580,21 @@ export const make = Effect.gen(function* () { Effect.withSpan("EnvironmentAuth.issueMcpClientSession"), ); + const renewMcpClientSession: EnvironmentAuth["Service"]["renewMcpClientSession"] = (input) => + input.peerLink + ? sessions + .renew({ + sessionId: input.sessionId, + ttl: MCP_CLIENT_SESSION_TTL, + ...(input.access === "read-only" ? {} : { runtimeModeCeiling: input.access }), + peerLink: true, + }) + .pipe( + Effect.mapError((cause) => new ServerAuthSessionTokenIssueError({ cause })), + Effect.withSpan("EnvironmentAuth.renewMcpClientSession"), + ) + : Effect.succeedNone; + const consumeMcpApprovalCode: EnvironmentAuth["Service"]["consumeMcpApprovalCode"] = ( code, access, @@ -1626,6 +1654,7 @@ export const make = Effect.gen(function* () { issueStartupPairingUrl, authenticateMcpClient, issueMcpClientSession, + renewMcpClientSession, consumeMcpApprovalCode, }); }); diff --git a/apps/server/src/auth/SessionStore.ts b/apps/server/src/auth/SessionStore.ts index 5717479a47ba9..2bd40e186ff7c 100644 --- a/apps/server/src/auth/SessionStore.ts +++ b/apps/server/src/auth/SessionStore.ts @@ -399,6 +399,20 @@ export class SessionStore extends Context.Service< readonly replaceSessionId?: AuthSessionId; }) => Effect.Effect; readonly verify: (token: string) => Effect.Effect; + /** + * Signs a live bearer session again with a later expiry, keeping its id, so everything bound + * to the session stays bound. None when it is revoked, expired, unknown or not a bearer session. + * The claims a row does not hold (`rtc`, `plk`) come from the caller's verified session. + */ + readonly renew: (input: { + readonly sessionId: AuthSessionId; + readonly ttl: Duration.Duration; + readonly runtimeModeCeiling?: RuntimeMode; + readonly peerLink?: boolean; + }) => Effect.Effect< + Option.Option<{ readonly token: string; readonly expiresAt: DateTime.Utc }>, + SessionCredentialInternalError + >; readonly issueWebSocketToken: ( sessionId: AuthSessionId, input?: { @@ -770,6 +784,49 @@ const make = Effect.gen(function* () { }, ); + const renew: SessionStore["Service"]["renew"] = Effect.fn("SessionStore.renew")( + function* (input) { + const { sessionId } = input; + const issueError = (cause: unknown) => new SessionCredentialIssueError({ sessionId, cause }); + const row = yield* authSessions.getById({ sessionId }).pipe(Effect.mapError(issueError)); + // A DPoP session's key binding is not re-signed here; only bearer sessions renew. + if ( + Option.isNone(row) || + row.value.revokedAt !== null || + row.value.method !== "bearer-access-token" + ) { + return Option.none(); + } + const now = yield* DateTime.now; + const expiresAt = DateTime.add(now, { milliseconds: Duration.toMillis(input.ttl) }); + const claims: SessionClaims = { + v: 1, + kind: "session", + sid: sessionId, + sub: row.value.subject, + scopes: row.value.scopes, + method: row.value.method, + ...(input.runtimeModeCeiling ? { rtc: input.runtimeModeCeiling } : {}), + ...(input.peerLink === true ? { plk: 1 as const } : {}), + iat: now.epochMilliseconds, + exp: expiresAt.epochMilliseconds, + }; + const encodedPayload = yield* encodeClaims(claims).pipe( + Effect.map(base64UrlEncode), + Effect.mapError(issueError), + ); + const signature = yield* signer.sign(encodedPayload); + // Only a session still live here moves; a revocation between the read and now wins. + const extended = yield* authSessions + .extend({ sessionId, expiresAt, now }) + .pipe(Effect.mapError(issueError)); + if (!extended) return Option.none(); + const session = yield* loadActiveSession(sessionId).pipe(Effect.orElseSucceed(Option.none)); + if (Option.isSome(session)) yield* emitUpsert(session.value); + return Option.some({ token: `${encodedPayload}.${signature}`, expiresAt }); + }, + ); + const verify: SessionStore["Service"]["verify"] = Effect.fn("SessionStore.verify")( function* (token) { if (devAuth?.matches(token)) { @@ -1145,6 +1202,7 @@ const make = Effect.gen(function* () { legacyCookieName, issue, verify, + renew, issueWebSocketToken, verifyWebSocketToken, listActive, diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 813f1ed59d164..18e66ef9f261f 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -296,6 +296,7 @@ export const make = Effect.gen(function* () { mcpModeLimitHeader: true, linkFence: true, linkedAppPairing: true, + linkRenewal: true, mcpApprovalCodes: true, mcpHostedApproval: true, shellThreadFamilies: true, diff --git a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts index 50f7d738c0b6d..38a46dca7b780 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/handlers.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/handlers.ts @@ -1,4 +1,4 @@ -import { mcpClientScopes, OrchestratorMcpFailure } from "@t3tools/contracts"; +import { AuthSessionId, mcpClientScopes, OrchestratorMcpFailure } from "@t3tools/contracts"; import { OrchestratorToolkit } from "./tools.ts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -66,6 +66,61 @@ export const linkAppPairing = Effect.map( ), ); +/** The calling link's own session, renewed in place, so a link never needs a new pairing code. */ +export const linkRenew = Effect.map( + Effect.serviceOption(EnvironmentAuth.EnvironmentAuth), + (environmentAuth) => + McpToolAccess.reads(() => + Effect.gen(function* () { + const { client } = yield* McpInvocationContext.McpInvocationContext; + if (client?.linked !== true) { + return yield* new OrchestratorMcpFailure({ + code: "capability_denied", + message: "Only a linked T3 Code environment can renew its link.", + }); + } + // The link's credential is the linked environment's, never one of its agents'. + if (client.senderThread !== undefined) { + return yield* new OrchestratorMcpFailure({ + code: "capability_denied", + message: "Only the linked environment can renew its link, not its agents.", + }); + } + if (Option.isNone(environmentAuth)) { + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "This environment cannot renew links here.", + }); + } + const renewed = yield* environmentAuth.value + .renewMcpClientSession({ + sessionId: AuthSessionId.make(client.sessionId), + access: client.access, + peerLink: true, + }) + .pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "The link could not be renewed here. The server log has details.", + }), + ), + ); + if (Option.isNone(renewed)) { + return yield* new OrchestratorMcpFailure({ + code: "capability_denied", + message: "This link's session was revoked or has ended here, so it cannot be renewed.", + }); + } + return { + accessToken: renewed.value.token, + expiresAt: DateTime.formatIso(renewed.value.expiresAt), + }; + }), + ), +); + const handlers = { orchestrator_capabilities: McpToolAccess.reads((input) => Effect.gen(function* () { @@ -291,9 +346,18 @@ const handlers = { return yield* service.interruptThread(scope, input); }), ), -} satisfies Omit, "t3_link_app_pairing">; +} satisfies Omit< + McpToolAccess.Handlers, + "t3_link_app_pairing" | "t3_link_renew" +>; export const layer = McpToolAccess.toLayer( OrchestratorToolkit, - Effect.map(linkAppPairing, (pairing) => ({ ...handlers, t3_link_app_pairing: pairing })), + Effect.all([linkAppPairing, linkRenew]).pipe( + Effect.map(([pairing, renew]) => ({ + ...handlers, + t3_link_app_pairing: pairing, + t3_link_renew: renew, + })), + ), ); diff --git a/apps/server/src/mcp/toolkits/orchestrator/tools.ts b/apps/server/src/mcp/toolkits/orchestrator/tools.ts index 64bdce98404fe..e27a98031e0b3 100644 --- a/apps/server/src/mcp/toolkits/orchestrator/tools.ts +++ b/apps/server/src/mcp/toolkits/orchestrator/tools.ts @@ -8,6 +8,7 @@ import { OrchestratorMcpEnvironmentUnlinkInput, OrchestratorMcpEnvironmentUnlinkResult, OrchestratorMcpLinkAppPairingResult, + OrchestratorMcpLinkRenewResult, OrchestratorMcpCreateThreadsInput, OrchestratorMcpCreateThreadsResult, OrchestratorMcpDelegateTaskInput, @@ -376,6 +377,19 @@ export const LinkAppPairingTool = Tool.make("t3_link_app_pairing", { .annotate(Tool.Destructive, false) .annotate(McpSchema.EnabledWhen, McpInvocationContext.onlyForLinks); +// Offered to another environment's link only, never to an agent. +export const LinkRenewTool = Tool.make("t3_link_renew", { + description: + "For a linked T3 Code environment: its link's credential, signed again with a later expiry. The link keeps its session here.", + success: OrchestratorMcpLinkRenewResult, + failure: OrchestratorMcpFailure, + failureMode: "return", + dependencies: [McpInvocationContext.McpInvocationContext], +}) + .annotate(Tool.Title, "Renew this link") + .annotate(Tool.Destructive, false) + .annotate(McpSchema.EnabledWhen, McpInvocationContext.onlyForLinks); + export const OrchestratorToolkit = Toolkit.make( OrchestratorCapabilitiesTool, DelegateTaskTool, @@ -398,4 +412,5 @@ export const OrchestratorToolkit = Toolkit.make( EnvironmentLinkTool, EnvironmentUnlinkTool, LinkAppPairingTool, + LinkRenewTool, ); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts index 3566598da64ac..2597e74336a92 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.test.ts @@ -15,6 +15,8 @@ import { modelAlertDestinations, modelAlertFor, overdueResumes, + type PeerLinkRenewalAlert, + recordLinkRenewal, recordModelAlert, recordUsageLimit, RESUME_GRACE_MS, @@ -633,3 +635,112 @@ it("a provider out of usage is tracked until its reset, and a healthy run clears ProviderExhaustion.clear("magpie-primary-codex"); assert.equal(ProviderExhaustion.until("magpie-primary-codex", 1_000), null); }); + +const linkAlert = (at: number): PeerLinkRenewalAlert => ({ + type: "peer-link-renewal", + id: `peer-link-renewal:environment-box:${at}`, + environmentId: "environment-a", + environmentLabel: "primary-r720", + linkedEnvironmentId: "environment-box", + linkedLabel: "ASUS", + linkSession: "peer-link:environment-box:1", + expiresAt: "2026-12-19T00:00:00.000Z", + detail: "ASUS did not answer.", + occurredAt: DateTime.formatIso(DateTime.makeUnsafe(at)), +}); + +it("a link that keeps failing to renew alerts once, until it renews", () => { + let state = EMPTY_MODEL_ALERT_STATE; + const enqueued: Array = []; + const step = ( + outcome: Parameters[1], + destinations: ReadonlyArray<"watcher" | "webhook"> = ["watcher", "webhook"], + ) => { + const result = recordLinkRenewal(state, outcome, destinations, 0); + state = result.state; + enqueued.push(result.enqueued); + }; + // Every hourly sweep fails; only the first one queues anything. + for (const at of [1, 2, 3]) step({ _tag: "failed", alert: linkAlert(at) }); + step({ _tag: "renewed", linkedEnvironmentId: "environment-box" }); + // A later failure after a renewal is news again. + step({ _tag: "failed", alert: linkAlert(4) }); + assert.deepEqual(enqueued, [true, false, false, false, true]); + assert.deepEqual( + state.outbox.map((entry) => [entry.alert.id, entry.destination]), + [ + ["peer-link-renewal:environment-box:1", "watcher"], + ["peer-link-renewal:environment-box:1", "webhook"], + ["peer-link-renewal:environment-box:4", "watcher"], + ["peer-link-renewal:environment-box:4", "webhook"], + ], + ); + // Another link failing is its own alert. + const other = recordLinkRenewal( + state, + { _tag: "failed", alert: { ...linkAlert(5), linkedEnvironmentId: "environment-air" } }, + ["watcher"], + 0, + ); + assert.equal(other.enqueued, true); + const message = watcherAlertMessage(linkAlert(1)); + assert.include(message, "Link alert from primary-r720"); + assert.include(message, "ASUS did not answer."); +}); + +it("a link alert counts only once queued, and only a delivered one keeps the link quiet", () => { + const WEEK = 7 * 24 * 60 * 60 * 1000; + // With nowhere to send it, nothing is marked, so the next sweep with a watcher alerts. + const nowhere = recordLinkRenewal( + EMPTY_MODEL_ALERT_STATE, + { _tag: "failed", alert: linkAlert(1) }, + [], + 0, + ); + assert.equal(nowhere.enqueued, false); + assert.deepEqual(nowhere.state.reported, {}); + const queued = recordLinkRenewal( + nowhere.state, + { _tag: "failed", alert: linkAlert(2) }, + ["watcher"], + 0, + ); + assert.equal(queued.enqueued, true); + + // Never delivered and gone from the outbox: the link alerts again. + const expired = recordLinkRenewal( + queued.state, + { _tag: "failed", alert: linkAlert(3) }, + ["watcher"], + WEEK + 1, + ); + assert.equal(expired.enqueued, true); + + // Delivered: quiet past the outbox's week, until it renews. + const delivered = afterDeliveryAttempt(queued.state, queued.state.outbox[0]!, true, 1); + assert.equal(delivered.outbox.length, 0); + const later = recordLinkRenewal( + delivered, + { _tag: "failed", alert: linkAlert(4) }, + ["watcher"], + WEEK + 1, + ); + assert.equal(later.enqueued, false); + + // Linking again names a new session, so its first failure is news. + const relinked = recordLinkRenewal( + delivered, + { _tag: "failed", alert: { ...linkAlert(5), linkSession: "peer-link:environment-box:2" } }, + ["watcher"], + 2, + ); + assert.equal(relinked.enqueued, true); + // A renewal clears every session's record for that link. + const renewed = recordLinkRenewal( + relinked.state, + { _tag: "renewed", linkedEnvironmentId: "environment-box" }, + [], + 3, + ); + assert.deepEqual(renewed.state.reported, {}); +}); diff --git a/apps/server/src/orchestration-v2/ModelAlertReactor.ts b/apps/server/src/orchestration-v2/ModelAlertReactor.ts index a4909326c6526..d9e1d37c09eca 100644 --- a/apps/server/src/orchestration-v2/ModelAlertReactor.ts +++ b/apps/server/src/orchestration-v2/ModelAlertReactor.ts @@ -82,8 +82,34 @@ export const ModelAlert = Schema.Struct({ }); export type ModelAlert = typeof ModelAlert.Type; -/** Everything the watcher thread hears: model alerts, thread events and usage limit incidents. */ -export const WatcherAlert = Schema.Union([ModelAlert, ThreadEventAlert, UsageLimitIncident]); +/** A link from this environment that could not renew its credential, so it will stop working. */ +export const PeerLinkRenewalAlert = Schema.Struct({ + type: Schema.Literal("peer-link-renewal"), + id: Schema.String, + environmentId: Schema.String, + environmentLabel: Schema.String, + linkedEnvironmentId: Schema.String, + linkedLabel: Schema.String, + /** + * Names the link's session here (its secret store entry, never the credential), so linking + * again starts a fresh alert rather than inheriting the old link's. + */ + linkSession: Schema.String, + /** When the link stops working, or stopped. */ + expiresAt: Schema.String, + /** Why the last renewal failed. */ + detail: Schema.String, + occurredAt: Schema.String, +}); +export type PeerLinkRenewalAlert = typeof PeerLinkRenewalAlert.Type; + +/** Everything the watcher thread hears: model alerts, thread events, usage limit incidents and links. */ +export const WatcherAlert = Schema.Union([ + ModelAlert, + ThreadEventAlert, + UsageLimitIncident, + PeerLinkRenewalAlert, +]); export type WatcherAlert = typeof WatcherAlert.Type; const Destination = Schema.Literals(["watcher", "webhook"]); @@ -329,6 +355,75 @@ export function recordThreadEvents( return { state: { ...state, reported, outbox }, enqueued }; } +const linkReportedPrefix = (linkedEnvironmentId: string) => `peer-link\n${linkedEnvironmentId}\n`; +const linkReportedKey = (alert: PeerLinkRenewalAlert) => + `${linkReportedPrefix(alert.linkedEnvironmentId)}${alert.linkSession}`; +/** A link alert's `reported` entry once some destination received it. */ +const linkDelivered = (id: string) => `delivered:${id}`; + +/** + * Records a link's renewal outcome. A failing link alerts once and stays quiet through every + * later failure while that alert is queued or once it was delivered. A renewal, or linking + * again, lets the next failure alert again, and so does an alert that expired from the outbox + * undelivered. Nothing counts as reported until something was queued. + */ +export function recordLinkRenewal( + state: ModelAlertState, + outcome: + | { readonly _tag: "renewed"; readonly linkedEnvironmentId: string } + | { readonly _tag: "failed"; readonly alert: PeerLinkRenewalAlert }, + destinations: ReadonlyArray, + nowMs: number, +): { readonly state: ModelAlertState; readonly enqueued: boolean } { + if (outcome._tag === "renewed") { + const prefix = linkReportedPrefix(outcome.linkedEnvironmentId); + const reported = Object.fromEntries( + Object.entries(state.reported).filter(([key]) => !key.startsWith(prefix)), + ); + if (Object.keys(reported).length === Object.keys(state.reported).length) { + return { state, enqueued: false }; + } + return { state: { ...state, reported }, enqueued: false }; + } + const { alert } = outcome; + const key = linkReportedKey(alert); + const previous = state.reported[key]; + if (previous !== undefined) { + const delivered = previous.startsWith(linkDelivered("")); + const queued = pruneOutbox(state.outbox, nowMs).some((entry) => entry.alert.id === previous); + if (delivered || queued) return { state, enqueued: false }; + } + const queued = enqueueAlert(state.outbox, alert, destinations, nowMs); + if (!queued.enqueued) return { state, enqueued: false }; + return { + state: { + ...state, + reported: { + ...Object.fromEntries( + Object.entries(state.reported) + .filter(([existing]) => existing !== key) + .slice(-(MAX_REPORTED - 1)), + ), + [key]: alert.id, + }, + outbox: queued.outbox, + }, + enqueued: true, + }; +} + +/** Marks a delivered link alert, so its link stays quiet until it renews or is linked again. */ +const markLinkDelivered = ( + reported: ModelAlertState["reported"], + entry: Delivery, +): ModelAlertState["reported"] => { + if (entry.alert.type !== "peer-link-renewal") return reported; + const key = linkReportedKey(entry.alert); + return reported[key] === entry.alert.id + ? { ...reported, [key]: linkDelivered(entry.alert.id) } + : reported; +}; + const pruneOutbox = (outbox: ReadonlyArray, nowMs: number) => outbox.filter((entry) => nowMs - entry.createdAt < OUTBOX_TTL_MS); @@ -367,6 +462,7 @@ export function afterDeliveryAttempt( return { ...state, incidentSentAt, + reported: delivered ? markLinkDelivered(state.reported, entry) : state.reported, outbox: pruneOutbox(state.outbox, nowMs).flatMap((candidate) => { if (!sameDelivery(candidate, entry)) return [candidate]; if (delivered) { @@ -574,8 +670,18 @@ function modelAlertMessage(alert: ModelAlert): string { ].join("\n\n"); } +function linkRenewalMessage(alert: PeerLinkRenewalAlert): string { + return [ + `Link alert from ${alert.environmentLabel} (environment ${alert.environmentId}): its link to ${alert.linkedLabel} (environment ${alert.linkedEnvironmentId}) could not renew its credential. It stops working at ${alert.expiresAt}, and then delegation and the shared inbox between the two machines stop.`, + `Last failure: ${alert.detail}`, + `Fix it: make sure ${alert.linkedLabel} is on, reachable and on a T3 Code build that renews links; ${alert.environmentLabel} retries every hour by itself. If its session there was revoked, link again. This link sends no further alert until it renews.`, + ].join("\n\n"); +} + export const watcherAlertMessage = (alert: WatcherAlert) => { switch (alert.type) { + case "peer-link-renewal": + return linkRenewalMessage(alert); case "thread-event": return threadEventMessage(alert); case "usage-limit-incident": @@ -640,6 +746,12 @@ export class ModelAlertReactor extends Context.Service< readonly flush: Effect.Effect; /** Judges every live lane for the time-based events: stuck. */ readonly sweep: Effect.Effect; + /** A link here renewed, or could not: a failing link tells the watcher once until it renews. */ + readonly linkRenewal: ( + outcome: + | { readonly _tag: "renewed"; readonly linkedEnvironmentId: string } + | { readonly _tag: "failed"; readonly alert: PeerLinkRenewalAlert }, + ) => Effect.Effect; } >()("t3/orchestration-v2/ModelAlertReactor") {} @@ -731,6 +843,11 @@ export const make = Effect.gen(function* () { * on another provider, or its parent replaced it. */ const currentAlert = Effect.fnUntraced(function* (alert: WatcherAlert) { + // A link that renewed while its alert waited has nothing left to say. + if (alert.type === "peer-link-renewal") { + const reported = (yield* Ref.get(state)).reported[linkReportedKey(alert)]; + return reported === alert.id || reported === linkDelivered(alert.id) ? alert : null; + } const stillCurrent = Effect.fnUntraced(function* ( threadId: string, runId: string | null, @@ -1228,6 +1345,27 @@ export const make = Effect.gen(function* () { ); }).pipe(Effect.catchCause((cause) => Effect.logWarning("watcher sweep failed", { cause }))); + const linkRenewal: ModelAlertReactor["Service"]["linkRenewal"] = Effect.fn( + "ModelAlertReactor.linkRenewal", + )( + function* (outcome) { + const current = yield* settings.getSettings; + const destinations = modelAlertDestinations({ + watcher: current.modelAlertWatcher, + webhookUrl: current.modelAlertWebhookUrl, + here: yield* environment.getEnvironmentId, + lineage: { parentThreadId: null, relationshipToParent: null }, + }); + const nowMs = yield* Clock.currentTimeMillis; + const enqueued = yield* update((state) => { + const result = recordLinkRenewal(state, outcome, destinations, nowMs); + return [result.enqueued, result.state]; + }); + if (enqueued) yield* Queue.offer(wake, undefined); + }, + Effect.catchCause((cause) => Effect.logWarning("recording a link renewal failed", { cause })), + ); + const start: ModelAlertReactor["Service"]["start"] = Effect.fn("ModelAlertReactor.start")( function* () { yield* forkParked( @@ -1253,7 +1391,7 @@ export const make = Effect.gen(function* () { }, ); - return ModelAlertReactor.of({ start, handle, flush, sweep }); + return ModelAlertReactor.of({ start, handle, flush, sweep, linkRenewal }); }); export const layer = Layer.effect(ModelAlertReactor, make); diff --git a/apps/server/src/peer/PeerLinkRenewal.test.ts b/apps/server/src/peer/PeerLinkRenewal.test.ts new file mode 100644 index 0000000000000..2229dac8cf13a --- /dev/null +++ b/apps/server/src/peer/PeerLinkRenewal.test.ts @@ -0,0 +1,229 @@ +import { NodeHttpServer } from "@effect/platform-node"; +import { expect, it } from "@effect/vitest"; +import { + EnvironmentId, + type OrchestratorMcpFailure, + OrchestratorMcpLinkRenewResult, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Ref from "effect/Ref"; +import * as Sink from "effect/Sink"; +import * as Stream from "effect/Stream"; +import { Toolkit } from "effect/ai"; +import * as TestClock from "effect/testing/TestClock"; + +import * as McpHttpServer from "../mcp/McpHttpServer.ts"; +import * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; +import * as McpToolAccess from "../mcp/McpToolAccess.ts"; +import { linkRenew } from "../mcp/toolkits/orchestrator/handlers.ts"; +import { LinkRenewTool } from "../mcp/toolkits/orchestrator/tools.ts"; +import * as ModelAlertReactor from "../orchestration-v2/ModelAlertReactor.ts"; +import * as PeerLinkRenewal from "./PeerLinkRenewal.ts"; +import * as PeerLinks from "./PeerLinks.ts"; +import * as PeerMcpClient from "./PeerMcpClient.ts"; +import { descriptorOf, layerLinkingEnvironment, linkTo, servePeer } from "./PeerLinks.testkit.ts"; + +const toolkit = Toolkit.make(LinkRenewTool); +const renewHandlers = McpToolAccess.toLayer( + toolkit, + Effect.map(linkRenew, (renew) => ({ t3_link_renew: renew })), +); + +type Outcome = Parameters[0]; + +/** A box that renews links, linked from A, with A's renewal sweep and the alerts it raised. */ +const setup = (capabilities: { readonly linkRenewal?: boolean } = { linkRenewal: true }) => + Effect.gen(function* () { + const box = descriptorOf("environment-box", "Box", capabilities); + const b = yield* servePeer(box, McpHttpServer.toolkitRegistration(toolkit, renewHandlers)); + const context = yield* Layer.build(layerLinkingEnvironment(descriptorOf("environment-a", "A"))); + const links = Context.get(context, PeerLinks.PeerLinks); + yield* linkTo(links, b, "full-access"); + const outcomes = yield* Ref.make>([]); + const renewal = yield* PeerLinkRenewal.make.pipe( + Effect.provide(context), + Effect.provide( + Layer.mock(ModelAlertReactor.ModelAlertReactor)({ + linkRenewal: (outcome) => Ref.update(outcomes, (seen) => [...seen, outcome]), + }), + ), + ); + const linkExpiry = links + .get(box.environmentId) + .pipe(Effect.map((link) => Option.getOrThrow(link).expiresAt)); + return { box, b, links, context, renewal, outcomes, linkExpiry }; + }); + +const days = (n: number) => DateTime.makeUnsafe(Duration.toMillis(Duration.days(n))); + +it.effect("renews a link inside its last 20 days in place, so it outlives its first 30", () => + Effect.scoped( + Effect.gen(function* () { + const { box, b, links, context, renewal, outcomes, linkExpiry } = yield* setup(); + expect(yield* linkExpiry).toEqual(days(30)); + const [session] = yield* b.linkedSessions; + const incarnation = yield* links.session(box.environmentId); + + // With 21 days left the sweep leaves the link alone. + yield* TestClock.adjust(Duration.days(9)); + yield* renewal.sweep; + expect(yield* linkExpiry).toEqual(days(30)); + expect(yield* Ref.get(outcomes)).toEqual([]); + + // With 19 days left it renews: the same session there, signed for 30 more days. + yield* TestClock.adjust(Duration.days(2)); + yield* renewal.sweep; + expect(yield* linkExpiry).toEqual(days(41)); + expect(yield* Ref.get(outcomes)).toEqual([ + { _tag: "renewed", linkedEnvironmentId: box.environmentId }, + ]); + const after = yield* b.linkedSessions; + expect(after.map((current) => current.sessionId)).toEqual([session!.sessionId]); + expect(after[0]!.expiresAt).toEqual(days(41)); + // The same incarnation here, so work bound to the session stays bound. + expect(yield* links.session(box.environmentId)).toEqual(incarnation); + + // Past the first 30 days the link still works, on the renewed token. + yield* TestClock.adjust(Duration.days(25)); + const again = yield* Context.get(context, PeerMcpClient.PeerMcpClient).call({ + environmentId: box.environmentId, + tool: "t3_link_renew", + arguments: {}, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + success: OrchestratorMcpLinkRenewResult, + }); + expect(DateTime.makeUnsafe(again.expiresAt)).toEqual(days(66)); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a peer that cannot renew alerts at once, with why", () => + Effect.scoped( + Effect.gen(function* () { + const { renewal, outcomes, linkExpiry } = yield* setup({}); + yield* TestClock.adjust(Duration.days(11)); + yield* renewal.sweep; + expect(yield* linkExpiry).toEqual(days(30)); + const [outcome] = yield* Ref.get(outcomes); + expect(outcome).toMatchObject({ + _tag: "failed", + alert: { + type: "peer-link-renewal", + environmentId: "environment-a", + linkedEnvironmentId: "environment-box", + linkedLabel: "Box", + expiresAt: DateTime.formatIso(days(30)), + }, + }); + expect(outcome?._tag === "failed" && outcome.alert.detail).toContain("cannot renew links"); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("a session revoked on the peer is not renewed and alerts", () => + Effect.scoped( + Effect.gen(function* () { + const { b, renewal, outcomes, linkExpiry } = yield* setup(); + const [session] = yield* b.linkedSessions; + expect(yield* b.auth.revokeSession(session!.sessionId)).toBe(true); + yield* TestClock.adjust(Duration.days(11)); + yield* renewal.sweep; + expect(yield* linkExpiry).toEqual(days(30)); + expect((yield* Ref.get(outcomes)).map((outcome) => outcome._tag)).toEqual(["failed"]); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it.effect("an expired link is not tried again and alerts that it needs linking again", () => + Effect.scoped( + Effect.gen(function* () { + const { box, b, links, renewal, outcomes } = yield* setup(); + const session = Option.getOrThrow(yield* links.session(box.environmentId)); + const before = (yield* Ref.get(b.bearers)).length; + yield* TestClock.adjust(Duration.days(31)); + yield* renewal.sweep; + expect((yield* Ref.get(b.bearers)).length).toBe(before); + const [outcome] = yield* Ref.get(outcomes); + expect(outcome).toMatchObject({ _tag: "failed", alert: { linkSession: session } }); + expect(outcome?._tag === "failed" && outcome.alert.detail).toContain("Link it again"); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + +it("a peer that is only off alerts in the link's last week; anything else alerts at once", () => { + const link = { expiresAt: days(30) }; + const transient = { transient: true, detail: "Box did not answer." }; + const permanent = { transient: false, detail: "revoked" }; + expect(PeerLinkRenewal.renewalDue(link, days(9))).toBe(false); + expect(PeerLinkRenewal.renewalDue(link, days(10))).toBe(true); + expect(PeerLinkRenewal.renewalDue(link, days(31))).toBe(true); + expect(PeerLinkRenewal.alertDue(transient, link, days(22))).toBe(false); + expect(PeerLinkRenewal.alertDue(transient, link, days(23))).toBe(true); + expect(PeerLinkRenewal.alertDue(permanent, link, days(11))).toBe(true); +}); + +const caller = ( + client: Partial | undefined, + thread?: McpInvocationContext.McpThreadCaller, +): McpInvocationContext.McpInvocationScope => ({ + environmentId: EnvironmentId.make("environment-box"), + requestNamespace: "caller", + thread, + client: + client === undefined + ? undefined + : { sessionId: "session", label: "Caller", access: "full-access", ...client }, + capabilities: new Set(["orchestration"]), + issuedAt: 0, + linkedEnvironments: true, +}); + +// Discovery offers the tool to links only. The handler must refuse everyone else on its own. +it.effect.each<{ + readonly name: string; + readonly scope: McpInvocationContext.McpInvocationScope; + readonly code: OrchestratorMcpFailure["code"]; +}>([ + { name: "an outside agent", scope: caller({}), code: "capability_denied" }, + { + name: "an agent's thread here", + scope: caller(undefined, { + threadId: ThreadId.make("thread:agent"), + providerSessionId: "provider:agent", + providerInstanceId: ProviderInstanceId.make("codex"), + }), + code: "capability_denied", + }, + { + name: "a link calling for one of its agent threads", + scope: caller({ + linked: true, + senderThread: { + environmentId: EnvironmentId.make("environment-a"), + threadId: ThreadId.make("thread:linked-agent"), + title: "Agent", + }, + }), + code: "capability_denied", + }, + // The link itself gets past both checks, to where this harness has no auth. + { name: "the link itself", scope: caller({ linked: true }), code: "orchestration_error" }, +])("$name calling t3_link_renew gets $code", ({ scope, code }) => + Effect.gen(function* () { + const built = yield* toolkit; + const { result } = yield* built + .handle("t3_link_renew", {}) + .pipe(Stream.unwrap, Stream.run(Sink.last()), Effect.flatMap(Effect.fromOption)); + expect(result).toMatchObject({ code }); + }).pipe( + Effect.provide(McpToolAccess.HandlersLayer.layer(renewHandlers)), + Effect.provideService(McpInvocationContext.McpInvocationContext, scope), + ), +); diff --git a/apps/server/src/peer/PeerLinkRenewal.ts b/apps/server/src/peer/PeerLinkRenewal.ts new file mode 100644 index 0000000000000..0cbfda176731d --- /dev/null +++ b/apps/server/src/peer/PeerLinkRenewal.ts @@ -0,0 +1,218 @@ +import { OrchestratorMcpLinkRenewResult, type PeerLink } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Duration from "effect/Duration"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Result from "effect/Result"; +import * as Schedule from "effect/Schedule"; +import type * as Scope from "effect/Scope"; + +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import * as ModelAlertReactor from "../orchestration-v2/ModelAlertReactor.ts"; +import { forkParked } from "../serverActivation.ts"; +import * as PeerLinks from "./PeerLinks.ts"; +import * as PeerMcpClient from "./PeerMcpClient.ts"; + +/** A link renews once this little of its 30 days is left, leaving weeks to retry a peer that is off. */ +const RENEW_WITHIN_DAYS = 20; +/** A peer that is only off or unreachable is retried quietly until the link has this little left. */ +const ALERT_TRANSIENT_WITHIN_DAYS = 7; +const FIRST_SWEEP_AFTER = Duration.minutes(2); +const SWEEP_EVERY = Duration.hours(1); +const RENEW_TIMEOUT = Duration.seconds(60); + +/** Why a renewal failed, and whether retrying later can fix it without a person. */ +interface RenewalFailure { + readonly transient: boolean; + readonly detail: string; +} + +/** Whether the sweep tries this link now: inside the renewal window, or already expired. */ +export const renewalDue = (link: Pick, now: DateTime.Utc) => + DateTime.isLessThanOrEqualTo(link.expiresAt, DateTime.add(now, { days: RENEW_WITHIN_DAYS })); + +/** Whether a failed renewal tells the watcher now, rather than retrying quietly. */ +export const alertDue = ( + failure: RenewalFailure, + link: Pick, + now: DateTime.Utc, +) => + !failure.transient || + DateTime.isLessThanOrEqualTo( + link.expiresAt, + DateTime.add(now, { days: ALERT_TRANSIENT_WITHIN_DAYS }), + ); + +/** + * Renews this environment's links before they expire. A link's credential is a 30-day session + * on the peer; the peer signs the same session again with a later expiry (`t3_link_renew`), so + * nobody pairs again and work bound to the session stays bound. A link that cannot renew tells + * the watcher once, through the model alert outbox, until a renewal succeeds. + */ +export class PeerLinkRenewal extends Context.Service< + PeerLinkRenewal, + { + readonly start: () => Effect.Effect; + /** Renews every due link once. */ + readonly sweep: Effect.Effect; + } +>()("t3/peer/PeerLinkRenewal") {} + +export const make = Effect.gen(function* () { + const links = yield* PeerLinks.PeerLinks; + const peers = yield* PeerMcpClient.PeerMcpClient; + const alerts = yield* ModelAlertReactor.ModelAlertReactor; + const environment = yield* ServerEnvironment.ServerEnvironment; + + /** The new expiry, or why there is none. */ + const attempt = Effect.fnUntraced(function* (link: PeerLink) { + const resolved = yield* links.resolve(link.environmentId).pipe( + Effect.mapError((error): RenewalFailure => ({ + transient: error.reason === "unreachable" || error.reason === "storage", + detail: error.message, + })), + ); + if (resolved.capabilities.linkRenewal !== true) { + return yield* Effect.fail({ + transient: false, + detail: `${resolved.link.label} runs a T3 Code version that cannot renew links. Update ${resolved.link.label}.`, + }); + } + // Full access here narrows nothing: the peer renews the session at the access it already has. + const renewed = yield* peers + .call({ + environmentId: link.environmentId, + tool: "t3_link_renew", + arguments: {}, + limits: { runtimeMode: "full-access", interactionMode: "default" }, + expectedIncarnation: resolved.incarnation, + success: OrchestratorMcpLinkRenewResult, + }) + .pipe( + Effect.timeoutOrElse({ + duration: RENEW_TIMEOUT, + orElse: () => + Effect.fail({ + code: "orchestration_error" as const, + message: `${resolved.link.label} did not answer within a minute.`, + }), + }), + Effect.mapError((failure): RenewalFailure => ({ + transient: failure.code !== "capability_denied" && failure.code !== "invalid_request", + detail: failure.message, + })), + ); + const expiresAt = DateTime.make(renewed.expiresAt); + if (Option.isNone(expiresAt)) { + return yield* Effect.fail({ + transient: false, + detail: `${resolved.link.label} answered in a shape this version does not understand. Update both environments.`, + }); + } + const kept = yield* links + .renewed({ + environmentId: link.environmentId, + incarnation: resolved.incarnation, + token: renewed.accessToken, + expiresAt: DateTime.toUtc(expiresAt.value), + }) + .pipe( + Effect.mapError((error): RenewalFailure => ({ transient: true, detail: error.message })), + ); + return { kept, expiresAt: DateTime.toUtc(expiresAt.value) }; + }); + + const renewOne = Effect.fnUntraced(function* (link: PeerLink, now: DateTime.Utc) { + // An expired link's token is refused there, so only linking again brings it back. + const expired = DateTime.isLessThanOrEqualTo(link.expiresAt, now); + const outcome = expired + ? Result.fail({ + transient: false, + detail: `The link expired before it could renew. Link it again with a new pairing code from ${link.label}.`, + }) + : yield* Effect.result(attempt(link)); + if (outcome._tag === "Success") { + if (outcome.success.kept) { + yield* Effect.logInfo("peer link renewed", { + linkedEnvironmentId: link.environmentId, + expiresAt: DateTime.formatIso(outcome.success.expiresAt), + }); + } else { + // Forgotten or linked again meanwhile: the new link is fresh, so nothing is left to alert. + yield* Effect.logInfo("peer link renewed after it was replaced; the renewal was not kept", { + linkedEnvironmentId: link.environmentId, + }); + } + yield* alerts.linkRenewal({ _tag: "renewed", linkedEnvironmentId: link.environmentId }); + return; + } + const failure = outcome.failure; + // The session names this link, so linking again starts a fresh alert. Forgotten between + // listing and now: nothing is left to renew or report. + const linkSession = yield* links.session(link.environmentId); + if (Option.isNone(linkSession)) return; + if (!expired) { + yield* Effect.logWarning("peer link renewal failed", { + linkedEnvironmentId: link.environmentId, + expiresAt: DateTime.formatIso(link.expiresAt), + transient: failure.transient, + detail: failure.detail, + }); + } + if (!alertDue(failure, link, now)) return; + const descriptor = yield* environment.getDescriptor; + const nowMs = yield* Clock.currentTimeMillis; + yield* alerts.linkRenewal({ + _tag: "failed", + alert: { + type: "peer-link-renewal", + id: `peer-link-renewal:${link.environmentId}:${nowMs}`, + environmentId: descriptor.environmentId, + environmentLabel: descriptor.label, + linkedEnvironmentId: link.environmentId, + linkedLabel: link.label, + linkSession: linkSession.value, + expiresAt: DateTime.formatIso(link.expiresAt), + detail: failure.detail, + occurredAt: DateTime.formatIso(now), + }, + }); + }); + + const sweep: PeerLinkRenewal["Service"]["sweep"] = Effect.gen(function* () { + if (!(yield* links.enabled)) return; + const now = yield* DateTime.now; + const stored = yield* links.stored; + for (const link of stored) { + if (!renewalDue(link, now)) continue; + // One link's defect never stops the others from renewing. + yield* renewOne(link, now).pipe( + Effect.catchCause((cause) => + Effect.logWarning("peer link renewal failed unexpectedly", { + linkedEnvironmentId: link.environmentId, + cause, + }), + ), + ); + } + }).pipe( + Effect.catchCause((cause) => Effect.logWarning("peer link renewal sweep failed", { cause })), + ); + + const start: PeerLinkRenewal["Service"]["start"] = Effect.fn("PeerLinkRenewal.start")( + function* () { + yield* forkParked( + Effect.sleep(FIRST_SWEEP_AFTER).pipe( + Effect.andThen(sweep.pipe(Effect.repeat(Schedule.spaced(SWEEP_EVERY)))), + ), + ); + }, + ); + + return PeerLinkRenewal.of({ start, sweep }); +}); + +export const layer = Layer.effect(PeerLinkRenewal, make); diff --git a/apps/server/src/peer/PeerLinks.ts b/apps/server/src/peer/PeerLinks.ts index 88ee16cf827f1..f79a18257a828 100644 --- a/apps/server/src/peer/PeerLinks.ts +++ b/apps/server/src/peer/PeerLinks.ts @@ -139,6 +139,17 @@ export class PeerLinks extends Context.Service< readonly get: ( environmentId: EnvironmentId, ) => Effect.Effect, PeerLinkError>; + /** + * Keeps a renewed token for the link session `incarnation` names, under that same name, so + * `session` does not change and work bound to it stays bound. False, keeping nothing, when the + * link was forgotten or linked again meanwhile. + */ + readonly renewed: (input: { + readonly environmentId: EnvironmentId; + readonly incarnation: string; + readonly token: string; + readonly expiresAt: DateTime.Utc; + }) => Effect.Effect; /** Forgets the link here. The peer still lists the session until it is revoked there. */ readonly unlink: (environmentId: EnvironmentId) => Effect.Effect; /** An address that answers as the linked peer, with its token, ready to call. */ @@ -792,6 +803,26 @@ const make = Effect.gen(function* () { }, ); + const renewed: PeerLinks["Service"]["renewed"] = Effect.fn("PeerLinks.renewed")( + function* (input) { + const row = yield* readRow(input.environmentId); + if (Option.isNone(row) || row.value.secret_name !== input.incarnation) return false; + // The old token and the new one name the same session there, so a call reading either works. + yield* secrets + .set(input.incarnation, new TextEncoder().encode(input.token)) + .pipe(Effect.catch(failStorage)); + const updated = yield* sql<{ environment_id: string }>` + UPDATE peer_environment_links SET expires_at = ${DateTime.formatIso(input.expiresAt)} + WHERE environment_id = ${input.environmentId} AND secret_name = ${input.incarnation} + RETURNING environment_id + `.pipe(Effect.catch(failStorage)); + if (updated.length > 0) return true; + // Forgotten or linked again since the read: that removed this name's token, or is about to. + yield* secrets.remove(input.incarnation).pipe(Effect.ignore); + return false; + }, + ); + const recordOutcome: PeerLinks["Service"]["recordOutcome"] = (environmentId, outcome) => DateTime.now.pipe( Effect.flatMap((now) => @@ -824,6 +855,7 @@ const make = Effect.gen(function* () { placementExclusions, session, get, + renewed, unlink, resolve, recordOutcome, diff --git a/apps/server/src/persistence/AuthSessions.ts b/apps/server/src/persistence/AuthSessions.ts index 04f10dd34b0b5..d140af3444203 100644 --- a/apps/server/src/persistence/AuthSessions.ts +++ b/apps/server/src/persistence/AuthSessions.ts @@ -81,6 +81,13 @@ export const RevokeAuthSessionInput = Schema.Struct({ }); export type RevokeAuthSessionInput = typeof RevokeAuthSessionInput.Type; +export const ExtendAuthSessionInput = Schema.Struct({ + sessionId: AuthSessionId, + expiresAt: Schema.DateTimeUtcFromString, + now: Schema.DateTimeUtcFromString, +}); +export type ExtendAuthSessionInput = typeof ExtendAuthSessionInput.Type; + export const RevokeOtherAuthSessionsInput = Schema.Struct({ currentSessionId: AuthSessionId, revokedAt: Schema.DateTimeUtcFromString, @@ -120,6 +127,10 @@ export class AuthSessionRepository extends Context.Service< readonly revoke: ( input: RevokeAuthSessionInput, ) => Effect.Effect; + /** Moves a live session's expiry; false when it is revoked, expired or unknown. */ + readonly extend: ( + input: ExtendAuthSessionInput, + ) => Effect.Effect; readonly revokeAllExcept: ( input: RevokeOtherAuthSessionsInput, ) => Effect.Effect, AuthSessionRepositoryError>; @@ -349,6 +360,20 @@ export const make = Effect.gen(function* () { `, }); + const extendSessionRows = SqlSchema.findAll({ + Request: ExtendAuthSessionInput, + Result: Schema.Struct({ sessionId: AuthSessionId }), + execute: ({ sessionId, expiresAt, now }) => + sql` + UPDATE auth_sessions + SET expires_at = ${expiresAt} + WHERE session_id = ${sessionId} + AND revoked_at IS NULL + AND expires_at > ${now} + RETURNING session_id AS "sessionId" + `, + }); + const proofKeyThumbprintRows = SqlSchema.findAll({ Request: GetAuthSessionByIdInput, Result: Schema.Struct({ proofKeyThumbprint: Schema.String }), @@ -497,6 +522,18 @@ export const make = Effect.gen(function* () { Effect.map((rows) => rows.length > 0), ); + const extend: AuthSessionRepository["Service"]["extend"] = (input) => + extendSessionRows(input).pipe( + Effect.mapError( + toPersistenceSqlOrDecodeError( + "AuthSessionRepository.extend:query", + "AuthSessionRepository.extend:decodeRows", + { sessionId: input.sessionId }, + ), + ), + Effect.map((rows) => rows.length > 0), + ); + const revokeAllExcept: AuthSessionRepository["Service"]["revokeAllExcept"] = (input) => revokeOtherSessionRows(input).pipe( Effect.mapError( @@ -553,6 +590,7 @@ export const make = Effect.gen(function* () { getById, listActive, revoke, + extend, revokeAllExcept, setLastConnectedAt, getProofKeyThumbprint, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index e209bbc5fa8d1..eee32fe9dab64 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -89,6 +89,7 @@ import * as McpHttpServer from "./mcp/McpHttpServer.ts"; import * as PeerForwarding from "./peer/PeerForwarding.ts"; import * as RemoteDelegation from "./peer/RemoteDelegation.ts"; import * as PeerLinkRequests from "./peer/PeerLinkRequests.ts"; +import * as PeerLinkRenewal from "./peer/PeerLinkRenewal.ts"; import * as PeerLinks from "./peer/PeerLinks.ts"; import * as PeerMcpClient from "./peer/PeerMcpClient.ts"; import * as McpSessionRegistry from "./mcp/McpSessionRegistry.ts"; @@ -867,12 +868,17 @@ const layerMakeRoutes = Layer.mergeAll( Layer.effectDiscard( ModelAlertReactor.ModelAlertReactor.pipe(Effect.flatMap((reactor) => reactor.start())), ), + // Renews links from here before their 30-day credentials expire; a link that cannot alerts once. + Layer.effectDiscard( + PeerLinkRenewal.PeerLinkRenewal.pipe(Effect.flatMap((renewal) => renewal.start())), + ), ).pipe( Layer.provide( PullRequestReviewGuideService.layer.pipe(Layer.provide(PullRequestReviewGuides.layer)), ), // delegate_task to a linked environment, from /mcp and the follower above. Layer.provide(RemoteDelegation.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), + Layer.provide(PeerLinkRenewal.layer), Layer.provide(ModelAlertReactor.layer.pipe(Layer.provide(ProjectionStoreV2.layer))), // Links agents ask for: shown in a thread from /mcp, answered over WebSocket. Layer.provide(PeerLinkRequests.layer), diff --git a/docs/internals/remote.md b/docs/internals/remote.md index 4a348382ac47b..6c8ed6b11ac13 100644 --- a/docs/internals/remote.md +++ b/docs/internals/remote.md @@ -103,6 +103,16 @@ Unlinking on the linking side only forgets the token. [`linkRequestTarget`](../../packages/client-runtime/src/linkRequest.ts) names a machine for it only when the client already reaches exactly one machine there. An address alone never moves an unexpired link to another address. +- **A link renews its session in place.** The linking side's hourly + [`PeerLinkRenewal`](../../apps/server/src/peer/PeerLinkRenewal.ts) sweep calls + the link-only `t3_link_renew` tool once 20 days or fewer are left. The peer + signs the same session again with a new 30-day expiry and extends its row, so + the session id that names linked work, its idempotency keys and the linking + side's incarnation all stay the same. Only peer-link bearer sessions renew, + and revoking the session there still ends the link. A failure that retrying + can fix (the peer is off or unreachable) alerts only in the link's last + week; anything else alerts at once. Each link sends one watcher alert until + it renews again. - **All callers on one side share the link's single session on the other**, so idempotency keys are hashed with the caller's namespace before they leave. Without that, two agents reusing a key would collide on the far side. diff --git a/docs/user/remote-access.md b/docs/user/remote-access.md index 6776b8a2a67bd..6e48e56964367 100644 --- a/docs/user/remote-access.md +++ b/docs/user/remote-access.md @@ -392,7 +392,12 @@ an address you trust: the code is sent there. - The link shows up in that machine's **Settings → Sharing & access** like any outside agent. Revoke it there to end it. **Forget** on this machine only removes it here. -- Links last 30 days. Link again with a new pairing code to renew one. +- A link's credential lasts 30 days, and the linking machine renews it by + itself once 20 days or fewer are left, with no pairing code. Both machines + need a T3 Code version that renews links. A link that cannot renew, because + the other machine stays off, runs an older T3 Code or revoked the link, sends + the watcher thread one alert. A revoked or expired link needs linking again + with a new pairing code. The app and its machine keep one list of your environments: what this machine links and what the app connects to. **Linked environments** shows a machine diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 19cd8d4273162..9bdabab1d9ce0 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -186,6 +186,8 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ linkFence: Schema.optionalKey(Schema.Boolean), /** An environment linked to this one can add it to its own client app, at no more than the link's access. */ linkedAppPairing: Schema.optionalKey(Schema.Boolean), + /** An environment linked to this one can renew its link's credential before it expires, without a pairing code. */ + linkRenewal: Schema.optionalKey(Schema.Boolean), /** Mints pairing codes that approve one agent sign-in at a set level (`AUTO-…`). */ mcpApprovalCodes: Schema.optionalKey(Schema.Boolean), /** Agent sign-ins on its T3 Connect address are approved on the hosted app, without a code. */ diff --git a/packages/contracts/src/orchestratorMcp.ts b/packages/contracts/src/orchestratorMcp.ts index c48c1dda76591..d297f09c37281 100644 --- a/packages/contracts/src/orchestratorMcp.ts +++ b/packages/contracts/src/orchestratorMcp.ts @@ -955,6 +955,13 @@ export const OrchestratorMcpLinkAppPairingResult = Schema.Struct({ }); export type OrchestratorMcpLinkAppPairingResult = typeof OrchestratorMcpLinkAppPairingResult.Type; +/** The same link session there, signed again with a later expiry. */ +export const OrchestratorMcpLinkRenewResult = Schema.Struct({ + accessToken: TrimmedNonEmptyString, + expiresAt: Schema.String, +}); +export type OrchestratorMcpLinkRenewResult = typeof OrchestratorMcpLinkRenewResult.Type; + export const OrchestratorMcpDeleteScheduledTaskInput = Schema.Struct({ environmentId: OrchestratorMcpEnvironmentTarget, scheduledTaskId: ScheduledTaskId,