From 1a4f8a78ea0e13e6b907844077707c45b12ba4f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:03:27 +0000 Subject: [PATCH 1/6] feat(lazurio): Buddy's thread signal for the sign by his avatar GET /api/lazurio/buddy/signal (orchestration:read) and `t3 lazurio buddy signal --json [--follow]` answer one versioned state of Buddy's thread: waiting on a pending request, failed (last settled turn failed or was interrupted without a stop request, or a bound task failed after it), answered (a turn completed after lastVisitedAt; since = the oldest unread answer, startedBy = person if any unread turn was theirs), otherwise quiet. A server without Buddy or before his thread exists answers 404, never quiet; --follow prints `unavailable` with the reason and reconnects while it cannot read. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/lazurio/agentCliConnection.ts | 25 ++ apps/server/src/lazurio/buddySignal.test.ts | 241 +++++++++++++++ apps/server/src/lazurio/buddySignal.ts | 289 ++++++++++++++++++ .../src/lazurio/buddySignalRead.test.ts | 253 +++++++++++++++ apps/server/src/lazurio/buddySurfaceCli.ts | 160 +++++++++- apps/server/src/lazurio/overlay.ts | 2 + scripts/lazurio-release-contract.test.mjs | 2 +- 7 files changed, 964 insertions(+), 8 deletions(-) create mode 100644 apps/server/src/lazurio/buddySignal.test.ts create mode 100644 apps/server/src/lazurio/buddySignal.ts create mode 100644 apps/server/src/lazurio/buddySignalRead.test.ts diff --git a/apps/server/src/lazurio/agentCliConnection.ts b/apps/server/src/lazurio/agentCliConnection.ts index b89e8a0d76df4..6b80d726c3ecb 100644 --- a/apps/server/src/lazurio/agentCliConnection.ts +++ b/apps/server/src/lazurio/agentCliConnection.ts @@ -192,6 +192,10 @@ export interface AgentCliConnection { route: string, body: unknown, ) => Effect.Effect<{ readonly status: number; readonly body: unknown }, AgentCliFailure>; + /** GET one of the overlay's routes as the person's client surface; every status comes back. */ + readonly getRoute: ( + route: string, + ) => Effect.Effect<{ readonly status: number; readonly body: unknown }, AgentCliFailure>; /** The live thread list, as the sidebar follows it. */ readonly subscribeShell: ( input: OrchestrationV2SubscribeShellInput, @@ -402,6 +406,27 @@ export const withAgentCliConnection = Effect.fn("LazurioAgentCli.withConnection" body: Option.getOrUndefined(decodeJson(text)), })), ), + getRoute: (route) => + ownerToken.pipe( + Effect.flatMap((token) => + httpClient + .execute( + HttpClientRequest.get(new URL(route, origin)).pipe( + HttpClientRequest.bearerToken(token), + ), + ) + .pipe( + Effect.flatMap((response) => + Effect.map(response.text, (text) => ({ status: response.status, text })), + ), + Effect.mapError(requestFailed), + ), + ), + Effect.map(({ status, text }) => ({ + status, + body: Option.getOrUndefined(decodeJson(text)), + })), + ), subscribeShell: (input) => Stream.unwrap( ws.pipe( diff --git a/apps/server/src/lazurio/buddySignal.test.ts b/apps/server/src/lazurio/buddySignal.test.ts new file mode 100644 index 0000000000000..f8f8b505340fd --- /dev/null +++ b/apps/server/src/lazurio/buddySignal.test.ts @@ -0,0 +1,241 @@ +import { assert, describe, it } from "@effect/vitest"; +import { MessageId, RunId, ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; + +import { + type BuddySignalInput, + type BuddySignalStarterMessage, + buddySignalMessageIds, + buddySignalOf, +} from "./buddySignal.ts"; + +const THREAD = ThreadId.make("buddy-thread"); +const at = (minute: number) => + DateTime.makeUnsafe(`2026-11-18T20:${String(minute).padStart(2, "0")}:00.000Z`); +const iso = (minute: number) => DateTime.formatIso(at(minute)); + +type Status = BuddySignalInput["runs"][number]["status"]; +/** A run started by `by`'s message `m`, settled at `minute` (null while it runs). */ +const run = (ordinal: number, status: Status, minute: number | null) => ({ + id: RunId.make(`run-${ordinal}`), + ordinal, + status, + userMessageId: MessageId.make(`m${ordinal}`), + completedAt: minute === null ? null : at(minute), +}); +const person = { createdBy: "user" as const }; +const agent = { createdBy: "agent" as const }; +const otherThread = { createdBy: "user" as const, senderThreadId: ThreadId.make("other") }; +const task = { createdBy: "user" as const, scheduledTaskId: ScheduledTaskId.make("task-1") }; + +const signal = ( + input: Partial> & { + readonly senders?: Record; + }, +) => { + const full = { + threadId: THREAD, + lastVisitedAt: null, + runs: [], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: [], + ...input, + }; + const messages = new Map( + Object.entries(input.senders ?? {}).map(([ordinal, sender]) => [ + MessageId.make(`m${ordinal}`), + sender, + ]), + ); + // The reader loads exactly the senders the signal names, so a test that leaves one out fails. + for (const id of buddySignalMessageIds(full)) + assert.isTrue(messages.has(id), `the test names the sender of ${id}`); + const { version, threadId, ...rest } = buddySignalOf({ ...full, messages }); + assert.strictEqual(version, 1); + assert.strictEqual(threadId, THREAD); + return rest; +}; + +describe("the sign by Buddy's avatar", () => { + it("is quiet for a new thread and once the person read every answer", () => { + assert.deepStrictEqual(signal({}), { + state: "quiet", + since: null, + request: null, + startedBy: null, + }); + assert.strictEqual( + signal({ + runs: [run(1, "completed", 5)], + lastVisitedAt: at(6), + senders: { 1: person }, + }).state, + "quiet", + ); + }); + + it("waits on a pending request, before a failure and an unread answer", () => { + const pendingRequests = [ + { kind: "user_input" as const, createdAt: at(9), runId: RunId.make("run-3") }, + { kind: "command" as const, createdAt: at(8), runId: RunId.make("run-3") }, + ]; + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2), run(2, "failed", 4), run(3, "running", null)], + pendingRequests, + senders: { 1: person, 2: person, 3: agent }, + }), + // The oldest request is the one it waits on; the run that asked is an agent's. + { state: "waiting", since: iso(8), request: { kind: "command" }, startedBy: "other" }, + ); + }); + + it("fails for a failed last turn or an interrupt nobody asked for, and a completed turn clears it", () => { + const failed = signal({ + runs: [run(1, "completed", 2), run(2, "failed", 4)], + lastVisitedAt: at(1), + senders: { 1: person, 2: person }, + }); + // The unread answer before it waits behind the failure. + assert.deepStrictEqual(failed, { + state: "failed", + since: iso(4), + request: null, + startedBy: "person", + }); + assert.strictEqual( + signal({ runs: [run(1, "interrupted", 3)], senders: { 1: task } }).state, + "failed", + ); + // Stopped on request: no fault, and nothing new either. + assert.strictEqual( + signal({ + runs: [run(1, "interrupted", 3)], + stopRequested: new Set([RunId.make("run-1")]), + lastVisitedAt: at(2), + senders: { 1: person }, + }).state, + "quiet", + ); + // A turn still running after the failure keeps it; a completed one clears it. + assert.strictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "running", null)], + senders: { 1: person }, + }).state, + "failed", + ); + assert.deepStrictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "completed", 5)], + lastVisitedAt: at(4), + senders: { 1: person, 2: person }, + }), + { state: "answered", since: iso(5), request: null, startedBy: "person" }, + ); + // A cancelled queued turn or a rollback settles nothing. + assert.strictEqual( + signal({ + runs: [run(1, "failed", 3), run(2, "cancelled", 4), run(3, "rolled_back", 5)], + senders: { 1: person }, + }).state, + "failed", + ); + }); + + it("fails for a bound task that failed after the last completed turn", () => { + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2)], + lastVisitedAt: at(3), + failedTaskRuns: [at(7), at(5)], + senders: { 1: person }, + }), + { state: "failed", since: iso(5), request: null, startedBy: "other" }, + ); + assert.strictEqual( + signal({ + runs: [run(1, "completed", 6)], + lastVisitedAt: at(7), + failedTaskRuns: [at(5)], + senders: { 1: person }, + }).state, + "quiet", + ); + }); + + it("answers from the oldest unread answer, the person's if they started any of them", () => { + // The person writes, Buddy answers, then three turns of another thread follow. + const runs = [ + run(1, "completed", 2), + run(2, "completed", 3), + run(3, "completed", 4), + run(4, "completed", 5), + ]; + assert.deepStrictEqual( + signal({ + runs, + lastVisitedAt: at(1), + senders: { 1: person, 2: otherThread, 3: otherThread, 4: task }, + }), + { state: "answered", since: iso(2), request: null, startedBy: "person" }, + ); + // All unread turns are others'; then the person's own turn completes. + const others = signal({ + runs: runs.slice(1), + lastVisitedAt: at(1), + senders: { 2: otherThread, 3: otherThread, 4: task }, + }); + assert.deepStrictEqual(others, { + state: "answered", + since: iso(3), + request: null, + startedBy: "other", + }); + assert.deepStrictEqual( + signal({ + runs: [...runs.slice(1), run(5, "completed", 8)], + lastVisitedAt: at(1), + senders: { 2: otherThread, 3: otherThread, 4: task, 5: person }, + }), + { ...others, startedBy: "person" }, + ); + // An agent's message, sent from outside any thread, is not the person's. + assert.strictEqual( + signal({ runs: [run(1, "completed", 2)], senders: { 1: agent } }).startedBy, + "other", + ); + }); + + it("reading while a turn runs leaves that turn's answer unread", () => { + const runs = [run(1, "completed", 2), run(2, "running", null)]; + assert.strictEqual( + signal({ runs, lastVisitedAt: at(3), senders: { 1: person } }).state, + "quiet", + ); + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 2), run(2, "completed", 6)], + lastVisitedAt: at(3), + senders: { 1: person, 2: person }, + }), + { state: "answered", since: iso(6), request: null, startedBy: "person" }, + ); + }); + + it("names no starter it could not read, rather than guessing one", () => { + assert.strictEqual( + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: null, + runs: [run(1, "failed", 2)], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: [], + messages: new Map(), + }).startedBy, + null, + ); + }); +}); diff --git a/apps/server/src/lazurio/buddySignal.ts b/apps/server/src/lazurio/buddySignal.ts new file mode 100644 index 0000000000000..9573f0e5261c3 --- /dev/null +++ b/apps/server/src/lazurio/buddySignal.ts @@ -0,0 +1,289 @@ +/** + * Lazurio overlay (plan LZD-0033): `GET /api/lazurio/buddy/signal`, the one source of the sign by + * Buddy's avatar, versioned for every reader. Absent Buddy or thread is a 404, never `quiet`. + */ +import { + AuthOrchestrationReadScope, + type MessageId, + type OrchestrationV2ConversationMessage, + type OrchestrationV2Run, + type OrchestrationV2RuntimeRequest, + type RunId, + type ThreadId, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; +import { HttpRouter, HttpServerResponse } from "effect/http"; + +import * as ServerConfig from "../config.ts"; +import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; +import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { isBuddySurfaceEnvironment } from "./buddySurface.ts"; +import { BUDDY_THREAD_ID, findSharedBuddyThread, sharedBuddyFolder } from "./buddySurfaceThread.ts"; +import { authenticateOverlayRoute } from "./routeAuth.ts"; + +export const BUDDY_SIGNAL_ROUTE = "/api/lazurio/buddy/signal"; + +export const BuddySignalState = Schema.Literals(["waiting", "failed", "answered", "quiet"]); +export type BuddySignalState = typeof BuddySignalState.Type; +export const BuddySignalStarter = Schema.Literals(["person", "other"]); +export type BuddySignalStarter = typeof BuddySignalStarter.Type; + +export const BuddySignal = Schema.Struct({ + version: Schema.Literal(1), + state: BuddySignalState, + threadId: Schema.String, + since: Schema.NullOr(Schema.String), + request: Schema.NullOr(Schema.Struct({ kind: Schema.String })), + startedBy: Schema.NullOr(BuddySignalStarter), +}); +export type BuddySignal = typeof BuddySignal.Type; + +type SignalRun = Pick< + OrchestrationV2Run, + "id" | "ordinal" | "status" | "userMessageId" | "completedAt" +>; +type SignalRequest = Pick & { + /** The run whose turn asked, where its node names one. */ + readonly runId: RunId | null; +}; +export type BuddySignalStarterMessage = Pick< + OrchestrationV2ConversationMessage, + "createdBy" | "senderThreadId" | "scheduledTaskId" +>; + +export interface BuddySignalInput { + readonly threadId: ThreadId; + readonly lastVisitedAt: DateTime.Utc | null; + readonly runs: ReadonlyArray; + readonly pendingRequests: ReadonlyArray; + /** Runs someone asked to stop (a `run_interrupt_request` item); any other interrupt is a fault. */ + readonly stopRequested: ReadonlySet; + /** The last failed run of each scheduled task bound to the thread. */ + readonly failedTaskRuns: ReadonlyArray; + readonly messages: ReadonlyMap; +} + +const millis = (value: DateTime.Utc) => DateTime.toEpochMillis(value); +const SETTLED = new Set(["completed", "failed", "interrupted"]); + +/** The last run that settled the thread's line; a stopped or cancelled one is no fault. */ +const lastSettledRun = (runs: ReadonlyArray) => + runs + .filter((run) => SETTLED.has(run.status) && run.completedAt !== null) + .toSorted((left, right) => left.ordinal - right.ordinal) + .at(-1); + +const isFault = (run: SignalRun, stopRequested: ReadonlySet) => + run.status === "failed" || (run.status === "interrupted" && !stopRequested.has(run.id)); + +const unreadCompletedRuns = (input: Pick) => + input.runs.filter( + (run) => + run.status === "completed" && + run.completedAt !== null && + (input.lastVisitedAt === null || millis(run.completedAt) > millis(input.lastVisitedAt)), + ); + +/** The user messages whose senders the signal needs, so a reader loads only those. */ +export function buddySignalMessageIds( + input: Omit, +): Array { + const byId = new Map(input.runs.map((run) => [run.id, run])); + const runs = [ + ...input.pendingRequests.flatMap((request) => + request.runId === null ? [] : [byId.get(request.runId)], + ), + lastSettledRun(input.runs), + ...unreadCompletedRuns(input), + ]; + return [...new Set(runs.flatMap((run) => (run === undefined ? [] : [run.userMessageId])))]; +} + +function starterOf( + run: SignalRun | undefined, + messages: BuddySignalInput["messages"], +): BuddySignalStarter | null { + if (run === undefined) return null; + const message = messages.get(run.userMessageId); + if (message === undefined) return null; + return message.createdBy === "user" && + message.senderThreadId === undefined && + message.scheduledTaskId === undefined + ? "person" + : "other"; +} + +/** + * Waiting on a pending request, else failed (the last settled turn failed or was interrupted + * without a stop request, or a bound task failed after it), else answered (completed after + * `lastVisitedAt`), else quiet. `since` is the oldest cause; `startedBy` is the person's if any. + */ +export function buddySignalOf(input: BuddySignalInput): BuddySignal { + const base = { version: 1 as const, threadId: input.threadId }; + const byId = new Map(input.runs.map((run) => [run.id, run])); + const iso = (value: DateTime.Utc) => DateTime.formatIso(value); + + const pending = input.pendingRequests.toSorted( + (left, right) => millis(left.createdAt) - millis(right.createdAt), + )[0]; + if (pending !== undefined) + return { + ...base, + state: "waiting", + since: iso(pending.createdAt), + request: { kind: pending.kind }, + startedBy: starterOf( + pending.runId === null ? undefined : byId.get(pending.runId), + input.messages, + ), + }; + + const settled = lastSettledRun(input.runs); + const lastCompleted = + settled?.status === "completed" && settled.completedAt !== null + ? millis(settled.completedAt) + : Number.NEGATIVE_INFINITY; + const taskFailure = input.failedTaskRuns + .filter((at) => millis(at) > lastCompleted) + .toSorted((left, right) => millis(left) - millis(right))[0]; + if (settled !== undefined && isFault(settled, input.stopRequested)) + return { + ...base, + state: "failed", + since: iso(settled.completedAt as DateTime.Utc), + request: null, + startedBy: starterOf(settled, input.messages), + }; + if (taskFailure !== undefined) + return { ...base, state: "failed", since: iso(taskFailure), request: null, startedBy: "other" }; + + const unread = unreadCompletedRuns(input).toSorted( + (left, right) => millis(left.completedAt!) - millis(right.completedAt!), + ); + const first = unread[0]; + if (first !== undefined) + return { + ...base, + state: "answered", + since: iso(first.completedAt!), + request: null, + startedBy: unread.some((run) => starterOf(run, input.messages) === "person") + ? "person" + : "other", + }; + + return { ...base, state: "quiet", since: null, request: null, startedBy: null }; +} + +/** Buddy's thread on this server: GEN3's one thread, GEN4's thread of the shared server. */ +const buddyThreadId = Effect.fn("LazurioBuddySignal.threadId")(function* () { + if (isBuddySurfaceEnvironment(yield* HostProcess.Environment)) + return { kind: "found", threadId: BUDDY_THREAD_ID } as const; + const buddyFolder = yield* sharedBuddyFolder; + if (buddyFolder === undefined) return { kind: "absent", error: "not-buddy" } as const; + const found = yield* findSharedBuddyThread(buddyFolder).pipe(Effect.option); + const thread = Option.getOrUndefined(found)?.thread ?? null; + return thread === null + ? ({ kind: "absent", error: "no-thread" } as const) + : ({ kind: "found", threadId: thread.id } as const); +}); + +/** The signal as the route answers it, or why there is none. */ +export const readBuddySignal = Effect.fn("LazurioBuddySignal.read")(function* () { + const target = yield* buddyThreadId(); + if (target.kind === "absent") return target; + const orchestrator = yield* OrchestratorV2; + const records = yield* orchestrator + .getThreadRecords(target.threadId, ["runs", "runtimeRequests"], { + runtimeRequestStatuses: ["pending"], + }) + .pipe(Effect.option); + if (Option.isNone(records) || records.value.thread.deletedAt !== null) + return { kind: "absent", error: "no-thread" } as const; + const { thread, runs, runtimeRequests } = records.value; + const pending = runtimeRequests.filter((request) => request.status === "pending"); + const { nodes } = + pending.length === 0 + ? { nodes: [] } + : yield* orchestrator + .getThreadRecords(target.threadId, ["nodes"], { + nodeIds: pending.map((request) => request.nodeId), + }) + .pipe(Effect.orDie); + const runOfNode = new Map(nodes.map((node) => [node.id, node.runId])); + const interrupted = runs.filter((run) => run.status === "interrupted").map((run) => run.id); + const { turnItems } = + interrupted.length === 0 + ? { turnItems: [] } + : yield* orchestrator + .getThreadRecords(target.threadId, ["turnItems"], { + turnItemTypes: ["run_interrupt_request"], + turnItemRunIds: interrupted, + }) + .pipe(Effect.orDie); + const tasks = yield* Effect.serviceOption(ScheduledTaskService); + const failedTaskRuns = Option.isNone(tasks) + ? [] + : (yield* tasks.value.list().pipe(Effect.orDie)).tasks.flatMap((task) => + task.threadId === target.threadId && + task.lastRunStatus === "failed" && + task.lastRunAt !== null + ? [DateTime.makeUnsafe(task.lastRunAt)] + : [], + ); + const input = { + lastVisitedAt: thread.lastVisitedAt, + runs, + pendingRequests: pending.map((request) => ({ + kind: request.kind, + createdAt: request.createdAt, + runId: runOfNode.get(request.nodeId) ?? null, + })), + stopRequested: new Set(turnItems.flatMap((item) => (item.runId === null ? [] : [item.runId]))), + failedTaskRuns, + }; + const messageIds = buddySignalMessageIds(input); + const messages = + messageIds.length === 0 + ? new Map() + : yield* orchestrator.getMessagesById(target.threadId, messageIds).pipe(Effect.orDie); + return { + kind: "signal", + signal: buddySignalOf({ ...input, threadId: target.threadId, messages }), + } as const; +}); + +export const layer = HttpRouter.use((router) => + Effect.gen(function* () { + // A handler sees only the request's context, so it takes its services from the layer's. + const orchestrator = yield* OrchestratorV2; + const config = yield* ServerConfig.ServerConfig; + const projects = yield* ProjectStoreV2; + const tasks = yield* Effect.serviceOption(ScheduledTaskService); + yield* router.add( + "GET", + BUDDY_SIGNAL_ROUTE, + Effect.gen(function* () { + yield* authenticateOverlayRoute(AuthOrchestrationReadScope); + const answer = yield* readBuddySignal().pipe( + Effect.provideService(OrchestratorV2, orchestrator), + Effect.provideService(ServerConfig.ServerConfig, config), + Effect.provideService(ProjectStoreV2, projects), + Option.isSome(tasks) + ? Effect.provideService(ScheduledTaskService, tasks.value) + : (effect) => effect, + ); + return answer.kind === "absent" + ? HttpServerResponse.jsonUnsafe({ error: answer.error }, { status: 404 }) + : HttpServerResponse.jsonUnsafe(answer.signal, { + headers: { "cache-control": "no-store" }, + }); + }), + ); + }), +); diff --git a/apps/server/src/lazurio/buddySignalRead.test.ts b/apps/server/src/lazurio/buddySignalRead.test.ts new file mode 100644 index 0000000000000..9b12d3e2fc659 --- /dev/null +++ b/apps/server/src/lazurio/buddySignalRead.test.ts @@ -0,0 +1,253 @@ +// @effect-diagnostics nodeBuiltinImport:off - Buddy's folder is a temporary folder under a temporary home. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeHttpPlatform from "@effect/platform-node/NodeHttpPlatform"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { + AuthOrchestrationReadScope, + AuthTerminalReadScope, + ProjectId, + ProviderInstanceId, + ThreadId, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; +import { HttpRouter } from "effect/http"; + +import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; +import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; +import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; + +import * as ServerConfig from "../config.ts"; +import * as ServerSettings from "../serverSettings.ts"; +import { type AgentCliConnection, agentCliFailure } from "./agentCliConnection.ts"; +import * as BuddySignal from "./buddySignal.ts"; +import { addProject, layerBuddyServer } from "./buddySurface.testkit.ts"; +import { followBuddySignal } from "./buddySurfaceCli.ts"; +import { layerManagedConfig, makeFolderDirectories } from "./managedFolder.testkit.ts"; +import { resolveBuddyThread } from "./buddySurfaceThread.ts"; + +const makeShared = () => { + const home = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-signal-home-")); + const folder = NodePath.join(home, "Lazurio"); + const buddy = NodePath.join(home, "Buddy"); + NodeFS.mkdirSync(folder); + NodeFS.mkdirSync(buddy); + return { home, folder, buddy }; +}; +const withShared = (effect: Effect.Effect, folder: string, buddy: string) => + Effect.flatMap(ServerConfig.ServerConfig, (config) => + effect.pipe( + Effect.provideService(ServerConfig.ServerConfig, { + ...config, + lazurioFolder: folder, + lazurioBuddyFolder: buddy, + }), + ), + ); + +describe("reading the sign by Buddy's avatar", () => { + it.live("says why there is none, and is quiet only for a thread that exists", () => { + const { home, folder, buddy } = makeShared(); + return Effect.gen(function* () { + // No Buddy on this server, then his project without his thread: never quiet. + assert.deepStrictEqual(yield* BuddySignal.readBuddySignal(), { + kind: "absent", + error: "not-buddy", + }); + yield* addProject(ProjectId.make("project:buddy"), buddy); + yield* (yield* ServerSettings.ServerSettingsService).updateSettings({ + projectSettingsOverrides: { + [ProjectId.make("project:buddy")]: { + defaultModelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "luna" }, + }, + }, + }); + assert.deepStrictEqual(yield* withShared(BuddySignal.readBuddySignal(), folder, buddy), { + kind: "absent", + error: "no-thread", + }); + const { threadId } = yield* withShared(resolveBuddyThread(), folder, buddy); + assert.deepStrictEqual(yield* withShared(BuddySignal.readBuddySignal(), folder, buddy), { + kind: "signal", + signal: { + version: 1, + state: "quiet", + threadId, + since: null, + request: null, + startedBy: null, + }, + }); + }).pipe( + Effect.provide(Layer.merge(layerBuddyServer, ServerSettings.layerTest())), + Effect.scoped, + Effect.provideService(HostProcess.Environment, { HOME: home }), + ); + }); + + it.live("on Buddy's own server, before his one thread exists, is no-thread", () => { + const { home } = makeShared(); + return Effect.gen(function* () { + assert.deepStrictEqual(yield* BuddySignal.readBuddySignal(), { + kind: "absent", + error: "no-thread", + }); + }).pipe( + Effect.provide(Layer.merge(layerBuddyServer, ServerSettings.layerTest())), + Effect.scoped, + Effect.provideService(HostProcess.Environment, { HOME: home, T3CODE_LAZURIO_BUDDY: "1" }), + ); + }); +}); + +describe("following the sign", () => { + const signal = (state: string, startedBy: string | null) => ({ + version: 1, + state, + threadId: "buddy-thread", + since: state === "quiet" ? null : "2026-11-18T20:02:00.000Z", + request: null, + startedBy, + }); + /** A connection whose route answers `answers` in turn and whose shell sends `updates`. */ + const connection = ( + answers: Array<{ status: number; body: unknown }>, + updates: number, + ): AgentCliConnection => + ({ + getRoute: () => Effect.sync(() => answers.shift() ?? { status: 500, body: undefined }), + subscribeShell: () => + Stream.fromIterable( + Array.from({ length: updates }, () => ({ + kind: "thread.updated" as const, + thread: { id: ThreadId.make("buddy-thread") }, + })), + ), + }) as unknown as AgentCliConnection; + + it.effect( + "prints each change, startedBy too, and unavailable when it cannot read, never silence", + () => + Effect.gen(function* () { + const lines: Array = []; + const connects = [ + // Starts quiet, an unchanged reading prints nothing, then others' and the person's answers. + connection( + [ + { status: 200, body: signal("quiet", null) }, + { status: 200, body: signal("quiet", null) }, + { status: 200, body: signal("answered", "other") }, + { status: 200, body: signal("answered", "person") }, + ], + 3, + ), + // The session expired. + connection([{ status: 401, body: undefined }], 0), + // Back, before his first visit. + connection([{ status: 404, body: { error: "no-thread" } }], 0), + ]; + const fiber = yield* followBuddySignal( + (use) => { + const next = connects.shift(); + return next === undefined + ? Effect.fail(agentCliFailure("server_not_running", "T3 Code is not running.")) + : use(next); + }, + (line) => Effect.sync(() => lines.push(JSON.parse(line))), + ).pipe(Effect.forkChild); + // Each reconnect waits at most half a minute. + for (let step = 0; step < 6; step += 1) yield* TestClock.adjust("30 seconds"); + const states = lines.map((line) => { + const { state, startedBy, reason } = line as Record; + return state === "unavailable" ? `unavailable:${reason}` : `${state}:${startedBy}`; + }); + assert.deepStrictEqual(states.slice(0, 7), [ + "quiet:null", + "answered:other", + "answered:person", + "unavailable:connection_lost", + "unavailable:unauthorized", + "unavailable:no_thread", + "unavailable:connection_lost", + ]); + assert.include(states.slice(7), "unavailable:server_not_running"); + yield* Fiber.interrupt(fiber); + }), + ); + + it.effect("ends with a failure on a server without Buddy", () => + Effect.gen(function* () { + const lines: Array = []; + const exit = yield* followBuddySignal( + (use) => use(connection([{ status: 404, body: { error: "not-buddy" } }], 0)), + (line) => Effect.sync(() => lines.push(line)), + ).pipe(Effect.exit); + assert.strictEqual(exit._tag, "Failure"); + assert.deepStrictEqual( + lines.map((line) => JSON.parse(line).reason), + ["not_buddy"], + ); + }), + ); +}); + +const { folder: authFolder } = makeFolderDirectories(); +const layerAuth = EnvironmentAuth.layer.pipe( + Layer.provide(ManagedPolicy.layerTest()), + Layer.provideMerge(SqlitePersistence.layerMemory), + Layer.provide(ServerSecretStore.layer), + Layer.provide(ServerEnvironment.layerIdentity), + Layer.provideMerge(layerManagedConfig(authFolder)), +); + +it.layer(layerAuth.pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))), { + excludeTestServices: true, +})("the sign's route", (it) => { + it.effect("needs a session that may read threads, and names a server without Buddy", () => + Effect.gen(function* () { + const auth = yield* EnvironmentAuth.EnvironmentAuth; + const config = yield* ServerConfig.ServerConfig; + // Refused before it reads anything, and without Buddy it reads no thread. + const handler = HttpRouter.toWebHandler( + BuddySignal.layer.pipe( + Layer.provideMerge(Layer.succeed(EnvironmentAuth.EnvironmentAuth, auth)), + Layer.provideMerge(Layer.mock(OrchestratorV2)({})), + Layer.provideMerge(Layer.mock(ProjectStoreV2)({})), + Layer.provideMerge(ServerConfig.layer(config)), + Layer.provideMerge(NodeHttpPlatform.layer), + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)), + ), + { disableLogger: true }, + ); + yield* Effect.addFinalizer(() => Effect.promise(() => handler.dispose())); + const reader = yield* auth.issueSession({ scopes: [AuthOrchestrationReadScope], label: "t" }); + const terminal = yield* auth.issueSession({ scopes: [AuthTerminalReadScope], label: "t" }); + const get = (token?: string) => + handler.handler( + new Request(`http://localhost${BuddySignal.BUDDY_SIGNAL_ROUTE}`, { + headers: token ? { authorization: `Bearer ${token}` } : {}, + }), + ); + yield* Effect.promise(async () => { + assert.strictEqual((await get()).status, 401); + assert.strictEqual((await get(terminal.token)).status, 403); + const answer = await get(reader.token); + assert.strictEqual(answer.status, 404); + assert.deepStrictEqual(await answer.json(), { error: "not-buddy" }); + }); + }), + ); +}); diff --git a/apps/server/src/lazurio/buddySurfaceCli.ts b/apps/server/src/lazurio/buddySurfaceCli.ts index 23cccb5ad5a54..ef1ddf8e2e828 100644 --- a/apps/server/src/lazurio/buddySurfaceCli.ts +++ b/apps/server/src/lazurio/buddySurfaceCli.ts @@ -1,16 +1,26 @@ /** - * Lazurio overlay (root decision 0210, plan DEV-6676): `t3 lazurio buddy thread`, which names the - * thread to post a wake into: Buddy's one thread (buddySurfaceThread.ts). It runs as an agent - * command (agentCli.ts): `--json` prints `{"threadId", "projectId"}` or `{"error": {"code", - * "message"}}`, and the exit code is 0, 1 for a refusal, 3 when T3 Code is not running. + * Lazurio overlay (root decision 0210): `t3 lazurio buddy thread` names Buddy's one thread + * (buddySurfaceThread.ts) and `t3 lazurio buddy signal` the sign by his avatar (buddySignal.ts). + * `--json` prints the answer or `{"error": {"code", "message"}}`; exit 1 refused, 3 not running. */ +import * as Cause from "effect/Cause"; +import * as Console from "effect/Console"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import { Command } from "effect/cli"; +import * as Stream from "effect/Stream"; +import { Command, Flag } from "effect/cli"; import { agentCommandFlags, runAgentCommand } from "./agentCli.ts"; -import { agentCliFailure } from "./agentCliConnection.ts"; +import { + type AgentCliConnection, + type AgentCliFailure, + agentCliFailure, + isAgentCliFailure, + withAgentCliConnection, +} from "./agentCliConnection.ts"; +import { BUDDY_SIGNAL_ROUTE, BuddySignal } from "./buddySignal.ts"; import { BUDDY_THREAD_ROUTE, BuddyThread } from "./buddySurfaceThread.ts"; const decodeBuddyThread = Schema.decodeUnknownOption(BuddyThread); @@ -44,7 +54,143 @@ const threadCommand = Command.make("thread", agentCommandFlags).pipe( ), ); +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeBuddySignal = Schema.decodeUnknownOption(BuddySignal); + +/** Buddy's signal now; a refusal fails with its route code (`not_buddy`, `no_thread`). */ +const readBuddySignal = (connection: Pick) => + connection.getRoute(BUDDY_SIGNAL_ROUTE).pipe( + Effect.flatMap(({ status, body }) => { + const signal = decodeBuddySignal(body); + if (status === 200 && Option.isSome(signal)) return Effect.succeed(signal.value); + if (status === 401 || status === 403) + return Effect.fail( + agentCliFailure("unauthorized", `T3 Code refused the session (HTTP ${status}).`), + ); + const refusal = Option.getOrUndefined(decodeRefusal(body)); + return Effect.fail( + agentCliFailure( + refusal?.error.replaceAll("-", "_") ?? "request_failed", + refusal?.message ?? `T3 Code answered HTTP ${status}.`, + ), + ); + }), + ); + +/** The line `--follow` prints while it cannot say Buddy's state: never a quiet one. */ +const unavailableLine = (failure: AgentCliFailure) => ({ + version: 1 as const, + state: "unavailable" as const, + reason: failure.code, + message: failure.message, +}); + +const FOLLOW_SESSION_TTL = Duration.hours(1); +// Scheduled tasks change no thread, so their failures are read on this beat. +const FOLLOW_REREAD = Duration.seconds(60); +const FOLLOW_RECONNECT = [1, 2, 5, 10, 30].map((seconds) => Duration.seconds(seconds)); + +/** + * Print Buddy's signal and then a line whenever any of it changes, `startedBy` and `since` included. + * While it cannot read it (T3 down, the session refused, the stream dropped) it prints + * `unavailable` with the reason and reconnects; only a server without Buddy ends it. + */ +export const followBuddySignal = Effect.fn("LazurioBuddySignal.follow")(function* ( + connect: ( + use: (connection: AgentCliConnection) => Effect.Effect, + ) => Effect.Effect, + print: (line: string) => Effect.Effect, +) { + let last: string | undefined; + let threadId: string | undefined; + const emit = (line: unknown) => { + const text = encodeJson(line); + if (text === last) return Effect.void; + last = text; + return print(text); + }; + let attempt = 0; + while (true) { + const exit = yield* Effect.exit( + connect((connection) => { + const read = readBuddySignal(connection).pipe( + Effect.flatMap((signal) => { + threadId = signal.threadId; + attempt = 0; + return emit(signal); + }), + // Before his first visit his thread does not exist yet: say so and keep following. + Effect.catchIf( + (failure) => failure.code === "no_thread", + (failure) => { + threadId = undefined; + return emit(unavailableLine(failure)); + }, + ), + ); + return read.pipe( + Effect.andThen( + Stream.merge( + connection + .subscribeShell({}) + .pipe( + Stream.filter( + (item) => + item.kind === "thread.updated" && + (threadId === undefined || item.thread.id === threadId), + ), + ), + // The beat after the first reading; the shell's end ends the connection. + Stream.tick(FOLLOW_REREAD).pipe(Stream.drop(1)), + { haltStrategy: "left" }, + ).pipe(Stream.runForEach(() => read)), + ), + Effect.andThen( + Effect.fail(agentCliFailure("connection_lost", "T3 Code ended the stream.")), + ), + ); + }), + ); + if (exit._tag === "Success") return; + const failure = Option.getOrUndefined(Cause.findErrorOption(exit.cause)); + const reported = isAgentCliFailure(failure) + ? failure + : agentCliFailure("request_failed", "Reading Buddy's signal failed."); + yield* emit(unavailableLine(reported)); + if (reported.code === "not_buddy") return yield* reported; + yield* Effect.sleep(FOLLOW_RECONNECT[Math.min(attempt, FOLLOW_RECONNECT.length - 1)]!); + attempt += 1; + } +}); + +const signalCommand = Command.make("signal", { + ...agentCommandFlags, + follow: Flag.Boolean("follow").pipe( + Flag.withDescription( + "Keep printing a JSON line whenever the signal changes, and `unavailable` while it cannot be read.", + ), + Flag.withDefault(false), + ), +}).pipe( + Command.withDescription( + "Print the sign by Buddy's avatar: whether he waits for the person, failed, or has answered.", + ), + Command.withHandler((flags) => + flags.follow + ? followBuddySignal( + (use) => withAgentCliConnection(flags, use, { sessionTtl: FOLLOW_SESSION_TTL }), + Console.log, + ) + : runAgentCommand( + flags, + readBuddySignal, + (signal) => + `Buddy is ${signal.state}${signal.since === null ? "" : ` since ${signal.since}`}.`, + ), + ), +); + export const buddyCommand = Command.make("buddy").pipe( Command.withDescription("Buddy's T3 Code."), - Command.withSubcommands([threadCommand]), + Command.withSubcommands([threadCommand, signalCommand]), ); diff --git a/apps/server/src/lazurio/overlay.ts b/apps/server/src/lazurio/overlay.ts index ca632c8b142d5..4179fcbc79b74 100644 --- a/apps/server/src/lazurio/overlay.ts +++ b/apps/server/src/lazurio/overlay.ts @@ -8,6 +8,7 @@ import * as Layer from "effect/Layer"; import * as ActingSenders from "./actingSenders.ts"; import * as ActingSendersRoute from "./actingSendersRoute.ts"; import * as AgentCliSendState from "./agentCliSendState.ts"; +import * as BuddySignal from "./buddySignal.ts"; import * as BuddySurfaceDays from "./buddySurfaceDays.ts"; import * as BuddySurfaceThreadRoute from "./buddySurfaceThreadRoute.ts"; import * as EnvironmentBrowser from "./environmentBrowser.ts"; @@ -27,6 +28,7 @@ export const layer = Layer.mergeAll( ModelRecommendationRoute.layer, ProviderRefreshRoute.layer.pipe(Layer.provide(ProviderRefresh.layer)), BuddySurfaceThreadRoute.layer, + BuddySignal.layer, AgentCliSendState.layer, Layer.mergeAll(BuddySurfaceDays.layer, ActingSendersRoute.layer).pipe( Layer.provide(ActingSenders.layer), diff --git a/scripts/lazurio-release-contract.test.mjs b/scripts/lazurio-release-contract.test.mjs index e5c5da049e82b..51c2ba81dd88c 100644 --- a/scripts/lazurio-release-contract.test.mjs +++ b/scripts/lazurio-release-contract.test.mjs @@ -2222,7 +2222,7 @@ NodeTest.test("a thread's tabs live in its holder file behind an overlay route", ); NodeAssert.match( overlay, - /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, + /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*BuddySignal\.layer,\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, ); NodeAssert.match( overlay, From dce0d460ce3afa35b185d143c7056f4ba9c05188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:08:35 +0000 Subject: [PATCH 2/6] fix(lazurio): the sign dates a task's failure by its completion and keeps it cleared Two timing faults in Buddy's thread signal: - a task's failure was compared with the last settled run only, so a turn stopped on request after the clearing completed turn made the old failure come back as `failed`; it is now compared with the latest completed turn, whatever settled after it; - a task's failure was dated by `lastRunAt`, which ScheduledTaskService records at the start; a task that started before a completed turn and failed after it was suppressed. It is now dated by the completion the service records as `updatedAt` (failedTaskRunsOf). Co-Authored-By: Claude Opus 5.5 --- apps/server/src/lazurio/buddySignal.test.ts | 47 +++++++++++++++++++++ apps/server/src/lazurio/buddySignal.ts | 35 ++++++++++----- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/apps/server/src/lazurio/buddySignal.test.ts b/apps/server/src/lazurio/buddySignal.test.ts index f8f8b505340fd..b71a51f4ffd41 100644 --- a/apps/server/src/lazurio/buddySignal.test.ts +++ b/apps/server/src/lazurio/buddySignal.test.ts @@ -7,6 +7,7 @@ import { type BuddySignalStarterMessage, buddySignalMessageIds, buddySignalOf, + failedTaskRunsOf, } from "./buddySignal.ts"; const THREAD = ThreadId.make("buddy-thread"); @@ -165,6 +166,52 @@ describe("the sign by Buddy's avatar", () => { ); }); + it("keeps a task's failure cleared by a completed turn when a later turn was stopped on request", () => { + // The task fails at :05, a turn completes at :06, another is stopped on request at :07. + assert.strictEqual( + signal({ + runs: [run(1, "completed", 6), run(2, "interrupted", 7)], + stopRequested: new Set([RunId.make("run-2")]), + lastVisitedAt: at(8), + failedTaskRuns: [at(5)], + senders: { 1: person, 2: person }, + }).state, + "quiet", + ); + }); + + it("dates a task's failure by its completion, not its start", () => { + const failed = (startedAt: number, completedAt: number) => ({ + threadId: THREAD, + lastRunStatus: "failed" as const, + lastRunAt: DateTime.formatIso(at(startedAt)), + updatedAt: DateTime.formatIso(at(completedAt)), + }); + // Started at :04, before a turn completed at :05, and failed at :07: a new failure. + const failedTaskRuns = failedTaskRunsOf([failed(4, 7)], THREAD); + assert.deepStrictEqual(failedTaskRuns, [at(7)]); + assert.deepStrictEqual( + signal({ + runs: [run(1, "completed", 5)], + lastVisitedAt: at(6), + failedTaskRuns, + senders: { 1: person }, + }), + { state: "failed", since: iso(7), request: null, startedBy: "other" }, + ); + // Another thread's task, and one that succeeded, are not this thread's failure. + assert.deepStrictEqual( + failedTaskRunsOf( + [ + { ...failed(4, 7), threadId: ThreadId.make("other") }, + { ...failed(4, 7), lastRunStatus: "succeeded" as const }, + ], + THREAD, + ), + [], + ); + }); + it("answers from the oldest unread answer, the person's if they started any of them", () => { // The person writes, Buddy answers, then three turns of another thread follow. const runs = [ diff --git a/apps/server/src/lazurio/buddySignal.ts b/apps/server/src/lazurio/buddySignal.ts index 9573f0e5261c3..343888547114b 100644 --- a/apps/server/src/lazurio/buddySignal.ts +++ b/apps/server/src/lazurio/buddySignal.ts @@ -9,6 +9,7 @@ import { type OrchestrationV2Run, type OrchestrationV2RuntimeRequest, type RunId, + type ScheduledTask, type ThreadId, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -118,6 +119,21 @@ function starterOf( : "other"; } +/** + * When each scheduled task bound to the thread last failed: the run's completion, which + * `ScheduledTaskService` records as `updatedAt` (`lastRunAt` is when it started). A later edit + * of the task moves `updatedAt` too. + */ +export const failedTaskRunsOf = ( + tasks: ReadonlyArray>, + threadId: ThreadId, +): ReadonlyArray => + tasks.flatMap((task) => + task.threadId === threadId && task.lastRunStatus === "failed" + ? [DateTime.makeUnsafe(task.updatedAt)] + : [], + ); + /** * Waiting on a pending request, else failed (the last settled turn failed or was interrupted * without a stop request, or a bound task failed after it), else answered (completed after @@ -144,10 +160,13 @@ export function buddySignalOf(input: BuddySignalInput): BuddySignal { }; const settled = lastSettledRun(input.runs); - const lastCompleted = - settled?.status === "completed" && settled.completedAt !== null - ? millis(settled.completedAt) - : Number.NEGATIVE_INFINITY; + // The latest completed turn clears a task's failure, also when a later turn was stopped. + const lastCompleted = Math.max( + Number.NEGATIVE_INFINITY, + ...input.runs.flatMap((run) => + run.status === "completed" && run.completedAt !== null ? [millis(run.completedAt)] : [], + ), + ); const taskFailure = input.failedTaskRuns .filter((at) => millis(at) > lastCompleted) .toSorted((left, right) => millis(left) - millis(right))[0]; @@ -229,13 +248,7 @@ export const readBuddySignal = Effect.fn("LazurioBuddySignal.read")(function* () const tasks = yield* Effect.serviceOption(ScheduledTaskService); const failedTaskRuns = Option.isNone(tasks) ? [] - : (yield* tasks.value.list().pipe(Effect.orDie)).tasks.flatMap((task) => - task.threadId === target.threadId && - task.lastRunStatus === "failed" && - task.lastRunAt !== null - ? [DateTime.makeUnsafe(task.lastRunAt)] - : [], - ); + : failedTaskRunsOf((yield* tasks.value.list().pipe(Effect.orDie)).tasks, target.threadId); const input = { lastVisitedAt: thread.lastVisitedAt, runs, From c53ee58cbf690d3175568d559bd0388a440c5c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:20:58 +0000 Subject: [PATCH 3/6] fix(lazurio): the sign dates a task's failure by when it was recorded, not by edits A task's `updatedAt` moves with every save or toggle, so a failure dated by it came back after an edit. The task keeps no completion time of its own, so the overlay records one (buddyTaskFailures): a Buddy server reads the task list at its start and on each change, and a failed run is new when the task's `runCount` changed, which only a finished run does. The time is kept in the server's state (stateDir/lazurio/buddy-task-failures.json), so a restart keeps it; a failure from before the record is dated by its run's start on the first reading. The signal reads failures from there. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/lazurio/buddySignal.test.ts | 33 ---- apps/server/src/lazurio/buddySignal.ts | 30 +--- .../src/lazurio/buddyTaskFailures.test.ts | 152 ++++++++++++++++++ apps/server/src/lazurio/buddyTaskFailures.ts | 139 ++++++++++++++++ apps/server/src/lazurio/overlay.ts | 3 +- scripts/lazurio-release-contract.test.mjs | 2 +- 6 files changed, 301 insertions(+), 58 deletions(-) create mode 100644 apps/server/src/lazurio/buddyTaskFailures.test.ts create mode 100644 apps/server/src/lazurio/buddyTaskFailures.ts diff --git a/apps/server/src/lazurio/buddySignal.test.ts b/apps/server/src/lazurio/buddySignal.test.ts index b71a51f4ffd41..8dfd87eb00221 100644 --- a/apps/server/src/lazurio/buddySignal.test.ts +++ b/apps/server/src/lazurio/buddySignal.test.ts @@ -7,7 +7,6 @@ import { type BuddySignalStarterMessage, buddySignalMessageIds, buddySignalOf, - failedTaskRunsOf, } from "./buddySignal.ts"; const THREAD = ThreadId.make("buddy-thread"); @@ -180,38 +179,6 @@ describe("the sign by Buddy's avatar", () => { ); }); - it("dates a task's failure by its completion, not its start", () => { - const failed = (startedAt: number, completedAt: number) => ({ - threadId: THREAD, - lastRunStatus: "failed" as const, - lastRunAt: DateTime.formatIso(at(startedAt)), - updatedAt: DateTime.formatIso(at(completedAt)), - }); - // Started at :04, before a turn completed at :05, and failed at :07: a new failure. - const failedTaskRuns = failedTaskRunsOf([failed(4, 7)], THREAD); - assert.deepStrictEqual(failedTaskRuns, [at(7)]); - assert.deepStrictEqual( - signal({ - runs: [run(1, "completed", 5)], - lastVisitedAt: at(6), - failedTaskRuns, - senders: { 1: person }, - }), - { state: "failed", since: iso(7), request: null, startedBy: "other" }, - ); - // Another thread's task, and one that succeeded, are not this thread's failure. - assert.deepStrictEqual( - failedTaskRunsOf( - [ - { ...failed(4, 7), threadId: ThreadId.make("other") }, - { ...failed(4, 7), lastRunStatus: "succeeded" as const }, - ], - THREAD, - ), - [], - ); - }); - it("answers from the oldest unread answer, the person's if they started any of them", () => { // The person writes, Buddy answers, then three turns of another thread follow. const runs = [ diff --git a/apps/server/src/lazurio/buddySignal.ts b/apps/server/src/lazurio/buddySignal.ts index 343888547114b..92038de97f1f6 100644 --- a/apps/server/src/lazurio/buddySignal.ts +++ b/apps/server/src/lazurio/buddySignal.ts @@ -9,7 +9,6 @@ import { type OrchestrationV2Run, type OrchestrationV2RuntimeRequest, type RunId, - type ScheduledTask, type ThreadId, } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; @@ -22,8 +21,8 @@ import { HttpRouter, HttpServerResponse } from "effect/http"; import * as ServerConfig from "../config.ts"; import { OrchestratorV2 } from "../orchestration-v2/Orchestrator.ts"; import { ProjectStoreV2 } from "../orchestration-v2/ProjectStore.ts"; -import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; import { isBuddySurfaceEnvironment } from "./buddySurface.ts"; +import { BuddyTaskFailures } from "./buddyTaskFailures.ts"; import { BUDDY_THREAD_ID, findSharedBuddyThread, sharedBuddyFolder } from "./buddySurfaceThread.ts"; import { authenticateOverlayRoute } from "./routeAuth.ts"; @@ -119,21 +118,6 @@ function starterOf( : "other"; } -/** - * When each scheduled task bound to the thread last failed: the run's completion, which - * `ScheduledTaskService` records as `updatedAt` (`lastRunAt` is when it started). A later edit - * of the task moves `updatedAt` too. - */ -export const failedTaskRunsOf = ( - tasks: ReadonlyArray>, - threadId: ThreadId, -): ReadonlyArray => - tasks.flatMap((task) => - task.threadId === threadId && task.lastRunStatus === "failed" - ? [DateTime.makeUnsafe(task.updatedAt)] - : [], - ); - /** * Waiting on a pending request, else failed (the last settled turn failed or was interrupted * without a stop request, or a bound task failed after it), else answered (completed after @@ -245,10 +229,10 @@ export const readBuddySignal = Effect.fn("LazurioBuddySignal.read")(function* () turnItemRunIds: interrupted, }) .pipe(Effect.orDie); - const tasks = yield* Effect.serviceOption(ScheduledTaskService); - const failedTaskRuns = Option.isNone(tasks) + const failures = yield* Effect.serviceOption(BuddyTaskFailures); + const failedTaskRuns = Option.isNone(failures) ? [] - : failedTaskRunsOf((yield* tasks.value.list().pipe(Effect.orDie)).tasks, target.threadId); + : yield* failures.value.failedAt(target.threadId); const input = { lastVisitedAt: thread.lastVisitedAt, runs, @@ -277,7 +261,7 @@ export const layer = HttpRouter.use((router) => const orchestrator = yield* OrchestratorV2; const config = yield* ServerConfig.ServerConfig; const projects = yield* ProjectStoreV2; - const tasks = yield* Effect.serviceOption(ScheduledTaskService); + const failures = yield* Effect.serviceOption(BuddyTaskFailures); yield* router.add( "GET", BUDDY_SIGNAL_ROUTE, @@ -287,8 +271,8 @@ export const layer = HttpRouter.use((router) => Effect.provideService(OrchestratorV2, orchestrator), Effect.provideService(ServerConfig.ServerConfig, config), Effect.provideService(ProjectStoreV2, projects), - Option.isSome(tasks) - ? Effect.provideService(ScheduledTaskService, tasks.value) + Option.isSome(failures) + ? Effect.provideService(BuddyTaskFailures, failures.value) : (effect) => effect, ); return answer.kind === "absent" diff --git a/apps/server/src/lazurio/buddyTaskFailures.test.ts b/apps/server/src/lazurio/buddyTaskFailures.test.ts new file mode 100644 index 0000000000000..ae69431cc712f --- /dev/null +++ b/apps/server/src/lazurio/buddyTaskFailures.test.ts @@ -0,0 +1,152 @@ +// @effect-diagnostics nodeBuiltinImport:off - the server's state is a temporary folder. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, describe, it } from "@effect/vitest"; +import { ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; + +import * as ServerConfig from "../config.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { + BuddyTaskFailures, + failuresOf, + layer as layerBuddyTaskFailures, + recordTaskFailures, +} from "./buddyTaskFailures.ts"; + +const THREAD = ThreadId.make("buddy"); +const at = (minute: number) => `2026-11-18T20:${String(minute).padStart(2, "0")}:00.000Z`; +const task = ( + runCount: number, + lastRunStatus: "failed" | "succeeded" | "running" | "never", + lastRunAt = at(4), +) => ({ + id: ScheduledTaskId.make("task-1"), + threadId: THREAD, + lastRunStatus, + lastRunAt, + runCount, +}); + +describe("when a task bound to Buddy's thread last failed", () => { + it("is when the list first reported the failed run, not the run's start, and an edit keeps it", () => { + // Started at :04, reported failed at :07. + const failed = recordTaskFailures({}, [task(1, "failed")], at(7)); + assert.deepStrictEqual(failuresOf(failed, THREAD), [DateTime.makeUnsafe(at(7))]); + // Saving or disabling the task at :09 reports the list again with the same run. + const edited = recordTaskFailures(failed, [task(1, "failed")], at(9)); + assert.deepStrictEqual(failuresOf(edited, THREAD), [DateTime.makeUnsafe(at(7))]); + }); + + it("dates a later failed run anew, and forgets a task once it no longer failed", () => { + const first = recordTaskFailures({}, [task(1, "failed")], at(5)); + const again = recordTaskFailures(first, [task(2, "failed", at(10))], at(12)); + assert.deepStrictEqual(failuresOf(again, THREAD), [DateTime.makeUnsafe(at(12))]); + assert.deepStrictEqual(recordTaskFailures(again, [task(3, "running")], at(13)), {}); + assert.deepStrictEqual(recordTaskFailures(again, [task(3, "succeeded")], at(14)), {}); + assert.deepStrictEqual(recordTaskFailures(again, [], at(15)), {}); + }); + + it("dates a failure from before the record by the run's start on the first reading after a start", () => { + const seen = recordTaskFailures({}, [task(1, "failed", at(3))], at(30), true); + assert.deepStrictEqual(failuresOf(seen, THREAD), [DateTime.makeUnsafe(at(3))]); + // A failure the file already holds keeps its time across the restart. + const kept = recordTaskFailures( + { "task-1": { threadId: THREAD, runCount: 1, failedAt: at(7) } }, + [task(1, "failed", at(3))], + at(30), + true, + ); + assert.deepStrictEqual(failuresOf(kept, THREAD), [DateTime.makeUnsafe(at(7))]); + }); + + it("belongs only to the thread the task is bound to", () => { + const other = recordTaskFailures( + {}, + [{ ...task(1, "failed"), threadId: ThreadId.make("other") }], + at(7), + ); + assert.deepStrictEqual(failuresOf(other, THREAD), []); + }); +}); + +const unused = () => Effect.die("unused in this test"); +const tasksListing = (tasks: ReadonlyArray>) => + Layer.succeed( + ScheduledTaskService, + ScheduledTaskService.of({ + list: () => Effect.succeed({ tasks } as never), + subscribeList: () => Stream.empty, + upsert: unused, + setEnabled: unused, + delete: unused, + deleteForThread: unused, + moveThreadTasks: unused, + runNow: unused, + rotateWebhookToken: unused, + listWebhookDeliveries: unused, + getWebhookDelivery: unused, + triggerWebhook: unused, + }), + ); + +describe("the record of a Buddy server's failed tasks", () => { + it.effect("is kept in the server's state, and a restart reads it back", () => { + const base = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-task-failures-")); + const failedAt = (tasks: ReadonlyArray>) => + Effect.flatMap(BuddyTaskFailures, (failures) => failures.failedAt(THREAD)).pipe( + Effect.provide( + layerBuddyTaskFailures.pipe( + Layer.provide(tasksListing(tasks)), + Layer.provide(ServerConfig.layerTest(base, base)), + ), + ), + Effect.provideService(HostProcess.Environment, { T3CODE_LAZURIO_BUDDY: "1" }), + Effect.provide(NodeServices.layer), + Effect.scoped, + ); + return Effect.gen(function* () { + // The start finds a failure the record does not know: dated by its run's start. + assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(3))]), [ + DateTime.makeUnsafe(at(3)), + ]); + const file = NodeFS.readdirSync(base, { recursive: true }).find((entry) => + String(entry).endsWith("buddy-task-failures.json"), + ); + assert.isDefined(file); + // A restart keeps the recorded time, whatever the task's start says now. + assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(1))]), [ + DateTime.makeUnsafe(at(3)), + ]); + }).pipe( + Effect.ensuring(Effect.sync(() => NodeFS.rmSync(base, { recursive: true, force: true }))), + ); + }); + + it.effect("watches nothing on a server without Buddy", () => + Effect.gen(function* () { + const base = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "buddy-task-failures-")); + const failures = yield* Effect.flatMap(BuddyTaskFailures, (service) => + service.failedAt(THREAD), + ).pipe( + Effect.provide( + layerBuddyTaskFailures.pipe( + Layer.provide(tasksListing([task(1, "failed")])), + Layer.provide(ServerConfig.layerTest(base, base)), + ), + ), + Effect.provideService(HostProcess.Environment, {}), + Effect.provide(NodeServices.layer), + Effect.scoped, + ); + assert.deepStrictEqual(failures, []); + NodeFS.rmSync(base, { recursive: true, force: true }); + }), + ); +}); diff --git a/apps/server/src/lazurio/buddyTaskFailures.ts b/apps/server/src/lazurio/buddyTaskFailures.ts new file mode 100644 index 0000000000000..b17dea5db0d75 --- /dev/null +++ b/apps/server/src/lazurio/buddyTaskFailures.ts @@ -0,0 +1,139 @@ +/** + * Lazurio overlay (plan LZD-0033): when each task last failed, for the sign by Buddy's avatar. + * A task keeps only its run's start and an `updatedAt` that edits move, so a failed run is + * recorded when the list reports a new `runCount`, which only a finished run changes. + */ + +import type { ScheduledTask, ThreadId } from "@t3tools/contracts"; +import { writeFileStringAtomically } from "@t3tools/shared/atomicWrite"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Ref from "effect/Ref"; +import * as Schema from "effect/Schema"; +import * as Stream from "effect/Stream"; + +import * as HostProcess from "@t3tools/shared/HostProcess"; + +import * as ServerConfig from "../config.ts"; +import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { isBuddySurfaceEnvironment } from "./buddySurface.ts"; +import { sharedBuddyFolder } from "./buddySurfaceThread.ts"; + +const TaskFailure = Schema.Struct({ + threadId: Schema.NullOr(Schema.String), + runCount: Schema.Number, + failedAt: Schema.String, +}); +const TaskFailuresFile = Schema.Record(Schema.String, TaskFailure); +export type TaskFailures = typeof TaskFailuresFile.Type; + +type TaskRow = Pick; + +/** + * The failures after one reading of the task list at `now`: a failed run not seen before is dated + * `now`, one already recorded keeps its time, any other task has none. On the first reading after + * a start (`firstSight`), a failure the file does not know predates this record and is dated by + * the run's start, the best the task keeps. + */ +export function recordTaskFailures( + recorded: TaskFailures, + tasks: ReadonlyArray, + now: string, + firstSight = false, +): TaskFailures { + const next: Record = {}; + for (const task of tasks) { + if (task.lastRunStatus !== "failed") continue; + const known = recorded[task.id]; + next[task.id] = + known !== undefined && known.runCount === task.runCount + ? { ...known, threadId: task.threadId } + : { + threadId: task.threadId, + runCount: task.runCount, + failedAt: firstSight && known === undefined ? (task.lastRunAt ?? now) : now, + }; + } + return next; +} + +const sameFailures = (left: TaskFailures, right: TaskFailures) => + Object.keys(left).length === Object.keys(right).length && + Object.entries(left).every(([id, failure]) => { + const other = right[id]; + return ( + other !== undefined && + other.threadId === failure.threadId && + other.runCount === failure.runCount && + other.failedAt === failure.failedAt + ); + }); + +/** When each task bound to `threadId` last failed. */ +export const failuresOf = ( + recorded: TaskFailures, + threadId: ThreadId, +): ReadonlyArray => + Object.values(recorded).flatMap((failure) => + failure.threadId === threadId ? [DateTime.makeUnsafe(failure.failedAt)] : [], + ); + +export class BuddyTaskFailures extends Context.Service< + BuddyTaskFailures, + { readonly failedAt: (threadId: ThreadId) => Effect.Effect> } +>()("t3/lazurio/buddyTaskFailures") {} + +const make = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { stateDir } = yield* ServerConfig.ServerConfig; + const filePath = path.join(stateDir, "lazurio", "buddy-task-failures.json"); + const decode = Schema.decodeUnknownOption(Schema.fromJsonString(TaskFailuresFile)); + const encode = Schema.encodeEffect(Schema.fromJsonString(TaskFailuresFile)); + const stored = yield* fs.readFileString(filePath).pipe( + Effect.map((text) => Option.getOrElse(decode(text), () => ({}))), + Effect.orElseSucceed(() => ({})), + ); + const state = yield* Ref.make(stored); + const tasks = yield* Effect.serviceOption(ScheduledTaskService); + // Only a server with Buddy's thread watches its tasks. + const buddy = + isBuddySurfaceEnvironment(yield* HostProcess.Environment) || + (yield* sharedBuddyFolder) !== undefined; + if (Option.isSome(tasks) && buddy) { + const record = Effect.fn("BuddyTaskFailures.record")( + function* (list: { readonly tasks: ReadonlyArray }, firstSight: boolean) { + const now = DateTime.formatIso(yield* DateTime.now); + const before = yield* Ref.get(state); + const after = recordTaskFailures(before, list.tasks, now, firstSight); + yield* Ref.set(state, after); + if (sameFailures(before, after)) return; + yield* fs.makeDirectory(path.dirname(filePath), { recursive: true, mode: 0o700 }); + yield* writeFileStringAtomically({ filePath, contents: `${yield* encode(after)}\n` }); + }, + Effect.ignoreCause({ log: true }), + ); + // The first reading is the start's, before any change is reported. + yield* tasks.value.list().pipe( + Effect.flatMap((list) => record(list, true)), + Effect.ignoreCause({ log: true }), + ); + yield* tasks.value.subscribeList().pipe( + Stream.runForEach((list) => record(list, false)), + Effect.ignoreCause({ log: true }), + Effect.forkScoped, + ); + } + return BuddyTaskFailures.of({ + failedAt: Effect.fn("BuddyTaskFailures.failedAt")(function* (threadId) { + return failuresOf(yield* Ref.get(state), threadId); + }), + }); +}); + +export const layer = Layer.effect(BuddyTaskFailures, make); diff --git a/apps/server/src/lazurio/overlay.ts b/apps/server/src/lazurio/overlay.ts index 4179fcbc79b74..94ba5de04e69f 100644 --- a/apps/server/src/lazurio/overlay.ts +++ b/apps/server/src/lazurio/overlay.ts @@ -11,6 +11,7 @@ import * as AgentCliSendState from "./agentCliSendState.ts"; import * as BuddySignal from "./buddySignal.ts"; import * as BuddySurfaceDays from "./buddySurfaceDays.ts"; import * as BuddySurfaceThreadRoute from "./buddySurfaceThreadRoute.ts"; +import * as BuddyTaskFailures from "./buddyTaskFailures.ts"; import * as EnvironmentBrowser from "./environmentBrowser.ts"; import * as ModelRecommendationRoute from "./modelRecommendationRoute.ts"; import * as ProviderRefresh from "./providerRefresh.ts"; @@ -28,7 +29,7 @@ export const layer = Layer.mergeAll( ModelRecommendationRoute.layer, ProviderRefreshRoute.layer.pipe(Layer.provide(ProviderRefresh.layer)), BuddySurfaceThreadRoute.layer, - BuddySignal.layer, + BuddySignal.layer.pipe(Layer.provide(BuddyTaskFailures.layer)), AgentCliSendState.layer, Layer.mergeAll(BuddySurfaceDays.layer, ActingSendersRoute.layer).pipe( Layer.provide(ActingSenders.layer), diff --git a/scripts/lazurio-release-contract.test.mjs b/scripts/lazurio-release-contract.test.mjs index 51c2ba81dd88c..61ecf9dcc5a7e 100644 --- a/scripts/lazurio-release-contract.test.mjs +++ b/scripts/lazurio-release-contract.test.mjs @@ -2222,7 +2222,7 @@ NodeTest.test("a thread's tabs live in its holder file behind an overlay route", ); NodeAssert.match( overlay, - /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*BuddySignal\.layer,\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, + /\nexport const layer = Layer\.mergeAll\(\s*ThreadTabsRoute\.layer\.pipe\(Layer\.provide\(environmentBrowser\)\),\s*ModelRecommendationRoute\.layer,\s*ProviderRefreshRoute\.layer\.pipe\(Layer\.provide\(ProviderRefresh\.layer\)\),\s*BuddySurfaceThreadRoute\.layer,\s*BuddySignal\.layer\.pipe\(Layer\.provide\(BuddyTaskFailures\.layer\)\),\s*AgentCliSendState\.layer,\s*Layer\.mergeAll\(BuddySurfaceDays\.layer, ActingSendersRoute\.layer\)\.pipe\(\s*Layer\.provide\(ActingSenders\.layer\),?\s*\),?\s*\)\.pipe\(Layer\.provide\(threadTabs\)\);/, ); NodeAssert.match( overlay, From 2a63ff33d3b3ea704ce52fdf3c87cfd414cffadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:28:02 +0000 Subject: [PATCH 4/6] fix(lazurio): a task failure the record does not know is dated by the reading that finds it On the first reading after a start, a failure the record did not hold was dated by its run's start, so a task that started before a completed turn and failed after it read as quiet after an upgrade or a missed record. It is now dated by that reading: a fresh failure always shows, and a stale one shows only until the next completed turn. Co-Authored-By: Claude Opus 5.5 --- .../src/lazurio/buddyTaskFailures.test.ts | 46 ++++++++++++++----- apps/server/src/lazurio/buddyTaskFailures.ts | 33 +++++-------- 2 files changed, 46 insertions(+), 33 deletions(-) diff --git a/apps/server/src/lazurio/buddyTaskFailures.test.ts b/apps/server/src/lazurio/buddyTaskFailures.test.ts index ae69431cc712f..24c76523047b7 100644 --- a/apps/server/src/lazurio/buddyTaskFailures.test.ts +++ b/apps/server/src/lazurio/buddyTaskFailures.test.ts @@ -4,15 +4,17 @@ import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, describe, it } from "@effect/vitest"; -import { ScheduledTaskId, ThreadId } from "@t3tools/contracts"; +import { MessageId, RunId, ScheduledTaskId, ThreadId } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as ServerConfig from "../config.ts"; import { ScheduledTaskService } from "../scheduledTasks/ScheduledTaskService.ts"; +import { buddySignalOf } from "./buddySignal.ts"; import { BuddyTaskFailures, failuresOf, @@ -53,15 +55,35 @@ describe("when a task bound to Buddy's thread last failed", () => { assert.deepStrictEqual(recordTaskFailures(again, [], at(15)), {}); }); - it("dates a failure from before the record by the run's start on the first reading after a start", () => { - const seen = recordTaskFailures({}, [task(1, "failed", at(3))], at(30), true); - assert.deepStrictEqual(failuresOf(seen, THREAD), [DateTime.makeUnsafe(at(3))]); + it("dates a failure the record does not know by the reading that finds it, also after a start", () => { + // Started at :04, a Buddy turn completed at :05, failed at :07; the server starts at :30 + // with no record: the failure is dated :30, after the turn, so the sign shows it. + const seen = recordTaskFailures({}, [task(1, "failed", at(4))], at(30)); + const failedTaskRuns = failuresOf(seen, THREAD); + assert.deepStrictEqual(failedTaskRuns, [DateTime.makeUnsafe(at(30))]); + const sign = buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(6)), + runs: [ + { + id: RunId.make("run-1"), + ordinal: 1, + status: "completed", + userMessageId: MessageId.make("m1"), + completedAt: DateTime.makeUnsafe(at(5)), + }, + ], + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns, + messages: new Map(), + }); + assert.strictEqual(sign.state, "failed"); // A failure the file already holds keeps its time across the restart. const kept = recordTaskFailures( { "task-1": { threadId: THREAD, runCount: 1, failedAt: at(7) } }, - [task(1, "failed", at(3))], + [task(1, "failed", at(4))], at(30), - true, ); assert.deepStrictEqual(failuresOf(kept, THREAD), [DateTime.makeUnsafe(at(7))]); }); @@ -112,17 +134,19 @@ describe("the record of a Buddy server's failed tasks", () => { Effect.scoped, ); return Effect.gen(function* () { - // The start finds a failure the record does not know: dated by its run's start. + // The start at :30 finds a failure the record does not know: dated by that reading. + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(at(30)))); assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(3))]), [ - DateTime.makeUnsafe(at(3)), + DateTime.makeUnsafe(at(30)), ]); const file = NodeFS.readdirSync(base, { recursive: true }).find((entry) => String(entry).endsWith("buddy-task-failures.json"), ); assert.isDefined(file); - // A restart keeps the recorded time, whatever the task's start says now. - assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(1))]), [ - DateTime.makeUnsafe(at(3)), + // A restart at :40 keeps the recorded time. + yield* TestClock.setTime(DateTime.toEpochMillis(DateTime.makeUnsafe(at(40)))); + assert.deepStrictEqual(yield* failedAt([task(1, "failed", at(3))]), [ + DateTime.makeUnsafe(at(30)), ]); }).pipe( Effect.ensuring(Effect.sync(() => NodeFS.rmSync(base, { recursive: true, force: true }))), diff --git a/apps/server/src/lazurio/buddyTaskFailures.ts b/apps/server/src/lazurio/buddyTaskFailures.ts index b17dea5db0d75..2b0794f3b4dbb 100644 --- a/apps/server/src/lazurio/buddyTaskFailures.ts +++ b/apps/server/src/lazurio/buddyTaskFailures.ts @@ -32,19 +32,17 @@ const TaskFailure = Schema.Struct({ const TaskFailuresFile = Schema.Record(Schema.String, TaskFailure); export type TaskFailures = typeof TaskFailuresFile.Type; -type TaskRow = Pick; +type TaskRow = Pick; /** - * The failures after one reading of the task list at `now`: a failed run not seen before is dated - * `now`, one already recorded keeps its time, any other task has none. On the first reading after - * a start (`firstSight`), a failure the file does not know predates this record and is dated by - * the run's start, the best the task keeps. + * The failures after one reading of the task list at `now`: a failed run not recorded before is + * dated `now`, also on the first reading after a start, since its completion is not kept anywhere + * (a stale failure then shows until the next completed turn, never a fresh one as quiet). */ export function recordTaskFailures( recorded: TaskFailures, tasks: ReadonlyArray, now: string, - firstSight = false, ): TaskFailures { const next: Record = {}; for (const task of tasks) { @@ -53,11 +51,7 @@ export function recordTaskFailures( next[task.id] = known !== undefined && known.runCount === task.runCount ? { ...known, threadId: task.threadId } - : { - threadId: task.threadId, - runCount: task.runCount, - failedAt: firstSight && known === undefined ? (task.lastRunAt ?? now) : now, - }; + : { threadId: task.threadId, runCount: task.runCount, failedAt: now }; } return next; } @@ -107,10 +101,10 @@ const make = Effect.gen(function* () { (yield* sharedBuddyFolder) !== undefined; if (Option.isSome(tasks) && buddy) { const record = Effect.fn("BuddyTaskFailures.record")( - function* (list: { readonly tasks: ReadonlyArray }, firstSight: boolean) { + function* (list: { readonly tasks: ReadonlyArray }) { const now = DateTime.formatIso(yield* DateTime.now); const before = yield* Ref.get(state); - const after = recordTaskFailures(before, list.tasks, now, firstSight); + const after = recordTaskFailures(before, list.tasks, now); yield* Ref.set(state, after); if (sameFailures(before, after)) return; yield* fs.makeDirectory(path.dirname(filePath), { recursive: true, mode: 0o700 }); @@ -119,15 +113,10 @@ const make = Effect.gen(function* () { Effect.ignoreCause({ log: true }), ); // The first reading is the start's, before any change is reported. - yield* tasks.value.list().pipe( - Effect.flatMap((list) => record(list, true)), - Effect.ignoreCause({ log: true }), - ); - yield* tasks.value.subscribeList().pipe( - Stream.runForEach((list) => record(list, false)), - Effect.ignoreCause({ log: true }), - Effect.forkScoped, - ); + yield* tasks.value.list().pipe(Effect.flatMap(record), Effect.ignoreCause({ log: true })); + yield* tasks.value + .subscribeList() + .pipe(Stream.runForEach(record), Effect.ignoreCause({ log: true }), Effect.forkScoped); } return BuddyTaskFailures.of({ failedAt: Effect.fn("BuddyTaskFailures.failedAt")(function* (threadId) { From e36d9b7adf4cb2a2cd12af59d2f994eaea3b0574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:06:12 +0000 Subject: [PATCH 5/6] test(lazurio): the sign's stale task failure shows once, on the first start without a record The known cost of dating an unrecorded failure by the reading that finds it, accepted for #232: a failure already cleared by a later turn shows as failed after the first start of this version, only until the next completed turn, and a later start with the record raises nothing. Co-Authored-By: Claude Opus 5.5 --- .../src/lazurio/buddyTaskFailures.test.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/apps/server/src/lazurio/buddyTaskFailures.test.ts b/apps/server/src/lazurio/buddyTaskFailures.test.ts index 24c76523047b7..97717856f7712 100644 --- a/apps/server/src/lazurio/buddyTaskFailures.test.ts +++ b/apps/server/src/lazurio/buddyTaskFailures.test.ts @@ -88,6 +88,38 @@ describe("when a task bound to Buddy's thread last failed", () => { assert.deepStrictEqual(failuresOf(kept, THREAD), [DateTime.makeUnsafe(at(7))]); }); + it("raises a stale failure once, on the first start without a record, until the next completed turn", () => { + // The known cost (Lazurio/t3code#232): failed at :04, cleared by a turn at :05, and this + // version first starts at :30 with no record. The failure is dated :30 and shows ... + const completed = (ordinal: number, minute: number) => ({ + id: RunId.make(`run-${ordinal}`), + ordinal, + status: "completed" as const, + userMessageId: MessageId.make(`m${ordinal}`), + completedAt: DateTime.makeUnsafe(at(minute)), + }); + const state = (runs: ReadonlyArray>, recordedAt: number) => + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(59)), + runs, + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: failuresOf( + recordTaskFailures({}, [task(1, "failed", at(3))], at(recordedAt)), + THREAD, + ), + messages: new Map(), + }).state; + assert.strictEqual(state([completed(1, 5)], 30), "failed"); + // ... only until the next completed turn, at :31. + assert.strictEqual(state([completed(1, 5), completed(2, 31)], 30), "quiet"); + // A later start reads the record back, so it raises nothing again. + const recorded = recordTaskFailures({}, [task(1, "failed", at(3))], at(30)); + const restarted = recordTaskFailures(recorded, [task(1, "failed", at(3))], at(45)); + assert.deepStrictEqual(failuresOf(restarted, THREAD), [DateTime.makeUnsafe(at(30))]); + }); + it("belongs only to the thread the task is bound to", () => { const other = recordTaskFailures( {}, From b2d213b574388eea222288cee0356415b96a371d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Such=C3=A1nek?= <16311043+immakermatty@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:03:59 +0000 Subject: [PATCH 6/6] fix(lazurio): a task's failure stays through its next start and success until a Buddy turn The record dropped a task's failure as soon as its status left `failed`: the next occurrence publishes `running` at once, and a successful run moves on to `succeeded`, so the sign went quiet although no Buddy turn had completed after the failure. A recorded failure now stays until the task fails anew (dated then) or is deleted; the signal's comparison with the latest completed turn is what clears it. Co-Authored-By: Claude Opus 5.5 --- .../src/lazurio/buddyTaskFailures.test.ts | 33 +++++++++++++++++-- apps/server/src/lazurio/buddyTaskFailures.ts | 13 ++++---- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/apps/server/src/lazurio/buddyTaskFailures.test.ts b/apps/server/src/lazurio/buddyTaskFailures.test.ts index 97717856f7712..1ee69da16f1b8 100644 --- a/apps/server/src/lazurio/buddyTaskFailures.test.ts +++ b/apps/server/src/lazurio/buddyTaskFailures.test.ts @@ -46,12 +46,39 @@ describe("when a task bound to Buddy's thread last failed", () => { assert.deepStrictEqual(failuresOf(edited, THREAD), [DateTime.makeUnsafe(at(7))]); }); - it("dates a later failed run anew, and forgets a task once it no longer failed", () => { + it("keeps a failure through a later start and success until a completed turn clears it", () => { + // Failed at :05; the next occurrence starts at :10 (running) and succeeds at :12. + const failed = recordTaskFailures({}, [task(1, "failed")], at(5)); + const running = recordTaskFailures(failed, [task(1, "running")], at(10)); + const succeeded = recordTaskFailures(running, [task(2, "succeeded")], at(12)); + assert.deepStrictEqual(failuresOf(succeeded, THREAD), [DateTime.makeUnsafe(at(5))]); + const completed = (minute: number) => ({ + id: RunId.make(`run-${minute}`), + ordinal: minute, + status: "completed" as const, + userMessageId: MessageId.make(`m${minute}`), + completedAt: DateTime.makeUnsafe(at(minute)), + }); + const state = (runs: ReadonlyArray>) => + buddySignalOf({ + threadId: THREAD, + lastVisitedAt: DateTime.makeUnsafe(at(59)), + runs, + pendingRequests: [], + stopRequested: new Set(), + failedTaskRuns: failuresOf(succeeded, THREAD), + messages: new Map(), + }).state; + // No Buddy turn completed after :05: still failed. + assert.strictEqual(state([completed(3)]), "failed"); + // A Buddy turn completed at :15 clears it. + assert.strictEqual(state([completed(3), completed(15)]), "quiet"); + }); + + it("dates a later failed run anew, and drops a deleted task's failure", () => { const first = recordTaskFailures({}, [task(1, "failed")], at(5)); const again = recordTaskFailures(first, [task(2, "failed", at(10))], at(12)); assert.deepStrictEqual(failuresOf(again, THREAD), [DateTime.makeUnsafe(at(12))]); - assert.deepStrictEqual(recordTaskFailures(again, [task(3, "running")], at(13)), {}); - assert.deepStrictEqual(recordTaskFailures(again, [task(3, "succeeded")], at(14)), {}); assert.deepStrictEqual(recordTaskFailures(again, [], at(15)), {}); }); diff --git a/apps/server/src/lazurio/buddyTaskFailures.ts b/apps/server/src/lazurio/buddyTaskFailures.ts index 2b0794f3b4dbb..c90c68306b656 100644 --- a/apps/server/src/lazurio/buddyTaskFailures.ts +++ b/apps/server/src/lazurio/buddyTaskFailures.ts @@ -36,8 +36,9 @@ type TaskRow = Pick = {}; for (const task of tasks) { - if (task.lastRunStatus !== "failed") continue; const known = recorded[task.id]; - next[task.id] = - known !== undefined && known.runCount === task.runCount - ? { ...known, threadId: task.threadId } - : { threadId: task.threadId, runCount: task.runCount, failedAt: now }; + if (task.lastRunStatus === "failed" && known?.runCount !== task.runCount) + next[task.id] = { threadId: task.threadId, runCount: task.runCount, failedAt: now }; + else if (known !== undefined) next[task.id] = { ...known, threadId: task.threadId }; } return next; }