diff --git a/apps/server/src/kin/KinServer.ts b/apps/server/src/kin/KinServer.ts index a0da08fa9618e..33bc02a3f5fef 100644 --- a/apps/server/src/kin/KinServer.ts +++ b/apps/server/src/kin/KinServer.ts @@ -5,8 +5,10 @@ import * as Option from "effect/Option"; import { KinIdentity } from "./chat/KinIdentity.ts"; import { KinChatService } from "./chat/KinChatService.ts"; import { KinChatHub } from "./chat/KinChatHub.ts"; +import { KinBudget } from "./lead/KinBudget.ts"; +import { KinEventQueue } from "./lead/KinEventQueue.ts"; -export type Services = KinIdentity | KinChatService; +export type Services = KinIdentity | KinChatService | KinBudget | KinEventQueue; export class KinServer extends Context.Service< KinServer, diff --git a/apps/server/src/kin/chat/layer.ts b/apps/server/src/kin/chat/layer.ts index c81f17b40b04c..e58558c0d850d 100644 --- a/apps/server/src/kin/chat/layer.ts +++ b/apps/server/src/kin/chat/layer.ts @@ -1,4 +1,7 @@ import * as Layer from "effect/Layer"; +import * as Budget from "../lead/KinBudget.ts"; +import * as Queue from "../lead/KinEventQueue.ts"; +import * as Cache from "../memory/MemoryCache.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as Repository from "./KinChatRepository.ts"; import * as Hub from "./KinChatHub.ts"; @@ -7,6 +10,7 @@ import { KinIdentity, KinIdentityConfig } from "./KinIdentity.ts"; export const layer = Layer.mergeAll( Service.layer, + Budget.layer.pipe(Layer.provideMerge(Queue.layer), Layer.provide(Cache.layer)), KinIdentity.layer.pipe(Layer.provide(KinIdentityConfig.layer)), ).pipe( Layer.provideMerge(Hub.layer), diff --git a/apps/server/src/kin/chat/rpcHandlers.test.ts b/apps/server/src/kin/chat/rpcHandlers.test.ts index b7e1e3cd08aad..ad3bd593c0b8e 100644 --- a/apps/server/src/kin/chat/rpcHandlers.test.ts +++ b/apps/server/src/kin/chat/rpcHandlers.test.ts @@ -8,6 +8,7 @@ import * as NodeSocket from "@effect/platform-node/NodeSocket"; import { AuthSessionId, AuthSettingsWriteScope, + AuthOrchestrationReadScope, KinChatError, KinMemberId, KinPersonId, @@ -18,7 +19,7 @@ import { KinAgentId, KinChatId, KinMessageId, - KinChatRpcs, + KinRpcs, KinOrgId, ThreadId, RpcScopeAuthorization, @@ -74,7 +75,7 @@ const probe = WsRpcGroup.omit( tag !== WS_METHODS.serverProbe, ), ); -const group = RpcGroup.make(...KinChatRpcs, ...probe.requests.values()).middleware( +const group = RpcGroup.make(...KinRpcs, ...probe.requests.values()).middleware( RpcScopeAuthorization, ); const seed = Effect.gen(function* () { @@ -531,3 +532,79 @@ it.effect("drops a failed waiting read without ending the chat list stream", () if (next._tag === "chat") assert.strictEqual(next.chat.lastSeq, 1); }).pipe(Effect.provide(root.pipe(Layer.provideMerge(Sqlite.layerMemory)))), ); + +it.effect( + "serves budget settings to the owner and refuses non-owner and foreign organizations", + () => + Effect.gen(function* () { + const { person, repo } = yield* seed; + const handlers = yield* makeKinChatHandlers(session); + const client = yield* RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.merge( + group.toLayer({ ...handlers, "server.probe": () => Effect.succeed({}) }), + RpcAuthorization.layer(AuthStandardClientScopes), + ), + ), + ); + const input = { orgId: person.orgId, limits: { leadUnits: 300, workerUnits: 600 } }; + assert.deepStrictEqual((yield* client["kin.budget.set"](input)).limits, input.limits); + assert.deepStrictEqual( + (yield* client["kin.budget.get"]({ orgId: person.orgId })).limits, + input.limits, + ); + assert.include( + yield* client["kin.budget.set"]({ ...input, orgId: KinOrgId.make("foreign") }).pipe( + Effect.flip, + ), + { reason: "not_found" }, + ); + const memberId = KinPersonId.make("non-owner"); + yield* repo.insertPerson({ + orgId: person.orgId, + personId: memberId, + handle: "member", + displayName: "Member", + githubLogin: null, + createdAt: "1970-01-01T00:00:00.000Z", + }); + const kin = yield* Kin.KinServer; + const services = Option.getOrThrow(kin.services); + const identity = Context.get(services, KinIdentity); + const memberHandlers = yield* makeKinChatHandlers(session).pipe( + Effect.provideService(Kin.KinServer, { + ...kin, + services: Option.some( + Context.add(services, KinIdentity, { + ...identity, + resolve: () => + Effect.succeed([{ _tag: "person", orgId: person.orgId, personId: memberId }]), + }), + ), + }), + ); + const memberClient = yield* RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.merge( + group.toLayer({ ...memberHandlers, "server.probe": () => Effect.succeed({}) }), + RpcAuthorization.layer(AuthStandardClientScopes), + ), + ), + ); + assert.include(yield* memberClient["kin.budget.set"](input).pipe(Effect.flip), { + reason: "not_allowed", + }); + const readOnly = yield* RpcTest.makeClient(group).pipe( + Effect.provide( + Layer.merge( + group.toLayer({ ...handlers, "server.probe": () => Effect.succeed({}) }), + RpcAuthorization.layer([AuthOrchestrationReadScope]), + ), + ), + ); + assert.strictEqual( + (yield* readOnly["kin.budget.set"](input).pipe(Effect.flip))._tag, + "EnvironmentAuthorizationError", + ); + }).pipe(Effect.provide(root.pipe(Layer.provideMerge(Sqlite.layerMemory)))), +); diff --git a/apps/server/src/kin/chat/rpcHandlers.ts b/apps/server/src/kin/chat/rpcHandlers.ts index c858b2131a226..7479ee779cb74 100644 --- a/apps/server/src/kin/chat/rpcHandlers.ts +++ b/apps/server/src/kin/chat/rpcHandlers.ts @@ -1,4 +1,4 @@ -import { KinChatError, KinChatRpcs } from "@t3tools/contracts"; +import { KinChatError, KinRpcs } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; @@ -7,10 +7,11 @@ import * as Stream from "effect/Stream"; import * as RpcGroup from "effect/rpc/RpcGroup"; import type { AuthenticatedSession } from "../../auth/EnvironmentAuth.ts"; import { KinServer } from "../layer.ts"; +import { KinBudget } from "../lead/KinBudget.ts"; import { KinIdentity } from "./KinIdentity.ts"; import { KinChatService } from "./KinChatService.ts"; -const group = RpcGroup.make(...KinChatRpcs); +const group = RpcGroup.make(...KinRpcs); const unavailable = new KinChatError({ reason: "unavailable", message: "Kin is unavailable." }); const isKinChatError = Schema.is(KinChatError); const mapError = (error: { readonly _tag: string }) => @@ -26,6 +27,7 @@ export const makeKinChatHandlers = Effect.fn("makeKinChatHandlers")(function* ( if (Option.isNone(kin) || Option.isNone(kin.value.services)) return yield* unavailable; return { identity: Context.get(kin.value.services.value, KinIdentity), + budget: Context.get(kin.value.services.value, KinBudget), chat: Context.get(kin.value.services.value, KinChatService), }; }); @@ -34,6 +36,22 @@ export const makeKinChatHandlers = Effect.fn("makeKinChatHandlers")(function* ( return { chat, people: yield* identity.resolve(session) }; }); return group.of({ + "kin.budget.get": (input) => + services.pipe( + Effect.flatMap(({ identity, budget }) => + identity + .resolve(session) + .pipe(Effect.flatMap((people) => budget.view(people, input.orgId))), + ), + Effect.mapError(mapError), + ), + "kin.budget.set": (input) => + services.pipe( + Effect.flatMap(({ identity, budget }) => + identity.resolve(session).pipe(Effect.flatMap((people) => budget.set(people, input))), + ), + Effect.mapError(mapError), + ), "kin.orgs.list": () => services.pipe( Effect.flatMap(({ identity, chat }) => diff --git a/apps/server/src/kin/lead/KinBudget.test.ts b/apps/server/src/kin/lead/KinBudget.test.ts new file mode 100644 index 0000000000000..501eae8bbf4c6 --- /dev/null +++ b/apps/server/src/kin/lead/KinBudget.test.ts @@ -0,0 +1,286 @@ +import { assert, describe, it } from "@effect/vitest"; +import { KinOrgId, KinPersonId, KinWakeupId } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlClient from "effect/sql/SqlClient"; +import * as TestClock from "effect/testing/TestClock"; +import * as Sqlite from "../../persistence/Sqlite.ts"; +import * as Persistence from "../persistence/KinPersistence.ts"; +import * as Chat from "../chat/KinChatRepository.ts"; +import * as Hub from "../chat/KinChatHub.ts"; +import * as Cache from "../memory/MemoryCache.ts"; +import * as Queue from "./KinEventQueue.ts"; +import * as Budget from "./KinBudget.ts"; +import { seed, orgId, agentId } from "./WakeupRunner.testkit.ts"; + +const database = Persistence.layer.pipe(Layer.provideMerge(Sqlite.layerMemory)); +const layer = Budget.layer.pipe( + Layer.provideMerge( + Layer.mergeAll(Queue.layer, Cache.layer, Hub.layer).pipe(Layer.provideMerge(Chat.layer)), + ), + Layer.provideMerge(database), +); +const owner = [{ _tag: "person" as const, orgId, personId: KinPersonId.make("owner") }]; +const usage = [ + { + providerTurnId: "turn", + usedTokens: 30, + turnTokenUsage: { + usageScope: "main_agent" as const, + usageStatus: "complete" as const, + hasSubagents: false, + inputTokens: 20, + cachedInputTokens: 5, + outputTokens: 10, + }, + }, +]; + +describe("KinBudget", () => { + it.effect( + "enforces every threshold boundary, worker headroom, and once-daily exhaustion notice", + () => + Effect.gen(function* () { + yield* seed; + const budget = yield* Budget.KinBudget; + const sql = yield* SqlClient.SqlClient; + yield* budget.set(owner, { orgId, limits: { leadUnits: 100, workerUnits: 10 } }); + for (const [remaining, allowed] of [ + [100, [true, true, true, true, true]], + [41, [true, true, true, true, true]], + [40, [true, true, true, false, true]], + [31, [true, true, true, false, true]], + [30, [true, true, true, false, false]], + [20, [true, true, true, false, false]], + [19, [true, true, false, false, false]], + [0, [true, false, false, false, false]], + ] as const) { + yield* sql`UPDATE kin_budget SET lead_units = ${100 - remaining} WHERE org_id = ${orgId}`; + const admissions = yield* Effect.forEach(["P0", "P1", "P2", "P3", "P4"] as const, (p) => + budget.admit(orgId, p), + ); + assert.deepStrictEqual( + admissions.map((a) => a.allowed), + [...allowed], + `${remaining}%`, + ); + assert.isTrue(admissions.every((a) => a.cheapOnly === (remaining === 0))); + } + assert.isTrue(yield* budget.admitWorker(orgId)); + yield* sql`UPDATE kin_budget SET worker_units = 10 WHERE org_id = ${orgId}`; + assert.isFalse(yield* budget.admitWorker(orgId)); + yield* budget.admit(orgId, "P0"); + assert.strictEqual( + (yield* sql`SELECT message_id FROM kin_messages WHERE org_id = ${orgId} ORDER BY message_id`) + .length, + 1, + ); + assert.deepStrictEqual( + yield* sql`SELECT exhausted_notified_at FROM kin_budget WHERE org_id = ${orgId} ORDER BY day`, + [{ exhausted_notified_at: 0 }], + ); + yield* budget.set(owner, { orgId, limits: { leadUnits: 200, workerUnits: 20 } }); + assert.isTrue((yield* budget.admit(orgId, "P3")).allowed); + yield* budget.set(owner, { orgId, limits: { leadUnits: 0, workerUnits: 0 } }); + assert.isFalse(yield* budget.admitWorker(orgId)); + assert.strictEqual( + (yield* sql`SELECT message_id FROM kin_messages WHERE org_id = ${orgId} ORDER BY message_id`) + .length, + 1, + ); + }).pipe(Effect.provide(layer)), + ); + it.effect("resets at owner-local midnight across both DST transitions", () => + Effect.gen(function* () { + yield* seed; + const budget = yield* Budget.KinBudget; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_trigger_settings VALUES (${orgId}, 'America/New_York', '09:00', '18:00', '02:00', 0, 1, '10:00', '{}')`; + for (const [start, reset, hours] of [ + ["2026-03-08T05:00:00.000Z", "2026-03-09T04:00:00.000Z", 23], + ["2026-11-01T04:00:00.000Z", "2026-11-02T05:00:00.000Z", 25], + ] as const) { + yield* TestClock.setTime(Date.parse(start)); + const before = yield* budget.get(orgId); + yield* sql`UPDATE kin_budget SET lead_units = 2000000 WHERE org_id = ${orgId} AND day = ${before.day}`; + assert.strictEqual((yield* budget.admit(orgId, "P3")).nextReset, Date.parse(reset)); + yield* TestClock.adjust(`${hours} hours`); + const after = yield* budget.get(orgId); + assert.notEqual(after.day, before.day); + assert.strictEqual(after.leadUnits, 0); + assert.isTrue((yield* budget.admit(orgId, "P3")).allowed); + } + }).pipe(Effect.provide(layer)), + ); + it.effect( + "charges fractional cached usage atomically once, preserving identical retries across days", + () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const budget = yield* Budget.KinBudget; + const wakeupId = KinWakeupId.make("wakeup"); + yield* sql`INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) VALUES (${orgId}, ${wakeupId}, ${agentId}, 'cheap', 0, 'running', '[]')`; + const first = yield* budget.charge({ orgId, wakeupId, usage }); + assert.strictEqual(first.units, 55.5); + assert.deepStrictEqual(yield* budget.charge({ orgId, wakeupId, usage }), first); + yield* TestClock.adjust("1 day"); + assert.deepStrictEqual(yield* budget.charge({ orgId, wakeupId, usage }), first); + assert.strictEqual((yield* budget.get(orgId)).leadUnits, 0); + assert.deepStrictEqual( + yield* sql`SELECT lead_units FROM kin_budget WHERE org_id = ${orgId} ORDER BY day`, + [{ lead_units: 55.5 }, { lead_units: 0 }], + ); + assert.deepStrictEqual( + yield* sql`SELECT tokens_json FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ tokens_json: first.tokensJson }], + ); + const full = [ + { ...usage[0]!, turnTokenUsage: { ...usage[0]!.turnTokenUsage, outputTokens: 11 } }, + ]; + const updated = yield* budget.charge({ orgId, wakeupId, usage: full }); + assert.strictEqual(updated.units, 59.5); + assert.deepStrictEqual(yield* budget.charge({ orgId, wakeupId, usage: full }), updated); + assert.strictEqual((yield* budget.get(orgId)).leadUnits, 4); + assert.include( + yield* budget + .charge({ orgId: KinOrgId.make("foreign"), wakeupId, usage }) + .pipe(Effect.flip), + { reason: "not_found" }, + ); + }).pipe(Effect.provide(layer)), + ); + it.effect( + "charges partial retries only by their positive delta and tolerates unusable usage", + () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + const budget = yield* Budget.KinBudget; + const cases = [ + { + id: "partial", + usedTokens: 30, + turnTokenUsage: { ...usage[0]!.turnTokenUsage, usageStatus: "partial" as const }, + units: 30, + }, + { id: "missing-fields", usedTokens: 30, turnTokenUsage: undefined, units: 30 }, + { + id: "unavailable", + usedTokens: 30, + turnTokenUsage: { ...usage[0]!.turnTokenUsage, usageStatus: "unavailable" as const }, + units: 30, + }, + { id: "no-usage", usedTokens: undefined, turnTokenUsage: undefined, units: 0 }, + { + id: "pi-normalized", + usedTokens: 150, + turnTokenUsage: { + ...usage[0]!.turnTokenUsage, + inputTokens: 140, + cachedInputTokens: 100, + }, + units: 90, + }, + { + id: "clamp", + usedTokens: undefined, + turnTokenUsage: { ...usage[0]!.turnTokenUsage, cachedInputTokens: 100 }, + units: 42, + }, + { + id: "other-usage", + usedTokens: undefined, + turnTokenUsage: { + ...usage[0]!.turnTokenUsage, + otherUsage: { inputTokens: 10, outputTokens: 2 }, + }, + units: 73.5, + }, + ...[-1, NaN, Infinity].map((bad, i) => ({ + id: `garbage-${i}`, + usedTokens: 30, + turnTokenUsage: { ...usage[0]!.turnTokenUsage, inputTokens: bad }, + units: 30, + })), + ]; + for (const row of cases) { + const wakeupId = KinWakeupId.make(row.id); + yield* sql`INSERT INTO kin_wakeups (org_id, wakeup_id, agent_id, model, started_at, status, event_ids) VALUES (${orgId}, ${wakeupId}, ${agentId}, 'cheap', 0, 'running', '[]')`; + const charge = yield* budget.charge({ + orgId, + wakeupId, + usage: [ + { + providerTurnId: row.id, + usedTokens: row.usedTokens, + ...(row.turnTokenUsage === undefined ? {} : { turnTokenUsage: row.turnTokenUsage }), + }, + ], + }); + assert.strictEqual(charge.units, row.units, row.id); + yield* (yield* Queue.KinEventQueue).finish({ orgId, wakeupId, status: "completed" }); + assert.deepStrictEqual( + yield* sql`SELECT status FROM kin_wakeups WHERE wakeup_id = ${wakeupId}`, + [{ status: "completed" }], + ); + assert.isNotNull( + (yield* sql`SELECT tokens_json FROM kin_wakeups WHERE wakeup_id = ${wakeupId}`)[0]! + .tokens_json, + ); + if (row.id === "partial") { + const full = yield* budget.charge({ orgId, wakeupId, usage }); + assert.strictEqual(full.units, 55.5); + assert.deepStrictEqual(yield* budget.charge({ orgId, wakeupId, usage }), full); + assert.deepStrictEqual(yield* budget.charge({ orgId, wakeupId, usage: [] }), full); + } + } + assert.strictEqual((yield* budget.get(orgId)).leadUnits, 411); + }).pipe(Effect.provide(layer)), + ); + it.effect("requires the same-org owner and validates limits before writing", () => + Effect.gen(function* () { + yield* seed; + const budget = yield* Budget.KinBudget; + const sql = yield* SqlClient.SqlClient; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) VALUES ('member', ${orgId}, 'member', 'Member', '1970-01-01T00:00:00.000Z')`; + const input = { orgId, limits: { leadUnits: 100, workerUnits: 200 } }; + assert.include( + yield* budget + .set([{ _tag: "person", orgId, personId: KinPersonId.make("member") }], input) + .pipe(Effect.flip), + { reason: "not_allowed" }, + ); + assert.include(yield* budget.set([], input).pipe(Effect.flip), { reason: "not_found" }); + for (const bad of [-1, 1.5, Infinity, NaN, Number.MAX_SAFE_INTEGER + 1]) { + assert.include( + yield* budget + .set(owner, { orgId, limits: { leadUnits: bad, workerUnits: 0 } }) + .pipe(Effect.flip), + { reason: "invalid_state" }, + ); + } + assert.deepStrictEqual((yield* budget.set(owner, input)).limits, input.limits); + const defaultOrg = KinOrgId.make("kin-default"); + const bootstrap = [ + { + _tag: "person" as const, + orgId: defaultOrg, + personId: KinPersonId.make("kin-default-owner"), + }, + ]; + yield* sql`INSERT INTO kin_orgs VALUES (${defaultOrg}, 'Default', '1970-01-01T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_people (person_id, org_id, handle, display_name, created_at) VALUES ('kin-default-owner', ${defaultOrg}, 'owner', 'Owner', '1970-01-01T00:00:00.000Z'), ('other-owner', ${defaultOrg}, 'other', 'Other', '1970-01-01T00:00:00.000Z')`; + const defaults = { orgId: defaultOrg, limits: input.limits }; + assert.deepStrictEqual((yield* budget.set(bootstrap, defaults)).limits, input.limits); + yield* sql`INSERT INTO kin_agents (agent_id, org_id, display_name, role, owner_person_id, created_at) VALUES ('default-lead', ${defaultOrg}, 'Lead', 'lead', 'other-owner', '1970-01-01T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('default-chat', ${defaultOrg}, 'team_group', 0, '1970-01-01T00:00:00.000Z')`; + yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) SELECT 'default-lead', ${defaultOrg}, name, level_since, models_json, worker_projects_json, 'default-chat' FROM kin_leads WHERE org_id = ${orgId}`; + assert.include(yield* budget.set(bootstrap, defaults).pipe(Effect.flip), { + reason: "not_allowed", + }); + yield* sql`UPDATE kin_agents SET owner_person_id = 'kin-default-owner' WHERE org_id = ${defaultOrg}`; + assert.deepStrictEqual((yield* budget.set(bootstrap, defaults)).limits, input.limits); + }).pipe(Effect.provide(layer)), + ); +}); diff --git a/apps/server/src/kin/lead/KinBudget.ts b/apps/server/src/kin/lead/KinBudget.ts new file mode 100644 index 0000000000000..7a7302f4b4e73 --- /dev/null +++ b/apps/server/src/kin/lead/KinBudget.ts @@ -0,0 +1,330 @@ +import { + KinBudget as Budget, + KinBudgetLimits, + KinChatError, + KinChatId, + KinMessageId, + type KinOrgId, + type KinWakeupId, + type TurnTokenUsage, + type OrchestrationV2ProviderTurnTokenUsage, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import { isSqlError } from "effect/sql/SqlError"; +import * as SqlClient from "effect/sql/SqlClient"; +import { toPersistenceSqlError, PersistenceSqlError } from "../../persistence/Errors.ts"; +import type { PersonIdentity } from "../chat/KinIdentity.ts"; +import * as Chat from "../chat/KinChatRepository.ts"; +import * as Hub from "../chat/KinChatHub.ts"; +import * as MemoryCache from "../memory/MemoryCache.ts"; +import * as Queue from "./KinEventQueue.ts"; + +type Priority = "P0" | "P1" | "P2" | "P3" | "P4"; +type Usage = { + readonly providerTurnId: string; + readonly turnTokenUsage?: TurnTokenUsage; + readonly tokenUsage?: OrchestrationV2ProviderTurnTokenUsage; + readonly usedTokens?: number | undefined; +}; +type BudgetError = KinChatError | PersistenceSqlError; +type Admission = { + readonly allowed: boolean; + readonly cheapOnly: boolean; + readonly nextReset: number; + readonly remaining: number; +}; +type Charge = { + readonly orgId: KinOrgId; + readonly wakeupId: KinWakeupId; + readonly day: string; + readonly units: number; + readonly tokensJson: string; +}; +// JSON has no representation for non-finite counters; retain their diagnostic values as strings. +const usageJsonValue = (value: unknown): unknown => { + if (typeof value === "number" && !Number.isFinite(value)) return String(value); + if (Array.isArray(value)) return value.map(usageJsonValue); + if (typeof value === "object" && value !== null) + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, usageJsonValue(item)]), + ); + return value; +}; +const defaults = { leadUnits: 2_000_000, workerUnits: 6_000_000 }; +const decodeLimits = Schema.decodeUnknownEffect(KinBudgetLimits); +const decodeStoredLimits = Schema.decodeUnknownEffect(Schema.fromJsonString(KinBudgetLimits)); +const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); +const notFound = new KinChatError({ reason: "not_found", message: "Kin budget record not found." }); +const notAllowed = new KinChatError({ + reason: "not_allowed", + message: "Kin budget operation not allowed.", +}); +const invalid = new KinChatError({ reason: "invalid_state", message: "Invalid Kin budget data." }); +const isChatError = Schema.is(KinChatError); +const isPersistenceError = Schema.is(PersistenceSqlError); +const mapError = (cause: unknown): BudgetError => + isSqlError(cause) + ? toPersistenceSqlError("KinBudget")(cause) + : isChatError(cause) || isPersistenceError(cause) + ? cause + : invalid; + +export class KinBudget extends Context.Service< + KinBudget, + { + readonly view: ( + people: ReadonlyArray, + orgId: KinOrgId, + ) => Effect.Effect; + readonly get: (orgId: KinOrgId) => Effect.Effect; + readonly set: ( + people: ReadonlyArray, + input: { readonly orgId: KinOrgId; readonly limits: KinBudgetLimits }, + ) => Effect.Effect; + readonly admit: (orgId: KinOrgId, priority: Priority) => Effect.Effect; + readonly admitWorker: (orgId: KinOrgId) => Effect.Effect; + readonly promptContext: (orgId: KinOrgId) => Effect.Effect; + readonly charge: (input: { + readonly orgId: KinOrgId; + readonly wakeupId: KinWakeupId; + readonly usage: ReadonlyArray; + }) => Effect.Effect; + } +>()("t3/kin/lead/KinBudget") {} + +const make = Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const cache = yield* MemoryCache.MemoryCache; + const queue = yield* Queue.KinEventQueue; + const chat = yield* Chat.KinChatRepository; + const hub = yield* Hub.KinChatHub; + const period = Effect.fn("KinBudget.period")(function* (orgId: KinOrgId) { + const now = yield* Clock.currentTimeMillis; + const zone = yield* cache.ownerTimeZone(orgId); + const local = DateTime.makeZonedUnsafe(now, { timeZone: zone }); + return { + day: DateTime.formatIsoDate(local), + nextReset: DateTime.toEpochMillis(DateTime.add(DateTime.startOf(local, "day"), { days: 1 })), + now, + }; + }); + const ensure = Effect.fn("KinBudget.ensure")(function* (orgId: KinOrgId, day: string) { + const rows = yield* sql`SELECT 1 FROM kin_orgs WHERE org_id = ${orgId}`; + if (rows.length === 0) return yield* notFound; + yield* sql`INSERT INTO kin_budget (org_id, day) VALUES (${orgId}, ${day}) ON CONFLICT DO NOTHING RETURNING org_id`; + }); + const snapshot = Effect.fn("KinBudget.snapshot")(function* (orgId: KinOrgId, day: string) { + const [row] = yield* sql<{ leadUnits: number; workerUnits: number; limits: string | null }>` + SELECT b.lead_units AS "leadUnits", b.worker_units AS "workerUnits", l.limits_json AS limits + FROM kin_budget b LEFT JOIN kin_budget_limits l ON l.org_id = b.org_id + WHERE b.org_id = ${orgId} AND b.day = ${day}`; + if (row === undefined) return yield* notFound; + return { + orgId, + day, + leadUnits: row.leadUnits, + workerUnits: row.workerUnits, + limits: row.limits === null ? defaults : yield* decodeStoredLimits(row.limits), + }; + }); + const get = Effect.fn("KinBudget.get")(function* (orgId: KinOrgId) { + const { day } = yield* period(orgId); + return yield* sql.withTransaction( + Effect.gen(function* () { + yield* ensure(orgId, day); + return yield* snapshot(orgId, day); + }), + ); + }, Effect.mapError(mapError)); + const share = (budget: Budget) => + budget.limits.leadUnits === 0 + ? 0 + : Math.max(0, budget.limits.leadUnits - budget.leadUnits) / budget.limits.leadUnits; + const notifyExhausted = Effect.fn("KinBudget.notifyExhausted")(function* ( + orgId: KinOrgId, + day: string, + now: number, + ) { + const [lead] = yield* sql<{ + chatId: string; + }>`SELECT owner_chat_id AS "chatId" FROM kin_leads WHERE org_id = ${orgId} ORDER BY agent_id LIMIT 1`; + if (lead === undefined) return; + const updated = + yield* sql`UPDATE kin_budget SET exhausted_notified_at = ${now} WHERE org_id = ${orgId} AND day = ${day} AND exhausted_notified_at IS NULL RETURNING org_id`; + if (updated.length === 0) return; + const message = yield* chat.insertMessage({ + orgId, + chatId: KinChatId.make(lead.chatId), + messageId: KinMessageId.make(`kin-budget:${orgId}:${day}`), + author: { _tag: "system" }, + kind: "text", + body: "Today's Kin lead budget is exhausted. Only urgent owner messages will run, on the cheap model. Raise the limits in settings to resume other work.", + intent: null, + mentions: [], + replyTo: null, + widgetId: null, + floorRunId: null, + status: "posted", + sentBy: null, + clientMessageId: null, + createdAt: DateTime.formatIso(DateTime.makeUnsafe(now)), + editedAt: null, + }); + yield* hub.publish({ type: "message.posted", orgId, message }); + }); + return KinBudget.of({ + get, + view: Effect.fn("KinBudget.view")(function* (people, orgId) { + return yield* sql.withTransaction( + Effect.gen(function* () { + const person = people.find((p) => p.orgId === orgId); + if (person === undefined) return yield* notFound; + const exists = + yield* sql`SELECT 1 FROM kin_people WHERE org_id = ${orgId} AND person_id = ${person.personId}`; + if (exists.length === 0) return yield* notFound; + return yield* get(orgId); + }), + ); + }, Effect.mapError(mapError)), + set: Effect.fn("KinBudget.set")(function* (people, input) { + const limits = yield* decodeLimits(input.limits); + const { day } = yield* period(input.orgId); + return yield* sql.withTransaction( + Effect.gen(function* () { + yield* ensure(input.orgId, day); + const person = people.find((p) => p.orgId === input.orgId); + if (person === undefined) return yield* notFound; + // The reserved bootstrap owner is authoritative before a lead is configured. + const owner = + yield* sql`SELECT 1 FROM kin_people p WHERE p.org_id = ${input.orgId} AND p.person_id = ${person.personId} + AND ((p.org_id = 'kin-default' AND p.person_id = 'kin-default-owner' + AND NOT EXISTS (SELECT 1 FROM kin_leads WHERE org_id = p.org_id)) OR EXISTS ( + SELECT 1 FROM kin_leads l JOIN kin_agents a ON a.org_id = l.org_id AND a.agent_id = l.agent_id + WHERE l.org_id = p.org_id AND a.owner_person_id = p.person_id))`; + if (owner.length === 0) return yield* notAllowed; + const json = yield* encodeJson(limits); + yield* sql`INSERT INTO kin_budget_limits (org_id, limits_json) VALUES (${input.orgId}, ${json}) ON CONFLICT DO NOTHING RETURNING org_id`; + yield* sql`UPDATE kin_budget_limits SET limits_json = ${json} WHERE org_id = ${input.orgId}`; + yield* queue.reconsiderBudget(input.orgId); + return yield* snapshot(input.orgId, day); + }), + ); + }, Effect.mapError(mapError)), + admit: Effect.fn("KinBudget.admit")(function* (orgId, priority) { + const { day, nextReset, now } = yield* period(orgId); + return yield* sql.withTransaction( + Effect.gen(function* () { + yield* ensure(orgId, day); + const remaining = share(yield* snapshot(orgId, day)); + if (remaining === 0) yield* notifyExhausted(orgId, day, now); + const allowed = + priority === "P0" || + (priority === "P4" + ? remaining > 0.3 + : remaining > 0 && + (priority === "P1" || (priority === "P2" ? remaining >= 0.2 : remaining > 0.4))); + return { allowed, cheapOnly: remaining === 0, nextReset, remaining }; + }), + ); + }, Effect.mapError(mapError)), + admitWorker: Effect.fn("KinBudget.admitWorker")(function* (orgId) { + const budget = yield* get(orgId); + return budget.workerUnits < budget.limits.workerUnits; + }), + promptContext: Effect.fn("KinBudget.promptContext")(function* (orgId) { + return `${Math.floor(share(yield* get(orgId)) * 100)}% of today left`; + }), + charge: Effect.fn("KinBudget.charge")(function* (input) { + let units = 0; + const valid = (value: number | undefined) => + value !== undefined && Number.isFinite(value) && value >= 0; + const count = (value: number | undefined) => (valid(value) ? value! : 0); + const ids = new Set(); + if (input.usage.length === 0) + yield* Effect.logWarning("Kin budget turn usage unavailable", { + orgId: input.orgId, + wakeupId: input.wakeupId, + }); + for (const turn of input.usage) { + if (ids.has(turn.providerTurnId)) continue; + ids.add(turn.providerTurnId); + const ledger = turn.turnTokenUsage; + const complete = + ledger?.usageStatus === "complete" && + valid(ledger.inputTokens) && + valid(ledger.outputTokens) && + (ledger.cachedInputTokens === undefined || valid(ledger.cachedInputTokens)); + if (!complete) { + yield* Effect.logWarning("Kin budget using fallback usage", { + orgId: input.orgId, + wakeupId: input.wakeupId, + usageStatus: ledger?.usageStatus ?? "missing", + hasUsedTokens: valid(turn.usedTokens), + }); + } + if (!complete && valid(turn.usedTokens)) { + units += count(turn.usedTokens); + continue; + } + const incoming = count(ledger?.inputTokens); + const cached = Math.min(incoming, count(ledger?.cachedInputTokens)); + if (cached !== count(ledger?.cachedInputTokens)) + yield* Effect.logWarning("Kin budget clamped cached usage", { + orgId: input.orgId, + wakeupId: input.wakeupId, + }); + units += + incoming - + cached + + 0.1 * cached + + 4 * count(ledger?.outputTokens) + + count(ledger?.otherUsage?.inputTokens) + + 4 * count(ledger?.otherUsage?.outputTokens); + } + if (!Number.isFinite(units)) { + yield* Effect.logWarning("Kin budget usage overflow", { + orgId: input.orgId, + wakeupId: input.wakeupId, + }); + units = 0; + } + const tokensJson = yield* encodeJson(usageJsonValue(input.usage)); + const { day } = yield* period(input.orgId); + return yield* sql.withTransaction( + Effect.gen(function* () { + const existing = + yield* sql`SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId}`; + if (existing.length === 0) return yield* notFound; + const [stored] = + yield* sql`SELECT org_id AS "orgId", wakeup_id AS "wakeupId", day, units, tokens_json AS "tokensJson" FROM kin_budget_charges WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId}`; + if (stored !== undefined && (stored.tokensJson === tokensJson || units <= stored.units)) + return stored; + const delta = Math.max(0, units - (stored?.units ?? 0)); + const chargedUnits = Math.max(units, stored?.units ?? 0); + yield* ensure(input.orgId, day); + const before = yield* snapshot(input.orgId, day); + if (!Number.isFinite(before.leadUnits + delta)) return yield* invalid; + yield* sql`INSERT INTO kin_budget_charges (org_id, wakeup_id, day, units, tokens_json) + VALUES (${input.orgId}, ${input.wakeupId}, ${day}, ${chargedUnits}, ${tokensJson}) + ON CONFLICT (org_id, wakeup_id) DO UPDATE SET units = excluded.units, tokens_json = excluded.tokens_json`; + yield* sql`UPDATE kin_budget SET lead_units = lead_units + ${delta} WHERE org_id = ${input.orgId} AND day = ${day}`; + yield* sql`UPDATE kin_wakeups SET tokens_json = ${tokensJson} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId}`; + return { + orgId: input.orgId, + wakeupId: input.wakeupId, + day: stored?.day ?? day, + units: chargedUnits, + tokensJson, + }; + }), + ); + }, Effect.mapError(mapError)), + }); +}); + +export const layer = Layer.effect(KinBudget, make); diff --git a/apps/server/src/kin/lead/KinEventQueue.ts b/apps/server/src/kin/lead/KinEventQueue.ts index 48443ef05ef7f..8fae9a32d440f 100644 --- a/apps/server/src/kin/lead/KinEventQueue.ts +++ b/apps/server/src/kin/lead/KinEventQueue.ts @@ -41,6 +41,7 @@ const ClaimInput = Schema.Struct({ }); type ClaimInput = typeof ClaimInput.Type; const ReleaseFields = { + budgetDeferred: Schema.optional(Schema.Boolean), orgId: KinOrgId, wakeupId: KinWakeupId, eventIds: Schema.optional(Schema.Array(KinEventId)), @@ -120,6 +121,7 @@ export class KinEventQueue extends Context.Service< | KinEventQueueUnknownAgent | KinEventQueueRetiredAgent >; + readonly reconsiderBudget: (orgId: KinOrgId) => Effect.Effect; readonly changes: (orgId: KinOrgId) => Effect.Effect>; readonly nextReady: ( orgId: KinOrgId, @@ -138,7 +140,7 @@ export class KinEventQueue extends Context.Service< readonly claim: ( input: ClaimInput, ) => Effect.Effect< - ReadonlyArray, + ReadonlyArray, | PersistenceSqlError | PersistenceDecodeError | KinEventQueueBusy @@ -373,11 +375,11 @@ const make = Effect.gen(function* () { return yield* new KinEventQueueBusy({ orgId: input.orgId }); const rows = yield* sql<{ json: string; - priority: string; + priority: "P0" | "P1" | "P2" | "P3"; received_at: number; event_id: string; }>` - UPDATE kin_events SET state = 'claimed', wakeup_id = ${input.wakeupId} + UPDATE kin_events SET state = 'claimed', wakeup_id = ${input.wakeupId}, budget_deferred = 0 WHERE org_id = ${input.orgId} AND event_id IN ( SELECT event_id FROM kin_events WHERE org_id = ${input.orgId} AND agent_id = ${input.agentId} AND state = 'queued' @@ -392,7 +394,11 @@ const make = Effect.gen(function* () { a.received_at - b.received_at || compare(a.event_id, b.event_id), ); - const events = yield* Effect.forEach(ordered, (row) => decodeEvent(row.json)); + const events = yield* Effect.forEach(ordered, (row) => + decodeEvent(row.json).pipe( + Effect.map((event) => ({ ...event, priority: row.priority })), + ), + ); const eventIds = yield* encodeEventIds(events.map((event) => event.eventId)); yield* sql`UPDATE kin_wakeups SET event_ids = ${eventIds} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId}`; @@ -430,7 +436,7 @@ const make = Effect.gen(function* () { const rows = yield* sql<{ event_id: string; state: string }>` UPDATE kin_events SET attempts = attempts + ${input.countAttempt ? 1 : 0}, state = CASE WHEN attempts + ${input.countAttempt ? 1 : 0} >= 3 THEN 'failed' ELSE 'queued' END, - wakeup_id = NULL, not_before = ${notBefore} + wakeup_id = NULL, not_before = ${notBefore}, budget_deferred = ${input.budgetDeferred === true ? 1 : 0} WHERE org_id = ${input.orgId} AND wakeup_id = ${input.wakeupId} AND state = 'claimed' AND (${ids ?? null} IS NULL OR event_id IN (SELECT value FROM json_each(${ids ?? null}))) AND EXISTS (SELECT 1 FROM kin_wakeups WHERE org_id = ${input.orgId} @@ -575,6 +581,13 @@ const make = Effect.gen(function* () { Effect.mapError(toQueuePersistenceError("KinEventQueue.nextReady")), ); return KinEventQueue.of({ + reconsiderBudget: Effect.fn("KinEventQueue.reconsiderBudget")(function* (orgId) { + yield* sql`UPDATE kin_events SET not_before = NULL, budget_deferred = 0 + WHERE org_id = ${orgId} AND state = 'queued' AND budget_deferred = 1`.pipe( + Effect.mapError(toPersistenceSqlError("KinEventQueue.reconsiderBudget")), + ); + yield* sql.afterCommit(notify(orgId)); + }), enqueue, claim, release, diff --git a/apps/server/src/kin/lead/LeadPrompt.test.ts b/apps/server/src/kin/lead/LeadPrompt.test.ts index 9087030e5699d..488f2874c9c27 100644 --- a/apps/server/src/kin/lead/LeadPrompt.test.ts +++ b/apps/server/src/kin/lead/LeadPrompt.test.ts @@ -123,10 +123,12 @@ it.effect( const second = yield* prompt.build({ ...input, wakeupId: KinWakeupId.make("next"), + budgetContext: "62% of today left", events: [routine], }); const prefix = (text: string) => text.split("--- KIN WAKE-UP ---")[0]; expect(prefix(first)).toBe(prefix(second)); + expect(second.split("--- KIN WAKE-UP ---")[1]).toContain("Budget: 62% of today left"); expect(first).toContain("1970-01-01T00:00:00.000Z"); expect(second).toContain("1970-01-01T01:00:00.000Z"); const current = yield* prompt.build(input); diff --git a/apps/server/src/kin/lead/LeadPrompt.ts b/apps/server/src/kin/lead/LeadPrompt.ts index 22c4c77bf72d0..7ac0c41a01c99 100644 --- a/apps/server/src/kin/lead/LeadPrompt.ts +++ b/apps/server/src/kin/lead/LeadPrompt.ts @@ -29,6 +29,7 @@ export interface KinLeadPromptInput { readonly wakeupId: KinWakeupId; readonly events: ReadonlyArray; readonly memoryHead: string; + readonly budgetContext?: string; } export class KinLeadPrompt extends Context.Service< @@ -131,6 +132,7 @@ export class KinLeadPrompt extends Context.Service< `Wake-up: ${encoded(input.wakeupId)}`, `Organization: ${encoded(orgId)}`, `Lead: ${encoded(agentId)}`, + ...(input.budgetContext === undefined ? [] : [`Budget: ${input.budgetContext}`]), "# Events", boundedEvents(events), ].join("\n\n"); diff --git a/apps/server/src/kin/lead/WakeupRunner.test.ts b/apps/server/src/kin/lead/WakeupRunner.test.ts index 8bbee9125bf2b..ef5df2302c96f 100644 --- a/apps/server/src/kin/lead/WakeupRunner.test.ts +++ b/apps/server/src/kin/lead/WakeupRunner.test.ts @@ -12,6 +12,8 @@ import { EnvironmentId, MessageId, KinWakeupId, + KinPersonId, + KinEvent, KinAgentId, ThreadId, RunId, @@ -29,6 +31,7 @@ import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as Budget from "./KinBudget.ts"; import * as KinIdentity from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; import * as MemoryConfig from "../memory/MemoryConfig.ts"; @@ -49,13 +52,16 @@ import * as EffectWorker from "../../orchestration-v2/EffectWorker.ts"; import { provideDeterministicTestRuntime } from "../../orchestration-v2/testkit/DeterministicRuntime.ts"; import { harness, seed, orgId, agentId, event } from "./WakeupRunner.testkit.ts"; +const decodeEvent = Schema.decodeSync(KinEvent); const decodeUsage = Schema.decodeEffect( Schema.fromJsonString( Schema.Array( Schema.Struct({ - inputTokens: Schema.Number, - cachedInputTokens: Schema.Number, - outputTokens: Schema.Number, + turnTokenUsage: Schema.Struct({ + inputTokens: Schema.Number, + cachedInputTokens: Schema.Number, + outputTokens: Schema.Number, + }), }), ), ), @@ -80,6 +86,128 @@ const run = Effect.fn("test.run")(function* () { }); describe("WakeupRunner", () => { + it.effect("uses the cheap model for exhausted P0 even when a routine is in the batch", () => + Effect.gen(function* () { + yield* seed; + const budget = yield* Budget.KinBudget; + const queue = yield* KinEventQueue.KinEventQueue; + const sql = yield* SqlClient.SqlClient; + yield* budget.set([{ _tag: "person", orgId, personId: KinPersonId.make("owner") }], { + orgId, + limits: { leadUnits: 0, workerUnits: 0 }, + }); + yield* queue.enqueue({ + ...event("urgent"), + _tag: "resume", + pausedAt: DateTime.makeUnsafe(0), + }); + yield* queue.enqueue(event("routine")); + yield* Fiber.join(yield* run()); + assert.deepStrictEqual( + yield* sql`SELECT model, status FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ model: "gpt-5.1-codex-mini", status: "completed" }], + ); + assert.deepStrictEqual( + yield* sql`SELECT event_id, state, attempts FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [ + { event_id: "routine", state: "queued", attempts: 0 }, + { event_id: "urgent", state: "done", attempts: 0 }, + ], + ); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect( + "defers P3 until the owner-local reset without attempts and re-admits it after limits rise", + () => + Effect.gen(function* () { + yield* seed; + const queue = yield* KinEventQueue.KinEventQueue; + const budget = yield* Budget.KinBudget; + const sql = yield* SqlClient.SqlClient; + const people = [{ _tag: "person" as const, orgId, personId: KinPersonId.make("owner") }]; + yield* sql`INSERT INTO kin_trigger_settings VALUES (${orgId}, 'America/New_York', '09:00', '18:00', '02:00', 0, 1, '10:00', '{}')`; + yield* budget.set(people, { orgId, limits: { leadUnits: 100, workerUnits: 100 } }); + yield* sql`UPDATE kin_budget SET lead_units = 85 WHERE org_id = ${orgId}`; + const pending = decodeEvent({ + ...event("deferred"), + receivedAt: "1970-01-01T00:00:00.000Z", + _tag: "watchdog", + finding: "Check", + itemIds: [], + }); + yield* queue.enqueue(pending); + yield* queue.enqueue( + decodeEvent({ + ...event("urgent"), + receivedAt: "1970-01-01T00:00:00.000Z", + _tag: "resume", + pausedAt: "1970-01-01T00:00:00.000Z", + }), + ); + yield* Fiber.join(yield* run()); + assert.deepStrictEqual( + yield* sql`SELECT state, not_before, attempts FROM kin_events WHERE org_id = ${orgId} AND event_id = 'deferred' ORDER BY event_id`, + [{ state: "queued", not_before: 18_000_000, attempts: 0 }], + ); + yield* budget.set(people, { orgId, limits: { leadUnits: 1000, workerUnits: 100 } }); + yield* Fiber.join(yield* run()); + assert.deepStrictEqual( + yield* sql`SELECT state, attempts FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [ + { state: "done", attempts: 0 }, + { state: "done", attempts: 0 }, + ], + ); + }).pipe(Effect.provide(harness()), provideDeterministicTestRuntime), + ); + it.effect("completes Pi wake-ups with disjoint snapshot counters and no turn ledger", () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* (yield* KinEventQueue.KinEventQueue).enqueue(event("pi")); + yield* Fiber.join(yield* run()); + const rows = yield* sql<{ + status: string; + tokens_json: string; + }>`SELECT status, tokens_json FROM kin_wakeups WHERE org_id = ${orgId}`; + assert.strictEqual(rows[0]!.status, "completed"); + assert.include(JSON.parse(rows[0]!.tokens_json)[0].tokenUsage, { + inputTokens: 2, + cachedInputTokens: 100, + }); + assert.strictEqual((yield* (yield* Budget.KinBudget).get(orgId)).leadUnits, 30); + assert.deepStrictEqual( + yield* sql`SELECT state, attempts FROM kin_events WHERE org_id = ${orgId}`, + [{ state: "done", attempts: 0 }], + ); + }).pipe(Effect.provide(harness({ piUsage: true })), provideDeterministicTestRuntime), + ); + it.effect( + "charges failed provider turns from the exact usage recorded on the failed wake-up", + () => + Effect.gen(function* () { + yield* seed; + const sql = yield* SqlClient.SqlClient; + yield* (yield* KinEventQueue.KinEventQueue).enqueue(event("failed-provider")); + yield* sql`UPDATE kin_events SET budget_deferred = 1, not_before = 0 WHERE org_id = ${orgId}`; + yield* Fiber.join(yield* run()); + const [row] = yield* sql<{ + status: string; + tokens_json: string; + }>`SELECT status, tokens_json FROM kin_wakeups WHERE org_id = ${orgId} ORDER BY wakeup_id`; + assert.strictEqual(row!.status, "failed"); + assert.deepStrictEqual( + yield* sql`SELECT budget_deferred, attempts FROM kin_events WHERE org_id = ${orgId} ORDER BY event_id`, + [{ budget_deferred: 0, attempts: 1 }], + ); + assert.strictEqual((yield* decodeUsage(row!.tokens_json)).length, 1); + assert.deepStrictEqual( + yield* sql`SELECT units, tokens_json FROM kin_budget_charges WHERE org_id = ${orgId} ORDER BY wakeup_id`, + [{ units: 55.5, tokens_json: row!.tokens_json }], + ); + assert.strictEqual((yield* (yield* Budget.KinBudget).get(orgId)).leadUnits, 55.5); + }).pipe(Effect.provide(harness({ failed: true })), provideDeterministicTestRuntime), + ); it.effect( "runs two wake-ups on one T3 thread with distinct provider threads and captures only each run's usage", () => @@ -95,8 +223,10 @@ describe("WakeupRunner", () => { thread_id: string; tokens_json: string; wakeup_id: string; - }>`SELECT thread_id, tokens_json, wakeup_id FROM kin_wakeups ORDER BY started_at`; + model: string; + }>`SELECT thread_id, tokens_json, wakeup_id, model FROM kin_wakeups ORDER BY started_at`; assert.strictEqual(rows.length, 2); + assert.isTrue(rows.every((row) => row.model === "gpt-5.1-codex")); assert.strictEqual(rows[0]?.thread_id, rows[1]?.thread_id); assert.match(rows[0]!.wakeup_id, /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/); const threads = yield* ThreadManagement.ThreadManagementService; @@ -107,9 +237,20 @@ describe("WakeupRunner", () => { for (const row of rows) { const usage = yield* decodeUsage(row.tokens_json); assert.deepStrictEqual(usage, [ - { inputTokens: 20, cachedInputTokens: 5, outputTokens: 10 }, + { turnTokenUsage: { inputTokens: 20, cachedInputTokens: 5, outputTokens: 10 } }, ]); } + const charged = yield* sql<{ + units: number; + tokens_json: string; + }>`SELECT units, tokens_json FROM kin_budget_charges WHERE org_id = ${orgId} ORDER BY wakeup_id`; + assert.strictEqual(charged.length, 2); + assert.isTrue(charged.every((charge) => charge.units === 55.5)); + assert.deepStrictEqual( + charged.map((charge) => charge.tokens_json).toSorted(), + rows.map((row) => row.tokens_json).toSorted(), + ); + assert.strictEqual((yield* (yield* Budget.KinBudget).get(orgId)).leadUnits, 111); assert.include(yield* resolveAgent(projection.thread.id).pipe(Effect.flip), { reason: "not_allowed", }); @@ -289,7 +430,10 @@ describe("WakeupRunner", () => { { status: "interrupted" }, ]); yield* Fiber.join(yield* run()); - yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL WHERE status = 'completed'`; + // Reconstruct a crash before the atomic usage capture and charge committed. + yield* sql`DELETE FROM kin_budget_charges WHERE org_id = ${orgId}`; + yield* sql`UPDATE kin_budget SET lead_units = 0 WHERE org_id = ${orgId}`; + yield* sql`UPDATE kin_wakeups SET status = 'running', ended_at = NULL, tokens_json = NULL WHERE status = 'completed'`; yield* sql`UPDATE kin_events SET state = 'claimed' WHERE state = 'done'`; const threads = yield* ThreadManagement.ThreadManagementService; const recoveredRunner = yield* WakeupRunner.WakeupRunner.pipe( @@ -341,6 +485,7 @@ describe("WakeupRunner", () => { tokens_json: string; }>`SELECT tokens_json FROM kin_wakeups WHERE status = 'completed'`; assert.strictEqual((yield* decodeUsage(usageRow!.tokens_json)).length, 2); + assert.strictEqual((yield* (yield* Budget.KinBudget).get(orgId)).leadUnits, 111); assert.deepStrictEqual(yield* sql`SELECT status FROM kin_wakeups ORDER BY status`, [ { status: "completed" }, { status: "interrupted" }, @@ -817,6 +962,8 @@ describe("WakeupRunner", () => { Layer.provide(Layer.mock(KinIdentity.KinIdentity)({ isAgentThread: () => false })), Layer.provide( Layer.mergeAll( + Layer.mock(Budget.KinBudget)({}), + Layer.mock(KinEventQueue.KinEventQueue)({}), Layer.mock(ThreadLaunch.ThreadLaunchService)({}), Layer.mock(ThreadManagement.ThreadManagementService)({}), Layer.mock(ProjectService.ProjectService)({}), diff --git a/apps/server/src/kin/lead/WakeupRunner.testkit.ts b/apps/server/src/kin/lead/WakeupRunner.testkit.ts index 5cee69733a6cb..8ed7653cd9439 100644 --- a/apps/server/src/kin/lead/WakeupRunner.testkit.ts +++ b/apps/server/src/kin/lead/WakeupRunner.testkit.ts @@ -33,6 +33,9 @@ import * as KinPersistence from "../persistence/KinPersistence.ts"; import * as KinChatRepository from "../chat/KinChatRepository.ts"; import * as KinIdentity from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; +import * as Budget from "./KinBudget.ts"; +import * as MemoryCache from "../memory/MemoryCache.ts"; +import * as Hub from "../chat/KinChatHub.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; @@ -84,7 +87,7 @@ export const seed = Effect.gen(function* () { yield* sql`INSERT INTO kin_chats (chat_id, org_id, kind, last_seq, created_at) VALUES ('owner-chat', 'org', 'team_group', 0, ${timestamp})`; const models = yield* encodeJson({ strong: selection, - cheap: selection, + cheap: { ...selection, model: "gpt-5.1-codex-mini" }, }); yield* sql`INSERT INTO kin_leads (agent_id, org_id, name, level_since, models_json, worker_projects_json, owner_chat_id) VALUES ('lead', 'org', 'Lead', 0, ${models}, '[]', 'owner-chat')`; }); @@ -108,6 +111,8 @@ export function harness( readonly releaseDone?: Deferred.Deferred; readonly hold?: Deferred.Deferred; readonly entered?: Deferred.Deferred; + readonly failed?: boolean; + readonly piUsage?: boolean; readonly prepare?: Memory.Memory["Service"]["prepareWakeup"]; } = {}, ) { @@ -164,6 +169,37 @@ export function harness( if (options.hold !== undefined) yield* Deferred.await(options.hold); const time = yield* DateTime.now; const id = ProviderTurnId.make(`turn-${input.runOrdinal}`); + if (!options.piUsage) + yield* PubSub.publish(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id, + providerThreadId: input.providerThread.id, + nodeId: input.rootNodeId, + runAttemptId: input.attemptId, + nativeTurnRef: null, + ordinal: input.runOrdinal, + status: "running", + startedAt: time, + completedAt: null, + tokenUsage: { + usedTokens: 22, + inputTokens: 18, + cachedInputTokens: 4, + outputTokens: 4, + updatedAt: DateTime.formatIso(time), + }, + turnTokenUsage: { + usageScope: "main_agent", + usageStatus: "partial", + hasSubagents: false, + inputTokens: 18, + cachedInputTokens: 4, + outputTokens: 4, + }, + }, + }); yield* PubSub.publish(events, { type: "provider_turn.updated", driver, @@ -174,14 +210,26 @@ export function harness( runAttemptId: input.attemptId, nativeTurnRef: null, ordinal: input.runOrdinal, - status: "completed", + status: options.failed ? "failed" : "completed", startedAt: time, completedAt: time, + ...(options.piUsage + ? {} + : { + turnTokenUsage: { + usageScope: "main_agent", + usageStatus: "complete", + hasSubagents: false, + inputTokens: 20, + cachedInputTokens: 5, + outputTokens: 10, + }, + }), tokenUsage: { usedTokens: 30, - inputTokens: 20, - cachedInputTokens: 5, - outputTokens: 10, + inputTokens: 2, + cachedInputTokens: options.piUsage ? 100 : 1, + outputTokens: 6, updatedAt: DateTime.formatIso(time), }, }, @@ -192,8 +240,18 @@ export function harness( providerThreadId: input.providerThread.id, providerTurnId: id, runOrdinal: input.runOrdinal, - status: "completed", - failure: null, + ...(options.failed + ? { + status: "failed" as const, + failureItemOrdinal: 1, + failure: { + class: "provider_error" as const, + message: "Fixture provider failure", + code: null, + retryable: false, + }, + } + : { status: "completed" as const, failure: null }), threadDisposition: "reusable", }); }), @@ -261,6 +319,15 @@ export function harness( ); const kin = Layer.mergeAll( KinEventQueue.layer, + Budget.layer.pipe( + Layer.provide( + Layer.mergeAll( + MemoryCache.layer, + KinEventQueue.layer, + Hub.layer.pipe(Layer.provideMerge(KinChatRepository.layer)), + ), + ), + ), KinLeadRegistry.layer, KinIdentity.KinIdentity.layer.pipe( Layer.provide(KinChatRepository.layer), diff --git a/apps/server/src/kin/lead/WakeupRunner.ts b/apps/server/src/kin/lead/WakeupRunner.ts index b229772b1db69..1a551a23f4176 100644 --- a/apps/server/src/kin/lead/WakeupRunner.ts +++ b/apps/server/src/kin/lead/WakeupRunner.ts @@ -31,6 +31,7 @@ import * as ProjectService from "../../project/ProjectService.ts"; import * as KinIdentity from "../chat/KinIdentity.ts"; import * as Memory from "../memory/Memory.ts"; import * as KinEventQueue from "./KinEventQueue.ts"; +import * as KinBudget from "./KinBudget.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; @@ -77,7 +78,6 @@ const RunningRow = Schema.Struct({ runId: Schema.NullOr(RunId), startedAt: Schema.Number, }); -const encodeUsage = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); const decodeLead = Schema.decodeUnknownEffect(LeadRow); const decodeRunning = Schema.decodeUnknownEffect(RunningRow); const ulid = Effect.gen(function* () { @@ -95,6 +95,7 @@ const ulid = Effect.gen(function* () { const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const queue = yield* KinEventQueue.KinEventQueue; + const budget = yield* KinBudget.KinBudget; const memory = yield* Memory.Memory; const hooks = yield* Effect.serviceOption(KinWakeupHooks); const prompt = yield* KinLeadPrompt; @@ -158,12 +159,13 @@ const make = Effect.gen(function* () { const turns = projection.providerTurns.filter( (turn) => turn.runAttemptId !== null && attempts.has(turn.runAttemptId), ); - const usage = turns.flatMap((turn) => - turn.tokenUsage === undefined ? [] : [{ providerTurnId: turn.id, ...turn.tokenUsage }], - ); - const tokens = yield* encodeUsage(usage); - yield* sql`UPDATE kin_wakeups SET tokens_json = ${tokens} - WHERE org_id = ${row.orgId} AND agent_id = ${row.agentId} AND wakeup_id = ${row.wakeupId} AND status = 'running'`; + const usage = turns.map((turn) => ({ + providerTurnId: turn.id, + ...(turn.turnTokenUsage === undefined ? {} : { turnTokenUsage: turn.turnTokenUsage }), + ...(turn.tokenUsage === undefined ? {} : { tokenUsage: turn.tokenUsage }), + usedTokens: turn.tokenUsage?.usedTokens, + })); + yield* budget.charge({ orgId: row.orgId, wakeupId: row.wakeupId, usage }); }); const wait = Effect.fn("WakeupRunner.wait")(function* ( row: typeof RunningRow.Type, @@ -295,7 +297,7 @@ const make = Effect.gen(function* () { }; yield* Effect.scoped( Effect.gen(function* () { - const events = yield* Effect.acquireRelease( + let events = yield* Effect.acquireRelease( queue.claim({ orgId, agentId, wakeupId, model: record.models.strong.model }), (_, exit) => cleanup(row, Exit.hasInterrupts(exit)).pipe( @@ -326,6 +328,29 @@ const make = Effect.gen(function* () { yield* queue.finish({ orgId, wakeupId, status: "failed" }); return yield* retire(current); } + let cheapOnly = false; + for (const priority of new Set(events.map((item) => item.priority))) { + const admission = yield* budget.admit(orgId, priority); + cheapOnly ||= admission.cheapOnly; + if (!admission.allowed) { + const deferred = new Set( + events.filter((item) => item.priority === priority).map((item) => item.eventId), + ); + yield* queue.release({ + orgId, + wakeupId, + eventIds: events + .filter((event) => deferred.has(event.eventId)) + .map((event) => event.eventId), + notBefore: admission.nextReset, + countAttempt: false, + budgetDeferred: true, + }); + events = events.filter((event) => !deferred.has(event.eventId)); + } + } + if (events.length === 0) return; + const budgetContext = yield* budget.promptContext(orgId); const pinned = yield* memory.prepareWakeup(orgId).pipe( Effect.tapError((error) => Effect.logError("Kin memory preparation failed", { @@ -353,11 +378,12 @@ const make = Effect.gen(function* () { wakeupId, events, memoryHead: pinned.head, + budgetContext, }); const urgent = yield* sql`SELECT 1 FROM kin_events WHERE org_id = ${orgId} AND wakeup_id = ${wakeupId} AND priority IN ('P0', 'P1') LIMIT 1`; const modelSelection = - urgent.length > 0 || events.some((event) => event._tag === "routine") + !cheapOnly && (urgent.length > 0 || events.some((event) => event._tag === "routine")) ? current.models.strong : current.models.cheap; const threadId = current.threadId ?? ThreadId.make(`kin:${yield* ulid}`); diff --git a/apps/server/src/kin/lead/layer.ts b/apps/server/src/kin/lead/layer.ts index 7f01a7671b1c7..ad30be0ad29b5 100644 --- a/apps/server/src/kin/lead/layer.ts +++ b/apps/server/src/kin/lead/layer.ts @@ -9,7 +9,6 @@ import * as KinChatRepository from "../chat/KinChatRepository.ts"; import * as Memory from "../memory/Memory.ts"; import * as MemoryConfig from "../memory/MemoryConfig.ts"; import * as KinPersistence from "../persistence/KinPersistence.ts"; -import * as KinEventQueue from "./KinEventQueue.ts"; import * as KinLeadRegistry from "./KinLeadRegistry.ts"; import { KinLeadPrompt } from "./LeadPrompt.ts"; import * as WakeupRunner from "./WakeupRunner.ts"; @@ -44,7 +43,7 @@ export const layer = Layer.effectDiscard( ).pipe( Layer.provide(WakeupRunner.layer), Layer.provide( - Layer.mergeAll(KinLeadRegistry.layer, KinEventQueue.layer, KinLeadPrompt.layer).pipe( + Layer.mergeAll(KinLeadRegistry.layer, KinLeadPrompt.layer).pipe( Layer.provideMerge(Layer.mergeAll(Memory.layer, KinChatRepository.layer)), ), ), diff --git a/apps/server/src/kin/persistence/KinMigrations.test.ts b/apps/server/src/kin/persistence/KinMigrations.test.ts index 41fedbd37fd51..06b1360b63dc7 100644 --- a/apps/server/src/kin/persistence/KinMigrations.test.ts +++ b/apps/server/src/kin/persistence/KinMigrations.test.ts @@ -14,6 +14,7 @@ const tableNames = [ "kin_agent_threads", "kin_agents", "kin_budget", + "kin_budget_charges", "kin_budget_limits", "kin_chat_members", "kin_chats", @@ -87,6 +88,7 @@ describe("Kin migrations", () => { { migration_id: 4, name: "MemoryReads" }, { migration_id: 5, name: "OneChatOwner" }, { migration_id: 6, name: "Pings" }, + { migration_id: 7, name: "Budget" }, ]); assert.deepStrictEqual( upstreamBefore.map((row) => [row.migration_id, row.name] as const), @@ -132,6 +134,7 @@ describe("Kin migrations", () => { { migration_id: 4, name: "MemoryReads" }, { migration_id: 5, name: "OneChatOwner" }, { migration_id: 6, name: "Pings" }, + { migration_id: 7, name: "Budget" }, ]); assert.deepStrictEqual(yield* upstreamHistory, upstreamBefore); assert.deepStrictEqual(yield* sql`SELECT * FROM kin_orgs`, [ @@ -200,6 +203,7 @@ describe("Kin migrations", () => { [4, "MemoryReads"], [5, "OneChatOwner"], [6, "Pings"], + [7, "Budget"], ]); const before = yield* kinSchema; assert.deepStrictEqual( @@ -220,12 +224,14 @@ describe("Kin migrations", () => { yield* sql`DROP TABLE kin_ping_deliveries`; yield* sql`DROP TABLE kin_ping_escalations`; yield* sql`DROP TABLE kin_ping_items`; - yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id = 6`; yield* sql`DROP INDEX kin_chat_members_one_owner`; - yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id = 5`; + yield* sql`DROP TABLE kin_budget_charges`; + yield* sql`ALTER TABLE kin_events DROP COLUMN budget_deferred`; + yield* sql`DELETE FROM kin_sql_migrations WHERE migration_id >= 5`; assert.deepStrictEqual(yield* runKinMigrations(), [ [5, "OneChatOwner"], [6, "Pings"], + [7, "Budget"], ]); assert.deepStrictEqual( yield* sql`SELECT member_id FROM kin_chat_members WHERE org_id = 'org' ORDER BY member_id`, @@ -244,6 +250,7 @@ describe("Kin migrations", () => { { migration_id: 4, name: "MemoryReads" }, { migration_id: 5, name: "OneChatOwner" }, { migration_id: 6, name: "Pings" }, + { migration_id: 7, name: "Budget" }, ]); assert.deepStrictEqual(yield* runKinMigrations(), []); assert.deepStrictEqual(yield* runMigrations(), []); @@ -304,6 +311,7 @@ describe("Kin migrations", () => { wakeup_id: "wake", not_before: null, attempts: 1, + budget_deferred: 0, }, kin_workers: { org_id: "org", diff --git a/apps/server/src/kin/persistence/KinMigrations.ts b/apps/server/src/kin/persistence/KinMigrations.ts index 845fce8d48d04..b5dfea8b629ee 100644 --- a/apps/server/src/kin/persistence/KinMigrations.ts +++ b/apps/server/src/kin/persistence/KinMigrations.ts @@ -7,6 +7,7 @@ import Triggers from "./migrations/K003_Triggers.ts"; import MemoryReads from "./migrations/K004_MemoryReads.ts"; import OneChatOwner from "./migrations/K005_OneChatOwner.ts"; import Pings from "./migrations/K006_Pings.ts"; +import Budget from "./migrations/K007_Budget.ts"; const run = Migrator.make({}); const loader = Migrator.fromRecord({ @@ -16,6 +17,7 @@ const loader = Migrator.fromRecord({ "4_MemoryReads": MemoryReads, "5_OneChatOwner": OneChatOwner, "6_Pings": Pings, + "7_Budget": Budget, }); export const runKinMigrations = Effect.fn("KinMigrations.run")(function* () { diff --git a/apps/server/src/kin/persistence/migrations/K007_Budget.ts b/apps/server/src/kin/persistence/migrations/K007_Budget.ts new file mode 100644 index 0000000000000..567621245bce2 --- /dev/null +++ b/apps/server/src/kin/persistence/migrations/K007_Budget.ts @@ -0,0 +1,17 @@ +import * as Effect from "effect/Effect"; +import * as SqlClient from "effect/sql/SqlClient"; + +export default Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`CREATE TABLE kin_budget_charges ( + org_id TEXT NOT NULL, + wakeup_id TEXT NOT NULL, + day TEXT NOT NULL, + units REAL NOT NULL CHECK (units >= 0), + tokens_json TEXT NOT NULL, + PRIMARY KEY (org_id, wakeup_id), + FOREIGN KEY (org_id, wakeup_id) REFERENCES kin_wakeups(org_id, wakeup_id), + FOREIGN KEY (org_id, day) REFERENCES kin_budget(org_id, day) + )`; + yield* sql`ALTER TABLE kin_events ADD COLUMN budget_deferred INTEGER NOT NULL DEFAULT 0 CHECK (budget_deferred IN (0, 1))`; +}); diff --git a/docs/kin/specs/lead.md b/docs/kin/specs/lead.md index f538035e621b7..ae166ea8debc5 100644 --- a/docs/kin/specs/lead.md +++ b/docs/kin/specs/lead.md @@ -22,7 +22,7 @@ Only one wake-up runs per organization at a time. Every wake-up counts against a | The same pattern in production | Scheduled tasks with `threadMode: "fresh_session"` call `sendToThread` with `freshContext: true` | `apps/server/src/scheduledTasks/ScheduledTaskService.ts` (around line 940), `packages/contracts/src/scheduledTask.ts` (`ScheduledTaskThreadMode`) | | Wait for a run, interrupt, stop children | `waitForThread` (with `untilDelegatedWorkSettled`), `interruptThread`, `stopDelegatedTasks`, `settleThread` | `ThreadManagementService.ts` | | Durable event stream | `EventSinkV2.stream({ threadId?, afterSequence?, eventType? })`. Internal workers use the unbounded form; `OrchestratorV2.streamStoredEventsFrom` is the bounded one meant for RPC subscribers | `apps/server/src/orchestration-v2/EventSink.ts` | -| Token usage per turn | `OrchestrationV2ProviderTurn.tokenUsage`; `ThreadTokenUsageSnapshot` has `inputTokens`, `cachedInputTokens`, `outputTokens`, `lastCachedInputTokens` | `packages/contracts/src/orchestrationV2.ts`, `packages/contracts/src/providerRuntime.ts` | +| Token usage per turn | `OrchestrationV2ProviderTurn.turnTokenUsage`; `TurnTokenUsage` is the cumulative turn ledger | `packages/contracts/src/orchestrationV2.ts`, `packages/contracts/src/providerRuntime.ts` | | Model choice | `ModelSelection { instanceId, model, options? }` | `packages/contracts/src/modelSelection.ts` | | Child agents | `delegate_task` → `delegated_task.request` command; requires an active parent run; completion wakes the parent (`completionWake: "always" \| "settled_only"`) | `apps/server/src/mcp/OrchestratorMcpService.ts` (around line 2490), `apps/server/src/mcp/toolkits/orchestrator/tools.ts`, `packages/contracts/src/orchestratorMcp.ts` | | MCP tools for agents | Toolkits registered in `layerTools`; `McpSchema.EnabledWhen` hides tools per caller (see `hiddenFromSubagents`); `McpInvocationContext` gives the calling `threadId`; lint rule `t3code/no-raw-mcp-registration` keeps registration in `McpHttpServer.ts` | `apps/server/src/mcp/McpHttpServer.ts` (`layerTools`, line ~1230), `apps/server/src/mcp/McpInvocationContext.ts`, `apps/server/src/mcp/McpToolAccess.ts` | @@ -219,7 +219,7 @@ Kin boot is optional: configuration, migrations, identity bootstrap and runner s The scheduled-task dispatch wrapper also contains Kin migration failures. When Kin persistence is unavailable, launch refusal is limited to `scheduled-task::webhook:` command ids. Sends classify the task using upstream `scheduled_tasks.schedule_json` and refuse only webhooks; an unreadable upstream row fails closed for that send only. Cron, interval, manual and ordinary dispatch remain available. Restoring the Kin database and restarting restores webhook dispatch. -A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's recorded token usage and provider-turn id, limited to the wake-up run's attempts, on success and failure. +A `MemorySyncConflict` fails the wake-up and leaves the events queued with the queue's five-minute counted-failure backoff. `MemoryCheckoutInvalid` does the same and logs an error; `KinWakeupHooks.memoryCheckoutInvalid` is the typed notification hook for LEAD-3. Writer commits/journal/reset and budget charging remain LEAD-3 and LEAD-10 integrations, not fabricated runner implementations. Until the writer lands, `memory_commit` remains null rather than claiming a writer commit. `tokens_json` contains each provider turn's cumulative `turnTokenUsage` ledger and fallback `usedTokens` and provider-turn id, limited to the wake-up run's attempts, on success and failure. ### The prompt and caching (gap 2) @@ -461,11 +461,14 @@ Thresholds live in code (`KinLevels.promotionRule`) and are shown on the card. T3 runs on the owner's subscriptions, so budget is counted in tokens, not money. Per organization, in `kin_budget` (**new**: `org_id, day, lead_units, worker_units, exhausted_notified_at`): -- `units = (inputTokens − cachedInputTokens) + 0.1 × cachedInputTokens + 4 × outputTokens`, summed over the run's provider turns. +- `units = (inputTokens − cachedInputTokens) + 0.1 × cachedInputTokens + 4 × outputTokens`, summed over the run's provider turns' cumulative `turnTokenUsage`, including `otherUsage` at input/output weights. Incomplete or missing field counts fall back to `usedTokens` at weight 1 and are logged; absent usage is recorded and logged. - `day` is the owner-local `YYYY-MM-DD`. Units are finite non-negative real numbers because cached input has fractional weight; limits are non-negative integers. Lasting limits live in `kin_budget_limits (org_id PRIMARY KEY, limits_json)`, where `limits_json` is `{ leadUnits, workerUnits }`; day rows hold counters only. `exhausted_notified_at` records the once-per-day exhaustion notice. - Defaults: 2,000,000 lead units and 6,000,000 worker units per day, reset at 00:00 in the owner's timezone from `KinTriggerSettings.timeZone` (triggers spec). The owner sets them in settings. - `admit(orgId, priority)` by share of the lead budget left: above 40% everything runs; 20–40% defers P3 to tomorrow; under 20% only P0–P1; at 0 only P0, on the cheap model, and the owner is told once. Dreaming (P4) runs only above 30%. -- `admitWorker(orgId)` refuses at 0 worker units left. The lead gets an error and tells the owner. +- Deferred P2/P3 events wait until the next owner-local reset unless limits change. +- A wake-up spanning midnight bills the owner-local day on which the charge is made. +- `admitWorker(orgId)` refuses at 0 worker units left. The lead gets an error and tells the owner. LEAD-9 owns charging `worker_units` from worker runs; it must not also count the lead turn's Claude `otherUsage`, which is already lead units. +- Providers without a per-turn `turnTokenUsage` ledger (currently Pi, Cursor and Grok) fall back to `usedTokens` at weight 1, which is their context size, so cache-heavy turns are over-billed until their adapters emit a turn ledger. - Deferred events stay queued with `not_before` and show in the morning brief ("3 checks postponed for budget"). `UsageLimitSources` (`apps/server/src/usage/UsageLimitSources.ts`) only reads cliproxy accounts, so plan headroom is not generally known. Kin uses its own counters. See open questions. diff --git a/packages/client-runtime/src/state/commandPermissions.test.ts b/packages/client-runtime/src/state/commandPermissions.test.ts index 79276f3851504..a1413d5d20ead 100644 --- a/packages/client-runtime/src/state/commandPermissions.test.ts +++ b/packages/client-runtime/src/state/commandPermissions.test.ts @@ -68,6 +68,34 @@ const setup = Effect.gen(function* () { }); describe("command permissions", () => { + it.effect("guards Kin budget mutations with the destination settings grant", () => + Effect.scoped( + Effect.gen(function* () { + const registry = yield* setup; + const permissions = createCommandPermissions(runtime, "kin.budget.set"); + registry.set(sessions(env), AsyncResult.success(grant(true))); + expect(registry.get(permissions.permissionAtom(env))).toBe(false); + expect((yield* permissions.authorize(registry, env).pipe(Effect.flip))._tag).toBe( + "EnvironmentAuthorizationError", + ); + registry.set( + sessions(env), + AsyncResult.success({ + ...grant(false), + scopes: [AuthSettingsWriteScope], + permissions: [AuthSettingsWriteScope], + }), + ); + expect(registry.get(permissions.permissionAtom(env))).toBe(true); + yield* permissions.authorize(registry, env); + registry.set(sessions(other), AsyncResult.success(grant(false))); + expect(registry.get(permissions.permissionAtom(other))).toBe(false); + expect((yield* permissions.authorize(registry, other).pipe(Effect.flip))._tag).toBe( + "EnvironmentAuthorizationError", + ); + }), + ), + ); it.effect("requires the destination read grant to refresh providers", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/contracts/src/kinLead.test.ts b/packages/contracts/src/kinLead.test.ts index 8da9df17a6f19..31944888b033a 100644 --- a/packages/contracts/src/kinLead.test.ts +++ b/packages/contracts/src/kinLead.test.ts @@ -184,6 +184,7 @@ describe("Kin lead contracts", () => { { schema: Kin.KinBudget, value: { ...budget, day: "tomorrow" } }, { schema: Kin.KinBudget, value: { ...budget, leadUnits: Infinity } }, { schema: Kin.KinBudgetLimits, value: { ...limits, workerUnits: Infinity } }, + { schema: Kin.KinBudgetLimits, value: { ...limits, leadUnits: Number.MAX_SAFE_INTEGER + 1 } }, { schema: Kin.KinOrgState, value: { ...state, paused: true } }, { schema: Kin.KinOrgState, diff --git a/packages/contracts/src/kinRpcs.ts b/packages/contracts/src/kinRpcs.ts index 35c30dc63732c..01aee69c132b2 100644 --- a/packages/contracts/src/kinRpcs.ts +++ b/packages/contracts/src/kinRpcs.ts @@ -1,9 +1,35 @@ -import { KinChatRpcs, KIN_CHAT_RPC_SCOPES } from "./kinChat.ts"; +import * as Rpc from "effect/rpc/Rpc"; +import * as Schema from "effect/Schema"; +import { KinBudget, KinBudgetGetInput, KinBudgetSetInput } from "./kinLead.ts"; +import { KinChatError, KinChatRpcs, KIN_CHAT_RPC_SCOPES } from "./kinChat.ts"; +import { + EnvironmentAuthorizationError, + AuthOrchestrationReadScope, + AuthSettingsWriteScope, +} from "./auth.ts"; -export const KinRpcs = [...KinChatRpcs] as const; -export const KIN_RPC_SCOPES = { ...KIN_CHAT_RPC_SCOPES } as const; +const KinBudgetRpcs = [ + Rpc.make("kin.budget.get", { + payload: KinBudgetGetInput, + success: KinBudget, + error: Schema.Union([KinChatError, EnvironmentAuthorizationError]), + }), + Rpc.make("kin.budget.set", { + payload: KinBudgetSetInput, + success: KinBudget, + error: Schema.Union([KinChatError, EnvironmentAuthorizationError]), + }), +] as const; +export const KinRpcs = [...KinChatRpcs, ...KinBudgetRpcs] as const; +export const KIN_RPC_SCOPES = { + ...KIN_CHAT_RPC_SCOPES, + "kin.budget.get": AuthOrchestrationReadScope, + "kin.budget.set": AuthSettingsWriteScope, +} as const; export const KIN_RPC_AGGREGATES = { + "kin.budget.get": "kin-budget", + "kin.budget.set": "kin-budget", "kin.orgs.list": "kin-chat", "kin.chats.subscribe": "kin-chat", "kin.chats.createDirect": "kin-chat",