Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions apps/server/src/kin/lead/layer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import * as Routines from "../triggers/routines.ts";
import * as DeferralFlow from "../triggers/deferralFlow.ts";
import * as GithubEvents from "../triggers/githubEvents.ts";
import * as Watchdogs from "../triggers/watchdogs.ts";
import * as GithubReaders from "../triggers/githubReaders.ts";
import * as OwnerCheck from "../triggers/ownerCheck.ts";
import * as TriggerScheduler from "../triggers/KinTriggerScheduler.ts";
import * as TriggerStore from "../triggers/KinTriggerStore.ts";
Expand Down Expand Up @@ -160,6 +161,7 @@ const withServices = <A, E, R>(entry: Layer.Layer<A, E, R>) =>
),
),
Layer.provide(KinPersistence.layer),
Layer.provide(GithubReaders.layer),
);

const registration = McpHttpServer.toolkitRegistration(KinLeadToolkit, handlers);
Expand Down
44 changes: 42 additions & 2 deletions apps/server/src/kin/triggers/KinTriggerService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
type MemoryItemHead,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Schema from "effect/Schema";
import * as SqlClient from "effect/sql/SqlClient";
Expand Down Expand Up @@ -82,10 +83,15 @@ const card = Effect.fn("test.deferralCard")(function* (org: string) {
it.effect("arms from front matter, re-arms on edits, and re-syncs without writes", () => {
let reads = 0;
const reader = {
read: () =>
readRepo: (_org: KinOrgId, _repo: string, numbers: ReadonlyArray<number>) =>
Effect.sync(() => {
reads++;
return { state: "open" as const, assignees: [], linkedPrs: [] };
return new Map(
numbers.map((number) => [
number,
{ state: "open" as const, assignees: [], linkedPrs: [] },
]),
);
}),
};
const linked = (overrides: Partial<MemoryItemHead> = {}) =>
Expand Down Expand Up @@ -311,6 +317,40 @@ it.effect("syncs each head once, retries failures, and isolates failing items",
}).pipe(Effect.provide(layer)),
);

it.effect("reads GitHub once per repo within one budget and asks for what it couldn't read", () => {
const calls: Array<readonly [string, ReadonlyArray<number>]> = [];
const reader = {
readRepo: (_org: KinOrgId, repo: string, numbers: ReadonlyArray<number>) =>
Effect.suspend(() => {
calls.push([repo, numbers]);
return Effect.never;
}),
};
const items = Array.from({ length: 20 }, (_, index) =>
deferral({
id: KinMemoryItemId.make(`def-${index + 1}`),
path: `deferrals/def-${index + 1}.md`,
links: [`https://github.com/acme/api/issues/${index + 1}`],
}),
);
return Effect.gen(function* () {
const { sql, service } = yield* setup();
const sync = yield* service.syncOrg(orgId, items).pipe(Effect.forkChild);
yield* TestClock.adjust("10 seconds");
assert.isDefined(sync.pollUnsafe(), "the whole sync fits in one 10 s budget");
yield* Fiber.join(sync);
assert.deepEqual(calls, [["acme/api", Array.from({ length: 20 }, (_, index) => index + 1)]]);
const runs = yield* sql<{ state: string; snapshot: string }>`
SELECT state, snapshot_json AS snapshot FROM kin_deferral_runs
WHERE org_id = ${orgId} ORDER BY item_id`;
assert.lengthOf(runs, 20);
for (const run of runs) {
assert.equal(run.state, "armed");
assert.include(run.snapshot, '"github":null');
}
}).pipe(Effect.provideService(KinDeferralGithubReader, reader), Effect.provide(layer));
});

it.effect("refuses deferral cards from the model and taps until the dispatcher lands", () =>
Effect.gen(function* () {
const { service } = yield* setup();
Expand Down
22 changes: 15 additions & 7 deletions apps/server/src/kin/triggers/KinTriggerService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import {
type DeferralPhase,
type DeferralPlan,
} from "./deferrals.ts";
import { type GithubLinkState, readGithubLinks } from "./deferralGithub.ts";
import { type GithubLinkState, readGithubLinks, readGithubLinkSets } from "./deferralGithub.ts";

export class KinTriggerSyncError extends Schema.TaggedError<KinTriggerSyncError>()(
"KinTriggerSyncError",
Expand Down Expand Up @@ -300,14 +300,22 @@ const make = Effect.gen(function* () {
if (pending.length === 0) return;
const timeZone = yield* cache.ownerTimeZone(orgId);
const now = yield* Clock.currentTimeMillis;
// Every read happens before any write, so the memory lock waits on GitHub at most once.
const github = yield* readGithubLinkSets(
orgId,
pending.map(([, item, plan]) => (plan.armed && item !== undefined ? item.links : [])),
);
const failures: Array<KinTriggerSyncError> = [];
for (const [itemId, item, plan] of pending)
yield* (
plan.armed && item !== undefined
? readGithubLinks(orgId, item.links)
: Effect.succeed(null)
for (const [index, [itemId, item, plan]] of pending.entries())
yield* apply(
orgId,
itemId,
item,
plan,
timeZone,
now,
plan.armed && item !== undefined ? (github[index] ?? null) : null,
).pipe(
Effect.flatMap((github) => apply(orgId, itemId, item, plan, timeZone, now, github)),
wrap(orgId),
Effect.catch((error) => {
failures.push(error);
Expand Down
112 changes: 87 additions & 25 deletions apps/server/src/kin/triggers/deferralGithub.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { KinOrgId } from "@t3tools/contracts";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";
import * as Schema from "effect/Schema";

export const GithubLinkState = Schema.Struct({
Expand All @@ -23,26 +24,32 @@ export class KinDeferralGithubError extends Schema.TaggedError<KinDeferralGithub

export interface KinDeferralGithubReaderShape {
/**
* Read-only: one issue or PR's state, assignees and linked PR numbers.
* Called under the org memory lock from syncOrg, so a real reader must be time-bounded.
* Read-only and batched: one call per repo for all of its numbers. Numbers GitHub doesn't have,
* or can't report in full, are left out. The caller bounds the time.
*/
readonly read: (
readonly readRepo: (
orgId: KinOrgId,
ref: GithubRef,
) => Effect.Effect<Omit<GithubLinkState, "link">, KinDeferralGithubError>;
repo: string,
numbers: ReadonlyArray<number>,
) => Effect.Effect<ReadonlyMap<number, Omit<GithubLinkState, "link">>, KinDeferralGithubError>;
}

// Kin has no GitHub client yet, so production reads fail and every recheck with
// GitHub links reports "couldn't verify", which asks instead of starting.
// githubReaders.ts replaces this when the Kin App is configured. Without it every read fails,
// so a recheck with GitHub links reports "couldn't verify", which asks instead of starting.
export const KinDeferralGithubReader = Context.Reference<KinDeferralGithubReaderShape>(
"t3/kin/triggers/deferralGithub/KinDeferralGithubReader",
{
defaultValue: () => ({
read: (orgId) => Effect.fail(new KinDeferralGithubError({ orgId, reason: "unavailable" })),
readRepo: (orgId) =>
Effect.fail(new KinDeferralGithubError({ orgId, reason: "unavailable" })),
}),
},
);

// Callers hold the org memory lock, so every GitHub read for one sync shares this budget.
const READ_BUDGET = "10 seconds";
const READ_CONCURRENCY = 4;

const urlRef = /^https:\/\/github\.com\/([\w.-]+\/[\w.-]+)\/(?:issues|pull)\/(\d+)\/?$/;
const shortRef = /^([\w.-]+\/[\w.-]+)#(\d+)$/;

Expand All @@ -53,24 +60,79 @@ export function githubRef(link: string): GithubRef | undefined {
return Number.isSafeInteger(number) && number > 0 ? { repo: match[1]!, number } : undefined;
}

/** Reads every GitHub link; `null` means at least one couldn't be read. */
export const readGithubLinks = Effect.fn("KinDeferralGithub.readLinks")(function* (
const refKey = (repo: string, number: number) => `${repo.toLowerCase()}#${number}`;

/**
* Reads the GitHub links of each list with one call per repo, all within one budget. A list is
* `null` when any of its links couldn't be read in time.
*/
export const readGithubLinkSets = Effect.fn("KinDeferralGithub.readLinkSets")(function* (
orgId: KinOrgId,
links: ReadonlyArray<string>,
linkSets: ReadonlyArray<ReadonlyArray<string>>,
) {
const reader = yield* KinDeferralGithubReader;
const states: Array<GithubLinkState> = [];
for (const link of links) {
const ref = githubRef(link);
if (ref === undefined) continue;
const state = yield* reader.read(orgId, ref).pipe(Effect.option);
if (state._tag === "None") return null;
states.push({
link,
state: state.value.state,
assignees: state.value.assignees.toSorted(),
linkedPrs: state.value.linkedPrs.toSorted((a, b) => a - b),
});
}
return states;
const byRepo = new Map<string, { readonly repo: string; readonly numbers: Set<number> }>();
for (const links of linkSets)
for (const link of links) {
const ref = githubRef(link);
if (ref === undefined) continue;
const key = ref.repo.toLowerCase();
const entry = byRepo.get(key) ?? { repo: ref.repo, numbers: new Set<number>() };
entry.numbers.add(ref.number);
byRepo.set(key, entry);
}
const read = new Map<string, Omit<GithubLinkState, "link">>();
yield* Effect.forEach(
byRepo.values(),
({ repo, numbers }) =>
reader
.readRepo(
orgId,
repo,
[...numbers].toSorted((a, b) => a - b),
)
.pipe(
Effect.tap((states) =>
Effect.sync(() => {
for (const [number, state] of states)
if (numbers.has(number)) read.set(refKey(repo, number), state);
}),
),
Effect.catch((error) =>
Effect.logWarning("Kin deferral GitHub read failed", {
orgId,
repo,
reason: error.reason,
}),
),
),
{ concurrency: READ_CONCURRENCY, discard: true },
).pipe(
Effect.timeoutOption(READ_BUDGET),
Effect.tap((done) =>
Option.isNone(done)
? Effect.logWarning("Kin deferral GitHub reads ran out of time", { orgId })
: Effect.void,
),
);
return linkSets.map((links): ReadonlyArray<GithubLinkState> | null => {
const states: Array<GithubLinkState> = [];
for (const link of links) {
const ref = githubRef(link);
if (ref === undefined) continue;
const state = read.get(refKey(ref.repo, ref.number));
if (state === undefined) return null;
states.push({
link,
state: state.state,
assignees: state.assignees.toSorted(),
linkedPrs: state.linkedPrs.toSorted((a, b) => a - b),
});
}
return states;
});
});

/** Reads every GitHub link; `null` means at least one couldn't be read. */
export const readGithubLinks = (orgId: KinOrgId, links: ReadonlyArray<string>) =>
readGithubLinkSets(orgId, [links]).pipe(Effect.map(([states]) => states ?? null));
21 changes: 10 additions & 11 deletions apps/server/src/kin/triggers/deferrals.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,7 @@ import * as Memory from "../memory/Memory.ts";
import * as MemoryConfig from "../memory/MemoryConfig.ts";
import * as MemoryWriter from "../memory/MemoryWriter.ts";
import * as DeferralFlow from "./deferralFlow.ts";
import {
KinDeferralGithubError,
KinDeferralGithubReader,
type GithubLinkState,
} from "./deferralGithub.ts";
import { KinDeferralGithubReader, type GithubLinkState } from "./deferralGithub.ts";
import * as Deferrals from "./deferrals.ts";
import * as TriggerScheduler from "./KinTriggerScheduler.ts";
import * as TriggerService from "./KinTriggerService.ts";
Expand Down Expand Up @@ -75,12 +71,15 @@ const layerFor = <E = never>(
);

const fakeGithub = (states: Map<string, Omit<GithubLinkState, "link">>) => ({
read: (org: typeof orgId, ref: { readonly repo: string; readonly number: number }) => {
const state = states.get(`${ref.repo}#${ref.number}`);
return state === undefined
? Effect.fail(new KinDeferralGithubError({ orgId: org, reason: "not_found" }))
: Effect.succeed(state);
},
readRepo: (_org: typeof orgId, repo: string, numbers: ReadonlyArray<number>) =>
Effect.succeed(
new Map(
numbers.flatMap((number) => {
const state = states.get(`${repo}#${number}`);
return state === undefined ? [] : [[number, state] as const];
}),
),
),
});
// Stands in for LEAD-11, the worker release gate and #44's GitHub client.
const openGates =
Expand Down
Loading