Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion apps/server/src/kin/KinServer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,15 @@ import { KinChatService } from "./chat/KinChatService.ts";
import { KinChatHub } from "./chat/KinChatHub.ts";
import { KinBudget } from "./lead/KinBudget.ts";
import { KinEventQueue } from "./lead/KinEventQueue.ts";
import type { KinGithubWebhookLock } from "./triggers/githubWebhooks.ts";

export type Services = KinIdentity | KinChatService | KinBudget | KinEventQueue | KinChatHub;
export type Services =
| KinIdentity
| KinChatService
| KinBudget
| KinEventQueue
| KinChatHub
| KinGithubWebhookLock;

export class KinServer extends Context.Service<
KinServer,
Expand Down
33 changes: 29 additions & 4 deletions apps/server/src/kin/contract/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import * as ScheduledTasks from "../../scheduledTasks/ScheduledTaskService.ts";
import * as Scheduler from "../../scheduling/Scheduler.ts";
import * as SecretRequests from "../../secrets/SecretRequests.ts";
import * as KinTriggerSink from "../triggers/KinTriggerSink.ts";
import * as KinTriggerStore from "../triggers/KinTriggerStore.ts";

const decodeEvent = Schema.decodeSync(OrchestrationV2DomainEventJson);

Expand Down Expand Up @@ -59,6 +60,8 @@ const projection = () => {
return emptyProjection(event);
};

export const CONTRACT_BOT_LOGIN = "kin-contract[bot]";

export const webhookInput = (overrides: Partial<typeof ScheduledTaskUpsertInput.Encoded> = {}) =>
decodeUpsert({
id: "kin-github",
Expand All @@ -80,18 +83,35 @@ export const withScheduledTasks = Effect.fn("KinContract.withScheduledTasks")(fu
service: ScheduledTasks.ScheduledTaskService["Service"];
launches: Queue.Queue<ThreadLaunch.ThreadLaunchInput>;
sends: Queue.Queue<ThreadManagement.ThreadManagementSendInput>;
launchDeliveries: Queue.Queue<ReadonlyArray<{ readonly receivedAt: unknown }>>;
launchDeliveries: Queue.Queue<
ReadonlyArray<{ readonly receivedAt: unknown; readonly secret: string }>
>;
launchInFlight: Queue.Queue<number>;
}) => Effect.Effect<A, E, SqlClient.SqlClient>,
policy?: ManagedPolicyDocument,
workers?: Map<string, Effect.Effect<void>>,
) {
const sql = yield* SqlClient.SqlClient;
const launchDeliveries =
yield* Queue.unbounded<ReadonlyArray<{ readonly receivedAt: unknown }>>();
yield* Queue.unbounded<
ReadonlyArray<{ readonly receivedAt: unknown; readonly secret: string }>
>();
const launches = yield* Queue.unbounded<ThreadLaunch.ThreadLaunchInput>();
const launchInFlight = yield* Queue.unbounded<number>();
const sends = yield* Queue.unbounded<ThreadManagement.ThreadManagementSendInput>();
const dependencies = Layer.mergeAll(
NodeCrypto.layer,
Scheduler.layer,
workers === undefined
? Scheduler.layer
: Layer.succeed(Scheduler.Scheduler, {
register: Effect.fn("KinContract.register")(function* <E2, R2>(
name: string,
work: Effect.Effect<void, E2, R2>,
) {
const context = yield* Effect.context<R2>();
workers.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
}),
}),
ManagedPolicy.layerTest(policy),
Layer.mock(ThreadLaunch.ThreadLaunchService)({
launch: Effect.fn("KinContract.launch")(function* (input) {
Expand All @@ -102,8 +122,13 @@ export const withScheduledTasks = Effect.fn("KinContract.withScheduledTasks")(fu
taskId: taskId ?? "",
deliveryId,
projectId: input.projectId,
botLogin: CONTRACT_BOT_LOGIN,
}).pipe(Effect.orDie);
yield* Queue.offer(launchDeliveries, rows);
const inFlight = yield* KinTriggerStore.inFlightDeliveries(sql, taskId ?? "").pipe(
Effect.orDie,
);
yield* Queue.offer(launchInFlight, inFlight.length);
yield* Queue.offer(launches, input);
const result = projection();
return { threadId: result.thread.id, projection: result, resumed: false };
Expand All @@ -126,6 +151,6 @@ export const withScheduledTasks = Effect.fn("KinContract.withScheduledTasks")(fu
);
return yield* Effect.gen(function* () {
const service = yield* ScheduledTasks.ScheduledTaskService;
return yield* body({ service, launches, sends, launchDeliveries });
return yield* body({ service, launches, sends, launchDeliveries, launchInFlight });
}).pipe(Effect.provide(Layer.fresh(ScheduledTasks.layer.pipe(Layer.provide(dependencies)))));
});
208 changes: 136 additions & 72 deletions apps/server/src/kin/contract/scheduledTask.kin-contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ import * as Effect from "effect/Effect";
import * as Queue from "effect/Queue";
import * as Schema from "effect/Schema";
import * as SqlClient from "effect/sql/SqlClient";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";

import * as Sqlite from "../../persistence/Sqlite.ts";
import { nextScheduledRunAt } from "../../scheduledTasks/Schedule.ts";
import { webhookInput, withScheduledTasks } from "./fixtures.ts";
import * as KinTriggerStore from "../triggers/KinTriggerStore.ts";
import { CONTRACT_BOT_LOGIN, webhookInput, withScheduledTasks } from "./fixtures.ts";

const decodeSchedule = Schema.decodeSync(ScheduledTaskUpsertSchedule);
const decodeReceivedAt = Schema.decodeUnknownEffect(CanonicalUtcTimestamp);
Expand Down Expand Up @@ -67,6 +69,7 @@ it.effect("preserves the upstream database columns Kin reads", () =>
for (const [table, columns] of [
["orchestration_v2_projection_runs", ["run_id", "thread_id", "status", "payload_json"]],
["orchestration_v2_projection_provider_turns", ["thread_id", "completed_at"]],
["scheduled_task_webhook_pending", ["task_id", "state"]],
] as const) {
const rows = yield* sql`PRAGMA table_info(${sql.literal(table)})`;
assert.includeMembers(
Expand All @@ -78,7 +81,16 @@ it.effect("preserves the upstream database columns Kin reads", () =>
const deliveries = yield* sql`PRAGMA table_info(scheduled_task_webhook_deliveries)`;
assert.includeMembers(
tasks.map((row) => row.name),
["task_id", "project_id", "thread_id", "thread_mode", "schedule_json"],
[
"task_id",
"project_id",
"thread_id",
"thread_mode",
"schedule_json",
"enabled",
"last_run_status",
"webhook_secret",
],
);
assert.includeMembers(
deliveries.map((row) => row.name),
Expand All @@ -89,78 +101,130 @@ it.effect("preserves the upstream database columns Kin reads", () =>

it.effect(
"dispatches signed GitHub deliveries with Kin's provenance, span and replay identity",
() =>
withScheduledTasks(({ service, launches, sends, launchDeliveries }) =>
Effect.gen(function* () {
const finished = yield* Deferred.make<void>();
const tracer = Tracer.make({
span: (options) => {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (...args) => {
end(...args);
if (span.name === "ScheduledTaskService.runWebhookDelivery")
Deferred.doneUnsafe(finished, Effect.void);
};
return span;
},
});
const { task } = yield* service.upsert(
yield* webhookInput({
schedule: {
type: "webhook",
signature: {
header: "x-hub-signature-256",
encoding: "hex",
prefix: "sha256=",
secret: "contract-secret",
() => {
const workers = new Map<string, Effect.Effect<void>>();
return withScheduledTasks(
({ service, launches, sends, launchDeliveries, launchInFlight }) =>
Effect.gen(function* () {
const finished = yield* Deferred.make<void>();
const tracer = Tracer.make({
span: (options) => {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (...args) => {
end(...args);
if (span.name === "ScheduledTaskService.runWebhookDelivery")
Deferred.doneUnsafe(finished, Effect.void);
};
return span;
},
});
const signature = {
header: "x-hub-signature-256",
encoding: "hex",
prefix: "sha256=",
secret: "contract-secret",
} as const;
const filters = [
{ path: "headers.x-github-event", op: "in", value: KinTriggerStore.GITHUB_EVENTS },
{ path: "body.sender.login", op: "notEquals", value: CONTRACT_BOT_LOGIN },
] as const;
const { task } = yield* service.upsert(
yield* webhookInput({
schedule: {
type: "webhook",
signature,
dedupe: { replayKey: KinTriggerStore.GITHUB_REPLAY_KEY },
filters,
},
dedupe: { replayKey: "{{headers.x-github-delivery}}" },
}),
);
const bodyText = JSON.stringify({
action: "opened",
number: 18,
sender: { login: "dev" },
});
const body = new TextEncoder().encode(bodyText);
const request = {
hookId: task.id,
token: task.webhook?.path.split("/").at(-1) ?? "",
method: "POST",
path: `/api/hooks/${task.id}`,
query: "",
body,
bodyText,
headers: {
"content-type": "application/json",
"x-github-event": "pull_request",
"x-github-delivery": "github-18",
"x-hub-signature-256": `sha256=${NodeCrypto.createHmac("sha256", "contract-secret").update(body).digest("hex")}`,
},
}),
);
const bodyText = JSON.stringify({ action: "opened", number: 18 });
const body = new TextEncoder().encode(bodyText);
const request = {
hookId: task.id,
token: task.webhook?.path.split("/").at(-1) ?? "",
method: "POST",
path: `/api/hooks/${task.id}`,
query: "",
body,
bodyText,
headers: {
"content-type": "application/json",
"x-github-event": "pull_request",
"x-github-delivery": "github-18",
"x-hub-signature-256": `sha256=${NodeCrypto.createHmac("sha256", "contract-secret").update(body).digest("hex")}`,
},
};
const accepted = yield* service.triggerWebhook(request).pipe(Effect.withTracer(tracer));
if (accepted._tag !== "accepted") return assert.fail(`Unexpected ${accepted._tag}`);
yield* Deferred.await(finished);
const launch = yield* Queue.take(launches);
const rows = yield* Queue.take(launchDeliveries);
assert.lengthOf(rows, 1);
const [row] = rows;
if (row === undefined) return assert.fail("Delivery missing at launch");
const receivedAt = yield* decodeReceivedAt(row.receivedAt);
assert.equal(DateTime.formatIso(receivedAt), row.receivedAt);
assert.equal(launch.initialMessage?.scheduledTaskId, task.id);
assert.equal(launch.commandId, `scheduled-task:${task.id}:webhook:${accepted.deliveryId}`);
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: accepted.deliveryId,
});
assert.equal(delivery.headers["x-github-event"], "pull_request");
assert.equal(delivery.body, bodyText);
assert.equal(delivery.outcome, "accepted");
const replay = yield* service.triggerWebhook(request);
assert.deepInclude(replay, { _tag: "accepted", outcome: "duplicate" });
assert.equal(yield* Queue.size(launches), 0);
assert.equal(yield* Queue.size(sends), 0);
}),
).pipe(Effect.provide(Sqlite.layerMemory)),
};
const accepted = yield* service.triggerWebhook(request).pipe(Effect.withTracer(tracer));
if (accepted._tag !== "accepted") return assert.fail(`Unexpected ${accepted._tag}`);
yield* Deferred.await(finished);
const launch = yield* Queue.take(launches);
const rows = yield* Queue.take(launchDeliveries);
assert.lengthOf(rows, 1);
assert.equal(yield* Queue.take(launchInFlight), 1);
const [row] = rows;
if (row === undefined) return assert.fail("Delivery missing at launch");
const receivedAt = yield* decodeReceivedAt(row.receivedAt);
assert.equal(DateTime.formatIso(receivedAt), row.receivedAt);
assert.equal(row.secret, signature.secret);
assert.equal(launch.initialMessage?.scheduledTaskId, task.id);
assert.equal(
launch.commandId,
`scheduled-task:${task.id}:webhook:${accepted.deliveryId}`,
);
const { delivery } = yield* service.getWebhookDelivery({
id: task.id,
deliveryId: accepted.deliveryId,
});
assert.equal(delivery.headers["x-github-event"], "pull_request");
assert.equal(delivery.body, bodyText);
assert.equal(delivery.outcome, "accepted");
const replay = yield* service.triggerWebhook(request);
assert.deepInclude(replay, { _tag: "accepted", outcome: "duplicate" });
assert.equal(yield* Queue.size(launches), 0);

// A coalesced delivery launches from the pending queue, so both in-flight arms match.
// The sink's lookup refuses it: a coalesce window could hold a delivery past a re-key.
const { task: queued } = yield* service.upsert(
yield* webhookInput({
id: "kin-github-queued",
schedule: {
type: "webhook",
signature,
dedupe: {
replayKey: KinTriggerStore.GITHUB_REPLAY_KEY,
coalesceKey: "{{headers.x-github-event}}",
coalesceWindowSeconds: 10,
},
filters,
},
}),
);
const window = yield* service.triggerWebhook({
...request,
hookId: queued.id,
token: queued.webhook?.path.split("/").at(-1) ?? "",
path: `/api/hooks/${queued.id}`,
});
assert.deepInclude(window, { _tag: "accepted", outcome: "accepted" });
yield* TestClock.adjust("10 seconds");
const worker = workers.get("webhook-coalescing");
if (worker === undefined) return assert.fail("No coalescing worker");
yield* worker;
assert.equal((yield* Queue.take(launches)).initialMessage?.scheduledTaskId, queued.id);
assert.lengthOf(yield* Queue.take(launchDeliveries), 0);
assert.equal(yield* Queue.take(launchInFlight), 2);
assert.equal(yield* Queue.size(sends), 0);
}),
undefined,
workers,
).pipe(Effect.provide(Sqlite.layerMemory));
},
);

it.effect("passes scheduledTaskId to sendToThread for bound tasks", () =>
Expand Down
3 changes: 2 additions & 1 deletion apps/server/src/kin/layer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export { KinServer } from "./KinServer.ts";
import type * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as KinMcp from "./mcp/layer.ts";
import * as GithubWebhooks from "./triggers/githubWebhooks.ts";

const enabled = Layer.effect(
KinServer,
Expand All @@ -33,7 +34,7 @@ const enabled = Layer.effect(
subscribeForViewer: Option.some(hub.subscribeForViewer),
};
}),
).pipe(Layer.provide(KinChat.layer));
).pipe(Layer.provide(Layer.merge(KinChat.layer, GithubWebhooks.lockLayer)));
const disabled = Layer.succeed(KinServer, {
taskCardUpdater: Layer.empty,
services: Option.none(),
Expand Down
Loading