From 5246c13621f5dd0cddbb928d9154ef109a6863f8 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 17:19:10 +0000 Subject: [PATCH 1/2] Add tool-agnostic rate-limit workaround for wiki workflow Wiki generation is multi-page and tool-call-heavy; host platforms like Factory may rate-limit mid-session and discard unwritten analysis held in context. Mirror the research workflow's crash-safe persistence rules: write each page immediately after reading its source, create phased checkpoint tasks via the host task system, and exit cleanly with a resume protocol on interrupt. Extract shared durability fragments into workflow-persistence.ts so future workflows can reuse the same host-agnostic guidance. --- .../skills/packs/codebase-wiki/SKILL.md | 2 + packages/server/src/mcp/tools/wiki-body.ts | 18 ++++- .../mcp/tools/workflow-persistence.test.ts | 37 ++++++++++ .../src/mcp/tools/workflow-persistence.ts | 74 +++++++++++++++++++ .../server/src/mcp/tools/workflow.test.ts | 3 + 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 packages/server/src/mcp/tools/workflow-persistence.test.ts create mode 100644 packages/server/src/mcp/tools/workflow-persistence.ts diff --git a/packages/server/assets/skills/packs/codebase-wiki/SKILL.md b/packages/server/assets/skills/packs/codebase-wiki/SKILL.md index b76a87567..d382a841c 100644 --- a/packages/server/assets/skills/packs/codebase-wiki/SKILL.md +++ b/packages/server/assets/skills/packs/codebase-wiki/SKILL.md @@ -32,6 +32,8 @@ wiki/ Don't free-hand it — call **`workflow({ kind: "wiki" })`** and follow the phased, STOP-gated guide. It auto-detects mode: a stubbed `OVERVIEW.md` (empty `source_commit`) → **generate** (survey → overview → architecture → modules → flows → concepts → link-graph audit); a stamped `source_commit` → **refresh** (diff `source_commit..HEAD`, update only affected pages, re-stamp). +**Persist incrementally (MUST).** Wiki generation is multi-page and tool-call-heavy — host platforms (Factory, Cursor, Claude, etc.) may rate-limit or end sessions mid-run. Write each page immediately after reading its source; never hold finished module write-ups in context for a trailing batch. The workflow guide carries the full crash-safe + resume rules; re-invoking `workflow({ kind: "wiki" })` picks up from partial pages already in `wiki/`. + **Two toolsets.** Read source code with NATIVE tools (`Read`/`Grep`/`Glob`/`Bash`) — OK MCP does not index non-markdown source. Author and audit the wiki with OK MCP verbs (`write`/`edit` for pages, `links`/`search` for the graph). Never hand-write wiki markdown with native `Write`/`Edit`. ## The two knobs diff --git a/packages/server/src/mcp/tools/wiki-body.ts b/packages/server/src/mcp/tools/wiki-body.ts index 744544dee..211d99437 100644 --- a/packages/server/src/mcp/tools/wiki-body.ts +++ b/packages/server/src/mcp/tools/wiki-body.ts @@ -17,6 +17,12 @@ * (`write`/`edit`/`links`/`search`/`exec`) that author and audit the markdown. */ +import { + buildSessionInterruptRecoverySection, + buildWikiCheckpointTasksSection, + buildWikiPersistAsYouGoSection, +} from './workflow-persistence.ts'; + export function buildWikiBody(contentDir: string): string { return `# Codebase Wiki — Generate + Refresh @@ -30,6 +36,12 @@ Content directory: \`${contentDir}\` (from \`.ok/config.yml\`). The wiki lives a **Prerequisite.** This guide assumes the \`codebase-wiki\` pack is seeded (\`ok seed --pack codebase-wiki\` → \`wiki/\` with \`architecture/ modules/ flows/ concepts/ guides/\`, each carrying folder frontmatter + a page template, plus \`wiki/OVERVIEW.md\` + \`wiki/log.md\`). If \`exec("ls -A ${contentDir}/wiki")\` shows the layout is missing, tell the user to seed first, then re-invoke. +${buildWikiPersistAsYouGoSection(contentDir)} + +${buildSessionInterruptRecoverySection( + 're-invoke `workflow({ kind: "wiki" })`, inventory partial pages under `wiki/`, read them back, skip finished pages, and continue from the first incomplete phase.', +)} + --- ## The two knobs @@ -66,7 +78,9 @@ Never invent paths — every source reference must point at a file you actually ## GENERATE — phased, STOP-gated (⛔ = wait for user confirmation) -Work the phases in order. Do not skip or batch ahead of a ⛔ gate. Each page is authored with OK \`write\`/\`edit\`; create from the seeded templates (\`write({ document: { path, template: "" } })\`) so pages start with the right skeleton, then fill the sections. +${buildWikiCheckpointTasksSection()} + +Work the phases in order. Do not skip or batch ahead of a ⛔ gate. Each page is authored with OK \`write\`/\`edit\`; create from the seeded templates (\`write({ document: { path, template: "" } })\`) so pages start with the right skeleton, then fill the sections. **Write each page before moving to the next** — see *Persist as you go* above. ### Phase 0 — Resolve profile + scope (⛔ STOP gate 0) @@ -168,6 +182,7 @@ Incremental by default — don't re-read the whole repo. - **Don't scaffold folders by hand** — the \`codebase-wiki\` pack already created \`wiki/\` with templates; if it's missing, seed first. - **Scale to \`depth\`** — don't write \`guides/\` or per-flow failure modes at \`tour\`; don't fold modules into architecture at \`exhaustive\`. - **Refresh is incremental** — diff \`source_commit..HEAD\` and touch only affected pages; full-regen only on large/structural diffs or when git is unavailable. +- **Don't batch writes at the end** — rate limits and session interrupts are expected on large wikis; persist each page as you finish it. --- @@ -175,6 +190,7 @@ Incremental by default — don't re-read the whole repo. - **Pack not seeded** — \`wiki/\` layout missing → tell the user to run \`ok seed --pack codebase-wiki\`, then re-invoke. Exit. - **Server down** — a \`write\`/\`links\` call reports the server is not running → tell the user to run \`ok start\` and retry. Exit cleanly; re-invoking resumes (already-written pages persist). +- **Host rate limit / session interrupt** — exit cleanly per *Host rate limits + session interrupts* above. List completed pages; leave the current phase task open if using the host task system. Re-invoking resumes from partial progress — do NOT restart from Phase 0 unless the user asks. - **User aborts at a ⛔ gate** → exit cleanly, leaving any already-written pages in place. - **Empty / unreadable repo** (no source detected in Phase 1) → tell the user there's nothing to document yet. Exit. `; diff --git a/packages/server/src/mcp/tools/workflow-persistence.test.ts b/packages/server/src/mcp/tools/workflow-persistence.test.ts new file mode 100644 index 000000000..d901163c3 --- /dev/null +++ b/packages/server/src/mcp/tools/workflow-persistence.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, test } from 'bun:test'; +import { + buildSessionInterruptRecoverySection, + buildWikiCheckpointTasksSection, + buildWikiPersistAsYouGoSection, + hostTaskSystemPhrase, +} from './workflow-persistence.ts'; + +describe('workflow-persistence — tool-agnostic durability fragments', () => { + test('hostTaskSystemPhrase names multiple hosts without picking one', () => { + const phrase = hostTaskSystemPhrase(); + expect(phrase).toContain('Factory'); + expect(phrase).toContain('Cursor'); + expect(phrase).toContain('Claude'); + }); + + test('buildWikiPersistAsYouGoSection stresses one-page-at-a-time writes', () => { + const section = buildWikiPersistAsYouGoSection('content'); + expect(section).toContain('PERSIST AS YOU GO'); + expect(section).toContain('write` each page immediately'); + expect(section).toContain('content/wiki'); + }); + + test('buildWikiCheckpointTasksSection lists phased tasks for GENERATE', () => { + const section = buildWikiCheckpointTasksSection(); + expect(section).toContain('Phase 0'); + expect(section).toContain('Phase 7'); + expect(section).toContain('REFRESH mode'); + }); + + test('buildSessionInterruptRecoverySection forbids native write bypass', () => { + const section = buildSessionInterruptRecoverySection('resume hint here'); + expect(section).toContain('rate-limit'); + expect(section).toContain('Never bypass OK'); + expect(section).toContain('resume hint here'); + }); +}); diff --git a/packages/server/src/mcp/tools/workflow-persistence.ts b/packages/server/src/mcp/tools/workflow-persistence.ts new file mode 100644 index 000000000..4a57e8d82 --- /dev/null +++ b/packages/server/src/mcp/tools/workflow-persistence.ts @@ -0,0 +1,74 @@ +/** + * Shared procedural fragments for MCP workflow bodies. + * + * Tool-agnostic guidance for crash-safe persistence and host rate-limit / + * session-interrupt recovery. Workflow bodies interpolate these into their + * markdown plans so agents on Factory, Cursor, Claude, or any other host get + * the same durability rules without host-specific code paths. + */ + +/** How to refer to the host's durable task/todo system without naming one product. */ +export function hostTaskSystemPhrase(): string { + return "your host's task system (`TaskCreate` in Claude; equivalent task/todo APIs elsewhere — Cursor, Factory, etc.)"; +} + +/** + * Generic rate-limit / session-interrupt recovery block. Callers pass a + * resume hint that tells the agent how to pick up where it left off. + */ +export function buildSessionInterruptRecoverySection(resumeHint: string): string { + return `## Host rate limits + session interrupts — exit cleanly, resume cheaply + +Host platforms may rate-limit tool calls, exhaust a session budget, or compact context mid-run. **This is normal for large wiki generation — design for it, don't fight it.** + +When the host signals rate limiting, tool-call budget exhaustion, context pressure, or an imminent session end: + +1. **Stop cleanly** — do NOT burst remaining work into a trailing batch held only in context. Finished units belong in the KB; unfinished units wait for the next session. +2. **Never bypass OK for wiki markdown** — native \`Write\`/\`Edit\` on in-scope wiki pages is still forbidden (loses attribution, backlinks, live preview). Rate limits are a pause signal, not an excuse to bypass MCP. +3. **Optional snapshot** — if you wrote substantial content this session, \`checkpoint({ summary: "Wiki partial — " })\` before exiting gives the user a named restore point in \`history\`. +4. **Tell the user what's done** — list pages written, what's next, and that re-invoking resumes without redoing finished pages. +5. **Resume** — ${resumeHint}`; +} + +/** + * Wiki-specific persist-as-you-go rules. Multi-page generation is the primary + * failure mode when hosts rate-limit mid-session. + */ +export function buildWikiPersistAsYouGoSection(contentDir: string): string { + return `## Persist as you go — the wiki IS your checkpoint + +⛔ **PERSIST AS YOU GO — crash-safe checkpoint rule.** Wiki generation is multi-page and tool-call-heavy. Host platforms may rate-limit or terminate sessions mid-run. The most expensive failure is completed analysis held in context, never written — discarded when the session died. The knowledge base is the checkpoint; these rules make every phase crash-safe: + +- **Create \`wiki/OVERVIEW.md\` skeleton early (Phase 2)** — stamp \`profile\` + \`source_commit\` + a nav map (placeholder links are fine) before module pages. Fill sections as you go; don't defer the whole hub to the end. +- **\`write\` each page immediately after reading its source** — one page at a time: read source → \`write\`/\`edit\` page → next. Never batch-survey the whole repo and write all pages in one trailing burst. +- **Interleave read work with writes** — read a module's source, write its page, then move on. Holding five module write-ups in context while still reading is the anti-pattern (see the platform skill's cadence note: durability beats batching). +- **After each page lands, update OVERVIEW nav links if needed** — don't defer all hub updates to Phase 7. +- **Structured notes that live only in your context are not persisted work** — if a section is worth keeping, it belongs in a wiki page via \`write\`/\`edit\`, not in chat or memory. + +On resume after any interrupt: re-invoke \`workflow({ kind: "wiki" })\`, inventory partial progress with \`exec("find ${contentDir}/wiki -name '*.md'")\` (or \`exec("ls -R ${contentDir}/wiki")\`), read each partial page via \`exec("cat …")\`, skip completed pages, continue from the first gap in phase order.`; +} + +/** Step 0 task list for wiki GENERATE mode — persists across context compaction on hosts that support tasks. */ +export function buildWikiCheckpointTasksSection(): string { + const taskPhrase = hostTaskSystemPhrase(); + return `## Step 0 — Create workflow checkpoint tasks (GENERATE mode) + +⛔ **ALWAYS THE FIRST ACTION** after mode detection confirms GENERATE (stub \`source_commit\`). Before any survey read or wiki write — create tasks via ${taskPhrase}. They persist across context compaction, make skipped phases visible, and show progress to the user. + +\`\`\` +TaskCreate: "Wiki: Resolve profile + scope (Phase 0)" → in_progress +TaskCreate: "Wiki: Survey codebase (Phase 1)" → pending, blocked by #1 +TaskCreate: "Wiki: Author OVERVIEW hub (Phase 2)" → pending, blocked by #2 +TaskCreate: "Wiki: Architecture pages (Phase 3)" → pending, blocked by #3 +TaskCreate: "Wiki: Module pages (Phase 4)" → pending, blocked by #4 +TaskCreate: "Wiki: Flow pages (Phase 5)" → pending, blocked by #5 +TaskCreate: "Wiki: Concept pages (Phase 6)" → pending, blocked by #6 +TaskCreate: "Wiki: Link-graph audit + log (Phase 7)" → pending, blocked by #7 +\`\`\` + +Use \`addBlockedBy\` (or equivalent) to enforce ordering. Mark each task \`completed\` as its phase finishes; mark the next \`in_progress\`. + +**REFRESH mode:** skip this step — jump to the *Refresh mode* section; create at most one task per affected page cluster if the host supports tasks. + +**Rate-limit interrupt:** mark the current phase task still \`in_progress\` (not \`completed\`) so the next session knows where to resume.`; +} diff --git a/packages/server/src/mcp/tools/workflow.test.ts b/packages/server/src/mcp/tools/workflow.test.ts index daad897c8..efe15acec 100644 --- a/packages/server/src/mcp/tools/workflow.test.ts +++ b/packages/server/src/mcp/tools/workflow.test.ts @@ -74,6 +74,9 @@ describe('workflow — kind discriminator + per-kind teaching errors', () => { expect(textOf(r)).toContain('# Codebase Wiki'); // The guide is interpolated with the resolved content.dir (mirrors discover). expect(textOf(r)).toContain('wiki/OVERVIEW.md'); + // Crash-safe persistence for host rate limits (Factory, Cursor, etc.). + expect(textOf(r)).toContain('PERSIST AS YOU GO'); + expect(textOf(r)).toContain('Host rate limits + session interrupts'); expect(r.structuredContent?.previewUrl).toBeNull(); }); From 37f19e502bb1cb37c57f3202183ca0cfdfb53d69 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 05:48:37 +0000 Subject: [PATCH 2/2] Address Copilot review on wiki rate-limit workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stabilize test assertion to match exact guide substring - Clarify hostTaskSystemPhrase doc comment; use single-quote style - Replace non-actionable exec("cat …") with concrete example path --- packages/server/src/mcp/tools/workflow-persistence.test.ts | 2 +- packages/server/src/mcp/tools/workflow-persistence.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/src/mcp/tools/workflow-persistence.test.ts b/packages/server/src/mcp/tools/workflow-persistence.test.ts index d901163c3..384139356 100644 --- a/packages/server/src/mcp/tools/workflow-persistence.test.ts +++ b/packages/server/src/mcp/tools/workflow-persistence.test.ts @@ -17,7 +17,7 @@ describe('workflow-persistence — tool-agnostic durability fragments', () => { test('buildWikiPersistAsYouGoSection stresses one-page-at-a-time writes', () => { const section = buildWikiPersistAsYouGoSection('content'); expect(section).toContain('PERSIST AS YOU GO'); - expect(section).toContain('write` each page immediately'); + expect(section).toContain('`write` each page immediately after reading its source'); expect(section).toContain('content/wiki'); }); diff --git a/packages/server/src/mcp/tools/workflow-persistence.ts b/packages/server/src/mcp/tools/workflow-persistence.ts index 4a57e8d82..73cfbd501 100644 --- a/packages/server/src/mcp/tools/workflow-persistence.ts +++ b/packages/server/src/mcp/tools/workflow-persistence.ts @@ -7,9 +7,9 @@ * the same durability rules without host-specific code paths. */ -/** How to refer to the host's durable task/todo system without naming one product. */ +/** How to refer to the host's durable task/todo system without requiring a single host product. */ export function hostTaskSystemPhrase(): string { - return "your host's task system (`TaskCreate` in Claude; equivalent task/todo APIs elsewhere — Cursor, Factory, etc.)"; + return 'your host\'s task system (`TaskCreate` in Claude; equivalent task/todo APIs elsewhere — Cursor, Factory, etc.)'; } /** @@ -45,7 +45,7 @@ export function buildWikiPersistAsYouGoSection(contentDir: string): string { - **After each page lands, update OVERVIEW nav links if needed** — don't defer all hub updates to Phase 7. - **Structured notes that live only in your context are not persisted work** — if a section is worth keeping, it belongs in a wiki page via \`write\`/\`edit\`, not in chat or memory. -On resume after any interrupt: re-invoke \`workflow({ kind: "wiki" })\`, inventory partial progress with \`exec("find ${contentDir}/wiki -name '*.md'")\` (or \`exec("ls -R ${contentDir}/wiki")\`), read each partial page via \`exec("cat …")\`, skip completed pages, continue from the first gap in phase order.`; +On resume after any interrupt: re-invoke \`workflow({ kind: "wiki" })\`, inventory partial progress with \`exec("find ${contentDir}/wiki -name '*.md'")\` (or \`exec("ls -R ${contentDir}/wiki")\`), read each partial page via \`exec("cat ${contentDir}/wiki/modules/.md")\` (substitute the actual path), skip completed pages, continue from the first gap in phase order.`; } /** Step 0 task list for wiki GENERATE mode — persists across context compaction on hosts that support tasks. */